diff --git a/docs/en/2.Development Guide/API Docs/openjiuwen.harness/subagents/browser_agent.md b/docs/en/2.Development Guide/API Docs/openjiuwen.harness/subagents/browser_agent.md index f71416d31c..12607ad480 100644 --- a/docs/en/2.Development Guide/API Docs/openjiuwen.harness/subagents/browser_agent.md +++ b/docs/en/2.Development Guide/API Docs/openjiuwen.harness/subagents/browser_agent.md @@ -14,10 +14,10 @@ The tools available to the sub-agent — both the Playwright MCP primitives and | [Capability allowlist](#browser-capabilities) | Playwright tools are grouped into named capabilities (`core`, `pdf`, `vision`, `devtools`, `config`, `network`, `storage`, `testing`). The caller selects capabilities per task; only the expanded tool allowlist is exposed to the model. | | [Runtime helper tools](../tools/browser_tools.md#runtime-helper-tools) | Page probes, batch interaction, custom actions, cancellation, and health checks, injected alongside the Playwright primitives. | | [Browser options](#browser-options) | Headless vs. headed operation, browser engine choice, session persistence, viewport/device emulation, proxies, and output directories. | -| [Browser instance isolation](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeconfigbrowserinstanceconfig) | `BrowserInstanceConfig` gives each agent its own browser (own MCP server, Chrome profile, and CDP port); agents sharing a key intentionally share one browser. | +| [Browser instance isolation](#class-openjiuwenharnesstoolsbrowser_moveruntimeconfigbrowserinstanceconfig) | `BrowserInstanceConfig` gives each agent its own browser (own MCP server, Chrome profile, and CDP port); agents sharing a key intentionally share one browser. | | [Driver modes](#driver-modes) | `managed` (launch a dedicated local Chrome), `remote` (attach to an existing CDP endpoint), or `extension` (drive a running browser through the Playwright MCP extension bridge). | -| [Progress and resumability](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeruntimebrowserruntimerail) | `BrowserRuntimeRail` persists task progress into the session, injects it back as continuation context on the next invocation, and converts incomplete runs into structured failure summaries. | -| [Guardrails](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeconfigbrowserrunguardrails) | Step, failure, and timeout limits with automatic retry and optional resume after a max-iterations stop. | +| [Progress and resumability](#class-openjiuwenharnesstoolsbrowser_moveruntimeruntimebrowserruntimerail) | `BrowserRuntimeRail` persists task progress into the session, injects it back as continuation context on the next invocation, and converts incomplete runs into structured failure summaries. | +| [Guardrails](#class-openjiuwenharnesstoolsbrowser_moveruntimeconfigrunguardrails) | Step, failure, and timeout limits with automatic retry and optional resume after a max-iterations stop. | | [Observability](#observability) | A dedicated `logs/browser_agent.log` file and redacted structured status telemetry (`[BROWSER_SUBAGENT]` records). | --- @@ -53,9 +53,9 @@ create_browser_agent( Create the browser sub-agent. On top of a plain deep agent, this factory: 1. Resolves the requested `browser_capabilities` against the trusted capability catalog (unknown names raise `ValueError` listing the available capabilities). -2. Builds a [`BrowserAgentRuntime`](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeruntimebrowseragentruntime) from `settings` (or from the model's client config / environment when `settings` is omitted). +2. Builds a [`BrowserAgentRuntime`](#class-openjiuwenharnesstoolsbrowser_moveruntimeruntimebrowseragentruntime) from `settings` (or from the model's client config / environment when `settings` is omitted). 3. Appends the [runtime helper tools](../tools/browser_tools.md#runtime-helper-tools) to any caller-provided `tools`. -4. Appends a [`BrowserRuntimeRail`](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeruntimebrowserruntimerail) to any caller-provided `rails`. +4. Appends a [`BrowserRuntimeRail`](#class-openjiuwenharnesstoolsbrowser_moveruntimeruntimebrowserruntimerail) to any caller-provided `rails`. The default system prompt (locale-specific, `cn`/`en`) instructs the agent to prefer compact probes over snapshots, to use `browser_run_code` only with a known selector, and to claim completion only when the outcome is evidenced on the page. @@ -76,9 +76,9 @@ The default system prompt (locale-specific, `cn`/`en`) instructs the agent to pr - **sys_operation** (SysOperation, optional): System operation. Default: `None`. - **language** (str, optional): `"cn"` or `"en"`; anything else falls back to `"cn"`. Default: `None`. - **prompt_mode** (str, optional): Prompt mode. Default: `None`. -- **settings** ([RuntimeSettings](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeconfigruntimesettings), optional): Full browser runtime override (provider, API key/base, model, MCP config, guardrails, instance). Default: resolved from `model` or environment variables. +- **settings** ([RuntimeSettings](#class-openjiuwenharnesstoolsbrowser_moveruntimeconfigruntimesettings), optional): Full browser runtime override (provider, API key/base, model, MCP config, guardrails, instance). Default: resolved from `model` or environment variables. - **browser_key** (str, optional): Shorthand for `browser_instance=BrowserInstanceConfig(key=...)`. Default: `None`. -- **browser_instance** ([BrowserInstanceConfig](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeconfigbrowserinstanceconfig) | dict, optional): Per-instance browser identity. A plain dict is accepted so the identity can travel as serializable `factory_kwargs`. Default: `None` (legacy shared, environment-driven browser). +- **browser_instance** ([BrowserInstanceConfig](#class-openjiuwenharnesstoolsbrowser_moveruntimeconfigbrowserinstanceconfig) | dict, optional): Per-instance browser identity. A plain dict is accepted so the identity can travel as serializable `factory_kwargs`. Default: `None` (legacy shared, environment-driven browser). - **browser_capabilities** (list[str], optional): Task-scoped capability names from the [capability catalog](#browser-capabilities). `core` is always included. `None` means no allowlist restriction. Default: `None`. - ****config_kwargs**: Additional configuration arguments forwarded to `create_deep_agent`. @@ -169,7 +169,7 @@ The sub-session ID for `browser_agent` is deterministic (`{parent_session}_sub_b ## Browser Capabilities -Module: `openjiuwen.harness.tools.browser_move.playwright_runtime.browser_capabilities` +Module: `openjiuwen.harness.tools.browser_move.surfaces.browser.capabilities` Playwright MCP tools are grouped into a trusted, explicit catalog of capabilities. Matching is by exact tool name — never by prefix — so a newly introduced Playwright tool is not exposed before its policy is reviewed. The main agent selects capability names per task; the resolver performs no task interpretation of its own. @@ -249,7 +249,7 @@ Headed operation is the safer default for sites with bot detection and for debug ### Timeouts - **Per tool call**: `PLAYWRIGHT_MCP_TIMEOUT_S` / `BROWSER_TIMEOUT_S` (default `180`) bounds each MCP tool invocation. -- **Per task attempt**: `BrowserRunGuardrails.timeout_s` (default `180`) bounds a whole delegated browser task; see [Guardrails](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeconfigbrowserrunguardrails). +- **Per task attempt**: `RunGuardrails.timeout_s` (default `180`) bounds a whole delegated browser task; see [Guardrails](#class-openjiuwenharnesstoolsbrowser_moveruntimeconfigrunguardrails). - **Per batch step**: `browser_batch_interact` clamps step timeouts to 250–30000 ms with a global batch cap of 90 s. ### Output directories @@ -258,7 +258,7 @@ The runtime resolves a working directory (`PLAYWRIGHT_RUNTIME_MCP_CWD`, default: --- -## class openjiuwen.harness.tools.browser_move.playwright_runtime.config.BrowserInstanceConfig +## class openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings.BrowserInstanceConfig ```python @dataclass(frozen=True) @@ -276,6 +276,7 @@ Per-instance browser identity, used to isolate one browser per agent. All fields - **profile_name** (str): Managed profile name. Empty falls back to the key, then `BROWSER_PROFILE_NAME`. - **cdp_url** (str): Explicit CDP endpoint for remote mode; wins over the shared environment endpoint. - **browser_binary** (str): Optional Chrome binary path override. +- **launch_args** (str): Playwright MCP launch arguments in the `PLAYWRIGHT_MCP_ARGS` format (a shell-style string or a JSON list). Empty defers to `PLAYWRIGHT_MCP_ARGS`, then the default; a caller that decides headless or isolated mode per run sets this instead of the environment. ### Driver Modes @@ -285,7 +286,7 @@ Per-instance browser identity, used to isolate one browser per agent. All fields --- -## class openjiuwen.harness.tools.browser_move.playwright_runtime.config.RuntimeSettings +## class openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings.RuntimeSettings ```python @dataclass(frozen=True) @@ -301,18 +302,18 @@ Resolved browser runtime settings, stored in `SubAgentConfig.factory_kwargs` and - **api_base** (str): Model API base URL. - **model_name** (str): Model name for the nested browser worker. - **mcp_cfg** (McpServerConfig): Playwright MCP server configuration (stdio; command defaults to `npx -y @playwright/mcp@0.0.78` with the full capability list enabled via `--caps=`). -- **guardrails** ([BrowserRunGuardrails](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeconfigbrowserrunguardrails)): Run guardrails. -- **instance** ([BrowserInstanceConfig](#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeconfigbrowserinstanceconfig), optional): Per-instance browser identity. +- **guardrails** ([RunGuardrails](#class-openjiuwenharnesstoolsbrowser_moveruntimeconfigrunguardrails)): Run guardrails. +- **instance** ([BrowserInstanceConfig](#class-openjiuwenharnesstoolsbrowser_moveruntimeconfigbrowserinstanceconfig), optional): Per-instance browser identity. Use `build_runtime_settings(instance)` to resolve everything from environment variables, or let `create_browser_agent` derive provider/key/base/model from the `Model` object's client config. --- -## class openjiuwen.harness.tools.browser_move.playwright_runtime.config.BrowserRunGuardrails +## class openjiuwen.harness.tools.browser_move.runtime.config.RunGuardrails ```python @dataclass -class BrowserRunGuardrails +class RunGuardrails ``` Limits applied to browser task runs. Built from environment variables by `build_browser_guardrails()`. @@ -329,7 +330,7 @@ Retryable transport failures (detached frame, target closed, page crash, `net::E --- -## class openjiuwen.harness.tools.browser_move.playwright_runtime.runtime.BrowserAgentRuntime +## class openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade.BrowserAgentRuntime ```python class BrowserAgentRuntime( @@ -338,7 +339,7 @@ class BrowserAgentRuntime( api_base: str, model_name: str, mcp_cfg: McpServerConfig, - guardrails: BrowserRunGuardrails, + guardrails: RunGuardrails, instance: BrowserInstanceConfig | None = None, allowed_tool_names: Iterable[str] | None = None, ) @@ -359,7 +360,7 @@ Runtime kernel shared by all browser helper tools. Owns the underlying `BrowserS --- -## class openjiuwen.harness.tools.browser_move.playwright_runtime.runtime.BrowserRuntimeRail +## class openjiuwen.harness.tools.browser_move.surfaces.browser.rails.runtime_rail.BrowserRuntimeRail ```python class BrowserRuntimeRail(runtime: BrowserAgentRuntime) diff --git a/docs/en/2.Development Guide/API Docs/openjiuwen.harness/tools/browser_tools.md b/docs/en/2.Development Guide/API Docs/openjiuwen.harness/tools/browser_tools.md index d0cbfb67b7..b542f116a1 100644 --- a/docs/en/2.Development Guide/API Docs/openjiuwen.harness/tools/browser_tools.md +++ b/docs/en/2.Development Guide/API Docs/openjiuwen.harness/tools/browser_tools.md @@ -112,9 +112,9 @@ Session-state access — cookies, localStorage, sessionStorage, and Playwright s ## Runtime Helper Tools -Module: `openjiuwen.harness.tools.browser_move.playwright_runtime.runtime_tools` +Module: `openjiuwen.harness.tools.browser_move.surfaces.browser.runner.tools` -Deterministic helpers backed by the shared [`BrowserAgentRuntime`](../subagents/browser_agent.md#class-openjiuwenharnesstoolsbrowser_moveplaywright_runtimeruntimebrowseragentruntime). They are always injected, regardless of the capability selection. +Deterministic helpers backed by the shared [`BrowserAgentRuntime`](../subagents/browser_agent.md#class-openjiuwenharnesstoolsbrowser_moveruntimeruntimebrowseragentruntime). They are always injected, regardless of the capability selection. | Tool | Description | |---|---| diff --git "a/docs/zh/2.\345\274\200\345\217\221\346\214\207\345\215\227/API\346\226\207\346\241\243/openjiuwen.harness/subagents/\346\265\217\350\247\210\345\231\250\345\255\220\346\231\272\350\203\275\344\275\223.md" "b/docs/zh/2.\345\274\200\345\217\221\346\214\207\345\215\227/API\346\226\207\346\241\243/openjiuwen.harness/subagents/\346\265\217\350\247\210\345\231\250\345\255\220\346\231\272\350\203\275\344\275\223.md" index 717b705765..cf3855a94e 100644 --- "a/docs/zh/2.\345\274\200\345\217\221\346\214\207\345\215\227/API\346\226\207\346\241\243/openjiuwen.harness/subagents/\346\265\217\350\247\210\345\231\250\345\255\220\346\231\272\350\203\275\344\275\223.md" +++ "b/docs/zh/2.\345\274\200\345\217\221\346\214\207\345\215\227/API\346\226\207\346\241\243/openjiuwen.harness/subagents/\346\265\217\350\247\210\345\231\250\345\255\220\346\231\272\350\203\275\344\275\223.md" @@ -17,7 +17,7 @@ | [浏览器实例隔离](#class-browserinstanceconfig) | `BrowserInstanceConfig` 为每个智能体分配独立浏览器（独立 MCP 服务器、Chrome 配置目录和 CDP 端口）；使用相同 key 的智能体有意共享同一个浏览器。 | | [驱动模式](#驱动模式) | `managed`（启动专属本地 Chrome）、`remote`（连接已有 CDP 端点）、`extension`（通过 Playwright MCP 扩展桥驱动正在运行的浏览器）。 | | [进度与可恢复性](#class-browserruntimerail) | `BrowserRuntimeRail` 将任务进度持久化到会话中，在下次调用时作为延续上下文注入，并将未完成的运行转换为结构化失败摘要。 | -| [Guardrails](#class-browserrunguardrails) | 步数、失败次数和超时限制，支持自动重试，以及在达到最大迭代后可选的恢复执行。 | +| [Guardrails](#class-runguardrails) | 步数、失败次数和超时限制，支持自动重试，以及在达到最大迭代后可选的恢复执行。 | | [可观测性](#可观测性) | 专用日志文件 `logs/browser_agent.log` 和经过脱敏的结构化状态遥测（`[BROWSER_SUBAGENT]` 记录）。 | --- @@ -167,7 +167,7 @@ main_agent = create_deep_agent(model=model, subagents=[browser_cfg]) ## 浏览器能力（Capabilities） -模块：`openjiuwen.harness.tools.browser_move.playwright_runtime.browser_capabilities` +模块：`openjiuwen.harness.tools.browser_move.surfaces.browser.capabilities` Playwright MCP 工具按可信的显式能力目录分组。匹配按工具名精确进行 —— 从不使用前缀匹配 —— 因此新引入的 Playwright 工具在其策略审查之前不会被暴露。主智能体按任务选择能力名称；解析器本身不做任何任务语义解释。 @@ -245,7 +245,7 @@ Managed 模式注意事项：重连时，运行时会拒绝复用启动参数与 ### 超时 - **单次工具调用**：`PLAYWRIGHT_MCP_TIMEOUT_S` / `BROWSER_TIMEOUT_S`（默认 `180`）限制每次 MCP 工具调用。 -- **单次任务尝试**：`BrowserRunGuardrails.timeout_s`（默认 `180`）限制一次完整的委派浏览器任务；参见 [Guardrails](#class-browserrunguardrails)。 +- **单次任务尝试**：`RunGuardrails.timeout_s`（默认 `180`）限制一次完整的委派浏览器任务；参见 [Guardrails](#class-runguardrails)。 - **批量步骤**：`browser_batch_interact` 将单步超时钳制在 250–30000 ms，整批上限 90 s。 ### 输出目录 @@ -256,7 +256,7 @@ Managed 模式注意事项：重连时，运行时会拒绝复用启动参数与 ## class BrowserInstanceConfig -模块：`openjiuwen.harness.tools.browser_move.playwright_runtime.config` +模块：`openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings` ```python @dataclass(frozen=True) @@ -276,6 +276,7 @@ class BrowserInstanceConfig | `profile_name` | `str` | Managed profile 名。为空时回退到 key，再回退到 `BROWSER_PROFILE_NAME` | | `cdp_url` | `str` | Remote 模式的显式 CDP 端点；优先于共享环境端点 | | `browser_binary` | `str` | 可选的 Chrome 可执行文件路径覆盖 | +| `launch_args` | `str` | Playwright MCP 启动参数，格式同 `PLAYWRIGHT_MCP_ARGS`（shell 风格字符串或 JSON 列表）。为空时遵循 `PLAYWRIGHT_MCP_ARGS`，再回退到默认值；按运行决定 headless 或 isolated 的调用方在此设置，而不写环境变量 | ### 驱动模式 @@ -287,7 +288,7 @@ class BrowserInstanceConfig ## class RuntimeSettings -模块：`openjiuwen.harness.tools.browser_move.playwright_runtime.config` +模块：`openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings` ```python @dataclass(frozen=True) @@ -305,20 +306,20 @@ class RuntimeSettings | `api_base` | `str` | 模型 API base URL | | `model_name` | `str` | 嵌套 browser worker 使用的模型名 | | `mcp_cfg` | `McpServerConfig` | Playwright MCP 服务器配置（stdio；命令默认 `npx -y @playwright/mcp@0.0.78`，通过 `--caps=` 启用完整能力列表） | -| `guardrails` | `BrowserRunGuardrails` | 运行 guardrails | +| `guardrails` | `RunGuardrails` | 运行 guardrails | | `instance` | `Optional[BrowserInstanceConfig]` | 每实例浏览器身份 | 使用 `build_runtime_settings(instance)` 从环境变量解析全部字段，或让 `create_browser_agent` 从 `Model` 对象的客户端配置推导 provider/key/base/model。 --- -## class BrowserRunGuardrails +## class RunGuardrails -模块：`openjiuwen.harness.tools.browser_move.playwright_runtime.config` +模块：`openjiuwen.harness.tools.browser_move.runtime.config` ```python @dataclass -class BrowserRunGuardrails +class RunGuardrails ``` 浏览器任务运行的限制。由 `build_browser_guardrails()` 从环境变量构建。 @@ -339,7 +340,7 @@ class BrowserRunGuardrails ## class BrowserAgentRuntime -模块：`openjiuwen.harness.tools.browser_move.playwright_runtime.runtime` +模块：`openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade` ```python class BrowserAgentRuntime( @@ -348,7 +349,7 @@ class BrowserAgentRuntime( api_base: str, model_name: str, mcp_cfg: McpServerConfig, - guardrails: BrowserRunGuardrails, + guardrails: RunGuardrails, instance: BrowserInstanceConfig | None = None, allowed_tool_names: Iterable[str] | None = None, ) @@ -371,7 +372,7 @@ class BrowserAgentRuntime( ## class BrowserRuntimeRail -模块：`openjiuwen.harness.tools.browser_move.playwright_runtime.runtime` +模块：`openjiuwen.harness.tools.browser_move.surfaces.browser.rails.runtime_rail` ```python class BrowserRuntimeRail(runtime: BrowserAgentRuntime) diff --git "a/docs/zh/2.\345\274\200\345\217\221\346\214\207\345\215\227/API\346\226\207\346\241\243/openjiuwen.harness/tools/\346\265\217\350\247\210\345\231\250\345\255\220\346\231\272\350\203\275\344\275\223\345\267\245\345\205\267.md" "b/docs/zh/2.\345\274\200\345\217\221\346\214\207\345\215\227/API\346\226\207\346\241\243/openjiuwen.harness/tools/\346\265\217\350\247\210\345\231\250\345\255\220\346\231\272\350\203\275\344\275\223\345\267\245\345\205\267.md" index 404e2e25ae..99cae40292 100644 --- "a/docs/zh/2.\345\274\200\345\217\221\346\214\207\345\215\227/API\346\226\207\346\241\243/openjiuwen.harness/tools/\346\265\217\350\247\210\345\231\250\345\255\220\346\231\272\350\203\275\344\275\223\345\267\245\345\205\267.md" +++ "b/docs/zh/2.\345\274\200\345\217\221\346\214\207\345\215\227/API\346\226\207\346\241\243/openjiuwen.harness/tools/\346\265\217\350\247\210\345\231\250\345\255\220\346\231\272\350\203\275\344\275\223\345\267\245\345\205\267.md" @@ -112,7 +112,7 @@ ## 运行时辅助工具 -模块：`openjiuwen.harness.tools.browser_move.playwright_runtime.runtime_tools` +模块：`openjiuwen.harness.tools.browser_move.surfaces.browser.runner.tools` 由共享的 [`BrowserAgentRuntime`](../subagents/浏览器子智能体.md#class-browseragentruntime) 支撑的确定性辅助工具。无论能力如何选择，它们始终被注入。 diff --git a/openjiuwen/core/common/logging/tool_call.py b/openjiuwen/core/common/logging/tool_call.py new file mode 100644 index 0000000000..2186eff307 --- /dev/null +++ b/openjiuwen/core/common/logging/tool_call.py @@ -0,0 +1,24 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +from __future__ import annotations + +from typing import Any, Callable, Optional + +ToolNameMatcher = Callable[[str], bool] +ToolCallLog = Callable[..., Any] + +# Process-wide like the loggers it routes to: agents are built by callers that never see the registrant. +_TOOL_CALL_LOGGERS: dict[ToolNameMatcher, ToolCallLog] = {} + + +def register_tool_call_logger(matches: ToolNameMatcher, log: ToolCallLog) -> None: + """Send the tool-call lines of tools whose name ``matches`` to ``log(message, *args)``.""" + _TOOL_CALL_LOGGERS[matches] = log + + +def tool_call_logger(tool_name: str) -> Optional[ToolCallLog]: + for matches, log in _TOOL_CALL_LOGGERS.items(): + if matches(tool_name): + return log + return None diff --git a/openjiuwen/core/context_engine/context/context_utils.py b/openjiuwen/core/context_engine/context/context_utils.py index f918fbbbac..4f2de7975e 100644 --- a/openjiuwen/core/context_engine/context/context_utils.py +++ b/openjiuwen/core/context_engine/context/context_utils.py @@ -1,6 +1,8 @@ # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +import hashlib +import re import threading import time import uuid @@ -18,6 +20,7 @@ CONTEXT_MESSAGE_ID_KEY = "context_message_id" CONTEXT_USAGE_STALE_KEY = "context_usage_stale" DEFAULT_CONTEXT_MAX_TOKENS = 200000 +_PATH_UNSAFE_CHARS = re.compile(r"[^A-Za-z0-9_.-]+") OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" OPENROUTER_MODEL_CACHE_TTL_SECONDS = 3600 _OPENROUTER_MODEL_CONTEXT_WINDOW_TOKENS: Dict[str, int] = {} @@ -164,6 +167,15 @@ class ContextUtils: All methods are static and stateless. """ + @staticmethod + def offload_session_dir_name(session_id: str) -> str: + """Directory name for a session's offload files; ids that are not path-safe get a digest suffix.""" + if session_id and not _PATH_UNSAFE_CHARS.search(session_id) and session_id not in {".", ".."}: + return session_id + digest = hashlib.sha256(session_id.encode("utf-8")).hexdigest()[:16] + readable = _PATH_UNSAFE_CHARS.sub("-", session_id).strip("-.")[:120] + return f"{readable}-{digest}" if readable else digest + @staticmethod def _get_positive_int_by_model_name(mapping: Optional[Dict[str, int]], model_name: str) -> Optional[int]: if not mapping: diff --git a/openjiuwen/core/context_engine/context/message_buffer.py b/openjiuwen/core/context_engine/context/message_buffer.py index cdbc474003..e62726bc84 100644 --- a/openjiuwen/core/context_engine/context/message_buffer.py +++ b/openjiuwen/core/context_engine/context/message_buffer.py @@ -6,6 +6,7 @@ from openjiuwen.core.foundation.llm import BaseMessage from openjiuwen.core.common.logging import logger +from openjiuwen.core.context_engine.context.context_utils import ContextUtils class ContextMessageBuffer: @@ -143,19 +144,26 @@ async def _reload_from_filesystem(self, offload_handle: str) -> List[BaseMessage return [] def _filesystem_reload_paths(self, offload_handle: str) -> List[str]: - """Return candidate filesystem paths for an offload handle.""" + """Return candidate filesystem paths for an offload handle; the folder of the literal id comes last.""" if not (self._workspace_dir and self._session_id): return [offload_handle] - offload_dir = os.path.join( - self._workspace_dir, "context", self._session_id + "_context", "offload" - ) - exact_path = os.path.join(offload_dir, offload_handle + ".json") - paths = [exact_path] - prefixed_paths = sorted(glob.glob(os.path.join(offload_dir, f"*_{offload_handle}.json"))) - paths.extend(path for path in prefixed_paths if path not in paths) + paths: List[str] = [] + for session_dir in self._offload_session_dirs(): + offload_dir = os.path.join(self._workspace_dir, "context", session_dir + "_context", "offload") + paths.append(os.path.join(offload_dir, offload_handle + ".json")) + prefixed_paths = sorted(glob.glob(os.path.join(glob.escape(offload_dir), f"*_{offload_handle}.json"))) + paths.extend(path for path in prefixed_paths if path not in paths) return paths + def _offload_session_dirs(self) -> List[str]: + session_dir = ContextUtils.offload_session_dir_name(self._session_id) + legacy = self._session_id + if legacy == session_dir or "/" in legacy or "\\" in legacy or legacy in {".", ".."}: + return [session_dir] + # Sessions stored before ids were made path-safe sit under the literal id (Linux and macOS). + return [session_dir, legacy] + def clear( self, offload_handle: str, diff --git a/openjiuwen/core/context_engine/processor/offloader/tool_result_budget_processor.py b/openjiuwen/core/context_engine/processor/offloader/tool_result_budget_processor.py index f75ddeae3e..25b805695f 100644 --- a/openjiuwen/core/context_engine/processor/offloader/tool_result_budget_processor.py +++ b/openjiuwen/core/context_engine/processor/offloader/tool_result_budget_processor.py @@ -253,12 +253,12 @@ def _is_already_offloaded(message: ToolMessage) -> bool: def _new_offload_handle_and_path(self, context: ModelContext) -> tuple[str, str | None]: offload_handle = uuid.uuid4().hex - session_id = context.session_id() + session_dir = ContextUtils.offload_session_dir_name(context.session_id()) workspace_dir = context.workspace_dir() file_prefix = self.config.offload_file_prefix or self.processor_type() file_name = f"{file_prefix}_{offload_handle}.json" if workspace_dir: - offload_path = os.path.join(workspace_dir, "context", f"{session_id}_context", "offload", file_name) + offload_path = os.path.join(workspace_dir, "context", f"{session_dir}_context", "offload", file_name) return offload_handle, offload_path return offload_handle, None diff --git a/openjiuwen/core/foundation/llm/model_clients/anthropic_model_client.py b/openjiuwen/core/foundation/llm/model_clients/anthropic_model_client.py index 7eae2f061a..4a69978b57 100644 --- a/openjiuwen/core/foundation/llm/model_clients/anthropic_model_client.py +++ b/openjiuwen/core/foundation/llm/model_clients/anthropic_model_client.py @@ -16,6 +16,7 @@ 3. messages """ +import asyncio import copy import importlib import json @@ -642,7 +643,15 @@ def connection_key(cls, model_client_config: ModelClientConfig) -> Tuple: ) def _client_cache_key(self) -> Tuple: - return self.connection_key(self.model_client_config) + """Connection identity plus the running event loop. + + Mirrors ``OpenAIModelClient._client_cache_key``; see there for why. + """ + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + return (*self.connection_key(self.model_client_config), loop) @classmethod def _build_request_headers( @@ -691,6 +700,7 @@ def _create_async_anthropic_client(self, timeout: Optional[float] = None) -> "an key = self._client_cache_key() client = self._client_cache.get(key) if client is None: + self._evict_stale_shared_clients() client = self._build_async_anthropic_client() self._client_cache[key] = client llm_logger.info( @@ -702,6 +712,23 @@ def _create_async_anthropic_client(self, timeout: Optional[float] = None) -> "an ) return client + @classmethod + def _evict_stale_shared_clients(cls) -> None: + """Drop cached clients whose event loop already closed, on a cache miss. + + Mirrors ``OpenAIModelClient._evict_stale_shared_clients``; see there for why. + """ + stale_keys = [ + key for key in list(cls._client_cache) + if isinstance(key[-1], asyncio.AbstractEventLoop) and key[-1].is_closed() + ] + for key in stale_keys: + cls._client_cache.pop(key, None) + if stale_keys: + logger.debug( + f"Dropped {len(stale_keys)} cached AsyncAnthropic client(s) from closed event loops" + ) + def _build_async_anthropic_client(self, timeout: Optional[float] = None) -> "anthropic.AsyncAnthropic": """Build a fresh ``AsyncAnthropic`` client with its own httpx connection pool.""" from anthropic import AsyncAnthropic, DefaultAsyncHttpxClient @@ -774,10 +801,16 @@ async def aclose_connections(cls, configs: Iterable[ModelClientConfig]) -> None: removed should stop consuming tokens at once. An in-flight request on a closed client surfaces as a normal model-call failure (not retried by LLMRetryRail, which only retries repetition/stream-timeout markers). + + Matches by connection-identity prefix, not exact equality: a key may + carry a trailing event-loop element, so one identity can have + several entries, all belonging to the removed credential. """ - keys = {cls.connection_key(cfg) for cfg in configs} + connection_keys = [cls.connection_key(cfg) for cfg in configs] closed = 0 - for key in keys: + for key in list(cls._client_cache): + if not any(key[: len(conn_key)] == conn_key for conn_key in connection_keys): + continue client = cls._client_cache.pop(key, None) if client is None: continue diff --git a/openjiuwen/core/foundation/llm/model_clients/openai_model_client.py b/openjiuwen/core/foundation/llm/model_clients/openai_model_client.py index 207e376884..adfe06dd49 100644 --- a/openjiuwen/core/foundation/llm/model_clients/openai_model_client.py +++ b/openjiuwen/core/foundation/llm/model_clients/openai_model_client.py @@ -1,6 +1,7 @@ # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. +import asyncio import json from copy import deepcopy from dataclasses import dataclass @@ -381,7 +382,8 @@ class OpenAIModelClient(BaseModelClient): _MODEL_PARAM_RULES: tuple[ModelParamRule, ...] = _DEFAULT_MODEL_PARAM_RULES # Process-wide cache of long-lived ``AsyncOpenAI`` clients, bucketed by - # tenant/connection config so different api_key/api_base never share a + # tenant/connection config (and the event loop currently running -- see + # ``_client_cache_key``) so different api_key/api_base/loop never share a # client. Each cached client keeps its own httpx keep-alive connection pool # alive, so cache hits reuse established connections (no per-request # build/close). Shared across subclasses (OpenRouter/DashScope/DeepSeek) on @@ -462,7 +464,16 @@ def _apply_model_specific_params(self, model: Optional[str], params: dict) -> No params["extra_body"] = extra_body def _client_cache_key(self) -> Tuple: - return self.connection_key(self.model_client_config) + """Connection identity plus the running event loop. + + A client's transport is bound to its build-time loop; reusing it + after that loop closes raises "Event loop is closed" on request. + """ + try: + loop = asyncio.get_running_loop() + except RuntimeError: + loop = None + return (*self.connection_key(self.model_client_config), loop) def _resolved_api_key(self) -> str: return _resolved_api_key_for_config(self.model_client_config) @@ -1083,6 +1094,7 @@ def _create_async_openai_client(self, timeout: Optional[float] = None) -> "opena key = self._client_cache_key() client = self._client_cache.get(key) if client is None: + self._evict_stale_shared_clients() client = self._build_async_openai_client() self._client_cache[key] = client llm_logger.info( @@ -1093,6 +1105,24 @@ def _create_async_openai_client(self, timeout: Optional[float] = None) -> "opena ) return client + @classmethod + def _evict_stale_shared_clients(cls) -> None: + """Drop cached clients whose event loop already closed, on a cache miss. + + Dropped, not closed: closing from a different (the current) loop + would raise the same "Event loop is closed" error this avoids. + """ + stale_keys = [ + key for key in list(cls._client_cache) + if isinstance(key[-1], asyncio.AbstractEventLoop) and key[-1].is_closed() + ] + for key in stale_keys: + cls._client_cache.pop(key, None) + if stale_keys: + logger.debug( + f"Dropped {len(stale_keys)} cached AsyncOpenAI client(s) from closed event loops" + ) + def _build_async_openai_client(self, timeout: Optional[float] = None) -> "openai.AsyncOpenAI": """Build a fresh ``AsyncOpenAI`` client with its own httpx connection pool.""" from openai import AsyncOpenAI @@ -1171,10 +1201,16 @@ async def aclose_connections(cls, configs: Iterable[ModelClientConfig]) -> None: removed should stop consuming tokens at once. An in-flight request on a closed client surfaces as a normal model-call failure (not retried by LLMRetryRail, which only retries repetition/stream-timeout markers). + + Matches by connection-identity prefix, not exact equality: a key may + carry a trailing event-loop element, so one identity can have + several entries, all belonging to the removed credential. """ - keys = {cls.connection_key(cfg) for cfg in configs} + connection_keys = [cls.connection_key(cfg) for cfg in configs] closed = 0 - for key in keys: + for key in list(cls._client_cache): + if not any(key[: len(conn_key)] == conn_key for conn_key in connection_keys): + continue client = cls._client_cache.pop(key, None) if client is None: continue diff --git a/openjiuwen/core/single_agent/agents/react_agent.py b/openjiuwen/core/single_agent/agents/react_agent.py index cfda08e898..c07e6e76b1 100644 --- a/openjiuwen/core/single_agent/agents/react_agent.py +++ b/openjiuwen/core/single_agent/agents/react_agent.py @@ -24,12 +24,7 @@ from openjiuwen.core.common.exception.codes import StatusCode from openjiuwen.core.common.exception.errors import BaseError, build_error, raise_error from openjiuwen.core.common.logging import logger -try: - from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_logging import ( - browser_agent_log_info, - ) -except Exception: # pragma: no cover - browser runtime is optional here - browser_agent_log_info = None +from openjiuwen.core.common.logging.tool_call import tool_call_logger from openjiuwen.core.common.security.user_config import UserConfig from openjiuwen.core.foundation.prompt import PromptTemplate from openjiuwen.core.foundation.llm.schema.config import ( @@ -2104,11 +2099,8 @@ async def _execute_tool_call( for tool_call in tool_calls: log_args = self._summarize_tool_args_for_log(tool_call.name, tool_call.arguments) tool_name = str(tool_call.name or "") - is_browser_tool = self._is_browser_tool_name(tool_name) - if is_browser_tool and browser_agent_log_info is not None: - browser_agent_log_info("Executing tool: %s with args: %s", tool_name, log_args) - else: - logger.info("Executing tool: %s with args: %s", tool_name, log_args) + log = tool_call_logger(tool_name) or logger.info + log("Executing tool: %s with args: %s", tool_name, log_args) results = await self.ability_manager.execute( ctx=ctx, diff --git a/openjiuwen/harness/docs/features/F_05_browser-decision-policy-slot.md b/openjiuwen/harness/docs/features/F_05_browser-decision-policy-slot.md new file mode 100644 index 0000000000..c8fb9f22bf --- /dev/null +++ b/openjiuwen/harness/docs/features/F_05_browser-decision-policy-slot.md @@ -0,0 +1,72 @@ +# F_05 浏览器子代理的决策策略槽位 + +## 元信息 + +| 项 | 值 | +|---|---| +| 类型 | feature | +| 日期 | 2026-09-19 | +| 范围 | `openjiuwen/harness/schema/decision_policy.py`（新增）、`openjiuwen/harness/subagents/browser_agent.py`（`create_browser_agent` 直通）、`openjiuwen/harness/tools/browser_move/runtime/runtime.py`（`probe_for_policy` / `activate_page`）、`tests/unit_tests/harness/tools/browser_move/` | +| 测试基线 | 见「验证」 | +| 关联 spec | [[S_18_subagents-and-lifecycle]]（不变量 6 增补） | +| Refs | 首个仓库外实现：agtai/jiuwen-jev（结构化决策模型接入 `model` 槽位） | + +## 背景 + +浏览器子代理的每一步由一次聊天模型回合决定：模型读取 `` 文本，再以自由文本生成 +工具调用 JSON，且可能命名未注册的目标。结构化决策模型换一种方式：一次请求带页面状态与若干选择题， +返回选项编号，几百毫秒内给出下一步，且只能选给出的编号。要让这类模型接到 `model` 槽位，装配层需要 +三件事：识别它、把运行时交给它、不再把 LLM 专用的处理器套在它身上。本次只落地这三个槽位；具体策略 +实现（HTTP 客户端、页面探测脚本、提示文本）在仓库外。 + +## 数据结构 + +- `DecisionPolicyModel`：`@runtime_checkable` Protocol，仅要求 + `bind_runtime(runtime: BrowserAgentRuntime) -> None`。 +- `BrowserAgentRuntime.probe_for_policy(source, params) -> dict`：`source` 为 `(params) => ...` 页面 + 函数，`params` 为其唯一实参。成功时返回脚本结果，`elements` 每项获得 `target_id`，字典获得 + `generation_id`；失败返回 `{"ok": False, "error": , "elements": [], "page_state": }`。 +- `BrowserAgentRuntime.activate_page(url) -> bool`。 + +## 决策 + +1. **协议而非基类**。装配层按结构判定（`isinstance(model, DecisionPolicyModel)`），不 import 具体 + 实现：否则策略的 HTTP 依赖会进入每次浏览器子代理构造的导入图，且新增第二种策略必须回来改 + `browser_agent.py`。 +2. **策略模型原样使用**。不做温度副本（策略没有采样温度）；不注入三个 LLM 专用的 + ContextProcessor（它们生成面向聊天模型的 `` 文本与工具结果窗口）； + `enable_model_anomaly_detection_rail=False`（该 rail 按输出文本重复判定异常，对结构化决策无 + 意义）。调用方仍可通过 `rails=` 自带 rail。 +3. **探测复用 `_execute_probe_json`**。策略函数包成 + `async (page) => await page.evaluate(source, params)` 交给既有代码执行器；结果经 + `_annotate_probe_generation` 与 `PageState.register_interactives` 注册，因此 `browser_click(target_id)` + 等既有工具的目标校验与代际检查不变。元素的 `selector_hint_validated=True, match_count=1` 由策略 + 自己保证（它给每个控件打唯一属性）。 +4. **`activate_page` 用 Playwright 的 `bringToFront()`**。后台标签页把定时器节流到约 1 s 且不渲染 + 下拉菜单，靠新探测决策的策略会反复点同一控件。`url` 用于在同一 context 内挑选目标页，否则前置 + 当前页。 +5. **失败信封而非异常**。探测脚本错误、代码执行器未就绪都返回结构化失败，策略据此降级（例如判 + BLOCKED），子代理回合不崩溃。`asyncio.CancelledError` 照常上抛。 + +## 拒绝的方案 + +- **新增工具或 rail 承载策略**：策略要替代的是"决定下一步"本身，只有 `model` 槽位在那个位置；工具 + 与 rail 只能在回合前后旁路。 +- **装配层 import 具体策略类判定**：见决策 1。 +- **同步支持声明式路径**（`build_browser_agent_config` / `SubAgentConfig`）：该路径没有放置策略实例 + 的位置，留作已知遗留。 + +## 验证 + +- `tests/unit_tests/harness/tools/browser_move/test_browser_policy_probe.py`：探测元素注册为目标并可解析 + 为 CSS 选择器、脚本收到 `source` 与 `params`；脚本异常与执行器未就绪的失败信封；`activate_page` 的 + 前置脚本与失败返回。 +- `tests/unit_tests/harness/tools/browser_move/test_create_browser_agent.py`：结构化判定；策略模型直通 + （同一对象、`bind_runtime` 收到运行时、异常检测 rail 关闭、不注入 ContextProcessorRail）。 +- 基线：`pytest tests/unit_tests/harness/tools/browser_move -q` → 改动后 **539 passed, 2 warnings in 1.95s**（含本文新增 7 个用例）。 + +## 已知遗留 + +1. 声明式路径（`SubAgentConfig` / `TaskTool` 派生的 browser_agent）没有放置策略实例的位置，仍用父模型。 +2. `probe_for_policy` 每次调用与 `probe_interactives` 一样写一份 audit artifact；高频策略探测下需要 + 评估目录增长。 diff --git a/openjiuwen/harness/docs/features/F_07_browser-drivers.md b/openjiuwen/harness/docs/features/F_07_browser-drivers.md new file mode 100644 index 0000000000..731afe9c2e --- /dev/null +++ b/openjiuwen/harness/docs/features/F_07_browser-drivers.md @@ -0,0 +1,5216 @@ +# F_07 Browser drivers + +## Meta + +| Item | Value | +|---|---| +| Type | feature | +| Date | 2026-10-15 | +| Scope | `openjiuwen/harness/tools/browser_move/` (`contract/`, `drivers/`, `provisioning/`, `surfaces/browser/`, `surfaces/browser/runner/`, `runtime/`, `runtime/utils/`, `playwright_runtime_mcp_server.py`), `openjiuwen/harness/subagents/browser_agent.py`, `openjiuwen/core/common/logging/tool_call.py`, `openjiuwen/core/single_agent/agents/react_agent.py`, `openjiuwen/core/context_engine/` (offload directory name), `tests/unit_tests/harness/tools/browser_move/` | +| Test baseline | see "Verification" in each section | +| Related spec | [[S_20_browser-driver-contract]] | +| Refs | #driver-architecture | + +## Background + +`browser_move` is being moved onto a driver contract: the agent core sends typed commands and reads typed results +and errors, and each driver (browser-use now, Playwright MCP next) translates them for one library. This document +records the decisions taken while the pieces land, one section per piece; S_20 describes the resulting contract. + +## Phases 0–2 + +- **Preparation.** The open browser PRs were merged first; the browser-use driver was ported from its earlier + branch with import changes only. WebVoyager baselines were taken on the Playwright MCP path (smoke on this branch) + and on the browser-use path (on the earlier branch). Every MCP tool and every browser-use tool was mapped onto the + commands of the contract; the mapping was approved before the contract was written. Outcomes that changed the + plan: one tool spec may cover several commands with an `action` field (`browser_tabs`); `FillForm` and `Drop` + became browser commands because both drivers handle them; `RunScript` means page JavaScript on every driver, and + the unsafe Playwright-code tool gets no command; `Forward`, `Reload` and `GetText` were dropped; the non-core MCP + tools were not carried over. +- **Safety net.** Before production code moved: architecture tests with allowlists of today's violations (imports, + driver names, browser words, env reads, module size), golden tool and prompt snapshots per path, a test that pins + what the WebVoyager runner imports and sets, recorded `@playwright/mcp@0.0.78` answers, and rail tests on a + command-level fake driver. The env-read test counts reads through `runtime/utils/env.py` helpers and key tuples, not only + `os.getenv` literals. Variables the runner sets but the package never reads (`BROWSER_INSTRUCTION`, + `BROWSER_LOGS_DIR`) stay in the pinned list, marked as such. The browser-use sidecar is a separate process with + its own configuration; its env reads are a named exemption. +- **Contract and browser-use on it.** `contract/` (stdlib only; `contract/testing/` may import pytest for the + conformance plugin), browser commands and `BrowserContext` in `surfaces/browser/`, a conformance suite with a + local browser scene, `managed_browser` moved to `provisioning/` (old name kept as an alias, `ProvisionedDevice` + as data), and the browser-use driver rewritten as a `HandlerDriver`. Decisions made on the way: `PolicyDenied` and + `ActionIntercepted` are raised before dispatch; a stale ref on browser-use is detected while resolving the node, + before any input; browser-use hit-tests the click point and raises `ActionIntercepted` instead of clicking an + overlay; sidecar replies report dialogs, popups and navigations (a same-URL reload included); browser-use accepts + `Locator(css)`. Live conformance on browser-use passes; the MCP goldens did not change. + +## Driver middleware + +Three wrappers in `drivers/middleware.py`, each a `Driver` around an inner `Driver`, stacked as +`Tracing(RetryOnStale(Timeouts(driver, per_effect)), sink)`. Contract touch-ups that came with them: +`StaleRef.observation`, `ActionIntercepted.default_dispatched = False`, `__sensitive_fields__` on `TypeText`, +`RunScript` and `FormField`, and `ProvisionedDevice.cleanup = None` by default. + +### Decisions + +1. **Wrappers are drivers.** Each middleware subclasses `Driver` and forwards `info`, `start`, `stop` and `health`, + so any number of them compose and the composition root sees one `Driver`. They import only `contract/`, and the + architecture tests scan `drivers/middleware.py` like the core. +2. **Timeouts by effect, expiry counts as dispatched.** The bound comes from the command's `effect` (`observe()` + is READ); an unlisted effect has no bound. On expiry the inner call is cancelled with `asyncio.timeout`, and + `ActionTimeout(dispatched=True)` is raised, because a cancelled call may already have reached the screen. +3. **RetryOnStale re-observes but does not resend.** The plan allows a retry only when the stale ref can be remapped + through a ref kind the driver declares (`IndexRef` or `Locator`). The middleware sees neither the observation the + stale ref came from nor a mapping from a `DriverRef` to an index or a selector, and neither built driver offers + one. So it observes once and raises `StaleRef` with that observation attached, which saves the runtime a second + `observe()`. Errors with `dispatched=True` pass through untouched. +4. **Masking is declared on the command.** `Tracing` masks every field named in a class-level + `__sensitive_fields__` when the instance's `sensitive` is true, walking nested dataclasses and sequences (so a + `FormField` inside `FillForm.fields` is found). A surface command opts in by declaring the attribute; the + middleware has no list of command types. +5. **Trace the shape, not the content.** Command records carry the masked arguments. Result records carry `ok`, + `detail` and `context_changed`; observation records carry the context id and the element count; error records + carry the class, message and `dispatched`. `ActionResult.value`, element values and native text are left out, + because they can echo what a sensitive command typed. +6. **The sink is a callable taking a dict** (`TraceSink`), so the conformance suite's collector and a logging hook + both fit without an adapter. +7. **`ActionIntercepted` defaults to not dispatched.** Both drivers raise it before any input is sent; the default + now says so. `ProvisionedDevice.cleanup` defaults to `None`, since most devices need no cleanup. + +### Rejected options + +- **Remap by visible text in `RetryOnStale`.** Re-targeting a stale `DriverRef` through `Locator(text=name)` could + hit a different element with the same name, which breaks "every driver means the same thing by the same command". +- **Remember the last observation in `RetryOnStale` to build a remap.** The wrapper would hold state the runtime's + page state already owns, for a remap no driver can use. +- **A per-command masking table inside `Tracing`.** Every new surface command would need an edit to the middleware, + which the plan forbids. +- **Gating the timeout on `Wait.seconds`.** Not done: the profile's READ bound has to be longer than the longest + `Wait` the model may ask for. + +### Verification + +- `tests/unit_tests/harness/tools/browser_move/drivers/test_middleware.py`: per-effect bounds, `dispatched=True` and + cancellation of the inner call, no bound for an unlisted effect; one re-observe whose observation the raised + `StaleRef` carries, no retry of dispatched errors, one send even when the driver stays stale; masking of + `TypeText`, `RunScript`, `FormField` and a test-local command with `__sensitive_fields__`; traced errors; the + stack passing `info`, `start`, `stop`, `health` through; the conformance suite on the fake world through the + full stack. +- `pytest tests/unit_tests/harness/tools/browser_move/drivers/test_middleware.py -q`: **50 passed, 1 skipped** + (the `IndexRef` happy path the fake scene does not provide). +- `pytest tests/unit_tests/harness/tools/browser_move -q`: **1225 passed, 59 skipped** (live suites without the + sidecar interpreter, and the browser-use tool snapshot). +- Live browser-use conformance, driver wrapped in `Tracing` + (`BROWSER_USE_SIDECAR_PYTHON=`): **45 passed, 0 skipped**; the two sensitive-trace + cases that skipped before now run. + +### Known leftovers + +1. Settled in "Environment boundary" (G11): the transport now sends `cancel` when the caller cancels the request. +2. Settled (G10): `RunScript.args` is a sensitive field. +3. Settled for sensitive commands (G10): their error message is not traced. Messages of other commands are traced + as they are. +4. The fake-surface end-to-end test expects a stale `PointRef` to be retried once; with no remap there is no retry, + so that expectation needs a remap rule for points first. + +## Settings + +`CoreSettings` in `runtime/config.py` and `BrowserSettings` in `surfaces/browser/config.py` parse the +environment variables, split by owner (S_20 "Settings"). This step adds the parsers and moves the public facades +(`RuntimeSettings`, `build_runtime_settings`, `build_playwright_mcp_config`, `build_browser_guardrails`, +`resolve_playwright_mcp_cwd`, `BrowserInstanceConfig`) onto them; no other module reads them yet. + +### Decisions + +1. **Parsers take the mapping.** `from_env(environ)` reads only what it is given; `from_process_env()` is the one + `os.environ` access per class. Tests pass a dict while `os.environ` holds conflicting values. +2. **Same precedence and quirks as the code they replace.** Each field reads the same key tuple in the same order, + with the same defaults. Kept on purpose: `PLAYWRIGHT_MCP_ARGS=""` means no arguments (not the default); the + worker tool timeout takes the first *set* key and falls back to 240 s if that value is invalid, without trying + the next key; the log mirror and raw-audit flags are on for any value but a false word; + `OPENJIUWEN_BROWSER_AGENT_LOG_FILE` set to an empty string means the current directory. +3. **One parse path.** The facades call `from_process_env()` instead of keeping their own reads, so the existing + config tests, the MCP goldens and the runner contract test exercise the parsers. +4. **`BROWSER_DRIVER` warns in one place.** Both classes read it: `CoreSettings` treats a legacy value as "default + driver" silently, `BrowserSettings` maps it to `provisioning` and warns. Building both gives one warning. +5. **`CoreSettings` does not reject an unsupported `MODEL_PROVIDER`.** The harness path takes its model from the + parent agent and calls only `build_browser_guardrails` / `build_playwright_mcp_config`; a `MODEL_PROVIDER` meant + for another tool must not break it. `build_runtime_settings` raises as before. +6. **`topology` is derived from `driver`**, a property, not a field read from the environment: the default per + driver is decided (delegated worker for Playwright MCP, direct for browser-use) and no variable exists; the + choice is measured in the benchmark phase. (Superseded by "Phase 6: topology setting": `topology` is a field + read from `BROWSER_TOPOLOGY`, and `""` means the default the driver's registration declares. The measurement + is in "Default topology for browser-use: delegated worker": both built-in drivers now register + `delegated_worker`.) +7. **No `tracing` field.** No variable switches tracing; the composition root passes the `Tracing` sink. The + existing raw-audit switch is `log_audit_raw`. +8. **`driver_mode` stays a field.** `BrowserInstanceConfig` appends `provisioning` and `driver` after its eight + positional fields; `__post_init__` copies whichever of `driver_mode` / `provisioning` is set into the other and + refuses two different values. `BrowserInstanceConfig(driver_mode=...)` and the dict form used by the teams + manifest keep working. +9. **Lazy where the old code was lazy.** `selector_cache_path=None` means the default under the home directory, and + `log_file` defaults to a relative path the logger resolves: computing `Path.home()` at parse time fails when the + home variables are absent (the existing tests clear the environment). +10. **The parsers do not call `runtime/utils/env.py`.** Its helpers keep serving the callers not switched yet; the parsing + logic is private to each config module. (Superseded by "Environment boundary", decision 2: both parsers now share + the pure helpers of `runtime/utils/env.py`.) `runtime/config.py` still re-exports `parse_command_args`, + `load_repo_dotenv` and the defaults it always exported. +11. **One allowlisted import.** `runtime/config.py` imports `surfaces.browser.config`, because the public + `RuntimeSettings` facade composes `BrowserSettings`; recorded in the import-boundary allowlist as legacy until + the facade can take the surface settings from the surface registry. The env-read test also learns env names + from the string constants of the config modules: parsers that read a mapping no longer show up as `os.getenv` + or `runtime/utils/env` calls, and the names must stay known for reads elsewhere to be caught. (The import is gone since + "Composition root", decision 4.) + +### Rejected options + +- **One settings class with a prefix per owner.** A second surface would edit it. +- **`CoreSettings` importing the provisioning names** to recognise legacy `BROWSER_DRIVER` values: the core would + depend on the browser surface; it keeps the three legacy names as a private literal. +- **`driver_mode` as a read-only property over `provisioning`.** Breaks the keyword argument and the dict coercion + that callers outside the package use. +- **New `BROWSER_TOPOLOGY` / tracing variables.** Not in the plan and nothing would read them. (`BROWSER_TOPOLOGY` + exists since "Phase 6: topology setting"; no tracing variable.) +- **Moving the runtime-MCP-server and child-process variables now.** Their owner is not clear (questions below). + +### Inventory + +Every variable the package reads today. Owner: **core** (`CoreSettings`), **browser** (`BrowserSettings`), +**exempt** (the browser-use sidecar process), **question** (not parsed yet). "Read in" is the module that reads it +today, relative to `browser_move/`. + +| Variables (precedence order) | Read in | Meaning | Default | Owner | Field | +|---|---|---|---|---|---| +| `BROWSER_DRIVER` | `runtime/config.py`, `service.py` | driver; legacy values = provisioning | `playwright_mcp` | core (browser reads legacy values) | `driver` | +| `BROWSER_TOPOLOGY` | `runtime/config.py` | `direct` / `delegated_worker`; empty = the driver's registered default | `""` | core | `topology` | +| `MODEL_PROVIDER`, `MODEL_CLIENT_PROVIDER` | `runtime/utils/env.py`, `browser_tools.py` | model provider | inferred from base URL, then keys, then `openai` | core | `provider` | +| `API_KEY`, `MODEL_API_KEY`, then per provider `OPENROUTER_API_KEY` / `SILICONFLOW_API_KEY` / `DASHSCOPE_API_KEY` / `OPENAI_API_KEY` | `runtime/utils/env.py`, `browser_tools.py` | model API key | `""` | core | `api_key` | +| `API_BASE`, `MODEL_API_BASE`, then per provider `OPENROUTER_BASE_URL`, `OPENROUTER_API_BASE`, `SILICONFLOW_BASE_URL`, `SILICONFLOW_API_BASE`, `DASHSCOPE_BASE_URL`, `DASHSCOPE_API_BASE`, `OPENAI_BASE_URL`, `OPENAI_API_BASE` | `runtime/utils/env.py`, `browser_tools.py` | model API base; also infers the provider | the provider's public URL | core | `api_base` | +| `MODEL_NAME` | `runtime/utils/env.py` | model name | `anthropic/claude-sonnet-4.5` | core | `model_name` | +| `BROWSER_WORKER_TEMPERATURE`, `BROWSER_MODEL_TEMPERATURE`, `MODEL_TEMPERATURE` | `agents.py` | worker sampling, 0–2 | `0.2` | core | `worker_temperature` | +| `BROWSER_WORKER_TOP_P`, `BROWSER_MODEL_TOP_P`, `MODEL_TOP_P` | `agents.py` | worker sampling, 0–1 | `0.1` | core | `worker_top_p` | +| `PLAYWRIGHT_TOOL_TIMEOUT_S`, `PLAYWRIGHT_MCP_TIMEOUT_S`, `BROWSER_TIMEOUT_S` | `agents.py` | worker per-tool-call timeout, s | `240` | core | `tool_timeout_s` | +| `BROWSER_TIMEOUT_S`, `PLAYWRIGHT_TOOL_TIMEOUT_S` | `runtime/utils/env.py`, `playwright_runtime_mcp_server.py` | task timeout, s (≥ 1) | `600` | core | `guardrails.timeout_s` | +| `BROWSER_GUARDRAIL_MAX_STEPS` | `runtime/config.py` | step limit (≥ 1) | `100` | core | `guardrails.max_steps` | +| `BROWSER_GUARDRAIL_MAX_FAILURES` | `runtime/config.py` | failure limit (≥ 0) | `4` | core | `guardrails.max_failures` | +| `BROWSER_GUARDRAIL_RETRY_ONCE` | `runtime/config.py` | retry once | `true` | core | `guardrails.retry_once` | +| `BROWSER_GUARDRAIL_RESUME_ON_MAX_ITERATIONS` | `runtime/config.py` | resume at the step limit | `false` | core | `guardrails.resume_on_max_iterations` | +| `OPENJIUWEN_BROWSER_AGENT_LOG_FILE` | `browser_logging.py` | agent log file; false word = none | `logs/browser_agent.log` | core | `log_file` | +| `OPENJIUWEN_BROWSER_AGENT_LOG_LEVEL` | `browser_logging.py` | agent log level | `INFO` | core | `log_level` | +| `OPENJIUWEN_BROWSER_AGENT_LOG_MIRROR_COMMON` | `browser_logging.py` | also log to the common logger | `false` | core | `log_mirror_common` | +| `OPENJIUWEN_BROWSER_AGENT_AUDIT_RAW` | `browser_logging.py` | store raw audit payloads | `false` | core | `log_audit_raw` | +| `BROWSER_SUBAGENT_STATUS_LOG` | `status_logging.py` | subagent status log | on | core | `status_log` | +| `BROWSER_PROVISIONING` (new), legacy `BROWSER_DRIVER` | — | where the browser comes from | `remote` | browser | `provisioning` | +| `PLAYWRIGHT_MCP_CDP_ENDPOINT`, `PLAYWRIGHT_CDP_URL` | `runtime/config.py` | CDP endpoint | `""` | browser | `cdp_url` | +| `PLAYWRIGHT_MCP_CDP_HEADERS`, `PLAYWRIGHT_CDP_HEADERS` | `runtime/config.py` | CDP headers | `""` | browser | `cdp_headers` | +| `PLAYWRIGHT_MCP_CDP_TIMEOUT`, `PLAYWRIGHT_CDP_TIMEOUT_MS` | `runtime/config.py` | CDP connect timeout | `""` | browser | `cdp_timeout` | +| `BROWSER_MANAGED_HOST` | `service.py` | managed Chrome host | `127.0.0.1` | browser | `managed_host` | +| `BROWSER_MANAGED_PORT` | `service.py` | managed Chrome debug port | `9333` | browser | `managed_port` | +| `BROWSER_MANAGED_KILL_EXISTING` | `service.py` | kill a running Chrome, use its default profile | `false` | browser | `managed_kill_existing` | +| `BROWSER_MANAGED_ARGS` | `service.py` | extra Chrome arguments (JSON list or shell string) | none | browser | `managed_args` | +| `BROWSER_MANAGED_USER_DATA_DIR` | `service.py` | Chrome user-data dir | per profile | browser | `user_data_dir` | +| `BROWSER_PROFILE_NAME` | `service.py` | profile name | `jiuwenclaw` | browser | `profile_name` | +| `BROWSER_PROFILE_STORE_PATH` | `service.py` | profile store file | `/.browser/profiles.json` | browser | `profile_store_path` | +| `BROWSER_MANAGED_BINARY` | `service.py` | Chrome binary | discovered | browser | `browser_binary` | +| `PLAYWRIGHT_MCP_ARGS` | `runtime/config.py` | Playwright MCP arguments | `-y @playwright/mcp@0.0.78` | browser | `launch_args` | +| `PLAYWRIGHT_MCP_COMMAND` | `runtime/config.py` | Playwright MCP command | `npx` | browser | `mcp_command` | +| `PLAYWRIGHT_MCP_TIMEOUT_S`, `BROWSER_TIMEOUT_S` | `runtime/config.py` | MCP client timeout, s (≥ 1) | `600` | browser | `mcp_timeout_s` | +| `PLAYWRIGHT_RUNTIME_MCP_CWD`, `BROWSER_RUNTIME_MCP_CWD`, `PLAYWRIGHT_RUNTIME_WORKDIR`, `BROWSER_RUNTIME_WORKDIR` | `runtime/config.py` | Playwright MCP working dir, profile root | current dir | browser | `mcp_cwd` | +| `PLAYWRIGHT_MCP_BROWSER` | `runtime/config.py` | Playwright browser name | `""` (`chrome` with CDP) | browser | `mcp_browser` | +| `PLAYWRIGHT_MCP_DEVICE` | `runtime/config.py`, `service.py` | device emulation (not with CDP) | `""` | browser | `mcp_device` | +| `PLAYWRIGHT_MCP_EXTENSION` | `runtime/config.py` | extension mode | `false` | browser | `mcp_extension` | +| `PLAYWRIGHT_MCP_EXTENSION_TOKEN` | `runtime/config.py` | extension token | `""` | browser | `mcp_extension_token` | +| `PLAYWRIGHT_BROWSERS_PATH`, `HTTP_PROXY`, `HTTPS_PROXY`, `NO_PROXY`, `PLAYWRIGHT_MCP_ENV_JSON` | `runtime/config.py` | Playwright MCP child env (JSON wins) | `{}` | browser | `mcp_env` | +| `BROWSER_UPLOAD_ROOT` | `runtime/utils/env.py`, `drivers/browser_use/transport.py` | upload sandbox root | none | browser | `upload_root` | +| `BROWSER_USE_SIDECAR_PYTHON` | `drivers/browser_use/transport.py` | sidecar interpreter | `.venvs/browser-use` | browser | `sidecar_python` | +| `OPENJIUWEN_BROWSER_SELECTOR_CACHE` | `site_profiles.py` | site selector cache | `~/.openjiuwen/browser_selector_cache.json` | browser (prefix: question 5) | `selector_cache_path` | +| `BROWSER_UPLOAD_ROOT` | `drivers/browser_use/sidecar/session_adapter.py` | upload root inside the sidecar | none | exempt | — | +| `PLAYWRIGHT_RUNTIME_MCP_*` / `BROWSER_RUNTIME_MCP_*` (`ENABLED`, `SERVER_ID`, `SERVER_NAME`, `CLIENT_TYPE`, `SERVER_PATH`, `HOST`, `PORT`, `PATH`, `COMMAND`, `ARGS`, `TIMEOUT_S`), `PLAYWRIGHT_RUNTIME_MCP_AUTO_HTTP_FALLBACK`, `PLAYWRIGHT_RUNTIME_MCP_TRANSPORT`, `PLAYWRIGHT_RUNTIME_MCP_LOG_LEVEL`, `PLAYWRIGHT_RUNTIME_MCP_STATELESS_HTTP`, `PLAYWRIGHT_RUNTIME_LOG_DIR`, `BROWSER_RUNTIME_LOG_DIR` | `browser_tools.py`, `playwright_runtime_mcp_server.py` | the browser runtime served as an MCP server | various | core (G2) | `runtime_server` (`RuntimeServerSettings`) | +| whole environment copy; `ALL_PROXY`; `PATH`, `SystemRoot`, `TEMP`, `TMP`, `USERPROFILE`, `HOMEDRIVE`, `HOMEPATH`, `LOCALAPPDATA`, `APPDATA`, `HOME`, `TMPDIR`; every `BROWSER_USE_*` | `browser_tools.py`, `drivers/browser_use/transport.py` | environment handed to child processes | — | pure functions of a passed mapping (G3) | — | +| `LOCALAPPDATA`, `ProgramFiles`, `ProgramFiles(x86)`, `ProgramW6432` | `provisioning/managed_browser.py` | where Chrome is installed; default Chrome user-data dir | — | browser (G4) | `browser_binary_candidates`, `local_app_data` | +| `AGENT_QUERY`, `AGENT_SESSION_ID` | `runtime/main.py` | demo entry point input | `""`, `demo-browser-session` | core reader (G5) | `read_demo_inputs()` | + +"Read in" is the state before the environment boundary step; owners of the former question rows were decided in +the wave 1 review (G2–G5). Counts at the settings step: 37 core names, 33 browser names plus the new +`BROWSER_PROVISIONING`, 1 exempt read, 45 names in questions +(28 runtime-MCP-server, 12 child-process plus the `BROWSER_USE_*` family, 3 more OS install paths, 2 demo). Names +read by both objects: `BROWSER_DRIVER`, `BROWSER_TIMEOUT_S`, `PLAYWRIGHT_MCP_TIMEOUT_S`. + +### Verification + +- `tests/unit_tests/harness/tools/browser_move/test_core_settings.py`, `surfaces/test_browser_settings.py`: every + parsed name with its default, an explicit value and, where one exists, an invalid value; the `BROWSER_DRIVER` + table, one `DeprecationWarning`, `BROWSER_PROVISIONING` winning; `from_env` ignoring a conflicting `os.environ`. +- `tests/unit_tests/harness/tools/browser_move/test_settings_facades.py`: `BrowserInstanceConfig` field order and the + `driver_mode` alias; `build_runtime_settings` equal to the two settings objects; unsupported provider still + rejected by it and not by the guardrail / MCP builders; extension provisioning reaching the MCP config. +- The three new files: **108 passed**. `pytest tests/unit_tests/harness/tools/browser_move -q`: **1333 passed, 59 + skipped** (same skips as before; goldens and runner contract unchanged). Harness tests that build the browser + subagent (`subagent_runtime/`, `task_loop/`, `test_deep_agent*.py`, `test_subagent_rail.py`, `tools/`): **248 + passed**. + +### Known leftovers + +1. Every module outside the two config modules still reads the environment itself; the env-read allowlist is + unchanged. (Settled by "Environment boundary".) +2. Facades now fail earlier on some invalid input: an unknown `BROWSER_DRIVER` raises when the guardrails or the + runtime settings are built (before: when the browser service started, and not at all when the instance carried a + `driver_mode`); a `BROWSER_MANAGED_PORT` that is not a positive integer raises whenever `BrowserSettings` is + built (before: only when managed Chrome started without a key or an explicit port). A legacy `BROWSER_DRIVER` + now warns from `build_playwright_mcp_config`, `build_runtime_settings` and `resolve_playwright_mcp_cwd`. +3. Questions on the names not parsed yet: + 1. The runtime-MCP-server variables configure `browser_move` served as an MCP server to another agent (client + registration in `browser_tools.py`, server entry point). Prefixes say browser, meaning says entry point. The + client reads both `PLAYWRIGHT_RUNTIME_*` and `BROWSER_RUNTIME_*`, the server only `PLAYWRIGHT_RUNTIME_*`, with + different defaults (path, log dir); one field per name pair would let the server honour the aliases. + 2. Child-process environments copy the whole environment or a list of OS variables. Proposal: they become pure + functions of the mapping the entry point passed to `from_env`, not settings fields. + 3. Chrome install discovery reads OS install paths; it belongs with the managed provisioner. + 4. `AGENT_QUERY` / `AGENT_SESSION_ID` are inputs of the demo entry point, not settings. + 5. `OPENJIUWEN_BROWSER_SELECTOR_CACHE` is browser by meaning but outside `ENV_PREFIXES`; core names also start + with `BROWSER_` (`BROWSER_DRIVER`, guardrails, timeouts), so the prefixes declare a namespace, not ownership. + +## Provisioning + +`provisioning/__init__.py` defines the `Provisioner` protocol and registers `managed`, `remote` and `extension`; +`ManagedBrowserProvisioner` implements all three (S_20 "Provisioning"). `BrowserService` no longer builds the +managed profile, resolves the port or starts Chrome: it takes a provisioner from the registry. The MCP path keeps +the same Chrome, flags, ports and profiles. + +### Decisions + +1. **One class, three registered names.** `ManagedBrowserProvisioner` already exists and keeps the Chrome-handle + API that the process registry, the heartbeat and reset use (`stop`, `is_endpoint_ready`, `clear`, + `owns_process`); the deprecated `ManagedBrowserDriver` alias keeps pointing at a class whose old constructor + still works. Remote and extension are a few lines each. +2. **Synchronous protocol.** The Chrome start is synchronous; the service calls `provision` in a worker thread, as + it called `start` before. +3. **The provisioner instance is the Chrome handle**, one instance per device. The service keeps it as its managed + driver, so the process registry still holds the same kind of object. `release(device)` runs the cleanup and is + what the composition root will call. +4. **Adopting a live Chrome and the healthy-driver check stay in the service**, before provisioning, in the old + order. The order is observable: an unkeyed service adopts the store's selected live Chrome even while its own + Chrome is healthy, and stops its own. `provision(settings)` cannot see the service's current driver; moving both + steps would need the current device as an extra argument (not the protocol) or per-service state in the + provisioner. +5. **The service passes its profile store** (`create(name, profile_store=...)`). The store caches the file; two + store objects in one service would overwrite each other's writes. Without a store the provisioner opens + `profile_store_path` or `mcp_cwd/.browser/profiles.json`, which is what the composition root needs. +6. **Instance overrides are mapped onto `BrowserSettings` in the service** (`dataclasses.replace`). A keyed + instance without a port gets `managed_port=0`, which the provisioner turns into a free port. The environment + parser refuses 0, so only this mapping produces it; no `key` field is needed. +7. **Device shapes.** `kind="chrome"` for all three, as in the plan. Extension: no endpoint, `extension=True` and + the token in `extras`. Remote: the CDP url plus headers and timeout. All three carry `launch_args` (Playwright MCP + arguments); Chrome's arguments are `managed_args`, following the settings naming. +8. **The service receives `BrowserSettings`** through the new keyword `browser_settings`; when it is absent the + service builds them once with `from_process_env()` at construction. `mcp_cwd` also comes from them instead of the + `resolve_playwright_mcp_cwd` facade. The type is imported from `runtime/config.py`, which already + imports it (allowlisted), so the core gains no import of the surface. +9. **Chrome install paths from the environment move to settings** (review decision G4): + `BrowserSettings.browser_binary_candidates` holds the `chrome.exe` paths under `LOCALAPPDATA`, `ProgramFiles`, + `ProgramFiles(x86)` and `ProgramW6432`, computed in `from_env` (Windows only). The home-based path and the fixed + macOS / Linux paths stay in the provisioner: `Path.home()` at parse time fails when the home variables are absent + (Settings decision 9). The lookup order is unchanged. + +### Rejected options + +- **Three small classes.** More code, and `ManagedBrowserProvisioner` would still be needed as the Chrome handle. +- **Adoption and health check inside `provision`.** Changes the order above, or needs the current device. +- **`provision(settings, current_device)`.** Not the protocol of the plan. +- **A `key` field in `BrowserSettings`.** A setting with no variable behind it; port 0 covers the one keyed rule the + provisioner needs. The other keyed rule (adopt only the own profile) stays with adoption in the service. +- **Provisioning `remote` / `extension` in the service.** Nothing to start; writing the endpoint again would differ + from the MCP config builder (device check, browser default). +- **Renaming the `env_map` reads in the service** so the env-read scanner stops seeing them. They touch the MCP + child's env mapping, not the process environment; hiding them from the scanner proves nothing. + +### Verification + +- `test_provisioning.py` (+17): registered names; remote and extension devices without a Chrome start; managed + device, profile reuse and rebuild, free port, kill-existing user-data dir, store fallbacks, release, install paths + from the settings. `surfaces/test_browser_settings.py` (+3, one skipped per OS): `browser_binary_candidates`. + `test_service.py` (+8): registry call with the service store, instance overrides, port precedence, no + provisioning for remote / extension, given settings winning over the environment, environment read once. +- Tests changed because the code they patched or called moved (no assertion weakened): `test_service.py` (three + tests patch `PROVISIONERS.create` instead of `service.ManagedBrowserProvisioner`; two profile-building tests call + `_build_profile(service._provisioning_settings())`; the stored-binary test runs the real provisioner with `start` + patched and builds the service under the cleared environment); `test_browser_instance_isolation.py` (six tests + patch `managed_browser._allocate_free_port` and call `_build_profile(...)` instead of + `BrowserService._build_managed_profile`); `drivers/test_conformance_browser_use_live.py` (passes the settings' + install paths to `_candidate_chrome_binaries`). +- Architecture allowlists: core imports of concrete modules 2 entries -> 1 (`service.py -> + provisioning.managed_browser` removed); env reads in `service.py` 13 descriptors / 22 reads -> 3 / 3, in + `provisioning/managed_browser.py` 4 / 5 -> 1 / 1; `service.py` browser words 93 -> 92, driver names 16 -> 14, + lines 1977 -> 1904. +- `pytest tests/unit_tests/harness/tools/browser_move -q`: **1360 passed, 60 skipped** (before: 1333 / 59; MCP + goldens and runner contract unchanged). Harness tests that build the browser subagent: **262 passed**. + +### Known leftovers + +1. `provisioning/managed_browser.py` still reads `LOCALAPPDATA` for the default Chrome user-data dir (kill-existing + without a user-data dir); G4 covered the install paths only. (Settled by "Environment boundary": + `BrowserSettings.local_app_data`.) +2. The three `service.py` entries left in the env-read allowlist touch the Playwright MCP child's env mapping. They + go when the MCP driver owns its config. +3. The service stops Chrome through the handle API, not `release(device)`: the process registry shares handles + among services and calls `stop` itself. +4. `service.py` imports `BrowserSettings` through `runtime/config.py`; when that module stops importing + the surface (review decision G1), the service needs another way to the type. (Settled by "Composition root", + decision 4.) +5. Visible differences, all from reading settings once: the browser variables are read at service construction, + not at Chrome start; a legacy `BROWSER_DRIVER` also warns when the service is built; `BROWSER_DRIVER` set to a + driver name no longer raises in the service; an invalid `BROWSER_MANAGED_PORT` raises at service construction in + every mode; the profile-store writes around a start run in the worker thread. +6. `PLAYWRIGHT_MCP_EXTENSION=true` with `remote` or `managed` puts Playwright MCP into extension mode (config + builder), while the provisioner is chosen by `provisioning` alone and hands out a remote or managed device. + (Without an explicit provisioning, settled by "Composition root", decision 5.) +7. An invalid instance `driver_mode` still raises "BROWSER_DRIVER must be one of: remote, managed, extension". + (Settled by "Composition root", decision 6.) +8. Settings leftover 3.3 (Chrome install discovery) is settled by decision 9; Settings leftover 1 no longer applies + to `service.py`. + +## Environment boundary + +Only `runtime/config.py` and `surfaces/browser/config.py` read the process environment (plan invariant +9); every other module receives settings or values from them (S_20 "Environment boundary"). Wave 1 review decisions +G2, G3, G5, G7 are implemented here, together with the two middleware follow-ups G10 and G11. + +### Decisions + +1. **Settings come from the caller; a missing one is built through the config module.** Entry points (`main.py`, + the runtime MCP server) build the settings once and pass them down; `BrowserAgentRuntime` and `BrowserService` + take them as keywords. Public functions that are called without settings today (tests, the harness subagent + factory, `browser_agent.py`) fall back to `X.from_process_env()` at the moment they read the environment before, + so the read timing does not move. `browser_agent.py` is not changed in this step. +2. **`runtime/utils/env.py` is pure (G7).** It keeps the defaults and one set of parse helpers used by both config + modules; the duplicated `_first` / `_int` / argument split / truthy sets / provider aliases in the config modules + are gone. Helpers that read the environment and have no caller left are deleted (`first_non_empty_env`, + `resolve_int_env`, `resolve_bool_env`, `resolve_model_name`, `resolve_browser_timeout_s`, `resolve_upload_root`, + `resolve_model_settings`); `load_repo_dotenv` and `resolve_repo_dotenv_path` move to + `runtime/config.py`, which already re-exported the former. +3. **`RuntimeServerSettings` nested in `CoreSettings` (G2)**, with two parsers keeping the client and server + readers apart: the client takes the `BROWSER_RUNTIME_*` aliases, the server does not (except the log dir, which + it always took). `port` stays a string and `args` a raw string, so an invalid port still fails where it is used, + and the client's own argument splitting (`posix` by OS, whitespace fallback) is unchanged. +4. **Child-process environments are pure functions in the config modules (G3)**: `runtime_server_child_env(environ)` + and `runtime_server_environ(environ)`; callers pass `process_environ()`. The old passthrough loop of + `_build_child_env` copied each listed variable onto a copy of the same environment and is dropped as a no-op. The + sidecar environment stays in `transport.py`: it copies OS and `BROWSER_USE_*` names from the given mapping and + takes the upload root from `BrowserSettings.upload_root` (resolved; the sidecar resolves it again). +5. **The runtime MCP server stops writing `os.environ`.** Its tool timeout defaults (`BROWSER_TIMEOUT_S` and + `PLAYWRIGHT_TOOL_TIMEOUT_S` = 600) are applied to the mapping its settings are parsed from, and the settings go + to the runtime, the service and the worker; the worker tool timeout under the server stays 600 s. Two + module-level slots hold the settings `main()` built, next to the server's existing `_runtime` slot; without + `main()` the runtime reads the process environment as before. +6. **Demo inputs through `read_demo_inputs(environ)` (G5).** +7. **`browser_logging` stays an allowlisted exception.** Configuring the logger from `CoreSettings` at a composition + point needs process-global state (the plan forbids module-level state), and code that never passes a composition + point (core `react_agent`, the live driver, tests) would lose `./logs/browser_agent.log`. It moves with the + logging hook into `openjiuwen.core` (plan Phase 4); done in "Tool-call logging hook". +8. **`BrowserSettings.local_app_data`** carries `LOCALAPPDATA` for the default Chrome user-data dir, next to the + install paths of G4. +9. **G10**: `RunScript.args` joins `__sensitive_fields__`; `Tracing` leaves `message` out of the error record of a + sensitive command. +10. **G11**: `SidecarTransport.request` sends `cancel` for its id when it is cancelled, then re-raises. + +### Rejected options + +- **Configuring `browser_logging` from settings now** (decision 7). +- **Child-environment builders left in `browser_tools.py` / `transport.py` as allowlisted "mapping, not process + env" entries.** The names they spell are env names; the config modules are where env names are spelled. +- **Leaf defaults read from settings inside `runtime/controllers/` or `runtime/utils/`.** Both are core; they would import the + browser surface settings. They take the value; `None` means not configured. +- **A sentinel to tell "not passed" from "not configured" for the upload root.** Every caller in the package + passes the value. + +### Verification + +- `test_settings_passing.py` (+34): `RuntimeServerSettings` client and server parsing (alias order, defaults, + timeout, fallback flag, tri-state stateless flag, nesting); pure environment mappings; `main()` building the + settings once and passing them to the runtime; `build_runtime_settings`, the runtime, the runtime MCP config, the + worker agent, the status flag, the selector cache, upload paths, the sidecar environment and the browser-use + factory using the given values over a conflicting process environment. +- `drivers/test_middleware.py` (+4, G10), `test_browser_use_transport.py` (+1, G11). +- Existing tests changed because a patched symbol or a signature moved (no assertion weakened): + `test_file_uploads.py` (passes `upload_root` instead of patching `resolve_upload_root`), + `test_browser_use_upload_validation.py` (same), `test_config.py` (the runtime log test passes + `RuntimeServerSettings` and an empty environ instead of patching `_build_child_env`), + `test_browser_probe_cards.py` (the patched `get_selector_cache` takes the path), `test_browser_runtime_rail.py` + (the bare runtime gets `_core_settings`), `drivers/test_conformance_browser_use_live.py` (passes + `BrowserSettings.from_process_env()` to the factory, which no longer reads `BROWSER_USE_SIDECAR_PYTHON` itself). +- Architecture allowlists: env reads 14 files / 82 descriptors / 92 reads -> 3 / 9 / 9 (sidecar, `browser_logging`, + the `service.py` env mapping); `KEY_TUPLE_ENV_NAMES` removed (every name is now spelled in a config module); + browser words `config.py` 8 -> 9 and `main.py` 4 -> 3 (the demo session id moved); module size `runtime.py` + 6383 -> 6396, `service.py` 1904 -> 1907 (settings keywords). +- `pytest tests/unit_tests/harness/tools/browser_move -q`: **1399 passed, 60 skipped** (before: 1360 / 60; MCP + goldens, tool/prompt snapshots and the runner contract unchanged). Harness tests that build the browser subagent: + **248 passed**. +- Fix after review: on Windows a `dict` snapshot of `os.environ` keeps only upper-case names, so the sidecar lost + `SystemRoot` (it exited before its hello; every live browser-use case failed) and the Chrome install roots lost + `ProgramFiles*`. `utils.env.env_value` looks a name up as given and upper-cased; the env-read rule now counts only + `runtime/utils/env.py` helpers that read the process environment. Live browser-use conformance: **45 passed**. + +### Known leftovers + +1. `browser_logging` reads the environment (decision 7). Resolved in "Tool-call logging hook". +2. The three `env_map` entries of `service.py` (Provisioning leftover 2). +3. `runtime/controllers/action.py` `list_upload_files`: ruff now reports ASYNC240 on its existing `upload_root.exists()`, + because the root became a typed parameter; behaviour unchanged. +4. Invalid input fails earlier in a few places: `ensure_execute_signature_compat`, `build_browser_worker_agent` and + `is_browser_subagent_status_log_enabled` without settings build `CoreSettings`, so an unknown `BROWSER_DRIVER` + raises there too. +5. Behaviour for callers outside the package: `register_builtin_actions()`, `resolve_upload_file_paths()` and a + `BrowserSelectorCache()` without a path no longer read `BROWSER_UPLOAD_ROOT` / `OPENJIUWEN_BROWSER_SELECTOR_CACHE`; + `create_browser_use_driver(device, None)` no longer honours `BROWSER_USE_SIDECAR_PYTHON`. `utils` no longer + exports the deleted helpers. +6. `python -m openjiuwen.harness.tools.browser_move.playwright_runtime_mcp_server` fails before this step's code at + the clients import, now `from drivers.playwright_mcp.clients.stdio_client import ...` (relative import beyond the + top-level package); not changed here. + +## Composition root + +`runtime/assembly.py` builds the driver stack from registry names: surface, provisioner, device, +driver, middleware, start (S_20 "Composition root"). The built-in drivers, surfaces and provisioners are +registered with lazy strings. The core no longer imports the browser surface. In this step the root ends at a +started driver; `create_browser_agent` and the Playwright MCP path keep their wiring. + +### Decisions + +1. **The surface factory returns `SurfaceBundle(settings, profile=None)`.** The root needs the surface settings now + and the profile later. A bundle lets the browser factory add its profile without changing the factory's return + type or the root's `surface.settings` access. `SurfaceProfile` already has a `settings` field, but it cannot be + built yet (it needs commands and a renderer), and a profile with placeholder fields would be a stub. +2. **The registry packages re-export their registry.** `from ..drivers import DRIVERS` imports the package that + registers the built-ins and gives back the filled registry; the root needs no import only for its side effect. + `drivers/__init__.py` re-exports `DRIVERS`, `surfaces/__init__.py` re-exports `SURFACES` and + `provisioning/__init__.py` re-exports `PROVISIONERS`. +3. **"Not built yet" lives in `Registry`.** `playwright_mcp` is registered with its final string. Resolving a string + whose own module is missing raises `ModuleNotFoundError` with that message; a missing dependency inside an + existing module is raised unchanged. A raising placeholder factory would need an edit when the driver lands, and + the same message also helps a third-party registration that has a typo in it. +4. **G1 / H5: the core types surface settings as `SurfaceSettings`.** This protocol has one member, + `provisioning`, and lives in `contract/surface.py`. `config.py` gets the browser settings with + `browser_surface_settings(environ=None)`, which is `SURFACES.create("browser", ...)`. `service.py`, `runtime.py`, + `main.py` and the MCP server use that function or take the settings from their caller. The private MCP config + builder `_playwright_mcp_config` takes `Any`: it reads a dozen browser fields, the protocol does not declare them, + and it moves into the Playwright MCP driver. +5. **H4: `PLAYWRIGHT_MCP_EXTENSION=true` without a provisioning selects `extension`**, in + `BrowserSettings.from_env`. An explicit `BROWSER_PROVISIONING` wins, and so does a legacy `BROWSER_DRIVER` + provisioning value, because it is an explicit choice in the deprecated spelling. One visible effect: the + browser service identity now says `extension` instead of `remote` for that environment. +6. **H6:** the instance validation error names `BrowserInstanceConfig.provisioning (driver_mode)` and + `BROWSER_PROVISIONING`. +7. **Per-effect timeouts are `core.tool_timeout_s` for every effect.** That is today's per-tool-call bound + (`PLAYWRIGHT_TOOL_TIMEOUT_S`, default 240 s), so it is longer than the longest `Wait` (G8). The surface profile + supplies per-effect values once it exists. +8. **Provisioning runs in a worker thread.** The provisioner protocol is synchronous (Provisioning, decision 2), so + the root calls `provision` and `release` through `asyncio.to_thread`, as the service does. +9. **The trace sink writes to the debug log**, one record per call, already masked by `Tracing`. The root gets + no sink parameter: no caller needs a different sink yet, and the tests read the stack and the log. +10. **Failure handling.** A failure after provisioning releases the device. A failed `start` also stops the + stack, because a driver may hold part of its connection. `close()` releases the device even when `stop` + raises. +11. **`Registries` is a frozen dataclass passed in by keyword.** `None` means the three default registries. + There is no module-level default object and no other state. + +### Rejected options + +- **The surface factory returns `SurfaceProfile` with only `settings` filled.** The required fields would need + placeholders. +- **The surface factory returns the bare settings now and the profile later.** That changes the return type in + Phase 4, and every caller of the factory would change with it. +- **A placeholder `playwright_mcp` factory that raises.** It is stub code, and it must be edited when the driver + lands. +- **Side-effect imports (`from .. import drivers # noqa`) in the root.** Decision 2 makes the dependency explicit. +- **`await provisioner.provision(...)`.** The protocol is synchronous (Provisioning, decision 2). +- **A `trace_sink` parameter on `assemble_driver`.** Speculative; nothing passes one. +- **Typing `_playwright_mcp_config` with `SurfaceSettings`.** It reads attributes the protocol does not declare. + +### Verification + +- `test_assembly.py` (+20): call order with fake registries (surface with the given environ, provisioner by the + settings' provisioning, provision, driver with device and settings, start); middleware order; every `Effect` + bounded by `tool_timeout_s` (`ActionTimeout`, dispatched); refusal of a driver that does not serve the surface, + with the device released and the driver not started; the device released when the driver factory, `start` or + `stop` fails; no driver when `provision` fails; trace records in the debug log. Also the built-in names, the + browser surface returning its settings and no profile, the default registries reaching `playwright_mcp` ("not + built yet"), and a subprocess check that importing the package and then the root loads no `browser_use`, browser-use + or Playwright MCP driver, browser surface or managed-browser module. +- `contract/test_registry.py` (+2): "not built yet", and a missing dependency raised unchanged. + `surfaces/test_browser_settings.py` (+4, H4). `test_settings_facades.py` (+1, G1: the facades take the settings + from the surface registry, once per call, with the process environment). `test_service.py` (+1, H6). +- `test_assembly_live.py` (+1, `live`): managed Chrome from the provisioner (headless, free port, temporary + profile and store) and the browser-use driver started through `assemble_driver`; `observe()` on `about:blank`; + after `close()` the CDP endpoint is gone and no Chrome process uses the profile. A probe run saw 11 Chrome + processes before `close()` and 0 after it. +- Tests changed for moved symbols (no assertion weakened): `test_service.py` imports `BrowserSettings` from + `surfaces/browser/config.py`. +- Architecture allowlists: core imports of concrete modules 1 entry -> 0 (`config.py -> surfaces.browser.config` + removed); module size `service.py` 1907 -> 1910 (H6 message); browser words, driver names and env reads + unchanged; the root itself has no browser word, driver name or env read. +- `pytest tests/unit_tests/harness/tools/browser_move -q`: **1429 passed, 61 skipped** (before: 1401 / 60; the new + skip is the live test without the sidecar interpreter; MCP goldens, tool/prompt snapshots and the runner + contract unchanged). Harness tests that build the browser subagent: **262 passed**. Live, with + `BROWSER_USE_SIDECAR_PYTHON` set: browser-use conformance **45 passed**, `test_assembly_live.py` **1 passed**. + +### Known leftovers + +1. `create_browser_agent`, the browser service and the runtime MCP server do not build through the root yet. The + service still provisions managed Chrome itself and adopts a live Chrome from the profile store (H1); the root + does not adopt. +2. Per-effect timeouts come from `CoreSettings.tool_timeout_s` until the surface profile supplies them. Done: see + "Agent construction, instance settings and restart on a dead device". +3. `mypy` on `config.py`, `service.py` and `runtime.py` reports 13 more errors than before (26 instead of 13). They + are `attr-defined` (and one `type-var` on `dataclasses.replace`) where browser code in the core reads browser + fields on a `SurfaceSettings`. `make type-check` is advisory. The errors go away when that code moves to the + surface and the MCP driver, or earlier if the stored settings are typed `Any`. +4. A legacy `BROWSER_DRIVER=remote|managed` wins over `PLAYWRIGHT_MCP_EXTENSION=true` (decision 5), so in that + environment the device and the MCP config still disagree, as before. + +## Playwright MCP result decoding + +`drivers/playwright_mcp/snapshot.py` parses `@playwright/mcp` text (sections, page header, tab lists, ARIA YAML) +and `drivers/playwright_mcp/result.py` decodes one tool result into an `ActionResult` or a typed error (S_20 +"Playwright MCP"). They are pure functions; the driver class that calls them comes next. The recorded fixtures +grew from 14 to 33 cases so that every decoding rule has a real server answer behind it. + +### Decisions + +1. **`decode` takes `before`, the `BrowserContext` before the call.** The server marks neither a navigation by + click nor a popup: the result of a link click differs from a plain click only in the URL under `### Page`, and a + new tab only shows up as a longer `### Open tabs` list. A pure decoder cannot see a change without the previous + state, so the caller passes it; without it, only navigate, back, tab commands and dialogs set + `context_changed`. The task named the signature `decode(tool_name, raw_result, *, output_dir)`; `before` is a + keyword-only addition with a `None` default. +2. **New tabs are the tabs past `len(before.tabs)`.** The server numbers tabs by position and appends new ones; + `browser_tabs new` and a `target="_blank"` click both add the last index. A tab list is shown only with more + than one tab, so a result with `### Page` and no list means one tab with id `"0"`, and `tabs` always contains + the focused tab. +3. **Every `browser_tabs` result is a context change**, `list` included. The driver uses `browser_tabs` only for + open, switch and close (observing reads the tabs from the snapshot), so telling `list` apart would need the + arguments for no gain. +4. **I4: the linked snapshot file is read under `output_dir`**, the server's working directory, because the links + are relative to it. The resolved path must stay inside it; a missing, unreadable or empty file gives + `observation=None`. The server writes the file empty while a dialog blocks the snapshot, so an empty file is + not an empty page. +5. **I5 on the recorded call logs.** `performing click action` / `click action done` → `ActionTimeout(dispatched=True)` + (recorded: a click on a link whose server never answers times out waiting for the navigation). A log of only + actionability waits (`waiting for locator(…)`, `locator resolved to`, `attempting/retrying … action`, + `waiting for element to be …`, `element is …`, scrolling, `waiting Nms`, `waiting for getBy…(…) to be visible`) + → `dispatched=False`; any other line → `True`. "Performed" is checked before "intercepted": a click retried past + an overlay and then performed was dispatched. +6. **`ConnectionLost` on two wordings.** After the browser died during a call, the next call fails with + `Protocol error (Target.createTarget): Failed to open a new tab` and the one after with + `Target page, context or browser has been closed`. Both mean the browser is gone, and a restart is the recovery. +7. **A call refused by an open dialog returns `ok=False`** (`Tool "…" does not handle the modal state`, with the + `### Modal state` in `detail`). The server sends nothing in that case; browser-use returns `ok=False` for an + action while a dialog is open, and the two drivers should agree. Not a listed error rule. +8. **`detail` drops `### Result`, `### Snapshot` and `### Events`.** The result is in `value` (or `surface` for a + tab list), the snapshot in `observation`; events are not reported (the loop guard used to strip them). The + remaining text, `### Ran Playwright code`, `### Page`, `### Open tabs` and `### Modal state` included, is what the + model sees today. +9. **A file chooser is not a context change.** It is a modal state but not a dialog: refs stay valid and + `HandleDialog` does not answer it. It stays visible in `detail`. +10. **`ActionIntercepted.by` uses the browser-use form** `tag#id.class "text"`, built from the element preview in + the call log, so a rail reads one format from both drivers. +11. **Element values only for form controls.** `value` is filled for textbox, searchbox, combobox, spinbutton and + slider, from the inline text, a child `text:` line (how a filled textbox appears) or the `[selected]` option. + A paragraph's text goes to `extra["text"]`, because `value` means a field value on browser-use. + +### Rejected options + +- **The driver compares the URL and the tabs itself.** It would put part of the decoding rules in the driver; + `before` keeps them in one place and testable on fixtures. +- **Passing the `browser_tabs` arguments to tell `list` from `new`/`select`/`close`.** Not needed (decision 3). +- **Reading `### Snapshot` file links with any path.** The text comes from a subprocess; the read is confined to + `output_dir`. +- **`DriverError(dispatched=True)` for the modal-state refusal.** Correct by the catch-all rule but it would make + the two drivers disagree on the same situation. +- **Treating every timeout as dispatched.** The recorded logs separate an action that never started from one that + ran, and I5 asks for that split. + +### Verification + +- New fixtures, recorded with the extended `record_mcp_fixtures.py` (three new pages, `popup.html`, + `upload.html`, `slow.html`, and a `/hang` route that answers after 30 s): `click_stale_ref`, + `snapshot_with_boxes`, `fill_form`, `snapshot_after_fill_form`, `select_option`, `evaluate_result`, + `evaluate_error`, `screenshot_element`, `wait_for_timeout`, `click_blocked_by_dialog`, `click_opens_tab`, + `snapshot_two_tabs`, `click_navigates`, `navigate_back`, `file_chooser_open_by_click`, `file_upload`, + `click_timeout_after_action`, `evaluate_new_tab_failed`, `evaluate_browser_closed`. The new cases run after the + old ones, which re-record byte-identical; two full re-recordings produced identical files, the screenshot PNG + included. +- `drivers/playwright_mcp/test_snapshot.py` (13) and `test_result.py` (58): every fixture through the parser or + the decoder with its outcome (ok, `context_changed`, or the error class and `dispatched`); the form page refs + `e1`–`e18` plus `f1e1`–`f1e3`; boxes, filled values, checked state, YAML quoting, tab lists; `### Events` and file + links gone from `detail`; same-URL reload, link click and popup as context changes with `opened_contexts`; a + linked snapshot read from a temporary `output_dir`, `None` when missing, empty or outside it; the blocker + description; screenshot value and size. +- `pytest tests/unit_tests/harness/tools/browser_move -q`: **1538 passed, 61 skipped** (before: 1429 / 61; +71 + parser and decoder tests, +38 fixture well-formedness cases). + +### Known leftovers + +1. The driver class, its handlers and the factory are not built; nothing calls the decoder yet. +2. Tabs are identified by position. A command that closes one tab and opens another in the same call keeps the + count and is not seen as a change; a tab closed before the focused one shifts the ids. +3. The decoder reads the linked snapshot file synchronously; the driver should call it off the event loop. +4. No `ConnectionLost` wording for a dead MCP server process: that is a transport error, not text, and belongs to + the driver. A browser closed by a tool call between calls is not an error at all: the server starts a new + browser on the next call (a stale ref is what the model then sees). +5. `last_status` and the scroll pixels of `BrowserContext` stay at their defaults: the server reports neither. + +## Playwright MCP driver + +`drivers/playwright_mcp/driver.py`: `PlaywrightMcpDriver(HandlerDriver)` over a `@playwright/mcp` server it starts +and owns, plus the registry factory `create_playwright_mcp_driver`. Nothing outside tests uses it yet; the agent +still runs the old Playwright MCP path. + +### Decisions + +1. **The factory reuses today's server config builder.** `_playwright_mcp_config` became the public + `playwright_mcp_config(settings, instance, required_capability)`; the factory passes the device endpoint as the + instance CDP URL. One builder, so the driver launches the same server the runtime does. +2. **The clients return the raw `CallToolResult`.** `call_tool_result` is the transport part of the old + `call_tool`; `call_tool` keeps its text extraction on top, so the runtime path is unchanged. The decoder needs the + content blocks and `isError`, not the joined text. The driver itself calls the single-attempt `call_tool_once` + (see "Playwright MCP live conformance"). +3. **Tool hints come from the server.** `start()` reads `tools/list` once and keeps the description of each mapped + tool; whether they or our own texts are better is a benchmark question. +4. **`native_text` of `observe()` is the whole `browser_snapshot` text**, so a renderer can reproduce today's model + input exactly; an observation read from a linked file carries the file's YAML. +5. **The driver keeps the last `BrowserContext`** from observe and every result and passes it to `decode`; that is how + a navigating click and a popup become context changes. +6. **Coordinates are screenshot pixels at scale 1** (screenshots are taken at CSS scale), so `PointRef` goes to the + coordinate tools unchanged. +7. **`count=2` is `doubleClick`; more is `Unsupported`.** `TypeText` without `clear` must be `slowly` (the tool + replaces the content otherwise). + +### Rejected options + +- **A second copy of the MCP command-line logic in the driver.** Two builders would drift; decision 1. +- **Going through `ToolMgr` and the client patch.** The driver owns its client; the patch dies with the old path. + +### Verification + +- `drivers/playwright_mcp/test_driver.py` (81) on a fake MCP session: pinned `info.actions`; one case per handler + with the exact tool and arguments; `Unsupported` cases send nothing; observe with and without screenshots; + start/stop/health; context tracking across calls; the factory's config equals `build_playwright_mcp_config` for + remote, managed and extension devices. +- `test_assembly.py`: the `playwright_mcp` registry name now builds a `PlaywrightMcpDriver`. +- `pytest tests/unit_tests/harness/tools/browser_move -q`: **1619 passed, 61 skipped**; live browser-use + conformance and assembly: 46 passed. + +### Known leftovers + +1. Fixed with the live conformance run: the re-sent tool call after a timeout or transport error, and every client + exception read as `ConnectionLost`. Live conformance now runs (next section). + +## Playwright MCP live conformance + +The conformance suite runs live on `PlaywrightMcpDriver` +(`drivers/playwright_mcp/test_conformance_playwright_mcp_live.py`): the registry factory over a headless Chrome with +a fresh profile, `@playwright/mcp@0.0.78` attached over CDP, `Tracing` around the driver, the browser scene. Every +failure of the first run was a driver or decoder bug or the test's server setup; the suite and the scene are +unchanged. + +### Decisions + +1. **One attempt per tool call.** The clients' `call_tool_result` reconnects and sends the call again after a + timeout or a transport error, so a click could run twice (invariant 7). The clients got `call_tool_once` (no + reconnect, no retry) and a `connected` property; the driver uses only those. Transport timeout → + `ActionTimeout(dispatched=True)`; no connection → `ConnectionLost(dispatched=False)`, nothing sent; any other + failure → `ConnectionLost(dispatched=True)`. The retrying `call_tool` stays for the runtime path. +2. **`observe()` asks for boxes and strips them from the text.** Without `boxes` the snapshot has no bounds, so no + `PointRef` can be built from an observation, although the driver offers the coordinate tools. With `boxes` and + ` [box=…]` removed, the text is byte for byte today's `browser_snapshot` text (checked live on both scene pages), + so the model input does not change. +3. **A `browser_wait_for` timeout is dispatched.** Its call log (`waiting for getByText(…) to be visible`) reads + like an actionability wait, which made it `dispatched=False`; but the wait is the command itself. +4. **Sensitive commands drop the echoed code from `detail`.** The server echoes what it ran + (`### Ran Playwright code`, e.g. `fill('')`), and `Tracing` records `detail`. `decode(..., + sensitive=True)` leaves the section out for `TypeText` and `RunScript` with `sensitive=True` and for a `FillForm` + with a sensitive field. Other commands keep it: it is part of what the model reads today. +5. **The live test starts the server with `--allow-unrestricted-file-access`.** The server reads upload and drop + files only under its working directory and its output directory; the scene keeps its upload file in its own + temporary directory. The driver's production command line is unchanged (Known leftovers 1). + +### Rejected options + +- **Masking the typed text inside `detail`.** The server quotes it as a JavaScript string literal, so escaped forms + would slip through; dropping the code section is exact. +- **Dropping `PointRef` from the driver's handled refs.** Today's MCP agent clicks by coordinates in vision mode. +- **Adding `--allow-unrestricted-file-access` to the driver's command line.** It also opens `file://` navigation; a + security choice, not a driver fix. +- **A server working directory that contains the scene's files** (the system temp directory). The server would write + its output files into a shared directory the test cannot clean up. + +### Verification + +- `drivers/playwright_mcp/test_conformance_playwright_mcp_live.py`: first run 9 failed, 42 passed (the three + `PointRef` happy paths: no bounds; `Drop` ×2 and `UploadFile`: file outside the server roots; the timeout case: + `dispatched=False`; sensitive text in the trace ×2). After the fixes: **51 passed** in two runs; no + `@playwright/mcp` node process left after either run. +- `test_assembly_live.py`: `BROWSER_DRIVER=playwright_mcp` over a managed Chrome: `assemble_driver` starts the + server, `observe()` works, `close()` leaves no process the test started (`npx`, the server, Chrome) and frees the + CDP port. Both assembly cases: 2 passed. +- New unit tests, each failing on the code before its fix: a call that times out on a fake session is sent once and + raises `ActionTimeout(dispatched=True)`; a closed transport is not re-sent; no connection sends nothing; observe + sends `boxes` and returns bounds with box-free text; `without_boxes`; `wait_for_timeout` is dispatched; a + sensitive call's `detail` has no code (decoder; `TypeText`, `FillForm`, `RunScript` through the driver). +- Browser-use live conformance: **45 passed** in two runs. `pytest tests/unit_tests/harness/tools/browser_move -q`: + **1681 passed, 61 skipped** (the MCP live module and the MCP assembly case included; goldens unchanged). + +### Known leftovers (differences between the drivers for Phase 4) + +1. **Upload and drop files must be under the MCP server's working directory** (`mcp_cwd`) or its output directory; + browser-use reads any path (only under `upload_root` when that is set) and answers a refused path with + `ok=False`. The MCP refusal is `DriverError(dispatched=False)` ("Open dialogs and refused file access"). Phase 4 + decides whether the runtime stages files into the workspace, or the server gets roots or the flag. +2. **`Wait(until_text, seconds)` ignores `seconds` on MCP**: `browser_wait_for` takes no timeout for a text, the + server's own applies (5 s on 0.0.78). browser-use waits up to `seconds`. +3. **Context ids are tab positions on MCP** and shift when an earlier tab closes; browser-use ids are target ids. +4. **Bounds only from `observe()`**: the observation that comes free with an MCP action result has none, so a + `PointRef` is built from an `observe()`. +5. **`detail` differs**: MCP carries `### Ran Playwright code`, `### Page` and `### Modal state` there; browser-use + none of them. A sensitive MCP call's `detail` has no code section. +6. **An open dialog blocks every other MCP tool** (`ok=False`, the modal state in `detail`). +7. **An MCP transport timeout comes after `mcp_timeout_s`** (600 s by default); the `Timeouts` middleware + (`tool_timeout_s`) normally ends the call first, also as `ActionTimeout(dispatched=True)`. +8. **`observe()` with boxes costs one box per element** on large pages; measured in the benchmark phase. +9. **The live MCP test runs wherever `npx` and Chrome exist** (gated on them, not on an environment variable) and + needs network the first time `npx` fetches the server. + +### Live test gating + +The live Playwright MCP cases (conformance and assembly) run only with `BROWSER_MOVE_LIVE_MCP=1`, as the browser-use +live cases need `BROWSER_USE_SIDECAR_PYTHON`. Without the gate they ran in every unit run on a host with `npx` and +Chrome (about 2.5 minutes, network access for `npx`). `Wait(until_text, seconds)` on this driver uses the server's own +text-wait deadline (5 s); S_20 allows a driver without a deadline for a text wait to use its own. + +## Open dialogs and refused file access + +`BrowserContext.dialog`, an `observe()` that works while a JavaScript dialog blocks the page (both drivers), and +the Playwright MCP file-access refusal counted as not dispatched. Decision E13 of the Phase 2 review and K2 of the +Phase 5 close. + +### Decisions + +1. **`dialog` is a string on `BrowserContext`, `": "`.** It is browser data, so it lives in the + surface context, not in `Observation` or `ActionResult` (S_20, rule 3). It is set in every observation and every + action result taken while a dialog blocks the page, and `None` otherwise. A dialog armed by an earlier + `HandleDialog` is answered at once and never shows. +2. **browser-use: the sidecar answers `observe` from cached target info.** browser-use keeps URL, title and tabs as + cached target data, which needs no CDP call and works while the dialog holds the page; the DOM, the screenshot + and the `document.title` script do not. The reply has `dialog` and no elements, and the driver then runs no + script and takes no screenshot. Wire version 3: act replies also carry `url` and `dialog` (the dialog still open + when the reply is built), so a click that opens a dialog and an action a dialog refused both get + `surface = BrowserContext(url, dialog)`. +3. **Playwright MCP: the tab list stands in for the snapshot.** While a modal state is open the server refuses + `browser_snapshot` like every page tool (`Tool "browser_snapshot" does not handle the modal state` plus + `### Modal state`; no `### Page`, no snapshot file; recorded as `snapshot_during_dialog`). `browser_tabs` is not + a page tool and still answers, with the URL, the tabs, an empty title (the server cannot read it) and + `### Modal state` (`tabs_list_during_dialog`). So `observe()` sends one `browser_tabs list` after a refused + snapshot; `native_text` is the refusal text, which is what the model reads today when it asks for a snapshot + during a dialog. A file chooser takes the same path (empty elements, no `dialog`). +4. **The decoder reads the dialog from `### Modal state`**: `["" dialog with message ""]: can be + handled by …`. The server does not escape the message, so it ends at the first `"]: can be handled by`. A call a + dialog refuses stays `ok=False` and gets `surface` = `before` with the dialog: nothing was sent, so the page is + the one before the call. +5. **The conformance `dialog` case checks an observation.** `check_context_change` observes right after the + command when the case has a `check`, before the cleanup answers the dialog. The browser scene's check wants + `"confirm: Proceed?"` in the click's result surface and in that observation; the suite stays surface-neutral. +6. **A refused file access is `DriverError(dispatched=False)`.** `@playwright/mcp` 0.0.78 checks every upload and + drop path before it touches the page; the decoder matches `File access denied: … is outside allowed roots` + right after the stale-ref rule. The fixture `drop_outside_roots` comes from the recorder's server, which runs + without `--allow-unrestricted-file-access`: a drop of a file in a second temporary directory (``). + +### Rejected options + +- **MCP: the blocked observation from the last known context.** No extra call, but it misses a navigation that + happened with the dialog (a dialog on load) and is not the server's own view; the tab list is a real read. +- **MCP: the linked snapshot file while a dialog is open.** `browser_snapshot` returns no link then; the empty + linked file belongs to action results, which already give `observation=None`. +- **browser-use: keep `ObserveError` and let the runtime build the dialog view from the last act reply.** Every + caller would need that special case; the driver knows the dialog. +- **A `dialog` field on `Observation` or `ActionResult`.** Browser-only data that a non-browser driver would fake. + +### Verification + +- Unit tests, each new one failing on the code before the change (15 failed on the code before): + - sidecar (`test_browser_use_sidecar.py`): `observe` during a dialog returns URL, title, tabs, the dialog and no + elements and sends nothing to the page; act replies carry `url` and the open `dialog` (opened, refused, handled); + an armed dialog leaves `dialog` empty. The `observe` row of "page reads fail at once" is gone. + - browser-use driver: the act result surface with an open dialog (opened, refused); no dialog, no surface; + `observe` during a dialog sends only `observe`. + - MCP decoder: `dialog` in the surface of `dialog_open_by_click` and `tabs_list_during_dialog`, `before` plus the + dialog for a refused click, none for a file chooser, a message with quotes and a line break; + `drop_outside_roots` is `DriverError(dispatched=False)`. + - MCP driver: `observe` during a dialog sends `browser_snapshot` then `browser_tabs list`, no screenshot. + - scene: the `dialog` check needs the dialog in both the result and the observation. Conformance self-test: a + driver whose observation hides the dialog fails the case. +- Fixtures: three new cases, recorded live; the 33 earlier fixtures re-recorded byte-identical. +- `pytest tests/unit_tests/harness/tools/browser_move -q`: **1653 passed, 113 skipped**. +- Live, with the new `dialog` check: browser-use **45 passed**; Playwright MCP **51 passed**. + +### Known leftovers + +1. **browser-use navigation replies carry no `dialog`.** A page that opens a dialog on load shows it in the next + `observe()` or action result; the MCP navigation result has it (`### Modal state`). +2. **The MCP title is empty during a dialog** (the server cannot read it); browser-use reports its cached title. +3. **An MCP file chooser gives an observation without elements and without `dialog`**; the modal state is only in + `native_text`. browser-use has no file-chooser state. +4. **Where upload and drop files may live on MCP is unchanged** ("Playwright MCP live conformance", Known leftovers + 1); only the refusal is now `dispatched=False`. + +## Tool-call logging hook + +`core/single_agent/agents/react_agent.py` imported `browser_logging` from the harness to send browser tool-call +lines to `browser_agent.log` (core → harness, the wrong direction). The hook now lives in core +(`openjiuwen/core/common/logging/tool_call.py`), the harness registers the browser log with it, and +`browser_logging` is configured from `CoreSettings` instead of the environment (S_20 "Browser-agent log and the +tool-call logging hook"). This resolves Environment boundary decision 7 and its Known leftover 1. + +### Decisions + +1. **Core asks for a logger by tool name.** `tool_call_logger(name)` returns the registered `log(message, *args)` + whose matcher accepts the name, or `None`; `ReActAgent` falls back to the core logger. The line + (`Executing tool: %s with args: %s`) and the argument redaction stay in core. +2. **The registry is one module-level dict in core.** Agents are built by callers that never see the harness (the + DeepAgent, the worker, an application's own agent given the runtime MCP tools); routing per agent would need each + of them to pass the logger. Registering the same matcher again replaces its entry, so repeated agent builds do + not stack. +3. **`configure_browser_agent_logging(settings)` applies the four logging fields and registers the browser + matcher.** Called by `create_browser_agent` (`CoreSettings.from_process_env()`, as that factory has no core + settings), `build_browser_worker_agent` (its settings) and `register_browser_runtime_mcp_server` + (`CoreSettings.from_env(environ)`): the three places that give an agent browser tools in this repo. The matcher + is the one core used (`browser_` anywhere or a `mcp_playwright` prefix, case-insensitive). +4. **The applied settings live on the stdlib logger, not in a new module variable.** The logger + `openjiuwen.browser_agent` is already process-wide; level, propagation and the file handler were already set on + it. The audit flag is stored on it too, and "no log file" is read from the absence of the marked file handler. +5. **Until the first configuration, the first use configures from `CoreSettings.from_process_env()`.** The + browser-use driver and transport fetch the logger at import time; the MCP clients, the runtime and tests log + without passing a composition point. Defaults alone would drop an env-configured path (or create + `./logs/browser_agent.log` when the file is disabled); the env read stays in the config module. +6. **Reconfiguration replaces the file handler.** Same path: the handler stays (no second truncation). Other path or + none: the old handler is closed. Before, a changed path added a second handler and both files were written. +7. **`task_tool.py` keeps its import.** It is harness importing harness (the `browser_agent` invoke line); not in + scope. + +### Rejected options + +- **Passing the tool-call logger into each agent (config field or constructor).** No module state, but agents built + outside the harness would stop routing browser lines; a behaviour change for the runtime MCP topology. +- **Registering at import of `browser_logging`.** Import side effect, and it could not take settings. +- **Defaults (`CoreSettings()`) before the first configuration** (decision 5). +- **Passing `audit_raw` from `runtime.py` to `write_browser_agent_audit_artifact`.** Removes the flag from the logger + but changes five call sites in `runtime.py`, which Phase 4 rewrites. + +### Verification + +- New tests: `tests/unit_tests/core/common/log/test_tool_call.py` (3: nothing registered, matching names only, + re-registration replaces), `tests/unit_tests/agent/react_agent/test_react_agent_tool_call_logging.py` (2: without a + registered logger every line goes to the core logger; with one, browser lines go to it and the rest to core), + `test_browser_logging.py` (7: file, level and mirroring from settings; reconfiguring moves the file; no file falls + back to the common logger; an unconfigured logger takes the process settings; configuring routes `browser_*` and + `mcp_playwright*` lines to the browser file; the two audit tests, moved from + `test_browser_subagent_status_logging.py` and switched from environment variables to settings). +- Architecture allowlists: `KNOWN_CORE_TO_HARNESS_IMPORTS` without `react_agent.py`; `KNOWN_ENV_READS` without + `browser_logging.py` (3 files / 9 reads -> 2 / 4). Browser-word counts and module sizes unchanged. +- `pytest tests/unit_tests/harness/tools/browser_move -q`: **1653 passed, 113 skipped** (goldens and runner contract + unchanged). Harness tests that mention `react_agent` or `browser_agent`, `tests/unit_tests/agent/react_agent/` and + `tests/unit_tests/core/common/log/`: **759 passed, 2 skipped**. +- Full unit suite (without `agent_teams/cli/` and `agent_rl/online/test_ppo_executor.py`, which need + `prompt_toolkit` / `omegaconf`): 101 failed, 20959 passed, 23 errors; every failure and error also fails on the + commit before this change (Windows paths and symlinks, `grep`, missing `chromadb`, Pulsar). +- Live: browser-use conformance **45 passed**, Playwright MCP conformance **51 passed**. + +### Known leftovers + +1. `CoreSettings.from_process_env()` raises on an unknown `BROWSER_DRIVER`; the import-time fetch in + `drivers/browser_use/driver.py` and `transport.py` now runs it, so such a value fails at import of the + browser-use driver instead of later. +2. Core keeps its own `_is_browser_tool_name` for the argument redaction in `_summarize_tool_args_for_log`; the + routing matcher is a copy in `browser_logging.py`. +3. A process that logs browser tool calls through a `ReActAgent` without passing one of the three registration + points now writes them to the core log. +4. The registry and the logger state are process-wide; tests reset them (`_TOOL_CALL_LOGGERS`, the marked handler and + the audit attribute). + +### Logger fallback reads only the log settings + +Before anything configures it, the browser-agent logger takes its settings from the process environment on first +use. It used to parse the whole `CoreSettings`, so an invalid unrelated value (an unknown `BROWSER_DRIVER`) raised at +import time in every module that fetches the logger (`drivers/browser_use/driver.py`, `transport.py`). +`CoreSettings.logging_from_process_env()` reads only the four log fields; everything else keeps its default. + +## Page state, driver session and policy rail + +Phase 4 items 3, 4 and 6 and the security rail of item 5: `BrowserPageState` learns contract types, +`DriverSession` recovers from a lost connection, `ActionPolicy` checks commands before the driver (S_20 "Runtime +state and recovery"). Nothing in `runtime.py`, `service.py`, `agents.py` or `browser_agent.py` calls them yet; the +agent switch wires them. + +### Decisions + +1. **One page state, extended in place.** `record`, `check`, `generation_of`, `resolve`, `reset` and the + `RefResolver` attributes sit next to the text-snapshot API, which the runtime still uses. The per-ref body of + `register_ax_snapshot` became `_register_ax_ref`, shared by text snapshots and typed elements; the regex path is + deleted only when the runtime stops reading snapshot text. +2. **New-generation triggers, in order:** `context_changed`; an address change without the flag (today's + backstop); a context id change; an element-set change. The element change is the Jaccard distance over + `(ref, role)` pairs, strictly above `element_change_threshold` (default `0.5`, constructor parameter, no env + var). `Observation.fingerprint` is not used: it hashes names too, so any text change would bump. +3. **A context change forgets every context.** `last_observation`, `surface_context`, `contexts` and + `current_context` are dropped, then refilled from the outcome (J8: MCP context ids are positions). Without the + drop, the next observation would be compared with the previous document and could bump a second time. +4. **`record` takes an `Observation` too.** `observe()` returns an `Observation`, not an `ActionResult`; passing it + directly keeps one entry point and lets `free_observations` count only observations that came with the result + of another command (plan item 4). +5. **Observations replace the current AX targets.** An `Observation` is complete, like the full snapshot of + `capture_browser_state`; refs missing from it leave the current generation. +6. **Stamping includes element refs.** `Element.ref.generation` gets the generation too, so every ref in a stamped + observation can go through `check`. +7. **`check` reports every stale ref at once** (as `validate_reference_values` does), with today's AX-ref and + selector wording; point and index refs get the same shape. A ref with an empty generation passes. +8. **`resolve(target_id, generation)` returns `DriverRef` for AX targets and `Locator(css)` for validated probe + selectors;** a link-only target is a `ValueError` (the primary-link rail turns those into `Navigate`). +9. **The session restarts the same driver object.** Re-provisioning gives a new `ProvisionedDevice`, but the + driver is not rebuilt: the provisioners normally hand back the same endpoint (remote: the configured `cdp_url`; + managed: the port stored in the profile; extension: none). `DriverAssembly` gained `surface_settings` so the session can call + `provision` again; the session replaces its assembly with the new device so `close()` releases it. +10. **Re-provisioning is triggered by `start()` raising a `DriverError`;** `health()` is not consulted. +11. **The failed call raises `ConnectionLost(driver_restarted)` with the original `dispatched`;** the command + itself is never re-sent. The second loss in a task, or a failed restart, raises a `BaseError` + (`DEEPAGENT_RUNTIME_ERROR`, `driver_lost`), which is not a `DriverError`, so a tool cannot render it as an + ordinary tool error. `begin_task()` resets the counter. +12. **`ActionPolicy` takes `builders` keyed by the model-facing tool name.** The tool factory does not exist yet; + a name → decoder map is what it produces, whatever prefix the driver declares. +13. **Denial as in `BrowserRuntimeRail._deny_tool_call`, with the Phase 4 text.** Per-call `_skip_tool_calls` + (the base rail's `_skip_tool` is invocation-wide and parallel calls share `ctx.extra`), message + `### Error\n`, metadata `executed/state_changed/denied`, `PolicyDenied` as the tool result. The runtime + payload of today's denial (`runtime_state`, codes) belongs to the browser rail, not to the policy. +14. **A `ValueError` from parsing or decoding is a denial,** so a malformed `browser_tabs` index never reaches the + driver (J8). + +### Rejected options + +- **A new `PageState` module next to the old one.** The plan says extend; two registries would disagree on the + generation. +- **Rebuilding the driver from the registry after re-provisioning.** Needs the driver name, the registries and the + middleware chain in the session (a second composition root) for a case no provisioner produces today. +- **`health()` after a restart to detect a dead device.** The MCP server answers `ping` with Chrome gone, so it + would not decide anything there; the next command reports the loss instead. +- **Using the base `_apply_reject`.** It sets the invocation-wide `_skip_tool`. +- **Letting the rail's `PolicyDenied` propagate as an exception.** The harness would treat it as a tool failure + after the call started; the plan wants it before anything runs. + +### Verification + +- New tests: `test_browser_page_state_record.py` (18: each trigger incl. a same-address reload via + `context_changed`, the threshold and its parameter, a role change, stamping, AX targets from elements, stale refs + nested in `FillForm`, stale point/locator refs, unknown current refs, contexts replaced on change, the + free-observation counter, `resolve`, `reset`, the page state as a decoder's `RefResolver`), + `test_driver_session.py` (8, `FakeDriver`: restart with message and reset, the command sent once, second loss + fails, `begin_task`, re-provisioning, failed restart, `observe`, `close` after re-provisioning), + `test_action_policy.py` (7: callback registered, denial message/metadata/skip/`PolicyDenied`, allow, command and + surface context passed (D15), other commands and unknown tools, decode error, no driver call through the ability + manager). +- Existing `test_browser_page_state.py` (25) unchanged and green. +- Browser-word allowlist: `runtime/page_state.py` 26 → 29 (the address backstop); the two new modules + have none. +- `pytest tests/unit_tests/harness/tools/browser_move tests/unit_tests/harness/rails + tests/unit_tests/harness/test_deep_agent_rail_event_routing.py` (live assembly deselected): **2762 passed, 111 + skipped** (architecture, goldens and runner contract included). +- Live: `test_assembly_live.py` **2 passed** (`DriverAssembly` field). + +### Known leftovers + +1. Nothing calls `record`/`check`/`DriverSession`/`ActionPolicy` yet; `capture_browser_state` still observes after + actions, so the saved observations are counted only once the tools use `record`. +2. The address backstop and `contexts` read `url`, `title` and `tabs` from the surface context by attribute name: + the core cannot import `BrowserContext`. They move with `page_state.py` into the browser surface (Phase 9). +3. A re-provisioned device with a different endpoint would need a rebuilt driver (decision 9). Done: see "Agent + construction, instance settings and restart on a dead device" (decisions 1–3 replace decisions 9 and 10). +4. The browser profile's `messages` (`driver_restarted`, `driver_lost`) and `policy_rules` come with the profile + task; the session reads the two messages at construction. Done: see "Browser surface profile". + +## Browser surface profile + +Phase 4 item 1 (the profile half): the contract gains what a model-facing tool needs (native tools, tool specs with +a schema and a decoder, a ref resolver, a renderer, contextual policy rules), and `create_browser_surface` returns a +filled `SurfaceProfile` (S_20 "Surface types", "Browser surface profile"). The tool factory that turns the profile +into tools comes next; nothing in `runtime.py`, `service.py`, `agents.py` or `browser_agent.py` reads the profile +yet. + +### Decisions + +1. **Native tools live in `DriverInfo`, not in the profile (Q1).** `native_tools` (spec name → `NativeTool`) and + `tool_name_prefix` come from the driver, so `surfaces/` never names a driver and the MCP schemas follow the + server version. `SurfaceProfile.native_tools` stays as the plan's extension point for surface-level formats + (a computer-use tool) and is empty for the browser. +2. **`RefResolver` sits in `contract/refs.py`** next to the ref types it returns, so `driver.py` (`NativeTool`) + and `surface.py` (`ToolSpec`) can both use it without importing each other. It names `contexts` and + `current_context` as well as `resolve` and `generation_of`: `browser_close` and a close without an index need + the focused context. +3. **`ContentBlock` is `str | Screenshot`.** The only image a browser tool returns is a screenshot; a separate + image type would duplicate `Screenshot`. +4. **`PolicyRule.check(action, context)` (D15)** stays a dataclass field holding a callable, so `rule.check(...)` + reads the same as a method. +5. **`SurfaceProfile.capability_tools` next to `capability_catalog`** (decided with Aleh, 2026-10-10). The pairs + (C26) cannot say which tools an entry selects (`browser_click` and `browser_mouse_click_xy` are both `Click`), + and the factory in the core cannot import `surfaces/browser/capabilities.py`. The factory reads the names from + `capability_tools` and a spec's ref kinds from the entry's pairs. +6. **`BrowserContext.tab_urls` and `current_tab`** (decided with Aleh, 2026-10-10). The tab list the design asks + for (`- i: (current) [title](url)`) needs every tab's URL and the focused tab; `ContextRef` carries neither and + the MCP decoder drops both. The renderer uses the fields now; the drivers fill them later. Until then a line + shows a URL and `(current)` only when known. +7. **One tool spec module for the browser dialect.** The 17 browser-use schemas are copied verbatim from + 857239db; the golden is the oracle. `browser_tabs` adds `new` to its enum (Q6) and keeps the description. The + vision, pdf, console and network tools get specs without schema or decoder: the catalog needs their commands, + and only a driver's native tool serves them today. +8. **Decoders map ids without asking the driver.** `target_id` goes through `RefResolver.resolve`, `ref` becomes + `DriverRef(ref, generation_id)`, `selector` becomes `Locator(css)`, in that precedence and at the generation the + model sent; staleness is the page state's `check`. Ignored arguments (`wait_until`, `timeout_ms`, drag `steps`) + are accepted (C16, C30); `steps` and `timeout_ms`-like integers are still validated where today's tool did. + Error texts keep today's wording. +9. **A close without an index closes the current tab** (design §2, MCP semantics); 857239db required an index. + A select needs an index; one outside `contexts` is a `ValueError` (J8). +10. **`renderer.result(Observe(), r)` is the tab list.** `browser_snapshot` renders through `observation()`; + `result()` with a plain `Observe` only answers `browser_tabs list`, so the renderer needs no flag. An `Observe` + with a screenshot renders the detail and the image. +11. **A failed result renders as an error.** `ActionResult(ok=False)` shows `detail`, prefixed with `### Error` + unless it already starts with it (the MCP modal refusal does; browser-use failure details do not). +12. **Messages:** `driver_restarted` is the plan's wording; `context_changed` is today's stale-ref sentence + (`runtime.py`, as a format string with `{generation}`); `driver_lost` has no wording today and is new. +13. **The catalog keeps only served names.** An entry lists the public tool names that a spec or the + `browser_find` extra tool serves; entries with no command (`advanced_code`, `unsafe_dev`, `config`, `storage`, + `testing`) stay as keys with no names and no pairs, so the factory can log them once (C25). +14. **`scene` stays `None`.** The conformance suite takes `browser_scene()` directly; importing the scene's HTTP + server with every profile is not needed yet. + +### Rejected options + +- **Copying the MCP schemas into the profile keyed by driver name.** Needs a driver name in `surfaces/` and drifts + with the server. +- **Replacing the catalog pairs by names and adding ref kinds to `ToolSpec`.** A larger contract change, and C26 + pairs are what the skip-and-log rule reads. +- **Choosing the current tab by title.** Ambiguous (two tabs with the same title in the fixtures) and loses the + other tabs' URLs. +- **A tab-list flag on `Observe`.** A model-facing detail in a shared command; the renderer method already tells + the two cases apart. +- **Rendering the free observation of a result.** Q14: kept for the page state, not shown. + +### Verification + +- Contract: `contract/test_tool_types.py` (9): native tools and prefix defaults, `HandlerDriver.info` keeping them, + decoders with a resolver, the renderer protocol on a profile, `PolicyRule.check` with the context, + `capability_tools` default. +- `surfaces/test_browser_tools.py` (252): each of the 17 dialect specs equals its entry in all 11 browser_use + golden views (187 comparisons), the golden tools without a spec are exactly cancel, clear-cancel, find and + recall, 33 decoder cases, 25 invalid-argument cases. +- `surfaces/test_browser_renderer.py` (57): the recorded MCP answers through the result decoder (14 action + results, 5 tab lists, 9 errors, 2 refusals, 3 snapshots) and FakeDriver results; one test name per expected + difference of design §4 (snapshot link, Events, plain error, stale ref with observation, intercepted message, + RunScript values, tab lists, element description, boxes, modal refusal). +- `surfaces/test_browser_profile.py` (13): fields, timeouts, messages, no driver names, catalog entries and + pairs, `core` equal to `CORE_BROWSER_TOOL_NAMES`. +- `test_assembly.py`: the browser surface now returns its profile (one assertion changed). +- `pytest tests/unit_tests/harness/tools/browser_move`: **2018 passed, 113 skipped** (architecture, goldens and + runner contract included). +- Live: browser-use conformance **45 passed**, Playwright MCP conformance **51 passed**. + +### Known leftovers + +1. MCP part done with the tool factory (the MCP decoder fills `tab_urls`/`current_tab`). Browser-use + `SwitchContext` and `CloseContext` results carry no surface at all, so their tab list is empty until the + browser-use helpers task. +2. Done with the tool factory: the MCP driver fills `DriverInfo.native_tools` at `start()`. +3. `browser_find` has no spec; its description and schema come from the native tool (MCP) or the 857239db text + (browser-use) when it becomes an extra tool. +4. Browser-use results have no `detail` for most commands, so many render as no text; the browser-use wire + fixtures show what the model reads there. + +## Tool factory and Playwright MCP native tools + +Phase 4 item 1 (the factory half): the Playwright MCP driver offers its server tools as native tools, and +`ToolFactory` turns the browser profile plus a driver's `DriverInfo` into model-facing `CommandTool`s (S_20 "Tool +factory", "Native tools"). The tool-list goldens are rebuilt from the factory alone. No agent uses the factory yet; +nothing in `runtime.py`, `service.py`, `agents.py` or `browser_agent.py` changed. + +### Decisions + +1. **MCP decoders are the inverse of the handlers.** `drivers/playwright_mcp/tools.py` holds one decoder per server + tool a capability can select (24). The test decodes every argument set the driver's handler test sends and checks + that the driver sends exactly those arguments again. A `target` matching `ARIA_REF` is a `DriverRef` at the + generation the page state last saw it in, anything else a `Locator(css)`; `doubleClick` is `count=2`; + `slowly` types without clearing (`clear=not slowly`), which is what the server does. +2. **Cosmetic and dropped arguments are accepted and ignored** (C11, C12, C16): `element` everywhere, `filename`, + the snapshot's `target/depth/boxes`, the screenshot's `target/element/type/scale` (element screenshots are + dropped), the click's `delay`. Points carry no generation (`PointRef(x, y, "")`): the model sends none and the + page-state check lets an empty generation pass. +3. **Native tools come from `tools/list` at `start()`.** Only tools with a decoder become `NativeTool`s; the schema + is the card the client already lists (`sanitize_json_schema`, the same path the goldens were captured through), + the description is verbatim. `tool_name_prefix` is `mcp_model_tool_prefix(server_name)`, the function the + ability manager uses (Q2: no instance-key suffix). +4. **The MCP result decoder fills `tab_urls` and `current_tab`** (Q-B) from every header: `PageHeader` gains + `tab_urls`; a page without `### Open tabs` has its own URL as the one entry. +5. **`build(capabilities, *, image_support, session, page_state)`.** The task wrote `build(capabilities, *, + image_support)`; a `CommandTool` needs the session and the page state at call time and the `ToolContext` for + extra tools is made of them, so they are arguments of `build`. `ToolFactory(profile, driver_info)` stays as + designed and selection does not depend on them. +6. **`Observe` counts as handled on every driver**: it is `Driver.observe`, never in `info.actions`. +7. **Rule 4 (generated schema) needs ref kinds.** Without that restriction browser-use got a `browser_mouse_wheel` + tool with an empty schema (`Scroll` is handled, the catalog pairs it with `None`), against C21 (no `Scroll` tool + on browser-use). A spec with no schema and no ref kinds to generate from is skipped and logged. +8. **Screenshot gate without tool names.** The core may not name tools (architecture tests). Without image support + the factory drops a spec whose only command is `Observe` and whose decoder, called with no arguments, asks for a + screenshot: `browser_take_screenshot` on both dialects; `browser_snapshot` and `browser_tabs` are untouched. +9. **Observe outcomes render by the tool.** A tool whose only command is `Observe` renders `renderer.observation()` + (the snapshot text); `browser_tabs` (several commands) renders `renderer.result()`, the tab list. +10. **`CommandRecord.surface` instead of `url`/`title` fields.** `url` is a browser word, forbidden in the core; + the typed `SurfaceContext` carries both, and the browser rail reads `record.surface.url` / `.title`. + `command` is the command type, not the instance, so no typed text or script sits in the record. +11. **The record carries the message metadata** (decided with Aleh, 2026-10-10). A tool cannot set + `ToolMessage.metadata`: the ability manager builds the message from `render_for_llm` after `invoke` + (`ability_manager.py`, "Build ToolMessage for successful execution"). `CommandRecord.message_metadata` gives + `success`, `executed`, `state_changed`, `denied` with today's meaning (`state_changed` for a success that may + change the screen, a timeout, or a context change; `executed=False` for anything stopped before the page, which + the context processors treat like today's denial); the rail copies it in `after_tool_call`. +12. **Stale before the driver.** A `StaleRef` from `page_state.check` costs one `observe()`; the observation is + recorded and rendered under the error (G9). If that observe fails, its own error is what the model reads. +13. **`browser_find` stays out of this task** (decided with Aleh, 2026-10-10): it has no command (C10) and becomes + an extra tool with the browser helpers. The factory goldens compare the 13 command tools and assert that + `browser_find` is the only MCP tool missing. +14. **Skips are logged once per factory**, keyed by name: an entry that selects no tool, a spec the driver cannot + run, a selected name with no spec that no extra tool serves. +15. **Images** use the multimodal shape MCP tools deliver; `source` is the driver name (today `mcp`). +16. **Tool-result golden at the renderer level**, as the design states: each recorded answer decoded with the result + decoder, the command with the native decoder, rendered with `result()` / `error()`; `browser_snapshot` answers + render as the observation the driver makes of them. `regenerate --results` rewrites the files and prints each + one's diff against the fixture's recorded `model_text`. The files are `.txt`, which the repository ignores, so + they are added with `git add -f`. + +### Rejected options + +- **A core hook copying tool-output metadata onto the tool message.** Touches `openjiuwen/core`; the record is + enough for the rail (Aleh chose the record). +- **Building `browser_find` now** to make `runner` byte for byte: it is the helpers task's extra tool. +- **A screenshot flag on `ToolSpec`, or a tool-name list in the factory.** A contract change, or tool names in the + core. +- **Probing every decoder with empty arguments.** Generated decoders raise `TypeError` without a ref; only + `Observe`-only specs can be screenshot tools. +- **`ToolFactory(profile, ToolContext)`.** The context would have to exist before selection, which needs neither + the session nor the page state. +- **Rendering the result golden through `CommandTool` on a fake server.** Snapshot and screenshot answers are + consumed inside `observe()`; a per-fixture golden would need invented second answers. + +### Golden comparison (factory alone, `image_support=False`) + +| View | Result | +|---|---| +| MCP `runner` | the 13 command tools equal the golden byte for byte; missing only `browser_find` | +| MCP `core`, `advanced_code`, `extended_interaction`, `pdf` | equal, minus the 4 helpers and `browser_find` | +| MCP `vision` | equal minus helpers, `browser_find`, `browser_mouse_down`, `browser_mouse_up` (C20) | +| MCP `unsafe_dev` | minus helpers, `browser_find`, `browser_run_code_unsafe` (C7) | +| MCP `devtools` / `network` / `storage` / `testing` / `config` | minus helpers, `browser_find` and the 12 / 5 / 17 / 5 / 1 tools with no command (C25) | +| browser-use, every view | equal except: `browser_cancel_run`, `browser_clear_cancel` dropped (Q3); `browser_close`, `browser_drag`, `browser_drop` only with `extended_interaction` (Q3); `browser_take_screenshot` only with image support (the MCP rule, now on both drivers); `browser_find`, `browser_recall_offload` not yet (extra tools); `browser_tabs` enum adds `new` (Q6) | + +### Verification + +- `drivers/playwright_mcp/test_tools.py` (78): round trip over the driver's 35 decodable handler cases, every spec + the server offers has a decoder and nothing else does, target and generation rules, ignored arguments, tabs, + 16 invalid-argument cases. +- `drivers/playwright_mcp/test_driver.py` (+3): native tools after `start()` (description, sanitized schema, + decoder), empty before, prefix. `test_result.py`: surfaces carry `tab_urls` / `current_tab` (expectations + updated). `surfaces/test_browser_renderer.py`: tab lists now come straight from the decoder (the helper that + filled the fields by hand is gone). +- `test_tool_factory.py` (32): selection order, ref-kind and command skips, logging once, native over profile form, + description fallbacks, generated schema, screenshot gate, extra tools from the `ToolContext`, builders for + `ActionPolicy` (a denial through the rail), the pipeline (decode, execute, record, render), stale ref before the + driver with one observation, free observation recorded not rendered, observation vs result rendering, image + blocks, typed errors, refused results, undecodable arguments, restart message, task failure, metadata table. +- `snapshots/test_tool_factory_golden.py` (37) and `snapshots/test_tool_result_golden.py` (37; 36 fixtures, 11 + identical to the recorded model text, the rest differing only by design §4 items 1, 2, 3, 9 and the screenshot + link). +- Fail-before: the changed decoder tests fail on the old decoder (14 failed); the new modules' tests fail at import. +- `pytest tests/unit_tests/harness/tools/browser_move`: **2205 passed, 113 skipped** (old MCP goldens, runner + contract, architecture included). +- Live: Playwright MCP conformance **51 passed**, browser-use conformance **45 passed**. + +### Known leftovers + +1. `browser_find` and the helper tools (probes, batch, recall) come as extra tools with the helpers task; the factory + goldens then compare all 14 / 18 tools. (Done: `core` 18 / 18; see "Batch and custom actions over commands".) +2. Nothing copies `CommandRecord.message_metadata` onto the tool message yet (the switch's rail work). +3. A screenshot tool renders through `observe()`: on MCP that is the snapshot text and the image (two server calls); + today the model gets the image and a file link. Only vision with image support sees it. (Rendering done: a + screenshot-only tool now shows the image only; see "Rails on command records".) +4. browser-use does not fill `tab_urls` / `current_tab` yet (helpers task). (Done; see "Helper infrastructure".) +5. `profile.rails` factories are not called by the factory; the rails task registers them with the `ToolContext`. +6. The browser-use tool-result golden needs the browser-use wire fixtures (helpers task). + +## Browser helpers over commands (probes, page metadata, find) + +Phase 4 helpers, first half: the page probes, the page metadata, `browser_find`, the decision-policy probe and +`activate_page` run as commands on the driver session (S_20 "Helpers"). They are new code in +`surfaces/browser/helpers/`, registered as the browser profile's `extra_tools`; the runtime's helpers (MCP code +runner) still serve the agent and are unchanged, as are their tests. The switch points the agent at the new ones. + +### Decisions + +1. **Built next to the old helpers** (O3, wave-3 review). The runtime keeps its MCP path until the switch; the new + tools reach the agent only through `profile.extra_tools` factories taking the `ToolContext`. +2. **Probes are page JavaScript** (C7). The page function is cut out of the existing builders' + `async (page) => { const params = …; return await page.evaluate((params) => {…}, params); }` with one anchored + regex, and run as `RunScript(code=, args=)`; the builders (clamping, site profiles, selector + cache records, generation id) stay the single source, untouched. Two attempts, retried when the value is not a + JSON object, as before. The result dict the model reads keeps today's keys. +3. **Every helper command is recorded** with `page_state.record`, as `CommandTool` does: the page state sees the + surface a driver reports with the script result. A probe result whose `url` differs from the page state's URL + resets the page state (advance and forget contexts, the rule `record` applies to a new address), because + browser-use reports no address with a script result; the old runtime only advanced the generation. +4. **Page metadata splits** into what the page state already has (URL, title, tabs from the last `BrowserContext`) + and one `RunScript` for page position and semantic state, the page-measure function cut out of the metadata + script. When a driver gives no `current_tab` (browser-use), the focused tab is the page state's current context. +5. **`activate_page` is `SwitchContext`** to the tab whose URL matches, else the current context, observing first + when no context is known. The Playwright MCP server's tab select calls `bringToFront()`, which is what the old + script did. The switch is recorded, so the generation advances (the old script did not touch the page state); + a policy probes again after activating, so its targets are fresh. +6. **`browser_find` searches the last observation** (C10) and ports the server's algorithm (`find.ts` of + `@playwright/mcp` 0.0.78): case-insensitive substring or `/pattern/flags`, 3 context lines, merged windows, + ancestor path, `...` off the match paths, `----` between snippets, the same `Found N matches for …` / + `No matches found for …` text. On Playwright MCP it searches the fenced YAML of `native_text`. With no last + observation it takes one (`Observe`), where the server would snapshot. +7. **One find tool for both dialects** (O3). Name, description and schema come from + `driver_info.native_tools["browser_find"]` when the driver lists it (Playwright MCP: byte-identical golden + entry), else the 857239db browser-use schema (byte-identical to the browser-use golden). The browser-dialect + form (`query`, `regex: bool`, `limit`, `generation_id`) is decoded into the same search; `limit` caps the matches. +8. **`NativeTool.build` is optional** (O3). The Playwright MCP driver lists `browser_find` with `build=None`; no tool + spec is named `browser_find`, so the factory never reaches it. +9. **Runner view = factory tools minus the probes.** The factory appends every extra tool to every build; the worker + removes the helpers after the build today, so the `runner` golden is compared with the probes removed. + +### Rejected options + +- **Refactoring `probes.py` into separate page-function constants.** The interactive probe is a 600-line f-string + with doubled braces; extracting it touches old code the runtime still uses. Cutting the function out of the + builder output keeps the old helpers byte-identical. +- **Re-reading URL, title and tabs inside the metadata script.** The design takes them from the last + `BrowserContext`; only position and semantic state need the page. +- **A fresh observation on every find**, as the server does: C10 searches the last observation and saves a snapshot + per call. +- **Not recording `SwitchContext`** to keep the generation: a real tab change must advance it, and the policy + probes after activating anyway. +- **The 857239db error for find without a snapshot** ("call browser_snapshot first"): costs the model a turn the + server never costs. + +### Golden comparison (factory alone, `image_support=False`) + +| View | Result | +|---|---| +| MCP `runner` | 14 / 14 byte for byte (factory tools with the two probes removed, as the worker does) | +| MCP `core` | 16 / 18 byte for byte: the 13 command tools, `browser_find`, both probes; `browser_batch_interact` and `browser_recall_offload` not built yet | +| MCP other views | equal minus `browser_batch_interact`, `browser_recall_offload` and the tools with no command (C7, C20, C25) | +| browser-use, every view | as before, plus both probes (C7), and `browser_find` identical; `browser_recall_offload` not yet | + +### `browser_find` result shape + +Live on the scene with `@playwright/mcp` 0.0.78, the tool's text equals the server's `### Result` section for +`text`, a `/regex/i` and a query with no match. Remaining differences: the server's answer also carries its page +header sections (not part of this tool's text); it searches a fresh snapshot, this tool the last observation (a page +changed by a script since is not seen); JavaScript regex syntax is read by Python `re` (flags `i`, `m`, `s`; `u`, +`y`, `d`, `v` ignored); a `/` inside a pattern is shown unescaped. On browser-use the text replaces the 857239db JSON +(`matches` with `target_id/ref`, `count`, `page_state`), and the search runs over browser-use's own snapshot text, +indented with tabs. + +### Verification + +- `surfaces/test_browser_helpers.py` (29, FakeDriver): probe cards equal the runtime helper cards; each probe sends + one `RunScript` with the page function and the clamped params, retries once on a malformed value, reports an + unparsable value, a refused script and a `DriverError`; a new address advances the generation; cards register + primary links; `probe_for_policy` sends the source and params verbatim and registers targets, failure envelopes; + `activate_page` picks the tab by URL, falls back to the current tab, observes first, returns `False` on errors; + page metadata sends one script and builds the tab list (with and without `current_tab`); find: card from the + driver or the browser-dialect schema, snippet with ancestor path, last observation reused, merged windows, no + match, four refused argument sets that send nothing, the `query` form with `limit`. +- `surfaces/test_browser_helpers_live.py` (3, `live`, gated like the suites): the interactive probe finds the scene's + save button on Playwright MCP and on browser-use; on Playwright MCP find equals the server's own answer for three + queries. +- `snapshots/test_tool_factory_golden.py` (38): `runner` 14 / 14 and `core` 16 / 18 byte for byte, the other views + with the listed differences. `drivers/playwright_mcp/test_driver.py` (+1): `browser_find` is a native tool with + no decoder. `surfaces/test_browser_profile.py`: `extra_tools`. `contract/test_tool_types.py` (+1). +- `pytest tests/unit_tests/harness/tools/browser_move`: **2237 passed, 116 skipped** (the old helper, policy-probe + and runtime tests untouched and green). +- Live: Playwright MCP conformance **51 passed**, browser-use conformance **45 passed**, helpers **3 passed**. + +### Known leftovers + +1. `browser_batch_interact` and `browser_recall_offload` are not extra tools yet, so `core` is 16 / 18. The batch tool + is the second helpers task; the recall tool needs the workspace, which `ToolContext` does not carry. (Recall done, + `core` 17 / 18; see "Helper infrastructure". Batch done, `core` 18 / 18; see "Batch and custom actions over commands".) +2. The helper tools return no `CommandRecord`, so nothing gives their tool messages metadata; the rails treat a + message without a record as a call that sent nothing (O6). A probe is a `RunScript` (`Effect.SCRIPT`), which + `message_metadata` would count as state-changing. (Done: records with the declared `READ` effect.) +3. The design's per-tool `effect`, `exempt_from_budget` and `progress` flags for extra tools are not declared yet. + (Done: `ToolFlags`.) +4. The factory does not check `NativeTool.build is None` itself; it is safe only because no spec is named + `browser_find`. +5. The probe page functions are cut out of the builder output by a regex; the switch can move them into their own + constants when the old runtime helpers are deleted. +6. browser-use still does not fill `tab_urls` / `current_tab`; `page_metadata` and `activate_page` fall back to the + page state's current context there (a tab cannot be picked by URL on browser-use). (Done; live test.) + +## Rails on command records + +Phase 4 item 5: the worker rails that read tool text get typed versions that read `CommandRecord`s (S_20 "Rails on +command records"), built next to the old ones like the helpers. The record gains what they need. The old rails, their +tests and the service registering them are unchanged; the switch registers the new classes, deletes the old ones and +gives the new ones the old names. + +### Decisions + +1. **`CommandRecord.blocked_by`** (O1) carries `ActionIntercepted.by`; the hint names the blocker from it instead of + parsing the Playwright call log. +2. **`CommandRecord.action`** (O2): the command with its sensitive fields masked. The rule moved out of `Tracing` into + `masked` in `drivers/middleware.py` (no browser words, already importable from the core); `Tracing` traces + `masked(command)`, so both share one rule. The guard compares commands, not tool arguments. +3. **An `Observe` record keeps its observed elements as `value`** (decided with Aleh, 2026-10-10). The old guard + compared snapshot text, so a loading page whose progress bar moves is not a repeat; the record had nothing that + changes with it (the generation advances only on large changes). The elements are typed, so no text is compared; + an unchanged screen is still a repeat. +4. **A call without a record compares by tool name and arguments** (decided with Aleh, 2026-10-10). O6 says such a + call is a failed call that sent nothing; keying it on the call keeps today's behaviour (identical undecodable calls + form a streak) and reads no tool text. The hint rail leaves it alone. +5. **"Identical" = same masked command and same result** `(ok, error type, value, surface context)`. A script keeps + the old normalisation: whitespace-free code with digits masked, value with digits masked, so counter variants + (`() => 'done1'`, `'done2'` …) form one streak as before. `detail` is not compared, so the `### Events` and + console counts the old fingerprint stripped play no part. +6. **Same behaviour list** (D6): note from the 3rd identical call, tools withheld for the next model call after the + 5th, reset on a different call, no-op scripts as one streak, same-turn calls counted in order (F1 stays), state + reset per invoke. Only the model-facing message changes: the note is appended to plain text, as there is no + `{"result": …}` envelope any more. +7. **The typed guard subclasses the old one** and overrides `after_tool_call`; `before_invoke` / `before_model_call` + are shared, so the withholding and the finish section cannot drift. The switch folds them into one class. +8. **Hint wording** per design §4 item 5: `Click not delivered: e57 is covered by .` The verb is the command + type (`Click`, `Hover`, `TypeText` …) and the target is the ref the model used, not Playwright's resolved + element. The consent hint is keyed on `blocked_by`; the hints and the retry sentence are the old ones (one shared + constant). Messages that are not strings are left alone, as before. +9. **`FinalAnswerRecoveryRail` is unchanged**: it reads the model output and the recovery flag, never tool text. + Its tests run again with the snapshot tool as a `CommandTool`. +10. **A screenshot-only `Observe` tool renders the image only** (O4): `renderer.result` on the screenshot, where it + showed the snapshot text and the image. A tool that only sometimes takes a screenshot still renders the + observation. +11. **The image block is labelled with the surface name** (O8): `profile.name` (`browser`) instead of the driver name + (`playwright_mcp`; the old path wrote `mcp`). `ToolContext` is unchanged (the helper tests build it + positionally); `CommandTool` takes the label. +12. **`command_record(tool_result)`** in the tool factory is the one way rails read a record. + +### Rejected options + +- **Observe identity by a new `Observation.fingerprint` field** on the record: it covers ref, role and name only, so + a progress value change would count as a repeat. +- **Observe never counted as a repeat**: five identical snapshots would no longer withhold tools. +- **A call without a record resetting the streak**: a loop on undecodable arguments would never be stopped. +- **Carrying the error object in the record**: one string is what the hint needs; the record stays plain data. +- **Rewording the retry sentence** to drop the tool name: model-visible text change beyond the design's item 5. +- **A `surface` field on `ToolContext`** for the image label: the helper tests construct it positionally. +- **Standalone typed guard** duplicating `before_invoke` / `before_model_call`: two copies of the withholding logic + until the switch. + +### Verification + +- `test_browser_repeated_call_guard_records.py` (52 passed, 1 strict xfail): every case of the old guard tests under + the same name, on records from `CommandTool` over `FakeDriver` with Playwright MCP tool names and arguments + (`command_tools.py`), plus one case showing the surface context is part of the result. Worker loops end with tools + withheld; same-turn press-key batch still withholds (F1 pinned); no extra model round trip with and without the + rail. Mutation checks: dropping value/surface from the identity fails 5 cases, dropping the script normalisation + fails 3. +- `test_browser_intercepted_action_hint_records.py` (21 passed, 1 strict xfail): every old case under the same name + (blockers from the old fixtures in the form the drivers report, ref / locator / hover targets, untouched errors, + observer keeps the raw error, no extra model round trip). +- `test_browser_final_answer_recovery_records.py` (15): every old case with the snapshot tool as a `CommandTool`. +- The two service-registration cases are strict xfails: they turn into failures once the service registers the + typed rails, so the switch has to remove the marks. +- `test_tool_factory.py` (36): one test runs a sensitive `TypeText` through `Tracing` and a `CommandTool` and finds + the mask in both; `blocked_by`, the `Observe` value, the image-only screenshot tool, the surface-name label, + `command_record`. Six of them fail on the previous code. +- `pytest tests/unit_tests/harness/tools/browser_move`: **2329 passed, 116 skipped, 2 xfailed** (old rail tests, + factory and tool-result goldens, runner contract and architecture included; tool-result goldens unchanged: they + render at the renderer level, where O4 and O8 do not apply). +- Live (after the record change): Playwright MCP conformance **51 passed**, browser-use conformance **45 passed**. + +### Known leftovers + +1. The service still registers the text-based rails; the switch registers `CommandRepeatGuardRail` and + `CommandInterceptedHintRail` (removing the two xfail marks), deletes the old classes and renames the new ones. +2. The retry sentence names `browser_snapshot`, a tool name in a core module (allowlisted with the old rail); a + surface-provided wording would remove it. +3. The hint verb is the command type name (`TypeText not delivered`); a surface may want friendlier verbs. +4. Helper tools (probes, find) return no record, so the guard compares their calls by tool name and arguments. + (Done: they return records; see "Helper infrastructure".) +5. F1 (same-turn batches counted as a loop) is unchanged, as decided; a fix belongs to the benchmark phase. + +## Helper infrastructure (records, workspace, tabs, fresh observation, points, upload staging) + +Phase 4 helpers, the infrastructure the batch tool builds on (wave-4 review P4–P7, P9, P11; wave-1 L3; Q9): helper +results carry command records, one shared step runs and records a command, the recall tool joins the extra tools, +browser-use reports tab addresses and dialogs after navigation, `browser_find` searches a current observation, +points carry the generation of their screenshot, and the Playwright MCP driver stages upload files. The old agent +path is unchanged. + +### Decisions + +1. **`ToolFlags(effect, exempt_from_budget, progress)`** (P4, design §6) in `tool_factory.py`, declared by each extra + tool as its `flags` class attribute: probes and find `READ`, not exempt, progress (today's + `_is_browser_progress_tool` counts every `browser_*` helper); recall `exempt_from_budget=True` (today's + `_is_replan_exempt_tool`). The flags stay on the tools; the record carries only the effect. +2. **Helper results carry a `CommandRecord` with the declared effect** (P4). A probe is a `RunScript` + (`Effect.SCRIPT`), but its record says `READ`, so `message_metadata` does not count it as a state change. The + payload moved under `data["content"]`; `render_payload_text` renders a non-string `content` as the JSON it + rendered before, so the model reads the same text. Find's record is an `Observe` with `dispatched` = whether it + observed and `value` = its answer; an argument error leaves no record (nothing was sent). +3. **One shared step** (P7): `ToolContext.execute(action)` / `observe(command)` run through the session and record in + the page state. `CommandTool`, the probes and the page helpers use them; the batch tool will. `CommandRecord.of` / + `failed` build the records for `CommandTool` and the helpers alike. +4. **`ToolContext.workspace` and `language`** (P5), keyword arguments of `ToolFactory.build` with defaults, so the + existing callers keep working. `language` is not in the review's wording; the recall tool's description is + language-dependent and the goldens are English, and `build_agent_parts` takes a `language` already. The default + `"cn"` is the recall tool's and `resolve_language`'s. +5. **The recall tool is a subclass** (`RecallOffloadTool(BrowserOffloadRecallTool)`) taking the `ToolContext`; the old + tool and its module are untouched. It sends no command, so it has no record. The worker's golden has no recall + tool: the runner removes it with the probes. +6. **browser-use tab fields from the tab rows the sidecar already sends** (P6): `tab_urls` from each row's `url`, + `current_tab` = the first row marked active (the sidecar's URL heuristic). No wire change for tabs. `OpenContext` + already listed the tabs; `SwitchContext` and `CloseContext` now list them too (one more request), so every tab + command reports the new tab list. A failure of that list after the switch is `dispatched=True`: the switch ran. +7. **Navigation replies carry the open dialog** (L3): the sidecar's `_nav_result` adds `dialog` (the same + `_open_dialog()` the act replies use, cached target info only), the driver puts it in the result surface. Wire + version 3 → 4. +8. **Fresh observation** (P9): `PageState.fresh_observation` is the last observation while it is of the current + generation and no command was recorded after it without a free observation, nor noted as dispatched and failed + (`note_dispatched`, called by `ToolContext.execute`). Any command counts, a `READ` one too: a probe is page + JavaScript whatever it declares. `browser_find` searches the fresh observation and observes when there is none. +9. **Points carry the generation of the last screenshot** (P11 / O7): the page state records it whenever an + observation with a screenshot is recorded and keeps it across generations; `RefResolver.screenshot_generation` + hands it to the decoders (MCP point tools, generated point schemas). `PageState.check` then refuses a point on an + older screenshot like a stale element ref. Before the first screenshot the generation is empty and nothing is + checked, as before. +10. **Upload staging inside the MCP driver** (Q9): `UploadFile` / `Drop` copy an existing file outside the server's + working directory to `output_dir/uploads/<12 hex>/` (a directory per file keeps the original name the page + sees) and send the copy. Files inside, missing files and directories go as given: the server's refusal + (`DriverError(dispatched=False)`, K2) stays for them. The live suite runs without + `--allow-unrestricted-file-access`, so its upload and drop cases go through staging. + +### Rejected options + +- **Flags in `AgentTools` or in `CommandRecord`**: a map in `AgentTools` duplicates what the tools carry; record + fields would be defaults on every command tool, and the recall tool has no record. +- **The record inside the probe payload dict**: the JSON the model reads would then contain it. +- **Stamping points in `CommandTool` after decoding**: the policy rail and the batch tool decode too; the resolver is + the one place every decoder asks. +- **Only non-`READ` commands outdating the observation**: probes run page JavaScript though declared `READ`; P9 + speaks of any dispatched command. +- **A new wire field for tab URLs**: the sidecar's tab rows already carry `url` and `active`. +- **Staging into the server's `.playwright-mcp` output directory**: the driver's `output_dir` is the server's working + directory, an allowed root on every configuration; `uploads/` under it needs no knowledge of `--output-dir`. +- **Deleting staged copies after the call**: a drop or file chooser may still read them; they live in the driver's + own output directory. +- **A server flag** (`--allow-unrestricted-file-access`): Q9 keeps the server's file check. + +### Golden comparison (factory alone, `image_support=False`, `language="en"`) + +| View | Result | +|---|---| +| MCP `runner` | 14 / 14 byte for byte (factory tools with the probes and recall removed, as the worker does) | +| MCP `core` | **17 / 18** byte for byte: only `browser_batch_interact` missing | +| MCP other views | equal minus `browser_batch_interact` and the tools with no command (C7, C20, C25) | +| browser-use, every view | as before plus both probes (C7); `browser_find` and `browser_recall_offload` identical | + +### Verification + +- Fail-before: each new test was run against the code before its feature commit in a separate worktree (no + stash): items 1–3 26 failures (the helper test module did not import), item 4 17, items 5–6 9, item 7 2 unit + failures and, live, the three upload / drop conformance cases refused with `File access denied`. +- `surfaces/test_browser_helpers.py` (+11): flags, `ToolContext.execute` / `observe`, probe records (result, + failure, unchanged model text), recall over the context's workspace and language, find records (observed, reused, + failed), find observing again after a command without an observation and after a new generation. +- `test_tool_factory.py` (+3): workspace and language reach extra tools; a point on an older screenshot never + reaches the driver; a dispatched failure outdates the observation. +- `test_browser_page_state_record.py` (+4): freshness, `note_dispatched`, older generation, screenshot generation. +- `drivers/playwright_mcp/test_tools.py` (+1), `drivers/playwright_mcp/test_driver.py` (+3), + `drivers/test_browser_use_driver.py` (+3, 6 updated), `drivers/test_browser_use_sidecar.py` (+8: every navigation + reply with and without a dialog), `snapshots/test_tool_factory_golden.py` (core 17 / 18). +- `pytest tests/unit_tests/harness/tools/browser_move`: **2372 passed, 117 skipped, 2 xfailed**. +- Live: Playwright MCP conformance **51 passed** (without unrestricted file access), browser-use conformance + **45 passed**, helpers **4 passed** (new: `activate_page` fronts a tab by its address on browser-use). + +### Known leftovers + +1. `browser_batch_interact` (next helpers task) makes `core` 18 / 18; it should run its steps through + `ToolContext.execute` and declare its flags. Done: see "Batch and custom actions over commands". +2. No rail reads `ToolFlags` yet; the switch moves `_is_browser_progress_tool`, `_is_replan_exempt_tool` and the + state-change checks onto them. +3. The recall tool leaves no record; a rail sees its message as a call that sent nothing. +4. Staged upload copies stay under `output_dir/uploads/`. Done: `stop()` removes them; see "Batch and custom actions over commands". +5. browser-use's `current_tab` is the first tab whose URL is the page URL; two tabs at one URL are not told apart. + +## Agent construction, instance settings and restart on a dead device + +Phase 4 item 6 and the composition-root part of plan section 3 ("One composition root"): `build_agent_parts` and +`build_agent` in `assembly.py`, `BrowserInstanceConfig` applied to the surface settings before anything is built, +per-effect timeouts from the profile, and the two restart follow-ups of the wave-2 review (N1 liveness before a +restart, N2 a new driver for a re-provisioned device). The fake-surface end-to-end test of plan section 4.3 runs +on it. Only tests reach `build_agent`; `create_browser_agent`, the service and the runtime are unchanged. + +### Decisions + +1. **The provisioner answers liveness** (N1): `Provisioner.is_alive(device)`. Managed and remote probe + `/json/version` (the check `is_endpoint_ready` already did, now one module function); a device without + an endpoint (extension; remote without `cdp_url`, where the driver launches its own browser) is always alive, + because provisioning it again returns the same empty device. +2. **Restart order:** `stop()` the old stack in every case, then ask the provisioner. Alive → `start()` the same + stack. Dead → release, provision, rebuild, start the new stack. A live device whose stack cannot start is a task + failure: the former fallback "re-provision when `start()` fails" is gone, since a new Chrome does not fix a driver + that cannot start, and on Playwright MCP `start()` succeeds over a dead Chrome anyway. +3. **The assembly rebuilds** (N2): `DriverAssembly.rebuild(device)` is the same closure that built the first stack + (driver factory, surface check, middleware with the profile timeouts), so stacks are built in one place and the + session holds no registry. The session swaps stack and device in its assembly; `close()` stops and releases the + new ones. A failing rebuild releases the new device before the task fails. +4. **`DriverAssembly` grows by appended defaults** (`rebuild=None`, `surface=None`); `surface_settings` stays. + Assemblies built by hand in the factory, helper and golden tests keep their four positional arguments. The tool + factory tests' hand-built assembly now has a provisioner that answers alive (decided with Aleh in-session); the + other hand-built assemblies never restart. +5. **The instance is applied before provisioning and the driver factory:** `assemble_driver(..., instance=)` calls + `surface.settings.with_instance(instance)` and puts the result into the bundle and into `profile.settings` (the + tool context reads the profile's). `BrowserSettings.with_instance` holds the rules of the browser service's + `_provisioning_settings` plus the two Playwright MCP overrides (`launch_args`, `cdp_url`), so + `test_browser_instance_isolation.py` semantics hold on the new path (distinct keys → distinct free ports and + profiles, explicit port and profile name win, no instance → environment values). +6. **The core passes the instance through untyped** (`Any`); `with_instance` is a method of the browser settings, + not of the `SurfaceSettings` protocol, so `contract/` is unchanged. +7. **No key in the MCP server name** (Q2): the Playwright MCP driver factory hands `playwright_mcp_config` only the + device endpoint, never the instance, so the server name and the tool prefix stay `mcp_playwright-official_` for + keyed instances. The old path's `build_playwright_mcp_config(instance)` still suffixes, until the switch. +8. **Per-effect timeouts from `profile.timeouts`** (I2's one line, now inside the stack builder). The browser + profile says 240 s for every effect, today's default; `PLAYWRIGHT_TOOL_TIMEOUT_S` no longer bounds driver calls + made through the root (it still bounds the worker's tool calls on the old path). An effect the profile leaves out + runs unbounded. +9. **`build_agent_parts(session, bundle, *, capabilities, image_support, language, workspace=None)`**: tools and + prompt sections from `ToolFactory.build` (workspace and language go to the tool context); rails = `ActionPolicy` + (the factory's builders, `profile.policy_rules`, the session's page state), `CommandRepeatGuardRail`, + `CommandInterceptedHintRail`, then each `profile.rails` factory called with the `ToolContext` of the same session. + The factory does not return its context, so the root builds an equal one. +10. **`build_agent` is async and returns `(agent, session)`.** It starts the driver before the agent exists (the MCP + native tools appear at `start()`); the caller owns `begin_task()` and `close()`. A failing construction closes + the session. The system prompt is the caller's text followed by the prompt sections, blank-line separated; + every other keyword goes to `create_deep_agent`. +11. **`DriverSession.page_state`** is readable, so the parts can be built from a session alone. `driver_session.py` + imports `DriverAssembly` for annotations only (the root now imports the session through the tool factory). +12. **The fake surface writes its own driver** (a `HandlerDriver` with `@handles(Click / Poke, refs={PointRef})`), not + `contract.testing.FakeDriver`: the plan derives the handled commands from the marks, and per-command scripted + outcomes make the scenario readable. The model is a `Model` subclass with a scripted `invoke`; the agent is a real + `DeepAgent` from `create_deep_agent`. The stale point comes from the page state: a click whose result changes + the context advances the generation, and the next point still carries the generation of the earlier screenshot. + +### Rejected options + +- **Liveness from `driver.health()`**: the MCP server answers with Chrome gone; the provisioner knows the device. +- **Rebuilding from the registries inside the session**: a second composition root (the reason it was rejected in + "Page state, driver session and policy rail"); the closure keeps one. +- **Keeping "re-provision when `start()` fails" next to the probe**: two triggers for one decision, and the probe + already covers the case. +- **Treating an assembly without a provisioner as alive** (to leave the tool factory test untouched): a branch for a + state only hand-built tests create. +- **`with_instance` on the `SurfaceSettings` protocol**: a contract change for one surface's identity type. +- **Registering `session.close` as the deep agent's task-resource cleanup**: that runs after every subagent task and + would stop the browser between tasks. +- **Returning only the agent**: the caller could neither close the device nor start a task. +- **The Phase 3 per-effect bound `core.tool_timeout_s`**: the plan puts timeouts in the profile so a slower surface + needs no core edit. + +### Fake-surface end-to-end test + +Six assertions on one scripted run (`look`, an off-board `poke`, `click`, a `poke` on the old screenshot, `look`, +`click` with a lost connection, then an answer): + +| Plan item | What the test sees | +|---|---| +| tool list | the model is offered `look`, `click` and `poke`; `click` and `poke` take `x, y` integers (schemas generated from `refs={PointRef}`) | +| image block | the request after `look` carries the screenshot as an `image_url` (the renderer's `Screenshot` block) | +| policy | `poke(500, 5)`: tool result `PolicyDenied`, the model reads `### Error\n(500, 5) is off the board`, no `Poke` reaches the driver | +| stale point | `poke(10, 10)` on the g0 screenshot after the generation moved to g1: exactly one `Observe()` and no `Poke` on the driver, record error `StaleRef`, the model reads the error with the fresh observation | +| lost connection | the driver sees `stop`, `start` once after the failing `click` (the device is alive, nothing provisioned again), the model reads `driver_restarted` | +| registries only | surface, driver and provisioner are registered in test-scoped `Registry` objects; no package file was changed for the test | + +**No seam needed a change:** nothing in `runtime/` besides the two files of this task, nothing in +`contract/` or `drivers/middleware.py`. + +### Verification + +- Fail-before (old `driver_session.py` in a separate worktree, new assembly and tests): 6 of the 12 session tests + fail, among them the lazily starting driver over a dead device. +- `test_driver_session.py` (12, rewritten around `is_alive` and `rebuild`), `test_assembly.py` (+4: rebuild, a + rebuild that does not serve the surface, the instance before provisioning and the driver, the plain MCP prefix for + a keyed instance; the timeout cases now come from the profile), `test_provisioning.py` (+4 functions, 5 cases: + liveness against a local CDP stub, a closed port, a managed device, devices without an endpoint), + `surfaces/test_browser_settings.py` (+7: `with_instance`), `test_build_agent.py` (6), + `architecture/fake_surface/` (6), `test_tool_factory.py` (hand-built assembly with a live provisioner). +- `pytest tests/unit_tests/harness/tools/browser_move`: **2393 passed, 117 skipped, 2 xfailed** (architecture, + old goldens, factory goldens and the runner contract included); the 12 harness test files outside it that use the + browser agent: **262 passed**. +- Live: `test_assembly_live.py` **3 passed** (managed Chrome with browser-use and with Playwright MCP through the + root, and new: a managed Chrome killed under Playwright MCP is detected dead, provisioned again, and the next + observation runs on a rebuilt driver); browser-use conformance **45 passed**. + +### Known leftovers + +1. **browser-use does not report a dead Chrome as `ConnectionLost`.** Killing the managed Chrome under browser-use + makes the next `observe` fail with a plain `DriverError` ("CDP still not connected"), so the session never + restarts. The driver's error mapping needs the connection case (checked live, not in a test). Done: + see "Batch and custom actions over commands". +2. A `StaleRef` raised by a driver reaches `CommandTool` with the fresh observation on the error + (`RetryOnStale`); the tool passes `observation=None` to `renderer.error` and does not record that observation in + the page state. The fake surface goes through the page-state path, which records it. Done: see "Batch and custom actions over commands". +3. `build_agent` registers no task hook: `begin_task()` and `close()` are the caller's until the switch wires the + facade's acquire/release. +4. `BrowserInstanceConfig.driver` is not read by the root; `core.driver` picks the driver. +5. `DriverAssembly.surface_settings` duplicates `surface.settings`; it stays while hand-built assemblies pass it + positionally. + +## The switch, step 1: the main agent on the driver session + +The first of the three switch sessions (design sections 1(c), 1(d), 2, 6 and 7; decisions O5, P8, Q14 and S1 of +wave 6; `switch-plan.md` S1), on the branch `p4-switch`. `create_browser_agent` keeps its signature; the agent it +builds runs on a `DriverSession` through the composition root, the factory tools and the record rails, and the old +MCP path of `runtime.py` is gone. The WebVoyager worker (`service.py`, `agents.py`) still runs the old path until S2. + +### Decisions + +1. **The facade keeps its signature and owns the session.** `BrowserAgentRuntime.ensure_runtime_ready` calls + `assemble_driver` with the given `CoreSettings` (else the process environment) and the instance, makes the + `DriverSession` and a `ToolContext`, and registers `CUSTOM_ACTIONS` on the controller. `mcp_cfg` stays a + constructor argument for the worker path; honouring it on the session is S2 (Q11). Two keyword-only arguments + were added, `workspace` and `language`, because the recall tool and the tool context need them. +2. **Everything arrives at the first invoke.** The design put the extra tools at construction; `browser_find` takes + its description from the driver's native tools, which exist only after `start()`, so the factory build (command + tools, extra tools, rails) runs in `BrowserRuntimeRail.before_invoke` where the MCP tools were added before. + `create_deep_agent` gets no browser tools; a caller listing tools before the first invoke sees its own only. + The `core` golden is byte-identical, which is what the rule protects. +3. **The parts' rails register dynamically**, through `agent.register_rail`, once per agent. The main agent thus + runs `ActionPolicy`, `CommandRepeatGuardRail` and `CommandInterceptedHintRail` as `build_agent_parts` returns + them; the design table named only `ActionPolicy` for the main agent, `build_agent_parts` (P4-T6) decided + otherwise, and one list serves both topologies. +4. **`ToolKind`**: what the rail knows before a call. It decodes the command with the factory's builder (one more + decode next to `ActionPolicy`'s and the tool's; all pure) and carries the schema's field names; after the call the + record's action and effect replace it. The classifiers (phase, action class, intent, read-only recovery, replan + exemption, progress, bounds, page-state attachment) read the kind; the strategy signature keeps the tool name + as a label so direct and batch clicks still compare equal. +5. **Capabilities from the allowlist.** The factory selects by capability entry; the facade derives the entries + whose tools are all in `allowed_tool_names`, the inverse of `resolve_browser_capabilities`, so the runner's + `allowed_tool_names=CORE_BROWSER_TOOL_NAMES` still means `core`. +6. **Argument normalisation by dialect.** Only a tool whose schema addresses elements by `target` (the MCP dialect) + gets the ref aliases, `[ref=e12]`, `target_id` and `generation_id` rewrites; the browser dialect decodes + `target_id` itself and needs its `generation_id`. The card primary-link rewrite is gated on the decoded `Click`. +7. **`field`, `fields` and `element` are dropped by schema.** All 69 server schemas of 0.0.78 forbid extra + properties, and the rail validates arguments against them; the keys are dropped only where the schema lacks them + (the evidence hints are read first). For `browser_drag` this also drops `element`, which the old path forwarded. +8. **Observation reuse (Q14):** `capture_browser_state` takes `page_state.fresh_observation` and observes only + otherwise; the free observation of the last mutating command is what the next capture reads. `observations_reused` + counts them for P4-T9. The metadata script runs after the observation and outdates it for `browser_find`, which + is fine: the next capture follows the next mutation, whose own free observation is fresh again. +9. **Capture errors.** A `DriverError` on the observation becomes `dom_error` (`page observation failed: …`); the + session's task failure after a second lost connection propagates instead of being folded into the state. +10. **The core imports the browser helpers.** `runtime.py` imports `CUSTOM_ACTIONS`, `probe_for_policy`, + `activate_page` and `page_metadata` from `surfaces/browser/helpers`, as the switch plan says, against plan + invariant 2 (core never imports a concrete surface). Phase 9's done-criterion is where the invariant must hold + ("`runtime/` never imports `surfaces/` except through the registry"); the import-boundary allowlist carries the + one entry (count 4) with that reason. The alternative, a hooks slot on `SurfaceProfile`, is a contract change + for Aleh to decide. +11. **URLs come from the record.** `_update_last_page`, `_evidence_variant_and_url` and the navigation completion + evidence read `url` from the record's surface context (`extract_result_url` is gone with the text parsing). +12. **A partial batch with a successful mutating step still counts as a state change** in the rail (the batch + record says `ok=False`, so its metadata alone would not refresh the browser state). +13. **The old goldens of the seven views with no-command tools are regenerated** (unsafe_dev, vision, devtools, + config, network, storage, testing): exactly the C7/C20/C25 removals; `core`, `runner`, `advanced_code`, + `extended_interaction` and `pdf` are byte-identical. The factory golden test compares byte for byte again and + keeps the removed names as the record of the decision. +14. **Batch typing on a driver without key-by-key typing (S1):** `TypeText(slowly=True)` refused with + `Unsupported(dispatched=False)` is sent again as `TypeText(clear=False)`, in the single-step path and the loop + (where the field was emptied first). A refusal the driver dispatched is not retried. +15. **`AgentParts.builders`** (appended, default empty) hands the factory's decoders to the runtime. + +### Rejected options + +- **Extra tools at construction with a lazily started session**: a `ToolContext` needs the started driver, and + `browser_find`'s native description is byte-identical only from the driver's tool list. +- **Keeping the name-based classifiers with prefix stripping**: the point of the switch; the kind carries the + command type and the effect on both dialects. +- **Normalising `target_id` for every tool**: breaks the browser dialect, whose decoders take `target_id` and need + `generation_id`. +- **Dropping `element` everywhere**: `browser_click` declares it; the schema decides. +- **A hooks slot on the profile for the policy and metadata helpers**: a contract change outside this session's + mandate; recorded as the open question. +- **Replacing `inputs.tool_result` with a dict in `after_tool_call`** (the old normalisation): the record rails + after this one need the `ToolOutput` with its record. + +### Golden status + +| View | Result | +|---|---| +| `core`, `runner`, `advanced_code`, `extended_interaction`, `pdf` | byte for byte | +| `unsafe_dev`, `vision`, `devtools`, `config`, `network`, `storage`, `testing` | regenerated: the 43 no-command tools of C7/C20/C25 removed, the prompt unchanged | +| runner contract (`test_wv_runner_contract.py`) | green | +| factory goldens | `core` 18/18, `runner` 14/14, every view byte for byte | + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q` on `p4-switch`: **2406 passed, 123 skipped, + 2 xfailed** (architecture, old goldens, factory goldens and the runner contract included), before the batch + fallback; `surfaces/test_browser_batch.py` 56 after it. +- The ten harness test files outside `browser_move` that build the browser agent: **234 passed**. +- Live: `test_assembly_live.py` 4 and Playwright MCP conformance 51 (**55 passed**); browser-use conformance + **45 passed**; live helpers on both drivers after the batch change **8 passed**. +- Deleted tests, each covered elsewhere or testing removed behaviour: the canonicalisation of tool names (gone); + the MCP text classifier, the RPC envelope and `extract_result_url` (records carry ok, error, value and surface); + the runtime's probe, batch, code-executor and MCP lookup cases (`surfaces/test_browser_helpers.py`, + `surfaces/test_browser_batch.py`); the single-step batch primitive through the client and rail; the controller's + LLM-fallback and script-builder cases (not ported, C28); the status-logging test that read the batch script. + +### Known leftovers + +1. S2: the worker still builds on `mcp_cfg` through the service (`ensure_started`, heartbeat, client patch); + `runtime_tools.py` helper classes call deleted runtime methods and serve only the worker's tool cards until the + worker gets the factory tools; `test_browser_runtime_tools.py` cases that invoke them through a mock runtime + still pass. `runtime_health` reports the service's flags, not `session.health()`. +2. S3: the context processors and `status_logging.py` still key on tool names; the old rail classes and their + tests stay next to the record rails; the client patch is still called in the facade constructor (row 4 of the + P5-T1 inventory). +3. A `browser_settings` given to the facade reaches the service only; the session's surface settings come from + the process environment through the surface registry. +4. The core import of the browser helpers (decision 10) is allowlisted until Phase 9. +5. `release_task_resources` keeps the session (and its browser) across tasks; the old path dropped the MCP + registration between tasks. + +## Batch and custom actions over commands + +The last helpers task before the switch (design section 6 and Q7, Q8; phase-0 decisions C28, C29; wave-5 review R1, +R5 to R8): `browser_batch_interact` becomes an extra tool that runs its steps as commands, the three Playwright-only +custom actions become functions of the tool context, and four small fixes land. The old agent path (runtime, service, +agents, controller, runtime tools) is unchanged; it still serves the agent until the switch. + +### Decisions + +1. **One step, one command** (design section 6 table). A step that maps to a command runs as that command on the + target's own ref: the `DriverRef` of a snapshot target is used as is (the old path first stamped it with + `browser_evaluate`), a probe target is `Locator(css)`. A card's primary link clicked alone is `Navigate`. Steps + with no single command (extractions, `set_checked`, `autocomplete`, the other waits, `press` with a locator) take + the loop, as the old path fell back to the script. +2. **Several steps run as a Python loop** (Q7): snapshot targets are stamped first, one page script checks every + target before anything acts (preflight), and each step locates its element again with one page script that + stamps it with a fresh mark, then acts on `Locator(css='[data-openjiuwen-target-id=""]')`. Re-locating at + each step keeps what the Playwright locators did (they were lazy queries, re-resolved on each action), at one + page script per step. +3. **Locator strategies in page JavaScript**: CSS, and stand-ins for Playwright's `getByRole` (explicit and implicit + roles, accessible name from `aria-labelledby`, `aria-label`, labels, placeholder, alt or text; hidden elements + left out), `getByLabel`, `getByPlaceholder`, `getByTestId` (`data-testid`) and `getByText` (the innermost elements + whose text matches), with `exact`. Strictness as before: more than one match fails at once. +4. **A missing or hidden target is waited for within the step timeout** at every check. The old script failed at + once on zero matches and waited only for visibility; the loop polls both, because a target can appear after an + earlier step. +5. **`type` and `autocomplete` empty the field first** (`Click`, `TypeText("")`, then `TypeText(slowly=True)`): + typing key by key does not clear on Playwright MCP (`pressSequentially`), and the old script cleared with + select-all and Backspace. `fill` is one `TypeText`. +6. **Waits**: `wait_for_url` and `wait_for_tab` poll `Observe` (the tab list and addresses are in + `BrowserContext`) and switch with `SwitchContext`; the other waits poll one page script each, with the old + `stable_ms` rule. `sleep` is `Wait(seconds)`; the pauses after a step stay local sleeps. +7. **Validation, stale-target refresh, preflight and the result keys are the old ones.** The controller's step + checks and the runtime's target contract run in that order, so the first error the model reads is unchanged. The + card and step contract live in `batch_steps.py`, the runner in `batch.py`: one module would pass the 1,500-line + limit. The read-selector check of the old runtime (exactly one visible match) is part of the preflight. +8. **The batch record** (R6): one `CommandRecord` for the call, `command` and `action` of the last command sent, + `effect` the strongest of what ran with page scripts and observations counted as `READ` (they only find, mark and + read) and the order `READ` < `SCRIPT` < `ACT` < `NAVIGATE`, `ok` as the payload says, `dispatched` and + `context_changed` from any command, `value` the payload. The tool declares `ToolFlags(effect=ACT)`. +9. **A lost connection ends the batch** even with `continue_on_error`: the page state was reset by the restart. +10. **The batch card keeps "MCP/Playwright-style" in one property description**: the card is golden byte for byte, + so `batch_steps.py` is allowlisted in the driver-name scan with that reason (decided with Aleh). +11. **Custom actions** (Q8, C28): `CUSTOM_ACTIONS` maps the controller names to functions of the tool context with the + controller's parameters, aliases, error texts and result keys; the switch registers them on the controller. + Coordinates are one page script (`querySelector` and the bounding box, else the element with the visible text); + drag stamps both elements and sends `Drag` between the stamps, or between two points of the last screenshot; + file inputs get `UploadFile` on the element when the driver accepts a `Locator` for it (browser-use), else a + `Click` that opens the chooser and `UploadFile(None, paths)` answering it (Playwright MCP, which stages the + files). The choice reads `driver_info.actions`, so no driver is named. Script values are read as values: the + executor path no longer reads the result envelope (F11). +12. **R1**: an unclassified browser-use sidecar error that says `not connected` or `Reconnection wait timed out` is + `ConnectionLost(dispatched=True)`. browser-use raises a plain `ConnectionError` from its watchdogs once its + reconnect gives up, which the sidecar sends as the generic error code. +13. **R5**: `CommandTool` records the observation a driver's `StaleRef` carries and renders the recorded one. +14. **R7**: `message_metadata.executed` is `ok` for a `READ` record, `dispatched` otherwise. +15. **R8, narrowed**: the Playwright MCP driver removes on `stop()` the staging folders it created and then + `uploads/` only if empty. The working directory defaults to the process directory, where `uploads/` may be the + user's own folder; removing it whole, as R8 reads, was rejected with Aleh. + +### Rejected options + +- **One page script per batch** (the old shape): the design keeps one command per step so the rails, the middleware + and the page state see each action. +- **Re-using the preflight stamps for execution**: a stamp is lost when an earlier step re-renders the element; the + old locators were re-resolved too. +- **Stamping a snapshot target for a single step**: one command was the rule, and the driver resolves its own refs. +- **Typing `type` steps without clearing** (`TypeText(slowly=True)` alone): appends to a filled field on Playwright + MCP. +- **Polling `wait_for_url` with `location.href` only**: it cannot see or switch to another tab. +- **Applying drag offsets and `steps` / `delay_ms`**: `Drag` takes two refs; the offsets only move the reported + points and the two numbers are echoed. +- **Porting the LLM fallbacks** of the custom actions (`_build_run_code_task`): dead on 0.0.78, and they sent page + code to the worker. +- **Rewording the card text** to drop "MCP/": the core golden would no longer be byte for byte. + +### Step to command + +| Step | One step | In a loop | +|---|---|---| +| `click` | `Click(ref)`; card link: `Navigate(href)` | locate + `Click(stamp)`, one retry on a transient error | +| `fill` | `TypeText(ref, value)` | locate + `TypeText(stamp, value)` | +| `type` (or `fill` with `mode: type`) | `TypeText(ref, value, slowly=True)` | locate + `Click`, `TypeText("")`, `TypeText(value, clear=False, slowly=True)` | +| `autocomplete` | loop | locate + as `type` (or `Click`, `TypeText` with `fill_first`), pause, locate option + `Click` | +| `select_visible_text` | loop | locate option + `Click` | +| `press` | without a locator: `PressKey(key)` | with a locator: locate and focus, `PressKey(key)` | +| `select_option` | values / value: `SelectOption(ref, values)` | locate (option value at `index`) + `SelectOption(stamp, values)` | +| `set_checked` | loop | locate + `FillForm((FormField(stamp, CHECKBOX, "true" / "false"),))` | +| `wait_for_text` | `Wait(until_text=text)` | page script polled (body text, or an element whose text equals it with `exact`) | +| `wait_for_selector`, `_load_state`, `_first_card_title`, `_sort_state`, `_result_count`, `_dom_text_change`, `_stable` | loop | page script polled, `stable_ms` where the old step had it | +| `wait_for_url` | loop | `Observe` polled over the focused tab, then the others from the newest; `SwitchContext` to a match | +| `wait_for_tab` | loop | `Observe` polled (tab count and match); `SwitchContext` unless `activate: false` | +| `sleep` | `Wait(seconds=ms / 1000)` | `Wait(seconds=ms / 1000)` | +| `extract_text`, `extract_value` | loop | locate and read `innerText` / `value` in the same page script | +| `screenshot` | `Observe(include_screenshot=True, full_page)` | the same | + +### Ported and dropped batch tests + +`surfaces/test_browser_batch.py` keeps the old names where the case still applies: rejects more than 25 steps, +rejects empty or non-list steps, validates the schema and the known bad cases before anything runs, the target +contract for mutations, the transient click retry, extraction provenance, partial results, compact condition +observations, `wait_for_tab` activating the new tab, `wait_for_url` switching to it, preflight rejecting a later +ambiguous target, the read-selector check; the Playwright form stub became the realistic booking flow over commands. +Dropped, with what they checked: the script text (`page.getByLabel`, `waitForTimeout`, fail-fast wording, the +retry loop source), the code executor call and missing executor, the compact-RPC envelope and its metrics, and the +drag script text (custom actions). The custom-action cases for the code executor and the LLM fallback go with those +paths; the fenced coordinate result, empty paths and the default file input are ported. + +### Differences from the old path + +- The model reads `execution_mode: "commands"` for a multi-step batch (was `compact_rpc`) and `metrics` with + `command_count`, preflight and step times (was script build and transport numbers); one step reports + `metrics.command` (was `tool_name`). +- A read-selector that matches more than one element fails as `batch_preflight_failed: ...` (was + `batch_target_validation_failed: Read-only batch selector must match exactly one visible element ...`). +- `screenshot` no longer writes a file at `path`; it is an observation with a screenshot. +- `delay_ms` does not slow typing (no delay in `TypeText`); `wait_for_load_state` `networkidle` is `complete`. +- `session_id` / `request_id` come from the arguments only (the old tool also read the parent task's context). +- Custom actions: Playwright selector syntax other than `:has-text("...")` is not understood (CSS plus the text + fallback); drag offsets and steps are not applied; error texts say ` failed: ...` instead of + `browser_run_code failed: ...`. +- On browser-use `type` and `autocomplete` steps fail at the slow typing (`Unsupported`: the driver cannot type key + by key) after the click and the clearing; `fill` works on both drivers. + +### Golden comparison (factory alone, `image_support=False`, `language="en"`) + +| View | Result | +|---|---| +| MCP `runner` | 14 / 14 byte for byte (the worker removes both probes, the batch and recall) | +| MCP `core` | **18 / 18** byte for byte | +| MCP other views | equal minus the tools with no command (C7, C20, C25) | +| browser-use, every view | as before plus both probes and the batch (C7) | + +### Verification + +- Fail-before (a separate worktree at the commit before the fixes, new tests copied in): R5 1 failure, R7 2, R8 2, + R1 3. The batch and custom-action tests import modules that did not exist. +- `surfaces/test_browser_batch.py` (53), `surfaces/test_browser_custom_actions.py` (19), + `surfaces/test_browser_helpers.py` (extra tools and flags), `snapshots/test_tool_factory_golden.py` (core + 18 / 18, the runner removing the batch), `architecture/test_driver_names.py` (one allowlisted card text), + `test_tool_factory.py` (+3: the driver's stale observation, two read records), `drivers/playwright_mcp/test_driver.py` + (+2, one changed to read the copy before `stop()`), `drivers/test_browser_use_driver.py` (+3). +- `pytest tests/unit_tests/harness/tools/browser_move`: **2473 passed, 123 skipped, 2 xfailed** (architecture, old + goldens, factory goldens and the runner contract included); the 11 harness test files outside it that use the + browser agent: **248 passed**. +- Live: helpers 8 (new: a fill, check, click, wait and read-back batch per driver on the scene; a file uploaded and + an element dragged per driver), assembly 4 (new: managed Chrome killed under browser-use is found dead, provisioned + again, and the next observation runs on a rebuilt driver), browser-use conformance 45, Playwright MCP conformance + 51. + +### Known leftovers + +1. The batch and the custom actions serve no agent yet: the switch registers `CUSTOM_ACTIONS` on the controller and + drops the old batch path (the runtime tool, the controller action and its script). +2. `select_visible_text` cannot pass validation (it needs a `target_id` but takes no primary locator), as before. +3. The controller's own `browser_batch_interact` action (reachable through `browser_custom_action`) skips the + runtime's target contract on the old path; nothing replaces it here. +4. `type` and `autocomplete` steps need a driver that types key by key (not browser-use). +5. `wait_for_url` and `wait_for_tab` observe at every poll (a full snapshot on Playwright MCP). + +## The switch, step 2: the worker and the runner facade on the driver session + +The second switch session (design section 7; decisions P1, R2, R3, R9, Q10, Q11, Q12, F7; the S1 review's T2, T3, +T4; `switch-plan.md` S2), on `p4-switch`. The WebVoyager worker runs on the same `DriverSession` as the main +agent, built from the factory; `BrowserService` holds that session and provisions nothing itself; the facade +serves the runner on both drivers. + +### Decisions + +1. **One session, owned by the service.** `BrowserService` takes the facade's `BrowserPageState` and opens the + session in `ensure_runtime_ready` through `assemble_driver`; the facade asks the service for its session and + builds its tool context on it. The S1 interim state (a session on the facade and an MCP server registered by the + service, two servers during a runner task) is gone (T4). +2. **Task release stops the stack and keeps the device (T2).** `DriverSession.stop()` / `start()`: the last task + binding stops the driver stack (the MCP server with its `--isolated` context, the browser-use sidecar); the next + task starts it again, re-provisioning first when the device died. `begin_task()` runs in + `acquire_task_binding`. The worker is built again after a release (its tools are bound to the session, which + stays). +3. **The surface factory takes the instance and a settings override (R2, T3).** `factory(environ, instance=None, + settings=None)`; `assemble_driver(..., settings=)`; the composition root stops calling `with_instance`. A caller's + `browser_settings` reach the session, with the given `mcp_cfg` as `BrowserSettings.mcp_server` and the config's + `cwd` as `mcp_cwd`. +4. **A given `mcp_cfg` is honoured (Q11) without changing the prefix (Q2).** The Playwright MCP factory keeps the + given command line (`params`, `client_type`) under its own server name and id and the device environment. +5. **The instance's `driver` overrides `core.driver` (R3)** in `service.core`, which the facade uses to pick the + topology. +6. **Worker tools and rails (R9, P1).** `build_agent_parts(..., worker=True)`: the `core` catalog tools, both probes, + the batch and find; `browser_recall_offload` is flagged `ToolFlags(worker=False)` and stays main-agent only, so the + runner's fixed removal list keeps the worker at the golden's 14 tools. Rails: `ActionPolicy`, + `CommandRepeatGuardRail`, `CommandInterceptedHintRail` (the parts' order), then `FinalAnswerRecoveryRail`; the + two strict xfails flipped. `build_browser_worker_agent` takes `tools` and no MCP config or allowlist. +7. **The browser-use facade is the main agent (Q12).** `ensure_started()` on `direct` builds the agent through + `create_browser_agent(model, runtime=self, ...)` (new keyword: an existing facade to build on), the model from + the facade's settings with the core worker temperature and top-p; `run_browser_task` runs it once under the + guardrails' timeout and maps the answer (`ok`, `final`, `page` from the page state, `screenshot=None`, `error`, + `status` / `progress` from the rail's terminal payload). Option B (phase-0 T4-Q1) is `create_browser_agent` plus + `Runner.run_agent`, so the runner's browser-use path is that same agent, not a bare `build_agent` DeepAgent. +8. **Heartbeat, health (Q10).** The heartbeat loop, `_check_connection`, the client-patch call and the MCP + registration are gone from the service; `runtime_health` reports `session.health()` and `started`; the MCP + server's `browser_runtime_health` reads it. +9. **`shutdown()` closes the model clients (F7)** after the service: `OpenAIModelClient.aclose()`. +10. **The service no longer provisions.** `_ensure_managed_driver_started`, the profile-store adoption, the + managed-driver bookkeeping and `_validate_mcp_command` are gone; the provisioner (through the root) provisions + and adopts the profile with the settings' name. The registry keeps coordinating per identity: the session's + provisioner is its managed driver, so `reset_managed_browser_runtime` still stops a preserved Chrome. +11. `runtime_tools.py` keeps only the cancel, clear-cancel, custom-action, list-actions and health tools; the probe + and batch classes and `build_browser_runtime_tools` are gone (the factory's extra tools serve the worker). + `capabilities_of` moved next to the catalog. + +### Rejected options + +- **Direct = `build_agent` without `BrowserRuntimeRail`**: no `` protocol, no page-state + context, and not the option-B agent the smoke compares with. +- **Per-agent extra-tool selection by name in the service**: a browser word in core and a name list to maintain; + the tool declares it. +- **Keeping the service's own provisioning next to the root's**: two provisioning paths for one device. +- **Stopping the stack only when the registry's binding count drops** (the old MCP-binding rule): every service + now has its own stack. +- **Closing the session between tasks**: the old path kept Chrome (T2). + +### Behaviour changes to know + +- An unkeyed service no longer adopts the store's globally selected live Chrome when its own profile name differs + (the Phase 3 H1 order); the provisioner reuses the profile with its own name only, and the legacy store path + (`~/.jiuwenclaw/browser-move/.browser/profiles.json`) is not read. +- `runtime_health` has `detail` instead of `last_heartbeat_ok`. +- The worker's helper cards come from the factory (`language="en"` as before); `browser_custom_action` and + `browser_list_custom_actions` are still registered as runtime tools only. + +### Golden status + +| View | Result | +|---|---| +| every MCP view, prompts included | byte for byte (`test_tool_prompt_snapshot.py`) | +| runner contract (`test_wv_runner_contract.py`) | green: 17 tools before removal (14 core, probes, batch), 14 after, answer dict unchanged | +| factory goldens | `core` 18/18, `runner` 14/14 | + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q` on `p4-switch`: **2359 passed, 123 skipped, + 25 failed**; the 25 are the three old-rail test files S3 deletes (`test_browser_repeated_call_guard.py`, + `test_browser_intercepted_action_hint.py`, `test_browser_final_answer_recovery.py`), which build the worker with + the old signature and mock `ensure_runtime_ready` without a session. No xfail remains. +- The ten harness test files outside `browser_move` that build the browser agent: **234 passed**. +- Live: see the report (`.hq/reports/P4-T8-S2.md`). +- Rewritten under their names: `test_service.py` (the session in place of the managed driver), + `test_browser_service_registry.py`, `test_browser_instance_isolation.py` (the instance through the surface + factory and the provisioner), `test_browser_worker_allowlist.py`, `test_browser_prompt_tool_surface.py` (the + worker from the service), the three records tests (`assemble_driver` patched instead of `ensure_runtime_ready`). +- Deleted tests: the service's MCP command validation, registration, refresh and heartbeat cases (no MCP + registration, no heartbeat); its managed-driver and profile-adoption cases (the provisioner's, covered by + `test_provisioning.py` and the surface settings tests); the runtime-tool probe and batch cases and the two card + parity tests (the factory goldens pin the cards); `test_legacy_instance_still_adopts_selected_profile` (behaviour + dropped, above). + +### Known leftovers + +1. S3: the context processors and `status_logging.py` still key on tool names; the old rail classes and their + three test files remain (red on `p4-switch`); the client patch is still called in the facade constructor. +2. `main.py` lists the worker's tools through `Runner.resource_mgr.get_mcp_tool_infos`, which now returns none. +3. The registry's heartbeat-owner bookkeeping is unused (every service stops its own stack). +4. `OpenAIModelClient.aclose()` on `shutdown()` closes every cached client in the process; a host that keeps model + clients alive across a facade shutdown rebuilds them on the next call. + +## The switch, step 3: processors, logging, dead code, renames + +The third switch session (design sections 1(e) and 8; decisions P1, P2, R4; the S2 review's U4, U5, U6; +`switch-plan.md` S3), on `p4-switch`. Nothing in the agent keys on a tool name or parses result text any more; the +old rails and the last client-patch consumers are gone; the browser-use golden is compared for real. + +### Decisions + +1. **The rail writes the effect onto the tool message.** `BrowserRuntimeRail.after_tool_call` adds `effect` (the + `Effect` value) to the metadata of the surface's budgeted tools (factory tools and non-exempt helpers); a tool + outside the surface, or an exempt one (`browser_recall_offload`), gets none. The 46-name refresh set and the + 4-name observation set of `BrowserStateContextProcessor` become "effect other than `READ`" and "`READ`"; the + JSON content parsing of `success`/`ok`/`state_changed`/`denied` goes, metadata alone decides. +2. **The working context trusts the metadata** for tool errors (`success is False` → the payload's `error` or the + text); the harness's own execution-error prefixes stay (they carry no metadata); the `ok`/`success`/`isError` + markers and the `success=false`/`error=` scan go. +3. **The status log is keyed by command type.** `BrowserSubagentStatusLogger(command_of=)` gets the rail's decoder + for `tool_start` (`RunScript` → code redacted, `Navigate` → URL without query) and reads the command record for + `tool_end` (`kind="command"`, command, effect, ok, dispatched, error, text length, line count, script value + kind). `browser_batch_interact` keeps its step summaries by name: it is the surface's own extra tool, not a + driver name (design 1(c) row 24). The command summary carries no URL: that would put a browser word into core. +4. **Old rails deleted, new rails take the old names (P1).** `RepeatedCallGuardRail` and `InterceptedActionHintRail` + on MCP text (call-log parsing, `### Events` masking, evaluate name checks) are gone; `CommandRepeatGuardRail` and + `CommandInterceptedHintRail` are renamed to those names, the guard no longer a subclass. The three old test + files are deleted; their records twins, which hold every old test name, take the old file names. +5. **The retry sentence is the surface's (P2).** `profile.messages["retry"]` = "Then retry the action (take a fresh + browser_snapshot if the page changed)."; `InterceptedActionHintRail(retry)` and `explain_blocked_command(record, + retry)` take it; `build_agent_parts` passes the profile's. The fake surface says "Then poke again.". +6. **`DriverAssembly.surface_settings` dropped (R4)**; the session re-provisions from `assembly.surface.settings`. +7. **Client patch consumer 4 gone**: the facade constructor no longer calls `ensure_browser_runtime_client_patch`; + `test_heartbeat.py` (one test, the stdio client patch) is deleted. The patch itself (`browser_tools.py`) and the + module-level copies in `main.py` and the MCP server entry stay for Phase 7 (J1, J3). +8. **Registry heartbeat ownership deleted (U6)**: `heartbeat_owner`, `is_heartbeat_owner`, `_live_owner` and + `BrowserServiceRelease` (`close_mcp_binding`, `next_heartbeat_owner`) are gone; `activate_binding` and `release` + return nothing. Every service stops its own stack on its last release, so nothing read them since S2. +9. **`main.py` lists the factory tools (U5)**: `runtime.agent_parts(image_support=False).tools` after + `ensure_started()`, instead of `Runner.resource_mgr.get_mcp_tool_infos` (empty since S2); the `Runner` import + goes. The listing is informational and stays. +10. **A given MCP config keeps the old env semantics (U4).** `create_playwright_mcp_driver` keeps the given env, + drops `PLAYWRIGHT_MCP_DEVICE`, sets the CDP endpoint and defaults `PLAYWRIGHT_MCP_BROWSER` to `chrome` only when + the device has an endpoint, as `_inject_cdp_endpoint` did; the surface-computed env no longer overrides a given + `PLAYWRIGHT_MCP_BROWSER`. +11. **The browser-use golden is a real comparison.** The snapshot builder builds a view on browser-use + (`build_view(view, workdir, driver="browser_use")`, `BROWSER_DRIVER` set, the sidecar transport stubbed with a + blank page as the MCP transport is). `test_tool_prompt_snapshot.py` asserts, per capability view, the tools equal + the 857239db golden except the one catalog's differences (Q3: run control, screenshot, close, drag and drop + missing; the probes and the batch added; Q6: `browser_tabs` gains `new`) and the system prompt equals the + Playwright MCP golden's (Q4: one prompt). The browser-use `runner` golden has no agents (the old runner never ran + browser-use), so the runner view is not compared; the WebVoyager smoke on browser-use is P4-T9's. + +### Rejected options + +- **A processor rule on `state_changed` alone**: the rail sets it `True` for any unknown tool that ran (a file write + on the main agent would refresh the browser state); the `effect` key marks the surface's tools. +- **Counting exempt tools (`browser_recall_offload`) as observations**: a recall-only turn would run the compact + capture and advance the semantic no-progress count; they were in neither name set before. +- **Keying the status log by argument shape** (`code`/`url` keys): names the dialect instead of the command. +- **Keeping `_normalise_call_key`'s evaluate handling** for no-record calls: a call that decoded to no command has + no script to mask; tool name and canonical arguments are enough. +- **Fixing the heartbeat wording of the `browser_runtime_health` tool description**: model-facing text of a runtime + tool; left for a benchmark-phase prompt review. + +### Behaviour changes to know + +- The browser-state processor refreshes after the surface's non-`READ` tools only; `browser_custom_action` + (a runtime tool the factory does not know, so it carries no `effect`) no longer triggers a refresh, where the old + name set listed it. `GetConsole`/`GetNetwork` results (`READ` commands) now count as observations and run the + compact merge; the old observation set did not list them. +- A helper's failed JSON payload is retained in the working context by its `error` field, as before, but only when + the rail marked the message `success=False`. +- The status log's `tool_end` summary of a command tool changed shape: `kind="command"` with the record's fields + instead of `{"kind": "dict", "keys": ["command", "content"]}` (which S1 had left it at). + +### Golden status + +| Check | Result | +|---|---| +| every MCP view, prompts included (`test_tool_prompt_snapshot.py`) | byte for byte, no golden touched | +| runner contract (`test_wv_runner_contract.py`) | green | +| factory goldens (`test_tool_factory_golden.py`) | `core` 18/18, `runner` 14/14 | +| browser-use views through the agent | tools = golden minus Q3 plus the helpers, tabs + `new` (Q6); prompt = MCP golden (Q4), all 11 capability views | + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q` on `p4-switch`: **2287 passed, 111 skipped** + (whole suite green; the three old-rail files gone, 22 browser-use golden tests added, the 12 browser-use + well-formedness tests no longer skipped). One timing test, + `surfaces/test_browser_batch.py::test_a_condition_waits_until_its_value_holds_still_for_the_stable_time`, failed + twice out of about ten runs ("no scripted answer left for `Wait`"; the stable-time loop polled once more on a slow + run); untouched by this step. +- Architecture tests: the allowlists shrank for every removed entry (`intercepted_action.py`, + `repeated_call_guard.py`, `service_registry.py`, `status_logging.py` driver names; `browser_state_context_processor.py` + and `intercepted_action.py` MCP names; `runtime.py` lowered to its actual counts). No entry grew. +- The ten harness test files outside `browser_move` that build the browser agent: **234 passed**. +- Live: MCP conformance + live assembly + live helpers **63 passed**; browser-use conformance + live helpers + live + assembly **56 passed, 1 failed** (the MCP half of the assembly test counted two OneDrive updater processes that + started during its window as Chrome children; alone, both halves pass **4/4**). +- Rewritten under their names: `test_browser_state_context_processor.py` (effects on the tool messages; the 46-name + case is the three mutating effects), `test_browser_subagent_status_logging.py` (commands and a record), + `test_browser_runtime_rail.py` (the `effect` key), `test_browser_result_contracts.py` (a decodable mock runtime), + `test_browser_service_registry.py`, `test_settings_passing.py` (`main.py` lists the facade's parts), + `test_browser_profile.py` untouched (the new message is additive). Added: a working-context error from metadata, a + command result summary, the args summary keyed by the decoded command, a given MCP env keeping its browser, the + browser-use view comparisons. +- Deleted tests: the three MCP-text rail files (89 tests; every name lives on in the renamed twins); + `test_registry_assigns_one_heartbeat_owner_and_transfers_it`; `test_heartbeat.py`. + +### Known leftovers + +1. `BrowserRuntimeRail._emit_status` logs a status-logger failure with `exc_info=True`, which + `browser_agent_log_warning` does not accept; a logger exception would surface as a `TypeError` from the rail. + Pre-existing, found when a test's mock runtime could not decode a command; not changed here. +2. `browser_runtime_health`'s card description still says "heartbeat status". +3. `main.py` and `playwright_runtime_mcp_server.py` keep their module-level client-patch copies; `browser_tools.py` + keeps the patch and the public `register_browser_runtime_mcp_server` wrapper (Phase 7). +4. The browser-use `runner` golden has no agents: no golden pins the runner's browser-use (Direct) view. + +## The switch, step 4: scoped snapshots + +The fourth switch session, on `p4-switch`: the model's `browser_snapshot` `depth` and `target` reach the server +again, and a scoped snapshot is a partial observation of the page. + +### Background + +The MCP smoke after step 3 cost 60% more than its reference ($1.33 against $0.83 for the same three tasks). Every +`browser_snapshot` call decoded to `Observe()` and the driver always sent `{"boxes": true}`, so `{"depth": 1}` on +Allrecipes returned 56,933 characters where the old path returned 361. The model reads pages cheaply this way: in the +559-task baseline, 215 tasks (38%) passed snapshot arguments (`depth` 208, `target` 25, `filename` 6, `boxes` 2). +The Phase 0 decision to accept and ignore these arguments is reversed. + +### Decisions + +1. **`Observe` gains `depth: int | None = None` and `root: Ref | None = None`**; `scoped` is true when either is + set. A scoped observation is the subtree under `root`, at most `depth` levels deep: partial by contract. + `include_screenshot` and `full_page` stay as they were. The root is a ref like any other, so `page_state.check` + refuses a scoped observe on a stale ref before the driver sees it. +2. **The MCP dialect decodes `depth` → `depth` and `target` → `root`**, the target with the same rule as every + other decoder (`ARIA_REF` → `DriverRef`, else `Locator(css)`), the depth rounded like every number. `boxes`, + `filename` and `element` stay accepted and ignored (baseline counts: `boxes` 2, `filename` 6; `element` is the + model's own label of the target). +3. **The MCP driver sends a scoped snapshot exactly as the model asked, without `boxes`.** A full `observe()` keeps + `{"boxes": true}` for the bounds a `PointRef` needs. The recorded fixtures show `boxes` adds only the `[box=…]` + tokens (`snapshot_with_boxes` without them equals `snapshot_form_page` up to the frame counter), so the text + would be the same either way; the model's arguments verbatim need no stripping and the answer is the server's + byte for byte. Two fixtures were recorded with `@playwright/mcp@0.0.78` on the fixture page, `snapshot_depth_1` + (`{"depth": 1}`, 442 characters) and `snapshot_target` (`{"target": "e3"}`, the navigation, 362 characters); the + rendered golden of each equals its recorded `model_text`. A `PointRef` or a text `Locator` root is + `Unsupported(dispatched=False)` before the call; the subtree's elements have no bounds. +4. **The page state treats a scoped observation as partial.** No generation check (a subtree against the last full + observation would trip the Jaccard rule on an unchanged page), and `last_observation`, `fresh_observation`, + `current_context`, `contexts` and the address are left alone; the refs it shows join the current generation and + the refs it does not show stay valid. This is the old split: a model's `browser_snapshot` went through + `register_ax_snapshot` (add), only the runtime's own capture through `replace_ax_snapshot`. So + `capture_browser_state` observes afresh after a scoped snapshot unless a full observation is still fresh. +5. **browser-use refuses a scoped `Observe`** with `Unsupported(dispatched=False)` before anything is sent. The + browser dialect's `browser_snapshot` schema has no `depth` or `target`, so nothing on that driver builds one; the + refusal is for the contract. +6. **Model-facing schemas do not change.** The MCP `browser_snapshot` card is the server's own (it always listed + `depth` and `target`); the browser dialect's has `include_screenshot` only. Every snapshot golden stays byte for + byte. + +### Rejected options + +- **`boxes: true` on scoped snapshots too**, for the subtree's bounds: the text would be identical after + `without_boxes`, but nothing reads the bounds of a partial observation (points come from the fresh full one), and + the model's arguments verbatim are the simplest proof that the model reads what it read before. +- **A separate `ScopedObserve` command**: two READ commands for one server tool; the dialect, the factory and the + renderer would all branch on it. Two optional fields and a `scoped` property do the same. +- **Recording a scoped observation as the page's observation**: the next full observation's element distance against + a subtree would start a generation on an unchanged page, and `browser_find` and the compact capture would search a + subtree. +- **Updating the address and title from a scoped snapshot's header**: a later full observation would then miss a + navigation the model saw only through a scoped snapshot (the address backstop compares against the known one). +- **A conformance case "scoped observe returns only the subtree"**: not added, see Known leftovers 1. + +### Golden status + +| Check | Result | +|---|---| +| every MCP view, prompts included (`test_tool_prompt_snapshot.py`) | byte for byte, no golden touched | +| factory goldens (`test_tool_factory_golden.py`) `core` / `runner`, browser-use views | unchanged | +| tool-result goldens (`test_tool_result_golden.py`) | `snapshot_depth_1.txt` and `snapshot_target.txt` added, each equal to the recorded model text; the other 36 unchanged | + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2307 passed, 111 skipped** (2287 before this + step). New: the contract's `scoped` property and `iter_refs` over the root; the dialect's `depth` / `target` + decoding and the non-numeric `depth` error; the driver's four scoped-observe cases (depth alone, a `DriverRef` + root, a css root with a depth, a `PointRef` root refused unsent); the page state's four rules (generation and last + observation kept, refs added and kept, an outdated observation not made fresh, a stale root refused); + browser-use's refusal; the facade's `capture_browser_state` reusing a full snapshot and observing afresh after a + scoped one; the two fixtures in the well-formedness, decoder and result-golden suites. +- `fixtures/mcp/record_mcp_fixtures.py` records the two new cases right after `snapshot_form_page` (same page, + same refs); re-recording `snapshot_depth_1` twice gave identical files. +- Live, `BROWSER_MOVE_LIVE_MCP=1`: MCP conformance + live assembly + live helpers **57 passed, 6 skipped** (the six + browser-use-gated cases of those files, no sidecar interpreter set in that run). Live, `BROWSER_USE_SIDECAR_PYTHON` + set: browser-use conformance **45 passed**. +- A scripted run through the factory's `browser_snapshot` on the live MCP scene (managed Chrome, `@playwright/mcp@0.0.78`): + the full snapshot is 1,104 characters, `{"depth": 1}` 327 characters (the root's children, subtrees cut), and a + `browser_click` on the "Save order" ref taken from the earlier full snapshot still succeeds afterwards (title + `clicked`); the generation stays `g1` across the scoped snapshot and the click. + +### Known leftovers + +1. **No conformance case for scoped observations.** The suite decides what a driver handles from `info.actions`, + which `HandlerDriver` fills from the `@handles` marks of `execute` handlers; `Observe` runs through `observe()` + and is never listed there (the factory treats it as handled by every driver), and a `depth`-only scope carries no + ref kind for `ensure_supported` to check. Listing `Observe` in `info.actions` would also make the suite's + `_pairs` generate happy-path cases that `execute` an `Observe`. Saying "this driver scopes" needs a decision + (`Observe` in `info.actions` with the suite skipping its happy path, a new `DriverInfo` field, or something + else) before the case can expect `Unsupported` on browser-use and a subtree on Playwright MCP. +2. `depth` is rounded like every numeric argument (`round(float(depth))`); the server's schema says `number`, so a + fractional depth from the model is rounded rather than refused. + +## The switch, step 5: the server judges the refs the model sends + +The fifth switch session, on `p4-switch`: on the Playwright MCP dialect a `target` the model sends carries no +generation, so the page state never refuses it before the driver; the server's own "not found" is the stale answer. + +### Background + +The MCP smoke after step 4 passed 3/3 at $1.28 against the reference's $0.83. Allrecipes and Amazon came in under +the reference; Google Flights cost $1.17 against $0.58. The dialect's `_target` stamped every aria ref and css +selector the model sent with `generation_of(target)`, the generation the page state last saw it in, and `check` +refused it with `Stale AX ref f1e149 belongs to g3; current generation is g4.` before the driver was called; +`CommandTool` then rendered the error plus a fresh full observation (about 18,000 characters). On a single-page +application the generation advances often (DOM churn, address changes), so refs the server would still resolve were +refused: six distinct refs in the Google Flights run, none in the reference run. The old worker sent refs straight to +`@playwright/mcp`, which resolves `aria-ref` and css against the live page and answers `Ref ... not found in the +current page snapshot` only when the element is really gone (the driver decodes that to `StaleRef(dispatched=False)`, +fixture `click_stale_ref`). + +### Decisions + +1. **Who owns staleness is a property of the dialect, written in the generation its decoder stamps.** The page + state owns it for the refs it hands out or the model dates itself: `resolve`d targets, points on a screenshot, + the browser dialect's `ref` / `selector` with the model's `generation_id`, browser-use index refs. The Playwright + MCP server owns it for the aria refs and css the model sends on its dialect. S_20 says this once under Refs; the + MCP dialect, which lives in the driver package, expresses it; no core code learns about MCP. +2. **`_target` builds `DriverRef(target, "")` and `Locator(css=target, text=None, generation="")`.** `check` + already lets an empty generation pass (the rule a point used before the first screenshot), so the page state does + not change. `PointRef` keeps `screenshot_generation`, the browser dialect keeps the model's `generation_id`, + `resolve` keeps the target's generation. This reverses decision 1 of "Tool factory and Playwright MCP native + tools" (a `DriverRef` at the generation the page state last saw it in). +3. **The server's refusal stays the driver-raised `StaleRef`.** `RetryOnStale` observes once and attaches the + fresh observation; `CommandTool` renders the error with it and records the observation, as before. The model + reads the same shape of answer as the pre-refusal gave, now only for an element that is really gone. +4. **Nothing downstream needed the generation of an MCP ref.** `RetryOnStale` reads only `dispatched`; `resolve` + and the batch / find / custom helpers stamp `page_state.generation_id` themselves; the policy probe checks the + command against the surface context; `CommandRecord.generation` and `message_metadata` come from the result and + the page state, not from the refs. The factory's builder for a spec without a schema (`_generated_build`) still + stamps `generation_of`; no MCP tool goes through it, and it was left alone. +5. **Two runtime-rail tests that asserted the pre-refusal on the MCP path were rewritten**, not deleted: after a + navigation the click on the old ref now reaches the driver and the server's `StaleRef` renders with its fresh + observation; after a URL change the click on the old ref reaches the driver and succeeds. + +### Rejected options + +- **Teaching `check` about the MCP dialect** (a flag on the page state or on `DriverInfo` that skips the check for + aria refs): the page state would learn a driver's name, and the generation on the ref would say something untrue. +- **A forgiving page-state rule** (accept a ref one generation old): the server already does the only lookup that + matters, against the live page; a forgiving rule would still refuse refs that survived two generations and still + pass refs of elements that are gone. +- **Keeping the pre-refusal and shrinking the observation it renders**: cheaper, but still a wrong answer for an + element that exists; the cost was the symptom. + +### Golden status + +| Check | Result | +|---|---| +| every MCP view, prompts included (`test_tool_prompt_snapshot.py`) | byte for byte, no golden touched | +| factory goldens (`test_tool_factory_golden.py`) `core` / `runner`, browser-use views | unchanged | +| tool-result goldens (`test_tool_result_golden.py`), `click_stale_ref` included | unchanged (the stale wording is the server's, the rendered error never carried a generation) | + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2309 passed, 112 skipped** (2307 before this step; one wall-clock batch test, + `wait_for_stable`, failed once while the live suite ran beside it and passed alone). New: the dialect's + target without a generation (an aria ref and a css selector, the resolver reporting an older one); the page state + passing an MCP click and hover after `reset()`; the factory's `browser_click` reaching the `FakeDriver` after the + generation moved (`Click(DriverRef("e1", ""))`, the record stamped `g1`) and the driver's `StaleRef` rendering with + the observation it carries; the two rewritten rail tests. The `PointRef` pre-refusal test is unchanged and green. +- Live, `BROWSER_MOVE_LIVE_MCP=1`: MCP conformance + live assembly + live helpers **58 passed, 6 skipped** (57 before; the six browser-use-gated cases of those files). New live case + (`test_browser_helpers_live.py`): a full snapshot, `browser_click` on "Open popup" (a second tab, the generation + moves), `browser_tabs` `select 0` (moves again, nothing re-registered), then `browser_click` on the "Last button" + ref from the first snapshot: the page state reports the ref in an older generation than the current one (the + popup click's answer snapshot re-registered the first tab's refs once, the tab select did not), the click reaches the + server and the title becomes `last`. +- Live, `BROWSER_USE_SIDECAR_PYTHON` set: browser-use conformance **45 passed**. + +### Known leftovers + +1. `_generated_build` (a spec without a schema) still stamps `generation_of`; no built driver serves an element ref + through it today. +2. The live case proves the server resolves a ref the page state's generation moved past; it does not measure the + smoke. The next MCP smoke tells whether Google Flights returns to the reference cost. +3. The import block of `test_browser_runtime_rail.py` was unsorted before this step (`ruff --select I`); it was left + as found. + +## The switch: verification baseline + +The verification session of the switch, on `p4-switch` after step 5: the small fixes the step reviews left to it, the +Phase 4 done-when items with their evidence, and the numbers the merge is judged against. + +### Decisions + +1. **A status-logger failure stays inside the rail.** `BrowserRuntimeRail._emit_status` logged a status-logger + exception with `exc_info=True`, which `browser_agent_log_warning(message, *args)` does not take, so the except + branch itself raised `TypeError` out of the rail (step 3's leftover 1). The failure is now one warning line, + `[BROWSER_SUBAGENT_ERROR] method= error=`, and the invoke goes on; the helper's signature is unchanged. +2. **The runner's Direct agent writes to the process log.** The core logger sends every line of an invoke to + `openjiuwen.browser_agent` while `BrowserRuntimeRail` holds the browser log context, which `before_invoke` entered + for every invoke. So on the browser-use runner path the model client's `CompletionUsage(...)` lines and the + agent's `tokens={...}` line ended in `browser_agent.log`, while the worker, which has no such rail, writes them to + stdout, where the runner's `budget.py`, `analyze.py` and `run_one_task` read them. `BrowserAgentRuntime.dedicated_log` + (default `True`) is `False` for the agent the facade builds for itself in `_direct_agent`, and the rail enters the + context only when it is `True`. Nothing the model sees changes; `create_browser_agent`'s own agents are unchanged. +3. **The repeat guard starts at the first model call too.** A `DeepAgent` fires `before_invoke` and `after_invoke` + on its outer callback context only; its inner `ReActAgent` runs the model and tool hooks on a context of its own, + with another `extra` dict. `RepeatedCallGuardRail` kept its per-invoke state in `ctx.extra` from `before_invoke`, + so on every deep agent (`build_agent`'s, and the main agent, which is the browser-use Direct path) the hooks found + no state and the guard counted nothing, while the worker, a `ReActAgent`, was guarded. The live module caught it: + five identical `browser_evaluate` calls through `build_agent` on Playwright MCP left the tools on the seventh + request. The state now also starts in the first `before_model_call` that finds none (the first model call precedes + every tool call, and the model-call context is the inner invoke's own), where the finish section is cleared as + `before_invoke` does. The worker path is unchanged. +4. **The argument normalisation is gated on `target` fields alone.** `BrowserRuntimeRail.before_tool_call` applied the + Playwright MCP normalisation (the `ref` → `target` rename, the `target_id` resolution, `generation_id` dropped) to + every factory tool whose schema had any of `target`, `startTarget`, `endTarget`, `ref`, `startRef`, `endRef`. The + browser dialect declares `ref` as a field of its own, so on browser-use through the main agent every element + command the model sent as `{"ref": …, "generation_id": …}` became `{"target": …}` and the schema check denied it + (`generation_id` missing, `target` not permitted); nothing reached the driver. The observe-count run showed it: + seven model turns on browser-use, no `Click` or `TypeText` at the driver. The gate is now the three `target` + fields, as S_20 already said; the browser dialect's tools are left to their decoders. Unit tests had driven the rail + only with the MCP dialect. + +### Rejected options + +- **Leaving the context around the model call only** (off in `before_model_call`, on in `after_model_call`): the + context processors run after the before hooks and before the model call, so the browser capture and its raw audit + would run outside the context anyway, and the agent logs its `tokens={...}` line after the after hooks, inside it. +- **A `Model` wrapper that drops the context inside `invoke`**: `create_browser_agent` copies the model into a plain + `Model` for the browser temperature, so the wrapper would not survive construction. +- **`OPENJIUWEN_BROWSER_AGENT_LOG_MIRROR_COMMON` in the runner's launcher**: every browser line into the runner's + stdout, not only the model calls, and a change outside `browser_move`. + +### Behaviour changes to know + +- On the runner's Direct path no raw audit artifact (`OPENJIUWEN_BROWSER_AGENT_AUDIT_RAW`) is written: the audit needs + the browser log context. Tool-call lines still reach the browser log through the hook, as the worker's do. + +### Phase 4 done-when, with the evidence + +| Item | Evidence | +|---|---| +| the fake-surface end-to-end test passes | `architecture/fake_surface`: 6 passed; no edit in `runtime/`, `contract/` or `drivers/middleware.py` | +| MCP tool and prompt snapshot = the Phase 1 golden | against `refactor/driver-architecture`: `core`, `runner`, `advanced_code`, `extended_interaction`, `pdf` byte-identical; `config`, `devtools`, `network`, `storage`, `testing`, `unsafe_dev`, `vision` differ by 822 removed lines, 0 added, no `system_prompt` hunk: the 43 approved no-command tools (C7/C20/C25). Runner contract and factory goldens green | +| browser-use snapshot = the 857239db record, differences listed | the browser-use golden files are unchanged against the old branch; per view the agent's tools are the golden minus Q3 (run control, screenshot, close, drag and drop) plus the two probes and the batch, `browser_tabs` with `new` (Q6); the prompt is the MCP golden's (Q4) | +| PR #7–#11 behaviours on `FakeDriver` and on both drivers | unit: `test_browser_repeated_call_guard.py`, `test_browser_intercepted_action_hint.py`, `test_browser_final_answer_recovery.py`, `test_browser_worker_allowlist.py`, `test_browser_prompt_tool_surface.py`, `test_browser_runtime_rail.py`; live: `test_agent_rails_live.py` on each driver (below) | +| no MCP tool name and no MCP text parsing left in `runtime/` | no text parsing; the remaining name hits are listed under "Allowlists" with their reason and phase | +| the smoke tasks pass on both drivers | the head's WebVoyager smokes, outside this session | + +### The live module (`test_agent_rails_live.py`) + +Gated as the other live suites (`BROWSER_MOVE_LIVE_MCP=1`, `BROWSER_USE_SIDECAR_PYTHON`), a headless Chrome from the +conformance fixture attached with `BROWSER_PROVISIONING=remote`, the scene server, a `ScriptedModel(Model)` that +issues one tool call per turn while tools are offered (names resolved by suffix, so the MCP prefix needs no case) and +then answers. Per driver: `build_agent`'s agent navigates and runs `() => document.title` five times: tools on six +requests, none on the seventh, the repeated-call note in the fifth evaluate's message; `build_agent`'s agent +navigates, snapshots, runs `cover()` and clicks the "Covered button" by the ref the snapshot listed (`target` on the +MCP dialect, `ref` + `generation_id` on the browser dialect): `### Error\nClick not delivered: is covered by …` +ending in the profile's retry sentence; `BrowserService` on the live driver builds the worker with the 14 core tools +(the driver's prefix) and the two probes and the batch, and a prose final answer is recovered in one tool-free call +(`recovered_from=non_json_output`, `ok=True`). + +### Double observation (Q14), measured + +A scripted six-step run of the main agent (`create_browser_agent` on a `BrowserAgentRuntime`, `core` capability) on +the live scene, each driver attached to its own headless Chrome; a counting wrapper under the middleware stack logged +every driver `observe` and whether each `execute` result carried an observation. Steps: navigate, click "Save order", +type "Ada" into "Full name", snapshot, click "Last button", snapshot; seven model turns. + +| Point | Playwright MCP | browser-use | +|---|---|---| +| before the first model call (initial capture) | 1 observe | 1 observe | +| after `browser_navigate` | 0 (the result carried an observation, reused) | 1 (no observation in the result) | +| after `browser_click` | 0 (carried, reused) | 1 | +| after `browser_type` | 1 (`browser_type`'s result carries none) | 1 | +| `browser_snapshot` (the model's own) | 1 | 1 | +| after the second `browser_click` | 0 (carried, reused) | 1 | +| second `browser_snapshot` | 1 | 1 | +| **total driver observes** | **4** | **7** | +| `observations_reused` | 3 | 0 | + +`capture_browser_state` never observed right after an action whose result carried an observation (Q14 holds); each +capture also runs one `page_metadata` script. On Playwright MCP, `browser_navigate` and `browser_click` answers carry +the page snapshot and `browser_type`'s does not (server behaviour, 0.0.78); on browser-use no action result carries an +observation, so every mutating step is followed by one capture observe. + +### Allowlists: what stays in `runtime.py` and why + +Driver names 2 (was 3): `latest_browser_user_request` matches the `browser_use` regex twice (the import and the +call); the docstring "the worker on Playwright MCP" was reworded. MCP tool names 9, none a text parser: + +| Hit | Why it stays | Goes in | +|---|---|---| +| `browser_take_screenshot` ×4 in `_BROWSER_IMAGE_CAPABILITY_GUIDANCE` (en, cn) | model-facing prompt section | Phase 9 (the browser rail and its prompt text move to `surfaces/browser/`) | +| `browser_resume` ×2 | the run-context flag callers set to resume a task, not the MCP tool; renaming changes a public key | Phase 9 (rail move) | +| `browser_click$` → `browser_navigate` in `_rewrite_primary_link_click` | the card primary-link rewrite finds the dialect's navigation tool by its model-facing name; a name-free lookup needs a command → tool map from the factory (a contract change) or the move of this browser-only rewrite | Phase 9 (rail move) | +| `"source": "browser_evaluate"` in `_evaluate_field_provenance` | a provenance label the working context shows the model | Phase 9 | + +### Verification + +- `test_browser_runtime_facade.py`: the Direct runtime reports `dedicated_log=False` and its rail's `before_invoke` + leaves the browser log context unset; the default runtime enters it. +- `test_browser_repeated_call_guard.py`: a run whose `before_invoke` fired on another context is guarded from its + first model call (tools withheld after the fifth identical call, a stale finish section cleared). +- `test_browser_runtime_rail.py`: a browser-dialect `browser_click` with `ref` and `generation_id` passes + `before_tool_call` untouched (red on the old gate: the arguments came out as `{"target": …}`). +- `test_browser_runtime_rail.py`: a status logger that raises is logged with the hook name and the error and the + rail returns. +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2314 passed, 118 skipped** (2310 / 112 at the + start of the session; the live module adds six gated cases). +- The ten harness test files outside `browser_move` that build the browser agent: **234 passed**. +- Architecture: the driver-name allowlist of `runtime.py` 3 → 2; no entry grew (the browser-word ratchet caught two + wordings of this session's own lines, both reworded before commit). +- Live, `BROWSER_MOVE_LIVE_MCP=1` with the sidecar interpreter set: Playwright MCP conformance + live assembly + live + helpers + the new module **70 passed** (the seven browser-use-gated cases of the shared files included); with the + sidecar alone: browser-use conformance + live assembly + live helpers + the new module **54 passed, 10 skipped** + (the MCP-gated cases). Run at the commit before decision 4, which touched only the main agent's rail; that rail ran + live on both drivers afterwards in the observe-count run above. The new module alone: 3 and 3. +- `wait_for_stable` batch test, 30 runs each: 1 failure in 30 before the fix, 0 in 30 after. +- Goldens: no file moved. +### Known leftovers + +1. The four `runtime.py` MCP-name hits above, Phase 9. +2. On the runner's Direct path no raw audit artifact is written (decision 2). +3. The main agent's `BrowserRuntimeRail` registers the parts' rails in `before_invoke`, so on the DeepAgent their own + `before_invoke` never fires for that invoke; the guard copes (decision 3), `ActionPolicy` and + `InterceptedActionHintRail` keep no per-invoke state. +4. `ruff format` and `ruff check --select I` report pre-existing differences in `runtime.py` (quote style, blank + lines) and `test_browser_runtime_rail.py` (import order); left as found. + +## The switch: verification follow-up + +The head's browser-use smoke on the verification baseline (three tasks, Sonnet) lost one task to the runtime, not +to the model: on Allrecipes--20 the driver's `start()` failed inside `BrowserRuntimeRail.before_invoke` (the sidecar's +connect answered "No valid agent focus available - target may have detached and recovery failed"), core's callback +framework logged the exception and went on, and the main agent ran with no browser tools: the model wrote +`` as text until `max_tokens` (65,536 completion tokens) and the task ended `partial` +on `about:blank` after 304 s. The same smoke showed the runner's Direct agent built with the facade's default +`language="cn"` (the runner passes none), so an English task got the Chinese prompt and working context, where the +worker it replaces builds in English. + +### Decisions + +1. **A session that cannot start fails the invoke.** The callback framework swallows every callback exception + except `AbortError`, and re-raises an `AbortError`'s `cause` to the caller. `BrowserRuntimeRail.before_invoke` + now wraps `ensure_runtime_ready()`: any failure there becomes `AbortError(str(error), cause=error)`, so the + invoke fails before a tool is added or a model call is made, and the caller sees the driver error itself; the + runner's Direct answer maps it to `ok=False` with the error text (`_run_direct` already caught exceptions). The + rail is the only place a callback starts the session: the worker path (`BrowserService.ensure_started`) and the + facade's other entry points (`capture_browser_state`, the custom actions, the policy hooks) run outside + callbacks and already propagate. +2. **A detached target at connect is a lost connection, and a session start restarts once.** The browser-use + driver maps "No valid agent focus available" / "target may have detached" to `ConnectionLost(dispatched=True)`, + as it maps "CDP not connected" and "Reconnection wait timed out", so a mid-task loss restarts the session once. + `DriverSession.start()` gives a `ConnectionLost` from the stack's start the same single restart: `_start()` + (liveness, re-provision and rebuild when dead, start) is the plain start; `_restart(error)` stops the stack, + calls `_start()`, wraps any failure in the `driver_lost` task failure with that failure as cause, and resets the + page state; `_recover` and `start()` both call it. `_restart` no longer re-enters `start()`, so a mid-task loss + still gets exactly one restart. The loss counter is left to `begin_task()`, which runs right after `start()`. +3. **The Direct agent speaks the worker's language.** `WORKER_LANGUAGE = "en"` in `service.py` is the one place the + worker's language is written; `BrowserService.ensure_started` (the parts and the runtime tools) and + `BrowserAgentRuntime._direct_agent` read it. The facade's own `language` (default `cn`) still reaches its tool + context and the parts it builds, and `create_browser_agent`'s default stays `cn` (public API). + +### Rejected options + +- **Catching the start failure in `ensure_runtime_ready` and reporting it in the answer**: the invoke would still + run without tools; the abort is the framework's own channel for "do not continue". +- **Re-entering `start()` from `_restart`**: a mid-task loss would restart twice (once in `_recover`, once more + inside the restarted `start()`). +- **Retrying the first open (`assemble_driver`, before a `DriverSession` exists)**: not in the task; the Allrecipes + failure was that first open, so it now fails the invoke explicitly and gets no restart (leftover 1). +- **Passing `language="en"` from the runner**: outside `browser_move`, and every other direct constructor of the + facade (`main.py`, the MCP server) would have the same gap. +- **Changing the facade's default language to `en`**: the Direct agent must match the worker whatever the facade is + given; the default is the facade's, not the worker's. + +### Behaviour changes to know + +- A driver that cannot start fails the task at once with the driver's error (the runner sees `ok=False`, + `error="connect: …"`) instead of a model loop without tools up to the task timeout. +- On browser-use the detached-target error at connect or mid-task now restarts the session once, like a dead CDP. +- The runner's Direct agent reads the English prompt and working context. + +### Verification + +- `test_driver_session.py`: a start that loses the connection restarts once (`stop, start, stop, start`, page + state generation bumped, no provisioning on a live device); a second loss fails with `driver_lost` and the + `ConnectionLost` as cause. +- `test_browser_use_driver.py`: the detached-target message maps to `ConnectionLost(dispatched=True)` at connect + (the sidecar closed) and mid-task. +- `test_browser_runtime_rail.py`: a session that cannot start raises `AbortError` with the error as cause and its + text as reason; no tool added, no rail registered. +- `test_browser_runtime_facade.py`: on browser-use, the real Direct agent (the image probe off) on a session that + cannot start answers `ok=False` with the connect error and the model client is never called; a facade built + without a language builds its Direct agent with `language="en"`, the English working-context processor and the + English system prompt. +- All seven new tests were red on the previous commit and are green now. +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2321 passed, 118 skipped** (2314 / 118 before). +- Goldens: `git diff` against the baseline commit is empty for `snapshots/golden/`. +- Live, sidecar set (browser-use conformance + live assembly + live helpers + `test_agent_rails_live.py`): + **54 passed, 10 skipped** (the MCP-gated cases). Live, `BROWSER_MOVE_LIVE_MCP=1` with the sidecar set: MCP + conformance + the same three files **70 passed** (the seven browser-use-gated cases of the shared files included). + +### The scripted run that ends `blocked` on browser-use + +The six scripted steps (navigate, click "Save order", type "Ada", snapshot, click "Last button", snapshot) make the +same semantic progress on both drivers. The tracker's state is url, form values, selected filters, result count and +field coverage (the title is not part of it), so the navigation and the typing count as progress and the two clicks +and the two snapshots as no progress; the capture before the final model call is the third consecutive no-progress +observation and sets the tracker's `replan_required` (`three_consecutive_no_progress_states`, revision 7) on both +drivers, measured with the rail's hooks wrapped. On Playwright MCP the flag is cleared at once: the sync into the task +state counts the last recorded action as recovered when it added evidence, and the second `browser_snapshot` is a new +page observation there because the MCP snapshot text carries the page title (`Page Title: form:3:false`, then `Page +Title: last`: two observations, two hashes). On browser-use the two snapshots are one and the same observation (one +hash): the sidecar's DOM text has no title, and the scene's last button changes nothing but the title, so no evidence +is added, nothing recovers, the task state stays `replan_required`, and the final-answer validation adds +`semantic_replan_required` as a runtime blocker: `blocked`, `terminal_reason=runtime_blocked`, +`recommended_recovery=replan_from_current_page`. No contract rule breaks: the semantic state ignores the title by +design and the evidence signature is the driver's observation text; the difference is the two drivers' renderings of a +page whose only visible change is its title. No fix. + +### Known leftovers + +1. The first open of a session (`assemble_driver` → `stack.start()`, before a `DriverSession` exists) gets no + restart: a `ConnectionLost` there fails the invoke at once. The Allrecipes--20 failure was that case. +2. On the Direct path the facade's own `language` (the runner's `cn`) still reaches the `ToolContext` and the parts + built for the main agent (`browser_recall_offload` takes it); the tools' schemas and the profile text are + language-free. +3. Leftovers of the previous section stand (the `runtime.py` MCP-name hits, no raw audit on the Direct path, the + rails registered in `before_invoke`, the import order of `test_browser_runtime_rail.py`). + +## Phase 6: topology setting + +The topology, who runs a task behind the runner's facade (the main agent on the facade, or a worker through the +browser service), was `CoreSettings.topology`, a property derived from the driver: `delegated_worker` iff +`playwright_mcp`. That was the one comparison with a driver name left in the core's settings, and it made the +other topology unreachable for either driver. This step turns it into a setting with a per-driver default declared +by the driver's registration; it supersedes "Settings" decision 6 and its rejected `BROWSER_TOPOLOGY`. The strategy +objects that run each topology, and the facade rewire onto them, are the next step of the phase. + +### Decisions + +1. **`topology` is a field read from `BROWSER_TOPOLOGY`.** `direct` or `delegated_worker` after trimming and + lower-casing, `""` when unset or blank, `ValueError` naming both values otherwise: the shape of the + `BROWSER_DRIVER` parser, with `TOPOLOGIES` next to `DRIVERS` in `config.py`. The field does not depend on the + driver: `BROWSER_DRIVER=browser_use` alone still gives `""`. +2. **Unset means the driver's default, resolved where the driver is known.** `config.py` cannot ask the registry: + it imports only `contract` and `surfaces`, and the built-in registrations run when the composition root imports + `drivers/`, so a `CoreSettings` built in a settings test would see an empty registry and differ from one built + by the composition root. The parser keeps `""`; the facade's `ensure_started` reads + `core.topology or default_topology(core.driver)` on `service.core`, so an instance's `driver` override takes + its own default. This inline bridge is the smallest change that keeps the runner working; `topology.py` + (`topology_for(core)`) replaces it in the next step. +3. **The default lives in the registry, not in a table.** `register_driver(name, factory, *, + default_topology="direct")` records a string in `DRIVER_TOPOLOGIES`, module state next to `DRIVERS` in + `contract/registry.py`; `default_topology(name)` returns it or raises the `KeyError` of an unknown driver with + the registered names, like `Registry._factory`. A string keeps `contract/` stdlib-only; the `Registry` class is + generic over drivers, surfaces and provisioners and gets no driver-only field; a driver registered from another + repository declares its default in the same call. A duplicate name is refused by `DRIVERS.register` before the + default is recorded. +4. **Validation is split by layer.** `config.py` validates the environment value against `TOPOLOGIES`; `contract/` + stores and returns the string without judging it; the core maps a name to a behaviour (today the facade's + comparison with `direct`, next step `topology.py`, which raises for a default name no strategy implements). +5. **Both registrations spell their default.** `browser_use` passes `default_topology="direct"` although it is the + parameter's default, so the two facts sit side by side in `drivers/__init__.py`. + +### Rejected options + +- **A `DEFAULT_TOPOLOGY` dict in `drivers/__init__.py`.** The same two facts in a second structure keyed by the + same names; the core would import the `drivers` package to read it (today only the composition root does); a + third-party `register_driver` call could not declare its default. +- **Resolving the default in `CoreSettings.from_env`.** The registry may be empty at parse time (decision 2); the + settings would depend on import order. +- **A `default_topology` field on `Registry` or on `DriverInfo`.** The registry is generic over the three kinds; + `DriverInfo` exists only once a driver is built, and the default is needed before the session opens. +- **Keeping the property and reading the variable as an override only.** The property's `driver == + "playwright_mcp"` is the driver-name knowledge the registration now owns; the core keeps no table of drivers. +- **An enum of topologies in `contract/`.** The contract would judge a value that only the core gives meaning to; + `TOPOLOGIES` in `config.py` is the one allowed set. + +### Behaviour changes to know + +- `CoreSettings().topology` is `""`, not `"delegated_worker"`; a reader resolves it through `default_topology`. +- `BROWSER_TOPOLOGY` is read; an invalid value fails `CoreSettings.from_env`, so `from_process_env`, the runtime + MCP server and the WebVoyager runner fail at start, like an invalid `BROWSER_DRIVER`. +- With the variable unset nothing changes: Playwright MCP runs the delegated worker, browser-use runs the main agent + on the facade. +- `contract` exports `DRIVER_TOPOLOGIES` and `default_topology`; `register_driver` takes the keyword-only + `default_topology`. + +### Verification + +- `test_core_settings.py`: `BROWSER_TOPOLOGY` trimmed and lower-cased for both values, unset and blank give `""`, + an unknown value raises naming both values, the field does not depend on `BROWSER_DRIVER`; the defaults test + asserts `""`; the driver test no longer asserts a derived topology. +- `contract/test_registry.py`: `register_driver` records the given default and `direct` when none is given; an + unknown name lists the registered ones; a duplicate registration is refused before its default is recorded. +- `test_service.py`: the instance's driver override resolves to `direct` through `default_topology`. +- `test_browser_runtime_facade.py`: `ensure_started` on each driver with the field unset takes the registered + default (MCP: the worker through the service, browser-use: the main agent through `create_browser_agent`) and + with the field set takes the setting (MCP + `direct`, browser-use + `delegated_worker`). +- Architecture: green without an allowlist change. `config.py` is outside the driver-name scan (config modules are + exempt), so the deleted comparison changes no count; `BROWSER_TOPOLOGY` is learned as an env name from the + constants of `config.py`, and no other module reads it; `contract/` stays stdlib-only. +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2337 passed, 118 skipped** (2321 / 118 before; + 16 new tests). +- Goldens: `snapshots/golden/` unchanged (no file in `git status`). +- `ruff check` clean on every touched file. `ruff format --check` reports hunks in `runtime.py`, `test_service.py` + and `test_browser_runtime_facade.py` that predate this step (context-manager layout, a quote style); left as they + are. +- No live suite: no driver, provisioning or surface code changed. + +### Known leftovers + +1. The facade resolves the default inline in `ensure_started`; `topology.py` takes it over in the next step, with + the `ValueError` for a registered default name that no strategy implements. +2. The S_20 paragraph on the runner's entry points still describes the two branches inside the facade; it moves to + the `Topology` strategies with that step. +3. `create_browser_agent` (the harness main agent, which always calls the tools itself) is outside + `BROWSER_TOPOLOGY`'s scope; the variable governs the runner's facade only. + +## Phase 6: assembly pieces, the first-open retry and the worker's one builder + +The second step of Phase 6 gives the composition root the surface-neutral pieces every agent is built from, and +moves the worker's construction out of the service. Before it, `BrowserService` opened the session with +`assemble_driver` and built the worker in `ensure_started`; `build_agent_parts` created the three core rails over a +copy of the factory's builders, so on Playwright MCP (whose native tools exist only after `start()`) rails could +only be built after the driver started, which is why the main agent registered them at its first invoke +(`RepeatedCallGuardRail` grew a fallback for the invoke it missed). The worker prompt named "Playwright MCP" in two +sentences on both drivers. The design review (A1) kept `assembly.py` surface-neutral: no browser agent body goes +in, each agent kind keeps one builder that takes the neutral pieces. + +### Decisions + +1. **`assembly.py` gets the neutral pieces only.** `assemble_surface` (step 1 of `assemble_driver`, public so a + caller can hold the profile before a driver exists), `open_session` (the one way a session is opened), + `core_rails` (the policy, the guard, the hint over a live mapping), and `build_agent_parts` returning the + profile's rails only. The module still names no driver, surface or provisioner and carries no browser word. +2. **One retry of the first open.** `open_session` calls `assemble_driver` again once when the first call raises + `ConnectionLost`. The failed assembly has already stopped its stack and released its device, so the retry is a + fresh provision, the equivalent of `DriverSession._start` on a dead device; a second `ConnectionLost` or any + other error propagates. This is the T9b question 1 answer (the first-open failure of "The switch: verification + follow-up", leftover 1). The session's own restart stays as it was for a connection lost later. +3. **The policy rail reads a live mapping.** `ActionPolicy` keeps the `builders` mapping it is given instead of + `dict(builders)`, and no longer passes it as `BaseSecurityRail.tool_names`, which only feeds `get_tools()` and + `add_tools` and is never consulted at check time. A decoder added after construction is enforced from the next + call. This is what lets the next step build the rails at the agent's construction over the facade's mapping. +4. **The core rails come from `core_rails`, not from the parts.** `build_agent_parts` returns `profile.rails` only; + `build_agent` composes `core_rails(profile, parts.builders, page_state)` before them; `build_worker` registers + them, then the parts' rails, then `FinalAnswerRecoveryRail`, the order the service used. Until the facade + switches to construction-time rails (next step), `BrowserAgentRuntime.agent_parts` composes the core rails into + the parts it returns, over its live `_builders` mapping, so `BrowserRuntimeRail` registers the same rails at the + first invoke as before. That bridge is three lines and goes with the next step. +5. **The worker's builder is `agents.build_worker`.** `build_worker(session, bundle, *, provider, api_key, + api_base, model_name, max_steps, core, capabilities, tool_result_observer, screenshot_subdir, artifacts_subdir)` + is the body of the old `BrowserService.ensure_started`: the parts (`worker=True`, `image_support=False`, + `WORKER_LANGUAGE`), `build_browser_worker_agent`, the rails. It sits next to `build_browser_worker_agent` because + it is browser-worker code (A1), and `WORKER_LANGUAGE` moves there with it (the service no longer needs it; the + facade imports it from `agents.py`). `BrowserService.ensure_started` calls it; `_open_session` calls + `open_session` and keeps the registry registration (the lifecycle identity is a service concept). +6. **The model identity is a parameter, not `core`.** The design signature took only `core`, but the service's + `provider`, `api_key`, `api_base`, `model_name` and `guardrails` are its own constructor values, not + `service.core`'s (`create_browser_agent` resolves them from the browser subagent's model settings, and + `test_service.py` pins `max_steps` from the service's guardrails, not the core's). Taking them from `core` would + have changed which model the worker calls wherever the two differ. So `build_worker` takes them explicitly; + `core` supplies the worker temperature, top-p, tool timeout and logging as before. +7. **The driver names itself to the model: `DriverInfo.display_name`.** `"Playwright MCP"` from + `PlaywrightMcpDriver.describe`, `"browser-use"` from `BrowserUseDriver.describe`; a string, so `contract/` stays + stdlib-only. `build_browser_worker_system_prompt`, `build_browser_worker_agent` and + `refresh_browser_worker_system_prompt` take a required `driver_name`, rendered in the two sentences that named + the driver; `worker_driver_name(info)` falls back to `info.name` for a driver that leaves the field empty (the + fakes). The template keeps the article ("a `{driver_name}` action"), which is right for both names. The MCP + `runner` golden is byte for byte what it was. + +### Rejected options + +- **A driver-keyed table of prompt sentences in the surface profile.** The profile is built before any driver + exists and takes no driver input; a `{"playwright_mcp": ..., "browser_use": ...}` table in `surfaces/browser/` is + a driver name in the surface scope that Phase 7 would have to empty again. +- **A default for `driver_name`.** A default of `"Playwright MCP"` would keep the old text on browser-use silently + for any caller that forgot the argument; every caller now says which driver it renders for. +- **`build_worker` reading the service.** `build_worker(service)` would reach into the service's private + progress recorder and subdirectories, and the worker builder would import the service type. Explicit arguments + keep `agents.py` independent of `service.py` (which imports it). +- **`build_worker(..., core=)` as the model's source.** Decision 6. +- **Retrying inside `DriverSession`.** The session exists only once an assembly started; the first open has no + session to restart, and a retry in `assemble_driver` would also retry the surface-neutral e2e builder and the + live conformance opens that assert a single failure. +- **Keeping `tool_names=builders` on the policy.** `get_tools()` would then show the construction-time names, a + stale view once decoders arrive later; nothing reads it. + +### Behaviour changes to know + +- A first `start()` that loses its connection (Playwright MCP could not connect, browser-use's sidecar lost CDP) + provisions and starts once more before the task fails; the first attempt's device is released in between. Tests + that count `assemble_driver` calls on a failing first open now see two. +- The worker prompt on browser-use reads "browser-use tools" and "a browser-use action"; on Playwright MCP it is + unchanged. +- `build_browser_worker_system_prompt`, `build_browser_worker_agent` and `refresh_browser_worker_system_prompt` + require `driver_name`; `build_agent_parts(...).rails` holds the profile's rails only; `ActionPolicy.get_tools()` + is empty. +- `WORKER_LANGUAGE` is imported from `runtime.agents`, no longer from `service`. +- `BrowserService` imports neither `assemble_driver` nor `build_browser_worker_agent`: tests that stub either + patch `runtime.assembly.assemble_driver` or `runtime.agents.build_browser_worker_agent`. + +### Verification + +- `test_assembly.py`: `assemble_surface` applies the instance to the given settings and is what `assemble_driver` + starts from; `open_session` retries once (two provisions, the first device released, the session over the second + stack with the given page state), propagates a second `ConnectionLost` with both devices released, does not retry + another `DriverError`, and reads the profile's messages. +- `test_build_agent.py`: the parts hold the profile rails only; `core_rails` order and inputs (the mapping object + itself, the rules, the page state, the retry message); `build_agent` hands the three core rails before the + profile rail to `create_deep_agent`. +- `test_action_policy.py`: a decoder added to an initially empty mapping after construction is enforced. +- `test_build_worker.py` (new): the worker's tools (the prefixed core tools, the two probes, the batch, no recall); + the model identity, `max_steps`, `core`, the observer, the subdirectories and the driver name handed to + `build_browser_worker_agent`; the rail order with `FinalAnswerRecoveryRail` last and the policy over the + session's page state; the rendered prompt names the display name ("browser-use tools", "a browser-use action") + and the model takes the core's temperature and top-p; the parts are built with `worker=True` in + `WORKER_LANGUAGE`. +- Driver tests: `PlaywrightMcpDriver().info.display_name == "Playwright MCP"`, `BrowserUseDriver().info.display_name + == "browser-use"`. +- `test_service.py` worker tests unchanged in substance (patch targets moved to `assembly` / `agents`); + `test_browser_runtime_facade.py`: a session that cannot start on browser-use now awaits `assemble_driver` twice. + `test_browser_final_answer_recovery.py` opens a session before injecting its worker (the prompt refresh reads the + driver's name from the session). +- Architecture: `KNOWN_DRIVER_NAMES` ratcheted `runtime/agents.py` 4 → 1 (the worker card's + description, not model-facing, stays for Phase 7) and `runtime/service.py` 14 → 1; + `KNOWN_BROWSER_WORDS` `runtime/service.py` 92 → 82. `assembly.py` stays off both allowlists. +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2349 passed, 118 skipped** (2337 / 118 + before; 12 new tests). +- Goldens: `snapshots/golden/` unchanged (no file in `git status`); the MCP `runner` prompt renders byte for byte + through `display_name`. +- `ruff check` clean on every touched file. +- Live suites: run once at the end of the phase's third step (the session plan), recorded in "Phase 6: + verification baseline". + +### Known leftovers + +1. `BrowserAgentRuntime.agent_parts` composes the core rails into its parts (decision 4) until the next step + builds them at the main agent's construction. +2. `WORKER_LANGUAGE` lives in `agents.py` next to the builder that needs it; the design placed it in `topology.py`, + which would import `agents.py` (for `build_worker`) and so cannot be imported by it. The next step decides + whether `topology.py` re-exports it. +3. The worker card's description ("using Playwright MCP tools") and the `browser_custom_action` tool description + still name the driver; neither is model-facing on the worker. Phase 7. +4. `_run_task_once` reads the driver's name for the prompt refresh from the service's session, so a worker injected + without a session (tests) needs one opened first. + +## Phase 6: topology as a strategy, one construction path + +The third step of Phase 6 turns the facade's two branches into strategy objects, builds the main agent's core rails +at its construction, and takes the worker's construction and the facade's worker-tool registration out of the +service and the facade. Before it, `ensure_started` / `run_browser_task` branched on the topology name inside +`runtime.py` (the Direct agent, its run and its answer mapping lived there), the core rails reached the main agent +at its first invoke through `BrowserRuntimeRail` (so `RepeatedCallGuardRail` missed the first `before_invoke`), the +facade's own `language` leaked into the parts of agents built in another language, and `service.run_task` rebuilt +the worker itself through `ensure_started`. + +### Decisions + +1. **`topology.py` owns the two strategies; the facade delegates.** `Topology` is a protocol (`name`, `start`, + `run`, `drop`): `run` alone would not do, because the runner calls `ensure_started()` first, reads + `_service.browser_agent` between the two calls, and the runtime MCP server starts once and runs many tasks. + `Direct` owns the main agent (built in `start`, forgotten in `drop`) and its one-attempt run under + `asyncio.wait_for`; `DelegatedWorker` owns the worker's construction (`build_worker`), the runtime tools and the + hand-over to `service.run_task(worker=)`. `topology_for(core)` resolves `core.topology or + default_topology(core.driver)` and raises for a registered default no strategy implements; the facade keeps the + instance from `ensure_started` to `shutdown`. Neither strategy names a driver. +2. **The main agent stays built by `create_browser_agent` (A1).** `Direct.start` calls it with `runtime=` + and `WORKER_LANGUAGE`, as the facade did; `assembly.py` gets no browser agent body. The design's `build_main_agent` + was dropped by the review. +3. **The core rails are queued at the main agent's construction.** `create_browser_agent` injects + `[BrowserRuntimeRail(runtime), *runtime.core_rails()]` (then the context-processor rail); `DeepAgent.invoke` + registers queued rails in `_ensure_initialized()` before `BEFORE_INVOKE`, so every core rail sees the first + invoke. What made this possible is the policy over the facade's live `builders` mapping (previous step): the + rails exist before the driver starts, over a mapping `agent_parts` fills later. `BrowserRuntimeRail._add_agent_parts` + adds tools only; its `_railed` set is gone. The guard keeps its first-model-call fallback, which still serves a + guard registered after an invoke began. +4. **One language per agent.** `agent_parts(*, image_support, language)` is keyed by both; the rail asks for the + parts in the receiving agent's language (`resolve_language(agent.deep_config.language)`), `build_worker` in + `WORKER_LANGUAGE`. The facade's constructor `language` (default `cn`, public) serves only its own tool context. + The five runtime tools lose their dead `language` parameter. +5. **The worker loop stays in `service.py`; it reads the worker through a callable.** `run_task(..., *, worker)` + and `run_task_once(..., *, worker)` await `worker()` per attempt, the recovery call reuses that agent; + `_restart` is `reset()` alone, so the next `worker()` (= `DelegatedWorker.start` + the slot) rebuilds on a fresh + session. `ensure_started` and `_restart_browser_runtime` are gone from the service; the `browser_agent` slot + stays for the runner (D7). The ≈900-line loop moves in Phase 9 by `git mv`. +6. **The facade's new members are the pieces the strategies need, nothing else.** `surface` (lazy, through + `assemble_surface`, so a facade built with no core settings still reads the environment at first use, not at + construction), `builders` (the live mapping), `core_rails()`, `register_runtime_tools()` (the old tail of + `ensure_started`, idempotent), `main_agent_answer(...)`, a `dedicated_log` setter, `workspace` and + `capabilities`. The constructor and the runner's members are unchanged. +7. **The answer mapping stays on the facade.** `topology.py` is a core module under the browser-word ratchet + (`test_browser_words.py`: a new core file may carry none, and allowlists never grow); the runner's answer dict + is `page` / `url` / `title` by contract. So `Direct.run` calls `runtime.main_agent_answer(...)`, the old + `_direct_answer` made public; the keys are the facade's run contract and `page` comes from the facade's page + state, so it is the right owner anyway. +8. **`WORKER_LANGUAGE` is written in `agents.py` and re-exported by `topology.py`.** `topology.py` imports + `build_worker` from `agents.py`, so `agents.py` cannot import the constant from it; the design's "moved to + `topology.py`" is met by the re-export (`from .topology import WORKER_LANGUAGE` works, `main.py` uses it). +9. **Two new goldens pin the other topology.** `golden/playwright_mcp/runner_direct.json` (the main agent through + the runner mirror with `BROWSER_TOPOLOGY=direct`: role `main`, the 14 prefixed core tools plus the helpers the + runner cannot remove, the main-agent prompt) and `golden/browser_use/runner_worker.json` (the worker on + browser-use with `BROWSER_TOPOLOGY=delegated_worker`: role `worker`, the 14 unprefixed core tools after the + runner's removal, the prompt saying "browser-use tools"). `builder.RUNNER_VIEWS` maps each runner view to its + topology and role; `list_views(driver)` adds the other-topology view of each driver; `regenerate.py` writes + both. The twelve recorded browser-use goldens are never rewritten (they carry `source_commit`), and the tests + that compare recorded goldens to today's specs skip a golden without it. + +### Rejected options + +- **`drop` left out of the protocol (dropping the topology object instead).** The facade could forget the object; + the explicit `drop` keeps the Direct agent's lifetime readable next to `start`, and the task names it. +- **`DelegatedWorker.run` passing a restart callable into `run_task`.** Two hooks where one suffices: the worker + callable both returns the current worker and rebuilds it after a reset. +- **A worker-factory stored on the service.** The service would still "build" through it; the design rejected it. +- **The answer mapping in `topology.py`.** Decision 7. +- **A `drop` on `DelegatedWorker` that clears the service's slot.** The service already clears it in + `_clear_task_scoped_state` on release, reset and shutdown; a second owner of the slot is a second path. +- **A module-scoped Direct run in `test_wv_runner_contract.py`.** The test's own instrumentation reads + `worker.ability_manager` unguarded, so a Direct run needs an edit to `_run_one_task`, which must stay untouched + (it mirrors the runner). The `runner_direct` golden is built by the builder's own mirror of `run_one_task` + (same constructor, guarded removal, call), so the Direct path through the runner is pinned there instead. + +### Behaviour changes to know + +- `service.run_task`, `run_task_once` require `worker=`; `service.ensure_started` and the facade's + `_direct_agent` / `_run_direct` / `_direct_answer` are gone; `_direct_answer` is `main_agent_answer`. +- `agent_parts` requires `language=`; the runtime tools take no `language`; `BrowserRuntimeRail` registers no rail. +- A `create_browser_agent` agent has `ActionPolicy`, `RepeatedCallGuardRail` and `InterceptedActionHintRail` in its + rails before the first invoke; the policy's mapping is the facade's live `builders`. +- After `shutdown()` the next `ensure_started()` picks the topology again (an env change between tasks of one + process is not read: `service.core` is cached). +- `WORKER_LANGUAGE` is importable from `runtime.topology` and `runtime.agents`. + +### The other topology (D8): what differs + +| | Playwright MCP + `Direct` (through the runner) | browser-use + `DelegatedWorker` (through the runner) | +|---|---|---| +| Agent | the main agent (`create_browser_agent`), as the MCP capability goldens are built | the worker, as on MCP | +| Tool names | `mcp_playwright-official_browser_*` plus the helpers `browser_probe_interactives`, `browser_probe_cards`, `browser_batch_interact`, `browser_find`, `browser_recall_offload`; the runner removes nothing (no worker slot), so the agent keeps them | unprefixed `browser_*`; the three worker helpers are removed by the runner; `browser_custom_action` / `browser_list_custom_actions` raise in `remove_ability` and are swallowed, as on MCP | +| Prompt | the main-agent prompt (`DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT["en"]`) plus the rail's sections; no JSON contract, no `FinalAnswerRecoveryRail`; the rail's terminal synthesis | the worker prompt saying "browser-use tools" / "a browser-use action"; the tool names it cites are the worker's actual, unprefixed names | +| Answer | `main_agent_answer`: `page` from `page_state.surface_context`, `screenshot=None`, `status` / `progress` from the rail's result; no `attempt`, `failure_summary`, `progress_state`; one attempt under one `asyncio.wait_for` | `service.run_task`'s dict, identical keys to MCP; `page` from the worker's JSON (browser-use observation text has no title, so `title` may be empty) | +| Image support | from the model probe, so `browser_take_screenshot` may be offered where the worker never gets it | `False`, as on MCP | +| Logging | `dedicated_log=False`, usage lines in the process log | as on MCP | +| Progress | the rail's `` protocol | the worker's JSON progress fields | +| Loop heuristics | none (one attempt) | the transport markers are Playwright wording; browser-use's "CDP not connected" / "No valid agent focus" are handled by `DriverSession` as `ConnectionLost` before the loop sees them | +| Cosmetic | — | the worker card `id="agent.playwright.browser_worker"`, not model-facing | + +### Golden status + +- `golden/playwright_mcp/`: the 12 existing files byte for byte (regenerated and compared: "unchanged" for all + twelve; the seven stored with CRLF in the working tree were restored to their bytes); `runner_direct.json` new. +- `golden/browser_use/`: the 12 recorded files untouched; `runner_worker.json` new (no `source_commit`: built by + today's code). +- `git diff d062c3173 --stat -- snapshots/golden/` shows only the two new files. + +### Verification + +- `test_topology.py` (new): `topology_for` for both drivers with the field unset and set both ways, the instance's + driver override, a registered default no strategy implements; `Direct`: `start` once through + `create_browser_agent` with `WORKER_LANGUAGE`, `dedicated_log=False`, the service's model settings, `run` maps the + answer, the agent gets the core rails at construction over the facade's live mapping (and a policy built before + the session enforces the tools built after it), the log context, a session that cannot start fails before any + model call (two `assemble_driver` awaits: the first-open retry), failed and timed-out runs, `shutdown` drops the + agent; `DelegatedWorker`: `start` opens the session, builds the worker once with the service's model identity, + `max_steps`, `core`, the observer and the driver name, registers the two runtime tools, rebuilds after a task + release, `run` hands the task to `service.run_task` with a worker callable that rebuilds after `reset()`, a + browser-use-shaped fake yields a worker whose prompt says "browser-use tools" with unprefixed tools, `shutdown` + leaves no worker. The facade's pick tests of the previous step and the five Direct tests moved here. +- `test_browser_runtime_facade.py`: `surface` built once before any driver, `core_rails()` over the live mapping and + the profile, `agent_parts` keyed by image support and language with no rails, `register_runtime_tools()` once. +- `test_browser_runtime_rail.py`: no `register_rail` from the rail; the parts in the agent's language; a start + failure adds no tool. +- `test_service.py`: the `run_task(worker=)` tests (the callable awaited per attempt, `_restart` is `reset()` + alone, no `ensure_started`); the worker-build tests moved to `test_topology.py`. +- `test_create_browser_agent.py`: the runtime's core rails are in the agent's rails at construction, after the + browser rail. +- `test_wv_runner_contract.py`: green with no edit (DelegatedWorker is the MCP default). +- `test_tool_prompt_snapshot.py`: `runner_direct` built and compared with the other MCP views; the browser-use + `runner_worker` built and compared; `test_golden_files_cover_exactly_the_views` checks both directories; the + well-formedness test runs over the 12 recorded browser-use goldens. +- Ten harness files outside `browser_move` that build the browser agent (`test_deep_agent.py`, + `test_deep_agent_terminal_stream.py`, `subagent_runtime/test_control.py`, `subagent_runtime/test_session_manager.py`, + `task_loop/test_session_spawn_executor.py`, `test_subagent_rail.py`, `tools/subagent/test_subagent_tools.py`, + `tools/test_sessions_tools.py`, `tools/test_task_tool.py`, `tools/test_task_tool_kv_cache_affinity.py`, + `tools/test_tool_render_for_llm.py`): **248 passed**. +- Architecture: green without an allowlist change; `topology.py` carries no browser word and no driver name; + `runtime.py` / `service.py` counts did not grow. +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2365 passed, 118 skipped** (2349 / 118 after + the previous step). +- `ruff check` clean on every touched file (pre-existing import-order and `E402` hits in `browser_agent.py` and + `main.py` left as they are). +- Live suites: run once at the end of the phase's last step, recorded in "Phase 6: verification baseline". + +### Known leftovers + +1. The worker loop (`run_task`, `_run_task_once`, `_recover_final_answer`, the progress state, the heuristics) stays + in `service.py` until Phase 9 (`git mv` to `runtime/worker.py`). +2. The worker card description ("using Playwright MCP tools") and the `browser_custom_action` tool description + still name the driver; neither is model-facing on the worker. Phase 7. +3. `runtime.py` keeps `main_agent_answer` because of the browser-word ratchet (decision 7); Phase 9 decides where + the runner's answer contract lives when the loop moves. +4. The transport markers of the worker loop are Playwright wording (D8 table); Phase 7. + +## Phase 6: verification baseline + +The phase's last step: the live proof that each driver finishes one task through the facade in both topologies, the +live suites run once over the whole phase, and the numbers the phase is judged against. The WebVoyager smokes (one +task per driver in each topology, with a real model) are the head's, outside this session. + +### The live module (`test_topology_live.py`) + +Gated as the other live suites (`BROWSER_MOVE_LIVE_MCP=1` for Playwright MCP, `BROWSER_USE_SIDECAR_PYTHON` for +browser-use), the conformance fixture's headless Chrome attached with `BROWSER_PROVISIONING=remote`, the scene +server, and the `ScriptedModel` of `test_agent_rails_live.py`: one `browser_navigate` to the scene, then the answer. +Per driver, through `BrowserAgentRuntime(core_settings=)`: + +- `delegated_worker`: `ensure_started()` puts the worker in `service.browser_agent`; the scripted model is set on it + and the five runner helpers removed as `run_one_task` does; `run_browser_task` returns the runner's shape + (`ok`, `final`, `error`, `screenshot`, `page`, `session_id`, `request_id`) with `ok=True` and the scripted + `final`; the first request's tools are exactly the prefixed (MCP) or plain (browser-use) core tools; the + navigate's tool message is not an error. +- `direct`: `ensure_started()` builds the main agent and leaves the worker slot empty; the answer is `ok=True` + with the scripted text in `final` and the scene's origin in `page.url` (from the page state, on both drivers); + the first request's tools hold the core tools and the four main-agent helpers. + +Two things the module had to learn: + +1. **The main agent's model is a copy.** `create_browser_agent` copies the given model into a plain `Model` for the + browser temperature, so a `Model` subclass with a scripted `invoke` does not survive construction (the worker, + a `ReActAgent`, takes `set_llm`). The script lives in a `BaseModelClient` installed through + `create_model_client`, the snapshot builder's pattern, defined once per module (a second class with the same + `__client_name__` only warns, but the first live run showed the registry warning while hunting the next item). +2. **The processor registry is snapshotted per test.** `tests/conftest.py` and `tests/unit_tests/harness/conftest.py` + copy `ContextEngine._PROCESSOR_MAP` when a test starts and restore it when it ends. `Direct.start` imports + `browser_agent` lazily, which registers `BrowserStateContextProcessor` and `BrowserWorkingContextProcessor` at + import: in the first live run that happened inside the first Direct test, so its teardown dropped both and the + second Direct agent failed with "cannot find processor type 'BrowserStateContextProcessor'". The module imports + `browser_agent` at collection time. Any test that builds two main agents in one process after a lazy first + import has the same exposure; `main.py` and the runner import eagerly and are not affected. + +### Phase 6 done-when, with the evidence + +| Item | Evidence | +|---|---| +| `BROWSER_TOPOLOGY` with a per-driver default | step 1 of the phase ("Phase 6: topology setting"); `test_core_settings.py`, `contract/test_registry.py` | +| one construction path: `service.py` and `runtime.py` build no agent | `assembly.py` opens every session (`open_session`) and gives every agent its tools and rails; `create_browser_agent` and `build_worker` are the two builders; `test_topology.py`, `test_build_worker.py`, `test_create_browser_agent.py` | +| the core rails at the main agent's construction | `create_browser_agent` queues `runtime.core_rails()`; `test_create_browser_agent.py`, `test_topology.py` ("gets the core rails at construction"), `test_browser_runtime_rail.py` (no `register_rail`) | +| the MCP `runner` golden byte for byte; every existing golden unchanged | `regenerate.py`: "unchanged" for the twelve MCP views; `git diff d062c3173 --stat -- snapshots/golden/` shows only `playwright_mcp/runner_direct.json` and `browser_use/runner_worker.json` | +| each driver completes one task in the other topology | `test_topology_live.py`: 4 passed (MCP in `direct`, browser-use in `delegated_worker`, and each in its default) | +| both live sets green | below | + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2365 passed, 122 skipped** (the four new live + cases skip without the gates; 2321 / 118 at `d062c3173`). +- Live, both gates set, one run over `drivers/playwright_mcp/test_conformance_playwright_mcp_live.py`, + `drivers/test_conformance_browser_use_live.py`, `surfaces/test_browser_helpers_live.py`, `test_assembly_live.py`, + `test_agent_rails_live.py`, `test_topology_live.py`: **116 passed, 3 failed in 361.77s** (6:01), the three being + the new module's cases before the two fixes above (a `status` key the worker answer does not carry, and item 2); + then `test_topology_live.py` alone, both gates set: **4 passed in 17.42s**. The 115 cases of the existing suites + passed in the one run. +- Ten harness files outside `browser_move`: 248 passed (previous step). +- `ruff check` clean on the new module. + +### Known leftovers + +1. The head's four WebVoyager smokes (each driver in each topology, a real model, cost recorded) are the phase's + behavioural baseline; this session ran none. +2. The per-test processor-registry snapshot in the conftests drops processors registered by a lazy import during + a test (item 2 above). Worth a note in the conftest or an eager import where the browser agent is built lazily + (`Direct.start`); left as is. +3. `test_wv_runner_contract.py` pins the MCP default only; the Direct path through the runner is pinned by the + `runner_direct` golden and the live module, not by the contract test's mirror (previous section). +4. The `runner_worker` golden carries no `source_commit`; `surfaces/test_browser_tools.py` compares recorded goldens + only. When the browser-use goldens are re-recorded (Phase 7), the two conventions can merge. + +## Phase 7: legacy removed + +The last old paths die before the renames (Phase 8) and the split (Phase 9): the MCP client patch and everything +that only existed for the time the agents reached the browser through core's `ToolMgr`. Nothing the model reads +changed; every golden is byte for byte what it was at the Phase 6 close. + +### What went + +- **The MCP client patch** (`browser_tools.py`: `_ensure_openjiuwen_client_patch`, + `ensure_browser_runtime_client_patch`, `_client_registry`, `get_registered_client`) and its two module-level + copies (`main.py`, `playwright_runtime_mcp_server.py`). Its consumers were gone since the switch (P5-T1's + inventory, S1–S3); what remained was the process-wide side effect: after one patch call every stdio or + streamable-http MCP server any agent registered in the process got a `BrowserMove*Client`. Those servers get + core's clients again. +- **`register_browser_runtime_mcp_server`** and the local-server launcher it fell back to (`_start_local_server`, + `stop_local_browser_runtime_server`, `restart_local_browser_runtime_server`, port probing, the stdout/stderr log + handles, the http fallback and retry configs). The three public names leave `browser_move.__all__` and + `runtime.__all__`; `build_browser_runtime_mcp_config` stays as the client-side config of the runtime + server, and the system test registers that config through the core calls the wrapper made + (`Runner.resource_mgr.add_mcp_server`, `ability_manager.add`). +- **`_build_dialogue_compressor_config`** (`agents.py`), without a caller since Phase 4, and its three imports. +- **`BrowserCancelTool`, `BrowserClearCancelTool`, `BrowserRuntimeHealthTool`** (`runtime_tools.py`), with the + card that still said "heartbeat status": `register_runtime_tools` adds only the two custom-action tools, so the + three had no caller outside their tests; no golden carried any of the three cards. The facade's `cancel_run`, + `clear_cancel` and `runtime_health` stay for the server entry point. +- **The Playwright MCP launch defaults** (`DEFAULT_PLAYWRIGHT_MCP_COMMAND`, `DEFAULT_PLAYWRIGHT_MCP_ARGS`) moved + from `runtime/utils/env.py` into `surfaces/browser/config.py`, their only reader, and left the `utils` exports. +- **Driver names in wording**: the probe builders' docstrings ("the JavaScript", not the tool that once ran it), + the upload-path and action-controller docstrings and the package comment ("the server entry point"), the demo + entry's log lines ("Server command"), `browser_tools.py`'s own description. +- **The worker card** renders the session driver's display name (`"… using {driver_name} tools."`, the name + `build_worker` already passes for the prompt), so the text is unchanged on Playwright MCP and right on + browser-use; the custom-action card says "the direct browser tools". +- **`BROWSER_DRIVER`** is only a registry name; the legacy values keep today's mapping (below). + +### Decisions + +1. **The registration wrapper is deleted, not kept as a shim.** Plan Phase 7 item 1 names the patch; the task + names the wrapper with it (P5-T1 open question 1). Without the patch the wrapper was the core's two calls plus a + launcher for a server a host can start itself. +2. **The three unregistered runtime tools are deleted rather than reworded.** The task asked to check the + "heartbeat status" card against the goldens; the card was in none, and the tool had no caller. +3. **The driver-name pattern wants `browser_use` to end there** (`browser_use(?![a-z])`): the three entries that + counted `latest_browser_user_request` (`runtime.py`, `browser_working_context.py`, + `browser_working_context_rail.py`) go without touching the helper. +4. **The launch defaults move to the surface's config module**, where the driver-name rule allows a driver name. +5. **The legacy `BROWSER_DRIVER` values stay as they were, confirmed and measured.** `_provisioning` + (`surfaces/browser/config.py`) warns on every parse of a legacy value with `stacklevel=3`; Python's warning + registry shows a given message once per call site under the `default` filter (three parses from one site: + `managed`, `managed`, `remote` → two warnings), every time under `always` (what `pytest.warns` uses), and the + default filter hides `DeprecationWarning` outside `__main__`. Tested by `surfaces/test_browser_settings.py` (one + warning per parse for each of the three values, an explicit `BROWSER_PROVISIONING` wins) and + `test_settings_facades.py` (exactly one through `build_playwright_mcp_config`). There is no explicit + once-per-process guard; the plan asks for "a warning" and none was added. + +### Rejected options + +- **Renaming `latest_browser_user_request`** to satisfy the pattern: a production rename for a detector's false + positive, and Phase 8 is the rename PR. +- **Reading the runtime server's cwd from the given `environ`** instead of `resolve_playwright_mcp_cwd()` to + empty `browser_tools.py`'s entry: it would change which environment the cwd comes from when a caller passes a + mapping; the facade's name goes with Phase 8. +- **Keeping `stop_local_browser_runtime_server` / `restart_local_browser_runtime_server`** as exports: both acted + only on a process the deleted wrapper had started. +- **An explicit once-per-process guard** for the legacy-value warning: not in the plan; recorded as a question. + +### What stays on the allowlists, and why + +| File | Driver names | MCP names | Why it stays | Goes with | +|---|---|---|---|---| +| `runtime/runtime.py` | 0 (was 2) | 9 | `browser_take_screenshot` ×4 in the image-capability guidance (model-facing prompt bytes, en/cn); `browser_resume` ×2 (a public run-context key, not the tool); `browser_click` → `browser_navigate` ×2 (the primary-link rewrite needs the dialect's navigation tool by name); `"source": "browser_evaluate"` ×1 (evidence provenance the model reads) | Phase 9: the browser rail and its prompt text move to `surfaces/browser/` | +| `runtime/agents.py` | 0 (was 1) | 11 | the worker prompt names the tools it governs; pinned byte for byte by the `runner` golden | Phase 9: prompt sections move to `surfaces/browser/` | +| `runtime/browser_capabilities.py` | 2 | 69 | the capability catalog (tool names per capability; descriptions the main agent chooses from) | Phase 9: `surfaces/browser/capabilities.py` | +| `runtime/browser_tools.py` | 3 (was 23) | 0 | `resolve_playwright_mcp_cwd` ×2 (the config facade's name) and `"/mcp"` (the runtime server's own streamable-http path); the module is about the runtime served over MCP, not the driver | Phase 8 (the facade's name) | +| `runtime/service.py` | 1 | 0 | `PLAYWRIGHT_MCP_CDP_ENDPOINT` read from the given server config's env map, part of the lifecycle identity (S_20 names it) | Phase 9: the service split | +| `surfaces/browser/helpers/batch_steps.py` | 1 | — | card text pinned by the golden tool list | stays | +| import `runtime.py` → `surfaces.browser.helpers` | — | 4 | the facade's decision-policy hooks, page metadata and custom actions | Phase 9 | + +Counts, before → after this phase: driver names 15 entries / 52 hits → 4 entries / 7 hits; MCP tool names +5 entries / 92 hits → 3 entries / 89 hits; core imports of concrete modules 5 entries / 8 → 1 entry / 4; browser +words: `browser_tools.py` gone (10 → 0), nothing grew. Against the Phase 0 measurements (`runtime.py` 110 MCP +tool-name literals, `runtime/controllers/action.py` 16, `agents.py` 9): `runtime/controllers/action.py` 0, `runtime.py` 9 (the +four items above), `agents.py` 11 (the prompt), `browser_capabilities.py` 69 (the catalog); zero elsewhere outside +`drivers/playwright_mcp/`. + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2362 passed, 122 skipped** (2365 / 122 at the + Phase 6 close: four tests deleted with their code, one added for the worker card). +- The eleven harness files outside `browser_move` that build the browser agent: **248 passed**. +- Both live sets, one run, `BROWSER_MOVE_LIVE_MCP=1` and `BROWSER_USE_SIDECAR_PYTHON` set (MCP conformance, + browser-use conformance, helpers, assembly, rails, topology): **119 passed in 365.67s (0:06:05)** (51 MCP conformance, 45 browser-use conformance, helpers, assembly, rails, the four topology cases) +- Goldens: `git diff cb3e9065d --stat -- tests/unit_tests/harness/tools/browser_move/snapshots/golden/` is empty. +- `ruff check` clean on every touched line; unchanged pre-existing findings: `E402`/`I001` in + `playwright_runtime_mcp_server.py` and `main.py`, `ASYNC250` in `main.py`, 17 in `probes.py`, 2 in + `runtime/controllers/action.py`, `I001` in the system test. +- `python -m …playwright_runtime_mcp_server --help` still fails on the script-style import + (`runtime.config` → `from ..contract import SurfaceSettings`), as since P5-T1; not touched. + +### Known leftovers + +1. The allowlist table above: seven driver-name hits in four files, 89 MCP-name hits in three, one import entry. +2. No explicit once-per-process guard on the legacy `BROWSER_DRIVER` warning (decision 5). +3. Noticed, not touched (outside the plan's list of old MCP helpers): `BrowserService.build_progress_context` has + no caller; `record_tool_progress`, `record_worker_progress`, `get_progress_state` and `export_progress_state` + are called by tests only; `DEFAULT_BROWSER_UPLOAD_ROOT` (`runtime/utils/env.py`), `browser_agent_log_error`, + `install_browser_subagent_status_logging_async`, `BrowserPageState.replace_ax_snapshot` and four + `BrowserProfileStore` methods have no production caller. +4. The browser-use goldens were not re-recorded (Phase 6 verification leftover 4 named Phase 7; the task did not). +5. The server entry point's script-style imports (above). + +## Phase 8: renames + +`playwright_runtime/` is `runtime/`, with `controllers/` and `utils/` nested under it; `playwright_runtime_mcp_server.py` +keeps its name as the entry point. One `git mv` commit carries every import with it, so each file's history follows +(`git log --follow`); the old package name stays for one release as an alias package. No golden changed. + +### What moved + +- `playwright_runtime/*` → `runtime/*` (30 files), `controllers/*` → `runtime/controllers/*` (3), `utils/*` → + `runtime/utils/*` (4), `tests/system_tests/…/test_playwright_runtime.py` → `test_runtime.py`: 38 renames, 30 at + 100 % similarity, the rest differ by their imports only. +- Imports. Inside the package, the relative imports that crossed the old folder boundaries (`..controllers` and + `..utils` become `.controllers` and `.utils` in `config.py`, `runtime.py`, `service.py`; `..playwright_runtime.profiles` + becomes `..runtime.profiles` in `provisioning/managed_browser.py`; `...utils.env` and `...playwright_runtime.config` + become `...runtime.utils.env` and `...runtime.config` in the surface's `config.py` and `profile.py`; + `....utils.parsing` becomes `....runtime.utils.parsing` in the stdio client) and the absolute ones in the drivers, the + helpers, `status_logging.py` and the two `__init__.py`. The four harness callers (`rails/subagent/subagent_rail.py`, + `schema/decision_policy.py`, `subagents/browser_agent.py`, `tools/subagent/task_tool.py`). The unit tests, + `test_deep_agent_terminal_stream.py` and the two system tests. The server's script-style imports read + `runtime.config`, `runtime.controllers.action` and `runtime.runtime`, and the bare-name `sys.modules` aliases they + rely on are renamed with them (`runtime`, `runtime.controllers`, `runtime.controllers.action`). +- Architecture tests: the allowlist keys follow the files, counts unchanged; `_scan.py` knows one core folder + (`runtime`), one config-module pattern and one assembly module; the env-read test's env module is + `runtime/utils/env.py`. +- The 25 import blocks the rename left unsorted were re-sorted in the move commit; no other line changed. + +### The alias package + +`playwright_runtime/__init__.py` is the only file left under the old name. On import it walks `runtime/` with +`pkgutil.walk_packages`, imports every module (36: the top-level ones, `controllers.*`, `utils.*`) and registers each +under its old dotted name in `sys.modules`; it re-exports `runtime.__all__` through a module `__getattr__`; and it +warns once per process with a `DeprecationWarning` (`stacklevel=2`, so the importer is named). The flag is read +through `globals()`, so `importlib.reload` warns no second time. The runner-contract mirror +(`snapshots/test_wv_runner_contract.py`) keeps the old import path on purpose: it stands in for the WebVoyager +runner, the alias's real consumer, and is green unchanged. + +### Decisions + +1. **Eager aliasing, not a lazy proxy.** `import a.b.c` and `from a.b import c` resolve through `sys.modules`, and an + entry has to be the module object itself for `playwright_runtime.runtime.BrowserAgentRuntime is + runtime.runtime.BrowserAgentRuntime` to hold; a module can only be aliased once it exists. Importing all of + `runtime/` costs what the old consumers paid anyway (`runtime.py` pulls most of the package), and no module under + `runtime/` has import-time side effects (`main.py` only defines). +2. **An explicit once-per-process flag**, unlike the legacy `BROWSER_DRIVER` warning (Phase 7 decision 5): the plan + asks for a `DeprecationWarning` once. The flag lives in the alias's own namespace and survives `importlib.reload`; + evicting the module from `sys.modules` and importing it again counts as a fresh process (the test does exactly that + to see the first warning). +3. **The alias is outside the core scan.** `_scan.py` names `runtime` as the only core folder, so the alias is seen by + the env-read rule only (it has no reads); it spells neither browser words nor driver names. +4. **The bare-name aliases are renamed, not dropped.** `sys.modules["runtime"]`, `["runtime.controllers"]` and + `["runtime.controllers.action"]` replace the old three. They exist for the server's script-style imports; the + rename carries them, it does not change them. +5. **Removal.** The alias package goes one release after this one; its docstring says so. With it go + `test_playwright_runtime_shim.py` and the old path in the runner-contract mirror, once the WebVoyager runner imports + `runtime`. + +### Rejected options + +- **A `__path__` alias** (`__path__ = runtime.__path__`): the import system would load a second copy of each module + under the old name, not the same object. +- **A `sys.meta_path` finder** handing back the existing module from `create_module`: `_init_module_attrs` rewrites + `__spec__` on the module it is given, so the real module would carry the alias's spec. +- **Lazy module proxies in `sys.modules`**: not identity on the first access, and `importlib.import_module` returns + the proxy. +- **Fixing the server's script-style import failure** (`from ..contract import` under a top-level package, failing + since P5-T1): not in the task; the imports are renamed as they were. + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2374 passed, 122 skipped** (2362 / 122 at the + Phase 7 close plus the twelve alias tests). +- The eleven harness files outside `browser_move` that build the browser agent, `test_deep_agent_terminal_stream.py` + among them: **248 passed** (as at the Phase 7 close). +- Both live sets, one run, `BROWSER_MOVE_LIVE_MCP=1` and `BROWSER_USE_SIDECAR_PYTHON` set (MCP conformance, + browser-use conformance, helpers, assembly, rails, topology): **119 passed in 355.75s (0:05:55)** (51 MCP conformance, 45 browser-use conformance, helpers, assembly, rails, the four topology cases; as at the Phase 7 close). +- `uv run python -W always -c "import openjiuwen.harness.tools.browser_move.playwright_runtime.runtime"` prints one + `DeprecationWarning` from the alias. (A second one, `probes.py:265: invalid escape sequence`, is the pre-existing + compile-time warning, shown only when no bytecode is cached.) +- Goldens: `git diff 322a4f4d6 --stat -- tests/unit_tests/harness/tools/browser_move/snapshots/golden/` is empty. +- `git log --follow` on `runtime/runtime.py` (41 commits) and `runtime/utils/env.py` lists the history across the + move; the alias package is the only file under `playwright_runtime/`. +- `ruff check --select I` on the moved files: seven pre-existing `I001` before, six after (the server's import block + happens to be sorted under the new names; `controllers/action.py`, `browser_working_context_processor.py`, + `browser_working_context_rail.py`, `semantic_state.py`, `site_profiles.py`, `status_logging.py` stay). The alias + and its test are clean. The other pre-existing findings in touched files are unchanged (`E402` in the server, 12 + and 9 in the two MCP clients, 17 in `probes.py`, the test files). + +### Known leftovers + +1. `python -m …playwright_runtime_mcp_server` still fails on its script-style import, now `runtime.config` → + `from ..contract import SurfaceSettings`, as since P5-T1. +2. The server's `browser_custom_action` description still reads "Register your own via + controllers.action.register_action" (model-facing text of the runtime server, in no golden; not an import). +3. The server docstring's usage lines (`uv run python src/playwright_runtime_mcp_server.py`) predate the package + layout. +4. `test_runtime.py` keeps its function name `test_playwright_runtime_end_to_end`. +5. `sys.modules["runtime"]` is a more generic top-level name than `playwright_runtime` was: `setdefault` never + displaces a real top-level `runtime` module, but a later bare `import runtime` in the same process gets this + package. +6. The alias package and its test: remove in the next release (decision 5). + +## Phase 9: PR a, the standalone browser modules leave the core + +Everything under `runtime/` that reads pages, tabs or URLs on its own (the probes, the semantic state, the site +profiles, the working context, the state context processor, the status log, the capability catalog) moves to +`surfaces/browser/`; the record rails move to `runtime/rails/`; the browser profile store moves to `provisioning/`. +Every move is a `git mv` with its import fixes in the same commit, proved by goldens and by AST identity. Two things +are not moves and are recorded as such: the catalog merge, and the `--caps` list built at call time. + +### What moved + +| from `runtime/` | to | +|---|---| +| `probes.py` | `surfaces/browser/probes/cards.py` (`probes/__init__.py` re-exports the three builders) | +| `probe_semantics.py` | `surfaces/browser/probes/semantics.py` | +| `semantic_state.py` | `surfaces/browser/semantic/state.py` | +| `site_profiles.py`, `status_logging.py` | `surfaces/browser/` (same names) | +| `browser_working_context.py`, `…_processor.py`, `…_rail.py` | `surfaces/browser/working_context/store.py`, `working_context/processor.py`, `rails/working_context.py` | +| `browser_state_context_processor.py` | `surfaces/browser/state_processor.py` | +| `browser_capabilities.py` | `surfaces/browser/capabilities.py` (merge, below) | +| `profiles.py` | `provisioning/profiles.py` | +| `action_policy.py`, `intercepted_action.py`, `repeated_call_guard.py`, `final_answer_recovery.py` | `runtime/rails/` | + +15 modules: 14 renames (`git show -M --name-status`: six at 100 % similarity, the rest differ by their imports only) +and the merge. Import fixes: `runtime/{runtime,service,config,assembly,agents,__init__}.py`, +`surfaces/browser/helpers/{page,probes}.py`, `provisioning/managed_browser.py`, the moved files' own relative imports, +`subagents/browser_agent.py`, `rails/subagent/subagent_rail.py`, 37 test files, the two API docs that named +`runtime.browser_capabilities`. `runtime.__all__` loses the five browser names (the two processors with their +settings classes, the working-context rail); the alias package serves them (below). + +**The catalog merge.** `surfaces/browser/capabilities.py` was a 53-line surface module (`_SPECS`, `_REF_KINDS`, +`CAPABILITY_TOOLS`, `_pairs`, `CAPABILITY_CATALOG`); `runtime/browser_capabilities.py` was the catalog. The catalog +file moves onto the surface module's path and the five surface definitions are appended verbatim before `__all__`, +which now also lists `CAPABILITY_TOOLS` and `CAPABILITY_CATALOG` (`profile.py` imports them by name). Git shows it +as a rename only under `-M -B` (a modified path is never a rename candidate for `-M` alone); `git log --follow` on +the new path follows the surface module's history, the catalog's is behind `--follow -M -B` or the old path. + +### The Phase 8 leftovers + +The runtime server imports the package by its full name (`from openjiuwen.harness.tools.browser_move.runtime…`); +the `sys.path` block, the `SRC_ROOT` append in `runtime/__init__.py` and the three bare-name `sys.modules` aliases +(`runtime`, `runtime.controllers`, `runtime.controllers.action`) are gone, and so is the scanner's bare-name branch +that existed for them. `python -m …playwright_runtime_mcp_server --help` runs (Phase 8 leftover 1 closed, with 2, 3 +and 4: the `browser_custom_action` description names `runtime.controllers.action.register_action`, the server +docstring's usage line names the module, `test_runtime.py`'s function is `test_runtime_end_to_end`). + +### The alias package + +`_MOVED = {"browser_capabilities": "surfaces.browser.capabilities", "profiles": "provisioning.profiles"}` is aliased +eagerly after the walk, so `playwright_runtime.browser_capabilities` (the WebVoyager runner's import) is the +catalog module object. A second, name-keyed map `_MOVED_NAMES` serves the five names `runtime.__all__` dropped +through the alias's `__getattr__`. The runner-contract mirror is green unchanged. + +### The `--caps` list, built at call time + +`config.py` used to build `PLAYWRIGHT_MCP_CAPABILITY_NAMES` at import from the catalog. With the catalog under +`surfaces/browser/`, that import broke the composition-root invariant of S_20 ("importing the package … imports no +concrete driver, surface or provisioner"), checked at runtime by `test_assembly.py`: `openjiuwen.harness.tools` +reaches `config.py` through `subagent/task_tool.py` and `browser_logging.py`. Decision (head, checkpoint 1 review): +pull the PR e rewrite of `config.py` forward. `playwright_mcp_config` imports the catalog inside the function and +builds the names there (same filter, same order: every catalog entry except `core` and the policy-only ones); the +module constant goes (its one caller was that function). The same probe then showed a second edge the checkpoint +had not seen: `rails/subagent/subagent_rail.py`, reached from `openjiuwen.harness.tools` through `worktree/rails.py` +and `rails/__init__.py`, imported `DEFAULT_BROWSER_CAPABILITIES` at module level for the one method that lists the +browser agent's capabilities in the available-agents description. It imports the catalog inside that method, by +the same decision; no test patches the name on the module. The rest of PR e (the profile fields) still follows PR d. + +### Browser words: import statements no longer count + +A `from ..surfaces.browser.x import` line carries the word `browser` in the package path, and the three core +modules that import surface modules (`runtime.py`, `service.py`, `config.py`) grew by one hit per import line after +the move. Decision (head): that edge is the import-boundary rule's business; the browser-word rule blanks the lines +of every import node (`ast`, `lineno..end_lineno`) before counting. The three bumps revert and the recount tightens +two stale values (`controllers/action.py` 129 → 13, `runtime_tools.py` 33 → 5, both below their allowlist value +since earlier phases; the ratchet only fails on growth) and `browser_logging.py` (8 → 6, an import line). + +### Decisions + +1. **The move is proved, not reviewed.** Goldens byte for byte, and an AST identity check: every top-level + definition and every method of the old files has exactly one new node with an equal `ast.dump` (positions are + not in the dump) in the new tree; the merge and `runtime/__init__.py` (expected changes: `__all__`, + `__getattr__`) are checked the same way. +2. **The registration mechanism of the two context processors is unchanged.** The `@ContextEngine.register_processor()` + decorators stay on the classes in `state_processor.py` and `working_context/processor.py`; the registration point + is the import of those modules, which `browser_agent.py` performs. `surfaces/browser/__init__.py` stays + side-effect free: importing the processors there would pull `openjiuwen.core.context_engine` and the + working-context store into every `SURFACES.create("browser")`, `browser_surface_settings()` included. +3. **`status_logging.py` goes to the surface** although the design's inventory listed it under the core: only the + rail imports it, and it reads `url`/`page` result fields. +4. **`profiles.py` goes to `provisioning/`**, not to the surface (head decision Q6): the store serves the + provisioner, and `surfaces/` may not import `provisioning/`. +5. **The call-time catalog import is counted by the import-boundary scan.** Lazy imports count everywhere except in + `assembly.py` (S_20, the architecture tests), so the key `config.py → surfaces.browser.capabilities` stays at 3; + the allowlist neither grew nor shrank for it. Whether `config.py` joins the lazy-allowed set is a PR d question. +6. **A new rule**: `surfaces/` never imports a concrete driver or a provisioner (`drivers/middleware.py`, `contract/` + and `runtime/` are allowed). Zero violations today. + +### Rejected options + +- **Loosening the composition-root probe** to accept `surfaces.browser` and the catalog until PR e: the probe is + the runtime check of an S_20 invariant; the call-time import keeps it whole at the cost of one body edit the + design had scheduled anyway. +- **Leaving the catalog under `runtime/`**: keeps 69 MCP tool names and 12 browser words in the core and + contradicts the plan's tree. +- **A forwarding module `runtime/browser_capabilities.py`**: a `runtime → surfaces` import; the alias map covers + the one consumer outside the repo. +- **Counting import lines and bumping the three allowlist values**: an allowlist may not grow for arithmetic the + design itself lists; the edge is already a counted key of the import-boundary rule. + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move tests/unit_tests/harness/test_deep_agent_terminal_stream.py -q`: + **2398 passed, 122 skipped in 48.54s** (2393 / 122 at the Phase 8 close, plus four tests for the server entry, + the alias map and the surfaces rule, plus the `playwright_mcp_config` names test). +- AST identity (a scratch script, not committed; the old files at the Phase 8 close against the new tree): the 14 + renamed modules **382 old definitions accounted for, 0 problems**; the merge **35 / 35**; `runtime/__init__.py` 7 + accounted for with the two expected changes. +- Goldens: `git diff a632b30f9 --stat -- tests/unit_tests/harness/tools/browser_move/snapshots/golden/` is empty. +- `uv run python -m openjiuwen.harness.tools.browser_move.playwright_runtime_mcp_server --help`: exit 0, the + argparse help. +- `uv run python -W always -c "import …playwright_runtime.browser_capabilities, …playwright_runtime.runtime"`: one + `DeprecationWarning` from the alias; both resolve; `runtime` and `runtime.controllers` are no longer in + `sys.modules`. +- `ruff check --select I` over the package, the harness callers and the tests: 18 `I001` at the Phase 8 close → 10 + (the blocks the move touched were re-sorted; the ten left are pre-existing in files PR a did not edit). The + touched files are clean; the server's three `E402` are gone with the `sys.path` block. +- Allowlists, Phase 8 close → PR a: + +| list | before | after | +|---|---|---| +| browser words | 24 keys / 569 | 13 keys / 280: eleven keys gone with their files (−169); `final_answer_recovery` 2 and `repeated_call_guard` 8 under `runtime/rails/`; `runtime.py` 96 → 95, `service.py` 82, `config.py` 9, `browser_logging.py` 8 → 6, `controllers/action.py` 129 → 13, `runtime_tools.py` 33 → 5 (the recount without import lines) | +| MCP tool names | 3 keys / 89 | 2 keys / 20 (`browser_capabilities.py` 69 gone) | +| driver names | 4 keys / 7 | 4 keys / 7 (the catalog key follows the file) | +| core imports of concrete modules | 1 key / 4 | 8 keys / 19: `runtime.py → helpers` 4 (unchanged), `→ capabilities` 2, `→ semantic.state` 2, `→ site_profiles` 1, `→ status_logging` 2, `→ working_context.store` 3 (the last three are the rail's and leave with it in PR b); `service.py → capabilities` 2; `config.py → capabilities` 3 (call time, decision 5) | +| module size | 3 keys | 3 keys (`runtime/probes.py` 3010 → `surfaces/browser/probes/cards.py` 3010) | +| env reads | unchanged | unchanged | + +### Known leftovers + +1. `config.py → surfaces.browser.capabilities` stays a counted key (decision 5) until PR d decides whether a + call-time import in a config module is allowed, or PR e hands the catalog to the core through the profile. +2. The import allowlist has eight keys where the design predicted four: the three `runtime.py` keys for the rail's + imports and `semantic.state` at 2 (the rail still imports `price_interval_signature`) leave with the rail in PR b. +3. The pre-existing compile-time warning `invalid escape sequence '\s'` now names `surfaces/browser/probes/cards.py:265`. +4. The alias package, its test and the two map entries: remove in the next release (Phase 8 decision 5). + +## Phase 9: PR b, the rail leaves the core + +`runtime/runtime.py` becomes `runtime/session.py` and keeps the facade; `BrowserRuntimeRail` moves to +`surfaces/browser/rails/runtime_rail.py` as a hook class over five mixins; the prompt text (the worker prompt, the +image and date-picker sections) moves to `surfaces/browser/tools.py`. Three commits, each proved by goldens and AST +identity; after them the MCP tool-name allowlist is empty and no core module names a browser tool. + +### What moved + +| commit | from | to | +|---|---|---| +| rename | `runtime/runtime.py` | `runtime/session.py` (R100; imports in `main.py`, `topology.py`, `runtime_tools.py`, the two lazy `__init__` exports, the server, `browser_agent.py`, `decision_policy.py`, the tests, the two API docs; `_MOVED["runtime"] = "runtime.session"`) | +| cut | `BrowserRuntimeRail` (149 methods, 46 module constants, 3 helpers) | `rails/runtime_rail.py`: the class statement with 45 methods (the hooks, `__init__`, `_emit_status`, lifecycle, deadlines, progress, target normalisation) and 19 constants (15 once the prompt commit moves four on); `rails/terminal.py` `_TerminalMixin` 15 methods + 5 constants; `rails/results.py` `_ResultsMixin` 8 + 2; `probes/results.py` `_ProbeResultsMixin` 8; `semantic/phases.py` `_PhaseMixin` 44 + 9 constants + 2 helpers (and `_NUMBER_WORDS`); `semantic/evidence.py` `_EvidenceMixin` 29 + 3; `rails/keys.py` the 8 constants and `_contains_any_token` two or more of these use | +| prompt | `runtime_rail.py` (4 section constants), `runtime/agents.py` (`_BROWSER_WORKER_HELPER_TOOL_NAMES`, `build_browser_worker_system_prompt`) | `surfaces/browser/tools.py`, appended verbatim; `before_model_call` and `agents.py` import them | + +`runtime/session.py` keeps `_ACTIVE_BROWSER_RUNTIMES`, `ToolKind`, `_tool_args`, `BrowserAgentRuntime` and the two +reset functions (732 lines). The mixin modules import `rails/keys.py`, `runtime/session.py` (`ToolKind`, +`_tool_args`, the facade type) and the rest of what their methods use; `runtime_rail.py` imports the five mixins; +nothing imports `runtime_rail.py` but `browser_agent.py` and the tests. Every constant landed where its users are, +and the by-use placement agreed with the design's table for all 46 (the nine shared ones in `keys.py`). + +### The cut mechanism + +Each mixin is a plain class whose methods are the old methods, decorators and bodies included; the class statement +lists them before `AgentRail`, so `self._x(...)` resolves through the MRO exactly as before and no call site, no +`patch.object(BrowserRuntimeRail, ...)` target and no golden changes. No mixin defines a hook name (`before_*`, +`after_*`, `on_*`): `AgentRail.get_callbacks` sees the class alone. A `_NUMBER_WORDS` class attribute moves with its +only reader (`_infer_requested_result_count`, which reads it through `cls`). + +### Two bodies changed: the static methods that named the class + +The design's premise was "methods verbatim". Four call sites in two static methods named `BrowserRuntimeRail` to +reach a phase classmethod: `_recommended_recovery` (`rails/terminal.py`) calls `_missing_evidence_slots` and +`_unavailable_evidence_slots`; `_build_phase_state` (`semantic/phases.py`) calls `_infer_required_evidence_slots` +and `_infer_requested_result_count`. A mixin module cannot import `runtime_rail.py` (it is imported by it), so the +four tokens name `_PhaseMixin`, the class that now owns the callees; `terminal.py` imports it from `phases.py`. This +is why `_NUMBER_WORDS` is on `_PhaseMixin`: called through the mixin, `cls` is the mixin. The transitive `cls.` +closure of the four callees stays inside `_PhaseMixin`, checked before the cut. The AST check lists exactly these +two methods as changed and the class shell (new bases); everything else is identical. + +### Decisions + +1. **One mixin per target module, the class statement with the hooks** (design §2.1): no wrapper layer, no + `self → rail` renames, the ≈60 test call sites and patch targets untouched. +2. **A name, not a signature, for the four cross-references.** The alternative, turning the two static methods + into classmethods and writing `cls._x(...)`, changes two signatures and two decorators for the same effect; a + caller through the rail class would see no difference either way. Aleh can ask for the classmethod form. +3. **The prompt constants travel with the rail in the cut commit** and move on to `tools.py` in the prompt commit, + rather than staying in `session.py` for one commit with a `surfaces → runtime` import of private names. +4. **Tests that read or patch module attributes follow the code**: `browser_agent_log_warning` is patched on the + rail module (`_emit_status` lives there), `time` and `PolicyDenied` are read from it, the log-context key too; + the status-log source test reads `runtime_rail.py`. Module attributes the facade still uses (`Runner`, + `OpenAIModelClient`, `ToolKind`, the registry) stay read from `session.py`. +5. **Absolute imports in the new surface modules**, as the rest of `surfaces/browser/` outside the package-internal + relative ones; `session.py` keeps its relative imports (same folder). +6. **The alias package serves `playwright_runtime.runtime` as the session module** through `_MOVED`, so the + WebVoyager runner's `BrowserAgentRuntime` import resolves unchanged; the runner-contract mirror is green as it is. + +### Rejected options + +- **Cutting the facade** across modules (design §2.1): its browser hooks cannot leave without the PR e contract change. +- **Module-level functions with one-line methods** (design §2.1): ≈450 lines of wrappers, every body renamed. +- **A late import of `runtime_rail` at the bottom of `terminal.py` / `phases.py`** to keep the four tokens: the + cycle fails whenever the import starts from `runtime_rail.py` (the class is not defined yet). +- **Leaving `_NUMBER_WORDS` on the rail**: `_PhaseMixin._infer_requested_result_count` would raise on `cls`. +- **A forwarding `runtime/runtime.py`**: a `runtime → runtime` alias nobody outside the repo imports (the runner + imports the alias package); the shim covers it. + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move tests/unit_tests/harness/test_deep_agent_terminal_stream.py -q` + at the cut and at the prompt commit: **2399 passed, 122 skipped** (2398 / 122 after PR a plus the alias map case + for `runtime`); at the rename, the touched suites green (485 tests over two runs). +- AST identity (the scratch script): the cut, old `session.py` against the eight files, **262 old definitions + accounted for, 0 problems** (3 expected changed: the class shell, `_recommended_recovery`, `_build_phase_state`; 1 + expected moved: `_NUMBER_WORDS`); the prompt move, `agents.py` + `runtime_rail.py` + `tools.py` before and after, + **128 / 128**; the rename is R100. +- Method set: `vars(BrowserRuntimeRail) ∪ vars(mixins)` equals the old `vars(BrowserRuntimeRail)` (150 names); + no hook name on any mixin; MRO `BrowserRuntimeRail, _TerminalMixin, _ResultsMixin, _ProbeResultsMixin, + _PhaseMixin, _EvidenceMixin, AgentRail`. +- Goldens: `git diff a632b30f9 --stat -- tests/unit_tests/harness/tools/browser_move/snapshots/golden/` is empty. +- The eleven harness files outside `browser_move`: **248 passed** (as at the Phase 8 close). +- Both live sets, one run, `BROWSER_MOVE_LIVE_MCP=1` and `BROWSER_USE_SIDECAR_PYTHON` set: **100 passed in 296.16s (0:04:56)** (51 MCP conformance, 4 assembly, 45 browser-use conformance) and, in a second invocation, **19 passed in 74.87s** (helpers, rails, the four topology cases): 119, as at the Phase 8 close. +- `uv run python -W always -c "import …playwright_runtime.browser_capabilities, …playwright_runtime.runtime"`: one + `DeprecationWarning`; `playwright_runtime.runtime is runtime.session`. +- `ruff check --select I` over the package, the harness callers and the tests: 10 `I001`, as after PR a (the + blocks the moves touched were re-sorted; the ten left are pre-existing in files not edited). `ruff check` on the + touched files: only pre-existing findings (`E501` ×4 in the status-log test, `ASYNC250` in `main.py`). The new + modules are clean, and `ruff format --diff` over the eight files sums to the old file's 144 lines: no new drift. +- Allowlists, PR a → PR b: + +| list | after PR a | after PR b | +|---|---|---| +| browser words | 13 keys / 280 | 13 keys / 203: `runtime.py` 95 → `session.py` 39 (the design predicted ≈ 23: the rail's share was 56, not 73), `agents.py` 23 → 2 | +| MCP tool names | 2 keys / 20 | **empty** | +| driver names | 4 keys / 7 | unchanged | +| core imports of concrete modules | 8 keys / 19 | 6 keys / 14: `session.py → helpers` 4, `→ capabilities` 2, `→ semantic.state` 1 (`price_interval_signature` left with the rail), `service.py → capabilities` 2, `config.py → capabilities` 3, new `agents.py → surfaces.browser.tools` 2 (the design's §5.4 end state) | +| module size | 3 keys | 2 keys (`session.py` gone; `service.py` 1910 stale at 1570, untouched) | +| env reads | unchanged | unchanged | + +### Known leftovers + +1. The four `_PhaseMixin.` tokens in two static bodies (decision 2); the classmethod form is the alternative. +2. `session.py` keeps 39 browser words (its own `page`/`url` keys and `activate_page`); the text pass (PR d) and PR e + decide what stays. +3. The import allowlist is at the design's PR d end state already (6 keys); PR e empties it. +4. `KNOWN_OVERSIZED_MODULES` still says `service.py` 1910 (actual 1570); PR c removes the key. +5. The alias package: `_MOVED` has three entries now; remove with the package next release. + +## Phase 9: PR c, the service, the probes and the action specs + +Five code commits and this one: the five progress wrappers go; `BrowserService` is cut into the lifecycle +(`runtime/service.py`) and the worker loop (`runtime/worker.py`); the three probe builders get one module each and +the card template becomes a resource file; the browser action specs and the upload listing leave the controller +module for `surfaces/browser/helpers/custom.py`. After them the module-size allowlist is empty and the only browser +words left in `controllers/action.py` are the four of the builtin `browser_task` spec and the blocked-recursion error. + +### What changed + +| commit | from | to | +|---|---|---| +| deletion | `BrowserService.build_progress_context`, `record_tool_progress`, `record_worker_progress`, `get_progress_state`, `export_progress_state` (no package caller; mocks in three rail tests) | gone, with the mocks; the private helpers they wrapped stay, the loop uses them | +| worker cut | `service.py` 1,532 lines: `MAX_ITERATION_MESSAGE`, the two observer context variables, `BrowserTaskProgressState`, 33 loop methods | `runtime/worker.py` (1,033): the four verbatim plus `_WorkerLoop`, a plain mixin with the 33 methods; `class BrowserService(_WorkerLoop)` keeps 34 lifecycle methods (516 lines); the rail and its terminal mixin import the two names from `worker.py` | +| probes split | `surfaces/browser/probes/cards.py` 3,010 lines | `probes/metadata.py` (`build_browser_state_metadata_js`, 234), `probes/interactives.py` (`build_interactive_probe_js`, 636, imports `_clamp_int` from `cards.py`), `probes/cards.py` (`_clamp_int`, `build_card_probe_js`, 46) and `probes/cards.js` (2,113 lines, the raw template), shipped by a `pyproject` package-data glob; the package re-exports the three builders | +| action specs | `controllers/action.py` 600 lines: `_list_dir_files`, the `list_upload_files` action, the three browser specs and `upload_root` of `register_builtin_actions` | `helpers/custom.py`: `register_custom_action_specs(controller, *, upload_root)` (the specs, the upload action and `_list_dir_files`, verbatim apart from the receiver's name) and `register_custom_actions(controller, context, *, upload_root)` (those specs, then the `CUSTOM_ACTIONS` handlers over the context, the loop the facade ran inline); `action.py` keeps the registry and `ping`, `echo`, `browser_task`, `run_browser_task` (468 lines) | + +The facade's `ensure_runtime_ready` calls `register_builtin_actions()` then `register_custom_actions(...)`; the server +entry registers the builtin actions and then `register_custom_action_specs(get_default_controller(), upload_root=None)` +on its default controller, which is what it registered before (the upload root was never passed there). Registration +order is the old one on both controllers: `list_upload_files` is the tail of the action dict and the three specs plus +the upload spec are the tail of the spec dict, so registering them right after the builtin actions keeps both orders. + +### The card template as a resource + +`cards.py` reads `cards.js` once at import, `importlib.resources.files(__package__).joinpath("cards.js").read_text(encoding="utf-8").strip()`, +and `build_card_probe_js` substitutes the parameters into that string as before. Reading with universal newlines makes +the string equal to the old `r"""…""".strip()` whatever the checkout's line endings; the output of the three builders +was compared old against new on 19 parameter sets (the probe tests' grids, the defaults, `None`/string arguments, the +builtin site profiles, a selector cache record) and is byte-identical. The module-size test sees no `.js` file; the +wheel (`uv build --wheel`) carries `openjiuwen/harness/tools/browser_move/surfaces/browser/probes/cards.js`. + +### Decisions + +1. **The five wrappers are deleted** (head, Q8): one-line forwards to the private loop helpers with no package + caller; the tests that set them on a `MagicMock` service lose those lines and assert the same session state. +2. **`_WorkerLoop` is a mixin in `worker.py`, the same mechanism as the rail's cut**: methods verbatim, no call-site + change, the attributes set by `BrowserService.__init__` reached through `self` as before; `service.py` keeps its + name (thirteen test modules, the harness and the alias package import it) and `worker.py` imports nothing from it. +3. **`cards.js` is a package resource, not a 2,140-line Python module on the size allowlist** (head, Q2). +4. **`register_custom_actions` takes the handler loop out of the facade.** The design named only + `register_custom_action_specs` and had the facade import it next to `CUSTOM_ACTIONS`; that fifth imported name + would have raised the import allowlist key `session.py → surfaces.browser.helpers` from 4 to 5, and the step's + rule is that no allowlist grows. The facade now imports `register_custom_actions` in place of `CUSTOM_ACTIONS` + (the key stays at 4) and the loop it ran inline lives next to the actions it registers, which is the shape the + profile's custom-actions hook takes in PR e. `register_custom_action_specs` keeps the design's name and + signature for the server, which has no context. +5. **Tests follow the code**: the two service tests that patched `Runner.run_agent` through the service module + patch it through `runtime.worker`, where `Runner` is imported now; the controller test that expected the browser + specs from `register_example_actions` registers them as the server does; the upload tests register the builtin + actions and the specs in two calls; the settings test counts `list_upload_files` among the handlers the facade's + controller registers. +6. **`controllers/action.py` keeps 4 browser words, not the design's 1**: the `browser_task` spec's model-facing + text ("browser task", "the browser worker" twice) and the blocked-recursion error; all class B. + +### Rejected options + +- **Keeping `upload_root` on `register_builtin_actions` and moving the specs alone**: the upload listing is the + surface's action (its parameter and its text are browser-specific), so it leaves with the specs. +- **A `cards.py` of 2,140 lines on the size allowlist, or the raw string split over two modules** (design Q2). +- **Importing the helpers package by module** (`from ..surfaces.browser import helpers`) to keep the import count at + 4 with five names: it games the count and changes four call sites' spelling. +- **`_list_dir_files` into `runtime/utils/upload_paths.py`** next to the path resolution: it serves the surface's + action only and would add an import of the core from `custom.py` for one 14-line function. + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move tests/unit_tests/harness/test_deep_agent_terminal_stream.py -q` + after the last code commit: **2399 passed, 122 skipped in 49.69s** (as at the PR b close). Targeted runs at each + commit: the rail, service, recovery and terminal-stream suites (153) after the deletion; service, rail, + architecture, shim, topology, facade, recovery, isolation, registry, contracts, assembly and the terminal stream + (286) after the worker cut; the probe, surface, snapshot and architecture suites (821) after the probes split; + controller, uploads, settings, server entry, architecture, facade, snapshots, surfaces, runtime tools, worker + allowlist, create agent, assembly, shim, build agent (899) after the action-spec move. +- AST identity (the scratch script): the deletion, old `service.py` against new, **102 accounted for, 0 problems** + (5 expected missing); the worker cut, old `service.py` against `service.py` + `worker.py`, **97 / 97** (1 expected + changed: the class shell's new base); the probes split, old `cards.py` against the three modules, **4 / 4** + (`build_card_probe_js` and `_TEMPLATE` listed as the rewrite); the action-spec move, old `action.py` + `custom.py` + against new, **68 / 68** (the two `register_builtin_actions` signatures and `__all__` listed). +- Builder output equality: **19 cases, 0 differ**. Registration order: the facade controller for three upload roots + (none, a missing directory, a directory with files) and the module-level default controller, action order, spec + order, `list_actions`, `describe_actions` and the results of `list_upload_files`, `ping`, `echo`: **0 problems**. +- Goldens: `git diff a632b30f9 --stat -- tests/unit_tests/harness/tools/browser_move/snapshots/golden/` is empty. +- The wheel: `uv build --wheel` into a scratch directory, exit 0; the archive lists `.../probes/cards.js` next to the + seven probe modules. `python -m …playwright_runtime_mcp_server --help`: exit 0. The alias package: one + `DeprecationWarning`, `playwright_runtime.runtime is runtime.session`. +- `ruff check --select I` over the package, the harness callers and the tests: the same 27 `I001` in the same files + before and after the step (none in a file this step created or re-sorted). `ruff check` on the touched files: the + same findings as before minus four `E501` (lines that were inside the card template); the `ASYNC240` of the + `list_upload_files` closure moved with it to `custom.py`; `I001` in `controllers/action.py` is pre-existing. +- Allowlists, PR b → PR c: + +| list | after PR b | after PR c | +|---|---|---| +| browser words | 13 keys / 203 | 14 keys / 194: `service.py` 82 → 10 and `worker.py` 72 (the split), `controllers/action.py` 13 → 4 | +| MCP tool names | empty | empty | +| driver names | 4 keys / 7 | unchanged | +| core imports of concrete modules | 6 keys / 14 | unchanged (`session.py → helpers` 4: `register_custom_actions` replaces `CUSTOM_ACTIONS`) | +| module size | 2 keys | **empty** (`service.py` 1910 → 516, `cards.py` 3010 → 46) | +| env reads | unchanged | unchanged | + +### Known leftovers + +1. `worker.py` holds 72 browser words: the answer keys `page`/`url`, the model-facing resume and failure text, the + session-id prefix and the restart tokens that match driver error text (design §9, a logic item); the text pass + (PR d) takes the comments, docstrings and log lines, the rest is the documented residual. +2. `_TEMPLATE` is read when `probes.cards` is imported: a missing resource fails at import, not at the first probe. +3. The `ASYNC240` finding (a `Path` call in an async function) moved with the `list_upload_files` closure; the + `I001` of `controllers/action.py` (its `.base` import block) is older than this phase. Neither was touched. +4. The facade still imports four helper names, the catalog and the semantic tracker; `service.py` and `config.py` + the catalog; `agents.py` the worker prompt: PR e. + +## Phase 9: PR d, the end state of the architecture tests + +No move in this step: a text pass over the core's remaining browser words, the import rule the scanner could not see, +every allowlist at its end value, and the residual listed hit by hit. Three commits. + +### The text pass + +Thirteen core modules lose the browser words that named the surface in a docstring, a comment, a log line or a local +name: the service and the registry speak of the session, the device and the lifecycle identity; the logging module of +the agent log; the page state of the surface state; the worker loop's locals `page` and `url` are `location` and +`address` (the keys they read are unchanged); the recovery, guard and runtime-tool docstrings and five log lines drop +the word. Nothing the model reads, no key, no public attribute, parameter or message changed. Proof (a scratch +script): the old and new modules parse to equal ASTs once docstrings are dropped, the two renames and the seven log +strings are applied, for all thirteen files. Browser words in the core: 194 → 114 (14 keys → 10; `browser_logging.py`, +`controllers/base.py`, `main.py` and `service_registry.py` are at zero). + +### The registry rule + +`runtime/` reaches the registries (`DRIVERS`, `PROVISIONERS`, `SURFACES`) from `runtime/assembly.py` only, and +`SURFACES` also from `runtime/config.py` (`browser_surface_settings()`); the scanner ignored one-part import targets, +so the bare packages were invisible to the boundary test. The rule holds today with no exception. The design named +`surfaces` only; `drivers` and `provisioning` are the same kind of edge and hold with the same two importers, so the +rule covers the three. + +### Decisions + +1. **Class A only** (design §5.1): comments, docstrings, log lines, local names. A docstring that names a parameter + (`activate_page`'s ``url``) keeps the name; a message the model reads keeps its words even in a docstring-like + position (`RECOVERY_MESSAGE`, the stale-target errors, the `_noop_note`). +2. **The residual is accepted and listed** (Aleh, Q4): 114 hits in 10 core modules, each a data key, a public + attribute or parameter, the surface name, an identifier, driver error text the loop matches, or text the model + reads. No test narrowing. +3. **Local renames only where the local itself is the hit**: `page` → `location` and `url` → `address` in the worker + loop; `page_url`, `last_page`, `normalized_url` are not whole-word hits and stay. +4. **Log lines are class A**: five reworded (`dedicated agent log file enabled`, `Worker repeated the same tool call`, + `observation failed`, `Failed to reset active runtime`, `Failed to reset the managed runtime`, the two recovery + lines); no test asserts on any of them (checked before the pass). + +### Rejected options + +- **Narrowing the browser-word test** to comments, docstrings and log strings (design Q4 option b): the keys and the + model text would stop being counted, which hides the residual instead of documenting it. +- **Renaming the public attributes and keys** (`BrowserPageState.url`, `page_id`, the runner answer's `page`/`url`, + the surface name): logic and contract changes outside the phase. +- **Rewording the model-facing strings** (the resume text, the stale-target errors, the tool descriptions): the + goldens and the prompt snapshots pin some of them, and the rest is prompt engineering, not architecture. + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move tests/unit_tests/harness/test_deep_agent_terminal_stream.py -q` + after the test commit: **2400 passed, 122 skipped in 44.91s** (PR c's 2399 plus the registry test); the text pass + alone: 572 targeted + tests green (architecture, service, logging, config, page state, guard, recovery, runtime tools, registry, facade, + rail, topology, status logging, isolation, snapshots, terminal stream). +- The no-logic proof: 13 files, 0 differ after normalisation. +- Goldens unchanged; `ruff check` on the touched files: the pre-existing `ASYNC250` in `main.py` only. +- Allowlists at the end of PR d: + +| list | value | notes | +|---|---|---| +| browser words | 10 keys / 114 | the residual below | +| MCP tool names | **empty** | since PR b | +| driver names | 4 keys / 7 | `runtime/browser_tools.py` 3 (the server's client config: the config facade's name, the `/mcp` path), `runtime/service.py` 1 (the CDP endpoint of the given server config, the lifecycle identity), `surfaces/browser/capabilities.py` 2, `surfaces/browser/helpers/batch_steps.py` 1 | +| core imports of concrete modules | 6 keys / 14 | `session.py → helpers` 4, `→ capabilities` 2, `→ semantic.state` 1; `service.py → capabilities` 2; `config.py → capabilities` 3 (call-time); `agents.py → surfaces.browser.tools` 2; PR e empties the list | +| module size | **empty** | since PR c | +| registries | rule, no allowlist | `assembly.py` (three), `config.py` (`SURFACES`) | +| surfaces → drivers/provisioning | rule, no allowlist | 0 violations since PR a | +| env reads | unchanged | the two named exemptions | + +### The browser-word residual, hit by hit + +Classes: **B** text the model reads (prompt, tool description, error or message in a result); **C** a data key, a +public attribute or parameter, an identifier, the surface name, or driver text the loop matches. + +| module | line (hits) | what | class | why it stays | +|---|---|---|---|---| +| `agents.py` | 194 (1) | the worker's `AgentCard` description | B | the card text the agent and its callers see | +| `config.py` | 332 (1) | `CoreSettings.surface = "browser"` | C | the surface name, the registry key | +| `config.py` | 421 (1) | `"demo-browser-session"` | C | the demo entry's default session id | +| `config.py` | 466 (1) | `SURFACES.create("browser", …)` | C | the registry key (`browser_surface_settings`) | +| `controllers/action.py` | 175 (1) | `recursive_browser_task_blocked: browser workers must not …` | B | the error returned to the caller of `browser_custom_action` | +| `controllers/action.py` | 428, 431, 441 (3) | the `browser_task` / `run_browser_task` spec summary and `task` parameter text | B | `describe_actions` output, read by the model through `browser_list_custom_actions` | +| `page_state.py` | 200 (1) | `self.page_id = page_id or f"page-…"` | C | public attribute, constructor parameter, the id format | +| `page_state.py` | 210, 245 (2), 258, 443 (5) | `self.url` | C | public attribute (`BrowserPageState.url`), read by the rail and the facade | +| `page_state.py` | 242, 253, 255 (3) | `advance(*, url=)`, `observe(*, url=)` and the parameter read | C | public signatures | +| `page_state.py` | 273 (2), 290 (2), 447 (1) | `observe(url=payload.get("url"))`, `observe(url=address)` | C | the keyword and the observation payload key | +| `page_state.py` | 442 (1) | `getattr(surface, "url", "")` | C | `BrowserPageState.record` reading the surface by attribute (design §9: a logic item for the benchmark phase) | +| `page_state.py` | 268, 590, 639, 652 (4) | "Probe or snapshot the current page again", "Capture the current page snapshot again", "Probe the current page again" | B | the stale-target errors the model reads | +| `page_state.py` | 762 (2), 776 (2) | `"url": self.url` | C | the `export` / index keys of every result's `page_state` | +| `page_state.py` | 971 (1) | `payload.get("url")` | C | the observation payload key | +| `rails/final_answer_recovery.py` | 21 (1) | `RECOVERY_MESSAGE` naming the `page` field | B | the recovery prompt; it lists the answer schema's keys | +| `rails/repeated_call_guard.py` | 34 (1) | `document\.URL` in `_NOOP_ATOM` | C | the page JavaScript the guard recognises as a no-op | +| `rails/repeated_call_guard.py` | 89 (2), 90 (1) | `_noop_note` | B | the note the model reads when tools are withheld | +| `runtime_tools.py` | 26, 28, 49 (3) | the descriptions of `browser_custom_action` and `browser_list_custom_actions` | B | tool cards | +| `service.py` | 379 (1) | `f"browser-{uuid}"` | C | the default session id (`session_new`), the worker conversation id derives from it | +| `session.py` | 237 (1) | `parsed.get("url")` | C | the probe result key `_rewrite_primary_link_click` reads | +| `session.py` | 379 (1) | `dom_error = f"page observation failed: {exc}"` | B | returned in the capture result (`dom_error`) | +| `session.py` | 395 (3), 421 (3), 445 (3), 468 (3), 488 (3), 503 (2) | `"url": metadata.get("url") or exported["url"]` and the like | C | the `url` keys of the `capture_*` results and of the metadata / page-state / semantic-state exports they read | +| `session.py` | 562 (3) | `"page": {"url": surface.url …, "title": …}` | C | `main_agent_answer`'s keys and the surface attribute (the runner contract) | +| `session.py` | 595, 596, 598 (3) | `activate_page(self, url)`, its docstring naming ``url``, the call through | C | public parameter; PR e moves the hook behind the profile | +| `worker.py` | 70, 97 (2) | `last_page.get("url")` | C | the stored progress state's key (`from_dict`, `from_task_state`) | +| `worker.py` | 189 (1) | `"url": self.last_page_url` | C | `to_dict` key (the session state the rail reads) | +| `worker.py` | 231 (2), 233, 235, 248 (2), 251 (7) | "target page, context or browser has been closed", "navigation failed because browser has disconnected", "page crashed" | C | driver error text the retry and restart predicates match (design §9: a logic item) | +| `worker.py` | 257 (1) | `"browser-session"` | C | the default session id of the worker conversation id | +| `worker.py` | 312 (2), 313 (2) | `value.get("page")`, `value.get("url")`, `location.get("url")` | C | the worker answer's keys (`_extract_page_snapshot`) | +| `worker.py` | 341 (1) | `f"url={address}"` | B | a progress-summary line the model reads in the resume task | +| `worker.py` | 517, 603 (2), 736, 778 (2), 804 (7) | "Recent browser tool activity", "current logical browser session/tab", "Continue from the current browser state", "Last page: url=" | B | the progress context and the resume task text | +| `worker.py` | 660 (2), 675 (2), 842 (2), 982 (2), 1023 (2) | `"page": {"url": "", "title": ""}` | C | the runner answer's keys | +| `worker.py` | 677, 1009, 1025 (3) | "Browser worker did not return valid JSON output", "unknown browser execution error" | B | error text in the runner answer | +| `worker.py` | 891 (2), 905 (2), 909, 928 (2), 935, 936 (2), 955, 1005 (11) | `parsed.get("page")`, `location.get("url")`, `"page": {`, `"url": …`, `last_failure_page.get("url")` | C | the worker answer's keys and the runner answer's keys | + +Totals: `agents.py` 1, `config.py` 3, `controllers/action.py` 4, `page_state.py` 24, `final_answer_recovery.py` 1, +`repeated_call_guard.py` 4, `runtime_tools.py` 3, `service.py` 1, `session.py` 25, `worker.py` 48 = 114; B 33, C 81. + +### Known leftovers + +1. PR e removes the six import keys and, with the hooks behind the profile, `session.py`'s `activate_page(url)` and + `main_agent_answer`'s `surface.url` stay as the facade's public surface (the keys are the runner contract). +2. `BrowserPageState.record` reading `surface.url` / `title` by `getattr` and the restart predicates matching driver + error text are logic items (design §9), not this phase's. +3. `ASYNC250` in `main.py` (an `input()` call in the demo's async entry) is older than this phase; untouched. + +## Phase 9: PR e, the profile carries what the core needed from the surface + +A `feat` triple (code, tests, docs) plus one test fix. `SurfaceProfile` gains the fields the facade, the service, the +config and the agent builders took from `surfaces/browser/` by import: the facade hooks, the worker prompt and the +capability catalog. After it `runtime/` imports no module of `surfaces/` at all: the core-import allowlist is empty +and the registry (`assembly.py`, and `config.py` for the surface settings and the catalog) is the only way from the +core to a surface, which is the plan's done-when for Phase 9 that moves alone could not reach (design §11). + +### The contract change + +- `Capability(name, description, tool_names, policy_only=False)`: one catalog entry; `policy_only` marks an entry a + policy rule alone grants, which the driver has no feature to enable for. +- `SurfaceProfile` gains `capabilities: tuple[Capability, ...]` (the catalog in order, its first entry the default one + every agent gets), `custom_actions(controller, context)`, `page_metadata(context) -> (metadata, error)`, + `policy_probe(context, source, params)`, `activate(context, address) -> bool`, `semantic_tracker()` (a factory) + and `worker_prompt(screenshot_subdir, artifacts_subdir, *, tool_names=None, driver_name)`. Each is `None` (the + catalog empty) unless the surface fills it. Invariants: the agent runtime facade and the delegated worker call + them without checking them (a facade over a surface without hooks is a programming error); a surface driven + through `build_agent` alone fills the catalog only. +- The browser profile fills all seven: `DEFAULT_BROWSER_CAPABILITIES` (now a tuple of `Capability`; + `BrowserCapability` is that type under its old name, the three policy-only entries carry the flag and + `POLICY_ONLY_BROWSER_CAPABILITY_NAMES` is derived from them), `register_custom_actions`, `page_metadata`, + `probe_for_policy`, `activate_page`, `SemanticStateTracker`, `build_browser_worker_system_prompt`. + +### What the core reads now + +| module | before | after | +|---|---|---| +| `runtime/session.py` | `CORE_BROWSER_TOOL_NAMES`, `capabilities_of` at construction; `SemanticStateTracker()` at construction; `page_metadata`, `probe_for_policy`, `activate_page`, `register_custom_actions` imported | `capabilities` computed from `profile.capabilities` on first read (the default entry's tools when no allowlist was given, the same result as before); the tracker from `profile.semantic_tracker()` on first use; `ensure_runtime_ready` calls `profile.custom_actions(controller, context)`; the capture, probe and activate methods call the profile's hooks | +| `runtime/service.py` | `capabilities_of(...)` into an attribute nobody read | gone; `allowed_tool_names` is the allowlist as given (`None` when none) | +| `runtime/agents.py` | `build_browser_worker_system_prompt` imported; the refresh log named the helper subset | `worker_prompt` is a required keyword of `build_browser_worker_agent` and `refresh_browser_worker_system_prompt`; `build_worker` and the loop pass the session profile's; the log lists the tools the prompt was rendered for | +| `runtime/config.py` | the catalog imported inside `playwright_mcp_config` | `browser_surface(environ, settings) -> SurfaceBundle` (used by `browser_surface_settings` too); the `--caps` list is the catalog after its first entry minus `policy_only`, read through the registry with the given settings | +| `surfaces/browser/helpers/custom.py` | `register_custom_actions(controller, context, *, upload_root)` | `register_custom_actions(controller, context)`, the upload root from `context.settings`; `register_custom_action_specs` unchanged for the server | + +### Decisions + +1. **Typed fields on `SurfaceProfile`, not a `hooks` mapping** (the design's sketch): every hook has a signature the + core relies on; a mapping of callables hides them from the reader and the type checker. +2. **Optional fields, `None` by default, required by the facade without checks.** The alternative, required fields, + makes every test profile and the fake surface fill seven callables they never use; a no-op implementation per + hook in the contract would put the tracker's interface into the contract. The fake screen surface fills its + catalog (one `Capability`, `core`) and leaves the rest `None`: a screen has no metadata script, no probes, no + custom actions, no semantic state and no delegated worker; the E2E test runs unchanged. +3. **The default capability is the catalog's first entry.** The browser catalog lists `core` first and every test + profile keys its tools by `core`; a `default` flag or a profile field would say the same thing twice. The `--caps` + list is "the rest minus `policy_only`", and `policy_only` lives on `Capability` because the core cannot import + the surface's constant. +4. **Lazy `capabilities` and tracker.** Both need the profile; the facade assembles its surface bundle lazily + (`service.core` is read at first use, a documented invariant), so they are computed on first read. Every path + reads them after `ensure_runtime_ready`, which assembles the bundle anyway; no observable change. +5. **The service's own capability computation is deleted**, not moved: nothing read it. Its `allowed_tool_names` + is now the allowlist as given, `None` when the caller gave none (it was the core tool names); no package code + and no test read it in that case. +6. **`register_custom_actions(controller, context)`** reads the upload root from the context's settings so the hook + signature stays surface-neutral; in production the context's settings are the facade's (the service opens the + session with them). +7. **`config.py` reads the catalog through the registry per server-config build.** The Playwright MCP driver builds + its server config from settings alone (`drivers/playwright_mcp/driver.py`), so a catalog parameter would need + the same fallback; one registry read more per build makes a dataclass of constants. +8. **The feat commit carries the test adaptations its signature changes force** (the `worker_prompt` keyword in the + four tests that build the worker agent directly, the empty import allowlist), the tests commit the new + coverage, as the moves carried their import fixes. One test was missed: the facades test's fake surface factory + took `environ` alone (the registry documents `(environ, instance=None, settings=None)`) and returned no profile; + fixed in a following `test(harness)` commit, so the feat commit is red for that one test. +9. **The refresh log line lists all tool names**: the helper subset needed the surface's constant. + +### Rejected options + +- `hooks: Mapping[str, Callable]` (design sketch): decision 1. +- Required fields, or contract-level no-op defaults: decision 2. +- `Capability.default` or `SurfaceProfile.default_capability`: decision 3. +- Passing the catalog into `playwright_mcp_config` as a keyword: decision 7. +- Assembling the surface at construction to keep `capabilities` eager: reads the core settings at construction, + against the documented lazy read; the facade's other lazy properties already assemble on first use. + +### Verification + +- `uv run pytest tests/unit_tests/harness/tools/browser_move tests/unit_tests/harness/test_deep_agent_terminal_stream.py -q` + after the last commit: **2404 passed, 122 skipped in 51.20s (PR d's 2400 plus the four new tests)**. After the feat commit with its adaptations, 1368 targeted tests green + (architecture, contract, surfaces, config, service, facade, worker build, topology, recovery, hint, guard, + settings, create agent, capabilities, snapshots, assembly, rail, policy probe, semantic state, worker allowlist, + server entry, uploads, terminal stream); the new tests 93 with their files. +- The eleven harness files outside `browser_move`: **248 passed**. +- Both live sets, one run (`BROWSER_MOVE_LIVE_MCP=1`, `BROWSER_USE_SIDECAR_PYTHON` set): **119 passed in 353.63s (0:05:53): 51 MCP conformance, 45 browser-use conformance, the helpers, assembly, rails and the four topology cases, as at the Phase 8 and PR b closes**. +- Goldens: `git diff a632b30f9 --stat -- tests/unit_tests/harness/tools/browser_move/snapshots/golden/` is empty. +- The alias package: one `DeprecationWarning`, `playwright_runtime.runtime is runtime.session`; the server entry's + `--help` exits 0. +- `ruff check` on the touched files: the pre-existing `ASYNC240` only; `ruff check --select I` over the package, the + harness callers and the tests: 27 `I001`, the same files as at the PR b close. +- Allowlists at the end of Phase 9: + +| list | value | +|---|---| +| browser words | 10 keys / 114, the residual of PR d (unchanged by PR e) | +| MCP tool names | empty | +| driver names | 4 keys / 7: `runtime/browser_tools.py` 3, `runtime/service.py` 1, `surfaces/browser/capabilities.py` 2, `surfaces/browser/helpers/batch_steps.py` 1 | +| core imports of concrete modules | **empty** | +| module size | empty | +| registries | 0 violations (`assembly.py` three, `config.py` `SURFACES`) | +| surfaces → drivers/provisioning | 0 violations | +| env reads | unchanged | + +### The plan's done-when + +- No module in `runtime/` or `surfaces/` over 1,500 lines: holds (the size allowlist is empty). +- The import-boundary test with `surfaces/` in its rules: `surfaces/` imports `contract/` and `runtime/` helpers and + never a driver (holds); `runtime/` never imports `surfaces/` except through the registry (holds: the allowlist is + empty, and the registry rule names `assembly.py` and `config.py`). +- The browser-word allowlist empty: not reachable without renaming keys and public names; the documented residual + stands in its place (Aleh, Q4; F_07 "Phase 9: PR d"). +- The snapshots match: the goldens are byte for byte those of `a632b30f9`. +- The smoke tasks on both drivers: not run in this session (no WebVoyager run); the live conformance, assembly, + helper, rail and topology sets ran on both drivers (above). + +### Known leftovers + +1. `BrowserAgentRuntime` keeps its browser-flavoured public surface (`activate_page(url)`, the `capture_*` methods, + `main_agent_answer`'s keys): the runner contract, the residual. +2. `openjiuwen/harness/subagents/browser_agent.py` and `rails/subagent/subagent_rail.py` import the catalog module + directly: outside the package, they compose the browser agent and may. +3. `service.allowed_tool_names` is `None` when the caller gave no allowlist (decision 5). +4. Two registry reads per server-config build (decision 7). +5. The alias package: remove with the next release. + +## Phase 10 fixes + +The full WebVoyager runs on both drivers (code of the Phase 9 close) were read task by task; the failures on the +agent side are fixed here, one commit triple per fix. Benchmark-side failures (judge, geo, stale or blocked sites) +are out of scope. + +### Offload path on Windows (browser-use) + +`Direct.run` built the conversation id as `:direct::`. The context engine uses the conversation id +as its session id and the session id as a directory name (`/context/_context/offload/`); +Windows rejects `:` (`WinError 123`), so on browser-use every offloaded tool result fell back to memory, the model saw +only the preview, and `browser_recall_offload` answered "invalid browser runtime session" (226 of 559 tasks). The +delegated worker (MCP) never offloads, so its `:`-separated id never reached a path. + +Decisions: +1. **Both sides.** The id now uses `.` (`.direct..`), and the core maps the session id to a directory + name at the point the path is built: `ContextUtils.offload_session_dir_name` keeps a path-safe id + (`[A-Za-z0-9_.-]+`, not `.`/`..`) unchanged and turns any other id into one safe component plus a 16-hex sha256 + suffix, so ids that read alike (`a:b`, `a/b`) stay apart. +2. **One function for the writer and both readers**: the tool-result offloader (budget and window processors), the + core reload buffer (`OffloadMessageBuffer`) and the browser recall tool. The recall tool's own id pattern goes; + the mapping makes any id safe and the "inside the workspace" check stays. +3. **Path-safe ids keep their directory**: nothing changes for the ids that worked. + +Rejected options: +- Sanitising in the harness only (the conversation id): a caller's session id could still carry an illegal + character, and the task asked for the path to be safe where it is built. +- Changing every `_context` builder in the core (forked processors, session memory, compression dumps, + debug dumps): outside this fix; the forked tree and the session-memory paths are not used by the browser agent. + Leftover 1. +- A plain character replacement without a digest: `a:b` and `a-b` would share a directory, and the recall tool + would read another session's files by handle. + +Verification: before the fix the new real-filesystem tests failed with `WinError 123` (core reload, browser recall) +and the topology test with the old id; after it they pass. The browser_move unit suite, the terminal-stream test and +`tests/unit_tests/core/context_engine` together: 2922 passed, 122 skipped, 1 failed +(`test_compression_recall.py::test_recall_rejects_chunk_symlink_escape`, which fails identically on the code before +the fix: creating a symlink needs a privilege this Windows account lacks). Goldens unchanged. + +Known leftovers: +1. Other core writers of `_context` (`processor/base.py`, `offloader/message_offloader.py`, the forked + processors, session memory, compression and debug dumps) still use the raw id; with a path-safe id (all the + browser agent produces now) they and the mapped name agree. +2. The delegated worker's conversation id keeps `:`; it never reaches a path today. + +### Clipboard (security) + +Google Map--31 on browser-use: `navigator.clipboard.readText` failed, the agent pressed `Control+v` and pasted the +host's OS clipboard (a PowerShell command) into the Google Maps search box. The managed Chrome shares the host's +clipboard, so any paste hands a third-party page whatever the user last copied. + +Decisions: +1. **The browser profile's first `PolicyRule`s** (`surfaces/browser/policy.py`): paste chords on `PressKey` + (control/ctrl/meta/cmd/command/ControlOrMeta + `v`, shift + `Insert`, any case, extra modifiers allowed) and + clipboard reads in `RunScript` (`clipboard.read*` dotted or bracketed, `execCommand('paste')`). The surface owns + them, the policy rail applies them for both dialects (they decode to the same commands). +2. **The batch checks the same rules itself.** It executes commands through the `ToolContext`, past the rail; it calls + `policy_denial` before each command and fails the step with the reason. No contract change: the rules are the + surface's own constant and the batch is a surface helper. +3. **No `TypeText` rule**: both drivers insert the text literally (browser-use `insert_text`, MCP `browser_type`), so + a typed "Control+v" is text, not a chord. The task listed `TypeText`; a rule there would never fire. +4. **One message** for every case: the clipboard belongs to the host; read the value from the page and type it. + +Rejected options: +- An isolated clipboard for the managed Chrome (suggested in the run analysis): a provisioning change with no + portable Chrome switch for it; the rules close the observed path. +- Writing to the clipboard (`writeText`) denied too: it does not leak host data to the page; left allowed. + +Verification: before the fix the browser profile had `policy_rules == ()` and a batch `press` step with +`Control+v` reached the driver as `PressKey("Control+v")`; after it, 34 new cases pass (the rules, the rail on the +Playwright MCP and browser-use builders, the batch on a FakeDriver). Full unit suite 2439 passed, 122 skipped. +Goldens unchanged. + +Known leftovers: +1. Pass-through native tools without a decoder (the MCP server's `browser_run_code`, when the server exposes it) are + not checked: the rail checks decoded commands only. +2. The script rule is a text match: obfuscated code (`navigator[atob(...)]`) passes. The paste-chord rule is the + one that stops the observed leak. +3. Middle-click paste of the X11 primary selection is not covered (Windows and macOS have none). + +### Final-answer recovery (MCP worker) + +The recovery turn fired in 125 of 553 MCP tasks (baseline 83 of 559). Its message ("If the task is not fully +complete, set ok=false and say in final what you found and what is missing") made the worker re-audit answers it had +already given in prose; Apple--8, Apple--11 and ESPN--25 came back as hedges the judge failed. + +Decisions: +1. **A prose ending is restated, not re-audited.** `non_json_output` gets `RESTATE_MESSAGE`: restate the answer + already given as the final JSON, do not re-check, change it or add caveats, `ok=false` only if that reply said the + task could not be completed. The prose endings of both runs are finished answers (below), so restating is right. +2. **Empty output and the step limit keep the old message**: there is no answer to restate; reporting what was + found and what is missing is still the right request. Both messages share one phrase for the JSON fields, so + `RECOVERY_MESSAGE` is byte-identical to before. +3. **The tool list stays in the recovery request** (D-R1): dropping it changed the prompt prefix and paid a full cache + write for the whole conversation ($0.41 of $1.17 on one smoke task). The rail still runs no tool call of the + recovery turn (forced finish with the turn's text) and stops after one model call. The recovery call is told apart + by its message, no longer by an empty tool list. + +Why the worker ends in prose more often (investigated on the run logs, not fixed): +- All of the rise is prose: `non_json_output` 54 → 114 (9.7% → 20.6%); `empty_output` 28 → 7; step limit 1 → 4. On + the tasks both runs ran, 83 ended in prose only in the refactor run and 23 only in the baseline. +- The prose endings are complete answers ("I found a recipe that fits: …"); the task text asks for "your final + answer as text", the system prompt for JSON only. +- Same inputs: system prompt, user message template, the 14 tool schemas and the guard and hint texts are byte + identical, one user message per request, nothing injected before the final turn. +- No harness message precedes the prose turn (no denial, intercept hint or repeated-call hint). The model's usual + way to finish (a no-op script or the repeated-call hint, then JSON) fell from 254 to 210 endings, and after a real + tool result prose roughly doubled (evaluate 19% → 37%, snapshot 20% → 34%, find 9% → 29%), across sites. +- What the refactor changed in what the model sees: tools listed alphabetically (`browser_find` last), "### Events" + and the snapshot file links gone, evaluate strings shown raw instead of as a quoted JSON string, errors no longer + wrapped in `{"result": ...}`, larger action results. None of these is tied to the prose turns in the logs. +- The model also changed where its input was identical: after the byte-identical repeated-call hint, empty replies + fell from 27/134 to 5/133. The smokes point weakly to model drift (2 of 3 prose endings on Oct 10 with the old + result format, 0 of 3 on Sep 30 and Oct 8). +- Conclusion: not attributable to our code on this evidence; the decisive check is the baseline code rerun today on + the 86 refactor-only recovery tasks plus controls (about 20% prose means drift; about 10% means the refactor, then + A/B the tool order and the result formatting). Decision 1 handles the prose endings either way. + +Rejected options: +- One message for every reason: an empty reply or the step limit has no answer to restate. +- `tool_choice: none` in the recovery request: a request parameter change of its own; the rail's forced finish + already keeps tools from running. +- Changing the result formatting or tool order back on the strength of a correlation: unproven, and each moves every + golden. + +Verification: before the fix the empty-output and prose recovery tests failed on the tool list (no tools in the +request). Full unit suite at the fix: 2439 passed, 122 skipped. No golden carries a recovery message; goldens +unchanged. + +Known leftovers: +1. The prose-rate rise is not explained (above); the baseline rerun decides. +2. A recovery turn that answers with a tool call and no text yields no JSON; today's prose fallback applies. + +### Phase budgets on command records (browser-use main agent) + +The main agent's phase rail ended 16 browser-use tasks early (extraction 20/20: Amazon--1/15, ArXiv--11/41, +Coursera--32, Google Flights--6/7, Google Map--5/40, Wolfram Alpha--7; navigation 12/12: Google Search--11/14; +semantic replan: Apple--22, Booking--9/24) and nav-menu cards forced `must_finish` on Allrecipes--41. MCP passed +them all: its delegated worker has no phase budgets. Read first: Amazon--1 (size, ZIP and cart clicks inherited the +extraction phase; 15 distinct page scripts), Google Flights--6 (form clicks between probes counted as extraction), +Allrecipes--41 (the cards were `#mntl-fullscreen-nav_1-0 … li.mntl-fullscreen-nav__list-item`, kind `result`), +Google Search--11 (13 navigations, each to a new address). + +Decisions: +1. **An action leaves extraction.** A known `ACT` command that is not a form or filter command used to inherit the + current phase; while extracting it now goes to `form` (created on demand, as typing already did), elsewhere it + keeps the current action phase (a click while filtering stays filtering). +2. **Progress is free, stalls count.** An `ACT`/`NAVIGATE` command whose record shows `context_changed` or a new + generation (a new address, or an observation that changed past the page state's threshold) gets its attempt back. + This is what the step-limit-free MCP worker effectively has, and it keeps the budget as a guard against actions + that change nothing. Reads are never refunded. +3. **A page script costs once.** In extraction a `RunScript` counts once per distinct code; polling the same script + is free (the repeated-call guard handles identical repeats). +4. **A changed page is semantic progress.** The tracker's digest has no field for a dialog, a cart or a picker, so a + click that opened one scored `no_progress` and three of them forced a replan. The runtime now passes + `changed` (the generation or the last observation moved since the previous semantic observation); a repeated + digest with `changed` is progress. Revisits (A, B, A) stay revisits. +5. **Menus are not cards.** "nav" is a token of its own in the chrome check (`-nav_`, `nav__` matched; `canvas` not), + and a candidate inside `nav`, `[role=navigation]`, `[role=menu]` or `[role=menubar]` is dropped whatever its kind. + +Rejected options: +- Larger budgets: hides loops as well as work; the counting was wrong, not the numbers. +- Charging after the call instead of refunding: moves the budget check past the dispatch; the refund keeps the deny + before anything reaches the driver. +- Encoding the generation into the semantic digest: every state would look new, and the A-B-A revisit guard would + never fire. +- Keeping the kind exemptions (`shop`, `product`, …) for the nav-container check: a link inside a real menu is a menu + link whatever its kind. + +Verification: 11 of the 18 new cases fail before the fix (classification, script count, refunds, the runtime's +change mark, the tracker flag, both card cases); the 7 loop-guard cases pass before and after. The existing loop +cases of `test_browser_runtime_rail.py` stay green (99). Full unit suite 2454 passed, 122 skipped. Goldens +unchanged. + +Known leftovers: +1. Whether the 16 tasks now finish is for the rerun; the budgets themselves (12/24/20/20) are unchanged. +2. A navigation loop between distinct addresses (A, B, A, B) is refunded; it is bounded by the semantic revisit + guard and the step and time limits, not by the navigation budget. +3. The first navigation from a blank tab advances no generation and is charged. +4. Page scripts that mutate (a `.click()` in the code) still count as extraction reads. + +### browser-use driver gaps + +Three failures on browser-use came from the driver, not the agent: Wolfram Alpha--7 (`browser_evaluate` of +`document.title` answered "evaluate: Uncaught"), Google Map--35 (navigate, evaluate and observe "timed out after +30.0s" for the rest of the task, no recovery), Google Flights--6 (`button:has-text(...)` and `div[role=combobox] +:has-text(...)` selectors failed with "DOM Error while querying"). + +Decisions: +1. **`RunScript.code` is an expression, called when it is a function**, on both drivers: Playwright MCP's + `browser_evaluate` does `eval('(' + code + ')')` and calls a function value; the sidecar now does the same + (page-wide and element-scoped). A newline before the closing parenthesis keeps a trailing line comment harmless. + The error text is the first line of the JS error description. +2. **Two request timeouts in a row are a lost target** (browser-use only). The transport's 30 s request timeout + means the sidecar got no answer from the page; one can be a slow page, two in a row with nothing in between is a + hung tab, and the session's single restart is the only recovery. On Playwright MCP a click timeout is usually an + intercepted click on a live page, so the rule stays in the browser-use driver, not in the driver session. +3. **Playwright selectors are refused, not translated.** The sidecar resolves a `Locator` with `querySelector`; + translating `:has-text()` would be a second selector engine. The refusal is `Unsupported(dispatched=False)` and + names what works (a ref or `target_id`, plain CSS, find/probe for text). +4. **Conformance**: the scene's untargeted `RunScript` case is now a bare expression, so both drivers prove the + contract; the targeted case keeps the function form. + +Rejected options: +- Counting timeouts in `DriverSession` for every driver: decision 2. +- Reloading the page after the first timeout: the sidecar's request is still cancelled-pending; a restart is the + one recovery the session already owns. +- Supporting `:has-text()` by a text filter in the sidecar: decision 3. + +Verification: 11 new cases fail before the fix (the evaluate wrapper and the scoped script in node, the error text, +the second timeout, five Playwright selectors). Full unit suite 2474 passed, 122 skipped. Goldens unchanged. The +live browser-use conformance set runs at the end of the Phase 10 fixes. + +Known leftovers: +1. After the restart the managed Chrome is the same process; a tab that hangs the whole browser needs the + provisioner's dead-device path, which only `is_alive` decides. +2. The timeout count is per driver, reset by any reply or `start()`, not by task boundaries the driver cannot see. + +### Screenshot on browser-use (not done) + +Wolfram Alpha--18 needed an image (an equation). Stopped with questions for the head: the browser dialect already +has `browser_take_screenshot`, offered through `vision` and only with image support; in the Phase 10 runs neither +driver offered a screenshot tool, and the browser-use main agent ran with image input off ("Image input is +unavailable or unverified for this run"). Adding the tool to the browser dialect changes nothing until image input +is on for that agent, and the catalog is shared by both drivers (a `core` entry would reach MCP too). Open: whether +the browser-use agent runs with image input, and whether the tool is browser-use only (a per-dialect selection rule) +or `core` for both. + +### Image input (both topologies) + +Run `20261012-1926_adhoc_sonnet-5.5_t1p1_12c332832` (browser-use, Direct): all 58 task logs carry the "Image input is +unavailable or unverified" guidance although the image probe logged `supported=True` in the same processes. Three +causes: +1. The runtime rail resolved image support from `ctx.agent`. `before_invoke` fires on the `DeepAgent`, but the model + and tool hooks are bridged to the inner `ReActAgent` (`_BRIDGE_EVENTS`), which has no `deep_config`; the + resolver then finds no config and no model and answers False. The tools were built on the right answer, the + prompt section and the batch screenshot gate on the wrong one. +2. The probe ran in the background (`schedule_image_support_probe`). The runner runs one `asyncio.run` per task, + so the first task of every shard ended before the verdict landed. +3. The delegated worker was built with `image_support=False`, whatever the model. + +Decisions: +1. **The rail resolves against the agent `before_invoke` saw.** There is one rail instance per agent + (`create_browser_agent` builds it), so the agent is remembered there and the three readers agree. The resolution + order stays `_multimodal.py`'s: explicit `enable_read_image_multimodal`, per-model `supports_vision`, the probe + cache. +2. **Both topologies await `probe_image_support` before they build the agent**, as the probe module's docstring asks + of a caller with one `asyncio.run` per task. The verdict is cached per `(api_base, model_name)`, so only the + first build of a process waits (up to the probe's 5 s timeout, twice when the reasoning-off retry runs). + Inconclusive is `None`: not cached, treated as unsupported. +3. **The worker takes the verdict** (`build_worker(..., image_support=)`), which turns on the screenshot-only tools + and the image blocks in tool results together. The worker prompt already names `browser_take_screenshot` only + "when it is present", so it needs no change. +4. **Unit tests stub the probe** in a package conftest. Otherwise the topologies would call the model endpoint in + every test that starts a runtime, and the scripted models of the live tests would answer the probe. +5. **`browser_take_screenshot` moves from `vision` to `core`** (head's decision after a stop). The runner grants + `CORE_BROWSER_TOOL_NAMES`, so with the tool only in `vision` the runner's agents never got it. The factory emits + it only with image support, so an agent without image input sees the tools it saw before. The public constant + `CORE_BROWSER_TOOL_NAMES` grew by one name (15) and `VISION_BROWSER_TOOL_NAMES` lost it, so no two capabilities + share a tool. + +Rejected options: +- Passing `enable_read_image_multimodal=` to `create_browser_agent`: a `False` would override a model's + `supports_vision`, and the setting also switches the deep agent's `read_file`. +- Walking from the inner agent to the outer one: the inner agent holds no reference to it. +- Caching the verdict in the rail: the probe cache can change during the process; the rail reads it per call. + +Changed tests for decision 5: +- The two pinned assertions: the runner contract's core tuple, and the core size (with the screenshot now asserted + in core). The runner mirror is unchanged. +- Vision's tools and command pairs in the profile test. +- Five tests that compare an agent built without image support against the core names now leave the screenshot + out. + +Rejected for decision 5: the screenshot in both `core` and `vision`. The resolver and the allowlist de-duplicate +it, and three tests assume the entries share nothing. + +Verification: 6 new cases fail before the fix (the outer agent's verdict in the inner hooks, the awaited probe in +`Direct`, three probe verdicts on `DelegatedWorker`, `build_worker` with image support). The False cases pass +before and after. Full unit suite: 2495 passed, 122 skipped. The golden diff against the start of this fix is +empty. Live sets (Playwright MCP and browser-use conformance, helpers, assembly, agent rails, topology): 119 passed. +After decision 5: full unit suite 2495 passed, 122 skipped; golden diff still empty, the `vision` view included +(built without image support, it never showed the screenshot). No live run for decision 5: no driver code changed. + +Known leftovers: +1. The probe adds one model call per process before the first task. It is sent under `LlmObservationSuppression`; + whether the runner's usage parser counts it is not checked. +2. A model whose probe is inconclusive at the first build is probed again at the next build, never mid-task. + +### Image input: reruns + +The reruns with the image input fix found two problems. + +1. **No usage in the task logs.** The WebVoyager runner calls `ensure_started()` (the topology's `start`) before it + redirects stdout into the task's `agent_log.txt`. With the probe awaited in `start`, the first model call of the + process happened there and bound the model logger to the real stdout. In `refactor-mcp\results\20261012-2048_adhoc_…` + and `refactor-bu\results\20261012-2105_adhoc_…` every task log has 0 `CompletionUsage` lines (they went to + `shardN\stdout.log`), so cost and usage parsing and the budget guard saw $0. +2. **Worker images rejected.** With image support, the MCP worker's `browser_take_screenshot` result made the next + model call fail with provider 400s ("Invalid base64 image_url", "You uploaded an unsupported image") in 9 of 51 + tasks. + +Decisions: +1. **No model call in `start`.** `Direct.run` awaits the probe before `Runner.run_agent`. The agent built in `start` + needs no verdict, because the rail reads the probe cache per model call. +2. **The delegated worker has no image input again and does not probe.** `build_worker` keeps `image_support` + (default `False`) for the day the payload is fixed. + +Rejected options: +- Fixing the MCP screenshot payload in the same change: it needs its own investigation (leftover 1). + +Verification: 2 new cases fail before the fix (no probe in `Direct.start`, and the probe before the agent in `run`; +the worker probes nothing and has no image support). The live topology and agent-rails tests still expected the full +core names from agents without image input (missed when the screenshot moved into core, because that change had no +live run) and now leave the screenshot out. Full unit suite: 2493 passed, 122 skipped. The golden diff against +`12c332832` is empty. Live sets: 119 passed. + +Known leftovers: +1. **The MCP screenshot payload.** Find out what the driver puts in `Screenshot.png` and in the rendered image block + for the screenshot result of `@playwright/mcp@0.0.78`, and why providers reject it ("Invalid base64 image_url", + "You uploaded an unsupported image", 9 of 51 tasks). Until then the worker gets no image input. +2. The deep agent schedules its own background probe when `create_browser_agent` builds it in `start`. If that probe + has not finished when `Direct.run` awaits the probe, the probe request is sent twice. + +### Task rules in the prompt (both topologies) + +Eleven failures came from the model's choices, not from the harness: a date without a year read as this year +(Booking--9), a filter the task named left out (Booking--17 property type, ESPN--43 class year), "today/latest" +answered from memory (BBC--6, Apple--8, ESPN--19/25, Huggingface--19), the named source not opened (Google +Search--16, Reddit), a UI step the task named skipped (Google Search--33), one reading of an ambiguous question +(Wolfram Alpha--13), a misread entity not re-queried (Wolfram Alpha--30), answers hedged into failures, and 50 +calls of hand-built search URLs (Coursera--8). + +Decisions: +1. **One block, nine rules, today's date**, in `surfaces/browser/tools.py` with the other prompt text, en and cn. + No prompt gave the model a date before; the block gives the local date at render time. +2. **Both agents read the same block**: the worker prompt renders it (per task, as the prompt is re-rendered per + task), the main agent gets it as a rail prompt section like the image and date-picker guidance (per model call). +3. **Before the output contract** in the worker prompt, so the JSON instructions stay last. +4. **Goldens show ``**: the snapshot builder already replaces ISO dates; the worker-prompt hash pin freezes the + date instead. + +Rejected options: +- Putting the date in the task text by the runner: the rule must hold for every caller, not the benchmark only. +- A rail that rewrites year-less dates in the task: guesses the user's intent; the rule lets the model decide with + the date in hand. +- Separate blocks per topology: the same failures happened on both. + +Golden diff: every Playwright MCP view (`core`, the ten capability views, `runner`, `runner_direct`) and the +browser-use `runner_worker` view gain exactly the block in the system prompt, nothing else (one changed line per +file: the system prompt string). The worker-prompt hash pin moves to the new prompt with a fixed date. + +Verification: 13 new cases; full unit suite green after the regeneration (see the report); goldens regenerated with +`uv run python -m tests.unit_tests.harness.tools.browser_move.snapshots.regenerate`. + +Known leftovers: +1. The main agent is built once per process; the date in its section is fresh per model call, the worker's per task. +2. Whether the rules change outcomes is for the rerun; they add about 230 tokens to every request. + +### Phase 10 fixes: extraction budget + +After the step on command records (above), eight browser-use tasks still ended `partial` on +`extraction_phase_budget_exhausted` (20/20) in the run at 12c332832: Google Flights--6/7, Google Search--14, +Google Map--40, ArXiv--11/41, Apple--14, ESPN--20. In the run at 109f6fccf four of them failed the same way +(Google Flights--6, Google Search--14, Google Map--40, ArXiv--11); the other four passed. Playwright MCP's worker, +which has no phase budgets, passed all eight. + +Diagnosis. The call sequences were rebuilt from the shard `llm.log` (tool calls and results, offloaded results +read back from `work\context`) and replayed through `_consume_phase_budget` and `_record_phase_result`. For the +12c332832 run the replay denies the same call as the run in all eight tasks, so the accounting below is the +accounting the run had. Columns count the 20 charged extraction attempts: a read that added evidence (a new page +observation or probe result), a read that added nothing, a page script that clicks or sets a value. "After" is +the extraction count of the replay with the fix at the call the run was denied on. + +| Task | What the agent was doing | New evidence | Nothing new | Mutating script | After | +|---|---|---|---|---|---| +| Google Flights--6 | filling the form (route, dates, cabin), a script reading the inputs or options after each step | 14 | 3 | 3 | 7 | +| Google Flights--7 | the same form, a probe of the inputs after each field | 16 | 2 | 2 | 5 | +| Google Search--14 | Wikipedia and Transfermarkt profiles, one navigation and one read per page | 18 | 1 | 1 | 3 | +| Google Map--40 | a restaurant's reviews: reading tabs and menus, opening the sort menu | 12 | 4 | 4 | 9 | +| ArXiv--11 | three help pages, revisited, a different slice or pattern per read | 19 | 1 | 0 | 2 | +| ArXiv--41 | the store page and its share dialog | 14 | 4 | 2 | 7 | +| Apple--14 | the MacBook Pro configurator: option and price slices, radio clicks by script | 14 | 3 | 3 | 7 | +| ESPN--20 | stats tables: table addresses with filters, then the rows | 11 | 7 | 2 | 10 | + +The rule that spent the budget in every task is the same: each successful read in extraction cost one attempt +whether or not it added anything ("reads are never refunded", decision 2 of the step on command records). +Mutating page scripts counted as reads in addition (0 to 4 per task). No charge came from retries: denied calls +are never charged, and the failed clicks of these runs were charged to `form`, not extraction. So the budget was +not too low for the work; it counted the work. Without the reads that added evidence the tasks used 2 to 10 of +20. Apple--14 and ESPN--20 passed again at 109f6fccf with the same accounting; their failure at 12c332832 came +from longer read sequences in that run, not from the step on command records. + +Decisions: +1. **A read that adds evidence is free.** In the extraction phase a successful call whose result adds a + `structured_evidence` entry (a page observation from another source or with other text, a probe result) or a + newly covered field gets its attempt back in `_record_phase_result`, as an action with progress already did. + Reads that return what the agent already saw, failed reads and stalls still count; the deny stays before the + call. +2. **Only a charged call is refunded.** The rail notes per call whether `_consume_phase_budget` raised the phase + attempts (`charged` in the call's runtime state) and passes it to `_record_phase_result`, so a repeated page + script, which costs nothing, cannot take back an attempt it never paid. +3. **The budgets and the terminal stop stay.** 12/24/20/20 are unchanged and exhaustion still ends the task + `partial`: a run of reads that show nothing new is the loop the budget is for. + +Rejected options: +- Making exhaustion a warning and letting the agent continue to the step and time limits: the data shows wrong + charges, not a wrong stop; the warning would also drop the only bound on reads that show nothing new. +- A larger extraction budget, or one per task type or site: Google Search--14 and ArXiv--11 read 18 and 19 new + pages; any number would be the next one they reach. +- Not charging retries of commands that never dispatched: no extraction charge in the eight tasks came from one. +- Moving mutating page scripts to `form`: they are 0 to 4 per task and fit the budget; reclassifying scripts by + their code is a separate change (leftover 4 of the step on command records). +- Charging after the call instead of refunding: the same reason as before, the budget check must stay before the + dispatch. +- Dropping the per-code script rule now that new evidence is free: it would charge polling a script whose output + has not changed yet, and its test is a contract of the previous step. + +Verification: five new cases in `test_browser_phase_accounting.py`. Three replays through `FakeDriver` modelled on +Google Flights--6 (seven fields, a read of the inputs, a read of the options and a clicking script after each), +Google Search--14 (21 profile pages, one read each) and Apple--14 (17 option slices, a snapshot, three clicking +scripts) exhaust extraction 20/20 at the pre-fix code and end at 7, 0 and 3 attempts with the fix. The guard that +22 reads returning the same text still exhaust extraction passes before and after; the case that a repeated free +script earns no refund fails before (2 attempts, the first read was charged). The loop and budget cases of +`test_browser_runtime_rail.py` and `test_browser_phase_accounting.py` stay green. Browser_move unit suite and the +terminal stream test: 2498 passed, 122 skipped. Live sets: both sets in one run, 119 passed. Goldens unchanged. + +Known leftovers: +1. Whether the eight tasks now finish is for the rerun. +2. A screenshot's page observation is the constant text "1 image(s) attached as multimodal input.", so a + screenshot adds evidence only when the address changed. The 109f6fccf runs, the first with the screenshot in + the core capability, spent 10 to 12 attempts on screenshots in Google Flights--6 and Google Map--40; the replay + (which only moves the address on navigations) ends Google Map--40 at 16/20. Pages that change without a new + address can still spend the budget on screenshots. +3. The replay does not model click progress, so it cannot tell the form budget: replaying the 109f6fccf Google + Flights--6 run reaches form 24/24 at call 52, with every click charged. The real run refunds clicks that + changed the page. +4. A sequence of reads whose text always changes (a clock, a counter) is refunded; it is bounded by the semantic + loop detection and the step and time limits, not by the extraction budget. +5. The evidence store keeps the last 20 entries; a read repeating text older than that counts as new. +6. After a first navigation that lands on another host (the consent page), the refund leaves no attempts, and + the next action is denied once with "Navigate to it directly" (call 2 of Google Flights--6/7, Google Map--40, + ArXiv--11). One wasted call per task; unchanged here. + +## Phase 11: dependency direction + +Two reviews of 2026-10-12 and an import scan of the package found edges that point the wrong way: drivers importing +`runtime/` and `provisioning/`, the core and a surface helper importing `drivers/middleware.py` for a helper that +only masks contract types, provisioning importing the browser surface's settings, and a module at the package root +that belonged to no layer. Eight `refactor` commits move types and helpers to the layer that owns them; one `fix` +triple narrows the MCP driver's lost-connection mapping. No driver → `runtime/` edge is left. + +### Cross-layer imports + +| edge | before | after | +|---|---|---| +| drivers → `runtime/` | 5 files: the MCP driver (`runtime.config`), both MCP clients (`browser_logging`, `utils.parsing`), the browser-use driver (`browser_logging`, `utils.upload_paths`) and transport (`browser_logging`, `config.process_environ`, `utils.env`) | 0: the upload path helper is `drivers/browser_use/upload_paths.py` | +| drivers → `provisioning/` (`ProvisionedDevice`) | 2 | 0: `contract/device.py` | +| `runtime/` → `drivers/middleware.py` (`masked`) | `tool_factory.py` | 0: `contract/masking.py` | +| `surfaces/` → `drivers/middleware.py` (`masked`) | `helpers/batch.py` | 0 | +| `provisioning/` → `surfaces/browser/config.py` | `managed_browser.py` | 0: `ManagedBrowserSettings` protocol | +| `surfaces/` → package root (`offload_recall`) | `helpers/recall.py` | 0: `surfaces/browser/offload_recall.py` | +| `runtime/` → `drivers/` | `assembly.py` (registry, middleware) | `assembly.py`; `config.py` inside a function (the settings facade) | +| drivers → surface `commands` / `context` | 6 | 6 (allowed) | + +### Decisions + +1. **The Playwright MCP server config is the driver's.** `drivers/playwright_mcp/config.py` builds it from the + browser settings and keyword values (`cdp_url`, `launch_args`, `key`, `required_capability`); it does not know + `BrowserInstanceConfig`. `runtime/config.py` keeps `playwright_mcp_config`, `build_playwright_mcp_config` and + `RuntimeSettings.mcp_cfg` for their callers (the WebVoyager runner, the subagent builder) and maps the instance + onto the builder, importing it at call time; the import-time probe of `test_assembly.py` stays green. +2. **The server's `--caps` groups are a driver tuple.** The builder read them from the browser surface's catalog + through the registry; a driver may not import the catalog, and building the whole surface bundle from a driver + factory to read seven names is the wrong direction. `SERVER_CAPABILITIES` lists the seven groups in catalog + order; the existing test that compares the `--caps` argument with the catalog's non-policy entries holds the two + together. The settings-facade test no longer asserts that the builder calls the registry. +3. **The settings facade is the second named importer of `drivers/`.** The task names `assembly.py` as the only + exception and asks `config.py` to call the driver lazily; the rule lists both by name, `config.py` for imports + inside function bodies only. +4. **Drivers log through the core logger.** The core logger already sends every line written inside the browser + log context to `openjiuwen.browser_agent` (`default_impl._emit`), which is how the model client's lines reach the + browser agent log; the drivers' lines take the same route instead of a second one. Outside the context (the + WebVoyager Direct agent runs with `dedicated_log=False`) they go to the process log, which that runner captures; + with the log file off they no longer go to both. +5. **The sidecar environment comes from the settings.** `BrowserSettings.process_env` is the mapping the settings + were parsed from (hidden from `repr` and equality, so it never prints with the settings), passed by the factory + through `BrowserUseConfig.environ`; the transport still keeps only OS and `BROWSER_USE_*` names. The transport + has its own two-line case-insensitive lookup; `env_value` stays in `runtime/utils/env.py` for its one other user, + the surface settings. +6. **`sanitize_json_schema` lives in the stdio client**, its only caller; tests import it from there. +7. **`offload_recall.py` moves to `surfaces/browser/`** (its only user is the browser recall helper) as a pure + rename; the old path is a module alias that replaces itself in `sys.modules` with the moved module and warns + once, like `playwright_runtime/`. +8. **The MCP driver asks its client what a transport failure is.** Both clients already retry on the same markers; + the predicate is public as `is_transport_error`, and every other exception is + `DriverError(dispatched=False)`. The reviews ask for "not dispatched"; a result that fails to validate after the + server ran the tool would also land there, and nothing re-sends a command, so the flag only keeps the session + from restarting. + +9. **Upload paths stay a driver concern (Aleh, option c).** The task moved the resolution to the browser surface, + but on Playwright MCP the upload and drop commands come from the driver's native decoder and reach the driver + without passing surface code; the policy rail can only deny. `runtime/utils/upload_paths.py` moves to + `drivers/browser_use/upload_paths.py`, its only package user, by a pure rename. Relative upload names are + resolved under the upload root on browser-use only; on Playwright MCP the server resolves against its own roots, + as before. Confinement of upload paths is the surface policy rule planned with the safety fixes. + +### Rejected options + +- A contract hook that lets a surface replace a command before the page-state check, or a surface rail that + rewrites the `paths` argument before the decoder: both put path rewriting in the surface for one rule that a + policy check can enforce; Aleh chose to keep resolution in the driver. +- Building the browser surface bundle in the MCP driver factory to read the catalog: a driver → surface edge the + rules forbid, and a full profile build per driver. +- Routing driver logging through `get_browser_agent_logger()` passed in by the factory, or a child of the stdlib + logger `openjiuwen.browser_agent`: the first changes the factory signature, the second bypasses core logging + (`logging.getLogger` is not allowed in library code). +- A pre-filtered sidecar environment in the surface settings: the credential filter would move out of the + transport that owns the rule, or exist twice. +- Narrowing `ConnectionLost` by exception type (anyio, httpx, `McpError(CONNECTION_CLOSED)`): a third definition + next to the two the clients already retry on. + +### Verification + +New architecture rules in `architecture/test_import_boundaries.py`: drivers import only the contract, their own +package, the middleware and surface `commands`/`context` (empty allowlist); provisioning imports +only the contract; `runtime/` and `surfaces/` import `drivers/` only from `assembly.py` (and `config.py` lazily); +the package root holds only the four known modules. MCP driver: a `ValueError` from the client is a +`DriverError(dispatched=False)` (fails before the fix: `ConnectionLost`), a "broken pipe" is still `ConnectionLost`. +The old offload path resolves to the moved module and warns. Browser_move unit suite and the terminal stream test: +2504 passed, 122 skipped (also after the upload-path move). Goldens unchanged against `77091ac9e`. Live sets not run in this step (one gate run at +the end of the phase). + +### Known leftovers + +1. Relative upload names resolve under the upload root on browser-use only; the same name on Playwright MCP is + resolved by the server against its working directory. +2. The browser log context decides where driver lines go; a driver used outside an agent invoke logs to the process + log. + +## Phase 11: selection, offering, optional hooks + +Three `fix` triples from the reviews (I1, P1; P5; P3, I10, D19): a registered driver is selectable by name, a tool is +offered only when the driver can run what it needs, and a surface that leaves the browser hooks empty runs through the +production facade. + +### Any registered driver by name (M1) + +`CoreSettings` accepted only `playwright_mcp` and `browser_use` for `BROWSER_DRIVER`, so `register_driver` from another +package worked but the setting could not choose it; the fake-surface test only reached its driver through +`CoreSettings(driver=...)` in Python. + +Decisions: +1. **The registry is the only check.** `_driver` keeps legacy provisioning values mapped to `DEFAULT_DRIVER` and returns + any other name; `DRIVERS.create` raises `unknown driver ''; registered: ` when the stack is built. The + `DRIVERS` tuple of `config.py` is gone (`DEFAULT_DRIVER` replaces its first entry; nothing else read it). +2. **The error comes after provisioning.** The stack is built from the provisioned device, so an unknown name provisions + first and releases the device on the failure (existing behaviour of `assemble_driver`). Checking the name in the + parser would need the registry at parse time, when the built-in registrations may not have happened yet (the same + reason the topology default is resolved late). + +Rejected options: +- Validating against `DRIVERS.names()` in `CoreSettings.from_env`: the parser runs before `drivers/` is imported, so the + built-in names would be missing; importing `drivers/` from `config.py` adds a core → driver edge. + +Verification: `test_assembly.py` selects a driver registered in a test registry through +`CoreSettings.from_env({"BROWSER_DRIVER": "fake_driver"})` and `assemble_driver` (fails before: `ValueError` at parse), +and an unregistered name fails at build with the registered names and a released device; +`test_settings_facades.py`'s `BROWSER_DRIVER=cua` builds the runtime settings and fails at `assemble_driver` with +`unknown driver 'cua'; registered: browser_use, playwright_mcp`; `test_core_settings.py` keeps any name as given. +Unit suite: 2506 passed, 122 skipped. + +### Extra tools follow the driver (M5) + +Spec tools were offered only when the driver handles their commands (`_handled_kinds`), but `profile.extra_tools` were +always offered: a point-only driver on the browser surface would get two probe tools whose every call fails +`Unsupported`. + +Decisions: +1. **`ToolFlags.requires`, a tuple of command types.** The tool declares what it sends; the factory checks the types + against `DriverInfo.actions`, the same declaration the spec tools use, and logs the skip with the missing names. + Ref kinds are not checked: the probes and the batch send `RunScript` and refs they read from the page state. +2. **The batch requires every command its steps build.** A driver without one of them drops the whole tool rather + than offering a batch whose some steps fail; a per-step schema would change the tool card the goldens pin. +3. **Find and recall require nothing**: find only observes (every driver observes), recall reads files. +4. **Custom actions are not covered.** They are not extra tools: the facade registers them on its action controller + through `profile.custom_actions`; the step on optional hooks (M4) makes that hook optional instead. + +Rejected options: +- Deriving the requirement by calling the tool with a recording driver: tools build commands from page content, so a + dry run cannot see every path. +- Requiring commands on the tool instance after construction: the factory would build tools it then throws away, and + the constructor of some reads the workspace. + +Verification: a toy extra tool requiring `RunScript` is left out for a driver without it (fails before: offered); on +the browser profile a driver without `RunScript` gets only find and recall, with it all five extra tools. The flag +tests list each helper's requirements. Both built-in drivers handle every required command: goldens unchanged. +Unit suite: 2508 passed, 122 skipped. + +### Optional surface hooks (M4) + +The facade and the worker build called six profile hooks and the worker prompt without checking them; a non-browser +surface had to fake them to reach the production path. `SurfaceProfile.native_tools` had no reader, and the hooks +were typed `Any` / `Callable[..., Any]`. + +Decisions: +1. **Each hook is optional where it is called**, with the answer a caller can act on: no custom actions; metadata + `{}` without an error (the observation is still the state); a failure envelope from `probe_for_policy` in the + shape of its own failures, naming the surface; `False` from `activate_page`, its existing "could not" value. +2. **A null semantic tracker in the core** (`_NoSemanticState` in `runtime/session.py`) instead of a check at each of + the seven places that use the tracker; it never reports observable progress, so loop and replan logic that reads + it stays inert. +3. **A neutral worker prompt in the core** (`default_worker_prompt` in `runtime/agents.py`): the driver's display + name, the tool names when the loop re-renders it, the output folders, "answer with the result". It names no + surface, so the browser-word ratchet does not grow. +4. **`Protocol` types in the contract** for the hooks and a `ContextFactory` for `extra_tools` and `rails`; the + context argument stays `Any`, because its type (`ToolContext`) lives in `runtime/` and the contract imports only + itself. `semantic_tracker` keeps `Callable[[], Any]`: typing the tracker would put its interface into the + contract (rejected in Phase 9 PR e). + +Rejected options: +- A no-op implementation per hook in the contract: puts browser behaviour (metadata keys, tracker methods) into the + contract, as before. +- Raising `Unsupported` from `probe_for_policy` / `activate_page`: their callers (the decision policy, the runtime + rail) handle the failure envelope and `False` today, not an exception. + +Verification: a browser profile with all six hooks `None` registers no custom actions, answers the probe and the +activation without a driver call, and captures an observation with no observable progress; a worker built without +`worker_prompt` gets the neutral prompt (no "browser" in it). All fail before the fix (`'NoneType' object is not +callable`). Unit suite: 2512 passed, 122 skipped. Goldens unchanged. + +Known leftovers: +1. **The fake surface does not run through the facade.** `BrowserAgentRuntime` and `BrowserService` read browser + settings no profile field covers: `BrowserService.__init__` resolves `mcp_cwd` (screenshot and artifact folders) + and the lifecycle identity from `profile_name`, `managed_args` and the provisioning; `_open_session` does + `replace(settings, mcp_server=self.mcp_cfg, mcp_cwd=...)`; the facade takes a required `mcp_cfg` and defaults + its settings to `browser_surface_settings()` whatever `core.surface` is. With the screen surface's settings the + service fails at construction (`'ScreenSettings' object has no attribute 'mcp_cwd'`). Decided with Aleh (option + c): a surface-neutral service is out of scope for Phase 11; M4 is done with the hook-level tests, and the + fake-surface facade run is dropped from its done-when. The fake surface keeps running through `build_agent`. +2. The Direct topology builds its agent with `create_browser_agent`, which imports `BrowserRuntimeRail`, the browser + capability catalog and the browser context processors by name; only the delegated worker takes its prompt and + tools from the profile. `main_agent_answer` reads `surface.url` / `surface.title` (P11-T3 territory). +3. Custom actions are registered through the facade's controller, not the tool factory, so M5's `requires` does not + reach them. + +## Phase 11: surface-neutral page state + +`runtime/page_state.py` (`BrowserPageState`) mixed the generation and reference registry the core needs with browser +logic: card evidence fields (price, rating, shop, temperature...), captcha/login blocker patterns, card and probe +targets, URL/title tracking, and a move detected through `getattr(surface, "url" / "title" / "tabs")`. A `refactor` +commit splits it; one `fix` triple moves navigation detection onto a neutral contract member. The member map is in +the step report; S_20 "Page state" lists the result. + +Decisions: +1. **Subclass, chosen by the profile.** `PageState` (core) and `BrowserPageState(PageState)` (surface), with four + hooks and three class attributes. A component the core would call would need the same hooks and an extra object + per call site; the browser helpers already take a page state and keep their type. +2. **`SurfaceProfile.page_state`, a zero-argument factory.** The task names the profile or the registry as the way; + the profile is where the rest of the surface's choices live. `assembly.new_page_state` builds it; `BrowserService` + makes it at first use from the surface bundle (it was made in the facade's constructor, before any surface), + and the facade reads the service's. +3. **`SurfaceContext.identity` is the one contract change** (confirmed proposal): a property, `""` on the base; + `BrowserContext.identity` is `url`. The core compares two non-empty identities, today's URL rule. The tab list and + the title are not contract members: the browser page state reads them from its own context type in + `_note_surface`. +4. **The browser keeps its messages and key order.** The core's stale-generation and unknown-ref hints are neutral + ("Observe again."); the browser subclass sets the old texts, and its `export()` builds the old dict, so tool + outputs and goldens do not move. +5. **`runtime.page_state.BrowserPageState` is not aliased.** It was not exported from `runtime`, a lazy alias in the + core module would import the surface, and every package and test import moved to + `surfaces/browser/page_state.py`. `CARD_EVIDENCE_FIELDS` moved with it. + +Rejected options: +- A second contract member for the context list (`SurfaceContext.contexts`): the core uses the list only as the + `RefResolver.contexts` a decoder reads, which only the surface's own decoders need; the subclass fills it. +- Keeping the URL in the core under a neutral name only: the move rule needs a value the surface defines, which is + what `identity` is. + +Verification: `test_page_state_identity.py`: a non-browser context with its own identity advances the core +generation on a change (fails before), not on the same or an empty identity, also through an observation; the +browser context's identity is its URL. The fake screen surface gets `ScreenContext` with its own identity and runs +on the core `PageState`. A guard keeps `url`, `title`, `tabs` and card field names out of `runtime/page_state.py`; +its browser-word allowlist entry is gone. Every existing page-state, batch, probe and facade test passes with the +import moved. Unit suite: 2519 passed, 122 skipped. Goldens unchanged against `77091ac9e`. + +Known leftovers: +1. The facade (`runtime/session.py`) still calls browser members on its page state (`set_requested_fields`, + `field_coverage`, `selector_primary_links`, `refresh_target_id`, `recovery_candidates`) and reads + `surface.url` / `surface.title` in `main_agent_answer`; with option (c) of the hook step the facade stays + browser-shaped. +2. The core `Target` still carries `selector` (a CSS locator, a contract `Locator` kind) and the source name `ax`. + +## Phase 11: safety and hygiene + +The safety and compatibility items of the reviews (C2–C5, S1, S2), one triple per logic change. + +### Upload confinement (C2) + +Any readable host file could reach a page: the model chose the paths, browser-use accepted any absolute path, and the +Playwright MCP driver copied files from anywhere into the server's folder, past the server's own root check. + +Decisions: +1. **A browser `PolicyRule` on `UploadFile` and `Drop`**, the mechanism the clipboard rules use: it runs before the + middleware on both dialects, and the batch and the custom upload action call the same rules, so a denied path + reaches neither driver nor the MCP staging copy. +2. **Allowed folders: the upload root and the agent workspace; with neither, the process cwd.** Absolute paths are + resolved with symlinks; a relative name must stay inside the root it is joined to (see the next section). +3. **The workspace reaches the rule through the settings.** A `PolicyRule` sees only the command and the context; + the profile builds its rules from its settings, and `BrowserSettings.workspace_root` is filled by the runtime + facade, the one place that knows both the workspace and the settings. The facade checks for the field by name, + so other settings pass unchanged. +4. **Relative names resolve on browser-use only** (decision 9 of the dependency step): the rule judges them under + the configured roots for both dialects; Playwright MCP keeps resolving them against its working directory. + +Rejected options: +- A surface rail with the `ToolContext` (it has the workspace): the policy rail exists for exactly this check, and + a second deny path would differ in messages and in the batch. +- Leaving the staging copy as the boundary: it runs after the decision to upload. + +Verification: the rule's cases (inside, outside, relative climbing out, `~`, quoted, no folders → cwd, a symlink +out of the root, skipped here because the account cannot create symlinks), the policy rail on both dialects for +upload and drop, `set_input_files` with an outside path (no command sent), the profile and the facade's workspace. +Unit suite: 2530 passed, 123 skipped. + +### No agent scaffolding in the test tree (C3) + +Running the suite from the test folder left `AGENT.md`, `HEARTBEAT.md`, `IDENTITY.md`, `SOUL.md`, `USER.md` and +`agents/`, `context/`, `memory/`, `messages/`, `skills/`, `todo/` there, and `logs/browser_agent.log` in every working +directory. Running each agent-building file from an empty directory showed two writers: the fake-surface run +(`build_agent` without a workspace) and the Direct-topology test whose session cannot start. Both pass a tmp +workspace now. The browser_move `conftest.py` sets `OPENJIUWEN_BROWSER_AGENT_LOG_FILE` to a tmp file before its +first package import (the agent log is configured from the process environment at its first use, which can be at +import), and a `pytest_sessionfinish` guard fails the run, naming the files, when `AGENT.md` or `SOUL.md` appears +under the test root. The stray untracked files were deleted after the guard was in place. Left as is: pytest's own +`report/index.html` (the project's `addopts`) and the facade tests' `screenshots/` folder under the runtime working +directory. + +### Old alias paths of the moved browser modules (C4) + +The nine modules that moved from `runtime/` to `surfaces/browser/` (`probes`, `probe_semantics`, `semantic_state`, +`browser_working_context` and its processor and rail, `browser_state_context_processor`, `site_profiles`, +`status_logging`) did not import under `playwright_runtime.`. The alias package's `_MOVED` map now serves +each as the moved module object; `probes` maps to the new package, whose `__init__` exports the three builders the +old module had. `playwright_runtime.page_state` stays the core module (the shim's identity rule for every module of +`runtime/`), so `BrowserPageState` under that path is a documented break for the release notes. +Verification: the shim test lists the nine (fail before: `ModuleNotFoundError`). Unit suite: 2540 passed, 123 +skipped. + +### Screenshots the providers rejected (C5) + +The 109f6fccf Playwright MCP run lost 9 of 51 tasks to "You uploaded an unsupported image" (OpenAI/Azure) and +"Invalid base64 image_url"; worker images were switched off as a precaution. The errors are in the judge's +verdict (`verifier_result.judge`), not in the agent's run, and in all nine the worker's answer had +`"screenshot": "screenshots/.png"`, a relative path that named no file (the model wrote the path; the +server saves a named screenshot under its working directory, where the service looks, and nothing was there). The +service passed an unloadable path through unchanged, and the runner sent that string to the judge model as an +image. The worker's own tool images were never at fault: `@playwright/mcp@0.0.78` returns the screenshot as a +base64 PNG block with `mimeType: image/png` (scaled to fit 1568 px, still PNG), `result._screenshot` keeps the bytes +in `Screenshot.png`, and the worker sends them as a `data:image/png` URL that decodes to the same PNG (checked with +the recorded fixture). + +Decisions: +1. **The answer's screenshot is an image or nothing**: http(s) links pass, data URLs pass when their bytes are an + image, local files become data URLs with the MIME type sniffed from the bytes (not the extension), everything + else is `None`. +2. **Worker images are on again, probed at the first run** like `Direct`: `start` still makes no model call; on a + yes the first run rebuilds the worker with image support. +3. **The rebuild keeps the caller's removals.** The runner removes the helper tools from the worker that + `ensure_started` builds; the strategy records the tools it built and removes from the rebuilt worker whatever + the caller removed (a follow-up fix after the first version restored the helpers). + +Rejected options: +- Building the worker with images at `start` from a cached verdict: each WebVoyager task runs in a fresh + process, so the cache is empty at `start`. +- Teaching the runner to check the screenshot: the runner is outside this repository, and our answer should not + carry a value it cannot load. + +Verification: `test_answer_screenshot.py` (the runs' missing paths are dropped; fail before), the fixture-based +worker image test in `test_build_worker.py`, the first-run probe and rebuild cases and the kept removals in +`test_topology.py`. Unit suite: 2557 passed, 123 skipped. Goldens unchanged. Live: the gate run at the end of the +phase; whether judge errors disappear is for the next WebVoyager run. + +### The tool timeout bounds the driver stack again (S1) + +`BROWSER_TIMEOUT_S` / `PLAYWRIGHT_TOOL_TIMEOUT_S` bounded only the delegated worker's tool calls; the stack used the +browser profile's 240 s for every effect, so on the Direct topology the setting did nothing. The composition root +now takes, per effect, the smaller of the profile's bound and `CoreSettings.tool_timeout_s`. A cap and not a +replacement: a surface that needs a shorter bound for one effect keeps it, and an effect the profile leaves unbounded +stays unbounded (no surface does that today). With the defaults (240 s both) nothing changes. Verification: a 3600 s +profile bound under a 0.05 s core timeout times out at 0.05 s and the reverse (fails before). Unit suite: 2559 +passed, 123 skipped. + +### Hygiene (S2) + +- The three modules moved to `surfaces/browser/` with their old import order (`semantic/state.py`, + `site_profiles.py`, `status_logging.py`, ruff I001) are sorted. +- S_20 loses its DRAFT banner: every section it announced has landed. +- Tests no longer name review phases (`Phase 10`, `Phase 9 PR d`); the browser-use golden keeps the commit it was + taken from as data, and its test checks the value is a commit hash instead of pinning it. +- `SidecarTransport` keeps the stderr drain task it starts (a task with no reference can be collected while it waits + for a line) and cancels it when it terminates the sidecar; a test starts a transport over a stderr that never ends + and checks the task is cancelled by `_terminate()` (fix → test → docs). + +Verification: unit suite 2560 passed, 123 skipped; goldens unchanged against `77091ac9e`; ruff clean on the touched +files except findings present before the phase (16 ASYNC109 and 3 F541 in the MCP clients, one E501 in +`transport.py`, one ASYNC240 in `helpers/custom.py`). + +### Relative upload names judged per root + +The first version of the rule joined a relative name to every root and accepted it when any joined path lay inside any +root. With the workspace inside the upload root (upload root `/home/u`, workspace `/home/u/ws`), `../x.txt` joined to +the workspace gives `/home/u/x.txt`, inside the upload root, so the rule allowed it, while browser-use joins the name to +the upload root and reads `/home/x.txt`, outside both. The rule now accepts a relative name only when the path joined +to a root lies inside that same root. + +Verification: a test with the workspace nested in the upload root denies `../x.txt` and allows `notes/../x.txt`; the +old rule allowed the first. Policy tests: 44 passed, 1 skipped (the symlink case). + +### Core imports that lead back into a surface + +The import-boundary rules looked only at imports inside the package. `Direct.start` in `runtime/topology.py` imports +`create_browser_agent` from `openjiuwen/harness/subagents/browser_agent.py`, and that module imports the browser +surface's runtime rail, capability catalog, state processor and working-context processor. The core thus depended on +the browser surface through a module the rules never read. + +Decision: a new rule reads every module outside the package that a core module imports and reports the import when +that module itself imports a concrete driver, surface or provisioner. The one hit is allowlisted with its reason; the +allowlist may only shrink, so a second such path fails the test. The edge stays until a surface-neutral runner exists +(the facade, the service and `Direct` are the browser runner, decision of step 2). + +Rejected: following the outside modules transitively. `openjiuwen.harness.deep_agent` imports `create_browser_agent` +inside its factory for named subagents (`browser_agent`, `browser_runtime`), so a transitive scan reports +`assembly.py` and every other core module that builds a deep agent, although none of them asks for the browser +agent. + +Verification: the rule finds exactly `("runtime/topology.py", "openjiuwen.harness.subagents.browser_agent")`, which +imports `surfaces.browser.capabilities`, `rails.runtime_rail`, `state_processor` and +`working_context.processor`; `deep_agent` and `factory` import none. Architecture tests: 24 passed. + +### Phase 11 live gate + +Both live sets once, at the end of the phase (`BROWSER_MOVE_LIVE_MCP=1`, `BROWSER_USE_SIDECAR_PYTHON` set): +`test_conformance_playwright_mcp_live.py`, `test_conformance_browser_use_live.py` and `test_topology_live.py`: +100 passed, none skipped, in 284 s. + +## Removal of the old import paths + +Decision (Aleh, 2026-10-13): the `playwright_runtime` alias package and the root module `offload_recall.py` are removed +now instead of after one release. Callers import the new modules: `runtime.`, `surfaces.browser.`, +`provisioning.profiles`. + +Known effect: agent-eval's WebVoyager runner (`bench/webvoyager/run.py`, outside this repository) imports +`playwright_runtime.config`, `playwright_runtime.runtime` and `playwright_runtime.browser_capabilities`; it has to move +to `runtime.config`, `runtime.session` and `surfaces.browser.capabilities` before its next run. The runner contract test +now pins the new paths. No module in this repository imported the old paths. + +Rejected: keeping the alias for one more release (the runner is the only known caller and its owner updates it with +this change). + +Verification: the shim tests and the offload alias test are removed; the root-module rule lists `__init__.py` and +`playwright_runtime_mcp_server.py`. Unit suite: 2536 passed, 123 skipped. + +## Removal of the ManagedBrowserDriver alias + +The deprecated module attribute `ManagedBrowserDriver`, an alias of `ManagedBrowserProvisioner` with a +deprecation warning, is removed together with its tests; no module in the repository used it. Provisioning +tests: 2532 unit tests pass. + +## A surface-neutral runner + +### Background + +`runtime/` held two things: the surface-neutral core (the composition root, the driver session, the page state, +the tool factory, the rails, the topologies) and the browser runner: the facade `BrowserAgentRuntime`, +`BrowserService` with the worker loop, the service registry keyed by browser fields, the custom-action tools and +controller, the browser agent log, the runtime server's client config, the browser-only settings facades and the +demo CLI. The runner required a Playwright MCP server config, defaulted its settings to the browser's, read +`mcp_cwd`, `profile_name` and `managed_args` from them and called `replace` on them, so a phone or desktop surface +could only call `assembly.build_agent`: no sessions, no cancellation, no restart, no worker loop, no output folders. +This reverses the "surface-neutral service is out of scope" decision of the optional-hooks step (option c). + +### Decisions + +1. **Moves first, then the split.** Three `refactor` commits move the browser modules into + `surfaces/browser/runner/` (`tools.py`, `actions.py` with the base contract folded in, `logging.py`, + `server_config.py`, `settings.py` with the browser-only half of `runtime/config.py`, `main.py`); the core still + imported them until the split, and those edges were allowlisted for the duration of the move and removed by it. + The task tool imports the browser agent log at its call site, so importing the harness tools no longer loads the + browser surface (the import-isolation test holds this). +2. **The settings facade is a config module.** `playwright_mcp_config` reaches the driver's config builder lazily; + the architecture rules treat `surfaces//runner/settings.py` as a config module (process-environment reads + allowed, driver names not counted) and move the lazy driver-import exception from `runtime/config.py` to it. + `runtime_server_environ` / `runtime_server_child_env` stay in the core: they read env names by literal, which + the env-read rule allows in config modules only, and name no browser word. +3. **`AgentRuntime` and `AgentService` take neutral inputs only**: the core settings, the complete surface bundle, + the model client fields, `RunGuardrails` (renamed; the runner package keeps `BrowserRunGuardrails` as a plain + alias), the allowlist, the workspace, the language, `output_dir` and a plain identity string. No `mcp_cfg`, no + instance, no browser default, no `replace` on the settings, no `getattr` for browser fields. +4. **The facade subclasses the core runtime** instead of wrapping it. The topology and the main-agent builder + receive one object: `Direct.start` passes `runtime` to the agent builder, and the browser rails need the facade's + captures on that same object. A wrapper would need a back-reference from the core runtime to the facade; a + subclass needs none, and every neutral member is reachable under its own name. +5. **The registry key is a string.** `BrowserServiceIdentity` moves to the facade with a `key` (the JSON of its + fields) and `from_key`; `reset_managed_browser_runtime` matches the browser fields of the live facades and of + every registry identity with a preserved driver through `from_key`; `find_managed_driver_identities` becomes + `managed_driver_identities()`. +6. **The answer's context.** `AgentRuntime.run_task` adds `context: {"identity", "metadata"}` to the answer of + either topology after the run: the metadata hook needs the tool context, which the runtime owns, so the worker + loop cannot add it. The metadata is captured for a successful answer only (a cancelled or failed agent task + must not be followed by a driver call); the identity is read after the capture. The core's `main_agent_answer` + has no `page`; the facade's override adds it from the browser context, so the WebVoyager answer keeps its keys. +7. **The worker slot is `agent`.** The runner's helper removal reads `runtime._service.agent` now + (`browser_agent` before); the removal itself is unchanged. +8. **The browser agent log is configured by the facade** (`BrowserAgentRuntime.__init__`, with its resolved core + settings) instead of the worker builder, which is core; `create_browser_agent` keeps its own call. +9. **The custom-action tools are registered by the facade** after `ensure_started` on the delegated topology, so + the topology no longer calls a browser method; `profile.custom_actions` is called by the facade too (its + argument is the browser runner's controller). +10. **The demo CLI moves with the settings** (`runner/main.py`): it builds a `BrowserAgentRuntime`, which is a + surface module now. + +### Rejected options + +- Composition (the facade holds an `AgentRuntime`): the topology would receive the neutral runtime and the + browser `main_agent` hook would need to find the facade from it (a module-level map or a back-reference). +- A registry keyed by any hashable identity object: the core would carry the browser identity type's shape in its + tests and docs; a string keeps it ignorant. +- Capturing the metadata on every answer: after `asyncio.wait_for` cancels the agent, a driver call could block up + to the effect timeout. +- Keeping `browser_agent` on the neutral service for the runner: a browser name on a core member. + +### Behaviour changes to know + +- The core settings are read at the facade's construction (`CoreSettings.from_process_env()` when not given), not + at the service's first use; a bad unrelated variable fails earlier. +- A generated session id is `session-` (`browser-` before). +- The browser agent log is configured when the facade is built, not when the worker is built. +- A successful answer costs one metadata probe (one driver script execution) on either topology. +- The `Direct` answer and the worker loop's answer carry `context`. + +### Verification + +- Step 2 (moves): unit suite `2532 passed, 123 skipped`; architecture tests 24 passed with the move allowlist; + goldens unchanged. +- Step 3 (the split): the facade tests pin the core answer without `page`, the browser answer with it and the + context the run adds (identity, metadata after a successful run only); the registry test pins that the identity + string round-trips; the service and registry tests build their service through the facade. Unit suite + `2573 passed, 123 skipped`; architecture tests 24 passed with `KNOWN_CORE_IMPORTS_OF_CONCRETE_MODULES` empty and + no `runtime/session.py` / `runtime/service.py` entry in the browser-word and driver-name allowlists; goldens + unchanged. + +### The main agent through the profile + +The one contract change of this feature: `SurfaceProfile.main_agent: MainAgentFactory | None`, a hook +`(runtime, model, *, language, max_iterations, workspace, capabilities) -> agent`. `Direct.start` calls it when +the profile sets it; without it the topology opens the session at `start` and builds the agent after the image +probe from the profile's parts through `assembly.agent_over`, which `build_agent` shares, with the probe's +verdict as `enable_read_image_multimodal` (no background probe). The browser profile sets the hook to +`surfaces/browser/runner/agent.main_agent`; that module holds what `create_browser_agent` did for an existing +runtime (the prompts, the card id, the sampling copy, `BrowserRuntimeRail`, the core rails, the context +processors, the task cleanup) as `build_main_agent`, and `create_browser_agent` keeps its signature, resolves +the capabilities, builds the facade when none is given and delegates. The core detour allowlist is empty. + +Decisions: +1. The hook receives the facade: the facade subclasses the core runtime (above), so the browser rails get the + object they need without a lookup. +2. The hook's `capabilities` are logged by the browser hook and not re-resolved: the runtime derived them from + its allowlist, and `create_browser_agent` keeps its own resolution and log for callers that start from + names. +3. Without a hook the agent is built in `run`, not `start`: `start` must make no model call and the parts + depend on the probe's verdict. + +Rejected: a hook that returns the agent's parts instead of the agent (the browser agent is a deep agent with +its own rails and processors; the parts are not enough); keeping the prompts in +`openjiuwen/harness/subagents/browser_agent.py` and importing them from the surface (a cycle: the subagent +module imports the surface). + +Verification: the topology tests patch the runner's builder and pin a profile without a hook (the session +opens at `start`, the agent is built from the parts after the probe with the core rails and the verdict); +the create-agent tests pin that the browser profile sets the hook and that the hook builds the browser agent +with the runtime's sampling and the task cleanup. Unit suite `2575 passed, 123 skipped`; architecture tests +24 passed with `KNOWN_CORE_DETOURS_INTO_CONCRETE_MODULES` empty; the browser prompts' two Playwright MCP +mentions are allowlisted in `surfaces/browser/runner/agent.py`; goldens unchanged. + +### Browser words out of the core rails + +The browser-word allowlist listed nine core modules. After the moves and the split seven are gone with their +modules; the rest are renamed where a name is all there is: the recovery message asks for "the final JSON object +required by your instructions" without listing the browser fields, the repeated-call note speaks of "the document's +title or address" and of calls that "do not change anything", the worker loop's prompt text ("the current logical +session", "the current state", "Recent tool activity", "Last location"), its error strings ("Worker did not return +valid JSON output", "unknown execution error") and the generated id prefix name no browser, the worker card is +"Worker that executes tasks using tools", and the default surface name is declared in +`surfaces/__init__.py` (`DEFAULT_SURFACE`) and imported by `CoreSettings`. No golden pins any of these texts; the +tests that pinned them follow. + +What stays, with the allowlist shrunk to it (`runtime/rails/repeated_call_guard.py: 1`, `runtime/worker.py: 37`): +the guard's no-op script atoms (`document.title`, `document.URL`, `location.href`: browser DOM reads the guard treats +as constants) and the worker loop's data: the worker's JSON `page` / `url` keys it reads and writes, the Playwright +restart and retry markers ("target page, context or browser has been closed", "page crashed", …) and the progress +record's `last_page` export. None is a name: each is browser knowledge the core applies, and moving it out needs +either a surface-provided definition (a contract member the task did not allow) or a behaviour change (dropping +the marker-driven retry, taking the location from the page state's identity instead of the worker's claim). +Decision (Aleh, 2026-10-15): they stay allowlisted; the allowlist may only shrink, and a later step that gives the +surface a way to provide them takes them out. + +Verification: unit suite `2537 passed, 123 skipped` (the suite of `browser_move` and the terminal-stream test); +architecture tests 24 passed; goldens unchanged. + +### The fake surface through the runtime + +The proof the feature exists for: `architecture/fake_surface/test_fake_surface_runtime.py` runs the screen surface's +scripted task (look, poke, look, click) through `AgentRuntime` on both topologies. The runtime takes +`CoreSettings(driver="screen_driver", surface="fake", topology=...)`, `SurfaceBundle(ScreenSettings, PROFILE)`, the +model client fields, `RunGuardrails`, a temporary workspace and output folder and the identity `screen:test`; the +test-scoped registries stand in for the composition root's defaults for the run, the scripted model is injected +at the topology's model builder (`Direct`) and at the worker builder (`DelegatedWorker`, whose worker is rebuilt +with image support on the first run), and the probe answers yes so the screenshot-only `look` tool is offered. +Checked: the answer (`ok`, `final`, `session_id`), `context == {"identity": "board", "metadata": {}}`, the tools +the model saw, the commands the driver saw with their generations, one provision and one release, the output +folder and the identity, and the core `PageState`. + +Rejected: a `registries` parameter on the runtime's constructor (an input the design did not list; patching the +composition root's defaults is what a test-scoped registry is for). + +Verification: unit suite `2547 passed, 123 skipped`; goldens unchanged. + +### Live gate + +Both live sets once, at the end (`BROWSER_MOVE_LIVE_MCP=1`, `BROWSER_USE_SIDECAR_PYTHON` set): +`test_conformance_playwright_mcp_live.py`, `test_conformance_browser_use_live.py` and `test_topology_live.py`: +`100 passed, 3 warnings in 239.36s (0:03:59)`, none skipped. Unit suite `2547 passed, 123 skipped`; architecture +tests 34 passed; goldens byte for byte unchanged against the start of the feature; ruff clean on every touched file +except findings present before it (`ASYNC250` in the demo CLI, `ASYNC240` in `helpers/custom.py`, `ASYNC109` in +`test_service.py`). + +### Known leftovers + +1. By decision (above), `runtime/worker.py` keeps its browser words: the worker's JSON `page` / `url` keys, the Playwright restart and + retry markers and the progress record's `last_page` export; the repeated-call guard its no-op script atoms. +## Review 1: upload authorization checks the path the driver sends + +The upload rule approved a relative name when `/` lay inside a root, but the drivers resolved the same +name elsewhere: browser-use tried the process working directory before the upload root, Playwright MCP joined it to +its own output directory. With `allowed/report.txt` and `outside/report.txt` and the process in `outside`, +authorizing `report.txt` dispatched `outside/report.txt`. + +Decisions: +1. **One resolution rule in the command module**, `commands.upload_paths(paths, upload_root)`: an absolute path as + given, a relative name only as `upload_root / name`, resolved with symlinks followed, and only while it stays + inside the upload root; a relative name without an upload root, or one that leaves the root, is a `ValueError` + whose message tells the model what to do. The policy rule calls it first and then judges every resolved path + against the upload root and the workspace. Both drivers call it with the `upload_root` of their surface settings + (`BrowserUseConfig.upload_root`; `PlaywrightMcpConfig` gained `upload_root`, filled by the factory) and never + consult the working directory or the output directory; a `ValueError` in a driver is `ActionResult(ok=False)`. +2. **The process directory is no longer a root for anything.** With neither upload root nor workspace configured + the rule denies every upload; the old fallback to the process cwd let a model upload whatever the process could + read from where it happened to run. +3. **The batch and the custom actions are untouched**: they keep calling `policy_denial`, which applies the rule. + +Rejected options: +- Keeping a per-driver resolver and only aligning the policy with browser-use: the Playwright MCP driver would + still resolve against its output directory, and two drivers would judge the same name differently. +- Resolving in the policy rail and rewriting the command's paths: the rail returns a verdict, not a command, and the + helpers that run commands without the rail would send the raw name again. + +Known effect: the sidecar's own `_normalize_upload_paths` still lists the process directory among its candidates +for a relative name, but the driver sends it absolute paths only, so that branch no longer runs. + +Verification: `upload_paths` alone (absolute, relative under the root with the cwd elsewhere, no root, climbing out, +quotes and `~`, a relative symlink out of the root); the policy rule and the rail on both dialects with the +reviewer's two files; the browser-use handler with a mocked transport (the sidecar receives `allowed/report.txt`; +without a root nothing is sent); the Playwright MCP handler with a fake session (the staged copy has the allowed +file's bytes; without a root nothing is sent, for upload and drop); `set_input_files` with the relative name and +with the climbing one. The batch has no upload or drop step, so it has no case. Unit suite: 2546 passed, 124 +skipped. Goldens unchanged. + +## Review 1: stored sessions under the old folder name + +`offload_session_dir_name` renamed the offload folder of an id with unsafe characters (readable part plus digest), +and `OffloadMessageBuffer` read the new name only; on Linux and macOS the old literal folder could exist from before +the renaming, and those messages were no longer found. + +Decision: a bounded legacy lookup in `OffloadMessageBuffer._filesystem_reload_paths`. When the safe name differs +from the id, the buffer also lists the old folder `_context/offload`, after the new one, but only for an +id without `/` or `\\` that is not `.` or `..`; any other id is looked up under the safe name only. Glob patterns +are escaped, so a literal id with glob characters never widens the search. New writes keep the new name. + +Rejected options: +- Migrating old folders on first read: a rename at read time is a side effect in a read path, and a concurrent + writer could be using the folder. +- Looking up the literal id unconditionally: an id with a separator would name a path outside the session's + folder, which is why the renaming exists. + +Verification: a fixture written under the old name is found; with both folders present the safe one wins; an id +with a separator, `../x`, `.` or `..` yields the safe path only; a path-safe id is looked up once. Context-engine +suite: 525 passed, 1 failed (`test_recall_rejects_chunk_symlink_escape`, WinError 1314: the account cannot create +symlinks; it fails the same way before this change). + +## Review 1: provisioning default and the Windows-only test + +The reviewer read the provisioning default as `managed`; the code gives `remote` (`_provisioning`: `implied` is +`remote` unless `PLAYWRIGHT_MCP_EXTENSION` is true), the behaviour before the driver work. The repository's +documents were checked for the claim: S_20 ("Settings", the `BROWSER_DRIVER` table), this document's settings +inventory and the `docs/en` and `docs/zh` browser-agent pages all state `remote`; no document names `managed` as +the default. S_20 "Provisioning" now states the default and the one implied override as an invariant, so the next +reader finds it where the provisioners are described and not only in a table row. (`.env.template` in the package +was not checked.) + +The test `test_default_chrome_user_data_dir_uses_the_given_local_app_data` (`test_settings_passing.py`) set +`os.name` to `nt` to simulate Windows; on macOS and Linux `pathlib` then tries to build a `WindowsPath` and fails. +The test now skips outside Windows (`platform.system() != "Windows"`, the file's existing pattern) and no longer +patches `os.name`: the helper reads the real `os.name`, so the simulation added nothing on Windows and broke +elsewhere. Rejected: rewriting with `PureWindowsPath`, which would test a path join the helper does not make. + +## Review 1: the clipboard rule enforced in the page + +`_CLIPBOARD_READ` is a regex over the script text. `() => { const c = navigator.clipboard; return c.readText(); }` +passes it and read the host clipboard. + +Decisions: +1. **Enforce in the page, keep the regex as early feedback.** `probes/clipboard_guard.js` runs in every document + before the page's scripts: `Clipboard.prototype.read` / `readText` (and `navigator.clipboard`'s own) become + functions that reject with the `CLIPBOARD_DENIED` text, `execCommand("paste")` returns `false`, on + `Document.prototype` and the document. The properties are frozen (`configurable: false`), so neither the page + nor a model script restores them. The regex stays: a denied call costs no round trip and the model gets the + reason before anything runs. +2. **Playwright MCP installs it through `--init-script`.** `npx -y @playwright/mcp@0.0.78 --help` lists + `--init-script ` ("path to JavaScript file to add as an initialization script. The script will be + evaluated in every page before any of the page's scripts"). The driver appends the argument when it starts a + stdio server; the kept config is unchanged. A `streamable-http` server is remote and gets none. +3. **browser-use installs it in the sidecar.** The driver reads the resource and sends it as `init_script` of the + `connect` request; the sidecar calls `Page.addScriptToEvaluateOnNewDocument` on every CDP session it obtains and + evaluates the script once on the document already open, because that document loaded before the registration. + One install per session id. +4. **A resource read, not an import.** Both drivers name the probes package as a string to + `importlib.resources.files`; the dependency rule of S_20 is unchanged and the architecture test sees no import. + `pyproject.toml` already ships `probes/*.js`. + +Rejected options: +- Widening the regex (aliases, bracket access, `Reflect.get`): a text rule never covers a language; the page is the + only place the rule can hold. +- `--grant-permissions` tuning or a clipboard permission denial: Chrome's permission is per origin and the managed + profile may already hold grants; a permission also does not cover `execCommand("paste")`. +- Installing the guard from the policy rail with a `RunScript` after every navigation: a page's own scripts run + first, and a document the rail never sees (a popup) is unguarded. + +Known leftover: on Playwright MCP a document already open when the server attaches over CDP (before the first +`Navigate`) runs no init script; the agent's first command is a navigation, and the sidecar path covers this case +by evaluating the script on attach. + +Verification: the resource is shipped and carries the policy message; with `node` the guard rejects the aliased +read with exactly `CLIPBOARD_DENIED`, answers `false` to `paste` and leaves `copy` alone; the Playwright MCP driver +starts its server with `--init-script ` (the stdio client's parameters recorded); the +browser-use driver's `connect` carries the script text; the sidecar installs it once per session and evaluates it +on the current document, nothing without a script. Live: `clipboard_read_case` and `paste_command_case` on both +drivers. The first live run failed every browser-use case at `connect` (`SyntaxError: Unexpected token ';'`): the +sidecar evaluates the guard through its expression wrapper, so the script is one expression without a trailing +semicolon. Live after that: Playwright MCP 54 passed; browser-use 48 passed (`BROWSER_MANAGED_PORT=9343`). + +## Review 1: `TypeText(slowly=True)` on browser-use + +The driver declared `TypeText` but raised `Unsupported` for `slowly=True`, so the two drivers were not +interchangeable on that variant (the batch worked around it by retrying without `slowly`). + +Decision: the sidecar's `type_text` takes `slowly` and, when set, sends one `Input.insertText` per character with +a 30 ms pause (`_SLOW_TYPING_DELAY_S`); the driver passes the flag and no longer refuses it. `clear` stays +independent of `slowly` on this driver (clear first when asked, then type), whereas Playwright MCP's `browser_type` +never clears when it types slowly; the batch sends `clear=False` with `slowly=True`, so it sees the same behaviour +on both. Six lines in the sidecar. + +Rejected: `Input.dispatchKeyEvent` per character, which needs a key layout for every character and differs for +IME input; `insertText` types what the text says. + +Verification: the driver sends `slowly` on the wire (true and false); the sidecar inserts `A`, `d`, `a` one at a +time with `slowly` and `Ada` at once without; the unsupported list no longer holds the variant; live +`slow_typing_case` on both drivers (same observed value, the form title). + +## Review 1: the recall tool reads the literal session folder too + +`BrowserOffloadRecallTool` resolved the offload folder through `offload_session_dir_name` only, so a session stored +under its literal id before ids were made path-safe could not be recalled, the same gap the message buffer had. +It now tries the path-safe folder first and the literal id's folder after it, under the same condition (the names +differ, the id holds no path separator and is not `.` or `..`); every candidate keeps the workspace, symlink and +parent checks. Verification: an artifact written under `user@host_context` is recalled for session `user@host`; +the existing colon-id test covers the path-safe name. + +## Review 1: the sidecar takes absolute upload paths only + +The driver resolves upload paths through `commands.upload_paths` and sends absolute paths, so the sidecar's own +resolution (the process working directory first, then `BROWSER_UPLOAD_ROOT`) was dead and read the environment +for nothing. `_normalize_upload_paths` now refuses a relative path and checks existence and readability of the +absolute ones; the environment read and its allowlist entry are gone. Verification: the sidecar, driver and upload +tests, 235 passed. + +## Default topology for browser-use: delegated worker + +### Background + +"Settings" decision 6 left the browser-use default at `direct` (the main agent on the facade) and deferred the +choice to the benchmark phase; Phase 6 made it the driver's registered default, so `BROWSER_DRIVER=browser_use` +without `BROWSER_TOPOLOGY` ran the main agent while Playwright MCP ran the worker. The WebVoyager runs of +2026-10-15 at eb5454b10 (`C:\Users\olegs\jiuwen-jev\runs\refactor-bu\results\20261015-*`, Sonnet 5.5 through +OpenRouter, `max_steps` 50, `timeout_s` 600) put browser-use at 511 of the 528 valid tasks against Playwright +MCP's 525, and the reruns below show the gap is the Direct topology's budgets, not the driver. + +### The measurement + +1. **23 browser-use failures rerun on Direct** (`20261015-1339_bu_critical`): judge SUCCESS 5 (21.7%), agent ok + 14, 9 answers the judge rejected; $11.02, $0.479 per task; 655 model calls; 10 780 744 of 13 202 332 input + tokens cached (82%). Seven runs ended `browser_task_incomplete`, four of them at the 50-call budget + (Amazon--15, Apple--16, Google Search--14, Huggingface--19), and two died in the stack (Google Map--40, + Wolfram Alpha--18); the per-site averages sit at 200-250 s for Amazon, Apple, Coursera, ESPN and Google Map, + against the 240 s simple-task deadline of `surfaces/browser/rails/runtime_rail.py`. +2. **The 14 left rerun on `BROWSER_TOPOLOGY=delegated_worker`** (`20261015-1410_bu_budget10`, 10 tasks, and + `20261015-1421_adhoc`, 4 tasks): judge SUCCESS 8 of 10 and 1 of 4, 9 of 14 together; $2.13 and $1.14, $0.213 + and $0.286 per task ($0.23 together, less than half of Direct's $0.479); 6 820 739 of 7 182 458 and 3 166 308 + of 3 380 803 input tokens cached (95% and 94%). The worker loop's retries and the worker budget finish the + tasks Direct abandoned (Amazon--15, Apple--2, ArXiv--11, ESPN--40, Google Map--40, Google Search--14, + Huggingface--19, Google Flights--6; Google Map--31 the judge accepts with `ok=False`). +3. **The 9 remaining** of the 23 fail on Playwright MCP too or are geo, site or judge cases: Apple--4 and + Apple--16 (configurations and prices the store no longer lists), Google Search--6 (a Trends range that does + not exist), Google Map--5 and Wolfram Alpha--18 (`ok=True` answers the judge rejects), and the four not + rerun (Amazon--38, Coursera--3, Wolfram Alpha--33: `ok=True` answers the judge rejected on Direct; + Google Search--15: one call, 6 s). + +### Decisions + +1. **`browser_use` registers `default_topology="delegated_worker"`.** One word in `drivers/__init__.py`; both + built-in drivers now spell the same default and `BROWSER_TOPOLOGY=direct` remains the explicit override + (`topology_for` is unchanged). Every caller of the facade's `ensure_started()` on browser-use now runs the + worker through the browser service, with the worker's prompt, retries and JSON progress. +2. **The golden builder derives the runner views from the registered default.** `RUNNER_VIEWS` maps a view to + its topology (`runner` to `None`, the default); `ROLES` maps a topology to the recorded role; `runner_topology` + resolves a view on a driver through `default_topology`, and `list_views(driver)` adds `runner_direct` when the + default is the worker, else `runner_worker`. So both drivers now record `runner` (the worker) and + `runner_direct` (the main agent through the runner), and `regenerate.py` rewrites the browser-use runner views + and removes a stale one, leaving the twelve recorded capability goldens alone. +3. **The browser-use `runner` golden gains the worker agent.** It was recorded at 857239db with no agents (the + old runner never ran browser-use) and compared with nothing; built by today's code it is the old + `runner_worker.json` byte for byte except `"view"`, and it is now compared like the MCP runner view. + +### Rejected options + +- **Raising the Direct budgets** (more steps, a longer simple-task deadline). The Direct reruns at 50 steps and + 240 s already cost $0.479 per task; the worker finishes the same tasks for half the money (cache hit 95% + against 82%), so a bigger Direct budget would buy the passes at the wrong price. + The budgets stay where they are for `BROWSER_TOPOLOGY=direct`. +- **Keeping `runner_worker.json` next to a rebuilt `runner.json`.** The two would be the same document under two + names; the "other topology" of browser-use is now `direct`. +- **Leaving `runner.json` as recorded.** A golden that claims the runner's default view and holds no agents pins + nothing; the well-formedness check over recorded goldens now covers the twelve capability views only. + +### Behaviour changes to know + +- `default_topology("browser_use")` is `"delegated_worker"`; `topology_for(CoreSettings(driver="browser_use"))` + is `DelegatedWorker`. `BROWSER_TOPOLOGY=direct` gives `Direct` on either driver. +- With the variable unset, browser-use opens the session at `ensure_started()` and puts the worker in + `service.agent`; the main agent on the facade needs the explicit `direct`. +- `builder.RUNNER_VIEWS` values are topology names or `None`; `builder.ROLES` and `builder.runner_topology` are + new; `list_views("browser_use")` ends with `runner`, `runner_direct`. + +### Golden status + +- `golden/browser_use/runner.json`: rebuilt (role `worker`, the 14 unprefixed core tools, the worker prompt saying + "browser-use tools"); equal to the former `runner_worker.json` except `"view"`. +- `golden/browser_use/runner_direct.json`: new (role `main`, the 18 tools of the MCP `runner_direct` view with + unprefixed names, the same system prompt as `golden/playwright_mcp/runner_direct.json`). +- `golden/browser_use/runner_worker.json`: removed. +- `golden/playwright_mcp/`: "unchanged" for all thirteen views; the files `regenerate.py` rewrote with LF were + restored to their committed bytes. + +### Verification + +- `test_service.py`: the instance's driver override resolves to `delegated_worker` through `default_topology`. +- `test_topology.py`: `topology_for` gives `DelegatedWorker` for browser-use with the field unset and `Direct` + with `direct`; the instance-override test patches `default_topology` to `direct` and asserts it is asked for + `browser_use`; the `Direct` tests build their runtime with `topology="direct"`. +- `test_tool_prompt_snapshot.py`: the browser-use `runner` and `runner_direct` views built and compared with + their goldens; `test_golden_files_cover_exactly_the_views` checks the new set; the well-formedness test runs + over the twelve recorded capability goldens. +- `uv run python -m tests.unit_tests.harness.tools.browser_move.snapshots.regenerate`: `unchanged` for the + thirteen MCP views; `removed browser_use/runner_worker.json`; `updated browser_use/runner.json`; + `updated browser_use/runner_direct.json`. +- `uv run pytest tests/unit_tests/harness/tools/browser_move -q`: **2551 passed, 130 skipped, 8 warnings in + 76.43s (0:01:16)** (2551 / 130 before: one comparison test replaced by two, one well-formedness case fewer). +- `ruff check` and `ruff format --check` clean on every touched file, except the pre-existing `ASYNC109` at + `test_service.py:261`. +- No WebVoyager run in this step: the measurement is the 2026-10-15 reruns above. + +## A final connection loss ends the task + +In the WebVoyager run of 2026-10-15 (`20261015-1339_bu_critical_sonnet-5.5_eb5454b10`, task Google Map--40, browser-use, +direct topology) the stack lost its connection a second time in the same task. `DriverSession` raised the `driver_lost` +error as designed, but nothing ended the task: `CommandTool` lets a `BaseError` through, the ability manager turned +it into an ordinary tool message (`Tool execution error: [123023] deepagent runtime error, reason: the browser +connection was lost again; the task cannot continue`) and the model went on: 10 more model calls after the final +loss, every tool call failing the same way, until the task deadline; the deadline then judged the evidence complete +and the answer's metadata hook met the dead stack, so the run ended in an exception instead of an answer. + +Decision: the session remembers the terminal loss, one core rail turns it into the end of the run, and each +topology's answer path reads the result it already understands. + +1. **`DriverSession.lost`.** The `driver_lost` message once a loss ended the task (a second loss, or a restart that + failed), `None` otherwise; `begin_task()` clears it with the loss counter. The errors raised are unchanged. +2. **`ConnectionLostRail`** (`runtime/rails/connection_lost.py`), the fourth core rail: `before_model_call` ends the + run with `{"output": , "result_type": "error", "error": }` when the session is lost. `core_rails` + takes a zero-argument session lookup, since the browser main agent's rails are built before the session exists. +3. **The answer.** The worker loop takes a run a rail ended with an error of its own as the blocked answer + (`ok=False`, `error` and `final` the text, `status` `blocked`), with no recovery call and no retry. The browser + main agent's rail marks the task `blocked` with `terminal_reason` `browser_connection_lost` in `after_invoke` + (the answer: `ok=False`, `status` `blocked`, `error` `browser_task_incomplete`, the `browser_result` carrying the + reason and the text as `summary`); the reason is retryable on its own and a resume drops the blocker. + +The one-restart rule and `contract/` are untouched. In the direct topology the loss may happen inside the state +capture the context processors run before a model call, after the stop has already let that call through: that +one model call still happens and its command fails; the next model call is skipped. + +Rejected options: +- Catching the error in `CommandTool` and returning a failure output: the model would retry just as in Google + Map--40, only with a prettier message; the batch helper, the probes and the runtime's own capture reach the + session without a `CommandTool`. +- An exception the agent loop lets escape: tool exceptions become tool messages and only cancellation escapes the + loop; a run that raises gives the direct answer no status and lets the worker loop's retry run the task again. +- Ending the run from the tool through the callback context (`accepts_tool_callback_context`): only `CommandTool` + would end it; the helpers and the state capture would not. +- A hook on `on_tool_exception` instead of `before_model_call`: the loss can happen outside a tool call (the state + capture, the metadata hook); the model-call gate is the one place every path passes, and it is what makes "no + further model call" a property rather than a case list. +- The stop in `BrowserRuntimeRail`: the worker has no such rail; the core rails are the rails both topologies + share, and a surface word in the core would fail the architecture tests. + +Verification: `uv run pytest tests/unit_tests/harness/tools/browser_move -q` → `2560 passed, 130 skipped`; +`git diff eb5454b10 --stat -- tests/unit_tests/harness/tools/browser_move/snapshots/golden/` → empty. The topology +test runs a fake driver that loses the connection on every command through the facade in both topologies: the +worker makes two model calls (the restart, then the final loss) and the direct main agent one (the state capture +takes the restart and, through the metadata hook, the final loss); both answers are `ok=False` with the lost-connection +reason, and the driver sees `execute, stop, start, execute` (worker) and `observe, stop, start, execute, execute` +(direct). The driver-session tests cover `lost` after a second loss and after a failed restart, and its reset by +`begin_task()`; the rail test the skipped model call; the service test the blocked answer without a recovery call; +the rail test on the direct side the `blocked` status over evidence, the retryable reason and the resume. diff --git a/openjiuwen/harness/docs/specs/S_18_subagents-and-lifecycle.md b/openjiuwen/harness/docs/specs/S_18_subagents-and-lifecycle.md index 8bf726a605..b58978a730 100644 --- a/openjiuwen/harness/docs/specs/S_18_subagents-and-lifecycle.md +++ b/openjiuwen/harness/docs/specs/S_18_subagents-and-lifecycle.md @@ -5,9 +5,9 @@ | 项 | 值 | |---|---| | 类型 | spec | -| 关联模块 | `openjiuwen/harness/subagents/`（8 文件）、`openjiuwen/harness/subagent_lifecycle.py`、`openjiuwen/harness/manifest/harness_elements.py`（subagent 构建器） | -| 最近一次修订日期 | 2026-09-03 | -| 关联 feature | N/A | +| 关联模块 | `openjiuwen/harness/subagents/`（8 文件）、`openjiuwen/harness/subagent_lifecycle.py`、`openjiuwen/harness/manifest/harness_elements.py`（subagent 构建器）、`openjiuwen/harness/schema/decision_policy.py`（`DecisionPolicyModel` 协议） | +| 最近一次修订日期 | 2026-09-19 | +| 关联 feature | [[F_05_browser-decision-policy-slot]] | ## 范围 / 边界 @@ -55,7 +55,12 @@ 这些是 code preset 的硬语义，新增 preset 不得绕过。 6. **browser agent 约束**：`build_browser_agent_config` 经 `_resolve_runtime_settings` 解析浏览器运行时；`_browser_model_with_temperature(model, temp)` - 设置浏览器模型温度；`_coerce_browser_instance` 归一实例形态。 + 设置浏览器模型温度；`_coerce_browser_instance` 归一实例形态。例外：`model` 满足 + `schema/decision_policy.py:DecisionPolicyModel`（`@runtime_checkable` 结构化 Protocol， + 只要求 `bind_runtime(runtime)`）时，`create_browser_agent` 原样使用该模型（不做温度副本）， + 不注入 LLM 专用的 ContextProcessorRail，关闭模型异常检测 rail，并以 + `bind_runtime(browser_backend)` 绑定运行时。判定按结构而非具体类；`browser_agent.py` + 不 import 任何具体策略实现，具体策略实现位于本仓库之外（`F_05`）。 7. **同步创建路径**：`create_*_agent(config, ...)` 是同步（复用 `create_deep_agent` 的 同步构造，`S_01` 不变量 3）；`enable_subagent_runtime` 时才进 `S_10` 的异步控制面。 8. **manifest 侧预设与 `subagents/` 预设同源**：`S_12` 的 `build_*_subagent` 是 diff --git a/openjiuwen/harness/docs/specs/S_20_browser-driver-contract.md b/openjiuwen/harness/docs/specs/S_20_browser-driver-contract.md new file mode 100644 index 0000000000..77201ead36 --- /dev/null +++ b/openjiuwen/harness/docs/specs/S_20_browser-driver-contract.md @@ -0,0 +1,1957 @@ +# S_20 Browser Driver Contract + +## Meta + +| Item | Value | +|---|---| +| Type | spec | +| Modules | `openjiuwen/harness/tools/browser_move/contract/` (`observation.py` / `refs.py` / `actions.py` / `results.py` / `errors.py` / `driver.py` / `registry.py` / `surface.py` / `device.py` / `masking.py` / `testing/`), `openjiuwen/harness/tools/browser_move/drivers/` (`__init__.py` / `middleware.py` / `playwright_mcp/snapshot.py` / `playwright_mcp/result.py` / `playwright_mcp/driver.py` / `playwright_mcp/config.py` / `playwright_mcp/tools.py` / `playwright_mcp/clients/`), `openjiuwen/harness/tools/browser_move/runtime/assembly.py`, `openjiuwen/harness/tools/browser_move/runtime/session.py`, `openjiuwen/harness/tools/browser_move/runtime/topology.py`, `openjiuwen/harness/tools/browser_move/runtime/page_state.py`, `openjiuwen/harness/tools/browser_move/runtime/driver_session.py`, `openjiuwen/harness/tools/browser_move/runtime/rails/action_policy.py`, `openjiuwen/harness/tools/browser_move/runtime/tool_factory.py`, `openjiuwen/harness/tools/browser_move/runtime/rails/repeated_call_guard.py`, `openjiuwen/harness/tools/browser_move/runtime/rails/intercepted_action.py`, `openjiuwen/harness/tools/browser_move/runtime/rails/final_answer_recovery.py`, `openjiuwen/harness/tools/browser_move/runtime/rails/connection_lost.py`, `openjiuwen/harness/tools/browser_move/runtime/config.py`, `openjiuwen/harness/tools/browser_move/runtime/service.py`, `openjiuwen/harness/tools/browser_move/runtime/service_registry.py`, `openjiuwen/harness/tools/browser_move/runtime/worker.py`, `openjiuwen/harness/tools/browser_move/runtime/utils/env.py`, `openjiuwen/harness/tools/browser_move/surfaces/browser/runner/` (`facade.py` / `settings.py` / `tools.py` / `actions.py` / `logging.py` / `server_config.py` / `main.py`), `openjiuwen/harness/tools/browser_move/surfaces/browser/offload_recall.py`, `openjiuwen/harness/tools/browser_move/provisioning/` (`__init__.py` / `managed_browser.py` / `profiles.py`), `openjiuwen/harness/tools/browser_move/surfaces/` (`__init__.py`; `browser/`: `commands.py` / `context.py` / `config.py` / `policy.py` / `profile.py` / `tools.py` / `renderer.py` / `capabilities.py` / `site_profiles.py` / `state_processor.py` / `status_logging.py` / `helpers/` / `probes/` / `semantic/` / `working_context/` / `rails/` / `scene/`), `openjiuwen/core/common/logging/tool_call.py`, `openjiuwen/core/context_engine/context/context_utils.py` (`offload_session_dir_name`) | +| Last revised | 2026-10-15 | +| Related feature | [[F_07_browser-drivers]] | + +## Scope + +The contract is the only thing the agent core and a driver both know. The core sends commands and reads typed +results and errors; a driver translates commands for one library and declares which commands it handles. As of +this revision the browser-use and Playwright MCP drivers implement the contract and pass the conformance suite +live, the middleware wraps any driver and the composition root builds a started driver stack, and an agent over +it, by name; the browser agent (`create_browser_agent`) runs on it: its runtime facade starts the driver +session and builds the factory tools at the first invoke. The WebVoyager worker still runs the old path. + +The core runtime package is `runtime/`, with `rails/` (the record rails: the policy rail, the repeated-call guard, +the intercepted-action hint, the final-answer recovery) and `utils/` under it; the browser runner is +`surfaces/browser/runner/`: the facade `BrowserAgentRuntime` (`facade.py`), the browser settings and the +`RuntimeSettings` facade (`settings.py`), the two custom-action tools (`tools.py`) and their controller +(`actions.py`), the browser agent log (`logging.py`), the runtime server's client config (`server_config.py`) and +the demo CLI (`main.py`). `playwright_runtime_mcp_server.py` keeps its name as the entry point and imports the +package by its full name (`python -m …playwright_runtime_mcp_server`). +Everything that reads pages, tabs or URLs lives under `surfaces/browser/`: the capability catalog, the site profiles, +the probe scripts (`probes/`), the semantic state (`semantic/`), the working context (`working_context/`), the state +context processor, the status log and the browser-only rails (`rails/`): the main agent's rail is +`rails/runtime_rail.py`, its mixins beside it (`rails/terminal.py`, `rails/results.py`, `probes/results.py`, +`semantic/phases.py`, `semantic/evidence.py`, the shared keys in `rails/keys.py`), and the prompt text (the +worker prompt, the image, date-picker and task-rules sections) is in `tools.py`; the core runtime is +`runtime/session.py` (`AgentRuntime`), the worker loop and its progress state `runtime/worker.py`, the service's +lifecycle `runtime/service.py` (`AgentService`) and the identity registry `runtime/service_registry.py`. The probe +scripts are one module each (`probes/metadata.py`, `probes/interactives.py`, `probes/cards.py` with its template in +the resource file `probes/cards.js`); the browser action specs and the upload listing are `helpers/custom.py`. +The architecture tests hold this layout: no module under `runtime/` or `surfaces/` over 1,500 lines; no +Playwright MCP tool name in the core; no driver name in the core (the runtime server's client config, +`surfaces/browser/runner/server_config.py`, and the facade's identity, which reads the CDP endpoint of the given +server config, name one); `contract/` +imports the standard library and itself; `surfaces/` imports no driver package and no provisioner; the core +reaches the registries (`DRIVERS`, `PROVISIONERS`, `SURFACES`) from `runtime/assembly.py` only; the core imports +no module of `drivers//`, `surfaces//` or `provisioning/`: the runtime, the service, the config and +the agent builders read the hooks, the worker prompt and the capability catalog from the surface profile, and the +registry is the only way from the core to a surface. The browser words left in the core are the worker loop's +data (the worker's JSON `page` / `url` keys, the Playwright restart and retry markers, the progress record's +`last_page` export) and the no-op script atoms of the repeated-call guard; the allowlist names them and may only +shrink. The default surface name is `surfaces.DEFAULT_SURFACE`, imported by the core settings. +The browser profile store is `provisioning/profiles.py`. The old `playwright_runtime` package and the root module +`offload_recall.py` are removed; callers import `runtime.`, `surfaces.browser.` or +`provisioning.profiles`. The WebVoyager runner uses `surfaces.browser.runner.settings.build_runtime_settings`, +`surfaces.browser.runner.facade.BrowserAgentRuntime` and `surfaces.browser.capabilities.CORE_BROWSER_TOOL_NAMES`, +and reads `runtime._service.agent` for its helper removal. + +The browser surface profile (tool specs, renderer, capability catalog) is specified under "Browser surface +profile"; the tool factory that turns it and the driver's native tools into model-facing tools under "Tool +factory". + +### Dependency rules + +Each layer imports only what it needs, and the architecture tests (`architecture/test_import_boundaries.py`) hold +every rule with an allowlist that may only shrink: + +- `contract/` imports the standard library and itself. It holds every type more than one layer shares: the + commands, refs, results and errors, the `Driver` base, the registries, the surface types, `ProvisionedDevice` + (`device.py`, the argument of every driver factory) and `masked` (`masking.py`, the masking rule of `Tracing`, + `CommandRecord.action` and the batch helper). +- A driver package `drivers//` imports the contract, its own package, `drivers/middleware.py`, the `commands` + and `context` modules of the surfaces it serves (it implements their commands and fills their context type), the + standard library, third-party packages and `openjiuwen.core`; never `runtime/`, `provisioning/` or any other + surface module. A driver's own configuration lives in its package (`drivers/playwright_mcp/config.py`); it logs + through the core logger, which the browser log context routes to the browser agent log. The driver allowlist + is empty. +- `provisioning/` imports the contract only (plus the standard library, third-party packages and + `openjiuwen.core`); a provisioner types the settings it reads with its own `Protocol`. +- `runtime/` and `surfaces/` import nothing under `drivers/`, except the composition root `runtime/assembly.py` + (it wraps the built driver with the middleware) and, inside function bodies only, the browser runner's settings + facade `surfaces/browser/runner/settings.py`, whose public `playwright_mcp_config` / + `build_playwright_mcp_config` / `RuntimeSettings.mcp_cfg` call the driver's config builder (the architecture + rules treat that module as a config module: it reads the process environment for the facades). + (it wraps the built driver with the middleware) and, inside function bodies only, `runtime/config.py`, whose + public `playwright_mcp_config` / `build_playwright_mcp_config` / `RuntimeSettings.mcp_cfg` call the driver's + config builder. +- A driver reads the browser surface's `probes/clipboard_guard.js` as a package resource + (`importlib.resources.files()`): a resource read, not an import, so the driver rule + above stands and the architecture test sees no import. No driver imports `probes`. +- The core does not reach a concrete driver, surface or provisioner through a module outside the package either: + a core import of an `openjiuwen` module that itself imports one of them is a finding; the allowlist is empty + (the main agent reaches the core through the profile's `main_agent` hook). +- The package root holds `__init__.py` and the runtime server entry point `playwright_runtime_mcp_server.py`; + every other module lives in a layer. + +A surface, driver or provisioner is added with new code and registry entries only. The fake-surface test +(`tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/`) holds this: a screen surface with its own +command, renderer and policy rule, a driver and a provisioner, all defined in the test and registered in +test-scoped registries, run a scripted task through `build_agent` on a real deep agent, and through +`AgentRuntime` on both topologies (`test_fake_surface_runtime.py`): the runtime takes the core settings, the +screen's bundle, a temporary output folder and an identity of its own, `Direct` builds the agent from the parts +(the profile sets no `main_agent`), `DelegatedWorker` the worker through the service, and the answer's +`context.identity` is the screen's. + +## Contract types + +### Rules that hold for every type + +1. **`contract/` imports only the standard library and itself.** No `browser_move` module, no third-party package. + It contains no browser word (`browser`, `page`, `tab`, `url`) in names, strings or comments. Both rules are + enforced by `tests/unit_tests/harness/tools/browser_move/architecture/`. +2. **Values are frozen dataclasses.** Mapping fields (`Element.extra`, `DriverInfo.versions` / `actions` / + `tool_hints`, profile maps) use `field(default_factory=dict)` and are left out of the hash; no default is shared + between instances. +3. **Surface-neutral core.** A field one driver would have to fake does not belong in `Observation` or + `ActionResult`. Per-surface data lives in a `SurfaceContext` subclass owned by the surface. +4. **No module-level state** except the three default registries in `registry.py`. + +### Observation (`observation.py`) + +What `Driver.observe()` returns: a snapshot of the focused context. A scoped `Observe` (`depth` or `root` set) gives a +partial one: the subtree under `root`, at most `depth` levels deep. It shows some of the page's elements, so the page +state never takes it for the page's observation (see "Page state"); a driver that cannot scope raises +`Unsupported(dispatched=False)` before anything is sent. + +| Type | Purpose | +|---|---| +| `Observation` | `context: ContextRef` (focused context), `generation` (stamped by the runtime's state tracker; drivers leave it `""`), `elements`, `screenshot`, `viewport`, `native_text` (the driver's own textual snapshot, shown or not by the renderer), `surface: SurfaceContext \| None`. Keyword-only. | +| `Observation.fingerprint` | 16-hex SHA-256 over the context id and each element's `(ref.ref, role, name)`. Generations, values, screenshots and the surface context are left out, so two observations of an unchanged scene match. A backstop for change detection; the primary signal is `ActionResult.context_changed`. | +| `Element` | `ref: DriverRef`, `role`, `name`, `value`, `bounds` (`(x, y, width, height)` in viewport pixels), `enabled`, `focused`, `extra`. | +| `Screenshot` | `png`, `width`, `height`, `scale` (image pixels per viewport pixel). `PointRef` coordinates are always in the screenshot's pixels. | +| `Viewport` | `width`, `height`. | +| `SurfaceContext` | Frozen base with one neutral member, `identity` (a property, `""` on the base): where the surface is, as the surface names it (`BrowserContext.identity` is its `url`; a phone context would give its package and activity). The core reads only `identity`, to detect a move; it passes the rest through to the surface's renderer, rails and page state. | + +### Refs (`refs.py`) + +| Type | Purpose | +|---|---| +| `DriverRef(ref, generation)` | The model-facing element ref (`e57`). | +| `PointRef(x, y, generation)` | A point in that generation's screenshot pixels. | +| `IndexRef(index, generation)` | Driver-internal index; never shown to the model. | +| `Locator(css, text, generation)` | A selector and/or visible text. | +| `ContextRef(id, kind, title)` | A context the driver can focus; `kind` is surface-defined. Not an element ref. | + +`Ref` is the union of the four element ref kinds; `REF_KINDS` is the same set as a tuple. One command takes several +ref kinds (one `Click`, not `ClickElement` / `ClickPoint`); which kinds a driver accepts is part of its +`DriverInfo.actions`. + +`RefResolver` (protocol) is what a tool's argument decoder asks the runtime: `contexts` (the open contexts, in +order), `current_context` (`None` when unknown), `screenshot_generation` (the generation of the last screenshot, +whose pixels a `PointRef` addresses; empty before the first), `resolve(model_id, generation) -> Ref` (the ref behind +a runtime-minted element id; `ValueError` for an id it does not know) and `generation_of(ref) -> str` (the generation +a driver ref was last seen in; the current one for a ref never seen). The page state implements it. A decoder builds +every `PointRef` with `screenshot_generation` (the model sends no generation for a point), so a point taken on an +older screenshot is stale like an element ref; an empty generation is never stale. + +Who owns staleness is a property of the dialect, written in the generation its decoder stamps. The page state owns +it for the refs it hands out or the model dates itself: a runtime-minted target (`resolve`), a point on a screenshot +(`screenshot_generation`), the browser dialect's `ref` / `selector` with the model's `generation_id`, and a +browser-use index ref. The Playwright MCP server owns it for the aria refs and css the model sends on its dialect, +which it resolves against the live page: those decode with an empty generation, the page state lets them through, +and the server's "not found" is the `StaleRef` the driver raises (`dispatched=False`), so the model gets the error +with a fresh observation either way. + +### Commands (`actions.py`, `surfaces//commands.py`) + +A command is a frozen dataclass subclassing `Action`, with a class-level `effect` (`Effect.READ`, `ACT`, +`NAVIGATE`, `SCRIPT`); middleware chooses timeouts and retry rules by effect. + +**Where a command lives.** `contract/actions.py` holds `Action`, `Effect` and the commands that make sense on any +screen: + +| Command | Fields | Effect | +|---|---|---| +| `Observe` | `include_screenshot=False`, `full_page=False`, `depth: int \| None = None`, `root: Ref \| None = None`; `scoped` is true when either is set | READ | +| `Click` | `target: Ref`, `button="left"`, `modifiers=()`, `count=1` | ACT | +| `TypeText` | `target: Ref \| None`, `text`, `submit=False`, `sensitive=False`, `clear=True`, `slowly=False` | ACT | +| `PressKey` | `key`, `target: Ref \| None = None` | ACT | +| `Scroll` | `target: Ref \| None = None`, `dx=0`, `dy=0` | ACT | +| `Hover` | `target: Ref` | ACT | +| `Drag` | `source: Ref`, `destination: Ref` | ACT | +| `Wait` | `seconds=None`, `until_text=None`, at least one set: `seconds` alone sleeps; `until_text` waits until the text appears, at most `seconds` (`None`: the driver's default; a driver whose library has no deadline for a text wait may use its own), then raises `ActionTimeout` | READ | +| `SwitchContext` / `CloseContext` | `context: ContextRef` | NAVIGATE | +| `HandleDialog` | `accept`, `prompt_text=None` | ACT | +| `RunScript` | `code`, `args=None`, `sensitive=False`, `target: Ref \| None = None` | SCRIPT | + +`RunScript.code` is a JavaScript expression: when its value is a function the driver calls it (with `args`, and the +element first when `target` is set), otherwise the value is the result (as Playwright MCP's `browser_evaluate`). + +A surface defines its own commands as `Action` subclasses in `surfaces//commands.py`. The browser surface +has `Navigate(url)`, `Back`, `OpenContext(url=None)` (NAVIGATE); `SelectOption(target, values)`, +`UploadFile(target: Ref | None, paths=())`, `FillForm(fields: tuple[FormField, ...])` with +`FormField(target, kind, value, sensitive=False)`, +`Drop(target, paths=(), data=())` (ACT); `GetPdf(filename=None)`, `GetConsole(level="info", all=False)`, +`GetNetwork(static=False, filter=None)` (READ). Drivers handle both kinds the same way; the core does not care +where a command class is defined, and a new surface adds its commands (`Tap`, `Swipe`) without editing +`contract/`. A command is added only when a driver handles a tool the model uses today. + +`sensitive=True` on `TypeText`, `RunScript` and a `FormField` means its text, code or value must never reach traces +or logs. Each of the three names its fields in a class-level `__sensitive_fields__` (`("text",)`, +`("code", "args")`, `("value",)`); `masked` (`contract/masking.py`) replaces the listed fields of any dataclass +whose `sensitive` is true, for `Tracing` and for `CommandRecord.action` alike, so a new surface command opts in by +declaring the attribute and a `sensitive` field. + +### ActionResult (`results.py`) + +`ok`, `value`, `detail`; `context_changed` (set by the driver when a new document loaded, including a reload of +the same address, the address changed, a dialog opened, or a context was opened, closed or switched); +`generation` (filled by the runtime after the call); `observation` (a fresh observation the driver got for free, +so the runtime skips a second `observe()`); `source_context` and `opened_contexts` (context ownership); +`surface` (the surface context after the command). Keyword-only. + +### Errors and the dispatched rule (`errors.py`) + +Every error is a `DriverError` with `dispatched: bool`: whether the command may have reached the screen. + +- **An error with `dispatched=True` is never retried automatically.** A retry may only follow an error raised + before anything was sent. +- **Unknown means dispatched.** When the driver cannot tell, `dispatched` is `True`; an unrecognized library error + becomes `DriverError(dispatched=True)`. +- `StaleRef` (the ref's generation is old) is raised only before dispatch and defaults to `dispatched=False`. + Its optional `observation` is a fresh observation taken after the error (`RetryOnStale` attaches it; drivers + leave it `None`). `Unsupported` (no handler, or a ref kind the handler does not accept) and `PolicyDenied` + (raised by the security rail before anything is sent, never by a driver) also default to `False`. +- `ActionIntercepted(by=...)` (`by` names the element that would receive the input) defaults to `False`: every + driver raises it before the input is sent, so it is not a `StaleRef` and is not retried either. The conformance + suite asserts it. +- All others default to `True`: `ElementNotFound`, `AmbiguousElement`, `ActionTimeout`, `NavigationError`, + `ConnectionLost`. + +Every constructor accepts an explicit `dispatched=` to override the default. + +### Driver (`driver.py`) + +- `Driver` (ABC): `info -> DriverInfo`, and async `start(config)`, `stop()`, `health() -> Health`, + `observe(Observe) -> Observation`, `execute(action) -> ActionResult`. The type of `config` belongs to the driver; + its factory builds it. +- `DriverInfo(name, surfaces: frozenset[str], versions, actions, observes, tool_hints, native_tools, + tool_name_prefix, display_name)`. `actions` maps each handled command type to the element ref kinds it accepts (empty for + commands without refs). `surfaces` lists the surfaces the driver serves; the composition root refuses a driver + that does not list the selected one. `observes` names the `Observation` fields the driver fills. `tool_hints` are + optional per-command descriptions. `native_tools` (default empty) maps a tool spec name to a + `NativeTool(description, schema, build=None)`: a tool the driver offers in its own format, with + `build(args, resolver) -> Action` decoding its arguments; `build` is `None` for a driver tool no command backs, + which the tool factory never emits itself (a surface extra tool may take its description and schema). The model + sees `tool_name_prefix` (default `""`) + the spec name. The Playwright MCP driver fills them after `start()`. + `display_name` (default `""`) is how a prompt names the driver to the model (`"Playwright MCP"`, + `"browser-use"`); a reader falls back to `name` when it is empty. It is the only driver-specific text a prompt in + the core carries, so the core names no driver itself. +- `@handles(Command, refs={DriverRef, ...})` marks a method as the handler of a command; marks stack. + `refs` accepts only element ref kinds. +- `HandlerDriver(Driver)`: `execute` dispatches by `type(action)` to the marked method. `info` is + `describe()` with `actions` replaced by the marks, so the supported commands cannot drift from the code. + A command with no handler, or a ref of a kind the handler does not accept (also inside nested values such as + `FillForm.fields`), raises `Unsupported(dispatched=False)` before any handler runs. Subclasses inherit handlers + and may re-mark a command. + +### Registries (`registry.py`) + +`Registry[T]`: `register(name, factory)` refuses a duplicate name; `factory` is a callable or a lazy import string +`"pkg.module:attr"` that is imported on the first `create(name, *args, **kwargs)`; an unknown name raises +`KeyError` listing the registered ones; `names()` keeps registration order. A lazy string whose module does not +exist raises `ModuleNotFoundError` " '' is registered but not built yet"; a missing module imported +*by* that module is raised unchanged. Three default registries — `DRIVERS`, `SURFACES`, `PROVISIONERS` — with +`register_driver`, `register_surface`, `register_provisioner`, so a driver, surface or provisioner can live in +another repository. Factory signatures: driver `factory(device, surface_settings) -> Driver`, surface +`factory(environ, instance=None, settings=None) -> SurfaceBundle` (`settings` is used instead of parsing `environ`; +`instance`, the surface's per-agent identity, is applied to the settings last), provisioner +`factory(**kwargs) -> Provisioner`. + +`register_driver(name, factory, *, default_topology="direct")` also records the driver's default topology in +`DRIVER_TOPOLOGIES: dict[str, str]`, module state next to `DRIVERS` (the `Registry` class is generic over the three +kinds and carries no driver-only field). `default_topology(name)` returns the recorded string or raises the +`KeyError` of an unknown driver, listing the registered names. A duplicate name is refused before anything is +recorded. `contract/` stores and returns the string without judging it: the settings parser validates the +environment value and the core maps the name to a behaviour. + +Nothing is registered in `contract/`; the packages that own the built-ins register them with lazy strings and +re-export the registry, so `from ..drivers import DRIVERS` gives a registry with the built-ins in it: + +| Package | Names | Factory | Default topology | +|---|---|---|---| +| `drivers/__init__.py` | `browser_use` | `drivers.browser_use.driver:create_browser_use_driver` | `delegated_worker` | +| | `playwright_mcp` | `drivers.playwright_mcp.driver:create_playwright_mcp_driver` | `delegated_worker` | +| `surfaces/__init__.py` | `browser` | `surfaces.browser.profile:create_browser_surface` | — | +| `provisioning/__init__.py` | `managed`, `remote`, `extension` | `provisioning.managed_browser:ManagedBrowserProvisioner` | — | + +Importing the package, a registry package or the composition root imports no concrete driver, surface or +provisioner module and not `browser_use`. + +### Surface types (`surface.py`) + +Data types only; their use is specified with the composition root and the tool factory. + +- `SurfaceSettings` (protocol): the settings a surface parses from its own variables. The core reads one attribute, + `provisioning` (a provisioner registry name), and passes the object on to the provisioner and the driver factory + unchanged. The core types surface settings with it and never imports a surface's settings class. +- `SurfaceBundle(settings, profile=None)`: what a surface factory returns. A surface without a profile returns its + settings only; the browser surface returns both. + +- `SurfaceProfile`: `name`, `commands`, `renderer: Renderer`, `tool_specs`, `capability_catalog` + (entry -> `(command, ref kind | None)` pairs; `None` is a call without an element ref), `capability_tools` + (the same entries -> the names of the tools they select), `extra_tools` and `rails` (each a `ContextFactory`: + called with the core's `ToolContext`, returns a tool or a rail), `policy_rules`, `timeouts` + (per `Effect`), `messages`, `settings`, `scene`; and the fields the agent runtime facade and the delegated + worker read: `capabilities` (the catalog, a tuple of `Capability` in order, its first entry the default one + every agent gets), `custom_actions(controller, context)`, `page_metadata(context) -> (metadata, error)`, + `policy_probe(context, source, params)`, `activate(context, address) -> bool`, `semantic_tracker()` (a + factory), `worker_prompt(screenshot_subdir, artifacts_subdir, *, tool_names=None, driver_name)`, + `page_state()` (a factory, "Page state") and `main_agent(runtime, model, *, language, max_iterations, + workspace, capabilities)`, typed by the `Protocol`s `CustomActions`, `Metadata`, `PolicyProbe`, `Activate`, + `WorkerPrompt` and `MainAgentFactory` in `surface.py`. Each is `None` (the catalog empty) unless the surface + fills it, and each is optional for the runtime and the worker: + without `custom_actions` no custom action is registered; without `page_metadata` the captures use `{}` and no + error; without `policy_probe` the facade answers `{"ok": False, "error": "surface '' has no policy probe", + "elements": [], "page_state": ...}`; without `activate` it answers `False`; without `semantic_tracker` it uses a + tracker that never observes progress (`{"progress": "unknown", "observable_progress": False}`, `latest` `{}`); + without `worker_prompt` the worker gets `runtime/agents.default_worker_prompt`, which names the driver and the + output folders and no surface; without `main_agent` the `Direct` topology builds the main agent from the + profile's parts ("Topology"). The unused `native_tools` field is gone (a driver's native tools are + `DriverInfo.native_tools`). +- `Capability(name, description, tool_names, policy_only=False)`: one catalog entry; `policy_only` marks an + entry a policy rule alone grants, which the driver has no feature to enable for. +- `ToolSpec(name, commands, description, prompt_section, schema=None, build=None)`: one model-facing tool; it may + cover several commands. `schema` is the JSON schema of its arguments; `build(args, resolver) -> Action` is the + pure decoder (raises `ValueError` for invalid arguments). A spec without either is served only by a driver's + native tool or a generated schema. +- `Renderer` (protocol): `observation(obs)`, `result(action, result)`, `error(action, error, observation)`, each + returning a tuple of `ContentBlock`s; `ContentBlock` is `str` (text) or `Screenshot` (an image). +- `PolicyRule(command_type, check)`: `check(action, context)` returns the reason to deny, or `None`; `context` is + the current surface context, `None` before the first observation. +- `Scene`: what the conformance suite runs a driver in — `surface`, `start` / `stop`, `enter()` (commands into the + initial state) and the case builders `happy(command, ref_kind, actions)`, `stale(actions)`, + `intercepted(actions)`, `timeout(actions)`, `context_changes` (name -> builder) and `sensitive` (name -> + `builder(actions, secret)`, a surface command that carries the secret marked sensitive). A builder gets the + driver's `info.actions` and returns a `SceneCase`, or the reason the scene cannot provide the case as a string; an + omitted builder returns a generic reason. See "Conformance suite". +- `SceneCase(steps, check, observe_between, cleanup)`: the suite observes once, then each step builds one command + from the observations taken so far; the last step is the command under test, the ones before it are setup steps. + With `observe_between=False` no observation is taken between the steps, so every step sees the first one (for a + setup step that opens a dialog, or one that changes the screen after the command under test has taken its refs). + `check(before, after, result)` tells whether the command under test took effect; in a context-change case it is + checked on an observation taken right after the command, before the cleanup. + +### FakeDriver (`contract/testing/fake.py`) + +A scripted `Driver` for tests: records every call as `Call(kind, payload)`, answers `observe` / `execute` from +queues in order (filled by the constructor and by `script(observations=..., results=...)`, which appends), raises +`Unsupported(dispatched=False)` for commands or ref kinds missing from its `info.actions` +(default: every shared command except `Observe`, all ref kinds; default surfaces `{"fake"}`), raises a scripted +`DriverError(dispatched=...)` or any exception via `fail_next`, and raises `ConnectionLost` from the n-th call +until the next `start()` via `lose_connection_on(n)`. Running out of script fails with `AssertionError`. + +## Conformance suite (`contract/testing/conformance.py`) + +A pytest plugin that holds invariants 6 and 7 of the plan: every driver means the same thing by the same command, and +a dispatched command is never repeated. It is surface-neutral; everything surface-specific comes from the `Scene`, +so a new driver or surface adds a test module and a scene, never an edit to the suite. + +Use: a test module sets `pytest_plugins = ["openjiuwen.harness.tools.browser_move.contract.testing.conformance"]` +and subclasses `DriverConformance` as `Test...` with `scene` (not started; the plugin starts it once per class), +`driver_actions` (what the happy paths are parametrized with when tests are collected; for a `HandlerDriver`, +`handled_actions()`) and a `conformance_driver_factory` fixture returning `factory(sink) -> Driver` (a new driver +per case; `start()` must need no argument). Every case starts a new driver in its own event loop, runs `enter()`, +plays its `SceneCase` and stops the driver. The plugin registers the `live` marker for runs against a real driver; +`pyproject.toml` registers it too, because a test module that marks itself `live` before loading the plugin would +otherwise get `PytestUnknownMarkWarning`. + +| Case | Passes when | Scene-dependent | +|---|---|---| +| surface | `scene.surface in info.surfaces` | no | +| declared actions | `info.actions` of the running driver equals `driver_actions` | no | +| happy path, one per `(command, ref kind)` of `info.actions` (`None` when the command takes no ref) | the command returns `ok=True` and `check(before, after, result)` holds | yes (`happy`) | +| stale ref | `StaleRef` with `dispatched=False` | yes (`stale`) | +| context change, one per `context_changes` entry | `result.context_changed`; with a `check`, an `observe()` right after the command (before the cleanup) passes it | yes | +| intercepted | `ActionIntercepted` with a non-empty `by` and `dispatched=False` | yes (`intercepted`) | +| timeout | the scene's case ends in `Wait(until_text=..., seconds=...)` for a text that never appears; it raises `ActionTimeout` with `dispatched=True`; a driver that retries into success fails here | yes (`timeout`) | +| unhandled command | the suite's own `UnhandledCommand` is not in `info.actions` and raises `Unsupported(dispatched=False)` | no | +| fingerprint | two `observe()` calls on the unchanged scene give the same `fingerprint` | no | +| sensitive text, trace, one per sensitive case: `type-text` and each `sensitive` entry | the secret leaves no text in the trace sink; `type-text` is `TypeText(sensitive=True)` built from the `TypeText` happy path; a case whose command does not carry the secret fails | yes (`happy`, `sensitive`) | +| sensitive text, logs, one per sensitive case | the same secret is in no log record of the process, captured at DEBUG including non-propagating loggers | yes (`happy`, `sensitive`) | + +A case the scene cannot provide is skipped with the scene's reason; a case needing a command (or ref kind) the driver +does not handle is skipped naming it. Nothing passes silently. The trace case skips when the factory writes nothing +to the sink; the test modules build their driver as `Tracing(driver, sink)`, so it runs. + +Live modules: `drivers/test_conformance_browser_use_live.py` (needs `BROWSER_USE_SIDECAR_PYTHON` and Chrome) and +`drivers/playwright_mcp/test_conformance_playwright_mcp_live.py` (needs the MCP command, `npx` by default, and +Chrome; `npx` fetches `@playwright/mcp@0.0.78` on first use). Both build the driver with its registry factory over +one headless Chrome with a fresh profile on a free CDP port, which the test starts and kills. + +### The browser scene (`surfaces/browser/scene/`) + +`browser_scene()` returns a `Scene(surface="browser")`. `start()` serves `scene/pages/` (`index.html`, +`second.html`, `data.json`; shipped as package data) with a stdlib HTTP server on `127.0.0.1:` and creates a temporary upload +file; `enter()` is `Navigate(index.html)`. Every element writes what happened to it into `document.title` +(`clicked`, `form:3:true`, `key:Enter`, ...), so checks read `BrowserContext.title`, `url` and `tabs`. Elements are +found by accessible name; a `Locator` uses the element's CSS id; a `PointRef` is the centre of `Element.bounds` +scaled to the screenshot. An `IndexRef` cannot be built (skipped). + +- Happy paths: every shared command and every browser command (`Navigate`, `Back`, `OpenContext`, `SelectOption`, + `UploadFile`, `FillForm`, `Drop`, `GetPdf`, `GetConsole`, `GetNetwork`) with `DriverRef`, `Locator` or `PointRef`. +- Stale ref: a `DriverRef` from the start document clicked after `Navigate(second.html)`. +- `RunScript` without a target is a bare expression (`document.title = 'script:ran'`); with a target, a function of + the element. +- Intercepted: the "Covered button" is visible when the suite observes the page, so a driver whose observation drops + occluded elements still gets a ref. A setup `RunScript` then calls the page's `cover()`, which puts an overlay over + it, and the click uses the ref from that first observation (`observe_between=False`). Only with `DriverRef` or + `Locator`; needs `RunScript`. +- Timeout: `Wait(until_text=, seconds=2.0)`; needs `Wait`. +- Sensitive: `form-field`, a `FillForm` that fills the "Full name" textbox with the secret, `sensitive=True`. +- Beyond the `Scene` fields, the module exports case builders the two live suites run through the suite's session + helpers: `clipboard_read_case` (a `RunScript` of `CLIPBOARD_ALIAS_SCRIPT`, `const c = navigator.clipboard; + c.readText()`, which the driver must report as an error carrying the `CLIPBOARD_DENIED` text) and + `paste_command_case` (a `RunScript` of `document.execCommand('paste')` whose value must be `False`), and + `slow_typing_case` (`TypeText(text="Ada", slowly=True)` into the empty "Full name" field; the title must read + `form:3:false`, what plain typing leaves). +- Context changes: `navigate`, `same-url-reload` (`Navigate` to the current address), `link-click`, `open-tab`, + `popup-click` (`window.open`), `switch-tab`, `close-tab`, `dialog` (a `confirm()`; its check wants + `BrowserContext.dialog == "confirm: Proceed?"` in the click's result and in an `observe()` while the dialog is + open). + +## Middleware (`drivers/middleware.py`) + +Three wrappers, each a `Driver` around an inner `Driver`, and the masking helper `Tracing` shares with the tool +factory. They know only `contract/` types, so they work for any +driver and any surface; the architecture tests scan the module like the core (no browser words, no driver names, no +concrete imports). `info`, `start`, `stop` and `health` pass through to the inner driver unchanged. The stack order +is `Tracing(RetryOnStale(Timeouts(driver, per_effect)), sink)`: the trace sees the error a timeout or a stale ref +finally produced, and the re-observe of `RetryOnStale` runs under the READ bound. + +- **`Timeouts(inner, per_effect: Mapping[Effect, float])`** bounds each `execute(action)` by `action.effect` and + each `observe()` by `READ`. An effect missing from the map is not bounded. On expiry the inner call is cancelled + and `ActionTimeout(dispatched=True)` is raised: the command may already have taken effect. A `Wait` longer than + the READ bound is cut by it. +- **`RetryOnStale(inner)`** catches `StaleRef` with `dispatched=False`, observes once (`Observe()`) and raises a new + `StaleRef` carrying that observation, chained to the original. It never resends the command: neither built driver + can remap a stale ref through another ref kind it declares, so there is nothing to retry with. An error with + `dispatched=True`, a `StaleRef` included, passes through untouched, without an observation (invariant 7). +- **`Tracing(inner, sink)`** writes one dict per `observe` / `execute` call to `sink` (`TraceSink`: any callable + taking that dict), after the call returns or raises a `DriverError`: + - `call` (`"observe"` / `"execute"`), `command` (class name), `args` (the fields of `masked(command)`, nested + dataclasses as dicts and sequences as lists); + - then `ok`, `detail`, `context_changed` for a result; `context` (id) and `elements` (count) for an observation, + whose element values and native text may show what a sensitive command typed; or `error` (class name), + `message`, `dispatched` for a `DriverError`, which is raised again. `message` is left out when the command is + sensitive (its `sensitive` is true and it lists `__sensitive_fields__`): a driver error can quote the typed text + or the script, e.g. a JS exception message. + + `ActionResult.value` and observation contents are never traced. Other exceptions (cancellation included) pass + through without a record. +- **`masked(value)`** (`contract/masking.py`, re-exported here with `MASK`) returns a copy of a command (or any + value) with `MASK` = `"***"` in place of each + `__sensitive_fields__` field of every dataclass in it whose `sensitive` is true (a `FormField` inside a + `FillForm` included); other values are copied unchanged. It is the one masking rule: `Tracing` traces its + output and `CommandTool` stores it as `CommandRecord.action`. + +## Settings (`runtime/config.py`, `surfaces/browser/config.py`, `surfaces/browser/runner/settings.py`) + +Environment variables are parsed into two frozen dataclasses, split by owner. Each has +`from_env(environ: Mapping[str, str])`, which reads only the mapping it is given, and `from_process_env()`, which +is `from_env(os.environ)` and the only place that touches the process environment. Besides the mapping, a parser +reads only the current directory (the default of `BrowserSettings.mcp_cwd`). Which variable feeds which field is +listed in [[F_07_browser-drivers]] ("Settings", inventory). + +- **`CoreSettings`** (core): `driver`, `surface` (`"browser"`, a registry name), `topology` (`direct`, + `delegated_worker`, or `""` = the driver's registered default; see `BROWSER_TOPOLOGY` below), `provider`, `api_key`, + `api_base`, `model_name`, `worker_temperature`, `worker_top_p`, `tool_timeout_s` (caps the driver stack's + per-effect timeouts, "Composition root" step 5), `guardrails` + (`RunGuardrails`), `log_file` (`None` = no dedicated file), `log_level`, `log_mirror_common`, + `log_audit_raw`, `status_log`, `runtime_server` (`RuntimeServerSettings`, client side). It has no field that only + a browser has. +- **`RuntimeServerSettings`** (core): the runtime MCP server, i.e. `browser_move` served over MCP + (`playwright_runtime_mcp_server.py`); `build_browser_runtime_mcp_config` (`surfaces/browser/runner/server_config.py`) turns the client + fields into the `McpServerConfig` a host registers through the core (`Runner.resource_mgr.add_mcp_server` and the + agent's ability manager). `browser_move` patches none of the core's MCP client classes and starts no local copy of + the server for a host: a host that wants the server running starts the entry point itself. Client fields: `enabled`, `server_id`, + `server_name`, `client_type` (as given; normalised by the caller), `server_path` (`""` = derived from host, port + and path), `command` (default: the current interpreter), `args` (raw string), `timeout_s` (`0` = no timeout, also + for a value that is not an integer), `host`, `port` (a string; parsed where it is used), `path`, `log_dir` + (`None` = `mcp_cwd/logs`), `auto_http_fallback`. Server fields: `transport`, `log_level`, `stateless_http` + (`None` = decided by the transport). Two parsers keep the two old readers apart: `from_env` (client) reads + `PLAYWRIGHT_RUNTIME_*` and then the `BROWSER_RUNTIME_*` alias; `server_from_env` (server entry point) reads only + `PLAYWRIGHT_RUNTIME_*` with the server's defaults (path `""`, log dir `/logs`), except the log dir, which also + takes `BROWSER_RUNTIME_LOG_DIR` as before. +- **`BrowserSettings`** (browser surface, a `SurfaceSettings`): `provisioning`, `cdp_url`, `cdp_headers`, `cdp_timeout`, `managed_host`, + `managed_port`, `managed_kill_existing`, `managed_args`, `user_data_dir`, `profile_name`, `profile_store_path`, + `browser_binary`, `browser_binary_candidates` (Chrome install paths under `LOCALAPPDATA`, `ProgramFiles`, + `ProgramFiles(x86)`, `ProgramW6432` on Windows; empty elsewhere and by default), `launch_args` (Playwright MCP + arguments; Chrome's own arguments are `managed_args`), `mcp_command`, `mcp_timeout_s`, `mcp_cwd`, + `mcp_browser`, `mcp_device`, `mcp_extension`, `mcp_extension_token`, `mcp_env` (read-only mapping), + `upload_root`, `sidecar_python`, `selector_cache_path` (`None` = the default under the home directory), + `local_app_data` (Windows `LOCALAPPDATA`, for the default Chrome user-data dir). The module + declares `ENV_PREFIXES = ("BROWSER_", "PLAYWRIGHT_")`. Another surface brings its own `surfaces//config.py` + with its own settings class and prefixes; `CoreSettings` does not change. + +`BROWSER_DRIVER` keeps its name and now selects the driver: + +| `BROWSER_DRIVER` | `CoreSettings.driver` | `BrowserSettings.provisioning` | +|---|---|---| +| unset or empty | `playwright_mcp` | `BROWSER_PROVISIONING`, default `remote` (`extension` when `PLAYWRIGHT_MCP_EXTENSION` is true) | +| `managed`, `remote`, `extension` (legacy) | `playwright_mcp` (`DEFAULT_DRIVER`) | `BROWSER_PROVISIONING` if set, else the legacy value; one `DeprecationWarning` per parse from `BrowserSettings` (Python's warning registry shows it once per value and call site) | +| any other name | that value | as for unset | + +The parser does not check the name: a driver registered from another package (`register_driver`) is selected by +setting `BROWSER_DRIVER` to its name. The driver registry checks it when the driver is built +(`assemble_driver` → `DRIVERS.create`): an unregistered name is `KeyError("unknown driver ''; registered: +")` after provisioning, and the device is released; `topology_for` raises the same error when no topology is +given. `BrowserInstanceConfig.driver` takes any registered name the same way. + +Values are compared after trimming and lower-casing. An explicit `BROWSER_PROVISIONING` or a legacy value wins over +`PLAYWRIGHT_MCP_EXTENSION`; without one, the provisioner and the Playwright MCP config agree on extension mode. `BROWSER_DRIVER_BACKEND` is not read. + +`BROWSER_TOPOLOGY` selects how the runner's facade runs a task, independently of the driver: `direct` (the main +agent on the facade) or `delegated_worker` (the worker through the browser service), compared after trimming and +lower-casing; `TOPOLOGIES = ("direct", "delegated_worker")` in `config.py` is the allowed set. Unset or blank gives +`CoreSettings.topology == ""`, which means the driver's registered default (`default_topology(driver)`, "Registries"). +The parser cannot resolve it: `config.py` imports only `contract` and `surfaces`, and the built-in registrations +happen when the composition root imports `drivers/`, so a `CoreSettings` built alone would see an empty registry. +The resolution happens where the driver is known, in `topology_for(service.core)` at the facade's +`ensure_started()` ("Topology"), so an instance's `driver` override takes its own default. + +Errors: `ValueError` for an unknown `BROWSER_TOPOLOGY` or `BROWSER_PROVISIONING`, a `BROWSER_MANAGED_PORT` that is not +a positive integer, and a `PLAYWRIGHT_MCP_ENV_JSON` that is not JSON. Every other invalid value falls back to the +next key or the default. `CoreSettings` keeps an unsupported `MODEL_PROVIDER` as given (the harness path takes its +model from the parent agent and must not fail on it); `build_runtime_settings` rejects it as before. + +Public facades; their signatures do not change: + +- `RuntimeSettings` keeps its fields; `build_runtime_settings(instance, *, core_settings=None, + browser_settings=None)` builds it from the given objects: `None` core settings are `CoreSettings.from_process_env()`, + `None` browser settings are `browser_surface_settings()`. `build_playwright_mcp_config`, + `build_browser_guardrails` and `resolve_playwright_mcp_cwd` read the same two objects. `playwright_mcp_config` + (`runtime/config.py`) maps the instance (`cdp_url`, `launch_args`, the sanitized key) onto the driver's builder + `drivers/playwright_mcp/config.playwright_mcp_config(settings, *, cdp_url, launch_args, key, + required_capability)`, imported when it is called. The builder's `--caps` list is the driver's + `SERVER_CAPABILITIES` (the server's capability groups, equal to the catalog's entries after the first that are + not `policy_only`, in catalog order; a test compares the two), then the caller's `required_capability`. +- `browser_surface(environ=None, settings=None)` is `SURFACES.create("browser", environ, None, settings)` and + `browser_surface_settings(environ=None)` its `.settings`, with the process + environment when `environ` is `None`. The core modules (`config.py`, `service.py`, `session.py`) and the entry + points get the browser settings through it or from their caller and type them as `SurfaceSettings`. +- `BrowserInstanceConfig` keeps its eight positional fields and appends `provisioning` and `driver` (`""` = from + settings). `driver_mode` is an alias of `provisioning`: whichever is given sets both; two different non-empty + values raise `ValueError`. `cdp_url` stays: the input for `remote`; for `managed` the endpoint comes from the Chrome that was started. In the + Playwright MCP config a non-blank `launch_args` replaces `BrowserSettings.launch_args` and a non-blank `cdp_url` + replaces `BrowserSettings.cdp_url`. +- `BrowserSettings.with_instance(instance)` returns the settings for one isolated browser, the rules the browser + service applies (below) in one place: `provisioning` = the instance's (lower-cased), else the settings'; any other + value than `remote`, `managed`, `extension` raises `ValueError` naming `BrowserInstanceConfig.provisioning`; + `managed_port` = the instance port when above 0, else 0 (a free port) for a keyed instance, else the settings + port; `profile_name` = the instance profile name, else the sanitized key, else the settings'; `cdp_url`, + `user_data_dir`, `browser_binary` = the instance's when not blank; `launch_args` = the instance's parsed + `PLAYWRIGHT_MCP_ARGS`-style string when not blank. `instance.driver` is read by the browser service (below), not + here. An empty instance returns equal settings. `create_browser_surface(environ, instance, settings)` applies it: + the bundle's settings and `profile.settings` are the instance-applied settings. +- `BrowserSettings.mcp_server: McpServerConfig | None` carries a server config given by a caller (the WebVoyager + runner's `mcp_cfg`, through the service); the Playwright MCP driver factory keeps its command line (below). + +### Environment boundary (invariant 9) + +Only the two config modules touch the process environment: `from_process_env()` of each settings class, +`browser_surface_settings()` without a mapping, `process_environ()` (a snapshot dict) and `load_repo_dotenv()` +(loads the repo `.env`), all in +`runtime/config.py` or `surfaces/browser/config.py`. Everything else receives a settings object or a +value from one. + +- `runtime/utils/env.py` holds pure parse helpers shared by both config modules (`first_value`, `int_value`, + `parse_command_args`, `is_truthy_env` / `is_falsy_env`, `normalize_provider`, `infer_provider_from_api_base`) + and driver-free defaults; it has no environment access. The Playwright MCP launch defaults (`npx`, + `-y @playwright/mcp@0.0.78`) live in `surfaces/browser/config.py`, their only reader. +- Environments of child processes are pure functions of a mapping the caller passes: + `runtime_server_child_env(environ)` (the runtime MCP server child: the whole mapping, API key and base copied to + the provider names, blocked proxies removed) and `runtime_server_environ(environ)` (the server's tool timeout + defaults, 600 s) in `runtime/config.py`; the browser-use sidecar's minimal environment in + `drivers/browser_use/transport.py`, built from OS and `BROWSER_USE_*` names of `BrowserSettings.process_env` (the + mapping the settings were parsed from, hidden from `repr` and equality) plus `BrowserSettings.upload_root`; the + driver factory passes both through `BrowserUseConfig`. +- Entry points build settings once: `surfaces/browser/runner/main.py` from `process_environ()` (its own inputs + through `read_demo_inputs(environ)`), `playwright_runtime_mcp_server.py` from + `runtime_server_environ(process_environ())`. They pass `core_settings` / `browser_settings` to + `BrowserAgentRuntime`, which completes the browser settings, hands the core settings and the surface bundle to + `AgentService` and + takes `selector_cache_path` for its selector cache; the custom actions read `upload_root` from the context's + settings. +- Public functions keep their signatures and gain keyword-only settings or values. Where a caller passes none, the + function builds them through the config module at the same moment it read the environment before + (`build_browser_runtime_mcp_config`, `ensure_execute_signature_compat`, `build_browser_worker_agent`, + `is_browser_subagent_status_log_enabled`). + Helpers whose value has an obvious "not configured" take the value and treat `None` as not configured: + `resolve_upload_file_paths(paths, *, upload_root)`, `register_custom_action_specs(controller, *, upload_root)`, + `get_selector_cache(path)`, `discover_sidecar_python(override)`. +- The env-read architecture test allows two exceptions: the browser-use sidecar (a separate process with its own + config boundary) and the one `env_map` read in `service.py` (the CDP endpoint of the given server config, part of + the lifecycle identity), which touches the Playwright MCP child's env mapping, not the process environment. + +### Browser-agent log and the tool-call logging hook + +`openjiuwen.core` imports nothing from `openjiuwen.harness` for logging. Core owns the hook +(`openjiuwen/core/common/logging/tool_call.py`); the harness registers the browser log with it. + +- `register_tool_call_logger(matches, log)` registers `log(message, *args)` for tool names for which + `matches(name)` is true; registering the same `matches` again replaces its `log`. `tool_call_logger(name)` returns + the first registered `log` whose matcher accepts `name`, or `None`. The registry is process-wide, like the loggers + it routes to: agents are built by callers that never see the harness. +- `ReActAgent` writes one line per tool call, `Executing tool: %s with args: %s` (name, redacted arguments), to + `tool_call_logger(name)` when one is registered and to the core logger otherwise. +- `configure_browser_agent_logging(settings: CoreSettings)` (`surfaces/browser/runner/logging.py`) applies + `log_file`, `log_level`, `log_mirror_common` and `log_audit_raw` to the stdlib logger `openjiuwen.browser_agent` + and registers browser tool names with the hook: `browser_` anywhere in the name or a `mcp_playwright` prefix, + case-insensitive. It is called where a browser runtime is built: `BrowserAgentRuntime.__init__` (with its + resolved core settings) and `create_browser_agent`. Without a call, no tool-call line goes to the browser log. +- The logger keeps what it was configured with (it is process-wide): the file handler, level, propagation and the + audit flag. Until the first `configure_browser_agent_logging`, the first use of the logger configures it from + `CoreSettings.from_process_env()`, so modules that fetch it at import time keep `./logs/browser_agent.log`. +- Reconfiguring with the same file keeps the open handler (the file is not truncated again); with another file the + old handler is closed and the new file is opened in write mode; with `log_file=None` there is no file, and the + lines written through `browser_agent_log_info` / `_warning` / `_error` also go to the common logger. `log_mirror_common` propagates browser lines to the parent + loggers. Raw audit artifacts are written only with `log_audit_raw`, a log file and the browser log context, next + to the log file under `browser_agent_audit/`. +- The browser log context (`openjiuwen/core/common/logging/browser_context.py`) is entered by + `BrowserRuntimeRail.before_invoke` for the invoke when the runtime's `dedicated_log` is true, the default; while it + is set, the core logger sends every line of the invoke, the model client's usage lines included, to + `openjiuwen.browser_agent` instead of the process log (mirrored to it only with `log_mirror_common`). The WebVoyager + runner's Direct agent runs with `dedicated_log=False`, so its model-call lines reach the captured process log as the + worker's do; its tool-call lines still reach the browser log through the hook, and no raw audit artifact is written + for it. +- Drivers and their clients import nothing from `runtime/`: they log through the core logger, so inside the browser + log context their lines (the MCP clients' call lines, the browser-use sidecar's log and stderr lines) land in the + browser agent log, and outside it in the process log. + +## Provisioning (`provisioning/`) + +A provisioner makes one device ready for a driver. It is picked by name, so the code that builds the agent never +handles a CDP url or a Chrome process itself. + +- `ProvisionedDevice(kind, endpoint, extras, cleanup=None)` (`contract/device.py`, re-exported by + `provisioning/`): frozen; `cleanup` is a no-argument callable, `None` when there is nothing to stop. +- `Provisioner` (protocol): `provision(surface_settings) -> ProvisionedDevice`, `release(device)`, which runs + `device.cleanup` if there is one, and `is_alive(device) -> bool`, whether the device still answers (the driver + session re-provisions a dead one). All three are synchronous; async callers run them in a worker thread. +- `provisioning/__init__.py` registers `managed`, `remote` and `extension` on `PROVISIONERS` as lazy import + strings. The registry name is the value of `BrowserSettings.provisioning`; `PROVISIONERS.create(name, **kwargs)` + returns a new provisioner. +- Invariant: the provisioning default is `remote` (`BrowserSettings.provisioning`, `_provisioning` in + `surfaces/browser/config.py`). The only implied override is `PLAYWRIGHT_MCP_EXTENSION` true → `extension`; + `managed` is never implied, it is chosen with `BROWSER_PROVISIONING=managed` (or the legacy `BROWSER_DRIVER` + value, or `BrowserInstanceConfig.provisioning`). + +### `ManagedBrowserProvisioner` (`provisioning/managed_browser.py`) + +All three names resolve to this class; `provision` branches on `settings.provisioning`. `kind` is `"chrome"`. Its +input is typed `ManagedBrowserSettings`, a `Protocol` in the same module listing the fields it reads; +`BrowserSettings` satisfies it structurally, so provisioning imports no surface module. + +| `provisioning` | Starts Chrome | `endpoint` | `extras` | `cleanup` | +|---|---|---|---|---| +| `managed` | yes, unless a Chrome already answers on the port | `http://:` | `profile` (`BrowserProfile`), `port`, `user_data_dir`, `managed_args`, `launch_args` | `stop` | +| `remote` | no | `cdp_url` (`""`: Playwright MCP launches its own browser) | `cdp_headers`, `cdp_timeout`, `launch_args` | `None` | +| `extension` | no | `""` | `extension=True`, `extension_token`, `launch_args` | `None` | + +Managed provisioning: + +1. Profile store: the one given to the constructor (`profile_store=`), else `profile_store_path`, else + `mcp_cwd/.browser/profiles.json`. +2. The stored profile named `profile_name` is reused if it is managed, has a port above 0, a user-data dir and the + same Chrome arguments (`managed_args`). Otherwise a profile is built: port `managed_port`, where 0 means a free + port (bind to port 0); user-data dir `user_data_dir`, else the OS default Chrome profile when + `managed_kill_existing`, else `mcp_cwd/.browser-profiles/`. +3. `browser_binary` replaces the stored binary. `""` means auto-detect: `chrome`, `google-chrome`, + `google-chrome-stable` on `PATH`, then `browser_binary_candidates`, then the per-user and fixed install paths of + the OS. A configured binary that is not Chrome or does not exist is an error, with no fallback. +4. The profile is stored and selected before the start and stored again with the endpoint after it. With + `managed_kill_existing`, Chrome processes using the user-data dir are killed first. The start waits up to 30 s + for `/json/version`; a Chrome that already answers on the port is used without being owned (`stop` leaves it + running). +5. The instance that provisioned a managed device owns its Chrome: `stop`, `is_endpoint_ready`, `clear` and + `owns_process` act on it. One instance per device. + +`is_alive(device)`: a device with an endpoint (managed, remote) is alive while `/json/version` answers +with a `webSocketDebuggerUrl` or `Browser` field (1.5 s timeout, the same check as `is_endpoint_ready`); a device +without one (extension, or remote without `cdp_url`, where the driver launches its own browser) is always alive: +there is nothing to probe or to provision again. + +### The service (`runtime/service.py`, `runtime/worker.py`, `runtime/service_registry.py`) + +`AgentService` is `AgentService(_WorkerLoop)`: `service.py` holds the lifecycle (construction, the identity's +registry lease, the task binding, cancellation, the session, screenshots, `shutdown`); `worker.py` holds +`_WorkerLoop`, a plain mixin with the worker loop (`run_task`, `run_task_once`, the retry and restart predicates, +the resume task, the failure summary, the progress recording), and `TaskProgressState`, `MAX_ITERATION_MESSAGE` +and the observer context variables; the browser rail imports the two names from `worker.py`, and `worker.py` +imports nothing from `service.py`. The progress state is reached through `set_progress_state`, +`clear_progress_state`, `should_treat_as_completed`, `build_failure_summary` and `observe_worker_tool_result`; +there are no other public progress methods. + +`AgentService(core, surface, *, provider, api_key, api_base, model_name, guardrails, allowed_tool_names=None, +output_dir, identity, cancel_store=None, page_state=None)` takes neutral inputs only: the core settings (complete; +a per-instance driver is already applied), the surface bundle whose settings are complete (for the browser the +facade fills the server config, the working folder and the workspace root before), the model client fields, +`RunGuardrails`, the allowlist, the folder the screenshots and artifacts go to (`output_dir/screenshots`, +`output_dir/artifacts`), the identity string the registry keys on, and the page state (`None`: +`new_page_state(surface.profile)` at construction). The service reads no environment variable and never calls +`replace` on the settings; `service.core` and `service.surface` are the given objects. + +The service holds the `DriverSession` the worker runs on and provisions nothing itself: + +- `ensure_runtime_ready()`: without a session, `open_session(service.core, process_environ(), page_state=, + settings=surface.settings)` ("Composition root"); the session's provisioner is registered with the process + registry under the identity. With a session whose stack was stopped by a task release, `session.start()`. Then + `begin_task()`. `started` is "the stack runs". +- `run_task(task, session_id=, request_id=, timeout_s=, *, worker)` runs the worker loop (the per-session lock, + the cancel check, the attempts, the max-iteration resume, the failure context, the JSON contract and the + final-answer recovery) on the agent `worker()` returns: `run_task_once(..., worker=)` awaits it once per attempt, + and the recovery call reuses that agent. A run a rail ended with an error of its own (`result_type` `error` and a + non-empty `error`, no JSON in the output: the `ConnectionLostRail` result) is the answer without a recovery call: + `ok=False`, `final` the output, `error` the text, `status` `blocked`, no retry (the text matches no restart + marker). `_restart` is `reset()` alone (the session closed, the `agent` slot + cleared), so the next attempt's `worker()` rebuilds the worker where it is built (`DelegatedWorker.start`, + "Topology"). The service builds no agent and starts none (there is no `ensure_started`); the `agent` slot is a + plain attribute the delegated topology fills and the runner's helper removal reads. `observe_worker_tool_result` + is the progress recorder a worker's builder takes as `tool_result_observer`; `screenshot_subdir` and + `artifacts_subdir` are readable. `session_new(session_id=None)` returns the given id, else `session-`. +- `acquire_task_binding()` counts a task and calls `session.begin_task()`; `release_task_binding()` on the last + binding stops the stack (`session.stop()`, the device stays), forgets the worker and the task state. `reset()` + closes the session (stop, release) and stops the identity's registered provisioners through + `reset_lifecycle_resources(identity, fallback_service=None)`, a classmethod the browser facade also calls for an + identity whose service is gone; `detach_lifecycle_resources()` is the per-service part of that. `shutdown()` is + `reset()` plus the release. + +`ServiceRegistry` (`runtime/service_registry.py`, the process instance `SERVICE_REGISTRY`) is keyed by the +identity string and knows nothing of what it encodes: `acquire`, `activate_binding`, `release`, +`register_managed_driver`, `unregister_managed_driver`, `begin_reset` (a `ServiceResetSnapshot` of the services +and preserved drivers of one identity), `discard_idle`, `managed_driver_identities()` (the identities holding a +preserved driver) and `clear()` (tests). + +## Composition root (`runtime/assembly.py`) + +`assemble_driver(core: CoreSettings, environ, *, instance=None, registries=None, settings=None) -> DriverAssembly` +builds the driver stack from names only and returns it started. `registries` is `Registries(drivers, surfaces, +provisioners)`; `None` means the three default registries with the built-ins. The module imports no concrete +driver, surface or provisioner, names none, and holds no state. `open_session` (below) calls it for the one session +the main agent and the worker share. + +1. `surface = assemble_surface(core, environ, instance=, settings=, registries=)`, which is + `registries.surfaces.create(core.surface, environ, instance, settings)`, a `SurfaceBundle`: the surface factory + parses `environ` unless `settings` is given, and applies `instance` (the surface's per-agent identity, + `BrowserInstanceConfig` for the browser) to the settings, so provisioning, the driver factory and the tool + context all see it. The composition root never touches the settings. `assemble_surface` is public so a caller + can hold the profile (its policy rules and messages) before any driver exists; a bundle built that way and the + session's bundle agree on `messages` (constants of the surface factory) and on what their `policy_rules` + decide (built from the same settings). +2. (merged into step 1.) +3. `provisioner = registries.provisioners.create(surface.settings.provisioning)`. +4. `device = provisioner.provision(surface.settings)`, run in a worker thread. +5. The stack for the device: `driver = registries.drivers.create(core.driver, device, surface.settings)`; when + `core.surface not in driver.info.surfaces`, `ValueError` naming the driver, the surface and the surfaces the + driver serves; `stack = Tracing(RetryOnStale(Timeouts(driver, per_effect)), sink)` with `per_effect[e] = + min(profile.timeouts[e], core.tool_timeout_s)`. Precedence: the surface sets a bound per effect, and the core's + tool timeout (`PLAYWRIGHT_TOOL_TIMEOUT_S`, else `PLAYWRIGHT_MCP_TIMEOUT_S`, else `BROWSER_TIMEOUT_S`, default + 240 s) caps every one of them; the delegated worker's tool calls are bounded by the same value once more + (`ensure_execute_signature_compat`). An effect missing from `profile.timeouts` runs unbounded; the sink writes + each trace record to the debug log. +6. `await stack.start()`. + +A failure in step 5 releases the device; a failure in `start` calls `close()`. A failure in `provision` leaves +nothing to release. + +`DriverAssembly(stack, device, provisioner, rebuild=None, surface=None)`: `close()` stops the stack, then releases +the device with `provisioner.release` in a worker thread, also when `stop` raises. `rebuild(device)` is step 5 for +another device (same driver factory, surface check and middleware), not started; `surface` is the bundle of step 2, +whose `settings` are what `provision` was called with. `DriverSession` provisions again from `surface.settings` and +builds the new stack with `rebuild`. Assemblies built by hand (tests) may leave `rebuild` and `surface` out. + +`open_session(core, environ, *, page_state, instance=None, settings=None, registries=None) -> DriverSession` is +the one way a session is opened: `assemble_driver(...)`, then `DriverSession(assembly, page_state, +assembly.surface.profile.messages)`. A `ConnectionLost` from that first `assemble_driver` (the stack's first +`start()` lost its connection; the failed assembly has already stopped its stack and released its device) runs +`assemble_driver` once more, a fresh provision; a second `ConnectionLost`, or any other error, propagates. Only the +first open is retried here: once the session exists, a lost connection is the session's own restart ("Driver +session"). + +### Agent construction + +Every agent gets its session, tools and rails from this module and is built by exactly one builder: the browser +main agent by `build_main_agent` (`surfaces/browser/runner/agent.py`; `create_browser_agent` in +`openjiuwen/harness/subagents/browser_agent.py` resolves the capabilities, builds the facade when none is +given and delegates to it), which queues `BrowserRuntimeRail(runtime)` and `runtime.core_rails()` as rails at +construction (before the first invoke; the tools arrive at the first invoke); the main agent of a surface +without a `main_agent` hook by `agent_over(parts, rails, model, *, language, workspace, system_prompt="", +**agent_options)` (a deep agent with the parts' tools and prompt sections, `rails` before the profile's own; +`build_agent` and the `Direct` topology share it); the worker by `build_worker` (`runtime/agents.py`). The +`Direct` topology builds the main agent through the profile's `main_agent` hook, else `agent_over`, +`DelegatedWorker` the worker through `build_worker` ("Topology"). Nothing else creates an agent or opens a +session. + +`core_rails(profile, builders, page_state, session_of) -> (ActionPolicy, RepeatedCallGuardRail, +InterceptedActionHintRail, ConnectionLostRail)`: `ActionPolicy(builders, profile.policy_rules, page_state)`, +`RepeatedCallGuardRail()`, `InterceptedActionHintRail(profile.messages["retry"])`, `ConnectionLostRail(session_of)`, +in that order. `builders` is kept as given, not copied ("Policy rail"), and `session_of` is a zero-argument lookup +of the `DriverSession` (`None` before it exists), so the rails can be built before the driver starts over a mapping +and a session that fill later; the inputs exist before the session: the profile's constants, the page state and +the lookup. + +`build_agent_parts(session, bundle, *, capabilities, image_support, language, workspace=None, worker=False) -> +AgentParts` (`tools`, `prompt_sections`, `rails`, `builders`): + +- `tools`, `prompt_sections`: `ToolFactory(bundle.profile, session.assembly.stack.info).build(capabilities, + image_support=, session=, page_state=session.page_state, workspace=, language=, worker=)`; with `worker=True` + the extra tools flagged `worker=False` are left out. +- `builders`: the factory's `AgentTools.builders`, for the runtime's `tool_kind`. +- `rails`: the profile's only, `factory(context)` for each `profile.rails` entry, `context` being the + `ToolContext` of the same session, profile, driver info, image support, workspace and language. The core rails are + not part of the parts: the builder of the agent takes them from `core_rails` and registers them before the + profile's. + +`build_agent(core, environ, *, model, capabilities, image_support, language, system_prompt="", workspace=None, +instance=None, registries=None, **agent_options) -> (DeepAgent, DriverSession)`: + +1. `assemble_driver(core, environ, instance=, registries=)`; +2. `DriverSession(assembly, BrowserPageState(), profile.messages)`; +3. `build_agent_parts(session, assembly.surface, ...)`; +4. `create_deep_agent(model, system_prompt=, + tools=, rails=[*core_rails(profile, parts.builders, session.page_state, lambda: session), *parts.rails], + workspace=, language=, + **agent_options)`. + +The driver is started before the agent exists (the Playwright MCP native tools exist only after `start()`). A +failure in steps 2–4 closes the session (stop, release). The caller owns the session: `begin_task()` per task, +`close()` when the agent is done. Neither function names a driver, a surface or a provisioner. + +`build_worker(session, bundle, *, provider, api_key, api_base, model_name, max_steps, core, capabilities, +tool_result_observer, screenshot_subdir, artifacts_subdir, image_support=False) -> ReActAgent` (`runtime/agents.py`, +async) is the worker's builder, called by `DelegatedWorker.start` with the default `False` ("Topology"): +`build_agent_parts(session, bundle, capabilities=, image_support=, language=WORKER_LANGUAGE, worker=True)`, so +image support decides both the screenshot-only tools (left out without it) and the image blocks in the tool results; +the worker prompt names `browser_take_screenshot` only as "when it is present"; `build_browser_worker_agent(provider=, api_key=, api_base=, model_name=, +max_steps=, tools=parts.tools, screenshot_subdir=, artifacts_subdir=, tool_result_observer=, settings=core, +driver_name=)`; then `register_rail` for `core_rails(bundle.profile, parts.builders, +session.page_state, lambda: session)`, the parts' rails and `FinalAnswerRecoveryRail()`, in that order. The model identity and +`max_steps` are the service's own values, not `core`'s: `core` gives the worker temperature, top-p, tool timeout +and logging. `WORKER_LANGUAGE` (`"en"`, written once in `agents.py`, re-exported by `topology.py`) is the language of +every agent the facade's entry points build: the worker, and the main agent in the `Direct` topology. + +The task rules (`browser_task_rules()` in `surfaces/browser/tools.py`: `{"en": ..., "cn": ...}`, nine rules under +"Task rules:", the first giving today's local date as `YYYY-MM-DD`) are one block both agents read: the worker +prompt renders the English block before its final-output contract, and the main agent's runtime rail adds it as the +`browser_task_rules` prompt section (priority 85, after the date-picker section) on every model call. Invariants: the +date is the one at render time (per task for the worker, per model call for the main agent); the block stays under +15 lines; the prompt goldens show the date as ``. + +The worker prompt (the profile's `worker_prompt`, `build_browser_worker_system_prompt(..., driver_name)` in +`surfaces/browser/tools.py`; `build_worker` hands it to `build_browser_worker_agent(..., worker_prompt=)` and the +loop re-renders it per task by `refresh_browser_worker_system_prompt(agent, worker_prompt=, ..., driver_name)`) +names the driver in two sentences, "with +`` tools and approved runtime helper tools only" and "required by a `` action", from +`worker_driver_name(info)`: `DriverInfo.display_name`, else `name`. On Playwright MCP the prompt reads exactly as +before (the `runner` golden); on browser-use it says "browser-use tools" / "a browser-use action". The prompt's +tool names (`browser_snapshot`, `browser_fill_form`, `browser_evaluate`, `browser_tabs`) are the same on both +drivers. + +## Browser surface profile (`surfaces/browser/`) + +`create_browser_surface(environ)` returns `SurfaceBundle(settings, profile)`; `profile.settings` is the same +`BrowserSettings`. No driver name appears in `surfaces/` (architecture test); tool names do. + +| Field | Value | +|---|---| +| `commands` | `Navigate`, `Back`, `OpenContext`, `SelectOption`, `UploadFile`, `FillForm`, `Drop`, `GetPdf`, `GetConsole`, `GetNetwork`; the function `upload_paths(paths, upload_root)`, the one resolution rule for upload and drop paths (see "Policy rail") | +| `renderer` | `BrowserRenderer` | +| `tool_specs` | `BROWSER_TOOL_SPECS` (`tools.py`) | +| `capability_catalog`, `capability_tools` | `capabilities.py` | +| `timeouts` | 240 s for every `Effect` | +| `messages` | `driver_restarted`: "the browser restarted; take a fresh observation"; `driver_lost`: "the browser connection was lost again; the task cannot continue"; `context_changed`: "Call browser_snapshot again and use refs from generation {generation}." (a format string); `retry`: "Then retry the action (take a fresh browser_snapshot if the page changed)." (the last line of the intercepted-action hint) | +| `extra_tools` | `helpers.EXTRA_TOOLS`: `ProbeInteractivesTool`, `ProbeCardsTool`, `BatchTool`, `FindTool`, `RecallOffloadTool` (see "Helpers") | +| `capabilities` | `DEFAULT_BROWSER_CAPABILITIES` (`capabilities.py`) | +| `custom_actions`, `page_metadata`, `policy_probe`, `activate` | `helpers.register_custom_actions`, `helpers.page_metadata`, `helpers.probe_for_policy`, `helpers.activate_page` | +| `semantic_tracker` | `SemanticStateTracker` (`semantic/state.py`) | +| `worker_prompt` | `tools.build_browser_worker_system_prompt` | +| `main_agent` | `runner/agent.main_agent`: `build_main_agent` with the worker's sampling (`core.worker_temperature`) | +| `policy_rules` | `policy.policy_rules(settings)`: the clipboard rules (`POLICY_RULES`) and the upload confinement to the settings' folders (see "Policy rail") | +| `rails` | empty | +| `scene` | `None` (the conformance suite takes `browser_scene()` directly) | + +`BrowserContext` also carries `tab_urls` (parallel to `tabs`) and `current_tab` (index of the focused tab); both are +empty when the driver does not report them. The Playwright MCP driver fills them from every header; browser-use does +not yet. + +### Tool specs (`tools.py`) + +The browser dialect: 17 specs whose name, description and schema equal the browser-use golden +(`snapshots/golden/browser_use/*.json`), except `browser_tabs`, whose `action` enum adds `new`. Element arguments +are `generation_id` (required) plus one of `target_id` (page-state target, through `RefResolver.resolve`), `ref` +(`DriverRef(ref, generation_id)`) or `selector` (`Locator(css, None, generation_id)`), in that precedence; +`browser_drag` uses the `source_` / `target_` prefixed forms. Boolean arguments accept booleans and the strings +`1/true/yes` and `0/false/no`. + +| Tool | Command | +|---|---| +| `browser_navigate` | `Navigate(url)`; `wait_until`, `timeout_ms` accepted and ignored | +| `browser_navigate_back` | `Back` | +| `browser_click` | `Click(target, button, count=max(1, click_count))` | +| `browser_type` | `TypeText(target, text, submit=press_enter, sensitive, clear)` | +| `browser_press_key` | `PressKey(key)` | +| `browser_take_screenshot` | `Observe(include_screenshot=True, full_page)` | +| `browser_snapshot` | `Observe(include_screenshot)` | +| `browser_tabs` | `list` (default) → `Observe()`; `new` → `OpenContext()`; `select` → `SwitchContext(contexts[index])`; `close` → `CloseContext(contexts[index])`, or the current context without `index` | +| `browser_close` | `CloseContext(current_context)` | +| `browser_select_option` | `SelectOption(target, (value or label,))` | +| `browser_evaluate` | `RunScript(function, args)` | +| `browser_drag` | `Drag(source, destination)`; `steps` checked and ignored | +| `browser_file_upload` | `UploadFile(target, paths)` | +| `browser_hover` | `Hover(target)` | +| `browser_handle_dialog` | `HandleDialog(accept, promptText or prompt_text)` | +| `browser_drop` | `Drop(target, paths, data=((mimeType or mime_type, data), ...))` | +| `browser_fill_form` | `FillForm(fields)`: `textbox`/`spinbutton`/`searchbox` → textbox, `slider`, `combobox`/`listbox`/`select` (value, else label), `checkbox`/`switch` → checkbox, `radio`; checked values `1/true/yes/on/checked` → `"true"` | + +Invalid arguments raise `ValueError` with the wording of the tools they replace (`'generation_id' is required`, +`exactly one of target_id, ref, or selector is required`, `'click_count' must be an integer`, ...); a tab index +outside `contexts` and a close with no current context are `ValueError` too. Seven specs have no schema and no +decoder: `browser_mouse_click_xy` (`Click`), `browser_mouse_move_xy` (`Hover`), `browser_mouse_drag_xy` (`Drag`), +`browser_mouse_wheel` (`Scroll`), `browser_pdf_save` (`GetPdf`), `browser_console_messages` (`GetConsole`), +`browser_network_requests` (`GetNetwork`). `browser_find` is not a spec: it is the `FindTool` extra tool over the +current observation (see "Helpers"). + +### Renderer (`renderer.py`) + +| Method | Output | +|---|---| +| `observation(obs)` | `native_text` verbatim, then the screenshot as an image block | +| `result(action, result)` with `ok=False` | `detail`, prefixed with `### Error` and a newline unless it starts with it | +| `result(RunScript, …)` | `### Result`, the value, then `detail`; a string as is, `None` as `undefined`, anything else `json.dumps(indent=2, ensure_ascii=False)` | +| `result(Observe() without screenshot, …)` | `### Result` and the tab list, then `detail` | +| `result(OpenContext / SwitchContext / CloseContext, …)` | `detail`; the tab list when `detail` is empty | +| `result(other, …)` | `detail` | +| `error(action, error, observation)` | `### Error` and the message; with an observation (the argument, or the one a `StaleRef` carries) a blank line and its text, then its image | + +The tab list is one line per tab of the result's surface context (or its observation's): `- : ` + `(current) ` +for `current_tab` + `[]` + `(<url>)` when known (`tab_urls[i]`, or the surface URL for the current tab). +A screenshot in `result.value` becomes an image block; free observations in a result are not rendered. Empty +text produces no block. + +### Capability catalog (`capabilities.py`) + +`CAPABILITY_TOOLS` maps every entry of `DEFAULT_BROWSER_CAPABILITIES` (the catalog, in the same module; +unchanged public API) to those of its tool names that a spec or the `browser_find` extra tool serves: `core` equals +`CORE_BROWSER_TOOL_NAMES`, which includes `browser_take_screenshot`; `vision` keeps the four `xy`/wheel tools; +`extended_interaction` keeps +`browser_close`, `browser_drag`, `browser_drop`; `pdf`, `devtools`, `network` keep one tool each; `advanced_code`, +`unsafe_dev`, `config`, `storage`, `testing` are empty. `CAPABILITY_CATALOG` gives each entry the +`(command, ref kind)` pairs of its specs: element tools take `DriverRef` and `Locator`, `browser_file_upload` and +`browser_evaluate` also `None`, the three `xy` tools `PointRef`, all others `None`. `DEFAULT_BROWSER_CAPABILITIES` +is a tuple of the contract's `Capability` (`BrowserCapability` is that type under its old name); its +`advanced_code`, `extended_interaction` and `unsafe_dev` entries are `policy_only`, and +`POLICY_ONLY_BROWSER_CAPABILITY_NAMES` is derived from them. The profile's `capabilities` is this tuple. +Invariants: no two entries share a tool name; `browser_take_screenshot` is in `core` and reaches an agent exactly +when the agent has image support (the factory's screenshot rule, "Tool factory"), so the WebVoyager runner, which +grants `CORE_BROWSER_TOOL_NAMES`, gets it with image input and not without. + +### Helpers (`surfaces/browser/helpers/`) + +Browser tools and functions built on the driver session through a `ToolContext` (see "Tool factory"); every command +they send goes through `context.execute` / `context.observe`, which record it in the page state. The factory appends +the five tools (`EXTRA_TOOLS`: both probes, the batch, find, recall) to a main-agent build, so the main agent gets +them at its first invoke; a worker build (`worker=True`) leaves out the tools flagged `worker=False` (recall), so +the worker gets both probes, the batch and find; the runtime facade calls `probe_for_policy`, `activate_page` and +`page_metadata` through the profile (`policy_probe`, `activate`, `page_metadata`). +`CUSTOM_ACTIONS` are not tools: they are the controller's custom actions as functions of the context. +`register_custom_actions(controller, context)`, the profile's `custom_actions`, registers them on the controller +when the runtime's session starts, for `browser_custom_action`: first `register_custom_action_specs(controller, +upload_root=context.settings.upload_root)` (the +specs of the three actions and the `list_upload_files` action with its spec over the upload root, `None` = not +configured), then the three handlers bound to the context. The server entry registers the specs on its default +controller the same way, with no upload root, after the builtin actions. The builtin actions (`ping`, `echo`, +`browser_task`, `run_browser_task`) stay in `surfaces/browser/runner/actions.py`, which names no browser spec. + +Each helper tool declares `flags: ToolFlags` (class attribute): the probes `ToolFlags(requires=(RunScript,))` (`READ`, +not exempt from budgets, progress, offered to a worker), find `ToolFlags()` (it only observes), the batch +`ToolFlags(effect=ACT, requires=(Click, TypeText, PressKey, SelectOption, FillForm, Navigate, Wait, SwitchContext, +RunScript))` (every command its steps build), the recall tool `ToolFlags(exempt_from_budget=True, worker=False)`. A probe or find result is +`ToolOutput(success, data={"content": <payload>, "command": CommandRecord}, error)`: the model reads `content` as +before (a probe's payload dict is rendered as JSON, find's text as is), and `command_record` finds the record. The +record's `effect` is the declared one (`READ`), not the command's: a probe runs `RunScript` but changes nothing the +rails count. + +| Helper | Commands | Contract | +|---|---|---| +| `ProbeInteractivesTool` (`browser_probe_interactives`) | `RunScript(code=<page function>, args=<params>)`, a second time if the value is not a JSON object | card equals the runtime helper's; `max_items` 1–30 (default 30); the probe page function of `probes/interactives.py` without its `async (page) => page.evaluate(...)` wrapper; result dict as the runtime helper's (`ok`, `error`, `url`, `title`, `generation_id`, `count`, `elements`, `page_state`, `diagnostics`, `audit`) | +| `ProbeCardsTool` (`browser_probe_cards`) | same | card equals the runtime helper's; `max_cards` 1–12 (default 12); the page function of `probes/cards.py` (its script is the resource `cards.js`, the parameters inlined at build time); selector cache records go in and out as before; primary links registered in `selector_primary_links` | +| `BatchTool` (`browser_batch_interact`) | one command for a step that maps to one; otherwise page scripts and one command per step | see below | +| `FindTool` (`browser_find`) | none, or one `Observe` when the page state has no fresh observation | see below | +| `RecallOffloadTool` (`browser_recall_offload`) | none | `BrowserOffloadRecallTool(context.workspace, language=context.language)`: card and behaviour of `surfaces/browser/offload_recall.py` (the old `browser_move.offload_recall` path is a deprecated alias of the same module); it reads `<workspace>/context/<ContextUtils.offload_session_dir_name(session id)>_context/offload/` and, when that name differs from the session id and the id holds no path separator, the literal id's folder after it (sessions stored before ids were made path-safe); the first is the directory the core's tool-result offloader writes (a path-safe id is its own name; any other id maps to one safe component plus a digest); no record. The core's `OffloadMessageBuffer` reloads from the same directory and, when the directory name differs from the id, afterwards from the old literal `<session id>_context/offload/` folder of sessions stored before the renaming, but only for an id without `/` or `\\` that is not `.` or `..`; new writes use the safe name only | +| `probe_for_policy(context, source, params)` | `RunScript(code=source, args=params)`, up to two | the F_05 result: elements become targets, `generation_id` added; failure envelope `{"ok": False, "error", "elements": [], "page_state"}` on a `DriverError`, a refused script or a result with `ok: false` | +| `activate_page(context, url) -> bool` | `Observe` when no context is known, then `SwitchContext(<tab whose URL is url, else the current context>)` | `False` on a `DriverError` or no context; the switch is recorded, so the generation advances | +| `CUSTOM_ACTIONS["browser_get_element_coordinates"]` | `Navigate` when `url` is given; one `RunScript` unless only the four coordinates are given | the controller's parameters and aliases; `querySelector` and the bounding box, else the element whose text matches (a Playwright `:has-text("...")` contributes its text); `{ok, source, target, error}` with the controller's error texts | +| `CUSTOM_ACTIONS["browser_drag_and_drop"]` | `Navigate` when `url` is given; elements: one `RunScript` that stamps both, then `Drag(Locator, Locator)` on the stamps; points: `Drag(PointRef, PointRef)` of the last screenshot | `{ok, message, source, target, steps, delay_ms, error}`; offsets only move the reported points, `steps` and `delay_ms` are echoed, not applied | +| `CUSTOM_ACTIONS["browser_set_input_files"]` | `UploadFile(Locator(css), paths)` when `driver_info.actions[UploadFile]` accepts `Locator`; else `Click(Locator(css))` then `UploadFile(None, paths)` | selector default `input[type="file"]`; `{ok, selector, paths, error}`, plus `hint` on a strict-mode error; a `DriverError` in any custom action is `{ok: False, error: "<action> failed: ...", session_id, request_id}` | +| `page_metadata(context) -> (dict, error)` | one `RunScript` (the page-measure function of the metadata script, no arguments) | `url`, `title` from the page state; `tabs` (`index`, `current`, `url`, `title`) from the last `BrowserContext` (current tab from `current_tab`, else the page state's current context; a tab URL from `tab_urls`, else the surface URL for the current tab); `page_position`, `semantic_state` from the script; `({}, error)` on a `DriverError` or an unparsable value | + +A probe result whose `url` differs from the page state's (non-empty) URL resets the page state (new generation, no +contexts) before its targets are registered: a script result does not report the address on every driver. Every +dict in the result with `selector_hint` or `generation_id` gets the current generation id. A `DriverError` becomes +the failure envelope (`browser_probe_* failed: <message>`) with `CommandRecord.failed` of the script; a task failure +from the session propagates. The record of a result describes the last script sent (`value` = its raw value). + +`BatchTool` (`batch.py`; card and step contract in `batch_steps.py`): card equal to the runtime tool's (golden byte +for byte). Before anything runs: the steps are validated (the controller's per-op contract, then the target contract: +a mutating op needs a `target_id`, a read op exactly one of `target_id` / `selector`; failure `batch_validation_failed: +<first error>` with `validation_errors`, no record); a stale `generation_id` refreshes probe targets (equivalent +current target, else one `RunScript` that checks the stale target's selector) and refuses snapshot refs and +model-written selectors outside conditions; targets resolve through the page state (`resolve_target`; a mutating op +refuses a probe target that is not visible, enabled and actionable). A failure there is +`batch_target_validation_failed: <message>` with `candidate_fresh_targets`. + +One step that maps to a command runs as that command alone, on the target's own ref (`DriverRef` of a snapshot +target, `Locator(css)` of a probe target): `click` -> `Click` (a card's primary link -> `Navigate`), `fill` / `type` -> +`TypeText` (`slowly` for `type` or a `delay_ms`), `select_option` with values -> `SelectOption`, `press` without a +target -> `PressKey`, `wait_for_text` -> `Wait(until_text)`, `sleep` -> `Wait(seconds)`, `screenshot` -> +`Observe(include_screenshot, full_page)`; `execution_mode: "primitive"`, `metrics.command` the command type name. + +Several steps (or one with no single command) run as a loop (`execution_mode: "commands"`): + +1. snapshot targets are stamped on the page (`RunScript` on the `DriverRef`, attribute `data-openjiuwen-target-id`) + and their page-state locator becomes the stamp selector; +2. preflight: one `RunScript` checks every target of every step (the primary target of click, fill, type, + autocomplete, select_option, set_checked, the extractions and a press with a locator; the option of + select_visible_text and of an autocomplete given by target): exactly one match (more fails at once), visible, + enabled unless an extraction, polled until the step timeout. A failure is `batch_preflight_failed: steps[i] + <message>` and nothing acts; +3. each step locates its target again with one `RunScript` that finds it (CSS, or page-side stand-ins for role + + name, label, placeholder, test id and text, with `exact`), checks the same conditions (polling while it is + missing or hidden), stamps it with a fresh mark and, when asked, focuses it or reads its text, its value or the + value of the option at an index; the step then sends its command on `Locator(css=<stamp>)`: `Click` (one retry + when the error reads detached, intercept, not stable or outside of the viewport), `TypeText`; for `type` and + `autocomplete` a `Click`, `TypeText("")` and `TypeText(slowly)` (typing key by key does not clear); `SelectOption`; + `FillForm` with a checkbox field for `set_checked`; `PressKey` after focusing. Waits poll a page script (selector + state, text, load state, first card title, sort state, result count, DOM text change, stable text, with + `stable_ms`) or, for `wait_for_url` / `wait_for_tab`, `Observe` (tabs from `BrowserContext`), then switch with + `SwitchContext`; `sleep` is `Wait(seconds)`; `screenshot` is `Observe(include_screenshot)`; +4. a failed step stops the batch unless the step is `optional` or `continue_on_error` is set; a `ConnectionLost` + always stops it. The batch has the controller's `global_timeout_ms`, checked before each step. + +The result keeps the controller's keys (`ok`, `status` completed / partial / failed, `error`, `action`, +`execution_mode`, `session_id`, `request_id`, `generation_id`, `steps` with `index`, `op`, `ok`, `status`, +`elapsed_ms`, `phase`, `error`, `extracted`, `field_provenance`, `conditions` with `observed`, `metrics`), then +`generation_id` (current), `generation_recovered_from` and `page_state`; extracted fields join the page state's field +coverage. The record: one `CommandRecord` for the batch, `command` and `action` of the last command sent, `effect` +the strongest of the commands that ran (page scripts and observations count as `READ`; `READ` < `SCRIPT` < `ACT` < +`NAVIGATE`), `ok` as the payload, `dispatched` / `context_changed` when any command was dispatched / changed the +context, `value` the payload; no record when nothing was sent. + +`FindTool`: name `driver_info.tool_name_prefix + "browser_find"`; description and schema from +`driver_info.native_tools["browser_find"]` when present (Playwright MCP: the server's, byte for byte), else the +browser-dialect schema (`query`, `regex: bool`, `limit` 1–100 default 20, `generation_id`). Arguments: `text` +(case-insensitive substring) or `regex` (a pattern, or `/pattern/flags` with `i`, `m`, `s`; `g` dropped), exactly +one; in the browser-dialect form `query` is the text, or the pattern when `regex` is `true`, and a stale +`generation_id` is refused. A bad argument is `### Error`, a newline and the message, and sends nothing (no +record). The search runs over `page_state.fresh_observation`, or a new `Observe` when there is none, over its +`native_text`, inside its fenced `yaml` block when it has one. The answer is `### Result`, a newline and the +server's text: `No matches found for <query>.`, or `Found N match(es) for <query>:` and one snippet per group of +matches (each match with 3 lines of context, merged when they touch, under the lines of its ancestors, `...` where +lines are skipped off the match paths), snippets separated by `----`; `<query>` is `"text"` or `/pattern/flags`. +The record: `command=Observe`, `effect=READ`, `ok=True`, `dispatched` = whether it observed, `value` = the answer +text; a `DriverError` while observing is the same error text with `CommandRecord.failed`. + +## Runtime state and recovery (`runtime/`) + +Three pieces sit between the model-facing tools and the driver stack; the runtime facade (below) wires them into +the main agent. + +### Page state (`runtime/page_state.py`, `PageState`; `surfaces/browser/page_state.py`, `BrowserPageState`) + +The core's `PageState` is the surface-neutral generation and target registry; it names no browser field. Generations +keep the `g<n>` format and one counter. A surface that keeps more state subclasses it and hands the class to the +core as `SurfaceProfile.page_state`; `assembly.new_page_state(profile)` builds the profile's (the core's own when +`None`), for `build_agent` and, at construction, for `AgentService.page_state`, which the runtime reads. No core +module imports a surface's page state. + +The core keeps: `page_id` (prefix `state-`), `generation` / `generation_id`, `identity`, `last_observation` / +`fresh_observation` / `note_dispatched`, `free_observations`, `screenshot_generation`, `surface_context`, +`contexts`, `current_context`, the ref and selector target registry (`Target`: id, generation, source, locator, ref, +selector, role, name, text, visibility and action flags), `record`, `check`, `generation_of`, `resolve`, +`resolve_target`, `get_target`, `validate_generation`, `advance`, `reset`, and `export()` (`page_id`, +`generation_id`, `interactives`). Hooks a subclass overrides: `_note_surface(surface)` (after each recorded surface +context), `_on_advance()`, `_on_observation(observation)` (after each registered observation, partial ones +included), `_forget_target(target)` (a target trimmed from the history); class attributes `target_type`, +`id_prefix`, and the two hints of the stale-generation and unknown-ref messages (`observe_again`, `snapshot_again`). + +`BrowserPageState(PageState)` adds what only the browser has: `url` (the core's `identity` under its browser name) +and `title`, the tab list (`contexts` from `BrowserContext.tabs`), probe and card targets (`BrowserTarget` adds +`href`, `region`, `kind`, `field_name`, `extractable`, `selected`), `register_interactives`, `register_cards`, +`CARD_EVIDENCE_FIELDS` and the field coverage, the requested fields, the captcha/login/access blockers, the primary +links, target refresh and rebind (`refresh_target_id`, `rebind_target`, `recovery_candidates`, +`update_target_locator`), the Playwright text-snapshot registration (`register_ax_snapshot`, +`replace_ax_snapshot`), `observe(url=, title=)`, `advance(url=, title=)`, and `export()` / `export_summary()` with +`url`, `title`, `cards`, `field_coverage`, `blockers` in their former key order. Its messages are the ones the model +read before the split. + +- `record(action, outcome) -> outcome`: `outcome` is an `ActionResult`, or the `Observation` of an `observe()`. A new + generation starts when `result.context_changed`, when the observation's context id differs from the current one, + when the Jaccard distance between the `(ref, role)` sets of the last and the new observation is above + `element_change_threshold` (constructor parameter, default `0.5`), or when the surface context's non-empty + `identity` differs from the known non-empty one without the flag (backstop; for the browser, the URL). A new generation drops `last_observation`, `surface_context`, + `contexts` and `current_context`: context ids are valid only until the next context change. The returned value is + stamped: `ActionResult.generation`, `Observation.generation` and every `Element.ref.generation`. Observation + elements replace the current generation's AX targets (same targets as a complete text snapshot). A non-empty + `identity` becomes the known one; the browser page state takes the context's `tabs` as `contexts` and its title; + `observation.context` becomes `current_context`. +- A scoped `Observe` (`action.scoped`) with an `Observation` outcome is partial: no generation check, no change to + `last_observation`, `current_context`, `contexts`, the address or `fresh_observation`. The refs it shows join the + current generation (new AX targets; existing ones are kept), the refs it does not show stay valid, as a model + snapshot registered them before the switch. The returned observation is stamped like any other. +- `free_observations` counts observations that came with an `ActionResult` of a command other than `Observe`. +- `fresh_observation`: `last_observation` while it is of the current generation and no command was recorded after + it without an observation, nor noted with `note_dispatched()` (a command that reached the surface and failed); + else `None`. A recorded observation (from `observe()` or free with a result) makes it fresh again. +- `screenshot_generation`: the generation of the last recorded observation with a screenshot (`""` before the first); + kept across generations, so `check` refuses a point on an older screenshot. +- `check(action)`: every ref in `iter_refs(action)` whose generation is older than the current one → + `StaleRef(dispatched=False)`, one message per stale ref (`Stale AX ref e1 belongs to g0; current generation is + g1.`). Refs with an empty generation and unknown refs of the current generation pass; the driver decides. On the + Playwright MCP dialect every element target the model sends is such a ref (Refs). +- `RefResolver`: `contexts`, `current_context`, `screenshot_generation`, `generation_of(ref)` (the generation the ref + was last registered in, the current one for a ref never seen) and `resolve(target_id, generation)` + (`resolve_target`, then `DriverRef` for an AX target, `Locator(css)` for a probe target with a validated selector, + `ValueError` otherwise). +- `reset()`: a new generation and no contexts or last observation. + +### Driver session (`driver_session.py`, `DriverSession`) + +`DriverSession(assembly, page_state, messages)` forwards `execute` and `observe` to `assembly.stack`; `messages` must +hold `driver_restarted` and `driver_lost` (read at construction). `page_state` is readable; `lost` is the `driver_lost` +message once a loss ended the task, else `None`. Sessions are opened by +`assembly.open_session`, which retries the first open once on `ConnectionLost` ("Composition root"); the table +below is what the session does with a connection lost after that. + +| Event | Behaviour | +|---|---| +| first `ConnectionLost` in a task (any `dispatched`) | `stop()` (driver errors ignored); then `provisioner.is_alive(device)`: alive → `start()` the same stack; dead → `provisioner.release(device)`, `provisioner.provision(surface_settings)`, `assembly.rebuild(new device)` (a failure here releases the new device), the session's assembly takes the new stack and device, `start()` the new stack; then `page_state.reset()` and the call raises `ConnectionLost(driver_restarted, dispatched=<original>)` | +| restart fails (`start`, `provision`, `rebuild`) | task failure: `BaseError` `DEEPAGENT_RUNTIME_ERROR` with `driver_lost`, the restart error as cause; `lost` is set | +| second `ConnectionLost` in the same task | task failure (the same `BaseError`, this loss as cause), no restart; `lost` is set | +| `begin_task()` | resets the loss counter and clears `lost` | +| `stop()` | `stack.stop()`; the device stays (between tasks) | +| `start()` | after `stop()`: `provisioner.is_alive(device)` → alive: `stack.start()`; dead: release, provision again, `rebuild`, start the new stack (the restart above without the stop). A `ConnectionLost` from that start gets the same single restart (stop, liveness, re-provision and rebuild when dead, start) and `page_state.reset()`; a second failure is the task failure above (`driver_lost`, the second error as cause). The loss counter is not touched | +| `health()` | `stack.health()` | +| `close()` | `assembly.close()` with the current (possibly rebuilt) stack and device | + +The failed command is never sent again. `health()` is not polled: the facade's `runtime_health` asks it. Liveness is +asked of the provisioner, not of the driver: a driver may start without touching its device (Playwright MCP +connects over CDP lazily). + +Invariant (after the single restart a second loss ends the task): once `lost` is set, no agent on the session makes +another model call. The `BaseError` still fails the call that met the loss (a factory tool, a helper, the runtime's +own capture or metadata call: the model may be mid-turn, and the ability manager turns a tool's exception into a +tool message), and `ConnectionLostRail` ("`ConnectionLostRail`") finishes the run before the next model call with +`{"output": <driver_lost>, "result_type": "error", "error": <driver_lost>}`. The worker loop takes that result as the +blocked answer ("The service"); the browser main agent's rail marks the task `blocked` with `terminal_reason` +`browser_connection_lost` ("`BrowserRuntimeRail`"). The restart rule is unchanged: one restart per task, the counter +and `lost` cleared by `begin_task()`. + +### Policy rail (`runtime/rails/action_policy.py`, `ActionPolicy`) + +`ActionPolicy(builders, rules, page_state)` is a `BaseSecurityRail` on `BEFORE_TOOL_CALL`. `builders` maps a +model-facing tool name to the pure argument decoder of its spec (`ToolSpec.build` / `NativeTool.build`); the rail +keeps the mapping it is given and reads it per call, so a decoder added after construction (the Playwright MCP +native tools exist only after `start()`) is enforced from then on. `BaseSecurityRail.tool_names` is not passed: +it only feeds `get_tools()`, never a check. For a tool in `builders` it parses the arguments (a JSON string or a +mapping), decodes the command with the page state as `RefResolver` and calls `rule.check(action, +page_state.surface_context)` for each rule whose `command_type` matches. +A reason, or a `ValueError` from parsing or decoding, denies the call: + +- `extra["_skip_tool_calls"][tool_call_id] = True` (per call; the ability manager returns the prepared result + without running the tool); +- `tool_msg`: content `### Error\n<reason>`, metadata `executed=False`, `state_changed=False`, `denied=True`; +- `tool_result`: `PolicyDenied(reason)` (`dispatched=False`). + +Tools without a decoder pass unchecked. + +The browser surface's rules (`surfaces/browser/policy.py`, `POLICY_RULES`) keep the host's clipboard out of pages: +the managed Chrome shares the OS clipboard. Invariants: a `PressKey` whose last token is `v` with a +control/ctrl/meta/cmd/command/ControlOrMeta modifier, or `Insert` with shift, is denied; a `RunScript` whose code +reads the clipboard (`clipboard.read`/`readText`, dotted or bracketed, or `execCommand('paste')`) is denied; the +reason is `CLIPBOARD_DENIED`, which tells the model to read the value from the page and type it. `TypeText` has no +rule: both drivers insert its text literally. + +The script rule is a text match and is early feedback only; the page enforces the same rule. The resource +`surfaces/browser/probes/clipboard_guard.js` runs in every document before the page's own scripts: it replaces +`Clipboard.prototype.read` / `readText` (and the instance's, when `navigator.clipboard` exists) with functions that +return a promise rejected with the `CLIPBOARD_DENIED` text, and makes `Document.prototype.execCommand("paste")` +(and the document's) return `false`; the properties are not configurable, so the page cannot restore them, and a +second install on the same document is a no-op. Each driver installs it at `start()` (see the drivers): a script +that reaches the clipboard through an alias (`const c = navigator.clipboard; c.readText()`) passes the regex and +is rejected in the page, on both drivers. Shipped as package data (`probes/*.js`). + +`upload_rules(upload_root, workspace_root)` confine the files a page receives. One resolution rule, +`surfaces/browser/commands.upload_paths(paths, upload_root)`, is applied by the rule and by both drivers, so the +path the rule judges is the path the driver sends. Invariants of `upload_paths`: wrapping quotes are stripped and +`~` expanded; an absolute path is resolved (symlinks followed) and returned as it is; a relative name is +`upload_root / name`, resolved, and must still lie inside the upload root (a `..` or a symlink that leaves it is +`ValueError`); a relative name without an upload root is `ValueError`; the process working directory and a +driver's own directory are never a root. The `ValueError` message tells the model what to do (give an absolute +path, or set `BROWSER_UPLOAD_ROOT` and name a file under it; `list_upload_files`). The rule denies with that +message, else checks every resolved path against the allowed roots, `BrowserSettings.upload_root` +(`BROWSER_UPLOAD_ROOT`) and `BrowserSettings.workspace_root`: a path outside both is denied; with neither +configured every path is. A `Drop` without paths passes. The reason names the paths and the allowed folders and +points the model at `list_upload_files`. `workspace_root` is the agent's workspace: the runtime facade sets it on +its settings (`dataclasses.replace`) when it has a workspace and the settings have the field, so the main agent, +the worker, the batch and the custom actions judge with the same folders; an agent built through `build_agent` +alone has none. `policy_rules(settings)` is `POLICY_RULES` plus these two. + +Helpers that execute commands without the rail check the same rules: the batch calls +`policy_denial(action, page_state.surface_context, policy_rules(context.settings))` before every command and fails +the step with the reason, so the command never reaches the driver (it has no upload or drop step today); +`set_input_files` checks its `UploadFile` the same way and answers `{"ok": False, "error": <reason>}` without a +command. + +## Tool factory (`runtime/tool_factory.py`) + +`ToolFactory(profile, driver_info).build(capabilities, *, image_support, session, page_state, workspace=None, +language="cn") -> AgentTools` turns the selected tool specs into one `CommandTool` each; `workspace` and `language` +only reach the `ToolContext`. `capabilities` are resolved entry names (the caller adds `core`, as +`resolve_browser_capabilities` does). The module names no surface, driver or tool (architecture tests). + +### Selection and the four rules + +For each entry, `profile.capability_tools[entry]` gives the spec names in order (a name is emitted once) and +`profile.capability_catalog[entry]` the `(command, ref kind)` pairs. Per selected spec: + +1. **Handled.** Every command of the spec is in `driver_info.actions` (`Observe` always is: every driver + observes), and when the entry pairs that command only with ref kinds, the driver accepts at least one of them. + Otherwise the spec is skipped. +2. **Native.** `driver_info.native_tools[spec.name]` exists → its description, schema and `build`. +3. **Profile form.** Else `spec.schema` and `spec.build`, with `spec.description`, else the first + `driver_info.tool_hints` entry of the spec's commands, else the command's docstring. +4. **Generated.** Else, for a spec with one command and ref kinds the driver accepts: a schema with `ref: string` + (`DriverRef`), `x`, `y: integer` (`PointRef`, with the resolver's `screenshot_generation`), `css: string` + (`Locator`), and a decoder that builds the command from the one given. A spec with no ref kinds to generate from + is skipped. + +The tool name is `driver_info.tool_name_prefix + spec.name`. Without `image_support`, a spec whose only command is +`Observe` and whose decoder, called with no arguments, asks for a screenshot is not emitted (today's rule for +`browser_take_screenshot`, without a tool name in the core); with image support that spec is a screenshot-only +tool. Each emitted spec's `prompt_section` is collected. Then +each `profile.extra_tools` factory is called with the `ToolContext` and its tool appended, unless its +`flags.requires` names a command missing from `DriverInfo.actions` (rule: the model is offered only tools the driver +can execute, for extra tools as for specs). An entry that selects no tool, a skipped spec, a skipped extra tool +(logged under the factory's name with the missing commands), and a selected name with no spec that no extra tool +serves (under its name or the prefixed one) are logged once per factory on `tool_logger`. + +`AgentTools(tools, prompt_sections, builders)`: `builders` maps each command tool's model-facing name to its +decoder; it is the map `ActionPolicy` takes. + +### `CommandTool` + +One instance per emitted tool, a core `Tool` (`ToolCard(id=name, name, description, input_params=schema)`). +`invoke(args)`: + +1. `action = build(args, page_state)`; a `ValueError` returns the failure `### Error\n<message>` and nothing is sent; +2. `page_state.check(action)`; a `StaleRef` makes one `session.observe(Observe())`, recorded, and the error is + rendered with that fresh observation; the driver never sees the stale command; +3. `context.execute(action)`, or `context.observe(action)` for an `Observe` (see `ToolContext`): the session runs it; +4. the same call records it with `page_state.record(action, outcome)`; +5. render: an `ActionResult` with `renderer.result`; an observation with `renderer.observation` when the tool's only + command is `Observe` and it is not a screenshot-only tool, else with `renderer.result` on `ActionResult(ok=True, + value=<screenshot>, observation, surface)` (the tab list of `browser_tabs list`; the image alone for a + screenshot-only tool); a `DriverError` with `renderer.error(action, error, None)`, except a `StaleRef` the driver + raised with a fresh observation (its retry observed afresh): that observation is recorded + (`page_state.record(Observe(), observation)`, so its refs get the current generation) and the recorded one is + passed to `renderer.error`. + +The output is a `ToolOutput`: `success = ok`; `data = {"content": <text blocks joined by newlines>, "command": +CommandRecord}`; on failure `error` is the same text. With `image_support`, image blocks go to `data["multimodal"]` +as `{"type": "image", "source": <profile name>, "source_path": <tool name>, "mime_type": "image/png", "data_url"}` +and the content gets the note `N image(s) attached as multimodal input.`, the shape MCP tools deliver; without it +image blocks are dropped. A task failure from the session (`BaseError`, second lost connection) propagates. + +### `CommandRecord` + +Frozen; never parsed from text. + +| Field | Value | +|---|---| +| `command` | the command type | +| `effect` | its `Effect`; an extra tool may give its declared one instead | +| `ok` | `ActionResult.ok`; `False` for an error | +| `dispatched` | `True` for a result; `error.dispatched` for an error | +| `context_changed` | `ActionResult.context_changed` | +| `error` | the `DriverError` type name, or `None` | +| `blocked_by` | `ActionIntercepted.by` for that error, else `None` | +| `surface` | the result's surface context; for an error, the page state's | +| `value` | `ActionResult.value`; for an `Observe`, the observed elements (stamped `Element`s) | +| `generation` | the generation stamped by `record` (the page state's current one for an error) | +| `action` | `masked(command)`: the command with its sensitive fields masked | + +`message_metadata` gives the tool-message metadata keys the context processors read: `success = ok`, `executed = +ok` for a `READ` record (a find over a current observation sent nothing but answered, as the server call did) and +`dispatched` for any other effect, `state_changed = context_changed or (effect != READ and (ok or error == "ActionTimeout"))`, `denied = +False` (denials come from `ActionPolicy`). A tool cannot set `ToolMessage.metadata` (core builds the message from +`render_for_llm` after `invoke`); a rail copies `message_metadata` onto it in `after_tool_call`. `BrowserRuntimeRail` +adds `effect` (the kind's `Effect` value) for the surface's budgeted tools, that is the factory tools and the extra +tools without `exempt_from_budget`; a message without `effect` belongs to a tool the context processors leave alone. + +`CommandRecord.of(action, result, *, effect=None, value=None)` builds the record of a result, +`CommandRecord.failed(action, error, page_state, *, effect=None)` the record of a `DriverError`; `CommandTool` and the +helpers use both. `command_record(tool_result)` returns the record from a tool output (`ToolCallInputs.tool_result` +in a rail), or `None`: arguments that decoded to no command, a tool that sends no command (recall), or a tool that +leaves no record. A rail treats a tool message without a record as a failed call that sent nothing. + +### `ToolContext` + +`ToolContext(driver, page_state, renderer, driver_info, settings, image_support, workspace=None, language="cn")`: +what `profile.extra_tools` and `profile.rails` factories receive; `driver` is the `DriverSession`. The factory builds +it from its arguments, `profile.renderer` and `profile.settings`. + +- `await context.execute(action)`: `driver.execute`, then `page_state.record`; a `DriverError` with + `dispatched=True` is noted (`page_state.note_dispatched()`) before it propagates. The one way a tool or helper runs + a command (the batch tool too). +- `await context.observe(command)`: `driver.observe`, then `page_state.record`. + +`ToolFlags(effect=READ, exempt_from_budget=False, progress=True, worker=True, requires=())`: what an extra tool +declares, as its `flags` attribute, for the rails that today decide by tool name (state change, replan budgets, +progress) and for the factory (`worker`: offered to a delegated worker too; `requires`: the command types the tool +sends, all of which the driver must handle for the tool to be offered). + +## Rails on command records + +Two rails read `CommandRecord`s, never tool text; the text-based rails of the same names are gone. +`build_agent_parts` hands them to the main agent (registered at its first invoke) and to the worker. Neither adds a +model call: each rewrites the tool message in `after_tool_call`, or empties the tools of the next model call +(invariant: a rail never adds a model round trip). + +### `RepeatedCallGuardRail` (`runtime/rails/repeated_call_guard.py`) + +State reset per invoke (`before_invoke`, and the first `before_model_call` that finds none: a deep agent fires +`before_invoke` on its outer callback context, and its inner agent's model and tool hooks share another `extra`), +the streak counted in `after_tool_call`, the finish section and the withholding in `before_model_call`. + +- **Identity.** Two calls are identical when their commands and their results are: `record.action` (masked) and + `(ok, error, value, surface)`. A `RunScript` compares its code without whitespace and with digits replaced by + `#`, and its value as sorted JSON with digits replaced; an `Observe` compares its observed elements. A message + without a record compares by tool name and arguments (canonical JSON; non-JSON text as is). +- **No-op scripts.** A `RunScript` whose whitespace-free code only returns literals, the document title or its + address (the same pattern as before) counts toward a no-op streak, whatever its result. +- **Counting.** The 3rd identical (or no-op) call in a row appends `### Repeated call` to the tool message, once; + after the 5th the next model call gets no tools and a finish instruction. Any other call resets the streak. + Calls of one model turn are counted in the order their `after_tool_call` runs, like separate turns. + +### `FinalAnswerRecoveryRail` (`runtime/rails/final_answer_recovery.py`) + +When the delegated worker ends without the final JSON, the worker loop asks once more in the same conversation +(`recovery_message(reason)`) and the rail stops that invoke after one model call, running none of its tool calls. +Invariants: the recovery request keeps the run's tool list (dropping it changes the prompt prefix and misses the +cache for the whole conversation); `non_json_output` (a prose ending) gets `RESTATE_MESSAGE`: restate the answer +already given as the JSON, without re-checking, changing it or adding caveats, `ok=false` only if that reply said +the task could not be completed; `empty_output` gets `RECOVERY_MESSAGE` and `max_iterations_reached` the step-limit +sentence plus `RECOVERY_MESSAGE` (no answer to restate: report what was found and what is missing). + +### `InterceptedActionHintRail` (`runtime/rails/intercepted_action.py`) + +`InterceptedActionHintRail(retry)` takes the surface's retry sentence (`profile.messages["retry"]`; the core names +no browser tool). For a record with `error == "ActionIntercepted"` and a non-empty `blocked_by`, and a string tool +message, it replaces the message with `explain_blocked_command(record, retry)`: + +``` +### Error +<command type> not delivered: <target> is covered by <blocked_by>. +<hint> +<retry> +``` + +`<target>` is the `DriverRef` id, the `Locator` css (or text), `(x, y)` for a `PointRef`, else `the target`. +`<hint>` is the cookie-consent hint when `blocked_by` matches the consent pattern, else the generic one; `<retry>` is +the sentence given at construction. The tool +output (`tool_result`) keeps the driver's own error. Other records and messages without a record are left alone. + +### `ConnectionLostRail` (`runtime/rails/connection_lost.py`) + +`ConnectionLostRail(session_of)` takes the session lookup `core_rails` was given. In `before_model_call`, when +`session_of()` is a session whose `lost` is set, it calls `ctx.request_force_finish({"output": <lost>, +"result_type": "error", "error": <lost>})`, so the model call is skipped and the run ends with that result ("Driver +session", the invariant). A missing or live session lets the call through. The rail reads no tool text and no +record: it is the one place that turns the session's terminal loss into the end of the run, for every agent +`core_rails` builds (the main agent of both topologies and the worker). A rail registered after it that reaches the +session in its own `before_model_call` (the browser context processors' state capture) can meet the loss after the +stop has let that call through; that model call's commands fail with the `BaseError`, and the next model call is +the one the stop skips. + +## The core runtime (`runtime/session.py`, `AgentRuntime`) + +`AgentRuntime(core, surface, *, provider, api_key, api_base, model_name, guardrails, allowed_tool_names=None, +workspace=None, language="cn", output_dir, identity, cancel_store=None, page_state=None)` is the surface-neutral +runner: it owns one `AgentService` built from the same inputs, works on the service's `DriverSession` from +`ensure_runtime_ready()` on (one session for the main agent and the worker) and runs each task through a topology. +It names no surface: what it needs from one it reads from `surface.profile`. + +- Plain views: `service`, `core`, `surface`, `workspace`, `language`, `page_state` (the service's), `session`, + `context` (the `ToolContext` of the started session, for the runtime's own calls), `topology` (`None` before + `ensure_started`), `builders` (the decoders of every factory tool built so far, one live mapping updated by + `agent_parts`; `ActionPolicy` reads it per call, "Policy rail", `tool_kind` too) and `dedicated_log`, settable: + `Direct.start` sets it `False` so the runner's agent writes to the process log, where the runner reads model + usage; agents from `create_browser_agent` keep `True`. +- `ensure_runtime_ready()`: `service.ensure_runtime_ready()`; when the service's session is a new one, a + `ToolContext` for the runtime's own calls (no image support, the given workspace and language) and the cached + parts dropped. +- `core_rails() -> (ActionPolicy, RepeatedCallGuardRail, InterceptedActionHintRail, ConnectionLostRail)`: + `assembly.core_rails(surface.profile, builders, page_state, lambda: self._session)` ("Agent construction"). A + policy built before the session exists enforces the tools built after it, through the live mapping; the + lost-connection stop reads the session through the lookup the same way. +- Capabilities are derived from `allowed_tool_names` on first read: the entries of the profile's catalog whose tool + names are all allowed; with no allowlist, the default entry's tools, and the service then holds + `allowed_tool_names=None`. +- `agent_parts(*, image_support, language)`: `build_agent_parts(session, assembly.surface, ..., language=)`, built + once per `(image_support, language)`; the receiving agent decides the language ("one language per agent": the + browser rail passes the agent's own, `build_worker` passes `WORKER_LANGUAGE`), and the constructor `language` + (default `cn`) serves only the runtime's own tool context. The parts hold no core rails. The runtime merges the + parts' `builders` into its live mapping and keeps each extra tool's `flags` to answer `tool_kind`. +- `tool_kind(tool_name, tool_args) -> ToolKind`: for a factory command tool, the command decoded from the arguments + with the page state as resolver (`action`, `effect = action.effect`, `fields` = the schema's property names; + arguments that do not decode leave `action` `None`, `effect` `ACT`); for an extra tool its `ToolFlags` + (`helper=True`); for an unknown tool `ToolKind()` (`known=False`, `ACT`). `command` and `read_only` are derived. + `progress` is `True` for every factory tool and as declared for a helper. +- `metadata() -> (mapping, error)`: the profile's `page_metadata(context)`; `({}, None)` for a surface without + the hook. +- `ensure_started()` picks the topology once, `topology_for(core)` ("Topology", `BROWSER_TOPOLOGY`), and calls its + `start(self)`; `run_task(task, session_id=, request_id=, timeout_s=)` is `ensure_started()`, the topology's + `run(self, task, session_id, request_id, timeout_s)`, then `answer["context"] = {"identity": <the current + context's identity, "" without one>, "metadata": <the profile's metadata, {} without the hook>}`. The metadata is + captured only for an answer with `ok` and a started session (no driver call after a failed, timed-out or + cancelled agent task); the identity is read after the capture. +- `main_agent_answer(session_id, request_id, *, result=None, error=None)`: the answer dict of a main-agent run, + for `Direct.run`: `ok` (no error, `result_type` not `"error"`), `session_id`, `request_id`, `final` (the + output), `screenshot=None`, `error`, and `status` / `progress` from the rail's `authoritative_browser_result` / + `progress_state` when present. No `page`: a surface's location is the surface's. +- `cancel_run`, `clear_cancel`, `runtime_health()` (`session.health()` when the stack runs, else `ok=False, + detail="not started"`, with `started` and the model settings), `shutdown()` (`service.shutdown()`, the session + view dropped, the topology dropped, then `OpenAIModelClient.aclose()` so the next event loop gets fresh + clients), `acquire_task_resources()` (the service binding), `release_task_resources() -> bool` (the service + release; the last binding stops the driver stack, keeps the device and resets the page state) and `reset()` + (closes the session through the service and resets the page state). + +The core never calls `profile.custom_actions`: its argument is the browser runner's action controller, so the +browser facade calls it. Every other hook is read where the core needs it. + +## The browser facade (`surfaces/browser/runner/facade.py`, `BrowserAgentRuntime`) + +`BrowserAgentRuntime(AgentRuntime)` keeps the constructor `create_browser_agent` and the WebVoyager runner use +(`provider, api_key, api_base, model_name, mcp_cfg, guardrails, instance=None, allowed_tool_names=None, *, +core_settings=None, browser_settings=None, workspace=None, language="cn"`) and turns it into the core's inputs: + +1. the core settings: `core_settings`, else `CoreSettings.from_process_env()` at construction, with the instance's + `driver`, when set, in place of `core.driver`; +2. the browser settings: `browser_settings`, else `browser_surface_settings()` (read once at construction); the + workspace root from `workspace`; the instance applied (`BrowserSettings.with_instance`); then `mcp_server=mcp_cfg` + and `mcp_cwd` (the config's `cwd`, else the settings'). The instance's `provisioning` / `driver_mode` is + checked first: a value other than `remote`, `managed`, `extension` raises `ValueError` naming + `BrowserInstanceConfig.provisioning` and `BROWSER_PROVISIONING`; +3. the lifecycle identity, `BrowserServiceIdentity(browser_key, profile_name, driver_mode, display_mode, + managed_args, browser_binary, user_data_dir, cdp_endpoint, server_id)` from the complete settings and the server + config (`display_mode` is `headless` / `headed` for a managed device by `--headless` in `managed_args`, else the + driver mode; `cdp_endpoint` is the config's `PLAYWRIGHT_MCP_CDP_ENDPOINT`); its `key`, the JSON of its fields, + is the registry string, and `from_key` rebuilds it; +4. `configure_browser_agent_logging(core)`; +5. the surface bundle: `assemble_surface(core, process_environ(), settings=<the complete settings>)`, built once, + before any driver, so the rails' profile constants exist at the main agent's construction. Invariant: the + session's bundle is built from the same settings, so the two agree on `messages` and on what `policy_rules` + decide; +6. `AgentRuntime.__init__` with those, `output_dir=mcp_cwd` and `identity=key`. + +On top of the core it adds: + +- `core_settings` (the resolved core), `lifecycle_identity`, `mcp_cfg`, `controller`, `page_id`, `generation_id`, + `export_page_state`, and the browser page-state views the rail reads: `set_task_requested_fields`, + `resolve_model_target_id`, `target_recovery_state`, `resolve_primary_link`. +- `ensure_runtime_ready()`: the core's, then, for a new session, the controller's builtin actions and the + profile's `custom_actions(controller, context)`. +- `ensure_started()`: the core's, then `register_runtime_tools()` when the topology is `DelegatedWorker`: + `browser_custom_action` and `browser_list_custom_actions` in `Runner.resource_mgr` (tag + `agent.playwright.runtime`), once; a tool that already exists there is accepted. The main agent never holds + these tools. They are the only `Tool` wrappers of the facade (`runner/tools.py`); cancellation and health are + reached through the runtime's methods, as the server entry point does. Neither card names a driver; the worker's + `AgentCard` renders the session driver's display name. +- `run_browser_task(task, session_id=, request_id=, timeout_s=)` is the core's `run_task`. +- `main_agent_answer(...)`: the core's answer plus `page: {"url", "title"}` from the browser context of the page + state. The answer the runner reads thus keeps its keys: `ok`, `session_id`, `request_id`, `final`, `page`, + `screenshot`, `error`, `status` / `progress` when reported, `context` from the core and, on the delegated + topology, the worker loop's `attempt`, `failure_summary`, `progress_state`. +- `capture_browser_state(action_group_id="")`, the browser state processor's provider: `page_state.fresh_observation`, + else one `Observe()` through the context (a `DriverError` becomes `dom_error`, `page observation failed: …`); + `observations_reused` counts the reuses; then the runtime's `metadata()`. The result keeps its keys (`ok`, `error`, + `url`, `title`, `tabs`, `page_position`, `semantic_state`, `semantic_progress`, `field_coverage`, `page_state`, + `dom=""`, `dom_error`, `audit`). `capture_reconciliation_browser_state` runs the metadata hook only; + `capture_compact_browser_state` sends nothing. A task failure from the session propagates. The semantic-state + tracker is the profile's `semantic_tracker()`, made on first use (a null tracker without the hook); the three + captures call `observe(state, action_group_id=..., changed=)` with `changed` true when the generation or the + last observation moved since the previous call. `semantic_progress`, `acknowledge_semantic_replan` and + `reset_semantic_task` read and reset it; `release_task_resources` (on the last binding) and `reset` reset it too. +- `probe_for_policy(source, params)` and `activate_page(url)`, the `DecisionPolicyModel` hooks, call the profile's + `policy_probe` and `activate` on the context; `run_custom_action` binds the runtime to the controller and + dispatches; `list_actions` lists the controller's actions. +- `shutdown()`: the core's, then the facade leaves the process-wide set of live facades. + +The module also holds that set and the process-wide resets: `reset_active_browser_runtimes()` resets every live +facade; `reset_managed_browser_runtime(*, browser_key, profile_name, display_mode, browser_binary)` resets the live +facades whose identity matches those browser fields (managed only), then `reset_registered_managed_browser(...)`, +which rebuilds the identity of every registry entry with a preserved driver (`BrowserServiceIdentity.from_key`) +and stops the matching ones through `AgentService.reset_lifecycle_resources`. + +Timing: the driver starts and the tools are added at the first invoke, in `BrowserRuntimeRail.before_invoke`, +where the MCP tools were added before; the deep agent is built with no browser tools and gets both the command +tools and the extra tools there (`browser_find` takes its description from the driver's native tools, which exist +only after `start()`). The rails are queued at construction (`create_browser_agent`), so every rail, the +repeated-call guard included, sees the first invoke's `before_invoke`. + +### Topology (`runtime/topology.py`) + +Who runs a task behind the runtime's entry points. `Topology` is a `Protocol`: `name`, `async start(runtime)`, +`async run(runtime, task, session_id, request_id, timeout_s) -> dict`, `drop()`. `topology_for(core)` maps +`core.topology or default_topology(core.driver)` to a strategy and raises `ValueError` naming the driver and the +name for a registered default no strategy implements. Neither strategy names a driver; the module carries no +browser word (it is a core module under the browser-word ratchet); the `runtime` it receives is an +`AgentRuntime`, the facade on the browser surface. + +- `Direct` (`name="direct"`, no built-in driver's default; `BROWSER_TOPOLOGY=direct` selects it): `start` sets + `runtime.dedicated_log = False` and builds the main agent once through the profile's `main_agent(runtime, model, + language=WORKER_LANGUAGE, max_iterations=guardrails.max_steps, workspace=runtime.workspace, + capabilities=runtime.capabilities)` with a + `Model` from the service's provider, key, base and model name (`temperature` / `top_p` from the core + settings' worker values); `start` makes no model call ("Invariant (image probe)"). With the browser hook no + session is opened: it opens at the agent's first invoke (`BrowserRuntimeRail.before_invoke`), as for any + `create_browser_agent` agent. Without a hook `start` opens the session (`runtime.ensure_runtime_ready()`) and + `run` builds the agent after the probe: `agent_over(runtime.agent_parts(image_support=<the verdict>, + language=WORKER_LANGUAGE), runtime.core_rails(), model, language=WORKER_LANGUAGE, workspace=, + max_iterations=guardrails.max_steps, enable_read_image_multimodal=<the verdict>)`. `run`: + `service.session_new(session_id)`, a request id, `await probe_image_support(<the same Model>)` + (the browser agent reads the verdict from the probe cache per model call, "`BrowserRuntimeRail`"), then + `Runner.run_agent(agent, {"query": task, + "conversation_id": "<sid>.direct.<rid>.<uuid>", "request_id": rid})` under `asyncio.wait_for(timeout_s or the + guardrails' timeout)`; the result, a timeout (`task_timeout: exceeded <n>s`) or an exception becomes the answer + through `runtime.main_agent_answer`. One attempt per task: the worker loop's retries are worker guardrails. + `drop` forgets the agent. Invariant: the conversation id is the context engine's session id and names the offload + directory, so its separators are path-legal on every OS (`:` is not on Windows). +- `DelegatedWorker` (`name="delegated_worker"`, the registered default of both built-in drivers): `start` is + `runtime.ensure_runtime_ready()` (the session opens here), then, when `service.agent` is `None`, + `service.agent = await build_worker(session, session.assembly.surface, provider=, api_key=, api_base=, + model_name=, max_steps=guardrails.max_steps, core=service.core, capabilities=runtime.capabilities, + tool_result_observer=service.observe_worker_tool_result, screenshot_subdir=, artifacts_subdir=, + image_support=<the strategy's verdict, False until a probe said yes>)`, records the tool names the worker was built + with (the browser facade registers its two custom-action tools after `ensure_started` on this topology). `run` + first awaits `probe_image_support` (as `Direct.run` does); + on the first yes it sets image support and empties the slot, remembering the tools the caller removed from the + started worker (the runner drops the helpers between `ensure_started()` and the task), so the worker is rebuilt + with the screenshot tool and image results and without those tools. Then `service.run_task(task=, session_id=, + request_id=, timeout_s=, worker=)` with `worker` an async callable that runs `start(runtime)` and returns + `service.agent`: after a `reset()` the slot is empty and the next attempt rebuilds the worker on a fresh + session. `drop` does nothing: + the worker lives in the service's slot, which the service clears on reset and shutdown. + +The worker's answer carries `screenshot` only as something a vision model reads as an image: an http(s) link, a +data URL whose bytes are an image, or a local file (resolved under the runtime working directory, then the process +cwd) that is a PNG, JPEG, GIF or WebP, turned into a data URL with the MIME type its bytes show +(`service._normalize_screenshot_value`). Anything else, a path that names no file included, becomes `None`: callers +(the WebVoyager judge) send the value to a model as an image. + +Invariant (image probe): `start` makes no model call on either strategy. The runner calls `ensure_started()` before it +redirects stdout into the task log, and the first model call binds the model logger to the stdout of that moment, so a +call in `start` sends every usage line of the task to the process stdout. `Direct.run` awaits `probe_image_support` +before the agent runs: a background probe (`schedule_image_support_probe`) dies with its event loop, and the runner +runs one `asyncio.run` per task. A conclusive verdict is cached per `(api_base, model_name)` for the process, so later +tasks pay nothing; an inconclusive probe (`None`: timeout, call failure, empty answer) is not cached, counts as no +image support, and is tried again at the next run. + +The runner reads `runtime._service.agent` between `ensure_started()` and `run_browser_task` to remove the +helpers; on `Direct` the slot is `None` and its guard skips the removal, so the main agent keeps the helpers it is +built with. `create_browser_agent` outside the runtime's entry points is not governed by the setting: that agent +always calls the tools itself. + +### `BrowserRuntimeRail` (`surfaces/browser/rails/runtime_rail.py`) + +The main agent's rail, built with the runtime and registered by `create_browser_agent`. It reads `ToolKind`s before +a call and `CommandRecord`s after it, never tool text. The class holds the hooks, the lifecycle and the target +normalisation; the rest of its methods are five plain mixin classes it inherits, in MRO order `_TerminalMixin` +(`rails/terminal.py`: model-protocol recovery, the terminal result), `_ResultsMixin` (`rails/results.py`: +tool-message rewriting), `_ProbeResultsMixin` (`probes/results.py`), `_PhaseMixin` (`semantic/phases.py`: phases, +strategy, completion) and `_EvidenceMixin` (`semantic/evidence.py`), then `AgentRail`. Invariants: no mixin +defines a hook name (`before_*`, `after_*`, `on_*`), so `AgentRail`'s dispatch sees the class alone; a mixin never +imports `runtime_rail.py` (the constants two of these modules share live in `rails/keys.py`, which imports nothing +from the package); the rail imports the facade (`runtime/session.py`), never the reverse. + +- Image support: `should_enable_read_image_multimodal` (`openjiuwen/harness/rails/_multimodal.py`: the agent's + `enable_read_image_multimodal`, else the model's `supports_vision`, else the probe cache, `None` counting as + unsupported) of the agent `before_invoke` saw. Invariant: the three readers (the `agent_parts` key in + `before_invoke`, the image-capability prompt section in `before_model_call`, the batch screenshot gate in + `before_tool_call`) resolve against that one agent. On a deep agent only `before_invoke` / `after_invoke` fire on + the `DeepAgent`; the model and tool hooks fire on its inner `ReActAgent`, which has no `deep_config`, so resolving + `ctx.agent` there always answered "unsupported". A rail whose hooks run with no `before_invoke` first resolves + `ctx.agent`. +- `before_invoke`: the browser log context for the invoke when `runtime.dedicated_log` is true (reset in + `after_invoke`), `ensure_runtime_ready()`, then `agent_parts(image_support=<the agent's read-image support>, + language=<the agent's own, resolve_language(agent.deep_config.language)>)`: a session that cannot start (or + restart) there raises core's `AbortError` with the start error as `cause`, so the invoke fails and the caller + sees that error (the callback framework re-raises the cause; the runner's Direct answer is `ok=False` with its + text); no tool is added and no model call is made. Any other callback failure the framework logs and swallows, + and the invoke would go on with no tools: every tool through `ability_manager.add_ability(card, tool)` on every + invoke (a re-add refreshes the instance). The rail registers no rail: the core rails are queued at the agent's + construction (`create_browser_agent`), so `RepeatedCallGuardRail.before_invoke` runs at the first invoke; the + guard's first-model-call fallback stays for a guard registered later (the worker's `register_rail` path). +- `before_tool_call`: a factory tool whose schema addresses elements by `target`, `startTarget` or `endTarget` + (the Playwright MCP dialect) has its arguments normalised: the `ref`, `startRef` and `endRef` aliases renamed, + `[ref=e12]` and `ref=e12` reduced to the ref, a `target_id` resolved through the page state to the target's ref + or selector (with `element` filled from its text), `generation_id` dropped, and `field`, `fields` and `element` + dropped when the schema does not declare them (the evidence hints are read first; every server schema forbids + extra properties). A `Click` whose target is a card's primary link becomes the navigation tool (`browser_click` + → `browser_navigate` in the name) with `{"url": <link>}`; a `t_g…` target left without a locator is denied. The + browser-dialect tools (`target_id`, `ref`, `selector`, a required `generation_id`) are left to their decoders. + Then the schema check, the batch screenshot gate without image support, and the phase budget as before, with the + kind in place of the name. +- `after_tool_call`: a `PolicyDenied` result or the rail's own denial sets `executed=False, denied=True`. Otherwise + `command_record(tool_result)`: with a record the tool message gets `record.message_metadata` (O5), a partial + batch with a successful mutating step counting as `state_changed`, and `effect` for a budgeted kind; without one, + a known tool sent nothing + (`executed=False`) and an unknown tool is read from `ToolOutput.success`. The evidence machinery reads one dict: + a helper's payload as is, else `{ok, error, result, url, title, generation_id, page_state}` with `result` the + script value of a `RunScript` or the rendered text, and `url`/`title` from `record.surface`. An `Observe` that is + not a screenshot replaces the message content with the compact page state; a `NAVIGATE` effect appends + `<browser_page_state>`; `Observe` and `RunScript` results and read-only helpers that count against budgets are + bounded at 12,000 characters. Progress is persisted for tools with `progress`. +- `after_invoke` on a result of type `error` that carries no progress block and is not the max-iteration result: + `_terminalize_invoke_failure(session, reason)`, status `partial` with evidence else `blocked`, the reason as + `terminal_reason` and a blocker. The reason is `browser_connection_lost` when the runtime's session has `lost` set + (then `blocked` whatever the evidence: the stack is gone), else `browser_execution_error`; the structured terminal + result follows (`error` `browser_task_incomplete`, the run's output as `summary`). `browser_connection_lost` is + retryable on its own (`_terminal_result_retryable`) and a resume drops it from the blockers: a new task restarts + the stack. +- Classifiers by kind: `NAVIGATE` → navigation; `TypeText`, `PressKey`, `SelectOption`, `FillForm`, `UploadFile` → + form; `RunScript` → script, its intent from the code as before; a read-only tool that counts against budgets → + extraction; a batch by its ops; `Scroll` → viewport exploration; `Observe` and read-only helpers → target + discovery (`browser_probe_cards` → structured extraction); `Click`, `Hover`, `Drag`, `Drop` → interaction; + `exempt_from_budget` → no phase or replan budget; read-only recovery for `READ`, `SwitchContext`, a script that + does not mutate, a batch of read-only ops. +- Phase budgets are charged on what each command did. Invariants: a known `ACT` command that is not a form or + filter command keeps the current phase, except while extracting, when it goes to `form` (created on demand); an + `ACT` or `NAVIGATE` command whose record is ok and shows `context_changed`, or a generation other than the one + stored when the call started, gets its attempt back (`_record_phase_result(..., progressed=True)`); in the + extraction phase a `RunScript` costs one attempt per distinct code (a sha256 prefix kept in the phase's + `scripts`), and a successful read that adds evidence (a new `structured_evidence` entry, such as a page + observation from another source or with other text, or a newly covered field) gets its attempt back. A refund + only returns an attempt the call was charged: the rail stores `charged` per call (the phase attempts grew in + `_consume_phase_budget`) and passes it as `_record_phase_result(..., charged=...)`, so a repeated script is + never refunded. Failed reads, reads that add nothing and stalled actions count; a mutating page script records + no page observation, so it counts unless it returns a requested field. They still exhaust their phase, and the budget check stays before the call. Exhaustion is terminal: status + `partial`, blocker `<phase>_phase_budget_exhausted`. + +### Context processors and the status log + +They read tool-message metadata and command records, never tool names or result text. All three live under +`surfaces/browser/`. The two processors register themselves with the context engine at import +(`@ContextEngine.register_processor()` on the class); the registration point is the import of +`state_processor.py` and `working_context/processor.py`, which `browser_agent.py` performs. `surfaces/browser/__init__.py` +stays side-effect free: importing them there would pull `openjiuwen.core.context_engine` and the working-context +store into every `SURFACES.create("browser")`, `browser_surface_settings()` included. + +- `BrowserStateContextProcessor` (`state_processor.py`) groups the completed tool calls of one model + turn by their messages' `effect`: a call with an effect other than `READ` whose message has `state_changed` not + `False` is a mutation (a full capture after the group, or the reconciliation capture when its `success` is + `False`); a group of `READ` effects alone with `success` not `False` is an observation (the compact capture, no + browser round trip). Messages without `effect`, with `denied`, or with `executed=False` are skipped. The name sets + and the JSON content parsing are gone. +- `BrowserWorkingContextStore` (`working_context/store.py`) retains a tool error from the message when the metadata + says `success` is `False` (the `error` field of a JSON payload, else the text) or the content starts with one of + the harness's execution-error prefixes; it no longer scans for `ok`, `success`, `isError` or `success=false` text. +- `BrowserSubagentStatusLogger` (`status_logging.py`) takes `command_of(tool_name, tool_args) -> str`, the rail's + decoder (the command type name, `""` for a helper or an unknown tool), and shapes `tool_start` arguments by the + command (`RunScript`: code length only; `Navigate`: the URL without its query) and `tool_end` results by the + command record (`kind="command"`, `command`, `effect`, `ok`, `dispatched`, `error`, the text length, `line_count` + for `Observe`, the value kind and length for `RunScript`). A helper's dict payload is shaped by its keys; + `browser_batch_interact`, the surface's own extra tool, keeps its step summaries by name. + +## Drivers + +Both drivers fill `BrowserContext` (`surfaces/browser/context.py`). While a JavaScript dialog blocks the page, +`dialog` is `"<type>: <message>"` (`"confirm: Proceed?"`) in every observation and in every action result +that reports it, and `observe()` returns what it can read without the page (the context, tabs, URL, the dialog; +no elements, no screenshot) instead of raising. Outside a dialog `dialog` is `None`. + +### browser-use (`drivers/browser_use/`) + +`BrowserUseDriver(HandlerDriver)` runs commands through the browser-use sidecar: a separate Python process +(`sidecar/`, its own venv; the agent-core process never imports `browser_use`) reached over NDJSON on stdio +(`transport.py`). `info`: `name="browser_use"`, `display_name="browser-use"`, `surfaces={"browser"}`, +`observes={elements, screenshot, native_text, surface}`, `versions` from the sidecar's `connect` reply. + +- Factory: `create_browser_use_driver(device, surface_settings)` builds `BrowserUseConfig(cdp_url=device.endpoint, + sidecar_python=..., upload_root=...)`, the last two from the `BrowserSettings` (`None` settings: neither); + `start(config=None)` spawns the sidecar with that interpreter (`None`: `.venvs/browser-use` in the repo) and + attaches to that CDP endpoint (an explicit config overrides); the `connect` request carries `init_script`, the + text of the clipboard guard, which the sidecar installs on every CDP session it obtains (the one at connect and + every later one, after a tab switch or for a new target) with `Page.addScriptToEvaluateOnNewDocument`, and runs + once on the document open at that moment, which loaded before the registration; a session is guarded once. + `UploadFile` and `Drop` resolve their paths with + `drivers/browser_use/upload_paths.resolve_upload_file_paths`, which applies `commands.upload_paths` with + `upload_root` (a relative name is a file under the upload root, never under the process directory) and then + reports missing or unreadable files; a `ValueError` of the rule is `ActionResult(ok=False, detail=<message>)` + and nothing is sent. The sidecar receives absolute paths only and refuses a relative one; it reads no environment for them. No surface code rewrites upload paths; confining + them is a surface policy rule's job. +- Handlers with `refs={DriverRef, Locator}`: `Click`, `TypeText` (target required; `slowly` inserts one character + per `Input.insertText` with a 30 ms pause, after the `clear` when asked, so `clear` and `slowly` are independent + here while `browser_type` on Playwright MCP never clears when it types slowly), `Hover`, `Drag`, + `SelectOption` (exactly one value), `UploadFile` (target required), `Drop`, `FillForm` (each field's + `sensitive` goes to the sidecar's `type_text`). With `refs={DriverRef}`: `Scroll` (one axis per command), + `RunScript` (page JS; with a target the element is stamped and passed as the first argument). Without refs: + `PressKey`, `HandleDialog`, `Wait` (sleep `seconds`, or poll page text for `until_text` up to `seconds`, default + 10 s), `Navigate`, `Back`, `OpenContext`, `SwitchContext`, `CloseContext`. Any other form raises + `Unsupported(dispatched=False)` before anything is sent. +- `DriverRef.ref` is the handle the sidecar mints in `observe()` (`bnid:<backend node id>`). A `Locator` is + accepted by CSS only, for the commands whose browser-use tools take a `selector`; it goes to the sidecar as its + selector ref, which resolves to the first match in the current document. A `Locator` with `text` raises + `Unsupported(dispatched=False)` (for `FillForm`, before any field is filled). The sidecar's index refs + (`IndexRef`) stay internal; no handler accepts them. +- `observe()` fills `elements`, `native_text` (browser-use's own page text), `screenshot` (a separate full-page + capture when `full_page`) and `BrowserContext` (url, title, tabs, `tab_urls`, `current_tab`, PDF viewer, scroll + pixels). A scoped `Observe` is `Unsupported(dispatched=False)`: browser-use observes whole pages, and the browser + dialect's `browser_snapshot` has no `depth` or `target`. The title is read from `document.title`, because + browser-use's own title lags behind it; the sidecar's + title is kept when the page runs no script. While a dialog is open the sidecar answers `observe` from + browser-use's cached target info only (URL, title, tabs) with `dialog` and no elements; the driver then runs no + script and takes no screenshot. + The focused context (`current_tab`) is the first tab whose URL is the page URL; with none, `current_tab` is + `None` and the context has an empty id. `generation` stays empty. +- Element and keyboard commands run in the sidecar as one action each (wire version 4). The action resolves its + ref, then sends its input as a task and watches the page while it runs: + - Dialogs. The sidecar owns `Page.javascriptDialogOpening` (it takes it back from browser-use's auto-accepting + popups watchdog before every action and observation). When a dialog opens that `HandleDialog` did not arm, + Chrome holds the input until the dialog closes; the action returns at once with `dialog_opened` (type and + message) and its input waits in the background. `HandleDialog` answers the dialog on the session that reported + it. While a dialog is open, `observe()` returns the page as above, screenshots and page scripts fail at once + (`DriverError`) instead of blocking, and any other action returns `ok=False` without sending input. Every act + reply carries the page URL and `dialog`, the dialog still open when the reply is built. + - Popups. A page that opens a popup can wait for the debugger until told to run, which held the click for 30 s. + When a page target is created during the action, the sidecar resumes the current page + (`Runtime.runIfWaitingForDebugger`) and reports the new pages as `new_targets`. + - Navigation. A navigation of the page's main frame that the action started (`Page.frameRequestedNavigation`, + current tab) gets up to 1 s (`_NAVIGATION_COMMIT_WAIT_S`) to commit (`Page.frameNavigated`). A committed one + is a `document_changed`, a same-URL reload from a click or a submit included; one that does not commit in + time counts only if the URL changed. + - Intercepted input. Before the mouse is used for `Click`, `Hover` and the `Drag` source, the sidecar hit-tests + the point with `elementFromPoint`. If another element is on top (not the target or inside it; a `<label>` of + the target is allowed), it raises `ActionIntercepted` with `by` = `tag#id.class "name"` of that element, and + nothing is sent. The `Drag` destination, `Drop` and the focusing click of `TypeText` are not hit-tested. +- `context_changed`: `Navigate`, `Back`, `OpenContext` always (a same-URL reload too); `SwitchContext` and + `CloseContext` always; other commands when the sidecar reports `document_changed`, a dialog, or new pages (which + become `opened_contexts`). `detail` names an opened dialog. `surface` is `BrowserContext(url, dialog)` when a + dialog is open after the command (one it opened, or one that refused it), else `None`. +- Navigation replies (`Navigate`, `Back`, and the sidecar's `go_forward` / `reload`) carry `dialog`, the dialog open + when the reply is built; `surface` is `BrowserContext(url, title, dialog)`. `OpenContext`, `SwitchContext` and + `CloseContext` list the tabs after the command (`list_tabs`, one more request) and report `tabs`, `tab_urls` and + `current_tab` in `surface` (switch and close: the title of the active tab); a failure of that list after a switch + or close is `dispatched=True`, since the tab command already ran. +- Errors: each sidecar error code maps to one typed error. `dispatched=False` only for errors the sidecar raises + before any input reaches the page: `StaleIndexError` and `StaleNodeError` (→ `StaleRef`; a node the CDP says is + not in the document, "does not belong to the document" / "No node with given id"), `ElementNotFound`, + `ActionIntercepted` (with `by`), `DriverNotConnected` (→ `ConnectionLost`) and `DriverUnsupported` (→ + `Unsupported`). Each action marks where its input starts; a stale or missing node after that point reaches the + driver as a plain `DriverError(dispatched=True)`. A browser that went away reaches the sidecar as browser-use's + own error once its reconnect gives up ("Reconnection failed - CDP still not connected", "Reconnection wait timed + out", "browser not connected"), and a target that detached under it fails the connect with "No valid agent + focus available - target may have detached"; an unclassified error whose message says `not connected`, + `Reconnection wait timed out`, `No valid agent focus available` or `target may have detached` becomes + `ConnectionLost(dispatched=True)`, so the driver session restarts (at `start()` too). Other codes it cannot + classify become `DriverError(dispatched=True)` and are logged raw. A script error carries the first line of the + JavaScript error (`TypeError: …`), not CDP's bare "Uncaught". +- Script source. The page-wide `evaluate` and the element-scoped script evaluate `(code)` and call the value only + when it is a function (the element-scoped one with the element and `args`; the element is also in scope for an + expression). +- Locators. A `Locator` goes to the sidecar as a CSS selector for `querySelector`; Playwright-only syntax + (`:has-text()`, `:text()`, `:text-is()`, `:visible`, `>>`, an engine prefix such as `text=` or `xpath=`) is + `Unsupported(dispatched=False)` with a message naming the supported forms (a ref or `target_id`, plain CSS). +- A hung page. Two transport timeouts in a row (requests that got no reply in 30 s) raise + `ConnectionLost(dispatched=True)` instead of the second `ActionTimeout`, so the driver session makes its single + restart; any reply, or `start()`, resets the count, and the driver's own `Wait` timeout does not count. +- Cancellation. One request is in flight at a time; after 30 s without a reply the transport sends the sidecar a + `cancel` for it and raises `ActionTimeout`. When the caller cancels the request instead (`Timeouts`), the + transport sends the same `cancel` and re-raises the cancellation, so the sidecar does not run the action alongside + the next one. A late reply is dropped by id. + +### Playwright MCP (`drivers/playwright_mcp/`) + +`snapshot.py` and `result.py` are the only code that reads `@playwright/mcp` text. Both are pure functions over +one tool result; they were written against `@playwright/mcp@0.0.78` and its recorded answers +(`tests/unit_tests/harness/tools/browser_move/fixtures/mcp/`). + +- **Sections.** A result is one or more `text` blocks (plus an `image` block for screenshots) made of `### Name` + sections: `Ran Playwright code`, `Open tabs`, `Page`, `Modal state`, `Snapshot`, `Result`, `Events`, `Error`. + `split_sections` keeps their order. +- **Boxes.** `without_boxes(text)` removes the ` [box=x,y,width,height]` annotations of a snapshot taken with + `boxes`; the result is byte for byte the text the server gives without `boxes` (checked live on the scene pages). +- **Header** (`PageHeader(url, title, tabs, focused, tab_urls)`). From `### Page` (`- Page URL:`, `- Page Title:`; the title + is missing while a dialog is open) and `### Open tabs`, or from a `browser_tabs` result list (`parse_tabs`). A + tab is `ContextRef(id=<index>, kind="tab", title)`; the `(current)` one is `focused`. The server lists tabs only + when more than one is open, so a `### Page` without `### Open tabs` means one tab, `"0"`. `tabs` always includes + the focused tab; `tab_urls` runs parallel to it (the page URL for the single tab). +- **Elements** (`parse_elements`). One `Element` per YAML node with a ref matching `ARIA_REF` = `(f\d+)?e\d+` + (iframe refs and the frame-prefixed refs of later pages included); nodes without a ref (`text:`, `option`) are + not elements. `DriverRef(ref, generation="")`, `role`, `name` (JSON-unescaped), `focused` from `[active]`, + `enabled` unless `[disabled]`, `bounds` from `[box=x,y,width,height]` (only when the snapshot was taken with + `boxes`). `value` only for `textbox`, `searchbox`, `combobox`, `spinbutton`, `slider`: the inline text, a child + `text:` line or, for a combobox, the `[selected]` option. Everything else goes to `extra`: the remaining + attributes (`level`, `cursor`, `checked`, …; flags as `True`), `/key: value` property lines (`url`, + `placeholder`) and other nodes' inline text (`text`). YAML-quoted lines (`'button "Note: read me" [ref=…]'`, + `: "42"`) are unquoted. +- **`decode(tool_name, raw_result, *, output_dir, before=None, sensitive=False) -> ActionResult`.** `raw_result` + is the `CallToolResult` as JSON (`content`, `isError`). `output_dir` is the server's working directory: file links + in the text are relative to it. `before` is the `BrowserContext` before the call. `sensitive` drops + `### Ran Playwright code`, which echoes typed text and scripts, before anything else is read. + - `value`: the screenshot from an `image` block (`Screenshot`, size from the PNG header); else `### Result` + parsed as JSON when it is JSON (`undefined` → `None`), the text otherwise; `None` for a `browser_tabs` list, + which goes to `surface`. + - `detail`: the remaining sections as text, without `### Result`, `### Snapshot` and `### Events`; `None` when + nothing remains. + - `surface`: `BrowserContext(url, title, tabs, tab_urls, current_tab, dialog)` from the header (`current_tab` is + the position of the focused tab); `None` when the result has no + `### Page` and is not a tab list (`browser_fill_form`, `browser_evaluate`, `browser_press_key`). `dialog` is + `"<type>: <message>"` from a `### Modal state` line `- ["<type>" dialog with message "<message>"]: can be + handled by …` (the server does not escape the message; it ends at the first `"]: can be handled by`). + - `observation`: from the inline `### Snapshot` YAML, or from the linked `page-*.yml` read under `output_dir`. + A link that resolves outside `output_dir`, a missing or unreadable file and an empty file (the server writes + one while a dialog blocks the snapshot) give `None`; the runtime then observes. `native_text` is the YAML + verbatim, `context` the focused tab, `surface` as above, `generation` empty. + - `context_changed`: always for `browser_navigate`, `browser_navigate_back` and `browser_tabs` (a same-URL + reload included); when `### Modal state` reports a dialog (a file chooser is not one); and, with `before`, + when the URL differs or the number of tabs changed. `opened_contexts`: the tabs past `len(before.tabs)`. + Without `before`, a click that navigates or opens a tab reports no change. +- **Errors.** `decode` raises; the message is the `### Error` text without ANSI escapes. Checked in this order: + + | Signal | Raised | `dispatched` | + |---|---|---| + | `Ref … not found in the current page snapshot` | `StaleRef` | `False` | + | `File access denied: … is outside allowed roots` (the server checks file paths before it touches the page) | `DriverError` | `False` | + | `Target page, context or browser has been closed`, `Failed to open a new tab` | `ConnectionLost` | `True` | + | call log shows `performing <x> action` or `<x> action done` | `ActionTimeout` on a timeout, else `DriverError` | `True` | + | call log has `… intercepts pointer events` | `ActionIntercepted(by=tag#id.class "text")` of the last blocker (`… in <container>` for a `from … subtree` blocker) | `False` | + | `browser_wait_for` timed out (the wait is the command, though its log reads like an actionability wait) | `ActionTimeout` | `True` | + | `TimeoutError:` / `Timeout Nms exceeded` | `ActionTimeout` | `False` if every call-log line is an actionability wait, else `True` | + | anything else | `DriverError` | `True` | + + Call-log lines are read without ANSI escapes and without the `N ×` fold. A call refused because a dialog is open + (`Tool "<name>" does not handle the modal state`) is not raised: it returns `ok=False` with the error and the + modal state in `detail`, as browser-use does for an action while a dialog is open; with `before` and a dialog, + `surface` is `before` with that `dialog` (nothing was sent, so the page is the one before the call). + +#### The driver (`drivers/playwright_mcp/driver.py`) + +- `create_playwright_mcp_driver(device, surface_settings)` builds the `@playwright/mcp` server config with + `drivers/playwright_mcp/config.playwright_mcp_config` (the builder the runner's `settings.py` public functions call), + with `device.endpoint` as the CDP endpoint. A config in `surface_settings.mcp_server` keeps its `params` (command, + args, cwd, timeout, env) and `client_type`; the factory's server name and id win, so the tool prefix never depends + on the caller. In the given env `PLAYWRIGHT_MCP_DEVICE` is dropped (no device emulation over CDP) and, when the + device has an endpoint, `PLAYWRIGHT_MCP_CDP_ENDPOINT` is set and `PLAYWRIGHT_MCP_BROWSER` defaults to `chrome`, as + the old service injection did. The composition root never sees the command line. +- `start()` owns one MCP client (`BrowserMoveStdioClient`, or `BrowserMoveStreamableHttpClient` for + `streamable-http`) and calls it directly, never through `ToolMgr`. A stdio server is started with + `--init-script <path of probes/clipboard_guard.js>` appended to the configured arguments (`@playwright/mcp` + evaluates the file in every page before the page's scripts); the config the driver keeps is unchanged, so + `DriverInfo.versions` still reads the version from it. A `streamable-http` server is not ours to start and gets + no guard. `start()` reads `tools/list` once and keeps each + mapped tool's description as `DriverInfo.tool_hints`, and every listed tool that has a decoder (`tools.py`) as + `DriverInfo.native_tools[name] = NativeTool(description, schema, build)`: the description verbatim, the schema + as the client lists it (`inputSchema` through the stdio client's `sanitize_json_schema`, so without `$schema`); + `browser_find` is listed the same way with `build=None` (no command backs it; the surface's `FindTool` takes its + description and schema). Before + `start()` `native_tools` is empty. `tool_name_prefix` is `mcp_model_tool_prefix(server_name)` + (`mcp_playwright-official_`), the prefix the ability manager gives MCP tools; an instance key never changes it. + `display_name` is `"Playwright MCP"`. `health()` pings the session. +- `observe()` calls `browser_snapshot` with `boxes: true` (and `browser_take_screenshot` when asked), so every + element has `bounds` and a `PointRef` can be built from an observation. `native_text` is the full text of that + result (header, open tabs, YAML) without the boxes (`without_boxes`): what the model reads today. Tabs come from + the snapshot header, never from a `browser_tabs list`, except while a modal state (a dialog or a file chooser) + makes the server refuse `browser_snapshot` and every other page tool: then one `browser_tabs list` (not a page + tool) gives `context` and `surface` (URL, tabs, the dialog; the title is empty), `elements` are empty, no + screenshot is taken and `native_text` is the refusal text. An observation that comes free with an action + result (the linked snapshot file) has no bounds. + A scoped `Observe` is sent exactly as the model asked and without `boxes`: `depth` → `depth`, `root` → `target` + (`DriverRef.ref` or the `Locator`'s css; a `PointRef` or a text `Locator` is `Unsupported(dispatched=False)` before + the call), so `native_text` is the server's answer byte for byte (fixtures `snapshot_depth_1`, `snapshot_target`) + and the elements of the subtree have no bounds. +- **One attempt per tool call.** The driver calls `call_tool_once` on its client, never the retrying `call_tool` + the runtime path uses: a call that fails is not sent again. A transport timeout (`mcp_timeout_s`) is + `ActionTimeout(dispatched=True)`; no connection is `ConnectionLost(dispatched=False)`; a transport failure (the + client's `is_transport_error`: closed or broken streams, a closed connection, read/write and broken-pipe errors, + a terminated session) is `ConnectionLost(dispatched=True)`. Any other client exception (a JSON-RPC error, a + serialization error) is `DriverError(dispatched=False)` with the exception type and message, so one bad argument + never restarts the browser. +- **Sensitive commands.** `TypeText(sensitive=True)`, `RunScript(sensitive=True)` and a `FillForm` with a sensitive + field are decoded with `sensitive=True`, so their `detail` has no echoed code; the error message of a failed + sensitive call can still quote it, and `Tracing` does not record that message. +- **Files.** The server reads upload and drop files only under its working directory and its output directory. + `UploadFile` and `Drop` first resolve their paths with `commands.upload_paths` and + `PlaywrightMcpConfig.upload_root` (the factory fills it from `surface_settings.upload_root`; a relative name is + a file under the upload root, never under the working directory); a `ValueError` of the rule is + `ActionResult(ok=False, detail=<message>)` and nothing is sent. Each resolved path that is an existing file + outside the working directory (`output_dir`) is staged: copied to `output_dir/uploads/<12 hex>/<file name>` (the + page sees the original name) and the copy's path is sent. Paths inside the working directory, missing files and + directories go as resolved; the server refuses those outside with `File access denied: … is outside allowed + roots` (decoded as `DriverError(dispatched=False)`). `stop()` removes the `uploads/<12 hex>/` folders this + driver created, then `uploads/` itself only when nothing else is left in it: the working directory defaults to + the process directory, which may hold a user's own `uploads/` folder. +- Every handler sends one tool call and decodes it with `decode(..., before=<last BrowserContext>)` in a worker + thread (the decoder may read the linked snapshot file). Commands and tools: Click → `browser_click` / + `browser_mouse_click_xy`; TypeText → `browser_type`; PressKey → `browser_press_key`; Hover → `browser_hover` / + `browser_mouse_move_xy`; Drag → `browser_drag` / `browser_mouse_drag_xy`; Scroll → `browser_mouse_wheel`; + SelectOption, FillForm, Drop, UploadFile (no target: answers the open file chooser), HandleDialog, Wait + (`browser_wait_for`), RunScript (`browser_evaluate`, page JavaScript), Navigate, Back, Open/Switch/CloseContext + (`browser_tabs`), GetPdf, GetConsole, GetNetwork. Element refs: `DriverRef`, `Locator(css)`; `PointRef` for the + coordinate tools. +- **Context ids are valid until the next context change.** On this driver they are tab positions (`"0"`, `"1"`), + which shift when an earlier tab closes; every open, switch and close reports `context_changed`. + +#### Native tools (`drivers/playwright_mcp/tools.py`) + +`DECODERS` maps each server tool a capability can select to the pure decoder of its arguments, the inverse of the +driver's handler: decoding a call the driver sent and sending the command again gives the same arguments. A +`target` that matches `ARIA_REF` becomes `DriverRef(target, "")`, any other target +`Locator(css=target, text=None, generation="")`: the server resolves both against the live page and judges +staleness itself, so the page state never refuses them (Refs). Coordinates become +`PointRef(round(x), round(y), resolver.screenshot_generation)` (the model sends no generation for a point, and the +page state refuses one on an older screenshot). + +| Server tool | Command | +|---|---| +| `browser_click` | `Click(target, button, modifiers, count=2 if doubleClick else 1)` | +| `browser_type` | `TypeText(target, text, submit, clear=not slowly, slowly)` | +| `browser_press_key` | `PressKey(key)` | +| `browser_hover` / `browser_drag` | `Hover(target)` / `Drag(startTarget, endTarget)` | +| `browser_select_option` | `SelectOption(target, values)` (at least one) | +| `browser_fill_form` | `FillForm(fields)`: each `FormField(target, FieldKind(type), value)` | +| `browser_drop` | `Drop(target, paths, data=<MIME type, value pairs>)` | +| `browser_file_upload` | `UploadFile(None, paths)` (answers the open file chooser) | +| `browser_handle_dialog` | `HandleDialog(accept, promptText)` | +| `browser_evaluate` | `RunScript(function, target=<target or None>)` | +| `browser_navigate` / `browser_navigate_back` | `Navigate(url)` / `Back()` | +| `browser_snapshot` | `Observe(depth=round(depth), root=<target or None>)`; `Observe()` without them | +| `browser_take_screenshot` | `Observe(include_screenshot=True, full_page=fullPage)` | +| `browser_tabs` | `list` → `Observe()`; `new` → `OpenContext(url)`; `select` → `SwitchContext(contexts[index])`; `close` → `CloseContext(contexts[index])`, the current context without `index` | +| `browser_close` | `CloseContext(current_context)` | +| `browser_mouse_click_xy` / `_move_xy` / `_drag_xy` | `Click(point, button, count=clickCount)` / `Hover(point)` / `Drag(start, end)` | +| `browser_mouse_wheel` | `Scroll(None, deltaX, deltaY)` | +| `browser_pdf_save` / `browser_console_messages` / `browser_network_requests` | `GetPdf(filename)` / `GetConsole(level, all)` / `GetNetwork(static, filter)` | + +Arguments that do not change the command are accepted and ignored: `element` (every tool), `filename`, the +snapshot's `boxes`, the screenshot's `target`, `element`, `type`, `scale`, and the click's `delay`. A missing +required argument, a wrong type (a non-numeric `depth` included), an unknown field type or tab action, a `select` +without index and an index outside `contexts` raise `ValueError`. `browser_find` (served by the surface's `FindTool`) and +`browser_wait_for` (not model-facing) have no decoder. + +## Relation to other specs + +- `S_05` (tools contract): the tools built from these commands are registered like every harness tool and go + through the permission engine. +- `S_08` (security engine): `ActionPolicy` checks `PolicyRule`s on commands before middleware and the driver and + denies with a `PolicyDenied` result. +- `S_18` (subagents): `create_browser_agent` is the public entry point; its signature does not change. diff --git a/openjiuwen/harness/rails/subagent/subagent_rail.py b/openjiuwen/harness/rails/subagent/subagent_rail.py index 78e90a2af8..70548dec51 100644 --- a/openjiuwen/harness/rails/subagent/subagent_rail.py +++ b/openjiuwen/harness/rails/subagent/subagent_rail.py @@ -5,7 +5,7 @@ from __future__ import annotations import asyncio -from typing import Callable, Collection, List, Optional, TYPE_CHECKING +from typing import TYPE_CHECKING, Callable, Collection, List, Optional from openjiuwen.core.common.logging import logger from openjiuwen.core.foundation.tool import ToolCard @@ -15,9 +15,6 @@ from openjiuwen.harness.rails.base import DeepAgentRail from openjiuwen.harness.schema.config import SubAgentConfig from openjiuwen.harness.tools import SessionToolkit, build_session_tools, create_task_tool -from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_capabilities import ( - DEFAULT_BROWSER_CAPABILITIES, -) from openjiuwen.harness.tools.subagent._control_registry import release_all_subagent_controls from openjiuwen.harness.tools.subagent.subagent_tools import build_subagent_tools @@ -363,6 +360,10 @@ def _build_available_agents_description(self, subagents: List[SubAgentConfig | " tools_str = self._extract_agent_tools(spec, agent_name) lines.append(f"- {agent_name}: {agent_desc} (Tools: {tools_str})") if agent_name == "browser_agent": + from openjiuwen.harness.tools.browser_move.surfaces.browser.capabilities import ( + DEFAULT_BROWSER_CAPABILITIES, + ) + lines.append(" Available Playwright capabilities:") for capability in DEFAULT_BROWSER_CAPABILITIES: tool_names = ", ".join(capability.tool_names) diff --git a/openjiuwen/harness/schema/decision_policy.py b/openjiuwen/harness/schema/decision_policy.py new file mode 100644 index 0000000000..8dc2ca443d --- /dev/null +++ b/openjiuwen/harness/schema/decision_policy.py @@ -0,0 +1,25 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Structural contract for model slots that decide their own steps against a live runtime. + +``harness/subagents`` is a generic assembly layer; concrete decision policies live outside this +repository. This protocol lets the assembly layer recognise such a model without importing any +concrete class. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Protocol, runtime_checkable + +if TYPE_CHECKING: + from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade import BrowserAgentRuntime + + +@runtime_checkable +class DecisionPolicyModel(Protocol): + """A model that decides browser steps itself and needs the live runtime to do it.""" + + def bind_runtime(self, runtime: "BrowserAgentRuntime") -> None: + """Attach the live runtime the policy will probe and act through.""" + ... diff --git a/openjiuwen/harness/subagents/browser_agent.py b/openjiuwen/harness/subagents/browser_agent.py index 31ad90fe67..32e371072b 100644 --- a/openjiuwen/harness/subagents/browser_agent.py +++ b/openjiuwen/harness/subagents/browser_agent.py @@ -4,47 +4,41 @@ from __future__ import annotations -import copy import dataclasses from typing import TYPE_CHECKING, Any, Dict, List, Optional from openjiuwen.core.common.logging import logger -from openjiuwen.core.context_engine import ToolResultWindowProcessorConfig from openjiuwen.core.foundation.llm.model import Model -from openjiuwen.core.foundation.llm.schema.config import ModelRequestConfig from openjiuwen.core.foundation.tool import McpServerConfig, Tool, ToolCard from openjiuwen.core.single_agent.rail.base import AgentRail from openjiuwen.core.single_agent.schema.agent_card import AgentCard from openjiuwen.core.sys_operation import SysOperation from openjiuwen.harness.deep_agent import DeepAgent -from openjiuwen.harness.factory import create_deep_agent -from openjiuwen.harness.rails.context_engineer import ContextProcessorRail from openjiuwen.harness.schema.config import SubAgentConfig -from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_state_context_processor import ( - BrowserStateContextProcessorConfig, +from openjiuwen.harness.schema.decision_policy import DecisionPolicyModel +from openjiuwen.harness.tools.browser_move.runtime.config import CoreSettings +from openjiuwen.harness.tools.browser_move.surfaces.browser.capabilities import ( + DEFAULT_BROWSER_CAPABILITIES, + resolve_browser_capabilities, ) -from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_working_context_processor import ( - BrowserWorkingContextProcessorConfig, +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.agent import ( + BROWSER_AGENT_CARD_ID, + DEFAULT_BROWSER_AGENT_DESCRIPTION, + DEFAULT_BROWSER_AGENT_MAX_ITERATIONS, + DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT, + DEFAULT_BROWSER_AGENT_TEMPERATURE, + browser_model_with_temperature, + build_main_agent, ) -from openjiuwen.harness.tools.browser_move.offload_recall import BrowserOffloadRecallTool -from openjiuwen.harness.tools.browser_move.playwright_runtime.config import ( +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade import BrowserAgentRuntime +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.logging import configure_browser_agent_logging +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings import ( BrowserInstanceConfig, RuntimeSettings, build_browser_guardrails, build_playwright_mcp_config, build_runtime_settings, ) -from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_capabilities import ( - DEFAULT_BROWSER_CAPABILITIES, - resolve_browser_capabilities, -) -from openjiuwen.harness.tools.browser_move.playwright_runtime.runtime import ( - BrowserAgentRuntime, - BrowserRuntimeRail, -) -from openjiuwen.harness.tools.browser_move.playwright_runtime.runtime_tools import ( - build_browser_runtime_tools, -) try: from openjiuwen.harness.prompts import resolve_language @@ -59,78 +53,6 @@ def resolve_language(language: Optional[str] = None) -> str: # type: ignore[mis BROWSER_AGENT_FACTORY_NAME = "browser_agent" -# Agent checkpoints are namespaced by AgentCard.id inside a conversation. -# Keep the default stable so reconstructing this subagent can restore its -# Session-backed working context on a same-conversation follow-up. -BROWSER_AGENT_CARD_ID = "openjiuwen.browser_agent" -DEFAULT_BROWSER_AGENT_TEMPERATURE = 0.4 -DEFAULT_BROWSER_AGENT_MAX_ITERATIONS = 100 -_BROWSER_MODEL_TEMPERATURE_MARKER = "_browser_agent_temperature" -_BROWSER_PARENT_MODEL_MARKER = "_browser_agent_parent_model" - -DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT_EN = ( - "You are a browser automation agent responsible for executing web tasks directly. " - "Choose the strategy at this agent level and use the available Playwright and runtime tools; " - "the runtime validates targets and outcomes but does not replace your task judgment.\n" - "Every model call includes one runtime-maintained <browser_working_context> followed by the latest " - "<browser_state>. Runtime outcomes and blockers are authoritative; inferred fields are extraction hints, " - "not additional user requirements. Source observations also provide usable evidence. " - "A fresh browser capture occurs initially and after a recognized page mutation; otherwise the cached " - "observation is reused. When the runtime directive requires replanning, change the strategy materially.\n" - "For a simple lookup, prefer a direct search-results URL when the engine and query are known. Use " - "browser_probe_interactives for page controls and browser_probe_cards for repeated results or products. " - "Use the compact PageState target_id with its generation_id directly; never reconstruct guessed CSS from " - "a target ID. When a card has primary_link or href, navigate directly to that URL.\n" - "Use browser_batch_interact when two or more deterministic actions or same-page field extractions are already " - "known. Use a primitive for one uncertain action. Prefer observable condition waits over fixed sleep. Use " - "browser_snapshot only when compact probes are insufficient, and browser_evaluate only for a small exact " - "target or computation. If an older result has a <persisted-output> marker, recall it only when its preview " - "does not contain the needed evidence; recalled targets are not executable after navigation.\n" - "Use canonical fields when convenient, but do not reread a page just to rename already observed facts. " - "Report genuinely unavailable values as unknown. One trustworthy page value or structured result is enough; " - "do not verify the same fact with multiple " - "tools. Stop immediately when the requested outcome is evidenced. The runtime determines final status, so " - "return a concise natural-language result rather than another progress object.\n" - "If an optional capability makes a browser_run_code tool visible, use it only when deterministic tools are " - "insufficient, and never dump the full document. Preserve session continuity and report a concrete blocker " - "when the available browser state or tools cannot complete the task." -) - -DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT_CN = ( - "你是浏览器自动化代理，负责直接完成网页任务。请在当前代理层决定策略并使用可见的 Playwright " - "和 runtime 工具；runtime 负责验证目标和结果，但不代替你的任务判断。\n" - "每次模型调用都会依次提供 runtime 维护的 <browser_working_context> 和最新 <browser_state>。" - "运行结果和阻断项是权威信息；推断字段只是提取提示，不是新增的用户要求。带来源的页面原文也可作为证据。" - "系统仅在初始调用和已识别的" - "页面变更后重新观察；runtime 要求重新规划时，应实质改变策略。\n" - "已知搜索引擎和关键词时，简单查询优先直接构造搜索结果 URL。页面控件使用 " - "browser_probe_interactives，重复结果或商品使用 browser_probe_cards。直接使用 PageState 返回的 " - "target_id 和 generation_id，禁止把 target_id 改写成猜测的 CSS。卡片包含 primary_link 或 href " - "时直接导航该 URL。\n" - "只有两个及以上确定动作或同页多字段提取时使用 browser_batch_interact；单个不确定动作使用基础工具。" - "优先等待可观察条件，不使用固定 sleep。紧凑 Probe 不足时再使用 browser_snapshot；" - "browser_evaluate 仅用于小范围精确目标或计算。旧结果出现 <persisted-output> 且预览不足时才恢复；" - "导航后恢复内容中的目标不可继续操作。\n" - "方便时使用规范字段，但不要仅为改写字段名而重读已经观察到的事实；字段确实不可获取时使用 unknown。" - "一个可信页面值或结构化结果已经足够，不要用多个工具重复验证同一事实。请求结果有证据后立即结束。" - "最终状态由 runtime 决定，只返回简洁自然语言结果，不再维护第二份进度对象。\n" - "只有可选能力明确暴露 browser_run_code 时才使用，并且仅限确定性工具不足的情况；禁止转储完整页面。" - "保持浏览器会话连续；现有页面或工具确实无法完成时，报告具体 blocker。" -) - -DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT: Dict[str, str] = { - "cn": DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT_CN, - "en": DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT_EN, -} - -DEFAULT_BROWSER_AGENT_DESCRIPTION_EN = ( - "Dedicated browser subagent that directly controls the browser with Playwright MCP tools." -) -DEFAULT_BROWSER_AGENT_DESCRIPTION_CN = "专用浏览器子代理，直接使用 Playwright MCP 工具执行网页任务。" -DEFAULT_BROWSER_AGENT_DESCRIPTION: Dict[str, str] = { - "cn": DEFAULT_BROWSER_AGENT_DESCRIPTION_CN, - "en": DEFAULT_BROWSER_AGENT_DESCRIPTION_EN, -} def _coerce_browser_instance( @@ -178,45 +100,6 @@ def _resolve_runtime_settings( return build_runtime_settings(instance) -def _browser_model_with_temperature(model: Model, temperature: float) -> Model: - """Copy a parent model descriptor and override sampling for browser work.""" - resolved_temperature = max(0.0, min(float(temperature), 2.0)) - if getattr(model, _BROWSER_MODEL_TEMPERATURE_MARKER, None) == resolved_temperature: - return model - parent_model = getattr(model, _BROWSER_PARENT_MODEL_MARKER, model) - - model_config = getattr(model, "model_config", None) - if isinstance(model_config, ModelRequestConfig): - browser_model_config = model_config.model_copy( - deep=True, - update={"temperature": resolved_temperature}, - ) - elif model_config is None: - browser_model_config = ModelRequestConfig(temperature=resolved_temperature) - else: - browser_model_config = copy.copy(model_config) - setattr(browser_model_config, "temperature", resolved_temperature) - - model_client_config = getattr(model, "model_client_config", None) - if issubclass(type(model), Model) and model_client_config is not None: - browser_model = Model( - model_client_config=model_client_config, - model_config=browser_model_config, - ) - setattr(browser_model, _BROWSER_MODEL_TEMPERATURE_MARKER, resolved_temperature) - setattr(browser_model, _BROWSER_PARENT_MODEL_MARKER, parent_model) - return browser_model - - # Lightweight test doubles and compatibility model descriptors may not be - # constructible as a concrete Model. Preserve their shape without mutating - # the parent object. - browser_model = copy.copy(model) - browser_model.model_config = browser_model_config - setattr(browser_model, _BROWSER_MODEL_TEMPERATURE_MARKER, resolved_temperature) - setattr(browser_model, _BROWSER_PARENT_MODEL_MARKER, parent_model) - return browser_model - - def build_browser_agent_config( model: Model, *, @@ -241,7 +124,7 @@ def build_browser_agent_config( """Build a SubAgentConfig that materializes as create_browser_agent().""" resolved_language = resolve_language(language) instance = _coerce_browser_instance(browser_instance, browser_key) - browser_model = _browser_model_with_temperature(model, temperature) + browser_model = browser_model_with_temperature(model, temperature) resolved_settings = _resolve_runtime_settings(browser_model, settings, instance) return SubAgentConfig( agent_card=card @@ -297,13 +180,15 @@ def create_browser_agent( browser_key: Optional[str] = None, browser_instance: Optional[BrowserInstanceConfig | Dict[str, Any]] = None, browser_capabilities: Optional[List[str]] = None, + runtime: Optional[BrowserAgentRuntime] = None, **config_kwargs: Any, ) -> DeepAgent: """Create the browser subagent with task-scoped capability context. ``browser_capabilities`` is resolved against the trusted capability catalog here. Core is always applied, including when the caller omits the - optional capability list. + optional capability list. ``runtime`` is an existing facade to build on + (the WebVoyager runner's); by default the agent gets its own. """ if browser_capabilities is not None and ( not isinstance(browser_capabilities, list) @@ -325,105 +210,38 @@ def create_browser_agent( ) resolved_language = resolve_language(language) - instance = _coerce_browser_instance(browser_instance, browser_key) - browser_model = _browser_model_with_temperature(model, temperature) - resolved_settings = _resolve_runtime_settings(browser_model, settings, instance) - - final_card = card or AgentCard( - id=BROWSER_AGENT_CARD_ID, - name="browser_agent", - description=DEFAULT_BROWSER_AGENT_DESCRIPTION.get( - resolved_language, - DEFAULT_BROWSER_AGENT_DESCRIPTION["cn"], - ), - ) - final_prompt = system_prompt or DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT.get( - resolved_language, - DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT["cn"], - ) - - runtime_kwargs: Dict[str, Any] = { - "provider": resolved_settings.provider, - "api_key": resolved_settings.api_key, - "api_base": resolved_settings.api_base, - "model_name": resolved_settings.model_name, - "mcp_cfg": resolved_settings.mcp_cfg, - "guardrails": resolved_settings.guardrails, - "instance": resolved_settings.instance, - "allowed_tool_names": resolved_capabilities.allowed_tool_names, - } - browser_backend = BrowserAgentRuntime(**runtime_kwargs) - injected_tools = build_browser_runtime_tools(browser_backend, language=resolved_language) - working_context_config = BrowserWorkingContextProcessorConfig( - language=resolved_language, - runtime_projection_only=True, - ) - injected_rails: List[AgentRail] = [ - BrowserRuntimeRail(browser_backend), - ] - injected_tools.append(BrowserOffloadRecallTool(workspace, language=resolved_language)) - - browser_state_processor = ( - "BrowserStateContextProcessor", - BrowserStateContextProcessorConfig(provider=browser_backend), + browser_model = ( + model if isinstance(model, DecisionPolicyModel) else browser_model_with_temperature(model, temperature) ) - browser_working_context_processor = ( - "BrowserWorkingContextProcessor", - working_context_config, - ) - browser_windowed_tool_names = [ - "browser_probe_interactives", - "browser_probe_cards", - "browser_snapshot", - "browser_find", - "browser_evaluate", - ] - browser_tool_result_window_processor = ( - "ToolResultWindowProcessor", - ToolResultWindowProcessorConfig( - tool_names=browser_windowed_tool_names, - keep_last_k=2, - trim_size=1000, - min_offload_chars=1000, - small_result_trim_size=1000, - ), - ) - caller_context_rails = [rail for rail in (rails or []) if isinstance(rail, ContextProcessorRail)] - if caller_context_rails: - for context_rail in caller_context_rails: - context_rail.add_processors( - [ - browser_tool_result_window_processor, - browser_state_processor, - browser_working_context_processor, - ] - ) - else: - injected_rails.append( - ContextProcessorRail( - processors=[ - browser_tool_result_window_processor, - browser_state_processor, - browser_working_context_processor, - ], - preset=False, - ) + configure_browser_agent_logging(CoreSettings.from_process_env()) + + if runtime is None: + instance = _coerce_browser_instance(browser_instance, browser_key) + resolved_settings = _resolve_runtime_settings(browser_model, settings, instance) + runtime = BrowserAgentRuntime( + provider=resolved_settings.provider, + api_key=resolved_settings.api_key, + api_base=resolved_settings.api_base, + model_name=resolved_settings.model_name, + mcp_cfg=resolved_settings.mcp_cfg, + guardrails=resolved_settings.guardrails, + instance=resolved_settings.instance, + allowed_tool_names=resolved_capabilities.allowed_tool_names, + workspace=workspace, + language=resolved_language, ) - - final_tools = list(tools or []) + injected_tools - final_mcps = list(mcps or []) - final_rails = list(rails or []) + injected_rails - - agent = create_deep_agent( - model=browser_model, - card=final_card, - system_prompt=final_prompt, - tools=final_tools, - mcps=final_mcps, + return build_main_agent( + runtime, + browser_model, + card=card, + system_prompt=system_prompt, + tools=tools, + mcps=mcps, subagents=subagents, - rails=final_rails, + rails=rails, enable_task_loop=enable_task_loop, max_iterations=max_iterations, + temperature=temperature, workspace=workspace, skills=skills, backend=backend, @@ -432,17 +250,13 @@ def create_browser_agent( prompt_mode=prompt_mode, **config_kwargs, ) - agent.register_task_resource_cleanup( - browser_backend.release_task_resources, - prepare=browser_backend.acquire_task_resources, - ) - return agent __all__ = [ "BROWSER_AGENT_CARD_ID", "BROWSER_AGENT_FACTORY_NAME", "DEFAULT_BROWSER_AGENT_MAX_ITERATIONS", + "DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT", "DEFAULT_BROWSER_AGENT_TEMPERATURE", "build_browser_agent_config", "create_browser_agent", diff --git a/openjiuwen/harness/tools/browser_move/__init__.py b/openjiuwen/harness/tools/browser_move/__init__.py index f8220bf908..da21e1ab7a 100644 --- a/openjiuwen/harness/tools/browser_move/__init__.py +++ b/openjiuwen/harness/tools/browser_move/__init__.py @@ -15,24 +15,16 @@ __all__ = [ "REPO_ROOT", "build_browser_runtime_mcp_config", - "register_browser_runtime_mcp_server", "reset_active_browser_runtimes", "reset_managed_browser_runtime", - "restart_local_browser_runtime_server", - "stop_local_browser_runtime_server", ] def __getattr__(name: str) -> Any: if name in {"reset_active_browser_runtimes", "reset_managed_browser_runtime"}: - module = import_module("openjiuwen.harness.tools.browser_move.playwright_runtime.runtime") + module = import_module("openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade") return getattr(module, name) - if name in { - "build_browser_runtime_mcp_config", - "register_browser_runtime_mcp_server", - "restart_local_browser_runtime_server", - "stop_local_browser_runtime_server", - }: - module = import_module("openjiuwen.harness.tools.browser_move.playwright_runtime.browser_tools") + if name == "build_browser_runtime_mcp_config": + module = import_module("openjiuwen.harness.tools.browser_move.surfaces.browser.runner.server_config") return getattr(module, name) raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/openjiuwen/harness/tools/browser_move/contract/__init__.py b/openjiuwen/harness/tools/browser_move/contract/__init__.py new file mode 100644 index 0000000000..ee7c228d93 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/__init__.py @@ -0,0 +1,130 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The driver contract: commands, observations, results, errors, the driver base and the registries.""" + +from .actions import ( + Action, + Click, + CloseContext, + Drag, + Effect, + HandleDialog, + Hover, + Observe, + PressKey, + RunScript, + Scroll, + SwitchContext, + TypeText, + Wait, +) +from .device import ProvisionedDevice +from .driver import Driver, DriverInfo, HandlerDriver, Health, NativeTool, ensure_supported, handles, iter_refs +from .errors import ( + ActionIntercepted, + ActionTimeout, + AmbiguousElement, + ConnectionLost, + DriverError, + ElementNotFound, + NavigationError, + PolicyDenied, + StaleRef, + Unsupported, +) +from .observation import Element, Observation, Screenshot, SurfaceContext, Viewport +from .refs import REF_KINDS, ContextRef, DriverRef, IndexRef, Locator, PointRef, Ref, RefResolver +from .registry import ( + DRIVER_TOPOLOGIES, + DRIVERS, + PROVISIONERS, + SURFACES, + Registry, + default_topology, + register_driver, + register_provisioner, + register_surface, +) +from .results import ActionResult +from .surface import ( + Capability, + ContentBlock, + PolicyRule, + Renderer, + Scene, + SceneCase, + SurfaceBundle, + SurfaceProfile, + SurfaceSettings, + ToolSpec, +) + +__all__ = [ + "DRIVERS", + "DRIVER_TOPOLOGIES", + "PROVISIONERS", + "REF_KINDS", + "SURFACES", + "Action", + "ActionIntercepted", + "ActionResult", + "ActionTimeout", + "AmbiguousElement", + "Capability", + "Click", + "CloseContext", + "ConnectionLost", + "ContentBlock", + "ContextRef", + "Drag", + "Driver", + "DriverError", + "DriverInfo", + "DriverRef", + "Effect", + "Element", + "ElementNotFound", + "HandleDialog", + "HandlerDriver", + "Health", + "Hover", + "IndexRef", + "Locator", + "NavigationError", + "NativeTool", + "Observation", + "Observe", + "PointRef", + "PolicyDenied", + "PolicyRule", + "ProvisionedDevice", + "PressKey", + "Ref", + "RefResolver", + "Registry", + "Renderer", + "RunScript", + "Scene", + "SceneCase", + "Screenshot", + "Scroll", + "StaleRef", + "SurfaceBundle", + "SurfaceContext", + "SurfaceProfile", + "SurfaceSettings", + "SwitchContext", + "ToolSpec", + "TypeText", + "Unsupported", + "Viewport", + "Wait", + "default_topology", + "ensure_supported", + "handles", + "iter_refs", + "register_driver", + "register_provisioner", + "register_surface", +] diff --git a/openjiuwen/harness/tools/browser_move/contract/actions.py b/openjiuwen/harness/tools/browser_move/contract/actions.py new file mode 100644 index 0000000000..ad63f00d94 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/actions.py @@ -0,0 +1,152 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Commands that make sense on any screen; a surface adds its own ``Action`` subclasses in its package.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from enum import Enum +from typing import Any, ClassVar + +from .refs import ContextRef, Ref + + +class Effect(str, Enum): + """What a command may do; middleware picks timeouts and retry rules by it.""" + + READ = "read" + ACT = "act" + NAVIGATE = "navigate" + SCRIPT = "script" + + +@dataclass(frozen=True) +class Action: + """Base of every command. Subclasses set ``effect``, and ``__sensitive_fields__`` to mask when ``sensitive``.""" + + effect: ClassVar[Effect] + + +@dataclass(frozen=True) +class Observe(Action): + """A snapshot of the focused context; ``depth`` or ``root`` scope it to a subtree, a partial observation.""" + + effect: ClassVar[Effect] = Effect.READ + + include_screenshot: bool = False + full_page: bool = False + depth: int | None = None + root: Ref | None = None + + @property + def scoped(self) -> bool: + return self.depth is not None or self.root is not None + + +@dataclass(frozen=True) +class Click(Action): + effect: ClassVar[Effect] = Effect.ACT + + target: Ref + button: str = "left" + modifiers: tuple[str, ...] = () + count: int = 1 + + +@dataclass(frozen=True) +class TypeText(Action): + """Type ``text`` into ``target`` (or the focused element when ``None``).""" + + effect: ClassVar[Effect] = Effect.ACT + __sensitive_fields__: ClassVar[tuple[str, ...]] = ("text",) + + target: Ref | None + text: str + submit: bool = False + sensitive: bool = False + clear: bool = True + slowly: bool = False + + +@dataclass(frozen=True) +class PressKey(Action): + effect: ClassVar[Effect] = Effect.ACT + + key: str + target: Ref | None = None + + +@dataclass(frozen=True) +class Scroll(Action): + """Scroll ``target`` (or the whole view when ``None``) by ``dx``/``dy`` pixels.""" + + effect: ClassVar[Effect] = Effect.ACT + + target: Ref | None = None + dx: int = 0 + dy: int = 0 + + +@dataclass(frozen=True) +class Hover(Action): + effect: ClassVar[Effect] = Effect.ACT + + target: Ref + + +@dataclass(frozen=True) +class Drag(Action): + effect: ClassVar[Effect] = Effect.ACT + + source: Ref + destination: Ref + + +@dataclass(frozen=True) +class Wait(Action): + """Sleep ``seconds``, or wait at most ``seconds`` (``None``: driver default) for ``until_text``.""" + + effect: ClassVar[Effect] = Effect.READ + + seconds: float | None = None + until_text: str | None = None + + def __post_init__(self) -> None: + if self.seconds is None and self.until_text is None: + raise ValueError("Wait needs seconds, until_text or both") + + +@dataclass(frozen=True) +class SwitchContext(Action): + effect: ClassVar[Effect] = Effect.NAVIGATE + + context: ContextRef + + +@dataclass(frozen=True) +class CloseContext(Action): + effect: ClassVar[Effect] = Effect.NAVIGATE + + context: ContextRef + + +@dataclass(frozen=True) +class HandleDialog(Action): + effect: ClassVar[Effect] = Effect.ACT + + accept: bool + prompt_text: str | None = None + + +@dataclass(frozen=True) +class RunScript(Action): + """Run ``code`` in the focused context, scoped to ``target`` if set. ``sensitive`` masks ``code`` and ``args``.""" + + effect: ClassVar[Effect] = Effect.SCRIPT + __sensitive_fields__: ClassVar[tuple[str, ...]] = ("code", "args") + + code: str + args: Any = field(default=None, hash=False) + sensitive: bool = False + target: Ref | None = None diff --git a/openjiuwen/harness/tools/browser_move/contract/device.py b/openjiuwen/harness/tools/browser_move/contract/device.py new file mode 100644 index 0000000000..a25458a5a6 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/device.py @@ -0,0 +1,18 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ProvisionedDevice: + """A device made ready by a provisioner, with the endpoint its driver connects to.""" + + kind: str + endpoint: str + extras: Mapping[str, Any] + cleanup: Callable[[], None] | None = None diff --git a/openjiuwen/harness/tools/browser_move/contract/driver.py b/openjiuwen/harness/tools/browser_move/contract/driver.py new file mode 100644 index 0000000000..279656835e --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/driver.py @@ -0,0 +1,147 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The driver interface, its self-description and the handler-based base class.""" + +from __future__ import annotations + +import dataclasses +from abc import ABC, abstractmethod +from collections.abc import Callable, Iterable, Iterator, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any, ClassVar, TypeVar + +from .actions import Action, Observe +from .errors import Unsupported +from .observation import Observation +from .refs import REF_KINDS, RefResolver +from .results import ActionResult + + +@dataclass(frozen=True) +class Health: + ok: bool + detail: str | None = None + + +@dataclass(frozen=True) +class NativeTool: + """A tool the driver offers in its own format; ``build`` decodes its arguments (``None``: no command backs it).""" + + description: str + schema: Mapping[str, Any] = field(hash=False) + build: Callable[[Mapping[str, Any], RefResolver], Action] | None = field(default=None, hash=False) + + +@dataclass(frozen=True) +class DriverInfo: + """Self-description: ``actions`` maps each handled command to the ref kinds it accepts.""" + + name: str + surfaces: frozenset[str] + versions: Mapping[str, str] = field(default_factory=dict, hash=False) + actions: Mapping[type[Action], frozenset[type]] = field(default_factory=dict, hash=False) + observes: frozenset[str] = frozenset() + tool_hints: Mapping[type[Action], str] = field(default_factory=dict, hash=False) + native_tools: Mapping[str, NativeTool] = field(default_factory=dict, hash=False) + tool_name_prefix: str = "" + display_name: str = "" + + +class Driver(ABC): + """Runs commands on one screen. ``config`` of ``start`` is the driver's own type, built by its factory.""" + + @property + @abstractmethod + def info(self) -> DriverInfo: ... + + @abstractmethod + async def start(self, config: Any = None) -> None: ... + + @abstractmethod + async def stop(self) -> None: ... + + @abstractmethod + async def health(self) -> Health: ... + + @abstractmethod + async def observe(self, command: Observe) -> Observation: ... + + @abstractmethod + async def execute(self, action: Action) -> ActionResult: ... + + +_HANDLES_MARK = "__contract_handles__" +_F = TypeVar("_F", bound=Callable[..., Any]) + + +def handles(command: type[Action], *, refs: Iterable[type] = ()) -> Callable[[_F], _F]: + """Mark a ``HandlerDriver`` method as the handler of ``command``; marks stack.""" + accepted = frozenset(refs) + unknown = accepted - set(REF_KINDS) + if unknown: + raise TypeError(f"not element ref kinds: {sorted(kind.__name__ for kind in unknown)}") + + def mark(method: _F) -> _F: + setattr(method, _HANDLES_MARK, getattr(method, _HANDLES_MARK, ()) + ((command, accepted),)) + return method + + return mark + + +def iter_refs(value: Any) -> Iterator[Any]: + """Yield every element ref inside a command, looking into nested dataclasses, tuples and lists.""" + if isinstance(value, REF_KINDS): + yield value + elif dataclasses.is_dataclass(value) and not isinstance(value, type): + for item in dataclasses.fields(value): + yield from iter_refs(getattr(value, item.name)) + elif isinstance(value, (tuple, list)): + for item in value: + yield from iter_refs(item) + + +def ensure_supported(driver_name: str, actions: Mapping[type[Action], frozenset[type]], action: Action) -> None: + """Raise ``Unsupported(dispatched=False)`` unless ``actions`` covers the command and every ref kind in it.""" + command = type(action) + accepted = actions.get(command) + if accepted is None: + raise Unsupported(f"{driver_name} does not handle {command.__name__}", dispatched=False) + for ref in iter_refs(action): + if type(ref) not in accepted: + raise Unsupported( + f"{driver_name} does not accept {type(ref).__name__} for {command.__name__}", dispatched=False + ) + + +class HandlerDriver(Driver): + """Dispatches ``execute`` by ``type(action)`` to ``@handles`` methods; ``info.actions`` comes from the marks.""" + + _handlers: ClassVar[Mapping[type[Action], tuple[str, frozenset[type]]]] = MappingProxyType({}) + + def __init_subclass__(cls, **kwargs: Any) -> None: + super().__init_subclass__(**kwargs) + handlers: dict[type[Action], tuple[str, frozenset[type]]] = {} + for klass in reversed(cls.__mro__): + for name, attribute in vars(klass).items(): + for command, refs in getattr(attribute, _HANDLES_MARK, ()): + handlers[command] = (name, refs) + cls._handlers = MappingProxyType(handlers) + + @classmethod + def handled_actions(cls) -> Mapping[type[Action], frozenset[type]]: + return MappingProxyType({command: refs for command, (_name, refs) in cls._handlers.items()}) + + @abstractmethod + def describe(self) -> DriverInfo: + """Everything but ``actions``, which ``info`` fills from the marks (a value set here is replaced).""" + + @property + def info(self) -> DriverInfo: + return dataclasses.replace(self.describe(), actions=self.handled_actions()) + + async def execute(self, action: Action) -> ActionResult: + ensure_supported(type(self).__name__, self.handled_actions(), action) + name, _refs = self._handlers[type(action)] + return await getattr(self, name)(action) diff --git a/openjiuwen/harness/tools/browser_move/contract/errors.py b/openjiuwen/harness/tools/browser_move/contract/errors.py new file mode 100644 index 0000000000..9dddeac0f3 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/errors.py @@ -0,0 +1,74 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Typed driver errors; ``dispatched`` (also when the outcome is unknown) means never retried automatically.""" + +from __future__ import annotations + +from typing import ClassVar + +from .observation import Observation + + +class DriverError(Exception): + """Base driver error. ``dispatched`` defaults to the class's ``default_dispatched`` (True unless stated).""" + + default_dispatched: ClassVar[bool] = True + + def __init__(self, message: str = "", *, dispatched: bool | None = None) -> None: + super().__init__(message) + self.dispatched: bool = type(self).default_dispatched if dispatched is None else dispatched + + +class StaleRef(DriverError): + """The ref belongs to an older generation. Raised only before anything is sent; may carry a fresh observation.""" + + default_dispatched: ClassVar[bool] = False + + def __init__( + self, message: str = "", *, dispatched: bool | None = None, observation: Observation | None = None + ) -> None: + super().__init__(message, dispatched=dispatched) + self.observation = observation + + +class ElementNotFound(DriverError): + pass + + +class AmbiguousElement(DriverError): + pass + + +class ActionIntercepted(DriverError): + """Another element would receive the input; ``by`` describes it.""" + + default_dispatched: ClassVar[bool] = False + + def __init__(self, message: str = "", *, by: str, dispatched: bool | None = None) -> None: + super().__init__(message, dispatched=dispatched) + self.by = by + + +class ActionTimeout(DriverError): + pass + + +class NavigationError(DriverError): + pass + + +class ConnectionLost(DriverError): + pass + + +class Unsupported(DriverError): + """The driver has no handler for the command, or the handler does not accept the ref kind.""" + + default_dispatched: ClassVar[bool] = False + + +class PolicyDenied(DriverError): + """Raised by the security rail, not by drivers, before anything is sent.""" + + default_dispatched: ClassVar[bool] = False diff --git a/openjiuwen/harness/tools/browser_move/contract/masking.py b/openjiuwen/harness/tools/browser_move/contract/masking.py new file mode 100644 index 0000000000..398aaebaf8 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/masking.py @@ -0,0 +1,33 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +from __future__ import annotations + +import dataclasses +from collections.abc import Mapping +from typing import Any, TypeVar, cast + +MASK = "***" + +_T = TypeVar("_T") + + +def is_sensitive(command: Any) -> bool: + return bool(getattr(command, "sensitive", False) and getattr(command, "__sensitive_fields__", ())) + + +def masked(value: _T) -> _T: + """``value`` with the ``__sensitive_fields__`` of every sensitive command, or part of one, set to ``MASK``.""" + if dataclasses.is_dataclass(value) and not isinstance(value, type): + hidden = getattr(value, "__sensitive_fields__", ()) if is_sensitive(value) else () + changes = { + item.name: MASK if item.name in hidden else masked(getattr(value, item.name)) + for item in dataclasses.fields(value) + if item.init + } + return dataclasses.replace(value, **changes) + if isinstance(value, (tuple, list)): + return cast(_T, type(value)(masked(item) for item in value)) + if isinstance(value, Mapping): + return cast(_T, {key: masked(item) for key, item in value.items()}) + return value diff --git a/openjiuwen/harness/tools/browser_move/contract/observation.py b/openjiuwen/harness/tools/browser_move/contract/observation.py new file mode 100644 index 0000000000..3b6fbf19f4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/observation.py @@ -0,0 +1,76 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""What a driver sees: one ``Observation`` per ``observe()`` call.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import Mapping +from dataclasses import dataclass, field +from typing import Any + +from .refs import ContextRef, DriverRef + + +@dataclass(frozen=True) +class SurfaceContext: + """Base of the per-surface context; the core reads only ``identity``, the surface's own code reads the rest.""" + + @property + def identity(self) -> str: + """Where the surface is, as the surface names it; a change between two non-empty values is a new generation.""" + return "" + + +@dataclass(frozen=True) +class Viewport: + """Size of the visible area, in the same pixels as ``Element.bounds``.""" + + width: int + height: int + + +@dataclass(frozen=True) +class Screenshot: + """A PNG image; ``PointRef`` coordinates are always in its pixels. ``scale`` is image pixels per viewport pixel.""" + + png: bytes + width: int + height: int + scale: float = 1.0 + + +@dataclass(frozen=True) +class Element: + """One element the model can point at. ``bounds`` is ``(x, y, width, height)`` in viewport pixels.""" + + ref: DriverRef + role: str + name: str + value: str | None = None + bounds: tuple[float, float, float, float] | None = None + enabled: bool = True + focused: bool = False + extra: Mapping[str, Any] = field(default_factory=dict, hash=False) + + +@dataclass(frozen=True, kw_only=True) +class Observation: + """A snapshot of the focused context; ``generation`` is stamped by the runtime, so drivers leave it empty.""" + + context: ContextRef + generation: str = "" + elements: tuple[Element, ...] = () + screenshot: Screenshot | None = None + viewport: Viewport | None = None + native_text: str | None = None + surface: SurfaceContext | None = None + + @property + def fingerprint(self) -> str: + """Backstop for change detection; ``ActionResult.context_changed`` is the primary signal.""" + payload = [self.context.id, [[e.ref.ref, e.role, e.name] for e in self.elements]] + encoded = json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8") + return hashlib.sha256(encoded).hexdigest()[:16] diff --git a/openjiuwen/harness/tools/browser_move/contract/refs.py b/openjiuwen/harness/tools/browser_move/contract/refs.py new file mode 100644 index 0000000000..5af37d7d73 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/refs.py @@ -0,0 +1,76 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Ways a command points at an element or a context; element refs carry their observation's ``generation``.""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import Protocol, Union + + +@dataclass(frozen=True) +class DriverRef: + """The model-facing element ref (``e57``) minted by the driver in an observation.""" + + ref: str + generation: str + + +@dataclass(frozen=True) +class PointRef: + """A point in the pixels of that generation's screenshot.""" + + x: int + y: int + generation: str + + +@dataclass(frozen=True) +class IndexRef: + """A driver-internal element index; never shown to the model.""" + + index: int + generation: str + + +@dataclass(frozen=True) +class Locator: + """A CSS selector and/or visible text that identifies one element.""" + + css: str | None + text: str | None + generation: str + + +@dataclass(frozen=True) +class ContextRef: + """A context the driver can focus: ``kind`` is surface-defined (for example a window or an activity).""" + + id: str + kind: str + title: str | None = None + + +Ref = Union[DriverRef, PointRef, IndexRef, Locator] +REF_KINDS: tuple[type, ...] = (DriverRef, PointRef, IndexRef, Locator) + + +class RefResolver(Protocol): + """What a tool's argument decoder asks the runtime about ids the model sends.""" + + @property + def contexts(self) -> tuple[ContextRef, ...]: ... + + @property + def current_context(self) -> ContextRef | None: ... + + @property + def screenshot_generation(self) -> str: + """The generation of the last screenshot, whose pixels a ``PointRef`` addresses; empty before the first.""" + + def resolve(self, model_id: str, generation: str) -> Ref: + """The ref behind a runtime-minted element id; raises ``ValueError`` for an id it does not know.""" + + def generation_of(self, ref: str) -> str: + """The generation a driver ref was last seen in; the current one for a ref never seen.""" diff --git a/openjiuwen/harness/tools/browser_move/contract/registry.py b/openjiuwen/harness/tools/browser_move/contract/registry.py new file mode 100644 index 0000000000..0079851c5e --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/registry.py @@ -0,0 +1,89 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Name -> factory registries; a factory is a callable or a lazy ``"pkg.module:attr"`` import string.""" + +from __future__ import annotations + +import importlib +from collections.abc import Callable +from typing import Any, Generic, TypeVar + +from .driver import Driver +from .surface import SurfaceBundle + +T = TypeVar("T") + + +class Registry(Generic[T]): + def __init__(self, kind: str) -> None: + self._kind = kind + self._entries: dict[str, Callable[..., T] | str] = {} + + def register(self, name: str, factory: Callable[..., T] | str) -> None: + if name in self._entries: + raise ValueError(f"{self._kind} {name!r} is already registered") + if isinstance(factory, str): + module, _, attribute = factory.partition(":") + if not module or not attribute: + raise ValueError(f"lazy {self._kind} factory must look like 'pkg.module:attr', got {factory!r}") + elif not callable(factory): + raise TypeError(f"{self._kind} factory for {name!r} must be callable or an import string") + self._entries[name] = factory + + def create(self, name: str, *args: Any, **kwargs: Any) -> T: + return self._factory(name)(*args, **kwargs) + + def names(self) -> tuple[str, ...]: + return tuple(self._entries) + + def _factory(self, name: str) -> Callable[..., T]: + try: + factory = self._entries[name] + except KeyError: + known = ", ".join(self._entries) or "none" + raise KeyError(f"unknown {self._kind} {name!r}; registered: {known}") from None + if isinstance(factory, str): + module, _, attribute = factory.partition(":") + try: + resolved: Any = importlib.import_module(module) + except ModuleNotFoundError as error: + if error.name is None or not f"{module}.".startswith(f"{error.name}."): + raise + raise ModuleNotFoundError( + f"{self._kind} {name!r} is registered but not built yet: module {module!r} does not exist", + name=module, + ) from error + for part in attribute.split("."): + resolved = getattr(resolved, part) + factory = self._entries[name] = resolved + return factory + + +DRIVERS: Registry[Driver] = Registry("driver") +SURFACES: Registry[SurfaceBundle] = Registry("surface") +PROVISIONERS: Registry[Any] = Registry("provisioner") +DRIVER_TOPOLOGIES: dict[str, str] = {} + + +def register_driver(name: str, factory: Callable[..., Driver] | str, *, default_topology: str = "direct") -> None: + """``factory(device, surface_settings) -> Driver``; the driver builds its own config.""" + DRIVERS.register(name, factory) + DRIVER_TOPOLOGIES[name] = default_topology + + +def default_topology(name: str) -> str: + try: + return DRIVER_TOPOLOGIES[name] + except KeyError: + known = ", ".join(DRIVER_TOPOLOGIES) or "none" + raise KeyError(f"unknown driver {name!r}; registered: {known}") from None + + +def register_surface(name: str, profile_factory: Callable[..., SurfaceBundle] | str) -> None: + """``profile_factory(environ, instance=None, settings=None)``; ``settings`` replaces ``environ``.""" + SURFACES.register(name, profile_factory) + + +def register_provisioner(name: str, factory: Callable[..., Any] | str) -> None: + PROVISIONERS.register(name, factory) diff --git a/openjiuwen/harness/tools/browser_move/contract/results.py b/openjiuwen/harness/tools/browser_move/contract/results.py new file mode 100644 index 0000000000..2b8828a328 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/results.py @@ -0,0 +1,27 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""What ``Driver.execute`` returns.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + +from .observation import Observation, SurfaceContext +from .refs import ContextRef + + +@dataclass(frozen=True, kw_only=True) +class ActionResult: + """The typed outcome of one command; drivers leave ``generation`` empty for the runtime to fill.""" + + ok: bool + value: Any = field(default=None, hash=False) + detail: str | None = None + context_changed: bool = False + generation: str = "" + observation: Observation | None = None + source_context: ContextRef | None = None + opened_contexts: tuple[ContextRef, ...] = () + surface: SurfaceContext | None = None diff --git a/openjiuwen/harness/tools/browser_move/contract/surface.py b/openjiuwen/harness/tools/browser_move/contract/surface.py new file mode 100644 index 0000000000..7bc4a893f4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/surface.py @@ -0,0 +1,190 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""What a surface hands to the core: its settings, its profile, its policy rules and its conformance scene.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable, Mapping, Sequence +from dataclasses import dataclass, field +from typing import Any, Protocol, Union + +from .actions import Action, Effect +from .errors import DriverError +from .observation import Observation, Screenshot, SurfaceContext +from .refs import RefResolver +from .results import ActionResult + + +class SurfaceSettings(Protocol): + """Settings a surface parses from its own environment variables; the core reads only ``provisioning``.""" + + @property + def provisioning(self) -> str: ... + + +@dataclass(frozen=True) +class ToolSpec: + """One model-facing tool. It may cover several commands (for example one tool with an operation argument).""" + + name: str + commands: tuple[type[Action], ...] + description: str = "" + prompt_section: str | None = None + schema: Mapping[str, Any] | None = field(default=None, hash=False) + build: Callable[[Mapping[str, Any], RefResolver], Action] | None = field(default=None, hash=False) + + +ContentBlock = Union[str, Screenshot] + + +class Renderer(Protocol): + """Turns what a driver returns into the content blocks the model reads (text and/or image).""" + + def observation(self, observation: Observation) -> tuple[ContentBlock, ...]: ... + + def result(self, action: Action, result: ActionResult) -> tuple[ContentBlock, ...]: ... + + def error( + self, action: Action, error: DriverError, observation: Observation | None + ) -> tuple[ContentBlock, ...]: ... + + +@dataclass(frozen=True) +class PolicyRule: + """Checked by the security rail before middleware; ``check`` returns a deny reason or ``None``.""" + + command_type: type[Action] + check: Callable[[Action, SurfaceContext | None], str | None] + + +SceneStep = Callable[[Sequence[Observation]], Action] +HandledActions = Mapping[type[Action], frozenset[type]] + + +@dataclass(frozen=True) +class SceneCase: + """A short script the conformance suite plays in a scene; the last step is the command under test.""" + + steps: tuple[SceneStep, ...] + check: Callable[[Observation, Observation, ActionResult], bool] | None = None + observe_between: bool = True + cleanup: tuple[SceneStep, ...] = () + + +def _not_provided(*_args: Any) -> str: + return "the scene does not provide this case" + + +@dataclass(frozen=True) +class Scene: + """A reproducible setting the conformance suite runs a driver in; a case builder returns a skip reason.""" + + surface: str + start: Callable[[], None] + stop: Callable[[], None] + enter: Callable[[], Sequence[Action]] = tuple + happy: Callable[[type[Action], type | None, HandledActions], SceneCase | str] = _not_provided + stale: Callable[[HandledActions], SceneCase | str] = _not_provided + intercepted: Callable[[HandledActions], SceneCase | str] = _not_provided + timeout: Callable[[HandledActions], SceneCase | str] = _not_provided + context_changes: Mapping[str, Callable[[HandledActions], SceneCase | str]] = field(default_factory=dict, hash=False) + sensitive: Mapping[str, Callable[[HandledActions, str], SceneCase | str]] = field(default_factory=dict, hash=False) + + +class ContextFactory(Protocol): + """Builds one tool or rail from the core's tool context (``runtime.tool_factory.ToolContext``).""" + + def __call__(self, context: Any, /) -> Any: ... + + +class CustomActions(Protocol): + def __call__(self, controller: Any, context: Any, /) -> None: ... + + +class Metadata(Protocol): + """``(metadata, error)`` of the current context; ``error`` is ``None`` on success.""" + + def __call__(self, context: Any, /) -> Awaitable[tuple[Mapping[str, Any], str | None]]: ... + + +class PolicyProbe(Protocol): + def __call__(self, context: Any, source: str, params: Mapping[str, Any], /) -> Awaitable[Mapping[str, Any]]: ... + + +class Activate(Protocol): + def __call__(self, context: Any, address: str, /) -> Awaitable[bool]: ... + + +class MainAgentFactory(Protocol): + """The surface's main agent over a started runtime; without it the runtime builds one from the profile's parts.""" + + def __call__( + self, + runtime: Any, + model: Any, + *, + language: str, + max_iterations: int, + workspace: Any, + capabilities: tuple[str, ...], + ) -> Any: ... + + +class WorkerPrompt(Protocol): + def __call__( + self, + screenshot_subdir: str, + artifacts_subdir: str, + *, + tool_names: tuple[str, ...] | None = None, + driver_name: str, + ) -> str: ... + + +@dataclass(frozen=True) +class Capability: + """One capability catalog entry: a grantable name and the tools it selects.""" + + name: str + description: str + tool_names: tuple[str, ...] + policy_only: bool = False + + +@dataclass(frozen=True) +class SurfaceProfile: + """The contract between the core and one surface; every hook is optional.""" + + name: str + commands: tuple[type[Action], ...] + renderer: Renderer + tool_specs: tuple[ToolSpec, ...] = () + capability_catalog: Mapping[str, frozenset[tuple[type[Action], type | None]]] = field( + default_factory=dict, hash=False + ) + capability_tools: Mapping[str, tuple[str, ...]] = field(default_factory=dict, hash=False) + extra_tools: tuple[ContextFactory, ...] = () + rails: tuple[ContextFactory, ...] = () + policy_rules: tuple[PolicyRule, ...] = () + timeouts: Mapping[Effect, float] = field(default_factory=dict, hash=False) + messages: Mapping[str, str] = field(default_factory=dict, hash=False) + settings: Any = field(default=None, hash=False) + scene: Scene | None = None + capabilities: tuple[Capability, ...] = () + custom_actions: CustomActions | None = None + page_metadata: Metadata | None = None + policy_probe: PolicyProbe | None = None + activate: Activate | None = None + semantic_tracker: Callable[[], Any] | None = None + worker_prompt: WorkerPrompt | None = None + page_state: Callable[[], Any] | None = None + main_agent: MainAgentFactory | None = None + + +@dataclass(frozen=True) +class SurfaceBundle: + """What a surface factory returns: the surface settings and, once the surface has one, its profile.""" + + settings: SurfaceSettings + profile: SurfaceProfile | None = None diff --git a/openjiuwen/harness/tools/browser_move/contract/testing/__init__.py b/openjiuwen/harness/tools/browser_move/contract/testing/__init__.py new file mode 100644 index 0000000000..840d0c53e6 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/testing/__init__.py @@ -0,0 +1,8 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Test support for drivers: ``FakeDriver``.""" + +from .fake import DEFAULT_ACTIONS, Call, FakeDriver + +__all__ = ["DEFAULT_ACTIONS", "Call", "FakeDriver"] diff --git a/openjiuwen/harness/tools/browser_move/contract/testing/conformance.py b/openjiuwen/harness/tools/browser_move/contract/testing/conformance.py new file mode 100644 index 0000000000..cfe636677d --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/testing/conformance.py @@ -0,0 +1,449 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Driver conformance suite (pytest plugin). PYTEST_DONT_REWRITE: test modules import it before pytest does.""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import logging +from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence +from dataclasses import dataclass +from typing import Any, ClassVar, TypeVar + +import pytest + +from ..actions import Action, Effect, Observe, TypeText, Wait +from ..driver import Driver, ensure_supported, iter_refs +from ..errors import ActionIntercepted, ActionTimeout, DriverError, StaleRef, Unsupported +from ..observation import Observation +from ..surface import HandledActions, Scene, SceneCase + +LIVE_MARKER = "live: runs the conformance suite against a real driver and a live scene" +SECRET_TEXT = "conformance-secret-5d1c" +TYPE_TEXT_CASE = "type-text" + +_E = TypeVar("_E", bound=DriverError) + + +class TraceSink: + """Collects what a driver, or the tracing middleware around it, writes to the trace.""" + + def __init__(self) -> None: + self.records: list[Any] = [] + + def __call__(self, record: Any) -> None: + self.records.append(record) + + @property + def text(self) -> str: + return "\n".join(repr(record) for record in self.records) + + +DriverFactory = Callable[[TraceSink], Driver] + + +@dataclass(frozen=True) +class UnhandledCommand(Action): + """A command no driver handles; the suite sends it to check the ``Unsupported`` path.""" + + effect: ClassVar[Effect] = Effect.READ + + +@dataclass(frozen=True) +class _Prepared: + action: Action + before: Observation + seen: tuple[Observation, ...] + + +def _name(kind: type | None) -> str: + return "no-ref" if kind is None else kind.__name__ + + +def _pairs(actions: HandledActions) -> list[tuple[type[Action], type | None]]: + pairs: list[tuple[type[Action], type | None]] = [] + for command in sorted(actions, key=lambda c: (c.__module__, c.__qualname__)): + kinds = sorted(actions[command], key=_name) + pairs.extend((command, kind) for kind in (kinds or [None])) + return pairs + + +def _brief(observation: Observation) -> str: + return f"context={observation.context!r} surface={observation.surface!r}" + + +def _skip_if_reason(scene: Scene, case: SceneCase | str) -> SceneCase: + if isinstance(case, str): + pytest.skip(f"scene {scene.surface!r}: {case}") + return case + + +def _skip_unless_handled(driver: Driver, action: Action) -> None: + try: + ensure_supported(driver.info.name, driver.info.actions, action) + except Unsupported as error: + pytest.skip(f"the case needs a command the driver does not handle: {error}") + + +@contextlib.asynccontextmanager +async def _session(factory: DriverFactory, scene: Scene, sink: TraceSink | None = None) -> AsyncIterator[Driver]: + driver = factory(sink if sink is not None else TraceSink()) + await driver.start() + try: + for action in scene.enter(): + await driver.execute(action) + yield driver + finally: + await driver.stop() + + +async def _prepare(driver: Driver, case: SceneCase, *, skip_unhandled_last: bool = True) -> _Prepared: + if not case.steps: + raise AssertionError("a scene case needs at least one step") + seen = [await driver.observe(Observe())] + for step in case.steps[:-1]: + action = step(tuple(seen)) + _skip_unless_handled(driver, action) + await driver.execute(action) + if case.observe_between: + seen.append(await driver.observe(Observe())) + action = case.steps[-1](tuple(seen)) + if skip_unhandled_last: + _skip_unless_handled(driver, action) + return _Prepared(action, seen[-1], tuple(seen)) + + +async def _cleanup(driver: Driver, case: SceneCase, seen: tuple[Observation, ...]) -> None: + if not case.cleanup: + return + with contextlib.suppress(DriverError, LookupError): + if case.observe_between: + seen = seen + (await driver.observe(Observe()),) + for step in case.cleanup: + await driver.execute(step(seen)) + + +async def _raised(driver: Driver, action: Action, expected: type[_E]) -> _E: + try: + result = await driver.execute(action) + except expected as error: + return error + except DriverError as error: + raise AssertionError( + f"expected {expected.__name__} for {action!r}, got {type(error).__name__}" + f"(dispatched={error.dispatched}): {error}" + ) from error + raise AssertionError(f"expected {expected.__name__} for {action!r}, got {result!r}") + + +async def _expect_error(factory: DriverFactory, scene: Scene, case_of: Callable, expected: type[_E]) -> _E: + async with _session(factory, scene) as driver: + case = _skip_if_reason(scene, case_of(driver.info.actions)) + prepared = await _prepare(driver, case) + try: + return await _raised(driver, prepared.action, expected) + finally: + await _cleanup(driver, case, prepared.seen) + + +async def check_surface(factory: DriverFactory, scene: Scene) -> None: + """The driver lists the scene's surface in ``info.surfaces``.""" + async with _session(factory, scene) as driver: + info = driver.info + assert scene.surface in info.surfaces, ( + f"{info.name} lists surfaces {sorted(info.surfaces)}, not the scene's {scene.surface!r}" + ) + + +async def check_declared_actions(factory: DriverFactory, scene: Scene, declared: HandledActions) -> None: + async with _session(factory, scene) as driver: + actual = dict(driver.info.actions) + assert actual == dict(declared), f"info.actions {actual!r} differs from driver_actions {dict(declared)!r}" + + +async def check_happy_path(factory: DriverFactory, scene: Scene, command: type[Action], ref_kind: type | None) -> None: + """The scene's happy path of ``command`` with ``ref_kind`` succeeds and takes effect.""" + async with _session(factory, scene) as driver: + case = _skip_if_reason(scene, scene.happy(command, ref_kind, driver.info.actions)) + prepared = await _prepare(driver, case, skip_unhandled_last=False) + action = prepared.action + assert type(action) is command, f"the scene built {action!r} for {command.__name__}" + kinds = {type(ref) for ref in iter_refs(action)} + assert kinds == ({ref_kind} if ref_kind else set()), ( + f"the scene built {action!r} for ref kind {_name(ref_kind)}" + ) + try: + result = await driver.execute(action) + assert result.ok, f"{action!r} returned ok=False: {result.detail}" + if case.check is not None: + after = await driver.observe(Observe()) + assert case.check(prepared.before, after, result), ( + f"{action!r} did not take effect: before {_brief(prepared.before)}, after {_brief(after)}" + ) + finally: + await _cleanup(driver, case, prepared.seen) + + +async def check_stale_ref(factory: DriverFactory, scene: Scene) -> None: + """A stale ref raises ``StaleRef`` before anything is sent.""" + error = await _expect_error(factory, scene, scene.stale, StaleRef) + assert error.dispatched is False, f"StaleRef must be raised before dispatch, got dispatched={error.dispatched}" + + +async def check_context_change(factory: DriverFactory, scene: Scene, name: str) -> None: + """The scene's context change ``name`` is reported as ``context_changed``; an observation then passes its check.""" + async with _session(factory, scene) as driver: + case = _skip_if_reason(scene, scene.context_changes[name](driver.info.actions)) + prepared = await _prepare(driver, case) + try: + result = await driver.execute(prepared.action) + after = None if case.check is None else await driver.observe(Observe()) + finally: + await _cleanup(driver, case, prepared.seen) + assert result.context_changed, f"{prepared.action!r} ({name}) did not report context_changed: {result!r}" + if after is not None: + assert case.check(prepared.before, after, result), ( + f"{prepared.action!r} ({name}) fails the scene's check: result {result!r}, after {_brief(after)}" + ) + + +async def check_intercepted(factory: DriverFactory, scene: Scene) -> None: + """A click another element receives raises ``ActionIntercepted`` naming that element, before dispatch.""" + error = await _expect_error(factory, scene, scene.intercepted, ActionIntercepted) + assert isinstance(error.by, str) and error.by.strip(), ( + f"ActionIntercepted.by must describe the element: {error.by!r}" + ) + assert error.dispatched is False, ( + f"ActionIntercepted means the input never reached the target, got dispatched={error.dispatched}" + ) + + +async def check_timeout(factory: DriverFactory, scene: Scene) -> None: + """A ``Wait`` for text that never appears raises ``ActionTimeout(dispatched=True)`` and is not retried.""" + async with _session(factory, scene) as driver: + case = _skip_if_reason(scene, scene.timeout(driver.info.actions)) + prepared = await _prepare(driver, case) + action = prepared.action + assert isinstance(action, Wait) and action.until_text is not None and action.seconds is not None, ( + f"the timeout case must end in Wait(until_text=..., seconds=...), the scene built {action!r}" + ) + try: + error = await _raised(driver, action, ActionTimeout) + finally: + await _cleanup(driver, case, prepared.seen) + assert error.dispatched is True, f"ActionTimeout must count as dispatched, got dispatched={error.dispatched}" + + +async def check_unhandled(factory: DriverFactory, scene: Scene) -> None: + """A command the driver does not handle is absent from ``info.actions`` and raises ``Unsupported`` unsent.""" + async with _session(factory, scene) as driver: + assert UnhandledCommand not in driver.info.actions, "info.actions lists a command no driver handles" + error = await _raised(driver, UnhandledCommand(), Unsupported) + assert error.dispatched is False, f"Unsupported must be raised before dispatch, got dispatched={error.dispatched}" + + +async def check_fingerprint(factory: DriverFactory, scene: Scene) -> None: + """Two observations of an unchanged scene have the same fingerprint.""" + async with _session(factory, scene) as driver: + first = await driver.observe(Observe()) + second = await driver.observe(Observe()) + assert first.fingerprint == second.fingerprint, ( + f"fingerprint changed on an unchanged scene: {first.elements!r} vs {second.elements!r}" + ) + + +def _type_text_case(scene: Scene, actions: HandledActions) -> SceneCase: + accepted = actions.get(TypeText) + if accepted is None: + pytest.skip("the driver does not handle TypeText") + reasons = [] + for kind in sorted(accepted, key=_name) or [None]: + case = scene.happy(TypeText, kind, actions) + if isinstance(case, SceneCase): + break + reasons.append(case) + else: + pytest.skip(f"scene {scene.surface!r}: {'; '.join(reasons)}") + last = case.steps[-1] + + def secret(seen: Sequence[Observation]) -> Action: + return dataclasses.replace(last(seen), text=SECRET_TEXT, sensitive=True) + + return dataclasses.replace(case, steps=case.steps[:-1] + (secret,)) + + +async def _send_secret(factory: DriverFactory, scene: Scene, sink: TraceSink, name: str) -> None: + async with _session(factory, scene, sink) as driver: + if name == TYPE_TEXT_CASE: + case = _type_text_case(scene, driver.info.actions) + else: + case = _skip_if_reason(scene, scene.sensitive[name](driver.info.actions, SECRET_TEXT)) + prepared = await _prepare(driver, case) + assert SECRET_TEXT in repr(prepared.action), f"the case {name!r} does not send the secret: {prepared.action!r}" + try: + await driver.execute(prepared.action) + finally: + await _cleanup(driver, case, prepared.seen) + + +async def check_sensitive_not_traced(factory: DriverFactory, scene: Scene, name: str = TYPE_TEXT_CASE) -> None: + """A sensitive value (``TypeText(sensitive=True)``, or the scene's case ``name``) leaves no text in the trace.""" + sink = TraceSink() + await _send_secret(factory, scene, sink, name) + if not sink.records: + pytest.skip("the driver factory wrote nothing to the trace sink (no tracing middleware around the driver)") + assert SECRET_TEXT not in sink.text, "sensitive text reached the trace" + + +class _ListHandler(logging.Handler): + def __init__(self) -> None: + super().__init__(logging.DEBUG) + self.messages: list[str] = [] + + def emit(self, record: logging.LogRecord) -> None: + try: + self.messages.append(record.getMessage()) + except (TypeError, ValueError): + self.messages.append(f"{record.msg!r} {record.args!r}") + + +@contextlib.contextmanager +def _captured_logs() -> Iterator[_ListHandler]: + root = logging.getLogger() + loggers = [root] + [lg for lg in logging.Logger.manager.loggerDict.values() if isinstance(lg, logging.Logger)] + handler = _ListHandler() + targets = [root] + [lg for lg in loggers[1:] if not lg.propagate] + levels = [(logger, logger.level) for logger in loggers] + for logger in loggers: + if logger is root or logger.level > logging.DEBUG: + logger.setLevel(logging.DEBUG) + for logger in targets: + logger.addHandler(handler) + try: + yield handler + finally: + for logger in targets: + logger.removeHandler(handler) + for logger, level in levels: + logger.setLevel(level) + + +async def check_sensitive_not_logged(factory: DriverFactory, scene: Scene, name: str = TYPE_TEXT_CASE) -> None: + """A sensitive value (as in ``check_sensitive_not_traced``) leaves no text in the logs of this process.""" + with _captured_logs() as handler: + await _send_secret(factory, scene, TraceSink(), name) + leaked = [message for message in handler.messages if SECRET_TEXT in message] + assert not leaked, f"sensitive text reached the logs: {leaked!r}" + + +def pytest_configure(config: pytest.Config) -> None: + config.addinivalue_line("markers", LIVE_MARKER) + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "conformance_pair" in metafunc.fixturenames: + actions = getattr(metafunc.cls, "driver_actions", None) + if actions is None: + raise pytest.UsageError(f"{metafunc.cls!r} must set driver_actions to parametrize the conformance suite") + pairs = _pairs(actions) + metafunc.parametrize("conformance_pair", pairs, ids=[f"{c.__name__}-{_name(k)}" for c, k in pairs]) + for fixture, names_of in ( + ("conformance_context_change", lambda scene: list(scene.context_changes)), + ("conformance_sensitive_case", lambda scene: [TYPE_TEXT_CASE, *scene.sensitive]), + ): + if fixture not in metafunc.fixturenames: + continue + scene = getattr(metafunc.cls, "scene", None) + if scene is None: + raise pytest.UsageError(f"{metafunc.cls!r} must set scene to parametrize the conformance suite") + names = names_of(scene) + metafunc.parametrize(fixture, names, ids=names) + + +@pytest.fixture(scope="class") +def conformance_scene(request: pytest.FixtureRequest) -> Iterator[Scene]: + """The class's ``scene``, started once for the class and stopped after it.""" + scene = getattr(request.cls, "scene", None) + if scene is None: + raise pytest.UsageError(f"{request.cls!r} must set scene") + scene.start() + try: + yield scene + finally: + scene.stop() + + +class DriverConformance: + """Subclass as ``Test...`` and set ``scene``, ``driver_actions`` and a ``conformance_driver_factory`` fixture.""" + + scene: ClassVar[Scene] + driver_actions: ClassVar[Mapping[type[Action], frozenset[type]]] + + def test_lists_the_scene_surface(self, conformance_scene: Scene, conformance_driver_factory: DriverFactory): + asyncio.run(check_surface(conformance_driver_factory, conformance_scene)) + + def test_declared_actions_match_the_driver(self, conformance_scene, conformance_driver_factory): + asyncio.run(check_declared_actions(conformance_driver_factory, conformance_scene, self.driver_actions)) + + def test_happy_path(self, conformance_scene, conformance_driver_factory, conformance_pair): + command, ref_kind = conformance_pair + asyncio.run(check_happy_path(conformance_driver_factory, conformance_scene, command, ref_kind)) + + def test_stale_ref_is_raised_before_dispatch(self, conformance_scene, conformance_driver_factory): + asyncio.run(check_stale_ref(conformance_driver_factory, conformance_scene)) + + def test_context_change_is_reported( + self, conformance_scene, conformance_driver_factory, conformance_context_change + ): + asyncio.run(check_context_change(conformance_driver_factory, conformance_scene, conformance_context_change)) + + def test_intercepted_click_names_the_receiver(self, conformance_scene, conformance_driver_factory): + asyncio.run(check_intercepted(conformance_driver_factory, conformance_scene)) + + def test_timeout_counts_as_dispatched(self, conformance_scene, conformance_driver_factory): + asyncio.run(check_timeout(conformance_driver_factory, conformance_scene)) + + def test_unhandled_command_is_unsupported(self, conformance_scene, conformance_driver_factory): + asyncio.run(check_unhandled(conformance_driver_factory, conformance_scene)) + + def test_observing_twice_gives_the_same_fingerprint(self, conformance_scene, conformance_driver_factory): + asyncio.run(check_fingerprint(conformance_driver_factory, conformance_scene)) + + def test_sensitive_text_stays_out_of_the_trace( + self, conformance_scene, conformance_driver_factory, conformance_sensitive_case + ): + asyncio.run( + check_sensitive_not_traced(conformance_driver_factory, conformance_scene, conformance_sensitive_case) + ) + + def test_sensitive_text_stays_out_of_the_logs( + self, conformance_scene, conformance_driver_factory, conformance_sensitive_case + ): + asyncio.run( + check_sensitive_not_logged(conformance_driver_factory, conformance_scene, conformance_sensitive_case) + ) + + +__all__ = [ + "LIVE_MARKER", + "SECRET_TEXT", + "TYPE_TEXT_CASE", + "DriverConformance", + "DriverFactory", + "TraceSink", + "UnhandledCommand", + "check_context_change", + "check_declared_actions", + "check_fingerprint", + "check_happy_path", + "check_intercepted", + "check_sensitive_not_logged", + "check_sensitive_not_traced", + "check_stale_ref", + "check_surface", + "check_timeout", + "check_unhandled", +] diff --git a/openjiuwen/harness/tools/browser_move/contract/testing/fake.py b/openjiuwen/harness/tools/browser_move/contract/testing/fake.py new file mode 100644 index 0000000000..cf260a9d2e --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/contract/testing/fake.py @@ -0,0 +1,135 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""A scripted ``Driver`` for tests: records every call and answers from queues, in order.""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Iterable +from dataclasses import dataclass +from typing import Any + +from ..actions import ( + Action, + Click, + CloseContext, + Drag, + HandleDialog, + Hover, + Observe, + PressKey, + RunScript, + Scroll, + SwitchContext, + TypeText, + Wait, +) +from ..driver import Driver, DriverInfo, Health, ensure_supported +from ..errors import ConnectionLost, DriverError +from ..observation import Observation +from ..refs import REF_KINDS +from ..results import ActionResult + +DEFAULT_ACTIONS: dict[type[Action], frozenset[type]] = { + command: frozenset(REF_KINDS) + for command in ( + Click, + TypeText, + PressKey, + Scroll, + Hover, + Drag, + Wait, + SwitchContext, + CloseContext, + HandleDialog, + RunScript, + ) +} + + +@dataclass(frozen=True) +class Call: + """One recorded call: ``kind`` is ``start``, ``stop``, ``observe`` or ``execute``.""" + + kind: str + payload: Any = None + + +class FakeDriver(Driver): + """Records every call and answers ``observe``/``execute`` from scripted queues, in order.""" + + def __init__( + self, + observations: Iterable[Observation] = (), + results: Iterable[ActionResult] = (), + *, + info: DriverInfo | None = None, + ) -> None: + self._info = info or DriverInfo(name="fake", surfaces=frozenset({"fake"}), actions=DEFAULT_ACTIONS) + self._observations: deque[Observation] = deque(observations) + self._results: deque[ActionResult] = deque(results) + self._errors: deque[BaseException] = deque() + self._lose_on: int | None = None + self._lose_dispatched = True + self._calls = 0 + self._connection_lost = False + self.calls: list[Call] = [] + self.started = False + + @property + def info(self) -> DriverInfo: + return self._info + + def script(self, *, observations: Iterable[Observation] = (), results: Iterable[ActionResult] = ()) -> None: + """Append answers to the end of the ``observe`` and ``execute`` queues.""" + self._observations.extend(observations) + self._results.extend(results) + + def fail_next(self, error: BaseException | None = None, *, dispatched: bool = False) -> None: + """Raise ``error`` (default: ``DriverError(dispatched=...)``) on the next observe/execute call.""" + self._errors.append(error if error is not None else DriverError("scripted", dispatched=dispatched)) + + def lose_connection_on(self, call: int, *, dispatched: bool = True) -> None: + """``call`` is 1-based; the connection stays lost until ``start()``.""" + self._lose_on = call + self._lose_dispatched = dispatched + + def kinds(self) -> list[str]: + return [call.kind for call in self.calls] + + async def start(self, config: Any = None) -> None: + self.calls.append(Call("start", config)) + self.started = True + self._connection_lost = False + + async def stop(self) -> None: + self.calls.append(Call("stop")) + self.started = False + + async def health(self) -> Health: + if self._connection_lost: + return Health(ok=False, detail="connection lost") + return Health(ok=self.started, detail=None if self.started else "not started") + + async def observe(self, command: Observe) -> Observation: + self.calls.append(Call("observe", command)) + return self._answer("observe", command, self._observations) + + async def execute(self, action: Action) -> ActionResult: + self.calls.append(Call("execute", action)) + ensure_supported(self._info.name, self._info.actions, action) + return self._answer("execute", action, self._results) + + def _answer(self, kind: str, payload: Any, queue: deque[Any]) -> Any: + self._calls += 1 + if self._calls == self._lose_on: + self._connection_lost = True + if self._connection_lost: + raise ConnectionLost("connection lost", dispatched=self._lose_dispatched) + if self._errors: + raise self._errors.popleft() + if not queue: + raise AssertionError(f"FakeDriver has no scripted answer left for {kind}({payload!r})") + return queue.popleft() diff --git a/openjiuwen/harness/tools/browser_move/controllers/__init__.py b/openjiuwen/harness/tools/browser_move/controllers/__init__.py deleted file mode 100644 index 5ac62598a5..0000000000 --- a/openjiuwen/harness/tools/browser_move/controllers/__init__.py +++ /dev/null @@ -1,47 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Controller package exports.""" - -import sys - -from .action import ( - ActionController, - bind_code_executor, - bind_runtime, - bind_runtime_runner, - browser_worker_action_context, - clear_code_executor, - clear_runtime_runner, - describe_actions, - get_default_controller, - list_actions, - register_action, - register_action_spec, - register_builtin_actions, - register_example_actions, - run_action, - validate_batch_steps, -) -from .base import BaseController - -__all__ = [ - "BaseController", - "ActionController", - "get_default_controller", - "bind_runtime", - "bind_runtime_runner", - "clear_runtime_runner", - "bind_code_executor", - "clear_code_executor", - "browser_worker_action_context", - "register_action", - "register_action_spec", - "register_builtin_actions", - "register_example_actions", - "list_actions", - "describe_actions", - "run_action", - "validate_batch_steps", -] - -sys.modules.setdefault("controllers", sys.modules[__name__]) diff --git a/openjiuwen/harness/tools/browser_move/controllers/action.py b/openjiuwen/harness/tools/browser_move/controllers/action.py deleted file mode 100644 index 9fc0633111..0000000000 --- a/openjiuwen/harness/tools/browser_move/controllers/action.py +++ /dev/null @@ -1,2539 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Runtime-native custom action controller for Playwright runtime. -It exposes a lightweight action registry consumed by the MCP wrapper. -""" - -from __future__ import annotations - -import asyncio -import contextlib -import contextvars -import copy -import inspect -import json -import sys -import time -from pathlib import Path -from typing import Any, Awaitable, Callable, Mapping, Protocol - -from openjiuwen.core.common.logging import logger -from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_logging import ( - browser_agent_log_info, - browser_agent_log_warning, -) -from .base import BaseController - -try: - # Normal package import (openjiuwen.harness.tools.browser_move.controllers.action) - from ..utils.env import resolve_upload_root - from ..utils.parsing import extract_json_object -except ImportError: # pragma: no cover - # When imported as top-level `controllers.action` (playwright_runtime/__init__.py - # appends browser_move/ to sys.path), `..utils` would go beyond the top-level - # package. Fall back to sibling top-level `utils.*` imports. - from utils.env import resolve_upload_root - from utils.parsing import extract_json_object - -ActionResult = dict[str, Any] -ActionHandler = Callable[..., Awaitable[Any] | Any] -CodeExecutor = Callable[[str], Awaitable[Any]] - -_BATCH_SUPPORTED_OPS = frozenset( - { - "click", - "fill", - "type", - "autocomplete", - "select_visible_text", - "press", - "select_option", - "set_checked", - "wait_for_selector", - "wait_for_text", - "wait_for_load_state", - "wait_for_url", - "wait_for_first_card_title", - "wait_for_sort_state", - "wait_for_result_count", - "wait_for_dom_text_change", - "wait_for_stable", - "wait_for_tab", - "sleep", - "extract_text", - "extract_value", - "screenshot", - } -) -_BATCH_LOCATOR_KEYS = ( - "target_id", - "ref", - "selector", - "role", - "label", - "placeholder", - "testid", -) -_BATCH_CONDITION_OPS = frozenset( - { - "wait_for_selector", - "wait_for_text", - "wait_for_load_state", - "wait_for_url", - "wait_for_first_card_title", - "wait_for_sort_state", - "wait_for_result_count", - "wait_for_dom_text_change", - "wait_for_stable", - "wait_for_tab", - } -) -_BATCH_PRIMARY_TARGET_OPS = frozenset( - { - "click", - "fill", - "type", - "autocomplete", - "select_option", - "set_checked", - "extract_text", - "extract_value", - } -) -_BATCH_OPTION_TARGET_OPS = frozenset({"autocomplete", "select_visible_text"}) -_BATCH_SELECTOR_CONDITION_OPS = frozenset( - { - "wait_for_selector", - "wait_for_first_card_title", - "wait_for_sort_state", - "wait_for_result_count", - "wait_for_dom_text_change", - "wait_for_stable", - } -) - - -def _locator_strategies(step: Mapping[str, Any], *, include_text: bool = True) -> list[str]: - strategies: list[str] = [] - for key in _BATCH_LOCATOR_KEYS: - if step.get(key) not in (None, ""): - strategies.append(key) - if include_text and step.get("text") not in (None, ""): - strategies.append("text") - return strategies - - -def _option_locator_strategies(step: Mapping[str, Any]) -> list[str]: - strategies: list[str] = [] - alias_groups = ( - ("option_target_id", "choose_target_id"), - ("option_ref", "choose_ref"), - ("option_selector", "choose_selector"), - ("option_role", "choose_role"), - ("option_text", "choose_text", "text_to_choose"), - ) - for aliases in alias_groups: - populated = [key for key in aliases if step.get(key) not in (None, "")] - if len(populated) > 1: - strategies.extend(populated) - elif populated: - strategies.append(populated[0]) - return strategies - - -def _validate_batch_locator_contract( - index: int, - op: str, - step: Mapping[str, Any], -) -> list[str]: - errors: list[str] = [] - locator_strategies = _locator_strategies( - step, - include_text=op - not in { - "wait_for_text", - "wait_for_sort_state", - "wait_for_first_card_title", - }, - ) - option_strategies = _option_locator_strategies(step) - if step.get("name") not in (None, "") and step.get("role") in (None, ""): - errors.append(f"steps[{index}].name requires role") - if op in _BATCH_PRIMARY_TARGET_OPS and not locator_strategies: - errors.append(f"steps[{index}] op={op} requires a target") - elif op in _BATCH_PRIMARY_TARGET_OPS and len(locator_strategies) > 1: - errors.append( - f"steps[{index}] op={op} requires exactly one locator strategy; received {', '.join(locator_strategies)}" - ) - if op == "press" and len(locator_strategies) > 1: - errors.append( - f"steps[{index}] op=press accepts at most one locator strategy; received {', '.join(locator_strategies)}" - ) - if op in _BATCH_SELECTOR_CONDITION_OPS: - invalid_locators = [key for key in locator_strategies if key != "selector"] - if invalid_locators: - errors.append(f"steps[{index}] op={op} accepts only selector; received {', '.join(invalid_locators)}") - elif op not in _BATCH_PRIMARY_TARGET_OPS and op != "press" and locator_strategies: - errors.append( - f"steps[{index}] op={op} does not accept a primary locator; received {', '.join(locator_strategies)}" - ) - if op in _BATCH_OPTION_TARGET_OPS and not option_strategies: - errors.append(f"steps[{index}] {op} requires an option target") - elif op in _BATCH_OPTION_TARGET_OPS and len(option_strategies) > 1: - errors.append( - f"steps[{index}] op={op} requires exactly one option locator strategy; " - f"received {', '.join(option_strategies)}" - ) - option_name = step.get("option_name") or step.get("choose_name") - option_role = step.get("option_role") or step.get("choose_role") - if option_name not in (None, "") and option_role in (None, ""): - errors.append(f"steps[{index}] option_name requires option_role") - return errors - - -def _validate_batch_op_requirements( - index: int, - op: str, - step: Mapping[str, Any], -) -> list[str]: - errors: list[str] = [] - if op in {"fill", "type", "autocomplete"} and "value" not in step: - suffix = "; use value, not text" if "text" in step else "" - errors.append(f"steps[{index}] op={op} requires value{suffix}") - option_value_keys = ( - "value", - "values", - "option_value", - "option_text", - "option_label", - "label_value", - "choose_text", - "index", - ) - has_option_value = False - for key in option_value_keys: - if step.get(key) is not None: - has_option_value = True - break - if op == "select_option" and not has_option_value: - errors.append(f"steps[{index}] select_option requires an option value") - if op == "press" and not str(step.get("key") or "").strip(): - errors.append(f"steps[{index}] press requires key") - if op == "wait_for_selector" and not step.get("selector"): - errors.append(f"steps[{index}] wait_for_selector requires selector") - if op == "wait_for_text" and not step.get("text"): - errors.append(f"steps[{index}] wait_for_text requires text") - if op == "wait_for_url" and not any( - step.get(key) for key in ("url", "expected_url", "url_contains", "url_pattern") - ): - errors.append(f"steps[{index}] wait_for_url requires a URL condition") - if op == "wait_for_first_card_title" and not step.get("selector"): - errors.append(f"steps[{index}] wait_for_first_card_title requires selector") - if op == "wait_for_sort_state" and not ( - step.get("selector") and any(step.get(key) is not None for key in ("expected_value", "value", "text")) - ): - errors.append(f"steps[{index}] wait_for_sort_state requires selector and expected value") - if op == "wait_for_result_count" and not ( - step.get("selector") and any(step.get(key) is not None for key in ("count", "min_count", "max_count")) - ): - errors.append(f"steps[{index}] wait_for_result_count requires selector and count") - if op == "wait_for_dom_text_change" and not step.get("selector"): - errors.append(f"steps[{index}] wait_for_dom_text_change requires selector") - if op == "sleep": - try: - sleep_ms = int(step.get("ms", step.get("time_ms", 0))) - except (TypeError, ValueError): - sleep_ms = -1 - if sleep_ms < 0: - errors.append(f"steps[{index}] sleep requires non-negative ms") - return errors - - -def validate_batch_steps(steps: Any) -> list[str]: - """Validate the strict per-operation Batch contract before browser execution.""" - if not isinstance(steps, list) or not steps: - return ["steps is required and must be a non-empty list"] - if len(steps) > 25: - return ["browser_batch_interact accepts at most 25 steps"] - - errors: list[str] = [] - for index, step in enumerate(steps): - if not isinstance(step, Mapping): - errors.append(f"steps[{index}] must be an object") - continue - op = str(step.get("op") or "").strip().lower() - if op not in _BATCH_SUPPORTED_OPS: - errors.append(f"steps[{index}].op is unsupported: {op or '<missing>'}") - continue - errors.extend(_validate_batch_op_requirements(index, op, step)) - errors.extend(_validate_batch_locator_contract(index, op, step)) - return errors - - -def _unwrap_browser_code_result(raw: Any) -> Any: - """Extract text/data payloads from common browser_run_code/MCP result shapes.""" - if isinstance(raw, dict): - if raw.get("__browser_compact_rpc__") is True: - return _unwrap_browser_code_result(raw.get("payload")) - - content = raw.get("content") - if isinstance(content, list): - texts: list[str] = [] - for item in content: - if isinstance(item, dict) and item.get("type") == "text": - texts.append(str(item.get("text") or "")) - if texts: - return "\n".join(texts) - - for key in ("result", "text", "data", "final"): - if key in raw: - return raw.get(key) - - return raw - - -def _browser_rpc_metrics(raw: Any) -> dict[str, Any]: - if not isinstance(raw, dict) or raw.get("__browser_compact_rpc__") is not True: - return {} - metrics = raw.get("rpc_metrics") - return dict(metrics) if isinstance(metrics, Mapping) else {} - - -def _compact_batch_step(item: Any) -> dict[str, Any] | None: - if not isinstance(item, Mapping): - return None - ok = bool(item.get("ok", False)) - compact = { - "index": item.get("index"), - "op": str(item.get("op") or ""), - "ok": ok, - "status": "completed" if ok else "failed", - "elapsed_ms": int(item.get("elapsed_ms") or 0), - } - for key in ("phase", "error"): - if item.get(key) not in (None, ""): - compact[key] = item.get(key) - return compact - - -def _compact_batch_conditions(parsed: Mapping[str, Any], steps: list[dict[str, Any]]) -> list[dict[str, Any]]: - supplied = parsed.get("conditions") - source = supplied if isinstance(supplied, list) else [] - if not source: - source = [item for item in steps if str(item.get("op") or "") in _BATCH_CONDITION_OPS] - - conditions: list[dict[str, Any]] = [] - for item in source: - compact = _compact_batch_step(item) - if compact is None: - continue - observed = item.get("observed") if isinstance(item, Mapping) else None - if observed not in (None, "", [], {}): - compact["observed"] = observed - conditions.append(compact) - return conditions - - -def _compact_extraction_provenance( - steps: list[dict[str, Any]], - generation_id: str, -) -> dict[str, dict[str, Any]]: - provenance: dict[str, dict[str, Any]] = {} - for item in steps: - if not isinstance(item, Mapping) or not item.get("ok"): - continue - if str(item.get("op") or "") not in {"extract_text", "extract_value"}: - continue - field_name = str(item.get("field") or "").strip() - if not field_name: - continue - provenance[field_name] = { - "selector": str(item.get("selector") or "")[:600], - "raw_text": str(item.get("raw_text") or "")[:1000], - "generation_id": generation_id, - } - return provenance - - -class RuntimeRunner(Protocol): - async def __call__( - self, - *, - task: str, - session_id: str | None = None, - request_id: str | None = None, - timeout_s: int | None = None, - ) -> dict[str, Any]: - ... - - -_ACTIONS: dict[str, ActionHandler] = {} -_ACTION_SPECS: dict[str, dict[str, Any]] = {} -_RUNTIME_RUNNER: RuntimeRunner | None = None -_CODE_EXECUTOR: CodeExecutor | None = None -_LOCK = asyncio.Lock() -_ctx_browser_worker_action: contextvars.ContextVar[bool] = contextvars.ContextVar( - "playwright_runtime_browser_worker_action", - default=False, -) -_RECURSIVE_BROWSER_ACTIONS = frozenset({"browser_task", "run_browser_task"}) - - -@contextlib.contextmanager -def browser_worker_action_context(): - token = _ctx_browser_worker_action.set(True) - try: - yield - finally: - _ctx_browser_worker_action.reset(token) - - -class ActionController(BaseController): - """Instance-scoped registry and dispatcher for custom actions.""" - - def __init__( - self, - *, - actions: dict[str, ActionHandler] | None = None, - action_specs: dict[str, dict[str, Any]] | None = None, - runtime_runner: RuntimeRunner | None = None, - code_executor: CodeExecutor | None = None, - lock: asyncio.Lock | None = None, - ) -> None: - self._actions: dict[str, ActionHandler] = actions if actions is not None else {} - self._action_specs: dict[str, dict[str, Any]] = action_specs if action_specs is not None else {} - self._runtime_runner: RuntimeRunner | None = runtime_runner - self._code_executor: CodeExecutor | None = code_executor - self._lock: asyncio.Lock = lock if lock is not None else asyncio.Lock() - - @property - def runtime_runner(self) -> RuntimeRunner | None: - return self._runtime_runner - - @property - def code_executor(self) -> CodeExecutor | None: - return self._code_executor - - def bind_runtime(self, runtime: Any) -> None: - run_browser_task = getattr(runtime, "run_browser_task", None) - if run_browser_task is None or not callable(run_browser_task): - raise ValueError("runtime must expose an async run_browser_task(...) method") - - async def _runner( - *, - task: str, - session_id: str | None = None, - request_id: str | None = None, - timeout_s: int | None = None, - ) -> dict[str, Any]: - return await run_browser_task( - task=task, - session_id=session_id, - request_id=request_id, - timeout_s=timeout_s, - ) - - self.bind_runtime_runner(_runner) - - def bind_runtime_runner(self, runner: RuntimeRunner | None) -> None: - self._runtime_runner = runner - - def clear_runtime_runner(self) -> None: - self.bind_runtime_runner(None) - - def bind_code_executor(self, executor: CodeExecutor | None) -> None: - self._code_executor = executor - - def clear_code_executor(self) -> None: - self._code_executor = None - - def register_action(self, name: str, handler: ActionHandler, *, overwrite: bool = True) -> None: - action_name = _normalize_action_name(name) - if not action_name: - raise ValueError("action name must be non-empty") - if not callable(handler): - raise TypeError("handler must be callable") - if not overwrite and action_name in self._actions: - raise ValueError(f"action already exists: {action_name}") - self._actions[action_name] = handler - - def register_action_spec( - self, - name: str, - *, - summary: str = "", - when_to_use: str = "", - params: dict[str, str] | None = None, - ) -> None: - action_name = _normalize_action_name(name) - if not action_name: - raise ValueError("action name must be non-empty") - self._action_specs[action_name] = { - "summary": (summary or "").strip(), - "when_to_use": (when_to_use or "").strip(), - "params": dict(params or {}), - } - - def list_actions(self) -> list[str]: - return sorted(self._actions.keys()) - - def describe_actions(self) -> dict[str, dict[str, Any]]: - result: dict[str, dict[str, Any]] = {} - for name in sorted(self._actions.keys()): - spec = self._action_specs.get(name, {}) - result[name] = { - "summary": str(spec.get("summary", "") or ""), - "when_to_use": str(spec.get("when_to_use", "") or ""), - "params": dict(spec.get("params", {}) or {}), - } - return result - - async def run_action( - self, - action: str, - session_id: str = "", - request_id: str = "", - **kwargs: Any, - ) -> ActionResult: - action_name = _normalize_action_name(action) - sid = (session_id or "").strip() - rid = (request_id or "").strip() - param_keys = ",".join(sorted(str(key) for key in kwargs.keys())) - logger.info( - f"CONTROLLER_ACTION start action={action_name} session_id={sid or '-'} " - f"request_id={rid or '-'} param_keys={param_keys or '-'}" - ) - - if _ctx_browser_worker_action.get() and action_name in _RECURSIVE_BROWSER_ACTIONS: - error = ( - "recursive_browser_task_blocked: browser workers must not invoke " - "browser_task/run_browser_task via browser_custom_action; return a JSON error instead" - ) - logger.warning( - f"CONTROLLER_ACTION blocked action={action_name} session_id={sid or '-'} " - f"request_id={rid or '-'} error={error}" - ) - return { - "ok": False, - "action": action_name, - "session_id": sid, - "request_id": rid, - "error": error, - } - - handler = self._actions.get(action_name) - if handler is None: - logger.warning( - f"CONTROLLER_ACTION unknown action={action_name} session_id={sid or '-'} request_id={rid or '-'}" - ) - return { - "ok": False, - "action": action_name, - "session_id": sid, - "request_id": rid, - "error": f"unknown action: {action_name}", - } - - try: - async with self._lock: - raw = await _maybe_await(handler(session_id=sid, request_id=rid, **kwargs)) - - response = dict(raw) if isinstance(raw, dict) else {"result": raw} - response.setdefault("ok", True) - response.setdefault("action", action_name) - response.setdefault("session_id", sid) - response.setdefault("request_id", rid) - response.setdefault("error", None) - _ok = bool(response.get("ok", False)) - _err = response.get("error") if not _ok else None - logger.info( - f"CONTROLLER_ACTION end action={action_name} session_id={sid or '-'} " - f"request_id={rid or '-'} ok={_ok}" + (f" error={_err!r}" if _err else "") - ) - return response - except Exception as exc: - logger.error( - f"CONTROLLER_ACTION error action={action_name} session_id={sid or '-'} " - f"request_id={rid or '-'} error={exc}" - ) - return { - "ok": False, - "action": action_name, - "session_id": sid, - "request_id": rid, - "error": str(exc), - } - - def register_builtin_actions(self) -> None: - register_builtin_actions(controller=self) - - def register_example_actions(self) -> None: - register_builtin_actions(controller=self) - - def snapshot(self) -> dict[str, Any]: - return { - "actions": dict(self._actions), - "action_specs": copy.deepcopy(self._action_specs), - "runtime_runner": self._runtime_runner, - "code_executor": self._code_executor, - } - - def restore(self, snapshot: dict[str, Any]) -> None: - self._actions.clear() - self._actions.update(dict(snapshot.get("actions", {}))) - self._action_specs.clear() - self._action_specs.update(copy.deepcopy(dict(snapshot.get("action_specs", {})))) - self._runtime_runner = snapshot.get("runtime_runner") - self._code_executor = snapshot.get("code_executor") - - -def _normalize_action_name(name: str) -> str: - return (name or "").strip().lower() - - -def _to_int_or_none(value: Any) -> int | None: - if value is None: - return None - try: - return int(value) - except Exception: - return None - - -def _normalize_offset(value: Any) -> dict[str, int] | None: - if value is None: - return None - if isinstance(value, dict): - x = _to_int_or_none(value.get("x")) - y = _to_int_or_none(value.get("y")) - else: - x = _to_int_or_none(getattr(value, "x", None)) - y = _to_int_or_none(getattr(value, "y", None)) - if x is None or y is None: - return None - return {"x": x, "y": y} - - -def _build_run_code_task(js_code: str, purpose: str) -> str: - tool_input = json.dumps({"code": js_code}, ensure_ascii=False) - return ( - f"Execute this browser operation: {purpose}.\n" - "Call browser_run_code exactly once with this JSON input:\n" - f"{tool_input}\n\n" - "Then return your required top-level response JSON. " - "Set its `final` field to the exact JSON result returned by browser_run_code." - ) - - -def _wrap_page_script(*parts: str) -> str: - return "async (page) => {\n" + "".join(parts) + "}" - - -def _build_selector_resolution_helpers(payload_json: str) -> str: - return ( - f" const params = {payload_json};\n" - " if (params.url && String(params.url).trim()) {\n" - " await page.goto(String(params.url).trim());\n" - " }\n" - " const getTextBox = async (query, role) => {\n" - " const term = String(query || '').trim().toLowerCase();\n" - " if (!term) return null;\n" - " return await page.evaluate(({ term, role }) => {\n" - " const all = Array.from(document.querySelectorAll('body *'));\n" - " const score = (el) => {\n" - " const text = String(el.textContent || '').trim().toLowerCase();\n" - " if (!text) return -1;\n" - " if (text === term) return 2;\n" - " if (text.includes(term)) return 1;\n" - " return -1;\n" - " };\n" - " const toVisibleBox = (el) => {\n" - " if (!el) return null;\n" - " const candidates = role === 'target' && el.parentElement ? [el.parentElement, el] : [el];\n" - " for (const candidate of candidates) {\n" - " const rect = candidate.getBoundingClientRect();\n" - " if (rect && rect.width > 0 && rect.height > 0) {\n" - " return { x: rect.x, y: rect.y, width: rect.width, height: rect.height };\n" - " }\n" - " }\n" - " return null;\n" - " };\n" - " const exactMatches = all.filter((el) => score(el) === 2);\n" - " for (const el of exactMatches) {\n" - " const box = toVisibleBox(el);\n" - " if (box) return box;\n" - " }\n" - " const fuzzyMatches = all.filter((el) => score(el) === 1);\n" - " for (const el of fuzzyMatches) {\n" - " const box = toVisibleBox(el);\n" - " if (box) return box;\n" - " }\n" - " return null;\n" - " }, { term, role });\n" - " };\n" - " const extractTextFromHasText = (s) => {\n" - " if (!s || typeof s !== 'string') return s;\n" - " const m = String(s).match(/:has-text\\s*\\(\\s*['\"]([^'\"]*)['\"]\\s*\\)/);\n" - " return m ? m[1] : s;\n" - " };\n" - " const getPoint = async (selector, offset, role) => {\n" - " let box = null;\n" - " if (selector) {\n" - " try {\n" - " const el = await page.$(selector);\n" - " if (el) box = await el.boundingBox();\n" - " } catch (_err) {\n" - " box = null;\n" - " }\n" - " }\n" - " if (!box) {\n" - " const textTerm = extractTextFromHasText(selector) || selector;\n" - " box = await getTextBox(textTerm, role);\n" - " }\n" - " if (!box) return null;\n" - " if (offset && Number.isFinite(offset.x) && Number.isFinite(offset.y)) {\n" - " return { x: Math.trunc(box.x + offset.x), y: Math.trunc(box.y + offset.y) };\n" - " }\n" - " return { x: Math.trunc(box.x + box.width / 2), y: Math.trunc(box.y + box.height / 2) };\n" - " };\n" - ) - - -def _build_coordinate_resolution_body() -> str: - return ( - " let source = null;\n" - " let target = null;\n" - " if (params.element_source || params.element_target) {\n" - " if (params.element_source) {\n" - " source = await getPoint(params.element_source, params.element_source_offset, 'source');\n" - " if (!source) {\n" - " return { ok: false, error: 'Failed to determine source coordinates from selector." - ' Use the exact visible text (e.g. "Learn more" not "More information")' - " or a valid CSS/Playwright selector.', source: null, target: null };\n" - " }\n" - " }\n" - " if (params.element_target) {\n" - " target = await getPoint(params.element_target, params.element_target_offset, 'target');\n" - " if (!target) {\n" - " return { ok: false, error: 'Failed to determine target coordinates from selector'," - " source, target: null };\n" - " }\n" - " }\n" - " } else {\n" - " const values = [params.coord_source_x, params.coord_source_y," - " params.coord_target_x, params.coord_target_y];\n" - " const allFinite = values.every((v) => Number.isFinite(v));\n" - " if (!allFinite) {\n" - " return { ok: false, error: 'Must provide either source/target selectors" - " or source/target coordinates' };\n" - " }\n" - " source = { x: Math.trunc(params.coord_source_x), y: Math.trunc(params.coord_source_y) };\n" - " target = { x: Math.trunc(params.coord_target_x), y: Math.trunc(params.coord_target_y) };\n" - " }\n" - " return { ok: true, source, target, error: null };\n" - ) - - -def _build_drag_operation_body() -> str: - return ( - " let source = null;\n" - " let target = null;\n" - " if (params.element_source && params.element_target) {\n" - " source = await getPoint(params.element_source, params.element_source_offset, 'source');\n" - " target = await getPoint(params.element_target, params.element_target_offset, 'target');\n" - " if (!source || !target) {\n" - " return { ok: false, error: 'Failed to determine source or target coordinates from selectors'," - " source, target };\n" - " }\n" - " } else {\n" - " const values = [params.coord_source_x, params.coord_source_y," - " params.coord_target_x, params.coord_target_y];\n" - " const allFinite = values.every((v) => Number.isFinite(v));\n" - " if (!allFinite) {\n" - " return { ok: false, error: 'Must provide either source/target selectors" - " or source/target coordinates' };\n" - " }\n" - " source = { x: Math.trunc(params.coord_source_x), y: Math.trunc(params.coord_source_y) };\n" - " target = { x: Math.trunc(params.coord_target_x), y: Math.trunc(params.coord_target_y) };\n" - " }\n" - " const steps = Math.max(1, Number.isFinite(params.steps) ? Math.trunc(params.steps) : 10);\n" - " const delayMs = Math.max(0, Number.isFinite(params.delay_ms) ? Math.trunc(params.delay_ms) : 5);\n" - " try {\n" - " await page.mouse.move(source.x, source.y);\n" - " await page.mouse.down();\n" - " for (let i = 1; i <= steps; i += 1) {\n" - " const ratio = i / steps;\n" - " const x = Math.trunc(source.x + (target.x - source.x) * ratio);\n" - " const y = Math.trunc(source.y + (target.y - source.y) * ratio);\n" - " await page.mouse.move(x, y);\n" - " if (delayMs > 0) {\n" - " await page.waitForTimeout(delayMs);\n" - " }\n" - " }\n" - " await page.mouse.move(target.x, target.y);\n" - " await page.mouse.move(target.x, target.y);\n" - " await page.mouse.up();\n" - " } catch (error) {\n" - " return {\n" - " ok: false,\n" - " error: `Error during drag operation: ${String(error)}`,\n" - " source,\n" - " target,\n" - " steps,\n" - " delay_ms: delayMs,\n" - " };\n" - " }\n" - " const message = params.element_source && params.element_target\n" - " ? `Dragged element '${params.element_source}' to '${params.element_target}'`\n" - " : `Dragged from (${source.x}, ${source.y}) to (${target.x}, ${target.y})`;\n" - " return { ok: true, message, source, target, steps, delay_ms: delayMs, error: null };\n" - ) - - -def _build_coordinate_script(payload: dict[str, Any]) -> str: - payload_json = json.dumps(payload, ensure_ascii=False) - return _wrap_page_script( - _build_selector_resolution_helpers(payload_json), - _build_coordinate_resolution_body(), - ) - - -def _build_drag_script(payload: dict[str, Any]) -> str: - payload_json = json.dumps(payload, ensure_ascii=False) - return _wrap_page_script( - _build_selector_resolution_helpers(payload_json), - _build_drag_operation_body(), - ) - - -def _build_set_input_files_script(selector: str, paths: list[str]) -> str: - selector_js = "'" + str(selector).replace("\\", "\\\\").replace("'", "\\'") + "'" - paths_json = json.dumps(paths) - return ( - "async (page) => {\n" - " try {\n" - f" await page.locator({selector_js}).setInputFiles({paths_json});\n" - f" return {{ ok: true, selector: {selector_js}, paths: {paths_json} }};\n" - " } catch (error) {\n" - " const msg = String(error);\n" - " if (msg.includes('strict mode violation')) {\n" - f" return {{ ok: false, error: msg, selector: {selector_js}, paths: {paths_json}," - " hint: 'Multiple file inputs matched. Use a more specific selector" - " (e.g. an id like #file-upload) targeting the visible input.' }};\n" - " }\n" - f" return {{ ok: false, error: msg, selector: {selector_js}, paths: {paths_json} }};\n" - " }\n" - "}" - ) - - -def _build_batch_interact_script(payload: dict[str, Any]) -> str: - """Build a Playwright function that executes compact deterministic browser steps. - - The model provides a structured plan; the runtime constructs the JS, so the - browser worker does not need to emit arbitrary page scripts for common form - and search-flow interactions. - """ - payload_json = json.dumps(payload, ensure_ascii=False) - return ( - "async (page) => {\n" - f" const payload = {payload_json};\n" - " const startedAt = Date.now();\n" - " const steps = Array.isArray(payload.steps) ? payload.steps : [];\n" - " const initialBrowserContext = page && typeof page.context === 'function' ? page.context() : null;\n" - " const initialTabCount = initialBrowserContext && typeof initialBrowserContext.pages === 'function'\n" - " ? initialBrowserContext.pages().length\n" - " : 1;\n" - " const defaultTimeout = Math.max(250, Number(payload.timeout_ms || 2500));\n" - " const defaultConditionTimeout = Math.max(\n" - " defaultTimeout,\n" - " Number(payload.condition_timeout_ms || 10000)\n" - " );\n" - " const conditionOps = new Set([\n" - " 'wait_for_selector', 'wait_for_text', 'wait_for_load_state', 'wait_for_url',\n" - " 'wait_for_first_card_title', 'wait_for_sort_state', 'wait_for_result_count',\n" - " 'wait_for_dom_text_change', 'wait_for_stable', 'wait_for_tab'\n" - " ]);\n" - " const defaultAfter = Math.max(0, Number(payload.wait_after_each_ms || 0));\n" - " const generationId = String(payload.generation_id || 'g0');\n" - " const results = [];\n" - " const sleep = async (ms) => {\n" - " const delay = Math.max(0, Number(ms || 0));\n" - " if (delay <= 0) return;\n" - " await page.waitForTimeout(delay);\n" - " };\n" - " const compactText = (value, maxLen = 1200) => String(value || '')\n" - " .replace(/\\s+/g, ' ')\n" - " .trim()\n" - " .slice(0, maxLen);\n" - " const hasTarget = (step) => !!(\n" - " step.selector || step.role || step.label || step.placeholder ||\n" - " step.text || step.testid\n" - " );\n" - " const selectAllKey = 'ControlOrMeta+A';\n" - " const locatorFromStep = (step) => {\n" - " if (step.selector) return page.locator(String(step.selector));\n" - " if (step.role) {\n" - " if (step.name !== undefined && step.name !== null && String(step.name).length > 0) {\n" - " return page.getByRole(\n" - " String(step.role),\n" - " { name: String(step.name), exact: !!step.exact }\n" - " );\n" - " }\n" - " return page.getByRole(String(step.role));\n" - " }\n" - " if (step.label) {\n" - " return page.getByLabel(\n" - " String(step.label),\n" - " { exact: !!step.exact }\n" - " );\n" - " }\n" - " if (step.placeholder) {\n" - " return page.getByPlaceholder(\n" - " String(step.placeholder),\n" - " { exact: !!step.exact }\n" - " );\n" - " }\n" - " if (step.testid) return page.getByTestId(String(step.testid));\n" - " if (step.text) {\n" - " return page.getByText(\n" - " String(step.text),\n" - " { exact: !!step.exact }\n" - " );\n" - " }\n" - " throw new Error('step needs selector, role, label, placeholder, testid, or text');\n" - " };\n" - " const optionLocatorFromStep = (step) => {\n" - " if (step.option_selector || step.choose_selector) {\n" - " return page.locator(\n" - " String(step.option_selector || step.choose_selector)\n" - " );\n" - " }\n" - " if (step.option_role || step.choose_role) {\n" - " const role = String(step.option_role || step.choose_role);\n" - " const name = step.option_name ?? step.choose_name ?? step.choose_text ?? step.option_text;\n" - " if (name !== undefined && name !== null && String(name).length > 0) {\n" - " return page.getByRole(\n" - " role,\n" - " { name: String(name), exact: !!step.exact }\n" - " );\n" - " }\n" - " return page.getByRole(role);\n" - " }\n" - " const chosenText = step.choose_text ?? step.option_text ?? step.text_to_choose ?? step.value;\n" - " if (chosenText !== undefined && chosenText !== null && String(chosenText).length > 0) {\n" - " return page.getByText(\n" - " String(chosenText),\n" - " { exact: !!step.exact }\n" - " );\n" - " }\n" - " throw new Error(\n" - " 'autocomplete/select_visible_text needs choose_text/option_text or option selector'\n" - " );\n" - " };\n" - " const firstLocator = (locator) => {\n" - " return locator && typeof locator.first === 'function' ? locator.first() : locator;\n" - " };\n" - " const validateTarget = async (locator, timeout, requireEnabled = true) => {\n" - " const matchCount = locator && typeof locator.count === 'function'\n" - " ? await locator.count()\n" - " : 1;\n" - " if (matchCount !== 1) {\n" - " throw new Error(`target must match exactly one element; matched ${matchCount}`);\n" - " }\n" - " const target = firstLocator(locator);\n" - " if (target && typeof target.waitFor === 'function') {\n" - " await target.waitFor({ state: 'visible', timeout });\n" - " }\n" - " const visible = target && typeof target.isVisible === 'function'\n" - " ? await target.isVisible()\n" - " : true;\n" - " const enabled = target && typeof target.isEnabled === 'function'\n" - " ? await target.isEnabled()\n" - " : true;\n" - " if (!visible) throw new Error('target is not visible');\n" - " if (requireEnabled && !enabled) throw new Error('target is not enabled');\n" - " return {\n" - " locator: target,\n" - " metadata: { match_count: matchCount, visible, enabled, generation_id: generationId }\n" - " };\n" - " };\n" - " const pollUntil = async (predicate, timeout, pollInterval, stableMs = 0) => {\n" - " const started = Date.now();\n" - " const interval = Math.max(50, Math.min(1000, Number(pollInterval || 100)));\n" - " let stableSince = 0;\n" - " let lastValue = null;\n" - " while (Date.now() - started <= timeout) {\n" - " const observation = await predicate();\n" - " if (observation && observation.ok) {\n" - " if (stableMs <= 0) return observation;\n" - " const serialized = JSON.stringify(observation.value);\n" - " if (serialized === lastValue) {\n" - " if (!stableSince) stableSince = Date.now();\n" - " if (Date.now() - stableSince >= stableMs) return observation;\n" - " } else {\n" - " lastValue = serialized;\n" - " stableSince = Date.now();\n" - " }\n" - " } else {\n" - " stableSince = 0;\n" - " lastValue = null;\n" - " }\n" - " await sleep(interval);\n" - " }\n" - " throw new Error(`condition timed out after ${timeout}ms`);\n" - " };\n" - " const clickWithTransientRetry = async (step, timeout, option = false, initialChecked = null) => {\n" - " const started = Date.now();\n" - " let lastError = null;\n" - " for (let attempt = 0; attempt < 2; attempt += 1) {\n" - " const remaining = Math.max(250, timeout - (Date.now() - started));\n" - " try {\n" - " const locator = option ? optionLocatorFromStep(step) : locatorFromStep(step);\n" - " const checked = attempt === 0 && initialChecked\n" - " ? initialChecked\n" - " : await validateTarget(locator, remaining, true);\n" - " await checked.locator.click({ timeout: remaining });\n" - " return checked;\n" - " } catch (error) {\n" - " lastError = error;\n" - " const message = String(error && error.message ? error.message : error).toLowerCase();\n" - " const transient = /(detached|not attached|intercept|not stable|" - "outside of the viewport)/.test(message);\n" - " if (!transient || attempt > 0 || Date.now() - started >= timeout) throw error;\n" - " await sleep(Math.min(100, Math.max(0, timeout - (Date.now() - started))));\n" - " }\n" - " }\n" - " throw lastError || new Error('click failed');\n" - " };\n" - " const summarizeVisible = async () => {\n" - " return await page.evaluate(() => {\n" - " const text = (document.body && document.body.innerText) ? document.body.innerText : '';\n" - " return text.replace(/\\s+/g, ' ').trim().slice(0, 1200);\n" - " });\n" - " };\n" - " const collectExtracted = (items) => {\n" - " const extracted = {};\n" - " for (const item of items) {\n" - " if (!item || !item.field) continue;\n" - " extracted[item.field] = item.text !== undefined ? item.text : item.value;\n" - " }\n" - " return extracted;\n" - " };\n" - " const preflightTargets = new Map();\n" - " const preflightStartedAt = Date.now();\n" - " let preflightElapsedMs = 0;\n" - " let preflightTargetCount = 0;\n" - " const preflightPrimaryOps = new Set([\n" - " 'click', 'fill', 'type', 'autocomplete', 'select_option', 'set_checked',\n" - " 'extract_text', 'extract_value'\n" - " ]);\n" - " const preflightKey = (index, kind) => `${index}:${kind}`;\n" - " const preflightTarget = async (index, kind, locator, timeout, requireEnabled = true) => {\n" - " preflightTargetCount += 1;\n" - " const checked = await validateTarget(locator, timeout, requireEnabled);\n" - " preflightTargets.set(preflightKey(index, kind), checked);\n" - " return checked;\n" - " };\n" - " const getCheckedTarget = async (index, kind, locator, timeout, requireEnabled = true) => {\n" - " const cached = preflightTargets.get(preflightKey(index, kind));\n" - " return cached || validateTarget(locator, timeout, requireEnabled);\n" - " };\n" - " for (let i = 0; i < steps.length; i += 1) {\n" - " const step = steps[i] || {};\n" - " const op = String(step.op || '').trim().toLowerCase();\n" - " const timeout = Math.max(250, Number(step.timeout_ms || defaultTimeout));\n" - " try {\n" - " if (preflightPrimaryOps.has(op) || (op === 'press' && hasTarget(step))) {\n" - " const requireEnabled = op !== 'extract_text' && op !== 'extract_value';\n" - " await preflightTarget(i, 'primary', locatorFromStep(step), timeout, requireEnabled);\n" - " }\n" - " if (op === 'select_visible_text') {\n" - " await preflightTarget(i, 'option', optionLocatorFromStep(step), timeout);\n" - " } else if (op === 'autocomplete' && step.resolved_option_target_id) {\n" - " await preflightTarget(i, 'option', optionLocatorFromStep(step), timeout);\n" - " }\n" - " } catch (error) {\n" - " const message = String(error && error.message ? error.message : error);\n" - " preflightElapsedMs = Date.now() - preflightStartedAt;\n" - " return {\n" - " ok: false,\n" - " status: 'failed',\n" - " error: `batch_preflight_failed: steps[${i}] ${message}`,\n" - " steps: [{\n" - " index: i, op, ok: false, phase: 'preflight', error: message,\n" - " elapsed_ms: Date.now() - startedAt, generation_id: generationId\n" - " }],\n" - " extracted: collectExtracted(results),\n" - " conditions: [],\n" - " elapsed_ms: Date.now() - startedAt,\n" - " internal_steps_elapsed_ms: 0,\n" - " preflight_elapsed_ms: preflightElapsedMs,\n" - " preflight_target_count: preflightTargetCount,\n" - " url: page.url(),\n" - " title: await page.title().catch(() => ''),\n" - " };\n" - " }\n" - " }\n" - " preflightElapsedMs = Date.now() - preflightStartedAt;\n" - " for (let i = 0; i < steps.length; i += 1) {\n" - " const step = steps[i] || {};\n" - " const op = String(step.op || '').trim().toLowerCase();\n" - " const fallbackTimeout = conditionOps.has(op) ? defaultConditionTimeout : defaultTimeout;\n" - " const timeout = Math.max(250, Number(step.timeout_ms || fallbackTimeout));\n" - " const stepStartedAt = Date.now();\n" - " const item = { index: i, op, ok: false, elapsed_ms: 0, generation_id: generationId };\n" - " try {\n" - " if (!op) throw new Error('missing op');\n" - " if (op === 'click') {\n" - " const prechecked = await getCheckedTarget(i, 'primary', locatorFromStep(step), timeout);\n" - " const target = await clickWithTransientRetry(step, timeout, false, prechecked);\n" - " item.target_validation = target.metadata;\n" - " } else if (op === 'fill' || op === 'type') {\n" - " const checked = await getCheckedTarget(i, 'primary', locatorFromStep(step), timeout);\n" - " item.target_validation = checked.metadata;\n" - " const target = checked.locator;\n" - " const value = String(step.value ?? step.text_value ?? '');\n" - " if (op === 'type' || step.mode === 'type') {\n" - " await target.click({ timeout });\n" - " try {\n" - " await page.keyboard.press(selectAllKey);\n" - " } catch (_err) {}\n" - " try {\n" - " await page.keyboard.press('Backspace');\n" - " } catch (_err) {}\n" - " await page.keyboard.type(\n" - " value,\n" - " { delay: Math.max(0, Number(step.delay_ms || 0)) }\n" - " );\n" - " } else {\n" - " await target.fill(value, { timeout });\n" - " }\n" - " } else if (op === 'autocomplete') {\n" - " const checked = await getCheckedTarget(i, 'primary', locatorFromStep(step), timeout);\n" - " item.target_validation = checked.metadata;\n" - " const target = checked.locator;\n" - " const value = String(step.value ?? step.query ?? step.text_value ?? '');\n" - " await target.click({ timeout });\n" - " try {\n" - " await page.keyboard.press(selectAllKey);\n" - " } catch (_err) {}\n" - " try {\n" - " await page.keyboard.press('Backspace');\n" - " } catch (_err) {}\n" - " if (step.fill_first) {\n" - " await target.fill(value, { timeout });\n" - " } else {\n" - " await page.keyboard.type(\n" - " value,\n" - " { delay: Math.max(0, Number(step.delay_ms || 0)) }\n" - " );\n" - " }\n" - " if (step.wait_after_type_ms !== undefined) {\n" - " await sleep(Math.max(0, Number(step.wait_after_type_ms || 0)));\n" - " }\n" - " const checkedOption = await getCheckedTarget(i, 'option', optionLocatorFromStep(step), timeout);\n" - " item.option_validation = checkedOption.metadata;\n" - " await clickWithTransientRetry(step, timeout, true, checkedOption);\n" - " } else if (op === 'select_visible_text') {\n" - " const checkedOption = await getCheckedTarget(i, 'option', optionLocatorFromStep(step), timeout);\n" - " item.target_validation = checkedOption.metadata;\n" - " await clickWithTransientRetry(step, timeout, true, checkedOption);\n" - " } else if (op === 'press') {\n" - " const key = String(step.key || 'Enter');\n" - " if (hasTarget(step)) {\n" - " const checkedTarget = await getCheckedTarget(i, 'primary', locatorFromStep(step), timeout);\n" - " item.target_validation = checkedTarget.metadata;\n" - " await checkedTarget.locator.press(key, { timeout });\n" - " } else {\n" - " await page.keyboard.press(key);\n" - " }\n" - " } else if (op === 'select_option') {\n" - " const checkedTarget = await getCheckedTarget(i, 'primary', locatorFromStep(step), timeout);\n" - " item.target_validation = checkedTarget.metadata;\n" - " const target = checkedTarget.locator;\n" - " if (step.values !== undefined) {\n" - " const values = Array.isArray(step.values)\n" - " ? step.values.map((item) => String(item))\n" - " : String(step.values);\n" - " await target.selectOption(values, { timeout });\n" - " } else {\n" - " const option = {};\n" - " if (step.value !== undefined || step.option_value !== undefined) {\n" - " option.value = String(step.option_value ?? step.value);\n" - " }\n" - " if (\n" - " step.label_value !== undefined ||\n" - " step.option_label !== undefined ||\n" - " step.option_text !== undefined ||\n" - " step.choose_text !== undefined\n" - " ) {\n" - " option.label = String(\n" - " step.label_value ?? step.option_label ?? step.option_text ?? step.choose_text\n" - " );\n" - " }\n" - " if (step.index !== undefined) option.index = Number(step.index);\n" - " if (!Object.keys(option).length) {\n" - " throw new Error(\n" - " 'select_option requires value, values, option_value, option_text, '\n" - " + 'option_label, label_value, choose_text, or index'\n" - " );\n" - " }\n" - " await target.selectOption(option, { timeout });\n" - " }\n" - " } else if (op === 'set_checked') {\n" - " const checked = step.checked === undefined ? true : !!step.checked;\n" - " const checkedTarget = await getCheckedTarget(i, 'primary', locatorFromStep(step), timeout);\n" - " item.target_validation = checkedTarget.metadata;\n" - " await checkedTarget.locator.setChecked(checked, { timeout });\n" - " } else if (op === 'wait_for_selector') {\n" - " if (!step.selector) throw new Error('wait_for_selector requires selector');\n" - " await firstLocator(page.locator(String(step.selector))).waitFor({\n" - " state: String(step.state || 'visible'),\n" - " timeout,\n" - " });\n" - " } else if (op === 'wait_for_text') {\n" - " if (!step.text) throw new Error('wait_for_text requires text');\n" - " await firstLocator(page.getByText(String(step.text), { exact: !!step.exact }))\n" - " .waitFor({ timeout });\n" - " } else if (op === 'wait_for_load_state') {\n" - " await page.waitForLoadState(\n" - " String(step.state || 'domcontentloaded'),\n" - " { timeout }\n" - " );\n" - " } else if (op === 'wait_for_url') {\n" - " const exactUrl = String(step.url ?? step.expected_url ?? '');\n" - " const containsUrl = String(step.url_contains ?? '');\n" - " const patternText = String(step.url_pattern ?? '');\n" - " const observation = await pollUntil(async () => {\n" - " const contextPages = typeof page.context === 'function' ? page.context().pages() : [page];\n" - " const candidates = [page, ...contextPages.filter((candidate) => candidate !== page).reverse()];\n" - " for (const candidate of candidates) {\n" - " const currentUrl = String(candidate.url());\n" - " let matched = exactUrl ? currentUrl === exactUrl : true;\n" - " if (containsUrl) matched = matched && currentUrl.includes(containsUrl);\n" - " if (patternText) matched = matched && new RegExp(patternText).test(currentUrl);\n" - " if (!matched) continue;\n" - " if (candidate !== page) {\n" - " page = candidate;\n" - " await page.bringToFront().catch(() => undefined);\n" - " item.tab_switched = true;\n" - " }\n" - " return { ok: true, value: currentUrl };\n" - " }\n" - " return { ok: false, value: String(page.url()) };\n" - " }, timeout, step.poll_interval_ms);\n" - " item.url = observation.value;\n" - " } else if (op === 'wait_for_first_card_title') {\n" - " const expected = String(step.expected_text ?? step.text ?? '');\n" - " const locator = page.locator(String(step.selector));\n" - " const observation = await pollUntil(async () => {\n" - " const count = await locator.count();\n" - " if (count < 1) return { ok: false, value: '' };\n" - " const title = compactText(await firstLocator(locator).innerText(), 300);\n" - " return { ok: !!title && (!expected || title.includes(expected)), value: title };\n" - " }, timeout, step.poll_interval_ms, Number(step.stable_ms || 0));\n" - " item.text = observation.value;\n" - " } else if (op === 'wait_for_sort_state') {\n" - " const locator = page.locator(String(step.selector));\n" - " const attribute = String(step.attribute || 'aria-sort');\n" - " const expected = String(step.expected_value ?? step.value ?? step.text ?? '');\n" - " const observation = await pollUntil(async () => {\n" - " if (await locator.count() !== 1) return { ok: false, value: '' };\n" - " const target = firstLocator(locator);\n" - " const value = attribute === 'text'\n" - " ? compactText(await target.innerText(), 300)\n" - " : String(await target.getAttribute(attribute) || '');\n" - " return { ok: value === expected, value };\n" - " }, timeout, step.poll_interval_ms);\n" - " item.value = observation.value;\n" - " } else if (op === 'wait_for_result_count') {\n" - " const locator = page.locator(String(step.selector));\n" - " const exactCount = step.count === undefined ? null : Number(step.count);\n" - " const minCount = step.min_count === undefined ? null : Number(step.min_count);\n" - " const maxCount = step.max_count === undefined ? null : Number(step.max_count);\n" - " const observation = await pollUntil(async () => {\n" - " const count = await locator.count();\n" - " const matched = exactCount !== null\n" - " ? count === exactCount\n" - " : (minCount === null || count >= minCount) && (maxCount === null || count <= maxCount);\n" - " return { ok: matched, value: count };\n" - " }, timeout, step.poll_interval_ms, Number(step.stable_ms || 0));\n" - " item.count = observation.value;\n" - " } else if (op === 'wait_for_dom_text_change') {\n" - " const locator = page.locator(String(step.selector));\n" - " const previousText = String(step.previous_text ?? step.initial_text ?? '');\n" - " const observation = await pollUntil(async () => {\n" - " if (await locator.count() !== 1) return { ok: false, value: '' };\n" - " const value = compactText(\n" - " await firstLocator(locator).innerText(), Number(step.max_chars || 1000)\n" - " );\n" - " return { ok: value !== previousText, value };\n" - " }, timeout, step.poll_interval_ms, Number(step.stable_ms || 0));\n" - " item.text = observation.value;\n" - " } else if (op === 'wait_for_stable') {\n" - " const locator = step.selector ? page.locator(String(step.selector)) : null;\n" - " const observation = await pollUntil(async () => {\n" - " const value = locator\n" - " ? compactText(await firstLocator(locator).innerText(), Number(step.max_chars || 1200))\n" - " : await summarizeVisible();\n" - " return { ok: true, value };\n" - " }, timeout, step.poll_interval_ms, Number(step.stable_ms || 500));\n" - " item.stable = true;\n" - " } else if (op === 'wait_for_tab') {\n" - " if (!page || typeof page.context !== 'function') {\n" - " throw new Error('wait_for_tab requires a Playwright browser context');\n" - " }\n" - " const minTabs = Math.max(1, Number(step.min_tabs || (initialTabCount + 1)));\n" - " const expectedUrl = String(step.url || step.expected_url || '');\n" - " const urlContains = String(step.url_contains || '');\n" - " const urlPattern = String(step.url_pattern || '');\n" - " const titleContains = String(step.title_contains || '');\n" - " const observation = await pollUntil(async () => {\n" - " const pages = page.context().pages();\n" - " const tabs = [];\n" - " for (let tabIndex = 0; tabIndex < pages.length; tabIndex += 1) {\n" - " const candidate = pages[tabIndex];\n" - " tabs.push({\n" - " index: tabIndex,\n" - " url: candidate.url(),\n" - " title: await candidate.title().catch(() => '')\n" - " });\n" - " }\n" - " const matches = tabs.filter((tab) => {\n" - " if (expectedUrl && tab.url !== expectedUrl) return false;\n" - " if (urlContains && !tab.url.includes(urlContains)) return false;\n" - " if (urlPattern) {\n" - " try { if (!(new RegExp(urlPattern)).test(tab.url)) return false; } catch (_) { return false; }\n" - " }\n" - " if (titleContains && !tab.title.includes(titleContains)) return false;\n" - " return true;\n" - " });\n" - " const matched = pages.length >= minTabs && matches.length > 0;\n" - " const matchedIndex = matched ? matches[matches.length - 1].index : -1;\n" - " return { ok: matched, value: { tabs, matched_index: matchedIndex } };\n" - " }, timeout, step.poll_interval_ms, Number(step.stable_ms || 0));\n" - " item.tabs = observation.value.tabs;\n" - " item.count = observation.value.tabs.length;\n" - " const matchedIndex = Number(observation.value.matched_index);\n" - " if (step.activate !== false && matchedIndex >= 0) {\n" - " const matchedPage = page.context().pages()[matchedIndex];\n" - " if (matchedPage) {\n" - " await matchedPage.bringToFront();\n" - " page = matchedPage;\n" - " item.url = page.url();\n" - " }\n" - " }\n" - " } else if (op === 'sleep') {\n" - " await sleep(Math.max(0, Number(step.ms || step.time_ms || 0)));\n" - " } else if (op === 'extract_text') {\n" - " const checkedTarget = await getCheckedTarget(i, 'primary', locatorFromStep(step), timeout, false);\n" - " item.target_validation = checkedTarget.metadata;\n" - " const rawText = await checkedTarget.locator.innerText({ timeout });\n" - " item.text = compactText(\n" - " rawText,\n" - " Number(step.max_chars || 500)\n" - " );\n" - " item.raw_text = String(rawText || '').slice(0, 1000);\n" - " item.selector = String(step.selector || '');\n" - " item.field = String(step.field || step.name || step.description || `field_${i + 1}`);\n" - " } else if (op === 'extract_value') {\n" - " const checkedTarget = await getCheckedTarget(i, 'primary', locatorFromStep(step), timeout, false);\n" - " item.target_validation = checkedTarget.metadata;\n" - " item.value = await checkedTarget.locator.inputValue({ timeout });\n" - " item.raw_text = String(item.value || '').slice(0, 1000);\n" - " item.selector = String(step.selector || '');\n" - " item.field = String(step.field || step.name || step.description || `field_${i + 1}`);\n" - " } else if (op === 'screenshot') {\n" - " const path = String(step.path || 'screenshots/batch_interact.png');\n" - " await page.screenshot({ path, fullPage: !!step.full_page });\n" - " item.path = path;\n" - " } else {\n" - " throw new Error(`unsupported op: ${op}`);\n" - " }\n" - " item.ok = true;\n" - " if (step.wait_after_ms !== undefined) {\n" - " await sleep(Math.max(0, Number(step.wait_after_ms || 0)));\n" - " } else if (defaultAfter > 0) {\n" - " await sleep(defaultAfter);\n" - " }\n" - " } catch (error) {\n" - " item.error = String(error && error.message ? error.message : error);\n" - " item.target = {\n" - " selector: step.selector || null,\n" - " role: step.role || null,\n" - " name: step.name || null,\n" - " label: step.label || null,\n" - " placeholder: step.placeholder || null,\n" - " text: step.text || null,\n" - " testid: step.testid || null,\n" - " choose_text: step.choose_text || step.option_text || null,\n" - " };\n" - " item.elapsed_ms = Date.now() - stepStartedAt;\n" - " results.push(item);\n" - " if (step.optional || payload.continue_on_error) continue;\n" - " return {\n" - " ok: false,\n" - " status: results.some((result) => result.ok) ? 'partial' : 'failed',\n" - " error: item.error,\n" - " steps: results,\n" - " extracted: collectExtracted(results),\n" - " conditions: results.filter((result) => conditionOps.has(result.op)),\n" - " elapsed_ms: Date.now() - startedAt,\n" - " internal_steps_elapsed_ms: results.reduce((sum, step) => sum + Number(step.elapsed_ms || 0), 0),\n" - " preflight_elapsed_ms: preflightElapsedMs,\n" - " preflight_target_count: preflightTargetCount,\n" - " url: page.url(),\n" - " title: await page.title().catch(() => ''),\n" - " };\n" - " }\n" - " item.elapsed_ms = Date.now() - stepStartedAt;\n" - " results.push(item);\n" - " }\n" - " const extracted = collectExtracted(results);\n" - " const failedSteps = results.filter((result) => result && !result.ok);\n" - " const conditions = results\n" - " .filter((result) => result && conditionOps.has(result.op))\n" - " .map((result) => {\n" - " const observed = {};\n" - " for (const key of ['url', 'text', 'value', 'count', 'stable', 'tabs']) {\n" - " if (result[key] !== undefined) observed[key] = result[key];\n" - " }\n" - " return {\n" - " index: result.index, op: result.op, ok: result.ok, error: result.error || null,\n" - " elapsed_ms: result.elapsed_ms, observed\n" - " };\n" - " });\n" - " return {\n" - " ok: failedSteps.length === 0,\n" - " status: failedSteps.length === 0 ? 'completed' : 'partial',\n" - " error: failedSteps.length === 0 ? null : 'one or more optional batch steps failed',\n" - " steps: results,\n" - " extracted,\n" - " conditions,\n" - " elapsed_ms: Date.now() - startedAt,\n" - " internal_steps_elapsed_ms: results.reduce((sum, step) => sum + Number(step.elapsed_ms || 0), 0),\n" - " preflight_elapsed_ms: preflightElapsedMs,\n" - " preflight_target_count: preflightTargetCount,\n" - " url: page.url(),\n" - " title: await page.title().catch(() => ''),\n" - " };\n" - "}" - ) - - -def _build_drag_payload( - *, - url: str = "", - source: str = "", - target: str = "", - element_source: str = "", - element_target: str = "", - element_source_offset: Any = None, - element_target_offset: Any = None, - source_x: Any = None, - source_y: Any = None, - target_x: Any = None, - target_y: Any = None, - coord_source_x: Any = None, - coord_source_y: Any = None, - coord_target_x: Any = None, - coord_target_y: Any = None, - steps: Any = None, - delay_ms: Any = None, -) -> dict[str, Any]: - source_selector = (element_source or "").strip() - target_selector = (element_target or "").strip() - if not source_selector: - source_selector = (source or "").strip() - if not target_selector: - target_selector = (target or "").strip() - - sx = _to_int_or_none(coord_source_x) - sy = _to_int_or_none(coord_source_y) - tx = _to_int_or_none(coord_target_x) - ty = _to_int_or_none(coord_target_y) - if sx is None: - sx = _to_int_or_none(source_x) - if sy is None: - sy = _to_int_or_none(source_y) - if tx is None: - tx = _to_int_or_none(target_x) - if ty is None: - ty = _to_int_or_none(target_y) - - return { - "url": (url or "").strip(), - "element_source": source_selector, - "element_target": target_selector, - "element_source_offset": _normalize_offset(element_source_offset), - "element_target_offset": _normalize_offset(element_target_offset), - "coord_source_x": sx, - "coord_source_y": sy, - "coord_target_x": tx, - "coord_target_y": ty, - "steps": _to_int_or_none(steps), - "delay_ms": _to_int_or_none(delay_ms), - } - - -def _has_selector_inputs(payload: dict[str, Any]) -> bool: - return bool((payload.get("element_source") or "").strip()) and bool((payload.get("element_target") or "").strip()) - - -def _has_source_selector(payload: dict[str, Any]) -> bool: - """True if at least element_source is set (for get_element_coordinates, target is optional).""" - return bool((payload.get("element_source") or "").strip()) - - -def _has_coordinate_inputs(payload: dict[str, Any]) -> bool: - return all( - payload.get(k) is not None for k in ("coord_source_x", "coord_source_y", "coord_target_x", "coord_target_y") - ) - - -def _normalize_timeout_s(value: Any) -> int | None: - parsed = _to_int_or_none(value) - if parsed is None or parsed <= 0: - return None - return parsed - - -def _list_dir_files(root: Path) -> list[dict[str, Any]]: - """Return a flat list of files under *root* with name, path, and size.""" - entries: list[dict[str, Any]] = [] - try: - for item in sorted(root.iterdir()): - if item.is_file(): - try: - size = item.stat().st_size - except OSError: - size = -1 - entries.append({"name": item.name, "path": str(item), "size_bytes": size}) - except OSError: - pass - return entries - - -async def _maybe_await(value: Any) -> Any: - if inspect.isawaitable(value): - return await value - return value - - -_DEFAULT_CONTROLLER = ActionController( - actions=_ACTIONS, - action_specs=_ACTION_SPECS, - runtime_runner=_RUNTIME_RUNNER, - lock=_LOCK, -) - - -def get_default_controller() -> ActionController: - return _DEFAULT_CONTROLLER - - -def bind_runtime(runtime: Any) -> None: - _DEFAULT_CONTROLLER.bind_runtime(runtime) - global _RUNTIME_RUNNER - _RUNTIME_RUNNER = _DEFAULT_CONTROLLER.runtime_runner - - -def bind_runtime_runner(runner: RuntimeRunner | None) -> None: - _DEFAULT_CONTROLLER.bind_runtime_runner(runner) - global _RUNTIME_RUNNER - _RUNTIME_RUNNER = runner - - -def clear_runtime_runner() -> None: - bind_runtime_runner(None) - - -def bind_code_executor(executor: CodeExecutor | None) -> None: - _DEFAULT_CONTROLLER.bind_code_executor(executor) - global _CODE_EXECUTOR - _CODE_EXECUTOR = executor - - -def clear_code_executor() -> None: - bind_code_executor(None) - - -def register_action(name: str, handler: ActionHandler, *, overwrite: bool = True) -> None: - _DEFAULT_CONTROLLER.register_action(name=name, handler=handler, overwrite=overwrite) - - -def register_action_spec( - name: str, - *, - summary: str = "", - when_to_use: str = "", - params: dict[str, str] | None = None, -) -> None: - _DEFAULT_CONTROLLER.register_action_spec( - name=name, - summary=summary, - when_to_use=when_to_use, - params=params, - ) - - -def list_actions() -> list[str]: - return _DEFAULT_CONTROLLER.list_actions() - - -def describe_actions() -> dict[str, dict[str, Any]]: - return _DEFAULT_CONTROLLER.describe_actions() - - -async def run_action( - action: str, - session_id: str = "", - request_id: str = "", - **kwargs: Any, -) -> ActionResult: - return await _DEFAULT_CONTROLLER.run_action( - action=action, - session_id=session_id, - request_id=request_id, - **kwargs, - ) - - -def _normalize_batch_interact_payload(kwargs: Mapping[str, Any]) -> dict[str, Any]: - payload = {key: value for key, value in kwargs.items() if value is not None} - - payload["session_id"] = str(payload.get("session_id") or "") - payload["request_id"] = str(payload.get("request_id") or "") - payload["continue_on_error"] = bool(payload.get("continue_on_error", False)) - return payload - - -def register_builtin_actions(controller: ActionController | None = None) -> None: - ctl = controller or _DEFAULT_CONTROLLER - - async def ping(session_id: str = "", request_id: str = "", **kwargs: Any) -> ActionResult: - return { - "ok": True, - "pong": True, - "session_id": session_id, - "request_id": request_id, - "meta": kwargs, - } - - async def echo( - session_id: str = "", - request_id: str = "", - text: str = "", - **kwargs: Any, - ) -> ActionResult: - return { - "ok": True, - "text": text, - "session_id": session_id, - "request_id": request_id, - "meta": kwargs, - } - - async def browser_task( - session_id: str = "", - request_id: str = "", - task: str = "", - timeout_s: int | None = None, - **kwargs: Any, - ) -> ActionResult: - runner = ctl.runtime_runner - if runner is None: - return { - "ok": False, - "error": "runtime_not_bound: call bind_runtime(...) before browser_task", - "session_id": session_id, - "request_id": request_id, - } - - task_text = (task or "").strip() - if not task_text: - return { - "ok": False, - "error": "missing required parameter: task", - "session_id": session_id, - "request_id": request_id, - } - - effective_timeout: int | None = None - if timeout_s is not None: - try: - parsed = int(timeout_s) - if parsed > 0: - effective_timeout = parsed - except Exception: - effective_timeout = None - - return await runner( - task=task_text, - session_id=session_id or None, - request_id=request_id or None, - timeout_s=effective_timeout, - ) - - async def browser_get_element_coordinates( - session_id: str = "", - request_id: str = "", - url: str = "", - source: str = "", - target: str = "", - *, - element_source: str = "", - element_target: str = "", - element_source_offset: Any = None, - element_target_offset: Any = None, - source_x: Any = None, - source_y: Any = None, - target_x: Any = None, - target_y: Any = None, - coord_source_x: Any = None, - coord_source_y: Any = None, - coord_target_x: Any = None, - coord_target_y: Any = None, - timeout_s: int | None = None, - **kwargs: Any, - ) -> ActionResult: - del kwargs - payload = _build_drag_payload( - url=url, - source=source, - target=target, - element_source=element_source, - element_target=element_target, - element_source_offset=element_source_offset, - element_target_offset=element_target_offset, - source_x=source_x, - source_y=source_y, - target_x=target_x, - target_y=target_y, - coord_source_x=coord_source_x, - coord_source_y=coord_source_y, - coord_target_x=coord_target_x, - coord_target_y=coord_target_y, - ) - if not (_has_source_selector(payload) or _has_coordinate_inputs(payload)): - return { - "ok": False, - "error": ( - "Missing location inputs. Provide at least element_source (element_target is optional), or " - "coord_source_x/coord_source_y/coord_target_x/coord_target_y. " - "Aliases source/target and source_x/source_y/target_x/target_y are also supported." - ), - } - js_code = _build_coordinate_script(payload) - code_executor = ctl.code_executor - if code_executor is not None: - try: - raw = await code_executor(js_code) - except Exception as exc: - return { - "ok": False, - "error": f"browser_run_code failed: {exc}", - "session_id": session_id, - "request_id": request_id, - } - parsed = extract_json_object(raw) - if not parsed: - return { - "ok": False, - "error": "Could not parse coordinate result JSON from browser_run_code output", - "raw_preview": str(raw)[:400], - } - return { - "ok": bool(parsed.get("ok", False)), - "source": parsed.get("source"), - "target": parsed.get("target"), - "error": parsed.get("error"), - } - # Fallback: route through LLM worker (requires runner to be bound) - runner = ctl.runtime_runner - if runner is None: - return { - "ok": False, - "error": "runtime_not_bound: call bind_runtime(...) before browser_get_element_coordinates", - "session_id": session_id, - "request_id": request_id, - } - task_prompt = _build_run_code_task(js_code, "resolve source/target coordinates") - runtime_result = await runner( - task=task_prompt, - session_id=session_id or None, - request_id=request_id or None, - timeout_s=_normalize_timeout_s(timeout_s), - ) - if not runtime_result.get("ok", False): - return { - "ok": False, - "error": runtime_result.get("error") or "runtime error", - "runtime": runtime_result, - } - parsed = extract_json_object(runtime_result.get("final")) - if not parsed: - final_preview = str(runtime_result.get("final", ""))[:400] - return { - "ok": False, - "error": "Could not parse coordinate result JSON from runtime final output", - "final_preview": final_preview, - "runtime": runtime_result, - } - return { - "ok": bool(parsed.get("ok", False)), - "source": parsed.get("source"), - "target": parsed.get("target"), - "error": parsed.get("error"), - "runtime": runtime_result, - } - - async def list_upload_files( - session_id: str = "", - request_id: str = "", - **kwargs: Any, - ) -> ActionResult: - del kwargs - upload_root = resolve_upload_root() - if upload_root is None: - return { - "ok": False, - "error": ( - "BROWSER_UPLOAD_ROOT is not configured. " - "Set this env var to the directory where uploadable files are stored." - ), - "files": [], - "session_id": session_id, - "request_id": request_id, - } - if not upload_root.exists(): - return { - "ok": False, - "error": f"Upload root directory does not exist: {upload_root}", - "files": [], - "upload_root": str(upload_root), - "session_id": session_id, - "request_id": request_id, - } - files = _list_dir_files(upload_root) - return { - "ok": True, - "upload_root": str(upload_root), - "files": files, - "session_id": session_id, - "request_id": request_id, - } - - async def browser_drag_and_drop( - session_id: str = "", - request_id: str = "", - url: str = "", - source: str = "", - target: str = "", - *, - element_source: str = "", - element_target: str = "", - element_source_offset: Any = None, - element_target_offset: Any = None, - source_x: Any = None, - source_y: Any = None, - target_x: Any = None, - target_y: Any = None, - coord_source_x: Any = None, - coord_source_y: Any = None, - coord_target_x: Any = None, - coord_target_y: Any = None, - steps: Any = None, - delay_ms: Any = None, - timeout_s: int | None = None, - **kwargs: Any, - ) -> ActionResult: - del kwargs - payload = _build_drag_payload( - url=url, - source=source, - target=target, - element_source=element_source, - element_target=element_target, - element_source_offset=element_source_offset, - element_target_offset=element_target_offset, - source_x=source_x, - source_y=source_y, - target_x=target_x, - target_y=target_y, - coord_source_x=coord_source_x, - coord_source_y=coord_source_y, - coord_target_x=coord_target_x, - coord_target_y=coord_target_y, - steps=steps, - delay_ms=delay_ms, - ) - if not (_has_selector_inputs(payload) or _has_coordinate_inputs(payload)): - return { - "ok": False, - "error": ( - "Missing drag inputs. Provide either " - "element_source + element_target, or " - "coord_source_x/coord_source_y/coord_target_x/coord_target_y. " - "Aliases source/target and source_x/source_y/target_x/target_y are also supported." - ), - } - js_code = _build_drag_script(payload) - code_executor = ctl.code_executor - if code_executor is not None: - try: - raw = await code_executor(js_code) - except Exception as exc: - return { - "ok": False, - "error": f"browser_run_code failed: {exc}", - "session_id": session_id, - "request_id": request_id, - } - parsed = extract_json_object(raw) - if not parsed: - return { - "ok": False, - "error": "Could not parse drag result JSON from browser_run_code output", - "raw_preview": str(raw)[:400], - } - return { - "ok": bool(parsed.get("ok", False)), - "message": parsed.get("message"), - "source": parsed.get("source"), - "target": parsed.get("target"), - "steps": parsed.get("steps"), - "delay_ms": parsed.get("delay_ms"), - "error": parsed.get("error"), - } - # Fallback: route through LLM worker (requires runner to be bound) - runner = ctl.runtime_runner - if runner is None: - return { - "ok": False, - "error": "runtime_not_bound: call bind_runtime(...) before browser_drag_and_drop", - "session_id": session_id, - "request_id": request_id, - } - task_prompt = _build_run_code_task(js_code, "drag and drop") - runtime_result = await runner( - task=task_prompt, - session_id=session_id or None, - request_id=request_id or None, - timeout_s=_normalize_timeout_s(timeout_s), - ) - if not runtime_result.get("ok", False): - return { - "ok": False, - "error": runtime_result.get("error") or "runtime error", - "runtime": runtime_result, - } - parsed = extract_json_object(runtime_result.get("final")) - if not parsed: - final_preview = str(runtime_result.get("final", ""))[:400] - return { - "ok": False, - "error": "Could not parse drag result JSON from runtime final output", - "final_preview": final_preview, - "runtime": runtime_result, - } - return { - "ok": bool(parsed.get("ok", False)), - "message": parsed.get("message"), - "source": parsed.get("source"), - "target": parsed.get("target"), - "steps": parsed.get("steps"), - "delay_ms": parsed.get("delay_ms"), - "error": parsed.get("error"), - "runtime": runtime_result, - } - - async def browser_batch_interact(**kwargs: Any) -> ActionResult: - """Execute multiple deterministic browser interactions with one LLM-visible tool call.""" - payload_args = _normalize_batch_interact_payload(kwargs) - - session_id = payload_args["session_id"] - request_id = payload_args["request_id"] - steps = payload_args.get("steps") - timeout_ms = payload_args.get("timeout_ms") - condition_timeout_ms = payload_args.get("condition_timeout_ms") - wait_after_each_ms = payload_args.get("wait_after_each_ms") - continue_on_error = bool(payload_args.get("continue_on_error", False)) - global_timeout_ms = payload_args.get("global_timeout_ms") - generation_id = str(payload_args.get("generation_id") or "g0") - - validation_errors = validate_batch_steps(steps) - if validation_errors: - return { - "ok": False, - "status": "failed", - "error": f"batch_validation_failed: {validation_errors[0]}", - "validation_errors": validation_errors, - "session_id": session_id, - "request_id": request_id, - } - - original_step_count = len(steps) - max_steps = 25 - safe_steps = steps - try: - per_step_timeout = int(timeout_ms or 2500) - except (TypeError, ValueError): - per_step_timeout = 2500 - per_step_timeout = max(250, min(30000, per_step_timeout)) - - try: - condition_timeout = int(condition_timeout_ms or 10000) - except (TypeError, ValueError): - condition_timeout = 10000 - condition_timeout = max( - per_step_timeout, - min(30000, condition_timeout), - ) - - try: - after_each = int(wait_after_each_ms or 0) - except (TypeError, ValueError): - after_each = 0 - after_each = max(0, min(5000, after_each)) - - if global_timeout_ms is None: - condition_steps = 0 - for step in safe_steps: - op = str(step.get("op") or "").strip().lower() - if op in _BATCH_CONDITION_OPS: - condition_steps += 1 - action_steps = len(safe_steps) - condition_steps - computed_timeout_ms = ( - action_steps * (per_step_timeout + after_each + 250) - + condition_steps * (condition_timeout + after_each + 250) - + 5000 - ) - effective_global_timeout_ms = min( - 120000, - max(5000, computed_timeout_ms), - ) - else: - try: - effective_global_timeout_ms = int(global_timeout_ms) - except (TypeError, ValueError): - effective_global_timeout_ms = 120000 - effective_global_timeout_ms = max( - 1000, - min(180000, effective_global_timeout_ms), - ) - - payload = { - "steps": safe_steps, - "timeout_ms": per_step_timeout, - "condition_timeout_ms": condition_timeout, - "wait_after_each_ms": after_each, - "continue_on_error": continue_on_error, - "original_step_count": original_step_count, - "executed_step_limit": max_steps, - "generation_id": generation_id, - } - build_started_at = time.perf_counter() - js_code = _build_batch_interact_script(payload) - build_elapsed_ms = int(max(0.0, (time.perf_counter() - build_started_at) * 1000)) - script_size_bytes = len(js_code.encode("utf-8", "ignore")) - code_executor = ctl.code_executor - - browser_agent_log_info( - "[BROWSER_BATCH] start session_id=%s request_id=%s steps=%s " - "timeout_ms=%s condition_timeout_ms=%s global_timeout_ms=%s " - "script_size_bytes=%s", - session_id or "-", - request_id or "-", - len(safe_steps), - per_step_timeout, - condition_timeout, - effective_global_timeout_ms, - script_size_bytes, - ) - - if code_executor is None: - browser_agent_log_warning( - "[BROWSER_BATCH] end ok=false session_id=%s request_id=%s error=%s", - session_id or "-", - request_id or "-", - "browser_code_executor_not_ready", - ) - return { - "ok": False, - "status": "failed", - "error": "browser_code_executor_not_ready", - "session_id": session_id, - "request_id": request_id, - "steps_requested": len(safe_steps), - "original_step_count": original_step_count, - "executed_step_limit": max_steps, - } - - executor_started_at = time.perf_counter() - try: - raw = await asyncio.wait_for( - code_executor(js_code), - timeout=effective_global_timeout_ms / 1000, - ) - except asyncio.TimeoutError: - error = f"browser_batch_interact timed out after {effective_global_timeout_ms}ms" - browser_agent_log_warning( - "[BROWSER_BATCH] end ok=false session_id=%s request_id=%s error=%s", - session_id or "-", - request_id or "-", - error, - ) - return { - "ok": False, - "status": "failed", - "error": error, - "session_id": session_id, - "request_id": request_id, - "steps_requested": len(safe_steps), - "original_step_count": original_step_count, - "executed_step_limit": max_steps, - "metrics": { - "script_build_elapsed_ms": build_elapsed_ms, - "executor_elapsed_ms": int( - max( - 0.0, - (time.perf_counter() - executor_started_at) * 1000, - ) - ), - "script_size_bytes": script_size_bytes, - }, - } - except Exception as exc: - error = f"browser_run_code failed: {exc}" - browser_agent_log_warning( - "[BROWSER_BATCH] end ok=false session_id=%s request_id=%s error=%s", - session_id or "-", - request_id or "-", - error, - ) - return { - "ok": False, - "status": "failed", - "error": error, - "session_id": session_id, - "request_id": request_id, - "steps_requested": len(safe_steps), - "original_step_count": original_step_count, - "executed_step_limit": max_steps, - "metrics": { - "script_build_elapsed_ms": build_elapsed_ms, - "executor_elapsed_ms": int( - max( - 0.0, - (time.perf_counter() - executor_started_at) * 1000, - ) - ), - "script_size_bytes": script_size_bytes, - }, - } - - executor_elapsed_ms = int(max(0.0, (time.perf_counter() - executor_started_at) * 1000)) - response_size_bytes = len(str(raw).encode("utf-8", "ignore")) - rpc_metrics = _browser_rpc_metrics(raw) - parse_started_at = time.perf_counter() - unwrapped = _unwrap_browser_code_result(raw) - parsed = extract_json_object(unwrapped) - parse_elapsed_ms = int(max(0.0, (time.perf_counter() - parse_started_at) * 1000)) - if not parsed: - error = "Could not parse browser_batch_interact result JSON from browser_run_code output" - raw_text = str(unwrapped) - browser_agent_log_warning( - "[BROWSER_BATCH] end ok=false session_id=%s request_id=%s error=%s raw_preview=%r", - session_id or "-", - request_id or "-", - error, - raw_text[:200], - ) - return { - "ok": False, - "status": "failed", - "error": error, - "raw_preview": raw_text[:400], - "session_id": session_id, - "request_id": request_id, - "steps_requested": len(safe_steps), - "original_step_count": original_step_count, - "executed_step_limit": max_steps, - "metrics": { - "script_build_elapsed_ms": build_elapsed_ms, - "executor_elapsed_ms": executor_elapsed_ms, - "result_parse_elapsed_ms": parse_elapsed_ms, - "script_size_bytes": script_size_bytes, - "response_size_bytes": response_size_bytes, - **rpc_metrics, - }, - } - - parsed.setdefault("session_id", session_id) - parsed.setdefault("request_id", request_id) - parsed.setdefault("action", "browser_batch_interact") - parsed.setdefault("original_step_count", original_step_count) - parsed.setdefault("executed_step_limit", max_steps) - parsed.setdefault("generation_id", generation_id) - completed = parsed.get("steps") or parsed.get("completed_steps") or [] - completed_steps = completed if isinstance(completed, list) else [] - had_step_errors = any(isinstance(item, dict) and item.get("ok") is False for item in completed_steps) - if had_step_errors: - parsed["status"] = ( - "partial" - if any(isinstance(item, dict) and item.get("ok") is True for item in completed_steps) - else "failed" - ) - parsed["ok"] = False - if not parsed.get("error"): - parsed["error"] = "one or more batch steps failed" - elif parsed.get("ok") is False: - parsed.setdefault("status", "failed") - else: - parsed["status"] = "completed" - parsed["ok"] = True - parsed["metrics"] = { - "script_build_elapsed_ms": build_elapsed_ms, - "executor_elapsed_ms": executor_elapsed_ms, - "result_parse_elapsed_ms": parse_elapsed_ms, - "script_size_bytes": script_size_bytes, - "response_size_bytes": response_size_bytes, - "internal_steps_elapsed_ms": int(parsed.get("internal_steps_elapsed_ms") or 0), - "preflight_elapsed_ms": int(parsed.get("preflight_elapsed_ms") or 0), - "preflight_target_count": int(parsed.get("preflight_target_count") or 0), - **rpc_metrics, - } - browser_agent_log_info( - "[BROWSER_BATCH] end ok=%s session_id=%s request_id=%s elapsed_ms=%s " - "internal_steps_elapsed_ms=%s executor_elapsed_ms=%s " - "script_size_bytes=%s response_size_bytes=%s " - "transport_response_size_bytes=%s completed_steps=%s error=%s", - bool(parsed.get("ok", False)), - session_id or "-", - request_id or "-", - parsed.get("elapsed_ms"), - parsed["metrics"]["internal_steps_elapsed_ms"], - parsed["metrics"]["executor_elapsed_ms"], - parsed["metrics"]["script_size_bytes"], - parsed["metrics"]["response_size_bytes"], - parsed["metrics"].get("transport_response_size_bytes", 0), - len(completed) if isinstance(completed, list) else "?", - parsed.get("error") or "-", - ) - - for item in completed_steps: - if isinstance(item, dict): - browser_agent_log_info( - "[BROWSER_BATCH] step index=%s op=%s ok=%s elapsed_ms=%s error=%s", - item.get("index"), - item.get("op"), - bool(item.get("ok", False)), - item.get("elapsed_ms"), - item.get("error") or "-", - ) - - compact_steps = [] - for item in completed_steps: - compact = _compact_batch_step(item) - if compact is not None: - compact_steps.append(compact) - extracted = parsed.get("extracted") - field_provenance = _compact_extraction_provenance( - completed_steps, - generation_id, - ) - compact_result = { - "ok": bool(parsed.get("ok", False)), - "status": str(parsed.get("status") or "failed"), - "error": parsed.get("error"), - "action": "browser_batch_interact", - "execution_mode": "compact_rpc", - "session_id": session_id, - "request_id": request_id, - "generation_id": generation_id, - "steps": compact_steps, - "extracted": dict(extracted) if isinstance(extracted, Mapping) else {}, - "field_provenance": field_provenance, - "conditions": _compact_batch_conditions(parsed, completed_steps), - "metrics": parsed["metrics"], - "_runtime_page": { - "url": str(parsed.get("url") or ""), - "title": str(parsed.get("title") or ""), - }, - } - return compact_result - - async def browser_set_input_files( - session_id: str = "", - request_id: str = "", - selector: str = "", - paths: list | None = None, - **kwargs: Any, - ) -> ActionResult: - del kwargs - effective_selector = (selector or "").strip() or 'input[type="file"]' - effective_paths = [str(p) for p in (paths or []) if p] - if not effective_paths: - return { - "ok": False, - "error": "paths is required and must be non-empty", - "session_id": session_id, - "request_id": request_id, - } - js_code = _build_set_input_files_script(effective_selector, effective_paths) - code_executor = ctl.code_executor - if code_executor is not None: - try: - raw = await code_executor(js_code) - except Exception as exc: - return { - "ok": False, - "error": f"browser_run_code failed: {exc}", - "session_id": session_id, - "request_id": request_id, - } - parsed = extract_json_object(raw) - if not parsed: - return { - "ok": False, - "error": "Could not parse set_input_files result JSON from browser_run_code output", - "raw_preview": str(raw)[:400], - } - return { - "ok": bool(parsed.get("ok", False)), - "selector": parsed.get("selector", effective_selector), - "paths": parsed.get("paths", effective_paths), - "error": parsed.get("error"), - } - # Fallback: route through LLM worker (requires runner to be bound) - runner = ctl.runtime_runner - if runner is None: - return { - "ok": False, - "error": "runtime_not_bound: call bind_runtime(...) before browser_set_input_files", - "session_id": session_id, - "request_id": request_id, - } - task_prompt = _build_run_code_task(js_code, f"set input files on {effective_selector}") - runtime_result = await runner( - task=task_prompt, - session_id=session_id or None, - request_id=request_id or None, - timeout_s=None, - ) - if not runtime_result.get("ok", False): - return { - "ok": False, - "error": runtime_result.get("error") or "runtime error", - "runtime": runtime_result, - } - parsed = extract_json_object(runtime_result.get("final")) - if not parsed: - return { - "ok": False, - "error": "Could not parse set_input_files result JSON from runtime final output", - "raw_preview": str(runtime_result.get("final", ""))[:400], - "runtime": runtime_result, - } - return { - "ok": bool(parsed.get("ok", False)), - "selector": parsed.get("selector", effective_selector), - "paths": parsed.get("paths", effective_paths), - "error": parsed.get("error"), - "runtime": runtime_result, - } - - ctl.register_action("ping", ping, overwrite=True) - ctl.register_action_spec( - "ping", - summary="Health check action.", - when_to_use="Use to verify controller dispatch and session/request threading.", - ) - ctl.register_action("echo", echo, overwrite=True) - ctl.register_action_spec( - "echo", - summary="Echoes provided text and metadata.", - when_to_use="Use for debugging payload passthrough through browser_custom_action.", - params={"text": "string: text to echo back"}, - ) - ctl.register_action("browser_task", browser_task, overwrite=True) - ctl.register_action_spec( - "browser_task", - summary="Runs a free-form browser task through runtime.run_browser_task.", - when_to_use="Use for generic website tasks when no specialized custom action applies.", - params={ - "task": "string: required task prompt for the browser worker", - "timeout_s": "int: optional positive timeout override", - }, - ) - ctl.register_action("run_browser_task", browser_task, overwrite=True) - ctl.register_action_spec( - "run_browser_task", - summary="Alias of browser_task.", - when_to_use="Same behavior as browser_task.", - params={ - "task": "string: required task prompt for the browser worker", - "timeout_s": "int: optional positive timeout override", - }, - ) - ctl.register_action( - "browser_get_element_coordinates", - browser_get_element_coordinates, - overwrite=True, - ) - ctl.register_action_spec( - "browser_get_element_coordinates", - summary="Resolves source/target screen coordinates by selectors or explicit coordinates.", - when_to_use=( - "Use when you need coordinates for one element (element_source only) or two (source + target). " - "element_target is optional." - ), - params={ - "url": "string: optional URL to open before resolving coordinates", - "element_source": "string: source selector/text alias (required for selector mode)", - "element_target": "string: target selector/text alias (optional)", - "coord_source_x": "int: source x coordinate", - "coord_source_y": "int: source y coordinate", - "coord_target_x": "int: target x coordinate", - "coord_target_y": "int: target y coordinate", - "source/target": "string aliases for element_source/element_target", - "source_x/source_y/target_x/target_y": "int aliases for coord_* fields", - }, - ) - ctl.register_action( - "browser_drag_and_drop", - browser_drag_and_drop, - overwrite=True, - ) - ctl.register_action_spec( - "browser_drag_and_drop", - summary="Performs drag-and-drop using selectors or explicit coordinates.", - when_to_use=("Use for drag-and-drop tasks instead of generic browser_run_task text-only instructions."), - params={ - "url": "string: optional URL to open before drag-and-drop", - "element_source": "string: source selector/text alias", - "element_target": "string: target selector/text alias", - "coord_source_x": "int: source x coordinate", - "coord_source_y": "int: source y coordinate", - "coord_target_x": "int: target x coordinate", - "coord_target_y": "int: target y coordinate", - "steps": "int: optional drag interpolation steps", - "delay_ms": "int: optional delay between drag steps", - "source/target": "string aliases for element_source/element_target", - "source_x/source_y/target_x/target_y": "int aliases for coord_* fields", - }, - ) - - ctl.register_action( - "browser_batch_interact", - browser_batch_interact, - overwrite=True, - ) - ctl.register_action_spec( - "browser_batch_interact", - summary=( - "Executes multiple deterministic page interactions in one Playwright code call " - "and returns compact per-step status." - ), - when_to_use=( - "Use for multi-field forms with several known controls, search boxes with autocomplete, " - "dropdown/date-picker flows, filter panels, and short click/type/wait/extract sequences. " - "Prefer this over many separate browser_click, browser_type, condition-wait, and " - "browser_evaluate turns when the next actions are already known. Do not use it for a single " - "uncertain click or when the page state must be inspected first." - ), - params={ - "steps": ( - "list[object]: required, 1-25 items. Each step has op plus target fields. Supported op values: " - "click, fill, type, autocomplete, select_visible_text, press, select_option, set_checked, " - "wait_for_selector, wait_for_text, wait_for_load_state, wait_for_url, " - "wait_for_first_card_title, wait_for_sort_state, wait_for_result_count, " - "wait_for_dom_text_change, wait_for_stable, wait_for_tab, extract_text, extract_value, " - "screenshot. Targets may use selector, role+name, label, placeholder, testid, or text. " - "For fill/type/autocomplete use value. For autocomplete also use choose_text/option_text " - "or option_selector/choose_selector. For native select_option, use value/option_value, " - "option_text/option_label/label_value, index, or values for MCP-style multi/single selection." - ), - "timeout_ms": "int: default timeout per step, default 2500, max 30000", - "condition_timeout_ms": ("int: default timeout for condition waits, default 10000, max 30000"), - "wait_after_each_ms": ("int: optional pause after successful steps, max 5000; prefer condition wait ops"), - "continue_on_error": "bool: when true, continue after failed optional/non-critical steps", - "global_timeout_ms": ( - "int: hard ceiling for the full batch, default derived from step count " - "and capped at 120000, explicit max 180000" - ), - }, - ) - ctl.register_action( - "browser_set_input_files", - browser_set_input_files, - overwrite=True, - ) - ctl.register_action_spec( - "browser_set_input_files", - summary=( - "Sets files on an <input type='file'> element. Requires prior page inspection" - " to select the correct input — do not call this without first reading the page snapshot." - ), - when_to_use=( - "Use for all file upload tasks. Does NOT require a file chooser dialog — sets files directly " - "on the DOM element. Call list_upload_files first to get absolute paths, then call this action. " - "IMPORTANT: pages may have multiple file inputs (e.g. a visible input plus a hidden Dropzone input). " - "Always prefer a specific selector such as '#file-upload' or 'input#id' over the generic " - "'input[type=\"file\"]' to avoid strict mode violations. If you have not yet inspected the page, " - "delegate this action to browser_run_task so the worker can read the page snapshot first." - ), - params={ - "selector": ( - "string: CSS/Playwright selector targeting exactly one file input " - "(default: 'input[type=\"file\"]'). Use a specific ID selector like '#file-upload' " - "when the page has more than one file input element." - ), - "paths": ( - "list[string]: absolute file paths to set on the input (required, non-empty). " - "Parameter name is 'paths' — not 'files', not 'file_paths'." - ), - }, - ) - ctl.register_action( - "list_upload_files", - list_upload_files, - overwrite=True, - ) - ctl.register_action_spec( - "list_upload_files", - summary="Lists files available for upload from the configured BROWSER_UPLOAD_ROOT directory.", - when_to_use=( - "Call this to discover what files are available and get their exact absolute paths " - "before calling browser_set_input_files to attach them to a file input. " - "Returns a list of {name, path, size_bytes} entries." - ), - ) - - -def register_example_actions(controller: ActionController | None = None) -> None: - register_builtin_actions(controller=controller) - - -__all__ = [ - "BaseController", - "ActionController", - "get_default_controller", - "bind_runtime", - "bind_runtime_runner", - "clear_runtime_runner", - "bind_code_executor", - "clear_code_executor", - "register_action", - "register_action_spec", - "register_builtin_actions", - "register_example_actions", - "browser_worker_action_context", - "list_actions", - "describe_actions", - "run_action", -] - -sys.modules.setdefault("controllers.action", sys.modules[__name__]) diff --git a/openjiuwen/harness/tools/browser_move/controllers/base.py b/openjiuwen/harness/tools/browser_move/controllers/base.py deleted file mode 100644 index 141d72537f..0000000000 --- a/openjiuwen/harness/tools/browser_move/controllers/base.py +++ /dev/null @@ -1,69 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Base controller contract for action dispatchers.""" - -from __future__ import annotations - -from abc import ABC, abstractmethod -from typing import Any - - -class BaseController(ABC): - """Abstract contract for controller implementations.""" - - @abstractmethod - def bind_runtime(self, runtime: Any) -> None: - """Bind a runtime object used by runtime-backed actions.""" - - @abstractmethod - def bind_runtime_runner(self, runner: Any | None) -> None: - """Bind a runtime runner callable.""" - - @abstractmethod - def clear_runtime_runner(self) -> None: - """Clear any currently bound runtime runner.""" - - @abstractmethod - def bind_code_executor(self, executor: Any | None) -> None: - """Bind a direct code executor callable: async (js_code: str) -> Any. - - When bound, browser actions that execute pre-built JS (drag-and-drop, - coordinate resolution) call this instead of going through an LLM worker. - """ - - @abstractmethod - def clear_code_executor(self) -> None: - """Clear the bound code executor.""" - - @abstractmethod - def register_action(self, name: str, handler: Any, *, overwrite: bool = True) -> None: - """Register an action handler.""" - - @abstractmethod - def register_action_spec( - self, - name: str, - *, - summary: str = "", - when_to_use: str = "", - params: dict[str, str] | None = None, - ) -> None: - """Register metadata for an action.""" - - @abstractmethod - def list_actions(self) -> list[str]: - """List registered action names.""" - - @abstractmethod - def describe_actions(self) -> dict[str, dict[str, Any]]: - """Return metadata for registered actions.""" - - @abstractmethod - async def run_action( - self, - action: str, - session_id: str = "", - request_id: str = "", - **kwargs: Any, - ) -> dict[str, Any]: - """Execute a registered action.""" diff --git a/openjiuwen/harness/tools/browser_move/drivers/__init__.py b/openjiuwen/harness/tools/browser_move/drivers/__init__.py index 285ed342d6..48f0134b79 100644 --- a/openjiuwen/harness/tools/browser_move/drivers/__init__.py +++ b/openjiuwen/harness/tools/browser_move/drivers/__init__.py @@ -1,2 +1,19 @@ # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Built-in drivers, registered by name with lazy import strings.""" + +from ..contract.registry import DRIVERS, register_driver + +register_driver( + "browser_use", + f"{__name__}.browser_use.driver:create_browser_use_driver", + default_topology="delegated_worker", +) +register_driver( + "playwright_mcp", + f"{__name__}.playwright_mcp.driver:create_playwright_mcp_driver", + default_topology="delegated_worker", +) + +__all__ = ["DRIVERS"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/__init__.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/__init__.py new file mode 100644 index 0000000000..285ed342d6 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/__init__.py @@ -0,0 +1,2 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/driver.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/driver.py new file mode 100644 index 0000000000..89bebebcdc --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/driver.py @@ -0,0 +1,590 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""``BrowserUseDriver``: the driver contract on top of the browser-use sidecar; imports no ``browser_use``.""" + +from __future__ import annotations + +import asyncio +import base64 +import dataclasses +import re +import struct +from collections.abc import Sequence +from dataclasses import dataclass, replace +from importlib import resources +from pathlib import Path +from types import MappingProxyType +from typing import Any, Mapping + +from openjiuwen.core.common.logging import logger as _logger +from openjiuwen.harness.tools.browser_move.contract import ( + ActionIntercepted, + ActionResult, + ActionTimeout, + AmbiguousElement, + Click, + CloseContext, + ConnectionLost, + ContextRef, + Drag, + DriverError, + DriverInfo, + DriverRef, + Element, + ElementNotFound, + HandleDialog, + HandlerDriver, + Health, + Hover, + Locator, + NavigationError, + Observation, + Observe, + PressKey, + ProvisionedDevice, + RunScript, + Screenshot, + Scroll, + StaleRef, + SwitchContext, + TypeText, + Unsupported, + Wait, + handles, +) +from openjiuwen.harness.tools.browser_move.drivers.browser_use.transport import SidecarTransport +from openjiuwen.harness.tools.browser_move.drivers.browser_use.upload_paths import resolve_upload_file_paths +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FieldKind, + FillForm, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext + +_CONTEXT_KIND = "tab" +_PROBES = "openjiuwen.harness.tools.browser_move.surfaces.browser.probes" +_TARGET_ATTRIBUTE = "data-openjiuwen-target-id" +_SCOPED_SCRIPT = ( + "(input) => {{ const element = document.querySelector(input.selector); " + "if (!element) {{ throw new Error('target element not found: ' + input.selector); }} " + "const value = ({code}\n); return typeof value === 'function' ? value(element, input.args) : value; }}" +) +_HAS_TEXT_JS = """(text) => { + const body = (document.body && document.body.innerText) ? document.body.innerText : ''; + return body.includes(String(text || '')); +}""" +_DOCUMENT_TITLE_JS = "() => document.title" +_WAIT_TIMEOUT_S = 10.0 +_WAIT_POLL_S = 0.1 +_CHECKED_VALUES = frozenset({"1", "true", "yes", "on", "checked"}) +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + +_SIDECAR_ERRORS: dict[str, tuple[type[DriverError], bool]] = { + "StaleIndexError": (StaleRef, False), + "ElementNotFound": (ElementNotFound, False), + "StaleNodeError": (StaleRef, False), + "AmbiguousElement": (AmbiguousElement, True), + "ActionTimeout": (ActionTimeout, True), + "NavigationError": (NavigationError, True), + "DriverNotConnected": (ConnectionLost, False), + "DriverConnectionError": (ConnectionLost, True), + "DriverConnectionLost": (ConnectionLost, True), + "DriverUnsupported": (Unsupported, False), + "ActionIntercepted": (ActionIntercepted, False), +} +_DISCONNECTED = re.compile( + r"\bnot connected\b|Reconnection wait timed out|No valid agent focus available|target may have detached", + re.IGNORECASE, +) + + +_HUNG_AFTER_TIMEOUTS = 2 +_PLAYWRIGHT_SELECTOR = re.compile( + r":(?:has-text|text|text-is|text-matches|visible|nth-match|left-of|right-of|above|below|near)\b|>>" + r"|^\s*(?:text|xpath|css|id|role|data-testid|internal:[a-z-]+)\s*=", + re.IGNORECASE, +) + + +@dataclass(frozen=True) +class BrowserUseConfig: + """What ``start()`` needs: the CDP endpoint of the browser and the connect timeout.""" + + cdp_url: str + timeout_s: float = 30.0 + sidecar_python: Path | None = None + upload_root: Path | None = None + environ: Mapping[str, str] = dataclasses.field(default_factory=dict, repr=False) + + +def _clipboard_guard() -> str: + return resources.files(_PROBES).joinpath("clipboard_guard.js").read_text(encoding="utf-8") + + +def _translate_error(method: str, exc: Exception) -> DriverError: + code = type(exc).__name__ + known = _SIDECAR_ERRORS.get(code) + if known is None and _DISCONNECTED.search(str(exc)): + return ConnectionLost(f"{method}: {exc}", dispatched=True) + if known is None: + _logger.warning("browser-use sidecar error on %s not classified: %s: %s", method, code, exc) + return DriverError(f"{method}: {exc}", dispatched=True) + error_type, dispatched = known + if error_type is ActionIntercepted: + return ActionIntercepted(str(exc), by=str(getattr(exc, "by", "")), dispatched=dispatched) + return error_type(str(exc), dispatched=dispatched) + + +def _ref_to_wire(ref: DriverRef | Locator, driver_generation: int) -> dict[str, Any]: + if isinstance(ref, Locator): + if ref.text is not None or not ref.css: + raise Unsupported("BrowserUseDriver addresses a Locator by css only", dispatched=False) + if _PLAYWRIGHT_SELECTOR.search(ref.css): + raise Unsupported( + f"BrowserUseDriver takes plain CSS selectors only, not Playwright selectors ({ref.css!r}): use a ref " + "or target_id from a snapshot or probe, or CSS such as #id, .class, tag[attr='value'], " + "a:nth-of-type(2); match text with a find or probe instead of :has-text() or text=", + dispatched=False, + ) + return {"kind": "selector", "css": ref.css, "nth": 0} + return {"kind": "driver", "handle": ref.ref, "driver_generation": driver_generation} + + +def _context_to_wire(context: ContextRef) -> dict[str, Any]: + return {"target_id": context.id, "url": "", "title": context.title or "", "active": False} + + +def _context_from_wire(data: dict[str, Any]) -> ContextRef: + return ContextRef(id=str(data["target_id"]), kind=_CONTEXT_KIND, title=str(data.get("title", ""))) + + +def _element_from_wire(data: dict[str, Any]) -> Element: + box = data.get("box") + value = data.get("value") + return Element( + ref=DriverRef(ref=str(data["driver_ref"]["handle"]), generation=""), + role=str(data.get("role") or ""), + name=str(data.get("name") or ""), + value=None if value is None else str(value), + bounds=None if box is None else (float(box["x"]), float(box["y"]), float(box["width"]), float(box["height"])), + extra={ + "index": int(data["index"]), + "tag": str(data.get("tag", "")), + "backend_node_id": data.get("backend_node_id"), + "frame_id": data.get("frame_id"), + "attributes": dict(data.get("attributes") or {}), + "visible": bool(data.get("visible", False)), + "scrollable": bool(data.get("scrollable", False)), + }, + ) + + +def _dialog_text(dialog: dict[str, Any] | None) -> str | None: + return None if dialog is None else f"{dialog.get('type') or 'dialog'}: {dialog.get('message', '')}" + + +def _screenshot_from_b64(data: str) -> Screenshot: + png = base64.b64decode(data) + width, height = struct.unpack(">II", png[16:24]) if png[:8] == _PNG_SIGNATURE and len(png) >= 24 else (0, 0) + return Screenshot(png=png, width=width, height=height) + + +def _tab_fields(raw_tabs: Sequence[dict[str, Any]]) -> dict[str, Any]: + active = [index for index, raw in enumerate(raw_tabs) if raw.get("active")] + return { + "tabs": tuple(_context_from_wire(tab) for tab in raw_tabs), + "tab_urls": tuple(str(tab.get("url", "")) for tab in raw_tabs), + "current_tab": active[0] if active else None, + } + + +def _observation_from_wire(data: dict[str, Any], screenshot_b64: str | None) -> Observation: + tab_fields = _tab_fields(data.get("tabs") or ()) + title = str(data.get("title", "")) + current = tab_fields["current_tab"] + focused = ContextRef(id="", kind=_CONTEXT_KIND, title=title) if current is None else tab_fields["tabs"][current] + return Observation( + context=focused, + elements=tuple(_element_from_wire(element) for element in data.get("elements") or ()), + screenshot=None if not screenshot_b64 else _screenshot_from_b64(screenshot_b64), + native_text=data.get("ax_text"), + surface=BrowserContext( + url=str(data["url"]), + title=title, + **tab_fields, + is_pdf_viewer=bool(data.get("is_pdf_viewer", False)), + pixels_above=int(data.get("pixels_above", 0)), + pixels_below=int(data.get("pixels_below", 0)), + dialog=_dialog_text(data.get("dialog")), + ), + ) + + +def _scroll_to_wire(dx: int, dy: int) -> tuple[str, int]: + if bool(dx) == bool(dy): + raise Unsupported("BrowserUseDriver scrolls along exactly one axis per command", dispatched=False) + if dy: + return ("down", dy) if dy > 0 else ("up", -dy) + return ("right", dx) if dx > 0 else ("left", -dx) + + +class BrowserUseDriver(HandlerDriver): + """Runs contract commands on a Chrome through the browser-use sidecar process.""" + + def __init__(self, config: BrowserUseConfig | None = None) -> None: + self._config = config + self._transport: SidecarTransport | None = None + self._driver_generation = 0 + self._versions: dict[str, str] = {} + self._timeouts = 0 + + def describe(self) -> DriverInfo: + return DriverInfo( + name="browser_use", + surfaces=frozenset({"browser"}), + versions=MappingProxyType(dict(self._versions)), + observes=frozenset({"elements", "screenshot", "native_text", "surface"}), + display_name="browser-use", + ) + + async def start(self, config: BrowserUseConfig | None = None) -> None: + config = config or self._config + if config is None: + raise DriverError("BrowserUseDriver.start() needs a BrowserUseConfig", dispatched=False) + if self._transport is not None: + await self.stop() + transport = SidecarTransport( + sidecar_python=config.sidecar_python, upload_root=config.upload_root, environ=config.environ + ) + try: + await transport.start(timeout_s=config.timeout_s) + result = await transport.request( + "connect", + {"cdp_url": config.cdp_url, "timeout_s": config.timeout_s, "init_script": _clipboard_guard()}, + timeout_s=config.timeout_s, + ) + except Exception as exc: + await transport.close() + raise _translate_error("connect", exc) from exc + self._config = config + self._transport = transport + self._timeouts = 0 + self._versions = { + "browser": str(result.get("browser_version", "")), + "protocol": str(result.get("protocol_version", "")), + "browser_use": str(result.get("backend_version", "")), + } + + async def stop(self) -> None: + transport, self._transport = self._transport, None + if transport is not None: + await transport.close() + + async def health(self) -> Health: + if self._transport is None: + return Health(ok=False, detail="not started") + try: + result = await self._transport.request("health", {}) + except Exception as exc: # noqa: BLE001 + return Health(ok=False, detail=str(exc)) + return Health(ok=bool(result.get("connected", False)), detail=result.get("error")) + + async def observe(self, command: Observe) -> Observation: + if command.scoped: + raise Unsupported("BrowserUseDriver observes the whole page only", dispatched=False) + viewport_screenshot = command.include_screenshot and not command.full_page + data = await self._request( + "observe", {"include_dom": True, "include_screenshot": viewport_screenshot, "cached": False} + ) + self._driver_generation = int(data.get("driver_generation", self._driver_generation)) + screenshot_b64 = data.get("screenshot_b64") + if data.get("dialog") is None: + try: + data = {**data, "title": str(await self._evaluate(_DOCUMENT_TITLE_JS, None))} + except DriverError: + pass + if command.include_screenshot and command.full_page: + screenshot_b64 = await self._request("screenshot", {"full_page": True, "clip": None}) + return _observation_from_wire(data, screenshot_b64) + + @handles(Click, refs={DriverRef, Locator}) + async def _click(self, action: Click) -> ActionResult: + data = await self._request( + "click", + { + "ref": self._ref(action.target), + "button": action.button, + "click_count": action.count, + "modifiers": list(action.modifiers), + }, + ) + return self._act_result(data) + + @handles(TypeText, refs={DriverRef, Locator}) + async def _type_text(self, action: TypeText) -> ActionResult: + if action.target is None: + raise Unsupported("BrowserUseDriver types only into a target element", dispatched=False) + data = await self._request( + "type_text", + { + "ref": self._ref(action.target), + "text": action.text, + "clear": action.clear, + "press_enter": action.submit, + "sensitive": action.sensitive, + "slowly": action.slowly, + }, + ) + return self._act_result(data) + + @handles(PressKey) + async def _press_key(self, action: PressKey) -> ActionResult: + return self._act_result(await self._request("press_key", {"keys": action.key})) + + @handles(Scroll, refs={DriverRef}) + async def _scroll(self, action: Scroll) -> ActionResult: + direction, amount = _scroll_to_wire(action.dx, action.dy) + ref = None if action.target is None else self._ref(action.target) + return self._act_result(await self._request("scroll", {"direction": direction, "amount": amount, "ref": ref})) + + @handles(Hover, refs={DriverRef, Locator}) + async def _hover(self, action: Hover) -> ActionResult: + return self._act_result(await self._request("hover", {"ref": self._ref(action.target)})) + + @handles(Drag, refs={DriverRef, Locator}) + async def _drag(self, action: Drag) -> ActionResult: + data = await self._request( + "drag", + {"source": self._ref(action.source), "target": self._ref(action.destination), "steps": 10, "delay_ms": 0}, + ) + return self._act_result(data) + + @handles(SelectOption, refs={DriverRef, Locator}) + async def _select_option(self, action: SelectOption) -> ActionResult: + if len(action.values) != 1: + raise Unsupported("BrowserUseDriver selects exactly one option per command", dispatched=False) + data = await self._request( + "select_option", {"ref": self._ref(action.target), "value": action.values[0], "label": None} + ) + return self._act_result(data) + + @handles(UploadFile, refs={DriverRef, Locator}) + async def _upload_file(self, action: UploadFile) -> ActionResult: + if action.target is None: + raise Unsupported("BrowserUseDriver needs the file input as target", dispatched=False) + existing, path_error = resolve_upload_file_paths(action.paths, upload_root=self._upload_root()) + if path_error: + return ActionResult(ok=False, detail=path_error) + data = await self._request("upload_files", {"ref": self._ref(action.target), "paths": existing}) + return self._act_result(data) + + @handles(Drop, refs={DriverRef, Locator}) + async def _drop(self, action: Drop) -> ActionResult: + existing: list[str] = [] + if action.paths: + existing, path_error = resolve_upload_file_paths(action.paths, upload_root=self._upload_root()) + if path_error: + return ActionResult(ok=False, detail=path_error) + data = await self._request( + "drop", + { + "ref": self._ref(action.target), + "paths": existing, + "data": [{"mimeType": mime, "data": value} for mime, value in action.data], + }, + ) + return self._act_result(data) + + @handles(FillForm, refs={DriverRef, Locator}) + async def _fill_form(self, action: FillForm) -> ActionResult: + context_changed = False + refs = [self._ref(field.target) for field in action.fields] + for index, (field, ref) in enumerate(zip(action.fields, refs)): + if field.kind in (FieldKind.CHECKBOX, FieldKind.RADIO): + method = "set_checked" + params = {"ref": ref, "checked": field.value.strip().lower() in _CHECKED_VALUES} + elif field.kind is FieldKind.COMBOBOX: + method, params = "select_option", {"ref": ref, "value": field.value, "label": None} + else: + method = "type_text" + params = { + "ref": ref, + "text": field.value, + "clear": True, + "press_enter": False, + "sensitive": field.sensitive, + } + try: + data = await self._request(method, params) + except DriverError as exc: + if index: + exc.dispatched = True + raise + step = self._act_result(data) + context_changed = context_changed or step.context_changed + if not step.ok: + return ActionResult( + ok=False, detail=f"fields[{index}] failed: {step.detail}", context_changed=context_changed + ) + return ActionResult(ok=True, context_changed=context_changed) + + @handles(HandleDialog) + async def _handle_dialog(self, action: HandleDialog) -> ActionResult: + data = await self._request("handle_dialog", {"accept": action.accept, "prompt_text": action.prompt_text}) + return self._act_result(data) + + @handles(Wait) + async def _wait(self, action: Wait) -> ActionResult: + if action.until_text is None: + await asyncio.sleep(action.seconds) + return ActionResult(ok=True) + timeout_s = _WAIT_TIMEOUT_S if action.seconds is None else action.seconds + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout_s + while not await self._evaluate(_HAS_TEXT_JS, action.until_text): + if loop.time() >= deadline: + raise ActionTimeout(f"text {action.until_text!r} did not appear within {timeout_s}s") + await asyncio.sleep(_WAIT_POLL_S) + return ActionResult(ok=True) + + @handles(RunScript, refs={DriverRef}) + async def _run_script(self, action: RunScript) -> ActionResult: + if action.target is None: + return ActionResult(ok=True, value=await self._evaluate(action.code, action.args)) + selector = await self._request( + "stamp", {"ref": self._ref(action.target), "attribute": _TARGET_ATTRIBUTE, "value": action.target.ref} + ) + value = await self._evaluate( + _SCOPED_SCRIPT.format(code=action.code), {"selector": str(selector), "args": action.args} + ) + return ActionResult(ok=True, value=value) + + @handles(Navigate) + async def _navigate(self, action: Navigate) -> ActionResult: + return self._nav_result(await self._navigate_wire(action.url, new_tab=False)) + + @handles(Back) + async def _back(self, action: Back) -> ActionResult: + del action + return self._nav_result(await self._request("go_back", {})) + + @handles(OpenContext) + async def _open_context(self, action: OpenContext) -> ActionResult: + before = {str(tab["target_id"]) for tab in await self._request("list_tabs", {})} + data = await self._navigate_wire(action.url or "about:blank", new_tab=True) + after = await self._request("list_tabs", {}) + opened = tuple(_context_from_wire(tab) for tab in after if str(tab["target_id"]) not in before) + return self._nav_result(data, raw_tabs=after, value=opened[0] if opened else None, opened_contexts=opened) + + @handles(SwitchContext) + async def _switch_context(self, action: SwitchContext) -> ActionResult: + data = await self._request("switch_tab", {"tab": _context_to_wire(action.context)}) + return await self._tab_result(data) + + @handles(CloseContext) + async def _close_context(self, action: CloseContext) -> ActionResult: + data = await self._request("close_tab", {"tab": _context_to_wire(action.context)}) + return await self._tab_result(data, source_context=action.context) + + async def _request(self, method: str, params: dict[str, Any]) -> Any: + if self._transport is None: + raise ConnectionLost("BrowserUseDriver is not started", dispatched=False) + try: + result = await self._transport.request(method, params) + except Exception as exc: + error = _translate_error(method, exc) + self._timeouts = self._timeouts + 1 if isinstance(error, ActionTimeout) else 0 + if self._timeouts >= _HUNG_AFTER_TIMEOUTS: + self._timeouts = 0 + raise ConnectionLost(f"{method}: {exc}; the page stopped answering", dispatched=True) from exc + raise error from exc + self._timeouts = 0 + return result + + async def _evaluate(self, source: str, args: Any) -> Any: + return await self._request( + "evaluate", {"source": source, "args": args, "await_promise": True, "return_by_value": True} + ) + + async def _navigate_wire(self, url: str, *, new_tab: bool) -> dict[str, Any]: + return await self._request( + "navigate", {"url": url, "wait_until": "load", "timeout_ms": None, "new_tab": new_tab} + ) + + def _upload_root(self) -> Path | None: + return self._config.upload_root if self._config is not None else None + + def _ref(self, ref: DriverRef | Locator) -> dict[str, Any]: + return _ref_to_wire(ref, self._driver_generation) + + def _act_result(self, data: dict[str, Any], *, context_changed: bool = False, **fields: Any) -> ActionResult: + self._driver_generation = int(data.get("driver_generation", self._driver_generation)) + detail = data.get("detail") or None + dialog = data.get("dialog_opened") + if dialog is not None: + note = f"{dialog.get('type') or 'dialog'} dialog opened: {dialog.get('message', '')!r}" + detail = f"{detail}; {note}" if detail else note + opened = tuple(_context_from_wire(target) for target in data.get("new_targets") or ()) + open_dialog = _dialog_text(data.get("dialog")) + return ActionResult( + ok=bool(data["ok"]), + detail=detail, + context_changed=( + context_changed or bool(data.get("document_changed", False)) or dialog is not None or bool(opened) + ), + opened_contexts=opened, + surface=None if open_dialog is None else BrowserContext(url=str(data.get("url", "")), dialog=open_dialog), + **fields, + ) + + async def _tab_result(self, data: dict[str, Any], **fields: Any) -> ActionResult: + result = self._act_result(data, context_changed=True, **fields) + try: + raw_tabs = await self._request("list_tabs", {}) + except DriverError as exc: + exc.dispatched = True + raise + title = next((str(tab.get("title", "")) for tab in raw_tabs if tab.get("active")), "") + surface = BrowserContext( + url=str(data.get("url", "")), title=title, **_tab_fields(raw_tabs), dialog=_dialog_text(data.get("dialog")) + ) + return replace(result, surface=surface) + + def _nav_result( + self, data: dict[str, Any], *, raw_tabs: Sequence[dict[str, Any]] = (), **fields: Any + ) -> ActionResult: + self._driver_generation = int(data.get("driver_generation", self._driver_generation)) + return ActionResult( + ok=True, + context_changed=True, + surface=BrowserContext( + url=str(data["url"]), + title=str(data.get("title", "")), + **_tab_fields(raw_tabs), + dialog=_dialog_text(data.get("dialog")), + ), + **fields, + ) + + +def create_browser_use_driver(device: ProvisionedDevice, surface_settings: Any) -> BrowserUseDriver: + """Driver factory (registry signature): the driver connects to ``device.endpoint`` on ``start()``.""" + if surface_settings is None: + return BrowserUseDriver(BrowserUseConfig(cdp_url=device.endpoint)) + return BrowserUseDriver( + BrowserUseConfig( + cdp_url=device.endpoint, + sidecar_python=surface_settings.sidecar_python, + upload_root=surface_settings.upload_root, + environ=surface_settings.process_env, + ) + ) + + +__all__ = ["BrowserUseConfig", "BrowserUseDriver", "create_browser_use_driver"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/__init__.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/__init__.py new file mode 100644 index 0000000000..285ed342d6 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/__init__.py @@ -0,0 +1,2 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/cdp.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/cdp.py new file mode 100644 index 0000000000..8b5373acca --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/cdp.py @@ -0,0 +1,584 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Raw CDP helpers for operations browser-use has no high-level API for; stdlib only.""" + +from __future__ import annotations + +import asyncio +import json +from typing import Any + +import exceptions + + +async def get_document_root_node_id(cdp_client: Any, session_id: str) -> int: + """Return the root node id of the current document.""" + result = await cdp_client.send.DOM.getDocument(params={"depth": 0}, session_id=session_id) + return int(result["root"]["nodeId"]) + + +async def query_selector_node_id(cdp_client: Any, session_id: str, selector: str) -> int | None: + """Return the CDP nodeId matching ``selector``, or None if not found.""" + root_node_id = await get_document_root_node_id(cdp_client, session_id) + result = await cdp_client.send.DOM.querySelector( + params={"nodeId": root_node_id, "selector": selector}, session_id=session_id + ) + node_id = int(result.get("nodeId", 0) or 0) + return node_id or None + + +async def describe_node_backend_id(cdp_client: Any, session_id: str, node_id: int) -> int: + """Resolve a CDP nodeId to its stable backendNodeId.""" + result = await cdp_client.send.DOM.describeNode(params={"nodeId": node_id}, session_id=session_id) + return int(result["node"]["backendNodeId"]) + + +_STALE_NODE_MESSAGES = ("does not belong to the document", "No node with given id") + + +async def resolve_object_id_by_backend_node(cdp_client: Any, session_id: str, backend_node_id: int) -> str: + """Resolve a backendNodeId to a Runtime objectId; raises ``StaleNodeError`` when it is gone.""" + try: + result = await cdp_client.send.DOM.resolveNode(params={"backendNodeId": backend_node_id}, session_id=session_id) + except Exception as exc: + if any(text in str(exc) for text in _STALE_NODE_MESSAGES): + message = f"backend node {backend_node_id} is not in the current document: {exc}" + raise exceptions.StaleNodeError(message) from exc + raise + object_id = (result.get("object") or {}).get("objectId") + if not object_id: + raise exceptions.StaleNodeError(f"backend node {backend_node_id} no longer resolves") + return str(object_id) + + +async def call_function_on_backend_node( + cdp_client: Any, + session_id: str, + backend_node_id: int, + function_declaration: str, + *, + arguments: list[dict[str, Any]] | None = None, + await_promise: bool = True, + return_by_value: bool = True, +) -> Any: + """Call ``function_declaration`` with ``this`` bound to the resolved node.""" + object_id = await resolve_object_id_by_backend_node(cdp_client, session_id, backend_node_id) + result = await cdp_client.send.Runtime.callFunctionOn( + params={ + "functionDeclaration": function_declaration, + "objectId": object_id, + "arguments": arguments or [], + "awaitPromise": await_promise, + "returnByValue": return_by_value, + }, + session_id=session_id, + ) + return _unwrap_js_result(result) + + +async def evaluate_expression( + cdp_client: Any, + session_id: str, + function_source: str, + args: Any = None, + *, + await_promise: bool = True, + return_by_value: bool = True, +) -> Any: + """Call a function expression with ``args``, or return any other expression's value.""" + expression = ( + f"(() => {{ const value = ({function_source}\n); " + f"return typeof value === 'function' ? value({json.dumps(args)}) : value; }})()" + ) + result = await cdp_client.send.Runtime.evaluate( + params={ + "expression": expression, + "awaitPromise": await_promise, + "returnByValue": return_by_value, + }, + session_id=session_id, + ) + return _unwrap_js_result(result) + + +async def dispatch_mouse_click( + cdp_client: Any, + session_id: str, + *, + x: float, + y: float, + button: str = "left", + click_count: int = 1, +) -> None: + """Dispatch a synthetic mouse press+release at page coordinates.""" + base = {"x": x, "y": y, "button": button, "clickCount": click_count, "modifiers": 0} + await cdp_client.send.Input.dispatchMouseEvent(params={**base, "type": "mousePressed"}, session_id=session_id) + await cdp_client.send.Input.dispatchMouseEvent(params={**base, "type": "mouseReleased"}, session_id=session_id) + + +async def dispatch_mouse_move( + cdp_client: Any, + session_id: str, + *, + x: float, + y: float, + buttons: int = 0, +) -> None: + """Dispatch a synthetic mouse move; ``buttons=1`` keeps an HTML5 drag active.""" + button = "left" if buttons & 1 else "none" + await cdp_client.send.Input.dispatchMouseEvent( + params={ + "type": "mouseMoved", + "x": x, + "y": y, + "button": button, + "buttons": buttons, + "clickCount": 0, + "modifiers": 0, + }, + session_id=session_id, + ) + + +async def set_intercept_drags(cdp_client: Any, session_id: str, enabled: bool) -> None: + """Enable/disable CDP drag interception (``Input.setInterceptDrags``).""" + await cdp_client.send.Input.setInterceptDrags(params={"enabled": bool(enabled)}, session_id=session_id) + + +async def dispatch_drag_event( + cdp_client: Any, + session_id: str, + *, + event_type: str, + x: float, + y: float, + data: dict[str, Any], +) -> None: + """Dispatch one ``Input.dispatchDragEvent`` (``dragEnter`` / ``dragOver`` / ``drop`` / ``dragCancel``).""" + await cdp_client.send.Input.dispatchDragEvent( + params={"type": event_type, "x": x, "y": y, "data": data, "modifiers": 0}, + session_id=session_id, + ) + + +def register_drag_intercepted(cdp_client: Any, callback: Any) -> Any: + """Register an ``Input.dragIntercepted`` handler; return an unregister callable.""" + + def _unregister_registry() -> None: + registry = getattr(cdp_client, "_event_registry", None) + if registry is not None and hasattr(registry, "unregister"): + registry.unregister("Input.dragIntercepted") + + register = getattr(cdp_client, "register", None) + input_reg = getattr(register, "Input", None) if register is not None else None + if input_reg is not None and hasattr(input_reg, "dragIntercepted"): + input_reg.dragIntercepted(callback) + return _unregister_registry + + registry = getattr(cdp_client, "_event_registry", None) + if registry is not None and hasattr(registry, "register"): + registry.register("Input.dragIntercepted", callback) + return _unregister_registry + + custom = getattr(cdp_client, "register_event", None) + if callable(custom): + custom("Input.dragIntercepted", callback) + unregister = getattr(cdp_client, "unregister_event", None) + if callable(unregister): + return lambda: unregister("Input.dragIntercepted") + return lambda: None + + raise exceptions.DriverUnsupported("CDP client cannot register Input.dragIntercepted handlers") + + +def _drag_data_from_intercept_event(event: Any) -> dict[str, Any]: + """Normalize ``Input.dragIntercepted`` payload to a CDP ``DragData`` dict.""" + if isinstance(event, dict): + data = event.get("data", event) + else: + data = getattr(event, "data", event) + if not isinstance(data, dict): + return {"items": [], "files": [], "dragOperationsMask": 1} + items = data.get("items") or [] + files = data.get("files") or [] + mask = data.get("dragOperationsMask", 1) + return { + "items": list(items) if isinstance(items, (list, tuple)) else [], + "files": list(files) if isinstance(files, (list, tuple)) else [], + "dragOperationsMask": int(mask if mask is not None else 1), + } + + +_HTML5_DND_JS = ( + "function(target) {" + " if (!target) { return { ok: false, error: 'missing drop target' }; }" + " const source = this;" + " const dt = new DataTransfer();" + " const opts = { bubbles: true, cancelable: true, dataTransfer: dt };" + " source.dispatchEvent(new DragEvent('dragstart', opts));" + " target.dispatchEvent(new DragEvent('dragenter', opts));" + " target.dispatchEvent(new DragEvent('dragover', opts));" + " const dropped = target.dispatchEvent(new DragEvent('drop', opts));" + " source.dispatchEvent(new DragEvent('dragend', opts));" + " return { ok: true, dropped: !!dropped };" + "}" +) + + +async def html5_drag_drop_via_js( + cdp_client: Any, + session_id: str, + *, + source_backend_node_id: int, + target_backend_node_id: int, +) -> dict[str, Any]: + """Fire the HTML5 drag events in the page when CDP drag interception did not happen.""" + target_object_id = await resolve_object_id_by_backend_node(cdp_client, session_id, target_backend_node_id) + result = await call_function_on_backend_node( + cdp_client, + session_id, + source_backend_node_id, + _HTML5_DND_JS, + arguments=[{"objectId": target_object_id}], + ) + if not isinstance(result, dict): + return {"ok": False, "error": "html5 js drag returned non-object"} + return result + + +async def perform_drag( + cdp_client: Any, + session_id: str, + *, + sx: float, + sy: float, + tx: float, + ty: float, + steps: int = 10, + delay_ms: int = 0, + source_is_html5: bool = False, + source_backend_node_id: int | None = None, + target_backend_node_id: int | None = None, +) -> dict[str, Any]: + """Drag from ``(sx, sy)`` to ``(tx, ty)``; an HTML5-draggable source succeeds only if a drop completed.""" + steps = max(1, int(steps)) + intercepted: asyncio.Future = asyncio.get_running_loop().create_future() + + def _on_intercepted(event: Any, _sid: Any = None) -> None: + if not intercepted.done(): + intercepted.set_result(_drag_data_from_intercept_event(event)) + + unregister = register_drag_intercepted(cdp_client, _on_intercepted) + try: + await set_intercept_drags(cdp_client, session_id, True) + await cdp_client.send.Input.dispatchMouseEvent( + params={ + "type": "mouseMoved", + "x": sx, + "y": sy, + "button": "none", + "buttons": 0, + "clickCount": 0, + "modifiers": 0, + }, + session_id=session_id, + ) + await cdp_client.send.Input.dispatchMouseEvent( + params={ + "type": "mousePressed", + "x": sx, + "y": sy, + "button": "left", + "buttons": 1, + "clickCount": 1, + "modifiers": 0, + }, + session_id=session_id, + ) + for step in range(1, steps + 1): + frac = step / steps + await dispatch_mouse_move( + cdp_client, + session_id, + x=sx + (tx - sx) * frac, + y=sy + (ty - sy) * frac, + buttons=1, + ) + if intercepted.done(): + break + if delay_ms: + await asyncio.sleep(delay_ms / 1000.0) + + if intercepted.done(): + data = intercepted.result() + await dispatch_drag_event(cdp_client, session_id, event_type="dragEnter", x=tx, y=ty, data=data) + await dispatch_drag_event(cdp_client, session_id, event_type="dragOver", x=tx, y=ty, data=data) + await dispatch_drag_event(cdp_client, session_id, event_type="drop", x=tx, y=ty, data=data) + await cdp_client.send.Input.dispatchMouseEvent( + params={ + "type": "mouseReleased", + "x": tx, + "y": ty, + "button": "left", + "buttons": 0, + "clickCount": 1, + "modifiers": 0, + }, + session_id=session_id, + ) + return {"ok": True, "detail": "dragged (html5 cdp)", "mode": "html5_cdp"} + + await cdp_client.send.Input.dispatchMouseEvent( + params={ + "type": "mouseReleased", + "x": tx, + "y": ty, + "button": "left", + "buttons": 0, + "clickCount": 1, + "modifiers": 0, + }, + session_id=session_id, + ) + + if source_is_html5: + if source_backend_node_id is None or target_backend_node_id is None: + return { + "ok": False, + "detail": ( + "HTML5 drag did not start (no Input.dragIntercepted); " + "cannot run JS fallback without source/target backend node ids" + ), + "mode": "html5_failed", + } + js_result = await html5_drag_drop_via_js( + cdp_client, + session_id, + source_backend_node_id=source_backend_node_id, + target_backend_node_id=target_backend_node_id, + ) + if js_result.get("ok"): + return {"ok": True, "detail": "dragged (html5 js)", "mode": "html5_js"} + error = js_result.get("error") or "HTML5 JS drag/drop produced no effect" + return {"ok": False, "detail": str(error), "mode": "html5_failed"} + + return {"ok": True, "detail": "dragged", "mode": "pointer"} + finally: + try: + await set_intercept_drags(cdp_client, session_id, False) + except Exception: # noqa: BLE001 + pass + try: + unregister() + except Exception: # noqa: BLE001 + pass + + +async def insert_text(cdp_client: Any, session_id: str, text: str) -> None: + """Insert text at the current focus/caret position.""" + await cdp_client.send.Input.insertText(params={"text": text}, session_id=session_id) + + +async def dispatch_key_press(cdp_client: Any, session_id: str, key: str) -> None: + """Dispatch a synthetic keyDown+keyUp for a single named key (e.g. ``"Enter"``).""" + await cdp_client.send.Input.dispatchKeyEvent(params={"type": "keyDown", "key": key}, session_id=session_id) + await cdp_client.send.Input.dispatchKeyEvent(params={"type": "keyUp", "key": key}, session_id=session_id) + + +async def request_node_id_from_object(cdp_client: Any, session_id: str, object_id: str) -> int: + """Resolve a Runtime remote objectId back to a CDP nodeId.""" + result = await cdp_client.send.DOM.requestNode(params={"objectId": object_id}, session_id=session_id) + return int(result["nodeId"]) + + +async def set_file_input_files( + cdp_client: Any, + session_id: str, + backend_node_id: int, + files: list[str], +) -> None: + """Attach files to ``<input type=file>`` by ``backendNodeId`` (``nodeId`` leaves a fakepath).""" + await cdp_client.send.DOM.setFileInputFiles( + params={"files": list(files), "backendNodeId": int(backend_node_id)}, + session_id=session_id, + ) + + +SUPPORTED_DROP_MIME_TYPES: frozenset[str] = frozenset( + { + "text/plain", + "text/uri-list", + "text/html", + } +) + + +def build_external_drag_data( + *, + paths: list[str] | None = None, + items: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Build a CDP ``DragData`` payload for an out-of-page drop; unsupported MIME types raise.""" + file_paths = [str(path) for path in (paths or []) if str(path or "").strip()] + mime_items: list[dict[str, str]] = [] + for raw in items or []: + if not isinstance(raw, dict): + raise exceptions.DriverUnsupported("drop data items must be objects with mimeType/data") + mime = str(raw.get("mimeType") or raw.get("mime_type") or "").strip().lower() + if not mime: + raise exceptions.DriverUnsupported("drop data item missing mimeType") + if mime not in SUPPORTED_DROP_MIME_TYPES: + supported = ", ".join(sorted(SUPPORTED_DROP_MIME_TYPES)) + raise exceptions.DriverUnsupported(f"drop MIME type {mime!r} is not supported (supported: {supported})") + mime_items.append({"mimeType": mime, "data": str(raw.get("data") or "")}) + if not file_paths and not mime_items: + raise exceptions.DriverError("drop requires at least one path or data item") + return { + "items": mime_items, + "files": file_paths, + "dragOperationsMask": 1, + } + + +async def perform_external_drop( + cdp_client: Any, + session_id: str, + *, + x: float, + y: float, + paths: list[str] | None = None, + items: list[dict[str, Any]] | None = None, +) -> dict[str, Any]: + """Drop external files or MIME data onto page coordinates with ``Input.dispatchDragEvent``.""" + data = build_external_drag_data(paths=paths, items=items) + await dispatch_drag_event(cdp_client, session_id, event_type="dragEnter", x=x, y=y, data=data) + await dispatch_drag_event(cdp_client, session_id, event_type="dragOver", x=x, y=y, data=data) + await dispatch_drag_event(cdp_client, session_id, event_type="drop", x=x, y=y, data=data) + file_count = len(data.get("files") or []) + item_count = len(data.get("items") or []) + parts: list[str] = [] + if file_count: + parts.append(f"{file_count} file(s)") + if item_count: + parts.append(f"{item_count} mime item(s)") + return { + "ok": True, + "detail": f"dropped {' + '.join(parts)} via CDP Input.dispatchDragEvent", + "files": list(data.get("files") or []), + "items": list(data.get("items") or []), + } + + +async def add_script_on_new_document(cdp_client: Any, session_id: str, source: str) -> None: + """Install ``source`` to run in every new document of the session before the page's own scripts.""" + await cdp_client.send.Page.addScriptToEvaluateOnNewDocument(params={"source": source}, session_id=session_id) + + +async def enable_page_domain(cdp_client: Any, session_id: str) -> None: + """Enable the Page domain (required for JS dialog events/handling).""" + await cdp_client.send.Page.enable(params={}, session_id=session_id) + + +async def handle_javascript_dialog( + cdp_client: Any, + session_id: str, + *, + accept: bool, + prompt_text: str | None = None, +) -> None: + """Accept or dismiss the currently open JS dialog via CDP.""" + params: dict[str, Any] = {"accept": bool(accept)} + if prompt_text is not None: + params["promptText"] = str(prompt_text) + await cdp_client.send.Page.handleJavaScriptDialog(params=params, session_id=session_id) + + +def register_javascript_dialog_opening(cdp_client: Any, callback: Any) -> Any: + """Register a ``Page.javascriptDialogOpening`` handler; return unregister callable.""" + + def _unregister_registry() -> None: + registry = getattr(cdp_client, "_event_registry", None) + if registry is not None and hasattr(registry, "unregister"): + registry.unregister("Page.javascriptDialogOpening") + + register = getattr(cdp_client, "register", None) + page_reg = getattr(register, "Page", None) if register is not None else None + if page_reg is not None and hasattr(page_reg, "javascriptDialogOpening"): + page_reg.javascriptDialogOpening(callback) + return _unregister_registry + + registry = getattr(cdp_client, "_event_registry", None) + if registry is not None and hasattr(registry, "register"): + registry.register("Page.javascriptDialogOpening", callback) + return _unregister_registry + + custom = getattr(cdp_client, "register_event", None) + if callable(custom): + custom("Page.javascriptDialogOpening", callback) + unregister = getattr(cdp_client, "unregister_event", None) + if callable(unregister): + return lambda: unregister("Page.javascriptDialogOpening") + return lambda: None + + raise exceptions.DriverUnsupported("CDP client cannot register Page.javascriptDialogOpening handlers") + + +def add_event_listener(cdp_client: Any, method: str, callback: Any) -> None: + """Call ``callback`` on ``method`` events, then the handler already there (cdp-use keeps one per event).""" + registry = getattr(cdp_client, "_event_registry", None) + if registry is None or not hasattr(registry, "register"): + raise exceptions.DriverUnsupported(f"CDP client cannot register {method} handlers") + previous = (getattr(registry, "_handlers", None) or {}).get(method) + + def chained(event: Any, session_id: Any = None) -> Any: + callback(event, session_id) + return None if previous is None else previous(event, session_id) + + registry.register(method, chained) + + +async def run_if_waiting_for_debugger(cdp_client: Any, session_id: str) -> None: + """Let a page that waits for the debugger run (a page opening a popup can wait so).""" + await cdp_client.send.Runtime.runIfWaitingForDebugger(params={}, session_id=session_id) + + +def _unwrap_js_result(result: dict[str, Any]) -> Any: + """Raise ``exceptions.EvaluateError`` on a JS exception, else return the value.""" + exception_details = result.get("exceptionDetails") + if exception_details: + exception = exception_details.get("exception") or {} + js_description = str(exception.get("description", "")) + js_message = js_description.split("\n", 1)[0] or str(exception_details.get("text", "unknown JS error")) + raise exceptions.EvaluateError(js_message, js_message=js_message, js_stack=js_description) + return (result.get("result") or {}).get("value") + + +__all__ = [ + "SUPPORTED_DROP_MIME_TYPES", + "add_event_listener", + "add_script_on_new_document", + "build_external_drag_data", + "call_function_on_backend_node", + "describe_node_backend_id", + "dispatch_drag_event", + "dispatch_key_press", + "dispatch_mouse_click", + "dispatch_mouse_move", + "enable_page_domain", + "evaluate_expression", + "get_document_root_node_id", + "handle_javascript_dialog", + "html5_drag_drop_via_js", + "insert_text", + "perform_drag", + "perform_external_drop", + "query_selector_node_id", + "register_drag_intercepted", + "register_javascript_dialog_opening", + "request_node_id_from_object", + "resolve_object_id_by_backend_node", + "run_if_waiting_for_debugger", + "set_file_input_files", + "set_intercept_drags", +] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/exceptions.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/exceptions.py new file mode 100644 index 0000000000..cf099cbfc7 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/exceptions.py @@ -0,0 +1,89 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Sidecar exceptions, named exactly as the wire error codes; stdlib only.""" + +from __future__ import annotations + + +class DriverError(Exception): + """Generic fallback error.""" + + +class DriverNotConnected(DriverError): + """A method was invoked before the session was connected.""" + + +class DriverConnectionError(DriverError): + """Attaching to the browser over CDP failed.""" + + +class DriverConnectionLost(DriverError): + """The CDP connection dropped mid-flight.""" + + +class ElementNotFound(DriverError): + """The referenced element does not resolve to any current DOM node.""" + + +class AmbiguousElement(DriverError): + """The referenced element matches more than one current DOM node.""" + + +class StaleIndexError(DriverError): + """A selector-map index no longer identifies the same node.""" + + +class StaleNodeError(DriverError): + """A backend node id no longer resolves.""" + + +class ActionTimeout(DriverError): + """A hands action did not complete within its timeout.""" + + +class NavigationError(DriverError): + """A navigation action failed.""" + + +class ObserveError(DriverError): + """Capturing a page state failed.""" + + +class EvaluateError(DriverError): + """In-page JavaScript evaluation failed.""" + + def __init__(self, message: str, *, js_message: str = "", js_stack: str = "") -> None: + super().__init__(message) + self.js_message = js_message + self.js_stack = js_stack + + +class DriverUnsupported(DriverError): + """The requested capability is not available.""" + + +class ActionIntercepted(DriverError): + """Another element would receive the input; ``by`` describes it. Raised before any input is sent.""" + + def __init__(self, message: str, *, by: str = "") -> None: + super().__init__(message) + self.by = by + + +__all__ = [ + "ActionIntercepted", + "ActionTimeout", + "AmbiguousElement", + "DriverConnectionError", + "DriverConnectionLost", + "DriverError", + "DriverNotConnected", + "DriverUnsupported", + "ElementNotFound", + "EvaluateError", + "NavigationError", + "ObserveError", + "StaleIndexError", + "StaleNodeError", +] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/keys.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/keys.py new file mode 100644 index 0000000000..2801e4914d --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/keys.py @@ -0,0 +1,30 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Playwright-style key names normalized to browser-use's key-combo format; stdlib only.""" + +from __future__ import annotations + +_MODIFIER_ALIASES: dict[str, str] = { + "control": "ctrl", + "ctrl": "ctrl", + "cmd": "cmd", + "command": "cmd", + "meta": "cmd", + "alt": "alt", + "option": "alt", + "shift": "shift", +} + + +def normalize_key_combo(keys: str) -> str: + """Normalize modifier tokens (``Control+A`` -> ``ctrl+A``); the final key keeps its case.""" + parts = [p for p in keys.split("+") if p] + if not parts: + return keys + *modifiers, final_key = parts + normalized_modifiers = [_MODIFIER_ALIASES.get(m.strip().lower(), m.strip().lower()) for m in modifiers] + return "+".join([*normalized_modifiers, final_key.strip()]) + + +__all__ = ["normalize_key_combo"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/main.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/main.py new file mode 100644 index 0000000000..92b1b7fc53 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/main.py @@ -0,0 +1,154 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Sidecar entry point: one JSON request per line on stdin, one reply per line on stdout.""" + +from __future__ import annotations + +import asyncio +import json +import os +import platform +import queue +import sys +import threading +from typing import Any + +import exceptions +import session_adapter +import wire + + +def _reconfigure_stdio() -> None: + """Force UTF-8 with plain ``\\n`` newlines on both stdio streams.""" + for stream in (sys.stdin, sys.stdout): + reconfigure = getattr(stream, "reconfigure", None) + if reconfigure is not None: + reconfigure(encoding="utf-8", newline="\n") + + +def _stdin_reader_thread(line_queue: queue.Queue) -> None: + """Blocking readline loop run on a dedicated thread; feeds an async-visible queue.""" + try: + for raw_line in sys.stdin: + line_queue.put(raw_line) + finally: + line_queue.put(None) + + +class _StdoutWriter: + """Serializes NDJSON writes to stdout across concurrently-running request tasks.""" + + def __init__(self) -> None: + self._lock = asyncio.Lock() + + async def write(self, payload: dict[str, Any]) -> None: + line = json.dumps(payload, ensure_ascii=False) + async with self._lock: + sys.stdout.write(line + "\n") + sys.stdout.flush() + + +def _error_code_for(exc: Exception) -> str: + name = type(exc).__name__ + return name if name in wire.ERROR_CODES else wire.DEFAULT_ERROR_CODE + + +def _error_data_for(exc: Exception) -> dict[str, Any] | None: + if isinstance(exc, exceptions.EvaluateError): + return {"js_message": exc.js_message, "js_stack": exc.js_stack} + if isinstance(exc, exceptions.ActionIntercepted): + return {"by": exc.by} + return None + + +async def _dispatch( + adapter: session_adapter.SessionAdapter, + writer: _StdoutWriter, + request_id: int, + method: str, + params: dict[str, Any], +) -> None: + if method not in wire.DRIVER_METHOD_NAMES: + await writer.write(wire.make_failure(request_id, "DriverUnsupported", f"unknown method {method!r}")) + return + handler = getattr(adapter, method, None) + if handler is None: + await writer.write(wire.make_failure(request_id, "DriverUnsupported", f"method {method!r} not implemented")) + return + try: + result = await handler(**params) + except asyncio.CancelledError: + await writer.write(wire.make_failure(request_id, "ActionTimeout", f"{method} cancelled")) + raise + except exceptions.DriverError as exc: + await writer.write(wire.make_failure(request_id, _error_code_for(exc), str(exc), _error_data_for(exc))) + except Exception as exc: # noqa: BLE001 + await writer.write(wire.make_failure(request_id, wire.DEFAULT_ERROR_CODE, str(exc))) + else: + await writer.write(wire.make_success(request_id, result)) + + +async def _run() -> None: + _reconfigure_stdio() + + adapter = session_adapter.SessionAdapter() + writer = _StdoutWriter() + await writer.write( + wire.make_hello( + browser_use_version=session_adapter.browser_use_version(), + python_version=platform.python_version(), + pid=os.getpid(), + ) + ) + + line_queue: queue.Queue = queue.Queue() + reader_thread = threading.Thread(target=_stdin_reader_thread, args=(line_queue,), daemon=True) + reader_thread.start() + + loop = asyncio.get_event_loop() + active_tasks: dict[int, asyncio.Task] = {} + + while True: + raw_line = await loop.run_in_executor(None, line_queue.get) + if raw_line is None: + break + line = raw_line.strip() + if not line: + continue + try: + message = json.loads(line) + except json.JSONDecodeError: + continue + + method = message.get("method") + request_id = message.get("id") + + if method == "cancel": + target_id = (message.get("params") or {}).get("id") + task = active_tasks.get(target_id) + if task is not None: + task.cancel() + continue + + if method == "shutdown": + for task in list(active_tasks.values()): + task.cancel() + if active_tasks: + await asyncio.gather(*active_tasks.values(), return_exceptions=True) + await adapter.close() + if request_id is not None: + await writer.write(wire.make_success(request_id, {})) + break + + task = asyncio.ensure_future(_dispatch(adapter, writer, request_id, method, message.get("params") or {})) + active_tasks[request_id] = task + task.add_done_callback(lambda _t, rid=request_id: active_tasks.pop(rid, None)) + + +def main() -> None: + asyncio.run(_run()) + + +if __name__ == "__main__": + main() diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/mapping.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/mapping.py new file mode 100644 index 0000000000..257406215c --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/mapping.py @@ -0,0 +1,111 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Map browser-use's state summary to wire observation dicts, duck-typed and stdlib only.""" + +from __future__ import annotations + +from typing import Any + + +def map_dom_rect(rect: Any) -> dict[str, float] | None: + """Map a DOMRect-like object to a wire ``Box`` dict, or None.""" + if rect is None: + return None + return { + "x": float(getattr(rect, "x", 0.0) or 0.0), + "y": float(getattr(rect, "y", 0.0) or 0.0), + "width": float(getattr(rect, "width", 0.0) or 0.0), + "height": float(getattr(rect, "height", 0.0) or 0.0), + } + + +def map_enhanced_node(index: int, node: Any, *, driver_generation: int) -> dict[str, Any]: + """Map one DOM node to a wire element dict stamped with ``driver_generation``.""" + ax_node = getattr(node, "ax_node", None) + role = getattr(ax_node, "role", None) if ax_node is not None else None + name = getattr(ax_node, "name", None) if ax_node is not None else None + attributes = dict(getattr(node, "attributes", None) or {}) + backend_node_id = int(getattr(node, "backend_node_id", 0) or 0) + return { + "index": int(index), + "backend_node_id": backend_node_id, + "driver_ref": {"handle": f"bnid:{backend_node_id}", "driver_generation": int(driver_generation)}, + "frame_id": getattr(node, "frame_id", None), + "tag": str(getattr(node, "node_name", "") or "").lower(), + "role": role, + "name": name, + "value": getattr(node, "node_value", None), + "attributes": attributes, + "box": map_dom_rect(getattr(node, "absolute_position", None)), + "visible": bool(getattr(node, "is_visible", False)), + "scrollable": bool(getattr(node, "is_scrollable", False)), + } + + +def map_tab(tab: Any, current_url: str) -> dict[str, Any]: + """Map one tab to a wire tab dict; a tab is active when its url is the current url.""" + tab_url = str(getattr(tab, "url", "") or "") + return { + "target_id": str(getattr(tab, "target_id", "") or ""), + "url": tab_url, + "title": str(getattr(tab, "title", "") or ""), + "active": bool(current_url) and tab_url == current_url, + } + + +def map_browser_state_summary( + summary: Any, + *, + driver_generation: int, + captured_at: float, + include_screenshot: bool = False, +) -> dict[str, Any]: + """Map a ``BrowserStateSummary``-like object to a wire ``Observation`` dict.""" + dom_state = getattr(summary, "dom_state", None) + selector_map = dict(getattr(dom_state, "selector_map", None) or {}) + elements = tuple( + map_enhanced_node(index, node, driver_generation=driver_generation) + for index, node in sorted(selector_map.items()) + ) + + ax_text: str | None = None + llm_representation = getattr(dom_state, "llm_representation", None) if dom_state is not None else None + if callable(llm_representation): + try: + ax_text = llm_representation() + except Exception: # noqa: BLE001 + ax_text = None + + current_url = str(getattr(summary, "url", "") or "") + tabs = tuple(map_tab(tab, current_url) for tab in (getattr(summary, "tabs", None) or [])) + + errors: list[str] = [str(e) for e in (getattr(summary, "browser_errors", None) or [])] + state_error = getattr(summary, "state_error", None) + if state_error: + errors.append(str(state_error)) + + screenshot_b64 = getattr(summary, "screenshot", None) if include_screenshot else None + + return { + "url": current_url, + "title": str(getattr(summary, "title", "") or ""), + "tabs": tabs, + "elements": elements, + "ax_text": ax_text, + "screenshot_b64": screenshot_b64, + "pixels_above": int(getattr(summary, "pixels_above", 0) or 0), + "pixels_below": int(getattr(summary, "pixels_below", 0) or 0), + "errors": tuple(errors), + "is_pdf_viewer": bool(getattr(summary, "is_pdf_viewer", False)), + "captured_at": float(captured_at), + "driver_generation": int(driver_generation), + } + + +__all__ = [ + "map_browser_state_summary", + "map_dom_rect", + "map_enhanced_node", + "map_tab", +] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/session_adapter.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/session_adapter.py new file mode 100644 index 0000000000..9b545a9a8e --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/session_adapter.py @@ -0,0 +1,1036 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The only sidecar module that imports ``browser_use``: wire dicts in, wire dicts out.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +import time +from pathlib import Path +from typing import Any + +import cdp +import exceptions +import keys as key_utils +import mapping + + +def _strip_upload_path_text(raw: str) -> str: + text = str(raw or "").strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"": + text = text[1:-1].strip() + return text + + +def _normalize_upload_paths(paths: list[str]) -> tuple[list[str], str | None]: + """Return ``(existing_absolute_paths, error_or_none)``; the driver sends absolute paths only.""" + existing: list[str] = [] + missing: list[str] = [] + unreadable: list[str] = [] + + for raw in paths or []: + text = _strip_upload_path_text(str(raw)) + if not text: + continue + path = Path(text).expanduser() + if not path.is_absolute(): + return [], f"upload path must be absolute: {text!r}" + try: + resolved = path.resolve() + except OSError: + missing.append(text) + continue + if not resolved.is_file(): + missing.append(str(resolved)) + continue + try: + with resolved.open("rb") as handle: + handle.read(1) + except OSError: + unreadable.append(str(resolved)) + continue + if str(resolved) not in existing: + existing.append(str(resolved)) + + if not existing and not missing and not unreadable: + return [], "upload_files requires at least one path" + err_parts: list[str] = [] + if missing: + listed = ", ".join(repr(p) for p in missing) + err_parts.append( + f"upload file(s) not found: {listed} " + "(create the file, use an absolute path, or place it under " + "BROWSER_UPLOAD_ROOT and call list_upload_files)" + ) + if unreadable: + listed = ", ".join(repr(p) for p in unreadable) + err_parts.append(f"upload file(s) not readable: {listed}") + if err_parts: + return existing, "; ".join(err_parts) + return existing, None + + +_DRIVER_REF_HANDLE_PREFIX = "bnid:" +_RESUME_TIMEOUT_S = 1.0 +_NAVIGATION_COMMIT_WAIT_S = 1.0 +_SLOW_TYPING_DELAY_S = 0.03 + + +def _mint_driver_ref(backend_node_id: int, driver_generation: int) -> dict[str, Any]: + """Mint this backend's ``DriverRef`` wire dict for a resolved CDP node.""" + return {"handle": f"{_DRIVER_REF_HANDLE_PREFIX}{int(backend_node_id)}", "driver_generation": int(driver_generation)} + + +def _parse_driver_ref_handle(handle: str) -> int: + """Recover the backend node id this adapter minted into ``handle``.""" + if not handle.startswith(_DRIVER_REF_HANDLE_PREFIX): + raise exceptions.ElementNotFound(f"driver ref handle {handle!r} was not minted by this backend") + try: + return int(handle[len(_DRIVER_REF_HANDLE_PREFIX) :]) + except ValueError as exc: + raise exceptions.ElementNotFound(f"driver ref handle {handle!r} does not encode a backend node id") from exc + + +def browser_use_version() -> str: + try: + import importlib.metadata + + return importlib.metadata.version("browser-use") + except Exception: # noqa: BLE001 + return "unknown" + + +class _Act: + def __init__(self, main_frame_id: str | None) -> None: + self.dispatched = False + self.main_frame_id = main_frame_id + self.navigation_requested = False + self.navigation_committed = asyncio.Event() + self.dialog: dict[str, str] | None = None + self.dialog_blocking = asyncio.Event() + self.new_targets: list[dict[str, Any]] = [] + self.target_created = asyncio.Event() + + def dispatching(self) -> None: + self.dispatched = True + + +_HIT_TEST_JS = ( + "function(x, y) {" + " const root = this.getRootNode ? this.getRootNode() : this.ownerDocument;" + " const hit = (root.elementFromPoint ? root : this.ownerDocument).elementFromPoint(x, y);" + " if (!hit || hit === this || this.contains(hit)) { return null; }" + " if (hit.tagName === 'LABEL' && hit.control === this) { return null; }" + " const name = (hit.getAttribute('aria-label') || hit.getAttribute('title') || hit.innerText || '')" + " .trim().replace(/\\s+/g, ' ').slice(0, 40);" + " const classes = typeof hit.className === 'string' ? hit.className.trim().split(/\\s+/).filter(Boolean) : [];" + " return hit.tagName.toLowerCase() + (hit.id ? '#' + hit.id : '')" + " + classes.slice(0, 2).map(c => '.' + c).join('') + (name ? ' \"' + name + '\"' : '');" + "}" +) + + +def _dialog_note(dialog: dict[str, Any]) -> str: + return f"a {dialog.get('type') or 'dialog'} dialog is open ({dialog.get('message', '')!r}); handle it first" + + +class SessionAdapter: + """Owns the browser-use session and translates wire calls into it.""" + + def __init__(self) -> None: + self._session: Any = None + self._cdp_session: Any = None + self._driver_generation = 0 + self._index_cache: dict[int, Any] = {} + self._dialog_armed: dict[str, Any] | None = None + self._dialog_pending: dict[str, Any] | None = None + self._dialog_listener_registered = False + self._dialog_unregister: Any = None + self._dialog_handler: Any = None + self._dialog_handle_tasks: set[asyncio.Task] = set() + self._current_act: _Act | None = None + self._waiting_inputs: set[asyncio.Task] = set() + self._listening_client: Any = None + self._init_script: str | None = None + self._guarded_sessions: set[str] = set() + + async def connect(self, *, cdp_url: str, timeout_s: float = 30.0, init_script: str | None = None) -> dict[str, Any]: + from browser_use.browser.profile import BrowserProfile + from browser_use.browser.session import BrowserSession + + profile = BrowserProfile(headless=None) + self._session = BrowserSession(cdp_url=cdp_url, is_local=False, browser_profile=profile) + await self._session.start() + self._init_script = init_script or None + await self._ensure_cdp_session() + await self._ensure_listeners() + + version_info: dict[str, Any] = {} + try: + version_info = await self._cdp_session.cdp_client.send.Browser.getVersion(session_id=None) or {} + except Exception: # noqa: BLE001 + version_info = {} + + try: + initial_url = await self._session.get_current_page_url() + except Exception: # noqa: BLE001 + initial_url = "" + + return { + "backend": "browser_use", + "browser_version": str(version_info.get("product", "") or ""), + "protocol_version": str(version_info.get("protocolVersion", "") or ""), + "backend_version": browser_use_version(), + "initial_url": initial_url, + } + + async def health(self) -> dict[str, Any]: + if self._session is None: + return {"connected": False, "url": "", "tab_count": 0, "latency_ms": 0.0, "error": "not connected"} + started = time.monotonic() + try: + url = await self._session.get_current_page_url() + tabs = await self._session.get_tabs() + latency_ms = (time.monotonic() - started) * 1000.0 + return {"connected": True, "url": url, "tab_count": len(tabs), "latency_ms": latency_ms, "error": None} + except Exception as exc: # noqa: BLE001 + latency_ms = (time.monotonic() - started) * 1000.0 + return {"connected": False, "url": "", "tab_count": 0, "latency_ms": latency_ms, "error": str(exc)} + + async def close(self) -> None: + """Detach only. MUST NOT KILL CHROME -- ``stop()``, never ``kill()``.""" + for task in list(self._waiting_inputs): + task.cancel() + if self._session is not None: + await self._session.stop() + self._session = None + self._cdp_session = None + self._index_cache = {} + self._guarded_sessions = set() + + async def observe( + self, *, include_dom: bool = True, include_screenshot: bool = False, cached: bool = False + ) -> dict[str, Any]: + if self._dialog_pending is not None: + return await self._dialog_observation() + await self._ensure_listeners() + summary = await self._session.get_browser_state_summary(include_screenshot=include_screenshot, cached=cached) + self._driver_generation += 1 + dom_state = getattr(summary, "dom_state", None) + self._index_cache = dict(getattr(dom_state, "selector_map", None) or {}) + return mapping.map_browser_state_summary( + summary, + driver_generation=self._driver_generation, + captured_at=time.time(), + include_screenshot=include_screenshot, + ) + + async def screenshot(self, *, full_page: bool = False, clip: dict[str, Any] | None = None) -> str: + self._refuse_while_dialog_open("screenshot", exceptions.ObserveError) + data = await self._session.take_screenshot(full_page=full_page, clip=clip) + return base64.b64encode(data).decode("ascii") + + async def list_tabs(self) -> list[dict[str, Any]]: + tabs = await self._session.get_tabs() + try: + current_url = await self._session.get_current_page_url() + except Exception: # noqa: BLE001 + current_url = "" + return [mapping.map_tab(tab, current_url) for tab in tabs] + + async def resolve(self, ref: dict[str, Any]) -> dict[str, Any]: + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + box = await self._element_box(backend_node_id) + tag, attributes, visible = await self._element_facts(backend_node_id) + return { + "backend_node_id": backend_node_id, + "driver_ref": _mint_driver_ref(backend_node_id, self._driver_generation), + "frame_id": ref.get("frame_id"), + "box": box, + "visible": visible, + "tag": tag, + "attributes": attributes, + } + + async def stamp(self, ref: dict[str, Any], *, attribute: str, value: str) -> str: + self._refuse_while_dialog_open("stamp", exceptions.EvaluateError) + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + cdp_session = await self._ensure_cdp_session() + await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(attr, val){ this.setAttribute(attr, val); }", + arguments=[{"value": attribute}, {"value": value}], + return_by_value=True, + ) + return f'[{attribute}="{value}"]' + + async def navigate( + self, + *, + url: str, + wait_until: str = "load", + timeout_ms: int | None = None, + new_tab: bool = False, + ) -> dict[str, Any]: + from browser_use.browser.events import NavigateToUrlEvent + + before_url = await self._safe_current_url() + event = self._session.event_bus.dispatch( + NavigateToUrlEvent(url=url, wait_until=wait_until, timeout_ms=timeout_ms, new_tab=new_tab) + ) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + return await self._nav_result(before_url) + + async def go_back(self) -> dict[str, Any]: + from browser_use.browser.events import GoBackEvent + + before_url = await self._safe_current_url() + event = self._session.event_bus.dispatch(GoBackEvent()) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + return await self._nav_result(before_url) + + async def go_forward(self) -> dict[str, Any]: + from browser_use.browser.events import GoForwardEvent + + before_url = await self._safe_current_url() + event = self._session.event_bus.dispatch(GoForwardEvent()) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + return await self._nav_result(before_url) + + async def reload(self) -> dict[str, Any]: + from browser_use.browser.events import RefreshEvent + + before_url = await self._safe_current_url() + event = self._session.event_bus.dispatch(RefreshEvent()) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + return await self._nav_result(before_url) + + async def click( + self, + ref: dict[str, Any], + *, + button: str = "left", + click_count: int = 1, + modifiers: list[str] | None = None, + ) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + box = await self._element_box(backend_node_id) + if box is None: + raise exceptions.ElementNotFound(f"element (backend_node_id={backend_node_id}) has no bounding box") + cx, cy = box["x"] + box["width"] / 2.0, box["y"] + box["height"] / 2.0 + await self._ensure_not_intercepted("click", backend_node_id, cx, cy) + cdp_session = await self._ensure_cdp_session() + act.dispatching() + for _ in range(max(1, click_count)): + await cdp.dispatch_mouse_click( + cdp_session.cdp_client, cdp_session.session_id, x=cx, y=cy, button=button, click_count=click_count + ) + return True, "clicked" + + return await self._act(body) + + async def type_text( + self, + ref: dict[str, Any], + text: str, + *, + clear: bool = True, + press_enter: bool = False, + sensitive: bool = False, + slowly: bool = False, + ) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + cdp_session = await self._ensure_cdp_session() + box = await self._element_box(backend_node_id) + act.dispatching() + if box is not None: + cx, cy = box["x"] + box["width"] / 2.0, box["y"] + box["height"] / 2.0 + await cdp.dispatch_mouse_click(cdp_session.cdp_client, cdp_session.session_id, x=cx, y=cy) + if clear: + await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(){ if ('value' in this) { this.value = ''; } else { this.textContent = ''; } " + "this.dispatchEvent(new Event('input', {bubbles: true})); }", + ) + if slowly: + for character in text: + await cdp.insert_text(cdp_session.cdp_client, cdp_session.session_id, character) + await asyncio.sleep(_SLOW_TYPING_DELAY_S) + else: + await cdp.insert_text(cdp_session.cdp_client, cdp_session.session_id, text) + if press_enter: + await cdp.dispatch_key_press(cdp_session.cdp_client, cdp_session.session_id, "Enter") + return True, "typed" if not sensitive else "typed (sensitive)" + + return await self._act(body) + + async def press_key(self, keys: str) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + from browser_use.browser.events import SendKeysEvent + + act.dispatching() + event = self._session.event_bus.dispatch(SendKeysEvent(keys=key_utils.normalize_key_combo(keys))) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + return True, "keys sent" + + return await self._act(body) + + async def select_option( + self, ref: dict[str, Any], *, value: str | None = None, label: str | None = None + ) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + from browser_use.browser.events import SelectDropdownOptionEvent + + node = await self._node_for_event(ref) + text = label if label is not None else (value or "") + act.dispatching() + event = self._session.event_bus.dispatch(SelectDropdownOptionEvent(node=node, text=text)) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + return True, "option selected" + + return await self._act(body) + + async def set_checked(self, ref: dict[str, Any], checked: bool) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + cdp_session = await self._ensure_cdp_session() + await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(want){ if (this.checked !== want) { this.checked = want; " + "this.dispatchEvent(new Event('click', {bubbles: true})); " + "this.dispatchEvent(new Event('change', {bubbles: true})); } }", + arguments=[{"value": bool(checked)}], + ) + return True, "checked state set" + + return await self._act(body) + + async def scroll(self, *, direction: str, amount: float, ref: dict[str, Any] | None = None) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + from browser_use.browser.events import ScrollEvent + + node = await self._node_for_event(ref) if ref is not None else None + act.dispatching() + event = self._session.event_bus.dispatch(ScrollEvent(node=node, direction=direction, amount=amount)) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + return True, "scrolled" + + return await self._act(body) + + async def upload_files(self, ref: dict[str, Any], paths: list[str]) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + normalized, path_error = _normalize_upload_paths(paths or []) + if path_error: + return False, path_error + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + cdp_session = await self._ensure_cdp_session() + facts = await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(){" + " const isFile = !!(this && this.tagName === 'INPUT' &&" + " String(this.type || '').toLowerCase() === 'file');" + " return {isFile: isFile, multiple: !!(isFile && this.multiple)," + " files: (isFile && this.files) ? this.files.length : 0};" + "}", + ) + if not isinstance(facts, dict) or not facts.get("isFile"): + return False, "upload target is not an <input type=file>" + if len(normalized) > 1 and not facts.get("multiple"): + return False, ( + f"upload target does not accept multiple files " + f"(got {len(normalized)} paths); pass a single path or a multiple= file input" + ) + act.dispatching() + await cdp.set_file_input_files( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + normalized, + ) + attached = await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(){ return (this.files && this.files.length) || 0; }", + ) + count = int(attached or 0) + if count < 1: + return False, ( + "file input has no files after attach " + "(DOM.setFileInputFiles did not stick; check path is readable by Chrome)" + ) + if count != len(normalized): + return False, f"file input has {count} file(s) after attach, expected {len(normalized)}" + return True, f"uploaded {count} file(s)" + + return await self._act(body) + + async def drag( + self, source: dict[str, Any], target: dict[str, Any], *, steps: int = 10, delay_ms: int = 0 + ) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + source_id = await self._resolve_ref_to_backend_node_id(source) + target_id = await self._resolve_ref_to_backend_node_id(target) + source_box = await self._element_box(source_id) + target_box = await self._element_box(target_id) + if source_box is None or target_box is None: + raise exceptions.ElementNotFound("drag source or target has no bounding box") + sx, sy = source_box["x"] + source_box["width"] / 2.0, source_box["y"] + source_box["height"] / 2.0 + tx, ty = target_box["x"] + target_box["width"] / 2.0, target_box["y"] + target_box["height"] / 2.0 + await self._ensure_not_intercepted("drag", source_id, sx, sy) + source_is_html5 = await self._element_is_html5_draggable(source_id) + cdp_session = await self._ensure_cdp_session() + act.dispatching() + result = await cdp.perform_drag( + cdp_session.cdp_client, + cdp_session.session_id, + sx=sx, + sy=sy, + tx=tx, + ty=ty, + steps=steps, + delay_ms=delay_ms, + source_is_html5=source_is_html5, + source_backend_node_id=source_id, + target_backend_node_id=target_id, + ) + return bool(result.get("ok")), str(result.get("detail") or "") + + return await self._act(body) + + async def hover(self, ref: dict[str, Any]) -> dict[str, Any]: + async def body(act: _Act) -> tuple[bool, str]: + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + box = await self._element_box(backend_node_id) + if box is None: + raise exceptions.ElementNotFound(f"element (backend_node_id={backend_node_id}) has no bounding box") + cx, cy = box["x"] + box["width"] / 2.0, box["y"] + box["height"] / 2.0 + await self._ensure_not_intercepted("hover", backend_node_id, cx, cy) + cdp_session = await self._ensure_cdp_session() + act.dispatching() + await cdp.dispatch_mouse_move( + cdp_session.cdp_client, + cdp_session.session_id, + x=max(0.0, cx - 4.0), + y=max(0.0, cy - 4.0), + ) + await cdp.dispatch_mouse_move(cdp_session.cdp_client, cdp_session.session_id, x=cx, y=cy) + return True, "hovered" + + return await self._act(body) + + async def handle_dialog(self, *, accept: bool, prompt_text: str | None = None) -> dict[str, Any]: + """Handle the open dialog, or arm the handler for the next one.""" + + async def body(act: _Act) -> tuple[bool, str]: + if self._dialog_pending is not None: + pending = self._dialog_pending + self._dialog_pending = None + cdp_session = await self._ensure_cdp_session() + await cdp.handle_javascript_dialog( + cdp_session.cdp_client, + pending.get("session_id") or cdp_session.session_id, + accept=bool(accept), + prompt_text=prompt_text, + ) + dialog_type = str(pending.get("type") or "dialog") + action = "accepted" if accept else "dismissed" + return True, f"{action} {dialog_type}" + self._dialog_armed = {"accept": bool(accept), "prompt_text": prompt_text} + return True, "armed for next dialog; call before the action that opens it" + + return await self._act(body, while_dialog=True) + + async def drop( + self, + ref: dict[str, Any], + *, + paths: list[str] | None = None, + data: list[dict[str, Any]] | None = None, + ) -> dict[str, Any]: + """Drop external files and/or MIME data onto an element (not element→element drag).""" + + async def body(act: _Act) -> tuple[bool, str]: + path_list = list(paths or []) + data_list = list(data or []) + if not path_list and not data_list: + return False, "drop requires at least one of paths or data" + normalized: list[str] = [] + if path_list: + normalized, path_error = _normalize_upload_paths(path_list) + if path_error: + return False, path_error + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + box = await self._element_box(backend_node_id) + if box is None: + raise exceptions.ElementNotFound(f"drop target (backend_node_id={backend_node_id}) has no bounding box") + cx, cy = box["x"] + box["width"] / 2.0, box["y"] + box["height"] / 2.0 + cdp_session = await self._ensure_cdp_session() + act.dispatching() + try: + result = await cdp.perform_external_drop( + cdp_session.cdp_client, + cdp_session.session_id, + x=cx, + y=cy, + paths=normalized, + items=data_list, + ) + except exceptions.DriverUnsupported: + raise + except exceptions.DriverError as exc: + return False, str(exc) + detail = str(result.get("detail") or "dropped") + if normalized: + attached = await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(){" + " if (this && this.tagName === 'INPUT' &&" + " String(this.type || '').toLowerCase() === 'file') {" + " return (this.files && this.files.length) || 0;" + " }" + " return null;" + "}", + ) + if isinstance(attached, int) and attached > 0: + detail = f"{detail}; file input now has {attached} file(s)" + return bool(result.get("ok")), detail + + return await self._act(body) + + async def switch_tab(self, tab: dict[str, Any]) -> dict[str, Any]: + before_url = await self._safe_current_url() + from browser_use.browser.events import SwitchTabEvent + + event = self._session.event_bus.dispatch(SwitchTabEvent(target_id=tab["target_id"])) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + self._cdp_session = None + self._dialog_listener_registered = False + self._dialog_unregister = None + return await self._act_result(before_url, ok=True, detail="switched tab") + + async def close_tab(self, tab: dict[str, Any]) -> dict[str, Any]: + before_url = await self._safe_current_url() + from browser_use.browser.events import CloseTabEvent + + event = self._session.event_bus.dispatch(CloseTabEvent(target_id=tab["target_id"])) + await event + await event.event_result(raise_if_any=True, raise_if_none=False) + return await self._act_result(before_url, ok=True, detail="closed tab") + + async def evaluate( + self, + source: str, + *, + args: Any = None, + await_promise: bool = True, + return_by_value: bool = True, + ) -> Any: + self._refuse_while_dialog_open("evaluate", exceptions.EvaluateError) + cdp_session = await self._ensure_cdp_session() + return await cdp.evaluate_expression( + cdp_session.cdp_client, + cdp_session.session_id, + source, + args, + await_promise=await_promise, + return_by_value=return_by_value, + ) + + async def wait_load_state(self, *, state: str = "load", timeout_ms: int) -> dict[str, Any]: + before_url = await self._safe_current_url() + await asyncio.sleep(min(timeout_ms / 1000.0, 5.0)) + return await self._act_result(before_url, ok=True, detail=f"waited for {state}") + + async def _ensure_cdp_session(self) -> Any: + if self._cdp_session is None: + self._cdp_session = await self._session.get_or_create_cdp_session() + await self._guard_document(self._cdp_session) + return self._cdp_session + + async def _guard_document(self, cdp_session: Any) -> None: + if not self._init_script or cdp_session.session_id in self._guarded_sessions: + return + await cdp.add_script_on_new_document(cdp_session.cdp_client, cdp_session.session_id, self._init_script) + # The document open at attach time loaded before the script was registered, so it gets it now. + await cdp.evaluate_expression( + cdp_session.cdp_client, cdp_session.session_id, self._init_script, await_promise=False + ) + self._guarded_sessions.add(cdp_session.session_id) + + async def _ensure_listeners(self) -> None: + cdp_session = await self._ensure_cdp_session() + if not self._dialog_listener_registered: + await cdp.enable_page_domain(cdp_session.cdp_client, cdp_session.session_id) + self._dialog_listener_registered = True + if self._dialog_handler is None: + self._dialog_handler = self._on_dialog + self._dialog_unregister = cdp.register_javascript_dialog_opening(cdp_session.cdp_client, self._dialog_handler) + if self._listening_client is not cdp_session.cdp_client: + client = cdp_session.cdp_client + cdp.add_event_listener(client, "Target.targetCreated", self._on_target_created) + cdp.add_event_listener(client, "Page.frameRequestedNavigation", self._on_navigation_requested) + cdp.add_event_listener(client, "Page.frameStartedNavigating", self._on_navigation_requested) + cdp.add_event_listener(client, "Page.frameNavigated", self._on_frame_navigated) + self._listening_client = client + + def _on_navigation_requested(self, event: Any, session_id: Any = None) -> None: + act = self._current_act + event = event or {} + if act is None or event.get("frameId") != act.main_frame_id: + return + if event.get("disposition", "currentTab") == "currentTab": + act.navigation_requested = True + + def _on_frame_navigated(self, event: Any, session_id: Any = None) -> None: + act = self._current_act + frame = (event or {}).get("frame") or {} + if act is not None and not frame.get("parentId") and frame.get("id") == act.main_frame_id: + act.navigation_committed.set() + + def _on_target_created(self, event: Any, session_id: Any = None) -> None: + info = (event or {}).get("targetInfo") or {} + act = self._current_act + if act is None or info.get("type") != "page": + return + target_id = str(info.get("targetId") or "") + if all(target["target_id"] != target_id for target in act.new_targets): + url, title = str(info.get("url") or ""), str(info.get("title") or "") + act.new_targets.append({"target_id": target_id, "url": url, "title": title, "active": False}) + act.target_created.set() + + def _on_dialog(self, event: Any, session_id: Any = None) -> None: + payload = event if isinstance(event, dict) else getattr(event, "__dict__", {}) or {} + if not isinstance(payload, dict): + payload = {} + info = {"type": str(payload.get("type") or "alert"), "message": str(payload.get("message") or "")} + act = self._current_act + if act is not None: + act.dialog = info + armed = self._dialog_armed + if armed is not None: + self._dialog_armed = None + self._dialog_pending = None + task = asyncio.create_task( + self._apply_armed_dialog( + accept=bool(armed.get("accept")), + prompt_text=armed.get("prompt_text"), + session_id=session_id, + ) + ) + self._dialog_handle_tasks.add(task) + task.add_done_callback(self._dialog_handle_tasks.discard) + else: + self._dialog_pending = {**info, "session_id": session_id} + if act is not None: + act.dialog_blocking.set() + + async def _apply_armed_dialog(self, *, accept: bool, prompt_text: str | None, session_id: Any = None) -> None: + try: + cdp_session = await self._ensure_cdp_session() + await cdp.handle_javascript_dialog( + cdp_session.cdp_client, + session_id or cdp_session.session_id, + accept=accept, + prompt_text=prompt_text, + ) + except Exception: # noqa: BLE001 + pass + + def _open_dialog(self) -> dict[str, str] | None: + pending = self._dialog_pending + return None if pending is None else {"type": pending["type"], "message": pending["message"]} + + async def _dialog_observation(self) -> dict[str, Any]: + tabs = await self.list_tabs() + return { + "url": await self._safe_current_url(), + "title": next((tab["title"] for tab in tabs if tab["active"]), ""), + "tabs": tabs, + "elements": [], + "dialog": self._open_dialog(), + "driver_generation": self._driver_generation, + } + + def _refuse_while_dialog_open(self, method: str, error: type[exceptions.DriverError]) -> None: + if self._dialog_pending is not None: + raise error(f"{method}: {_dialog_note(self._dialog_pending)}") + + async def _act(self, body: Any, *, while_dialog: bool = False) -> dict[str, Any]: + before_url = await self._safe_current_url() + if self._dialog_pending is not None and not while_dialog: + return await self._act_result(before_url, ok=False, detail=_dialog_note(self._dialog_pending)) + await self._ensure_listeners() + act = _Act(getattr(self._cdp_session, "target_id", None)) + self._current_act = act + task = asyncio.ensure_future(body(act)) + dialog_waiter = asyncio.ensure_future(act.dialog_blocking.wait()) + target_waiter = asyncio.ensure_future(act.target_created.wait()) + try: + waiters = {dialog_waiter, target_waiter} + while not task.done() and not dialog_waiter.done(): + done, _pending = await asyncio.wait({task, *waiters}, return_when=asyncio.FIRST_COMPLETED) + if target_waiter in done and target_waiter in waiters: + waiters.discard(target_waiter) + await self._resume_current_page() + if task.done(): + ok, detail = self._body_result(task, act) + await self._await_navigation(act) + else: + self._waiting_inputs.add(task) + task.add_done_callback(self._forget_waiting_input) + ok, detail = True, "the input waits until the dialog is handled" + finally: + dialog_waiter.cancel() + target_waiter.cancel() + if not task.done() and task not in self._waiting_inputs: + task.cancel() + self._current_act = None + return await self._act_result(before_url, ok=ok, detail=detail, act=act) + + @staticmethod + def _body_result(task: asyncio.Task, act: _Act) -> tuple[bool, str]: + try: + return task.result() + except (exceptions.StaleNodeError, exceptions.StaleIndexError, exceptions.ElementNotFound) as exc: + if not act.dispatched: + raise + raise exceptions.DriverError(f"{exc} (after input reached the page)") from exc + + @staticmethod + async def _await_navigation(act: _Act) -> None: + if act.navigation_requested and not act.navigation_committed.is_set(): + try: + await asyncio.wait_for(act.navigation_committed.wait(), _NAVIGATION_COMMIT_WAIT_S) + except asyncio.TimeoutError: + pass + + async def _resume_current_page(self) -> None: + cdp_session = await self._ensure_cdp_session() + try: + await asyncio.wait_for( + cdp.run_if_waiting_for_debugger(cdp_session.cdp_client, cdp_session.session_id), _RESUME_TIMEOUT_S + ) + except Exception: # noqa: BLE001 + pass + + def _forget_waiting_input(self, task: asyncio.Task) -> None: + self._waiting_inputs.discard(task) + if not task.cancelled(): + task.exception() + + async def _safe_current_url(self) -> str: + try: + return await self._session.get_current_page_url() + except Exception: # noqa: BLE001 + return "" + + async def _nav_result(self, before_url: str) -> dict[str, Any]: + url = await self._safe_current_url() + try: + title = await self._session.get_current_page_title() + except Exception: # noqa: BLE001 + title = "" + self._driver_generation += 1 + return { + "url": url, + "title": title, + "changed_document": url != before_url, + "driver_generation": self._driver_generation, + "dialog": self._open_dialog(), + } + + async def _act_result(self, before_url: str, *, ok: bool, detail: str, act: _Act | None = None) -> dict[str, Any]: + url = await self._safe_current_url() + changed = url != before_url or (act is not None and act.navigation_committed.is_set()) + if changed: + self._driver_generation += 1 + return { + "ok": ok, + "detail": detail, + "url": url, + "document_changed": changed, + "driver_generation": self._driver_generation, + "dialog_opened": None if act is None else act.dialog, + "dialog": self._open_dialog(), + "new_targets": [] if act is None else list(act.new_targets), + } + + def _node_from_index(self, index: int, driver_generation: int) -> Any: + if driver_generation != self._driver_generation: + raise exceptions.StaleIndexError( + f"index {index} was observed at driver_generation {driver_generation}, " + f"current is {self._driver_generation}" + ) + node = self._index_cache.get(index) + if node is None: + raise exceptions.StaleIndexError(f"index {index} is not present in the last observation") + return node + + async def _resolve_ref_to_backend_node_id(self, ref: dict[str, Any]) -> int: + """Resolve any wire ElementRef dict to a CDP backendNodeId.""" + kind = ref.get("kind") + if kind == "index": + node = self._node_from_index(int(ref["index"]), int(ref["driver_generation"])) + backend_node_id = getattr(node, "backend_node_id", None) + if backend_node_id is None: + raise exceptions.StaleIndexError(f"index {ref['index']} has no backend_node_id") + return int(backend_node_id) + if kind == "node": + return int(ref["backend_node_id"]) + if kind == "driver": + return _parse_driver_ref_handle(str(ref["handle"])) + if kind == "selector": + cdp_session = await self._ensure_cdp_session() + node_id = await cdp.query_selector_node_id(cdp_session.cdp_client, cdp_session.session_id, ref["css"]) + if node_id is None: + raise exceptions.ElementNotFound(f"no element matches selector {ref['css']!r}") + return await cdp.describe_node_backend_id(cdp_session.cdp_client, cdp_session.session_id, node_id) + if kind == "text": + return await self._resolve_text_ref(ref) + raise exceptions.ElementNotFound(f"unsupported element ref kind {kind!r}") + + async def _resolve_text_ref(self, ref: dict[str, Any]) -> int: + cdp_session = await self._ensure_cdp_session() + text = str(ref.get("text", "")) + role = ref.get("role") + find_fn = ( + "(args) => { " + "const text = args.text; const role = args.role; " + "const walker = document.createTreeWalker(document.body, NodeFilter.SHOW_ELEMENT); " + "let node = walker.currentNode; " + "while (node) { " + " if (node.textContent && node.textContent.includes(text) && " + " (!role || node.getAttribute('role') === role)) { return node; } " + " node = walker.nextNode(); " + "} " + "return null; }" + ) + result = await cdp_session.cdp_client.send.Runtime.evaluate( + params={ + "expression": f"({find_fn})({json.dumps({'text': text, 'role': role})})", + "awaitPromise": True, + "returnByValue": False, + }, + session_id=cdp_session.session_id, + ) + object_id = (result.get("result") or {}).get("objectId") + if not object_id: + raise exceptions.ElementNotFound(f"no element matches text {text!r}") + node_id = await cdp.request_node_id_from_object(cdp_session.cdp_client, cdp_session.session_id, object_id) + return await cdp.describe_node_backend_id(cdp_session.cdp_client, cdp_session.session_id, node_id) + + async def _node_for_event(self, ref: dict[str, Any] | None) -> Any: + """A real DOM node for browser-use's high-level events, fresh when possible.""" + if ref is None: + return None + if ref.get("kind") == "index": + return self._node_from_index(int(ref["index"]), int(ref["driver_generation"])) + backend_node_id = await self._resolve_ref_to_backend_node_id(ref) + cdp_session = await self._ensure_cdp_session() + await cdp.resolve_object_id_by_backend_node(cdp_session.cdp_client, cdp_session.session_id, backend_node_id) + for node in self._index_cache.values(): + if getattr(node, "backend_node_id", None) == backend_node_id: + return node + raise exceptions.ElementNotFound( + f"backend_node_id={backend_node_id} has no cached selector-map node for this action; " + "re-observe the page first" + ) + + async def _element_box(self, backend_node_id: int) -> dict[str, float] | None: + cdp_session = await self._ensure_cdp_session() + try: + rect = await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(){ const r = this.getBoundingClientRect(); " + "return {x: r.x, y: r.y, width: r.width, height: r.height}; }", + ) + except exceptions.StaleNodeError: + raise + except exceptions.DriverError: + return None + if not isinstance(rect, dict): + return None + return {k: float(rect.get(k, 0.0) or 0.0) for k in ("x", "y", "width", "height")} + + async def _ensure_not_intercepted(self, method: str, backend_node_id: int, x: float, y: float) -> None: + cdp_session = await self._ensure_cdp_session() + by = await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + _HIT_TEST_JS, + arguments=[{"value": x}, {"value": y}], + ) + if by: + raise exceptions.ActionIntercepted( + f"{method} at ({x:.0f}, {y:.0f}) would reach {by}, not the target; nothing was sent", by=str(by) + ) + + async def _element_is_html5_draggable(self, backend_node_id: int) -> bool: + """True when the node opts into HTML5 DnD (mouse-only drag would be a no-op).""" + cdp_session = await self._ensure_cdp_session() + try: + result = await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(){" + " if (this.getAttribute && this.getAttribute('draggable') === 'true') return true;" + " if (this.draggable === true) return true;" + " return false;" + "}", + ) + except exceptions.DriverError: + return False + return bool(result) + + async def _element_facts(self, backend_node_id: int) -> tuple[str, dict[str, str], bool]: + cdp_session = await self._ensure_cdp_session() + try: + facts = await cdp.call_function_on_backend_node( + cdp_session.cdp_client, + cdp_session.session_id, + backend_node_id, + "function(){ const attrs = {}; for (const a of this.attributes || []) { attrs[a.name] = a.value; } " + "const r = this.getBoundingClientRect(); " + "return {tag: this.tagName ? this.tagName.toLowerCase() : '', attributes: attrs, " + "visible: !!(r.width > 0 && r.height > 0)}; }", + ) + except exceptions.DriverError: + return "", {}, False + if not isinstance(facts, dict): + return "", {}, False + return ( + str(facts.get("tag", "")), + dict(facts.get("attributes", {}) or {}), + bool(facts.get("visible", False)), + ) + + +__all__ = ["SessionAdapter", "browser_use_version"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/wire.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/wire.py new file mode 100644 index 0000000000..51b58cf019 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/sidecar/wire.py @@ -0,0 +1,113 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Wire protocol constants shared verbatim by the driver and the sidecar; stdlib only.""" + +from __future__ import annotations + +WIRE_VERSION = 4 + +DRIVER_METHOD_NAMES: tuple[str, ...] = ( + "connect", + "health", + "close", + "observe", + "screenshot", + "list_tabs", + "resolve", + "stamp", + "navigate", + "go_back", + "go_forward", + "reload", + "click", + "type_text", + "press_key", + "select_option", + "set_checked", + "scroll", + "upload_files", + "drag", + "hover", + "handle_dialog", + "drop", + "switch_tab", + "close_tab", + "evaluate", + "wait_load_state", +) + +CONTROL_METHOD_NAMES: tuple[str, ...] = ("cancel", "shutdown") + +WIRE_METHOD_NAMES: frozenset[str] = frozenset(DRIVER_METHOD_NAMES) | frozenset(CONTROL_METHOD_NAMES) + +ERROR_CODES: tuple[str, ...] = ( + "DriverError", + "DriverNotConnected", + "DriverConnectionError", + "DriverConnectionLost", + "ElementNotFound", + "AmbiguousElement", + "StaleIndexError", + "StaleNodeError", + "ActionTimeout", + "NavigationError", + "ObserveError", + "EvaluateError", + "DriverUnsupported", + "ActionIntercepted", +) + +DEFAULT_ERROR_CODE = "DriverError" + +MAX_LINE_BYTES = 8 * 1024 * 1024 + + +def make_request(request_id: int, method: str, params: dict | None = None) -> dict: + """Build a request payload: ``{"id": ..., "method": ..., "params": ...}``.""" + return {"id": request_id, "method": method, "params": params or {}} + + +def make_success(request_id: int, result: object) -> dict: + """Build a success reply payload: ``{"id": ..., "ok": ...}``.""" + return {"id": request_id, "ok": result} + + +def make_failure(request_id: int, code: str, message: str, data: dict | None = None) -> dict: + """Build a failure reply payload: ``{"id": ..., "error": {...}}``.""" + error: dict[str, object] = {"code": code, "message": message} + if data: + error["data"] = data + return {"id": request_id, "error": error} + + +def make_hello(*, browser_use_version: str, python_version: str, pid: int) -> dict: + """Build the unsolicited hello line the sidecar emits first.""" + return { + "kind": "hello", + "wire": WIRE_VERSION, + "browser_use": browser_use_version, + "python": python_version, + "pid": pid, + } + + +def make_log(level: str, message: str) -> dict: + """Build an unsolicited log line.""" + return {"kind": "log", "level": level, "message": message} + + +__all__ = [ + "CONTROL_METHOD_NAMES", + "DEFAULT_ERROR_CODE", + "DRIVER_METHOD_NAMES", + "ERROR_CODES", + "MAX_LINE_BYTES", + "WIRE_METHOD_NAMES", + "WIRE_VERSION", + "make_failure", + "make_hello", + "make_log", + "make_request", + "make_success", +] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/transport.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/transport.py new file mode 100644 index 0000000000..7c97f374a8 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/transport.py @@ -0,0 +1,318 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""NDJSON-over-stdio transport to the browser-use sidecar process; never imports ``browser_use``.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import platform +from pathlib import Path +from typing import Any, Mapping + +from openjiuwen.core.common.logging import logger as _logger +from openjiuwen.harness.tools.browser_move.drivers.browser_use.sidecar import exceptions as driver_errors +from openjiuwen.harness.tools.browser_move.drivers.browser_use.sidecar import wire + +_SIDECAR_VENV_DIR = ".venvs/browser-use" +_REQUEST_TIMEOUT_S = 30.0 +_SHUTDOWN_GRACE_S = 5.0 + +_ERROR_CODE_TO_EXCEPTION: dict[str, type[driver_errors.DriverError]] = { + "DriverError": driver_errors.DriverError, + "DriverNotConnected": driver_errors.DriverNotConnected, + "DriverConnectionError": driver_errors.DriverConnectionError, + "DriverConnectionLost": driver_errors.DriverConnectionLost, + "ElementNotFound": driver_errors.ElementNotFound, + "AmbiguousElement": driver_errors.AmbiguousElement, + "StaleIndexError": driver_errors.StaleIndexError, + "StaleNodeError": driver_errors.StaleNodeError, + "ActionTimeout": driver_errors.ActionTimeout, + "NavigationError": driver_errors.NavigationError, + "ObserveError": driver_errors.ObserveError, + "EvaluateError": driver_errors.EvaluateError, + "DriverUnsupported": driver_errors.DriverUnsupported, + "ActionIntercepted": driver_errors.ActionIntercepted, +} + + +def _sidecar_dir() -> Path: + return Path(__file__).resolve().parent / "sidecar" + + +def _default_venv_python() -> Path: + repo_root = Path(__file__).resolve().parents[6] + if platform.system() == "Windows": + return repo_root / _SIDECAR_VENV_DIR / "Scripts" / "python.exe" + return repo_root / _SIDECAR_VENV_DIR / "bin" / "python" + + +def discover_sidecar_python(override: Path | None = None) -> Path: + """Resolve the sidecar interpreter path, or raise ``DriverUnsupported`` with a creation recipe.""" + if override is not None: + if override.is_file(): + return override + raise driver_errors.DriverUnsupported( + f"BROWSER_USE_SIDECAR_PYTHON={str(override)!r} does not point at an existing interpreter" + ) + + candidate = _default_venv_python() + if candidate.is_file(): + return candidate + + raise driver_errors.DriverUnsupported( + "No browser-use sidecar interpreter found. Create one with:\n" + " uv venv --python 3.12 .venvs/browser-use\n" + ' uv pip install --python .venvs/browser-use "browser-use==0.13.10"\n' + "or set BROWSER_USE_SIDECAR_PYTHON to an existing interpreter." + ) + + +def _env_value(environ: Mapping[str, str], key: str) -> str | None: + value = environ.get(key) + return environ.get(key.upper()) if value is None else value + + +def _sidecar_env(environ: Mapping[str, str], upload_root: Path | None) -> dict[str, str]: + """A minimal environment for the sidecar: no LLM credentials, ever.""" + env: dict[str, str] = {} + for key in ("PATH",): + value = environ.get(key) + if value is not None: + env[key] = value + if platform.system() == "Windows": + for key in ("SystemRoot", "TEMP", "TMP", "USERPROFILE", "HOMEDRIVE", "HOMEPATH", "LOCALAPPDATA", "APPDATA"): + value = _env_value(environ, key) + if value is not None: + env[key] = value + else: + for key in ("HOME", "TMPDIR"): + value = environ.get(key) + if value is not None: + env[key] = value + for key, value in environ.items(): + if key.startswith("BROWSER_USE_") and key != "BROWSER_USE_SIDECAR_PYTHON": + env[key] = value + if upload_root is not None: + env["BROWSER_UPLOAD_ROOT"] = str(upload_root) + env.setdefault("ANONYMIZED_TELEMETRY", "false") + return env + + +class SidecarTransport: + """One live sidecar process plus its NDJSON request/response cycle.""" + + def __init__( + self, + *, + sidecar_python: Path | None = None, + upload_root: Path | None = None, + environ: Mapping[str, str] | None = None, + ) -> None: + self._sidecar_python = sidecar_python + self._upload_root = upload_root + self._environ = environ or {} + self._process: asyncio.subprocess.Process | None = None + self._next_id = 1 + self._lock = asyncio.Lock() + self._hello: dict[str, Any] | None = None + self._closed = False + self._stderr_task: asyncio.Future[None] | None = None + + @property + def hello(self) -> dict[str, Any] | None: + return self._hello + + async def start(self, *, timeout_s: float = 30.0) -> dict[str, Any]: + """Spawn the sidecar and read its hello line.""" + python_path = discover_sidecar_python(self._sidecar_python) + main_path = _sidecar_dir() / "main.py" + self._process = await asyncio.create_subprocess_exec( + str(python_path), + str(main_path), + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + env=_sidecar_env(self._environ, self._upload_root), + limit=wire.MAX_LINE_BYTES, + ) + self._stderr_task = asyncio.ensure_future(self._drain_stderr()) + + assert self._process.stdout is not None + try: + raw_line = await asyncio.wait_for(self._readline_stdout(), timeout=timeout_s) + except asyncio.TimeoutError as exc: + await self._terminate() + raise driver_errors.DriverConnectionError("sidecar did not send hello within timeout") from exc + except driver_errors.DriverConnectionLost as exc: + await self._terminate() + raise driver_errors.DriverConnectionError(str(exc)) from exc + + if not raw_line: + await self._terminate() + raise driver_errors.DriverConnectionError("sidecar exited before sending hello") + + try: + message = json.loads(raw_line.decode("utf-8").strip()) + except json.JSONDecodeError as exc: + await self._terminate() + raise driver_errors.DriverConnectionError(f"sidecar hello line is not valid JSON: {raw_line!r}") from exc + + if message.get("kind") != "hello": + await self._terminate() + raise driver_errors.DriverConnectionError(f"expected hello line, got: {message!r}") + + if message.get("wire") != wire.WIRE_VERSION: + await self._terminate() + raise driver_errors.DriverConnectionError( + f"sidecar wire version {message.get('wire')!r} != expected {wire.WIRE_VERSION!r}" + ) + + self._hello = message + return message + + async def request( + self, method: str, params: dict[str, Any] | None = None, *, timeout_s: float | None = None + ) -> Any: + """Send one request, await its matching reply. One in-flight request at a time.""" + if self._process is None or self._process.stdin is None or self._process.stdout is None: + raise driver_errors.DriverNotConnected("sidecar transport has not been started") + + async with self._lock: + request_id = self._next_id + self._next_id += 1 + payload = wire.make_request(request_id, method, params) + line = json.dumps(payload, ensure_ascii=False) + "\n" + + try: + self._process.stdin.write(line.encode("utf-8")) + await self._process.stdin.drain() + except (BrokenPipeError, ConnectionResetError) as exc: + raise driver_errors.DriverConnectionLost("sidecar stdin pipe is closed") from exc + + effective_timeout = timeout_s if timeout_s is not None else _REQUEST_TIMEOUT_S + try: + reply = await asyncio.wait_for(self._read_reply(request_id), timeout=effective_timeout) + except asyncio.TimeoutError: + await self._cancel(request_id) + raise driver_errors.ActionTimeout(f"{method} timed out after {effective_timeout}s") from None + except asyncio.CancelledError: + await self._cancel(request_id) + raise + + return self._unwrap_reply(reply) + + async def _readline_stdout(self) -> bytes: + assert self._process is not None and self._process.stdout is not None + try: + raw_line = await self._process.stdout.readline() + except asyncio.LimitOverrunError as exc: + raise driver_errors.DriverConnectionLost("sidecar emitted a line exceeding the max line size") from exc + except ValueError as exc: + if "chunk is longer than limit" not in str(exc): + raise + raise driver_errors.DriverConnectionLost("sidecar emitted a line exceeding the max line size") from exc + if raw_line and len(raw_line) > wire.MAX_LINE_BYTES: + raise driver_errors.DriverConnectionLost("sidecar emitted a line exceeding the max line size") + return raw_line + + async def _read_reply(self, request_id: int) -> dict[str, Any]: + assert self._process is not None and self._process.stdout is not None + while True: + raw_line = await self._readline_stdout() + if not raw_line: + raise driver_errors.DriverConnectionLost("sidecar closed stdout unexpectedly") + try: + message = json.loads(raw_line.decode("utf-8").strip()) + except json.JSONDecodeError: + _logger.warning("browser-use sidecar emitted a non-JSON line: %r", raw_line[:200]) + continue + + if message.get("kind") == "log": + self._forward_log(message) + continue + if message.get("kind") == "hello": + continue + if message.get("id") == request_id: + return message + + async def _cancel(self, request_id: int) -> None: + if self._process is None or self._process.stdin is None: + return + try: + payload = wire.make_request(0, "cancel", {"id": request_id}) + self._process.stdin.write((json.dumps(payload) + "\n").encode("utf-8")) + await self._process.stdin.drain() + except (BrokenPipeError, ConnectionResetError): + pass + + @staticmethod + def _unwrap_reply(message: dict[str, Any]) -> Any: + if "ok" in message: + return message["ok"] + error = message.get("error") or {} + code = str(error.get("code", wire.DEFAULT_ERROR_CODE)) + exc_class = _ERROR_CODE_TO_EXCEPTION.get(code, driver_errors.DriverError) + message_text = str(error.get("message", "sidecar error")) + data = error.get("data") or {} + if exc_class is driver_errors.EvaluateError: + raise driver_errors.EvaluateError( + message_text, js_message=str(data.get("js_message", "")), js_stack=str(data.get("js_stack", "")) + ) + if exc_class is driver_errors.ActionIntercepted: + raise driver_errors.ActionIntercepted(message_text, by=str(data.get("by", ""))) + raise exc_class(message_text) + + def _forward_log(self, message: dict[str, Any]) -> None: + level_name = str(message.get("level", "info")).upper() + text = str(message.get("message", "")) + if level_name == "ERROR": + _logger.error("browser-use sidecar: %s", text) + elif level_name == "WARNING": + _logger.warning("browser-use sidecar: %s", text) + else: + _logger.info("browser-use sidecar: %s", text) + + async def _drain_stderr(self) -> None: + if self._process is None or self._process.stderr is None: + return + while True: + raw_line = await self._process.stderr.readline() + if not raw_line: + break + text = raw_line.decode("utf-8", errors="replace").rstrip() + if text: + _logger.warning("browser-use sidecar stderr: %s", text) + + async def close(self) -> None: + """Ask the sidecar to shut down, then terminate it if it does not exit in time.""" + if self._closed: + return + self._closed = True + if self._process is None: + return + try: + await self.request("shutdown", {}, timeout_s=_SHUTDOWN_GRACE_S) + except driver_errors.DriverError: + pass + await self._terminate() + + async def _terminate(self) -> None: + if self._stderr_task is not None: + self._stderr_task.cancel() + self._stderr_task = None + if self._process is None: + return + if self._process.returncode is None: + try: + self._process.terminate() + await asyncio.wait_for(self._process.wait(), timeout=_SHUTDOWN_GRACE_S) + except asyncio.TimeoutError: + with contextlib.suppress(ProcessLookupError): + self._process.kill() + self._process = None + + +__all__ = ["SidecarTransport", "discover_sidecar_python"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/browser_use/upload_paths.py b/openjiuwen/harness/tools/browser_move/drivers/browser_use/upload_paths.py new file mode 100644 index 0000000000..d4a7ed3d5d --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/browser_use/upload_paths.py @@ -0,0 +1,58 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Resolve the local file paths of an upload or a drop for the browser-use driver.""" + +from __future__ import annotations + +from pathlib import Path +from typing import Sequence + +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import upload_paths + + +def _try_open(path: Path) -> bool: + try: + with path.open("rb") as handle: + handle.read(1) + return True + except OSError: + return False + + +def resolve_upload_file_paths(paths: Sequence[str], *, upload_root: Path | None = None) -> tuple[list[str], str | None]: + """Return ``(absolute_existing_paths, error_or_none)``; relative names resolve under ``upload_root`` only.""" + try: + resolved = upload_paths(tuple(str(raw) for raw in paths or ()), upload_root) + except ValueError as exc: + return [], str(exc) + existing: list[str] = [] + missing: list[str] = [] + unreadable: list[str] = [] + for path in resolved: + if not path.is_file(): + missing.append(str(path)) + elif not _try_open(path): + unreadable.append(str(path)) + elif str(path) not in existing: + existing.append(str(path)) + + if not existing and not missing and not unreadable: + return [], "'paths' must contain at least one file path" + + err_parts: list[str] = [] + if missing: + listed = ", ".join(repr(p) for p in missing) + err_parts.append( + f"upload file(s) not found: {listed} " + "(create the file, use an absolute path, or place it under " + "BROWSER_UPLOAD_ROOT and call list_upload_files)" + ) + if unreadable: + listed = ", ".join(repr(p) for p in unreadable) + err_parts.append(f"upload file(s) not readable: {listed}") + if err_parts: + return existing, "; ".join(err_parts) + return existing, None + + +__all__ = ["resolve_upload_file_paths"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/middleware.py b/openjiuwen/harness/tools/browser_move/drivers/middleware.py new file mode 100644 index 0000000000..5d470111e9 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/middleware.py @@ -0,0 +1,145 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Wrappers every driver gets; each is a ``Driver`` around an inner ``Driver``.""" + +from __future__ import annotations + +import asyncio +import dataclasses +from collections.abc import Awaitable, Mapping +from typing import Any, Protocol, TypeVar + +from ..contract import ( + Action, + ActionResult, + ActionTimeout, + Driver, + DriverError, + DriverInfo, + Effect, + Health, + Observation, + Observe, + StaleRef, +) +from ..contract.masking import MASK, is_sensitive, masked + +_T = TypeVar("_T") + + +class TraceSink(Protocol): + def __call__(self, record: dict[str, Any]) -> None: ... + + +class _Wrapper(Driver): + def __init__(self, inner: Driver) -> None: + self.inner = inner + + @property + def info(self) -> DriverInfo: + return self.inner.info + + async def start(self, config: Any = None) -> None: + await self.inner.start(config) + + async def stop(self) -> None: + await self.inner.stop() + + async def health(self) -> Health: + return await self.inner.health() + + async def observe(self, command: Observe) -> Observation: + return await self.inner.observe(command) + + async def execute(self, action: Action) -> ActionResult: + return await self.inner.execute(action) + + +class Timeouts(_Wrapper): + """Bounds each call by the command's ``effect``; an effect without a bound runs unbounded.""" + + def __init__(self, inner: Driver, per_effect: Mapping[Effect, float]) -> None: + super().__init__(inner) + self._per_effect = dict(per_effect) + + async def observe(self, command: Observe) -> Observation: + return await self._bounded(command, self.inner.observe(command)) + + async def execute(self, action: Action) -> ActionResult: + return await self._bounded(action, self.inner.execute(action)) + + async def _bounded(self, command: Action, call: Awaitable[_T]) -> _T: + seconds = self._per_effect.get(command.effect) + if seconds is None: + return await call + try: + async with asyncio.timeout(seconds): + return await call + except TimeoutError as error: + raise ActionTimeout(f"{type(command).__name__} took longer than {seconds}s", dispatched=True) from error + + +class RetryOnStale(_Wrapper): + """On ``StaleRef`` before dispatch, observes once and re-raises it with that fresh observation attached.""" + + async def execute(self, action: Action) -> ActionResult: + try: + return await self.inner.execute(action) + except StaleRef as error: + if error.dispatched: + raise + fresh = await self.inner.observe(Observe()) + raise StaleRef(str(error), dispatched=False, observation=fresh) from error + + +def _traced(value: Any) -> Any: + if dataclasses.is_dataclass(value) and not isinstance(value, type): + return {item.name: _traced(getattr(value, item.name)) for item in dataclasses.fields(value)} + if isinstance(value, (tuple, list)): + return [_traced(item) for item in value] + if isinstance(value, Mapping): + return {key: _traced(item) for key, item in value.items()} + return value + + +class Tracing(_Wrapper): + """Writes one record per ``observe``/``execute`` to ``sink``, masking the ``__sensitive_fields__`` of commands.""" + + def __init__(self, inner: Driver, sink: TraceSink) -> None: + super().__init__(inner) + self._sink = sink + + async def observe(self, command: Observe) -> Observation: + record = self._command_record("observe", command) + try: + observation = await self.inner.observe(command) + except DriverError as error: + self._sink(record | self._error_record(error, command)) + raise + self._sink(record | {"context": observation.context.id, "elements": len(observation.elements)}) + return observation + + async def execute(self, action: Action) -> ActionResult: + record = self._command_record("execute", action) + try: + result = await self.inner.execute(action) + except DriverError as error: + self._sink(record | self._error_record(error, action)) + raise + self._sink(record | {"ok": result.ok, "detail": result.detail, "context_changed": result.context_changed}) + return result + + @staticmethod + def _command_record(call: str, command: Action) -> dict[str, Any]: + return {"call": call, "command": type(command).__name__, "args": _traced(masked(command))} + + @staticmethod + def _error_record(error: DriverError, command: Action) -> dict[str, Any]: + record = {"error": type(error).__name__, "dispatched": error.dispatched} + if not is_sensitive(command): + record["message"] = str(error) + return record + + +__all__ = ["MASK", "RetryOnStale", "Timeouts", "TraceSink", "Tracing", "masked"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/__init__.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/__init__.py new file mode 100644 index 0000000000..de25fd24e7 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/__init__.py @@ -0,0 +1,5 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +from .driver import PlaywrightMcpConfig, PlaywrightMcpDriver, create_playwright_mcp_driver + +__all__ = ["PlaywrightMcpConfig", "PlaywrightMcpDriver", "create_playwright_mcp_driver"] diff --git a/openjiuwen/harness/tools/browser_move/clients/__init__.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/__init__.py similarity index 100% rename from openjiuwen/harness/tools/browser_move/clients/__init__.py rename to openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/__init__.py diff --git a/openjiuwen/harness/tools/browser_move/clients/logging_utils.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/logging_utils.py similarity index 89% rename from openjiuwen/harness/tools/browser_move/clients/logging_utils.py rename to openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/logging_utils.py index 4d819316b3..f8a4274217 100644 --- a/openjiuwen/harness/tools/browser_move/clients/logging_utils.py +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/logging_utils.py @@ -1,10 +1,6 @@ # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Safe logging helpers for browser-move MCP clients. - -These helpers keep low-level transport logs useful without dumping generated -JavaScript, page snapshots, form values, credentials, or other user data. -""" +"""Logging helpers for the MCP clients that never write scripts, snapshots or form values.""" from __future__ import annotations @@ -70,10 +66,8 @@ def _redacted_text(value: Any) -> dict[str, Any]: text = "" if value is None else str(value) text_hash = "" if text: - text_hash = hashlib.sha256( - text.encode("utf-8", errors="ignore") - ).hexdigest()[:12] - + text_hash = hashlib.sha256(text.encode("utf-8", errors="ignore")).hexdigest()[:12] + return { "kind": type(value).__name__, "length": len(text), @@ -162,9 +156,7 @@ def summarize_tool_arguments_for_log(tool_name: str, arguments: Any) -> dict[str code_text = str(code) code_hash = "" if code_text: - code_hash = hashlib.sha256( - code_text.encode("utf-8", "ignore") - ).hexdigest()[:12] + code_hash = hashlib.sha256(code_text.encode("utf-8", "ignore")).hexdigest()[:12] return { "kind": "code_execution", @@ -175,11 +167,7 @@ def summarize_tool_arguments_for_log(tool_name: str, arguments: Any) -> dict[str } if "navigate" in lowered_name: - raw_url = ( - arguments.get("url") - or arguments.get("href") - or arguments.get("target") - ) + raw_url = arguments.get("url") or arguments.get("href") or arguments.get("target") return { "kind": "navigation", "keys": keys, @@ -217,10 +205,7 @@ def summarize_tool_arguments_for_log(tool_name: str, arguments: Any) -> dict[str key_str = str(key) key_lower = key_str.lower() - is_sensitive_key = any( - token in key_lower - for token in sensitive_key_tokens - ) + is_sensitive_key = any(token in key_lower for token in sensitive_key_tokens) if key_lower in _TEXT_VALUE_KEYS or is_sensitive_key: redacted_values[key_str] = _redacted_text(value) @@ -238,4 +223,4 @@ def summarize_tool_arguments_for_log(tool_name: str, arguments: Any) -> dict[str if redacted_values: summary["redacted_values"] = redacted_values - return summary \ No newline at end of file + return summary diff --git a/openjiuwen/harness/tools/browser_move/clients/stdio_client.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/stdio_client.py similarity index 71% rename from openjiuwen/harness/tools/browser_move/clients/stdio_client.py rename to openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/stdio_client.py index b57a02e14a..1d979a1e5f 100644 --- a/openjiuwen/harness/tools/browser_move/clients/stdio_client.py +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/stdio_client.py @@ -8,26 +8,48 @@ from openjiuwen.core.foundation.tool import McpToolCard from openjiuwen.core.foundation.tool.mcp.base import NO_TIMEOUT, preserve_mcp_tool_result_status from openjiuwen.core.foundation.tool.mcp.client.stdio_client import StdioClient -from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_logging import ( - browser_agent_log_info, - browser_agent_log_warning, -) -from ..utils.parsing import sanitize_json_schema + from .logging_utils import summarize_tool_arguments_for_log +_UNSUPPORTED_SCHEMA_KEYS = {"$schema", "$id", "$defs", "definitions", "$comment", "$anchor", "$vocabulary"} -class BrowserMoveStdioClient(StdioClient): - """browser_move extension of StdioClient. - Adds timeout resolution, retryable-error detection, auto-reconnect, and - retry/timeout wrapping around list_tools and call_tool. Also fixes the - missing exit-stack reset in connect/disconnect that would otherwise cause - failures on the second connection attempt. +def sanitize_json_schema(schema: Any) -> Any: + """Strip schema keywords OpenAI-compatible APIs reject; a nullable ``anyOf`` becomes the plain type.""" + if not isinstance(schema, dict): + return schema + + for kw in ("anyOf", "oneOf"): + variants = schema.get(kw) + if isinstance(variants, list) and len(variants) == 2: + non_null = [v for v in variants if v != {"type": "null"} and v.get("type") != "null"] + null_count = len(variants) - len(non_null) + if null_count == 1 and len(non_null) == 1: + merged = {k: v for k, v in schema.items() if k != kw} + merged.update(non_null[0]) + schema = merged + + cleaned = {k: v for k, v in schema.items() if k not in _UNSUPPORTED_SCHEMA_KEYS} + + if cleaned.get("type") is None and "type" in cleaned: + cleaned["type"] = "object" + + if "properties" in cleaned and isinstance(cleaned["properties"], dict): + cleaned["properties"] = {k: sanitize_json_schema(v) for k, v in cleaned["properties"].items()} + + for kw in ("items", "additionalProperties", "not"): + if kw in cleaned: + cleaned[kw] = sanitize_json_schema(cleaned[kw]) + + for kw in ("anyOf", "oneOf", "allOf"): + if kw in cleaned and isinstance(cleaned[kw], list): + cleaned[kw] = [sanitize_json_schema(v) for v in cleaned[kw]] + + return cleaned - Uses owner-task actor pattern to ensure async context manager enter/exit - occur in the same task, preventing anyio CancelScope cross-task exit - which causes CPU 100% usage. - """ + +class BrowserMoveStdioClient(StdioClient): + """StdioClient with timeouts, retries and reconnect; one owner task enters and exits the context.""" def __init__(self, config): super().__init__(config) @@ -36,15 +58,8 @@ def __init__(self, config): self._owner_close: asyncio.Event = asyncio.Event() self._connect_result: Optional[bool] = None self._connect_exception: Optional[Exception] = None - # Strong reference to an owner task that did not finish within - # _force_close's cancel grace period, kept so the task object and - # its exit stack are not GC'd while aclose() is still pending. self._leaked_owner_task: Optional[asyncio.Task] = None - # ------------------------------------------------------------------ - # Helpers - # ------------------------------------------------------------------ - def _resolve_timeout(self, timeout: float = NO_TIMEOUT, *, default_s: float = 60.0) -> float: """Return an effective timeout in seconds for MCP operations.""" try: @@ -66,7 +81,7 @@ def _resolve_timeout(self, timeout: float = NO_TIMEOUT, *, default_s: float = 60 return configured @staticmethod - def _is_retryable_transport_error(error: Exception) -> bool: + def is_transport_error(error: Exception) -> bool: name = type(error).__name__.lower() text = str(error).lower() markers = ( @@ -87,10 +102,6 @@ async def _reconnect(self, *, timeout: float = NO_TIMEOUT) -> bool: await self.disconnect(timeout=timeout) return await self.connect(timeout=timeout) - # ------------------------------------------------------------------ - # Overrides — include exit-stack reset fix missing from base class - # ------------------------------------------------------------------ - async def _run_owner(self): from mcp import ClientSession, StdioServerParameters from mcp.client.stdio import stdio_client @@ -153,7 +164,6 @@ async def _run_owner(self): self._read = None self._write = None self._exit_stack = AsyncExitStack() - # Owner task completed its own aclose() — no leak for this run. self._leaked_owner_task = None async def connect(self, *, timeout: float = NO_TIMEOUT) -> bool: @@ -175,7 +185,7 @@ async def connect(self, *, timeout: float = NO_TIMEOUT) -> bool: else: await asyncio.wait_for(self._owner_ready.wait(), timeout=timeout) except asyncio.TimeoutError: - logger.error(f"Stdio connection timeout") + logger.error("Stdio connection timeout") await self._force_close() return False @@ -202,11 +212,11 @@ async def disconnect(self, *, timeout: float = NO_TIMEOUT) -> bool: else: await asyncio.wait_for(self._owner_task, timeout=timeout) except asyncio.TimeoutError: - logger.error(f"Stdio disconnect timeout") + logger.error("Stdio disconnect timeout") await self._force_close() return False except asyncio.CancelledError: - logger.error(f"Stdio disconnect cancelled") + logger.error("Stdio disconnect cancelled") await self._force_close() raise except Exception as e: @@ -227,23 +237,13 @@ async def _force_close(self): try: await asyncio.wait_for(self._owner_task, timeout=2.0) except asyncio.CancelledError: - logger.warning( - "Stdio client owner task was cancelled during graceful close" - ) + logger.warning("Stdio client owner task was cancelled during graceful close") except Exception as e: - logger.warning( - "Stdio client owner task close raised exception: %r", e - ) - # Owner task did not finish within cancel grace period — it may - # still be stuck inside _exit_stack.aclose() (e.g., subprocess - # stdin closure blocked). Keep a strong reference so the task - # object is not GC'd while we clear our handle, and surface a - # warning so operators can detect subprocess pipe leaks. + logger.warning("Stdio client owner task close raised exception: %r", e) if not self._owner_task.done(): leaked_owner_task = self._owner_task logger.warning( - "Stdio client owner task did not terminate after cancel; " - "subprocess pipe may leak. task=%r", + "Stdio client owner task did not terminate after cancel; subprocess pipe may leak. task=%r", leaked_owner_task, ) except (asyncio.CancelledError, Exception): @@ -253,9 +253,6 @@ async def _force_close(self): self._client = None self._read = None self._write = None - # Reset the exit stack so a subsequent connect() starts fresh; the - # leaked owner task (if any) still holds the old stack and will run - # its own aclose() independently. self._exit_stack = AsyncExitStack() self._leaked_owner_task = leaked_owner_task self._is_disconnected = True @@ -294,22 +291,18 @@ async def list_tools(self, *, timeout: float = NO_TIMEOUT) -> List[Any]: if connected: continue logger.error(f"Stdio list_tools timed out after {effective_timeout:.1f}s") - raise RuntimeError( - f"Stdio list_tools timed out after {effective_timeout:.1f}s" - ) from e + raise RuntimeError(f"Stdio list_tools timed out after {effective_timeout:.1f}s") from e except Exception as e: - if attempt == 0 and self._is_retryable_transport_error(e): - logger.warning( - f"Stdio list_tools retry after reconnect: type={type(e).__name__}, repr={e!r}" - ) + if attempt == 0 and self.is_transport_error(e): + logger.warning(f"Stdio list_tools retry after reconnect: type={type(e).__name__}, repr={e!r}") connected = await self._reconnect(timeout=timeout) if connected: continue logger.error(f"Failed to list tools via Stdio: {e}") raise - async def call_tool(self, tool_name: str, arguments: dict, *, timeout: float = NO_TIMEOUT) -> Any: - """Call tool via Stdio, with auto-reconnect, timeout, and multi-content extraction.""" + async def call_tool_result(self, tool_name: str, arguments: dict, *, timeout: float = NO_TIMEOUT) -> Any: + """The ``CallToolResult`` of one call via Stdio, with auto-reconnect and timeout.""" if not self._session: connected = await self._reconnect(timeout=timeout) if not connected: @@ -318,7 +311,7 @@ async def call_tool(self, tool_name: str, arguments: dict, *, timeout: float = N effective_timeout = self._resolve_timeout(timeout) for attempt in range(2): try: - browser_agent_log_info( + logger.info( f"Calling tool '{tool_name}' via Stdio with arguments_summary: " f"{summarize_tool_arguments_for_log(tool_name, arguments)}" ) @@ -326,54 +319,22 @@ async def call_tool(self, tool_name: str, arguments: dict, *, timeout: float = N self._session.call_tool(tool_name, arguments=arguments), timeout=effective_timeout, ) - result_content = None - if tool_result.content and len(tool_result.content) > 0: - chunks = [] - for item in tool_result.content: - text = getattr(item, "text", None) - if text: - chunks.append(text) - continue - - uri = getattr(item, "uri", None) - if uri: - chunks.append(str(uri)) - continue - - data = getattr(item, "data", None) - if data is not None: - mime = ( - getattr(item, "mimeType", None) - or getattr(item, "mime_type", None) - or "application/octet-stream" - ) - chunks.append(f"[binary content: {mime}]") - continue - - chunks.append(str(item)) - - if chunks: - result_content = "\n".join(chunks) - browser_agent_log_info(f"Tool '{tool_name}' call completed via Stdio") - return preserve_mcp_tool_result_status(tool_result, result_content) + logger.info(f"Tool '{tool_name}' call completed via Stdio") + return tool_result except asyncio.TimeoutError as e: if attempt == 0: - browser_agent_log_warning( + logger.warning( f"Stdio tool call '{tool_name}' timed out after" f" {effective_timeout:.1f}s, retrying after reconnect" ) connected = await self._reconnect(timeout=effective_timeout) if connected: continue - logger.error( - f"Tool call timed out via Stdio: tool='{tool_name}', timeout={effective_timeout:.1f}s" - ) - raise RuntimeError( - f"Stdio tool call timed out for '{tool_name}' after {effective_timeout:.1f}s" - ) from e + logger.error(f"Tool call timed out via Stdio: tool='{tool_name}', timeout={effective_timeout:.1f}s") + raise RuntimeError(f"Stdio tool call timed out for '{tool_name}' after {effective_timeout:.1f}s") from e except Exception as e: - if attempt == 0 and self._is_retryable_transport_error(e): - browser_agent_log_warning( + if attempt == 0 and self.is_transport_error(e): + logger.warning( f"Stdio tool call '{tool_name}' retry after reconnect: type={type(e).__name__}, repr={e!r}" ) connected = await self._reconnect(timeout=timeout) @@ -383,9 +344,59 @@ async def call_tool(self, tool_name: str, arguments: dict, *, timeout: float = N f"Tool call failed via Stdio: type={type(e).__name__}, repr={e!r}", exc_info=True, ) - raise RuntimeError( - f"Stdio tool call failed for '{tool_name}': {type(e).__name__}: {e!r}" - ) from e + raise RuntimeError(f"Stdio tool call failed for '{tool_name}': {type(e).__name__}: {e!r}") from e + + @property + def connected(self) -> bool: + return self._session is not None + + async def call_tool_once(self, tool_name: str, arguments: dict) -> Any: + """The ``CallToolResult`` of one call via Stdio, never re-sent: a failed call may already have run.""" + if not self._session: + raise RuntimeError("Not connected to Stdio server") + logger.info( + f"Calling tool '{tool_name}' via Stdio with arguments_summary: " + f"{summarize_tool_arguments_for_log(tool_name, arguments)}" + ) + tool_result = await asyncio.wait_for( + self._session.call_tool(tool_name, arguments=arguments), + timeout=self._resolve_timeout(), + ) + logger.info(f"Tool '{tool_name}' call completed via Stdio") + return tool_result + + async def call_tool(self, tool_name: str, arguments: dict, *, timeout: float = NO_TIMEOUT) -> Any: + """Call tool via Stdio, with auto-reconnect and multi-content extraction.""" + tool_result = await self.call_tool_result(tool_name, arguments, timeout=timeout) + result_content = None + if tool_result.content and len(tool_result.content) > 0: + chunks = [] + for item in tool_result.content: + text = getattr(item, "text", None) + if text: + chunks.append(text) + continue + + uri = getattr(item, "uri", None) + if uri: + chunks.append(str(uri)) + continue + + data = getattr(item, "data", None) + if data is not None: + mime = ( + getattr(item, "mimeType", None) + or getattr(item, "mime_type", None) + or "application/octet-stream" + ) + chunks.append(f"[binary content: {mime}]") + continue + + chunks.append(str(item)) + + if chunks: + result_content = "\n".join(chunks) + return preserve_mcp_tool_result_status(tool_result, result_content) async def get_tool_info(self, tool_name: str, *, timeout: float = NO_TIMEOUT) -> Optional[Any]: """Get specific tool info via Stdio.""" diff --git a/openjiuwen/harness/tools/browser_move/clients/streamable_http_client.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/streamable_http_client.py similarity index 68% rename from openjiuwen/harness/tools/browser_move/clients/streamable_http_client.py rename to openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/streamable_http_client.py index 52fc775a59..7c80178275 100644 --- a/openjiuwen/harness/tools/browser_move/clients/streamable_http_client.py +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/clients/streamable_http_client.py @@ -2,16 +2,14 @@ # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. import asyncio from contextlib import AsyncExitStack -from typing import Any, List, Optional, Dict +from typing import Any, Dict, List, Optional from openjiuwen.core.common.logging import logger from openjiuwen.core.foundation.tool import McpServerConfig, McpToolCard from openjiuwen.core.foundation.tool.mcp.base import NO_TIMEOUT -from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_logging import ( - browser_agent_log_info, - browser_agent_log_warning, -) + from .logging_utils import summarize_tool_arguments_for_log + try: from openjiuwen.core.foundation.tool.mcp.client.streamable_http_client import StreamableHttpClient except ModuleNotFoundError: @@ -19,11 +17,7 @@ class BrowserMoveStreamableHttpClient(StreamableHttpClient): - """browser_move extension of StreamableHttpClient. - - Adds reconnect lock, retryable transport error detection, auto-reconnect, - and one retry for list_tools/call_tool after reconnect. - """ + """StreamableHttpClient with reconnect and one retry after it.""" def __init__( self, @@ -36,7 +30,7 @@ def __init__( self._reconnect_lock = asyncio.Lock() @staticmethod - def _is_retryable_transport_error(error: Exception) -> bool: + def is_transport_error(error: Exception) -> bool: name = error.__class__.__name__.lower() text = str(error).lower() markers = ( @@ -73,9 +67,7 @@ async def connect(self, *, retry_times: int = 1, timeout: float = NO_TIMEOUT) -> timeout=actual_timeout, auth=self._auth_provider, ) - self._read, self._write, self._get_session_id = await self._exit_stack.enter_async_context( - self._client - ) + self._read, self._write, self._get_session_id = await self._exit_stack.enter_async_context(self._client) self._session = await self._exit_stack.enter_async_context( ClientSession(self._read, self._write, sampling_callback=None) ) @@ -91,8 +83,7 @@ async def connect(self, *, retry_times: int = 1, timeout: float = NO_TIMEOUT) -> await self.disconnect(timeout=timeout) except Exception as e: logger.error( - f"Streamable HTTP connection failed to {self._server_path} " - f"(attempt {attempt}/{attempts}): {e}" + f"Streamable HTTP connection failed to {self._server_path} (attempt {attempt}/{attempts}): {e}" ) await self.disconnect(timeout=timeout) return False @@ -137,7 +128,7 @@ async def list_tools(self, *, timeout: float = NO_TIMEOUT) -> List[Any]: logger.info(f"Retrieved {len(tools_list)} tools from Streamable HTTP server") return tools_list except Exception as e: - if attempt == 0 and self._is_retryable_transport_error(e): + if attempt == 0 and self.is_transport_error(e): logger.warning(f"Streamable HTTP list_tools retry after reconnect: {e}") connected = await self._reconnect(timeout=timeout) if connected: @@ -145,8 +136,8 @@ async def list_tools(self, *, timeout: float = NO_TIMEOUT) -> List[Any]: logger.error(f"Failed to list tools via Streamable HTTP: {e}") raise - async def call_tool(self, tool_name: str, arguments: dict, *, timeout: float = NO_TIMEOUT) -> Any: - """Call tool via Streamable HTTP, with auto-reconnect on transport errors.""" + async def call_tool_result(self, tool_name: str, arguments: dict, *, timeout: float = NO_TIMEOUT) -> Any: + """The ``CallToolResult`` of one call via Streamable HTTP, with auto-reconnect on transport errors.""" if not self._session: connected = await self._reconnect(timeout=timeout) if not connected: @@ -154,52 +145,71 @@ async def call_tool(self, tool_name: str, arguments: dict, *, timeout: float = N for attempt in range(2): try: - browser_agent_log_info( + logger.info( f"Calling tool '{tool_name}' via Streamable HTTP with arguments_summary: " f"{summarize_tool_arguments_for_log(tool_name, arguments)}" ) tool_result = await self._session.call_tool(tool_name, arguments=arguments) - result_content = None - if tool_result.content and len(tool_result.content) > 0: - chunks = [] - for item in tool_result.content: - text = getattr(item, "text", None) - if text: - chunks.append(text) - continue - - uri = getattr(item, "uri", None) - if uri: - chunks.append(str(uri)) - continue - - data = getattr(item, "data", None) - if data is not None: - mime = ( - getattr(item, "mimeType", None) - or getattr(item, "mime_type", None) - or "application/octet-stream" - ) - chunks.append(f"[binary content: {mime}]") - continue - - chunks.append(str(item)) - - if chunks: - result_content = "\n".join(chunks) - browser_agent_log_info(f"Tool '{tool_name}' call completed via Streamable HTTP") - return result_content + logger.info(f"Tool '{tool_name}' call completed via Streamable HTTP") + return tool_result except Exception as e: - if attempt == 0 and self._is_retryable_transport_error(e): - browser_agent_log_warning( - f"Streamable HTTP tool call '{tool_name}' retry after reconnect: {e}" - ) + if attempt == 0 and self.is_transport_error(e): + logger.warning(f"Streamable HTTP tool call '{tool_name}' retry after reconnect: {e}") connected = await self._reconnect(timeout=timeout) if connected: continue logger.error(f"Tool call failed via Streamable HTTP: {e}") raise + @property + def connected(self) -> bool: + return self._session is not None + + async def call_tool_once(self, tool_name: str, arguments: dict) -> Any: + """The ``CallToolResult`` of one call via Streamable HTTP, never re-sent: a failed call may already have run.""" + if not self._session: + raise RuntimeError("Not connected to Streamable HTTP server") + logger.info( + f"Calling tool '{tool_name}' via Streamable HTTP with arguments_summary: " + f"{summarize_tool_arguments_for_log(tool_name, arguments)}" + ) + tool_result = await self._session.call_tool(tool_name, arguments=arguments) + logger.info(f"Tool '{tool_name}' call completed via Streamable HTTP") + return tool_result + + async def call_tool(self, tool_name: str, arguments: dict, *, timeout: float = NO_TIMEOUT) -> Any: + """Call tool via Streamable HTTP, with auto-reconnect and multi-content extraction.""" + tool_result = await self.call_tool_result(tool_name, arguments, timeout=timeout) + result_content = None + if tool_result.content and len(tool_result.content) > 0: + chunks = [] + for item in tool_result.content: + text = getattr(item, "text", None) + if text: + chunks.append(text) + continue + + uri = getattr(item, "uri", None) + if uri: + chunks.append(str(uri)) + continue + + data = getattr(item, "data", None) + if data is not None: + mime = ( + getattr(item, "mimeType", None) + or getattr(item, "mime_type", None) + or "application/octet-stream" + ) + chunks.append(f"[binary content: {mime}]") + continue + + chunks.append(str(item)) + + if chunks: + result_content = "\n".join(chunks) + return result_content + async def get_tool_info(self, tool_name: str, *, timeout: float = NO_TIMEOUT) -> Optional[Any]: """Get specific tool info via Streamable HTTP.""" tools = await self.list_tools(timeout=timeout) @@ -209,3 +219,13 @@ async def get_tool_info(self, tool_name: str, *, timeout: float = NO_TIMEOUT) -> return tool logger.warning(f"Tool '{tool_name}' not found via Streamable HTTP") return None + + async def ping(self, *, timeout: float = 5.0) -> bool: + """Return True if the server still answers.""" + if not self._session: + return False + try: + await asyncio.wait_for(self._session.list_tools(), timeout=timeout) + return True + except Exception: + return False diff --git a/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/config.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/config.py new file mode 100644 index 0000000000..a82546dd04 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/config.py @@ -0,0 +1,100 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from typing import Any + +from openjiuwen.core.foundation.tool import McpServerConfig + +SERVER_CAPABILITIES = ("pdf", "vision", "devtools", "config", "network", "storage", "testing") + + +def _ensure_capabilities(args: list[str], required_capabilities: Iterable[str]) -> list[str]: + normalized_args: list[str] = [] + capabilities: list[str] = [] + index = 0 + while index < len(args): + arg = args[index] + if arg == "--caps": + if index + 1 < len(args) and not args[index + 1].startswith("--"): + capabilities.extend(args[index + 1].split(",")) + index += 2 + continue + elif arg.startswith("--caps="): + capabilities.extend(arg.removeprefix("--caps=").split(",")) + index += 1 + continue + normalized_args.append(arg) + index += 1 + + capabilities = list(dict.fromkeys(capability.strip() for capability in capabilities if capability.strip())) + for capability in required_capabilities: + if capability not in capabilities: + capabilities.append(capability) + normalized_args.append(f"--caps={','.join(capabilities)}") + return normalized_args + + +def playwright_mcp_config( + settings: Any, + *, + cdp_url: str = "", + launch_args: Sequence[str] | None = None, + key: str = "", + required_capability: Iterable[str] = (), +) -> McpServerConfig: + """The ``@playwright/mcp`` server config from the browser settings; ``cdp_url`` wins over theirs.""" + args = _ensure_capabilities( + list(settings.launch_args if launch_args is None else launch_args), + (*SERVER_CAPABILITIES, *required_capability), + ) + extension_mode = settings.provisioning == "extension" or settings.mcp_extension + + env_map: dict[str, str] = dict(settings.mcp_env) + if extension_mode: + env_map["PLAYWRIGHT_MCP_EXTENSION"] = "true" + if settings.mcp_extension_token: + env_map["PLAYWRIGHT_MCP_EXTENSION_TOKEN"] = settings.mcp_extension_token + if "--extension" not in args: + args.append("--extension") + else: + cdp_endpoint = cdp_url or settings.cdp_url + if cdp_endpoint: + if settings.mcp_device: + raise ValueError("PLAYWRIGHT_MCP_DEVICE is not supported with CDP endpoint mode.") + env_map["PLAYWRIGHT_MCP_CDP_ENDPOINT"] = cdp_endpoint + if not settings.mcp_browser: + env_map["PLAYWRIGHT_MCP_BROWSER"] = "chrome" + if settings.cdp_headers: + env_map["PLAYWRIGHT_MCP_CDP_HEADERS"] = settings.cdp_headers + if settings.cdp_timeout: + env_map["PLAYWRIGHT_MCP_CDP_TIMEOUT"] = settings.cdp_timeout + + params: dict[str, Any] = { + "command": settings.mcp_command, + "args": args, + "cwd": str(settings.mcp_cwd), + } + if settings.mcp_timeout_s > 0: + params["timeout_s"] = settings.mcp_timeout_s + if env_map: + params["env"] = env_map + + server_id = "playwright_official_stdio" + server_name = "playwright-official" + if key: + server_id = f"{server_id}__{key}" + server_name = f"{server_name}-{key}" + + return McpServerConfig( + server_id=server_id, + server_name=server_name, + server_path="stdio://playwright", + client_type="stdio", + params=params, + ) + + +__all__ = ["SERVER_CAPABILITIES", "playwright_mcp_config"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/driver.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/driver.py new file mode 100644 index 0000000000..5040f53fcc --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/driver.py @@ -0,0 +1,535 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""``PlaywrightMcpDriver``: the driver contract on top of a ``@playwright/mcp`` server the driver owns.""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import json +import re +import shutil +import uuid +from dataclasses import dataclass +from importlib import resources +from pathlib import Path +from types import MappingProxyType +from typing import Any + +from openjiuwen.core.foundation.tool import McpServerConfig +from openjiuwen.core.foundation.tool.mcp.base import mcp_model_tool_prefix +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + ActionTimeout, + Click, + CloseContext, + ConnectionLost, + Drag, + DriverError, + DriverInfo, + DriverRef, + HandleDialog, + HandlerDriver, + Health, + Hover, + Locator, + NativeTool, + Observation, + Observe, + PointRef, + PressKey, + ProvisionedDevice, + RunScript, + Screenshot, + Scroll, + SwitchContext, + TypeText, + Unsupported, + Wait, + handles, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.clients import ( + BrowserMoveStdioClient, + BrowserMoveStreamableHttpClient, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.config import playwright_mcp_config +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.result import decode, result_text +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.snapshot import ( + parse_snapshot, + parse_tabs, + split_sections, + without_boxes, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.tools import DECODERS +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FillForm, + GetConsole, + GetNetwork, + GetPdf, + Navigate, + OpenContext, + SelectOption, + UploadFile, + upload_paths, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext + +_PACKAGE_VERSION = re.compile(r"^@playwright/mcp@(?P<version>.+)$") +_PING_TIMEOUT_S = 5.0 +_PROBES = "openjiuwen.harness.tools.browser_move.surfaces.browser.probes" + +_TOOL_OF: dict[type[Action], str] = { + Click: "browser_click", + TypeText: "browser_type", + PressKey: "browser_press_key", + Hover: "browser_hover", + Drag: "browser_drag", + Scroll: "browser_mouse_wheel", + SelectOption: "browser_select_option", + FillForm: "browser_fill_form", + Drop: "browser_drop", + UploadFile: "browser_file_upload", + HandleDialog: "browser_handle_dialog", + Wait: "browser_wait_for", + RunScript: "browser_evaluate", + Navigate: "browser_navigate", + Back: "browser_navigate_back", + OpenContext: "browser_tabs", + SwitchContext: "browser_tabs", + CloseContext: "browser_tabs", + GetPdf: "browser_pdf_save", + GetConsole: "browser_console_messages", + GetNetwork: "browser_network_requests", +} +_UNBACKED_TOOLS = frozenset({"browser_find"}) + + +@dataclass(frozen=True) +class PlaywrightMcpConfig: + """What ``start()`` needs: the server's command line, environment and working directory; the upload folder.""" + + server: McpServerConfig + upload_root: Path | None = None + + @property + def output_dir(self) -> Path: + return Path(self.server.params["cwd"]) + + @property + def version(self) -> str: + for arg in self.server.params.get("args") or (): + match = _PACKAGE_VERSION.match(str(arg)) + if match is not None: + return match.group("version") + return "" + + +def _target(ref: DriverRef | Locator) -> str: + if isinstance(ref, Locator): + if ref.text is not None or not ref.css: + raise Unsupported("PlaywrightMcpDriver addresses a Locator by css only", dispatched=False) + return ref.css + return ref.ref + + +def _aim(ref: DriverRef | Locator, what: str) -> dict[str, Any]: + target = _target(ref) + return {"element": f"{what} {target}", "target": target} + + +def _snapshot_args(command: Observe) -> dict[str, Any]: + if not command.scoped: + return {"boxes": True} + args: dict[str, Any] = {} + if command.depth is not None: + args["depth"] = command.depth + if isinstance(command.root, PointRef): + raise Unsupported("browser_snapshot scopes to an element ref or a css target, not a point", dispatched=False) + if command.root is not None: + args["target"] = _target(command.root) + return args + + +def _point(ref: PointRef) -> dict[str, int]: + return {"x": ref.x, "y": ref.y} + + +def _staged(paths: tuple[str, ...], upload_root: Path | None, root: Path) -> tuple[list[str], list[Path]]: + root = root.resolve() + staged = [] + created = [] + for source in upload_paths(paths, upload_root): + if source.is_relative_to(root) or not source.is_file(): + staged.append(str(source)) + continue + target = root / "uploads" / uuid.uuid4().hex[:12] / source.name + target.parent.mkdir(parents=True) + created.append(target.parent) + shutil.copy2(source, target) + staged.append(str(target)) + return staged, created + + +def _with_init_script(server: McpServerConfig) -> McpServerConfig: + """The server config with the clipboard guard as its init script; a remote server keeps its own setup.""" + if server.client_type == "streamable-http": + return server + guard = str(resources.files(_PROBES).joinpath("clipboard_guard.js")) + return McpServerConfig( + server_id=server.server_id, + server_name=server.server_name, + server_path=server.server_path, + client_type=server.client_type, + params={**server.params, "args": [*(server.params.get("args") or ()), "--init-script", guard]}, + ) + + +def _unstaged(directories: list[Path]) -> None: + for directory in directories: + shutil.rmtree(directory, ignore_errors=True) + with contextlib.suppress(OSError): + directories[0].parent.rmdir() + + +class PlaywrightMcpDriver(HandlerDriver): + """Runs contract commands on a Chrome through a ``@playwright/mcp`` server started by ``start()``.""" + + def __init__(self, config: PlaywrightMcpConfig | None = None) -> None: + self._config = config + self._client: BrowserMoveStdioClient | BrowserMoveStreamableHttpClient | None = None + self._context: BrowserContext | None = None + self._tool_hints: dict[type[Action], str] = {} + self._native_tools: dict[str, NativeTool] = {} + self._staged_dirs: list[Path] = [] + + @property + def config(self) -> PlaywrightMcpConfig | None: + return self._config + + def describe(self) -> DriverInfo: + version = self._config.version if self._config is not None else "" + return DriverInfo( + name="playwright_mcp", + surfaces=frozenset({"browser"}), + versions=MappingProxyType({"@playwright/mcp": version} if version else {}), + observes=frozenset({"elements", "screenshot", "native_text", "surface"}), + tool_hints=MappingProxyType(dict(self._tool_hints)), + native_tools=MappingProxyType(dict(self._native_tools)), + tool_name_prefix=mcp_model_tool_prefix(self._config.server.server_name) if self._config is not None else "", + display_name="Playwright MCP", + ) + + async def start(self, config: PlaywrightMcpConfig | None = None) -> None: + config = config or self._config + if config is None: + raise DriverError("PlaywrightMcpDriver.start() needs a PlaywrightMcpConfig", dispatched=False) + if self._client is not None: + await self.stop() + if config.server.client_type == "streamable-http": + client: BrowserMoveStdioClient | BrowserMoveStreamableHttpClient = BrowserMoveStreamableHttpClient( + config.server + ) + else: + client = BrowserMoveStdioClient(_with_init_script(config.server)) + if not await client.connect(): + raise ConnectionLost(f"could not start {config.server.server_name}", dispatched=False) + try: + cards = await client.list_tools() + except Exception as exc: + await client.disconnect() + raise ConnectionLost(f"{config.server.server_name}: {exc}", dispatched=False) from exc + descriptions = {card.name: card.description for card in cards} + self._tool_hints = {command: descriptions[tool] for command, tool in _TOOL_OF.items() if descriptions.get(tool)} + self._native_tools = { + card.name: NativeTool(card.description, card.input_params, DECODERS.get(card.name)) + for card in cards + if card.name in DECODERS or card.name in _UNBACKED_TOOLS + } + self._config = config + self._client = client + self._context = None + + async def stop(self) -> None: + client, self._client = self._client, None + if client is not None: + await client.disconnect() + staged, self._staged_dirs = self._staged_dirs, [] + if staged: + await asyncio.to_thread(_unstaged, staged) + + async def health(self) -> Health: + if self._client is None: + return Health(ok=False, detail="not started") + if await self._client.ping(timeout=_PING_TIMEOUT_S): + return Health(ok=True) + return Health(ok=False, detail="the server does not answer") + + async def observe(self, command: Observe) -> Observation: + raw = await self._call("browser_snapshot", _snapshot_args(command)) + decoded = await self._decode("browser_snapshot", raw) + if not decoded.ok: + return await self._observe_modal_state(raw) + elements, header = parse_snapshot(result_text(raw)) + if header is None: + raise DriverError("browser_snapshot reported no page", dispatched=True) + screenshot = None + if command.include_screenshot: + shot = await self._run("browser_take_screenshot", {"type": "png", "fullPage": command.full_page}) + screenshot = shot.value if isinstance(shot.value, Screenshot) else None + return Observation( + context=header.focused, + elements=elements, + screenshot=screenshot, + native_text=without_boxes(result_text(raw)), + surface=decoded.surface, + ) + + async def _observe_modal_state(self, refused: dict[str, Any]) -> Observation: + raw = await self._call("browser_tabs", {"action": "list"}) + listed = await self._decode("browser_tabs", raw) + header = parse_tabs(split_sections(result_text(raw)).get("Result", "")) + if header is None: + raise DriverError("browser_tabs reported no page", dispatched=True) + return Observation(context=header.focused, native_text=result_text(refused), surface=listed.surface) + + @handles(Click, refs={DriverRef, Locator, PointRef}) + async def _click(self, action: Click) -> ActionResult: + if isinstance(action.target, PointRef): + if action.modifiers: + raise Unsupported("browser_mouse_click_xy takes no modifiers", dispatched=False) + args: dict[str, Any] = _point(action.target) + if action.button != "left": + args["button"] = action.button + if action.count != 1: + args["clickCount"] = action.count + return await self._run("browser_mouse_click_xy", args) + if action.count > 2: + raise Unsupported("browser_click clicks once or twice", dispatched=False) + args = _aim(action.target, "click") + if action.count == 2: + args["doubleClick"] = True + if action.button != "left": + args["button"] = action.button + if action.modifiers: + args["modifiers"] = list(action.modifiers) + return await self._run("browser_click", args) + + @handles(TypeText, refs={DriverRef, Locator}) + async def _type_text(self, action: TypeText) -> ActionResult: + if action.target is None: + raise Unsupported("browser_type types only into a target element", dispatched=False) + if not action.clear and not action.slowly: + raise Unsupported("browser_type replaces the content unless it types slowly", dispatched=False) + args = {**_aim(action.target, "type into"), "text": action.text} + if action.submit: + args["submit"] = True + if action.slowly: + args["slowly"] = True + return await self._run("browser_type", args, sensitive=action.sensitive) + + @handles(PressKey) + async def _press_key(self, action: PressKey) -> ActionResult: + return await self._run("browser_press_key", {"key": action.key}) + + @handles(Hover, refs={DriverRef, Locator, PointRef}) + async def _hover(self, action: Hover) -> ActionResult: + if isinstance(action.target, PointRef): + return await self._run("browser_mouse_move_xy", _point(action.target)) + return await self._run("browser_hover", _aim(action.target, "hover")) + + @handles(Drag, refs={DriverRef, Locator, PointRef}) + async def _drag(self, action: Drag) -> ActionResult: + points = isinstance(action.source, PointRef), isinstance(action.destination, PointRef) + if all(points): + start, end = _point(action.source), _point(action.destination) + args = {"startX": start["x"], "startY": start["y"], "endX": end["x"], "endY": end["y"]} + return await self._run("browser_mouse_drag_xy", args) + if any(points): + raise Unsupported("PlaywrightMcpDriver drags between two refs or two points", dispatched=False) + start_target, end_target = _target(action.source), _target(action.destination) + args = { + "startElement": f"drag {start_target}", + "startTarget": start_target, + "endElement": f"drop on {end_target}", + "endTarget": end_target, + } + return await self._run("browser_drag", args) + + @handles(Scroll) + async def _scroll(self, action: Scroll) -> ActionResult: + return await self._run("browser_mouse_wheel", {"deltaX": action.dx, "deltaY": action.dy}) + + @handles(SelectOption, refs={DriverRef, Locator}) + async def _select_option(self, action: SelectOption) -> ActionResult: + args = {**_aim(action.target, "select in"), "values": list(action.values)} + return await self._run("browser_select_option", args) + + @handles(FillForm, refs={DriverRef, Locator}) + async def _fill_form(self, action: FillForm) -> ActionResult: + fields = [] + for field in action.fields: + target = _target(field.target) + fields.append( + { + "name": f"{field.kind.value} {target}", + "type": field.kind.value, + "target": target, + "value": field.value, + } + ) + sensitive = any(field.sensitive for field in action.fields) + return await self._run("browser_fill_form", {"fields": fields}, sensitive=sensitive) + + @handles(Drop, refs={DriverRef, Locator}) + async def _drop(self, action: Drop) -> ActionResult: + args = _aim(action.target, "drop on") + if action.paths: + try: + args["paths"] = await self._stage(action.paths) + except ValueError as exc: + return ActionResult(ok=False, detail=str(exc)) + if action.data: + args["data"] = dict(action.data) + return await self._run("browser_drop", args) + + @handles(UploadFile) + async def _upload_file(self, action: UploadFile) -> ActionResult: + if not action.paths: + return await self._run("browser_file_upload", {}) + try: + paths = await self._stage(action.paths) + except ValueError as exc: + return ActionResult(ok=False, detail=str(exc)) + return await self._run("browser_file_upload", {"paths": paths}) + + @handles(HandleDialog) + async def _handle_dialog(self, action: HandleDialog) -> ActionResult: + args: dict[str, Any] = {"accept": action.accept} + if action.prompt_text is not None: + args["promptText"] = action.prompt_text + return await self._run("browser_handle_dialog", args) + + @handles(Wait) + async def _wait(self, action: Wait) -> ActionResult: + if action.until_text is not None: + return await self._run("browser_wait_for", {"text": action.until_text}) + return await self._run("browser_wait_for", {"time": action.seconds}) + + @handles(RunScript, refs={DriverRef, Locator}) + async def _run_script(self, action: RunScript) -> ActionResult: + args: dict[str, Any] = {} + if action.target is not None: + args = _aim(action.target, "evaluate on") + function = action.code + if action.args is not None: + params = f"element, {json.dumps(action.args)}" if action.target is not None else json.dumps(action.args) + head = "(element)" if action.target is not None else "()" + function = f"{head} => ({action.code})({params})" + return await self._run("browser_evaluate", {**args, "function": function}, sensitive=action.sensitive) + + @handles(Navigate) + async def _navigate(self, action: Navigate) -> ActionResult: + return await self._run("browser_navigate", {"url": action.url}) + + @handles(Back) + async def _back(self, action: Back) -> ActionResult: + del action + return await self._run("browser_navigate_back", {}) + + @handles(OpenContext) + async def _open_context(self, action: OpenContext) -> ActionResult: + args: dict[str, Any] = {"action": "new"} + if action.url is not None: + args["url"] = action.url + result = await self._run("browser_tabs", args) + opened = result.surface.tabs[-1] if isinstance(result.surface, BrowserContext) else None + return dataclasses.replace(result, value=opened) + + @handles(SwitchContext) + async def _switch_context(self, action: SwitchContext) -> ActionResult: + return await self._run("browser_tabs", {"action": "select", "index": int(action.context.id)}) + + @handles(CloseContext) + async def _close_context(self, action: CloseContext) -> ActionResult: + result = await self._run("browser_tabs", {"action": "close", "index": int(action.context.id)}) + return dataclasses.replace(result, source_context=action.context) + + @handles(GetPdf) + async def _get_pdf(self, action: GetPdf) -> ActionResult: + args = {"filename": action.filename} if action.filename is not None else {} + return await self._run("browser_pdf_save", args) + + @handles(GetConsole) + async def _get_console(self, action: GetConsole) -> ActionResult: + args: dict[str, Any] = {"level": action.level} + if action.all: + args["all"] = True + return await self._run("browser_console_messages", args) + + @handles(GetNetwork) + async def _get_network(self, action: GetNetwork) -> ActionResult: + args: dict[str, Any] = {"static": action.static} + if action.filter is not None: + args["filter"] = action.filter + return await self._run("browser_network_requests", args) + + async def _run(self, tool: str, arguments: dict[str, Any], *, sensitive: bool = False) -> ActionResult: + return await self._decode(tool, await self._call(tool, arguments), sensitive=sensitive) + + async def _call(self, tool: str, arguments: dict[str, Any]) -> dict[str, Any]: + if self._client is None: + raise ConnectionLost("PlaywrightMcpDriver is not started", dispatched=False) + if not self._client.connected: + raise ConnectionLost(f"{tool}: not connected to the server", dispatched=False) + try: + result = await self._client.call_tool_once(tool, arguments) + except TimeoutError as exc: + raise ActionTimeout(f"{tool}: the server did not answer in time", dispatched=True) from exc + except Exception as exc: + if self._client.is_transport_error(exc): + raise ConnectionLost(f"{tool}: {exc}", dispatched=True) from exc + raise DriverError(f"{tool}: {type(exc).__name__}: {exc}", dispatched=False) from exc + return result.model_dump(mode="json", by_alias=True, exclude_none=True) + + async def _decode(self, tool: str, raw: dict[str, Any], *, sensitive: bool = False) -> ActionResult: + result = await asyncio.to_thread( + decode, tool, raw, output_dir=self._output_dir(), before=self._context, sensitive=sensitive + ) + if isinstance(result.surface, BrowserContext): + self._context = result.surface + return result + + async def _stage(self, paths: tuple[str, ...]) -> list[str]: + upload_root = self._config.upload_root if self._config is not None else None + staged, created = await asyncio.to_thread(_staged, paths, upload_root, self._output_dir()) + self._staged_dirs.extend(created) + return staged + + def _output_dir(self) -> Path: + return self._config.output_dir if self._config is not None else Path.cwd() + + +def create_playwright_mcp_driver(device: ProvisionedDevice, surface_settings: Any) -> PlaywrightMcpDriver: + """Registry factory; the server attaches to ``device.endpoint`` over CDP on ``start()``.""" + server = playwright_mcp_config(surface_settings, cdp_url=device.endpoint) + given = getattr(surface_settings, "mcp_server", None) + if given is not None: + env = {key: value for key, value in (given.params.get("env") or {}).items() if key != "PLAYWRIGHT_MCP_DEVICE"} + if device.endpoint: + env = {"PLAYWRIGHT_MCP_BROWSER": "chrome", **env, "PLAYWRIGHT_MCP_CDP_ENDPOINT": device.endpoint} + server = McpServerConfig( + server_id=server.server_id, + server_name=server.server_name, + server_path=given.server_path, + client_type=given.client_type, + params={**given.params, "env": env}, + ) + return PlaywrightMcpDriver(PlaywrightMcpConfig(server, upload_root=getattr(surface_settings, "upload_root", None))) + + +__all__ = ["PlaywrightMcpConfig", "PlaywrightMcpDriver", "create_playwright_mcp_driver"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/result.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/result.py new file mode 100644 index 0000000000..6dec01edb7 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/result.py @@ -0,0 +1,225 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Decode a @playwright/mcp tool result into an ``ActionResult`` or a typed driver error.""" + +from __future__ import annotations + +import base64 +import dataclasses +import json +import re +import struct +from collections.abc import Mapping +from pathlib import Path +from typing import Any + +from openjiuwen.harness.tools.browser_move.contract import ( + ActionIntercepted, + ActionResult, + ActionTimeout, + ConnectionLost, + ContextRef, + DriverError, + Observation, + Screenshot, + StaleRef, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.snapshot import ( + PageHeader, + parse_elements, + parse_header, + parse_tabs, + snapshot_yaml, + split_sections, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext + +_NAVIGATING_TOOLS = frozenset({"browser_navigate", "browser_navigate_back", "browser_tabs"}) +_DROPPED_SECTIONS = frozenset({"Result", "Snapshot", "Events"}) +_CODE_SECTION = "Ran Playwright code" +_PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" + +_ANSI = re.compile(r"\x1b\[[0-9;]*m") +_FILE_LINK = re.compile(r"^- \[[^\]]*\]\((?P<path>[^)]+)\)$", re.MULTILINE) +_DIALOG = re.compile(r'^- \["(?P<type>[^"]+)" dialog with message "(?P<message>.*?)"\]: can be handled by', re.M | re.S) +_MODAL_REFUSAL = re.compile(r'Tool "[\w-]+" does not handle the modal state') +_STALE_REF = re.compile(r"Ref \S+ not found in the current page snapshot") +_FILE_ACCESS_DENIED = re.compile(r"File access denied: .+ is outside allowed roots") +_CLOSED = re.compile(r"Target page, context or browser has been closed|Failed to open a new tab") +_TIMEOUT = re.compile(r"TimeoutError:|Timeout \d+ms exceeded") +_LOG_PREFIX = re.compile(r"^\s*(?:\d+ × )?(?:- )?") +_PERFORMED = re.compile(r"^performing \w+ action$|^\w+ action done$") +_INTERCEPTED = re.compile(r"^(?P<by><.+?)(?: from (?P<within><.+?) subtree)? intercepts pointer events$") +_ACTIONABILITY_WAIT = re.compile( + r"waiting for (?:locator|getBy\w+)\(.*" + r"|locator resolved to .*" + r"|attempting \w+ action" + r"|retrying \w+ action" + r"|waiting \d+ms" + r"|waiting for element to be .*" + r"|element is .*" + r"|element was detached from the DOM, retrying" + r"|scrolling into view if needed" + r"|done scrolling" +) +_START_TAG = re.compile(r"<([a-zA-Z][\w-]*)([^>]*)>([^<]*)") +_HTML_ATTRIBUTE = re.compile(r'([\w:-]+)="([^"]*)"') + + +def decode( + tool_name: str, + raw_result: Mapping[str, Any], + *, + output_dir: Path, + before: BrowserContext | None = None, + sensitive: bool = False, +) -> ActionResult: + """``raw_result`` is the ``CallToolResult`` as JSON; ``before`` is the context before the call.""" + blocks = raw_result.get("content") or () + text = result_text(raw_result) + parts = split_sections(text) + if sensitive: + parts.pop(_CODE_SECTION, None) + dialog = _dialog(parts.get("Modal state", "")) + if raw_result.get("isError") or "Error" in parts: + if _MODAL_REFUSAL.search(parts.get("Error", "")): + blocked = None if before is None or dialog is None else dataclasses.replace(before, dialog=dialog) + return ActionResult(ok=False, detail=_detail(parts), surface=blocked) + raise _error(tool_name, _ANSI.sub("", parts.get("Error") or text)) + + tab_list = parse_tabs(parts.get("Result", "")) if tool_name == "browser_tabs" else None + header = tab_list or parse_header(parts) + surface = None + if header is not None: + surface = BrowserContext( + url=header.url, + title=header.title, + tabs=header.tabs, + tab_urls=header.tab_urls, + current_tab=header.tabs.index(header.focused), + dialog=dialog, + ) + image = next((block for block in blocks if block.get("type") == "image"), None) + if image is not None: + value: Any = _screenshot(str(image["data"])) + elif tab_list is None and "Result" in parts: + value = _result_value(parts["Result"]) + else: + value = None + + opened: tuple[ContextRef, ...] = () + changed = tool_name in _NAVIGATING_TOOLS or dialog is not None + if before is not None and header is not None: + opened = header.tabs[len(before.tabs) :] + changed = changed or header.url != before.url or len(header.tabs) != len(before.tabs) + return ActionResult( + ok=True, + value=value, + detail=_detail(parts), + context_changed=changed, + observation=_observation(parts.get("Snapshot"), header, surface, output_dir), + opened_contexts=opened, + surface=surface, + ) + + +def result_text(raw_result: Mapping[str, Any]) -> str: + """The text blocks of a result, joined: what the model reads today.""" + blocks = raw_result.get("content") or () + return "\n".join(str(block.get("text", "")) for block in blocks if block.get("type") == "text") + + +def _dialog(modal_state: str) -> str | None: + match = _DIALOG.search(modal_state) + return None if match is None else f"{match.group('type')}: {match.group('message')}" + + +def _detail(parts: dict[str, str]) -> str | None: + kept = [f"### {name}\n{body}" if name else body for name, body in parts.items() if name not in _DROPPED_SECTIONS] + return "\n".join(kept) or None + + +def _result_value(body: str) -> Any: + if body == "undefined": + return None + try: + return json.loads(body) + except ValueError: + return body + + +def _screenshot(data: str) -> Screenshot: + png = base64.b64decode(data) + width, height = struct.unpack(">II", png[16:24]) if png[:8] == _PNG_SIGNATURE and len(png) >= 24 else (0, 0) + return Screenshot(png=png, width=width, height=height) + + +def _observation( + section: str | None, header: PageHeader | None, surface: BrowserContext | None, output_dir: Path +) -> Observation | None: + if section is None or header is None: + return None + yaml = snapshot_yaml(section) + if yaml is None: + yaml = _read_linked_snapshot(section, output_dir) + if not yaml: + return None + return Observation(context=header.focused, elements=parse_elements(yaml), native_text=yaml, surface=surface) + + +def _read_linked_snapshot(section: str, output_dir: Path) -> str | None: + link = _FILE_LINK.search(section) + if link is None: + return None + root = output_dir.resolve() + path = (root / link.group("path").replace("\\", "/")).resolve() + if not path.is_relative_to(root): + return None + try: + return path.read_text(encoding="utf-8") + except OSError: + return None + + +def _error(tool_name: str, message: str) -> DriverError: + if _STALE_REF.search(message): + return StaleRef(message, dispatched=False) + if _FILE_ACCESS_DENIED.search(message): + return DriverError(message, dispatched=False) + if _CLOSED.search(message): + return ConnectionLost(message, dispatched=True) + head, _, call_log = message.partition("Call log:") + lines = [_LOG_PREFIX.sub("", line).strip() for line in call_log.splitlines() if line.strip()] + timeout = bool(_TIMEOUT.search(head)) + if any(_PERFORMED.match(line) for line in lines): + return ActionTimeout(message, dispatched=True) if timeout else DriverError(message, dispatched=True) + blockers = [match for match in map(_INTERCEPTED.match, lines) if match is not None] + if blockers: + return ActionIntercepted(message, by=_describe_blocker(blockers[-1]), dispatched=False) + if timeout and tool_name == "browser_wait_for": + return ActionTimeout(message, dispatched=True) + if timeout: + only_waits = bool(lines) and all(_ACTIONABILITY_WAIT.fullmatch(line) for line in lines) + return ActionTimeout(message, dispatched=not only_waits) + return DriverError(message, dispatched=True) + + +def _describe_blocker(match: re.Match[str]) -> str: + by = _describe_element(match.group("by")) + within = match.group("within") + return by if within is None else f"{by} in {_describe_element(within)}" + + +def _describe_element(html: str) -> str: + """``tag#id.class "text"``, the form the browser-use driver reports in ``ActionIntercepted.by``.""" + match = _START_TAG.match(html) + if match is None: + return html + tag, raw_attributes, text = match.groups() + attributes = dict(_HTML_ATTRIBUTE.findall(raw_attributes)) + classes = attributes.get("class", "").split()[:2] + described = tag.lower() + (f"#{attributes['id']}" if attributes.get("id") else "") + described += "".join(f".{name}" for name in classes) + text = " ".join(text.split())[:40] + return f'{described} "{text}"' if text else described diff --git a/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/snapshot.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/snapshot.py new file mode 100644 index 0000000000..9e1ee080f4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/snapshot.py @@ -0,0 +1,177 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Parse @playwright/mcp text: result sections, the page header, tab lists and the ARIA snapshot YAML.""" + +from __future__ import annotations + +import json +import re +from dataclasses import dataclass +from typing import Any + +from openjiuwen.harness.tools.browser_move.contract import ContextRef, DriverRef, Element + +TAB_KIND = "tab" +ARIA_REF = re.compile(r"(?:f\d+)?e\d+") + +_SECTION = re.compile(r"^### (.+)$", re.MULTILINE) +_FENCED_YAML = re.compile(r"```yaml\n(.*?)```", re.DOTALL) +_PAGE_FIELD = re.compile(r"^- Page (URL|Title): ?(.*)$", re.MULTILINE) +_TAB_LINE = re.compile(r"^- (\d+): (\(current\) )?\[(.*)\]\((.*)\)$", re.MULTILINE) +_NODE_LINE = re.compile(r"^(?P<indent> *)- (?P<body>.*)$") +_PROPERTY = re.compile(r"/(?P<key>[\w-]+): ?(?P<value>.*)") +_NODE = re.compile( + r'(?P<role>[\w-]+)(?: "(?P<name>(?:[^"\\]|\\.)*)")?(?P<attrs>(?: \[[^\]]*\])*)(?::(?: (?P<text>.*))?)?' +) +_ATTRIBUTE = re.compile(r"\[([^\]=]+)(?:=([^\]]*))?\]") +_BOX = re.compile(r" \[box=[^\]]*\]") +_VALUE_ROLES = frozenset({"textbox", "searchbox", "combobox", "spinbutton", "slider"}) + + +@dataclass(frozen=True) +class PageHeader: + """The page and tabs a result reports; ``tabs`` lists every open tab, the focused one included.""" + + url: str + title: str + tabs: tuple[ContextRef, ...] + focused: ContextRef + tab_urls: tuple[str, ...] + + +def split_sections(text: str) -> dict[str, str]: + """``### Name`` sections in order; text before the first header is kept under ``""``.""" + parts: dict[str, str] = {} + matches = list(_SECTION.finditer(text)) + preface = text[: matches[0].start()] if matches else text + if preface.strip(): + parts[""] = preface.strip("\n") + for match, following in zip(matches, matches[1:] + [None]): + end = following.start() if following is not None else len(text) + parts[match.group(1).strip()] = text[match.end() : end].strip("\n") + return parts + + +def snapshot_yaml(section: str) -> str | None: + """The inline YAML of a ``### Snapshot`` section, or None when the section only links a file.""" + match = _FENCED_YAML.search(section) + return None if match is None else match.group(1) + + +def parse_tabs(body: str) -> PageHeader | None: + """Header from a tab list (``browser_tabs`` result or ``### Open tabs``); None when it lists no tab.""" + tabs: list[ContextRef] = [] + urls: list[str] = [] + focused: tuple[ContextRef, str] | None = None + for match in _TAB_LINE.finditer(body): + index, current, title, url = match.groups() + tab = ContextRef(id=index, kind=TAB_KIND, title=title) + tabs.append(tab) + urls.append(url) + if current: + focused = (tab, url) + if focused is None: + return None + return PageHeader( + url=focused[1], title=focused[0].title or "", tabs=tuple(tabs), focused=focused[0], tab_urls=tuple(urls) + ) + + +def parse_header(parts: dict[str, str]) -> PageHeader | None: + """Header from ``### Page`` and ``### Open tabs``; None when the result has no ``### Page``.""" + page = parts.get("Page") + if page is None: + return None + fields = dict(_PAGE_FIELD.findall(page)) + url, title = fields.get("URL", ""), fields.get("Title", "") + listed = parse_tabs(parts.get("Open tabs", "")) + if listed is not None: + return PageHeader(url=url, title=title, tabs=listed.tabs, focused=listed.focused, tab_urls=listed.tab_urls) + only = ContextRef(id="0", kind=TAB_KIND, title=title) + return PageHeader(url=url, title=title, tabs=(only,), focused=only, tab_urls=(url,)) + + +def without_boxes(text: str) -> str: + """The text of a ``browser_snapshot`` taken with ``boxes``, as the server gives it without them.""" + return _BOX.sub("", text) + + +def parse_snapshot(text: str) -> tuple[tuple[Element, ...], PageHeader | None]: + """Elements and header of a ``browser_snapshot`` result.""" + parts = split_sections(text) + yaml = snapshot_yaml(parts.get("Snapshot", "")) + return (parse_elements(yaml) if yaml else ()), parse_header(parts) + + +def parse_elements(yaml: str) -> tuple[Element, ...]: + """Every node with a ref; ``/key: value`` lines and a control's text or selected option fill its fields.""" + elements: list[dict[str, Any]] = [] + parents: list[tuple[int, dict[str, Any] | None]] = [] + for line in yaml.splitlines(): + match = _NODE_LINE.match(line) + if match is None: + continue + indent, body = len(match.group("indent")), _unquote_key(match.group("body")) + while parents and parents[-1][0] >= indent: + parents.pop() + parent = parents[-1][1] if parents else None + prop = _PROPERTY.fullmatch(body) + if prop is not None: + if parent is not None: + parent["extra"][prop.group("key")] = prop.group("value") + continue + node = _NODE.fullmatch(body) + if node is None: + parents.append((indent, None)) + continue + role = node.group("role") + name = json.loads(f'"{node.group("name")}"') if node.group("name") is not None else "" + attributes = {key: value or True for key, value in _ATTRIBUTE.findall(node.group("attrs"))} + text = _unquote_value(node.group("text") or "") + if parent is not None and parent["role"] in _VALUE_ROLES and parent["value"] is None: + if role == "text": + parent["value"] = text + elif role == "option" and "selected" in attributes: + parent["value"] = name + ref = attributes.pop("ref", None) + element = None + if isinstance(ref, str) and ARIA_REF.fullmatch(ref): + element = _element(ref, role, name, attributes, text) + parents.append((indent, element)) + if element is not None: + elements.append(element) + return tuple(Element(**element) for element in elements) + + +def _unquote_key(body: str) -> str: + if not body.startswith("'"): + return body + match = re.fullmatch(r"'((?:[^']|'')*)'(:.*)?", body) + return body if match is None else match.group(1).replace("''", "'") + (match.group(2) or "") + + +def _element(ref: str, role: str, name: str, attributes: dict[str, Any], text: str) -> dict[str, Any]: + extra: dict[str, Any] = {key: value for key, value in attributes.items() if key not in ("box", "active")} + if text and role not in _VALUE_ROLES: + extra["text"] = text + box = attributes.get("box") + return { + "ref": DriverRef(ref=ref, generation=""), + "role": role, + "name": name, + "value": text if text and role in _VALUE_ROLES else None, + "bounds": tuple(float(part) for part in box.split(",")) if isinstance(box, str) else None, + "enabled": "disabled" not in attributes, + "focused": "active" in attributes, + "extra": extra, + } + + +def _unquote_value(text: str) -> str: + if len(text) >= 2 and text[0] == text[-1] == '"': + try: + return str(json.loads(text)) + except ValueError: + return text + return text diff --git a/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/tools.py b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/tools.py new file mode 100644 index 0000000000..2dc4077f8c --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/drivers/playwright_mcp/tools.py @@ -0,0 +1,276 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Arguments of the @playwright/mcp tools decoded into commands: the inverse of the driver's handlers.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from types import MappingProxyType +from typing import Any + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + Click, + CloseContext, + ContextRef, + Drag, + DriverRef, + HandleDialog, + Hover, + Locator, + Observe, + PointRef, + PressKey, + RefResolver, + RunScript, + Scroll, + SwitchContext, + TypeText, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.snapshot import ARIA_REF +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FieldKind, + FillForm, + FormField, + GetConsole, + GetNetwork, + GetPdf, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) + +Builder = Callable[[Mapping[str, Any], RefResolver], Action] + +_FIELD_KINDS = {kind.value: kind for kind in FieldKind} + + +def _required(args: Mapping[str, Any], name: str) -> Any: + value = args.get(name) + if value is None: + raise ValueError(f"'{name}' is required") + return value + + +def _number(args: Mapping[str, Any], name: str, default: int | None = None) -> int: + value = args.get(name, default) + if value is None: + raise ValueError(f"'{name}' is required") + try: + return round(float(value)) + except (TypeError, ValueError): + raise ValueError(f"'{name}' must be a number") from None + + +def _strings(args: Mapping[str, Any], name: str) -> tuple[str, ...]: + value = args.get(name) or [] + if not isinstance(value, list): + raise ValueError(f"'{name}' must be a list of strings") + return tuple(str(item) for item in value) + + +def _target(value: Any, resolver: RefResolver) -> DriverRef | Locator: + target = str(value or "") + if not target: + raise ValueError("'target' is required") + if ARIA_REF.fullmatch(target): + return DriverRef(target, "") + return Locator(css=target, text=None, generation="") + + +def _point(args: Mapping[str, Any], x: str, y: str, resolver: RefResolver) -> PointRef: + return PointRef(_number(args, x), _number(args, y), resolver.screenshot_generation) + + +def _current_context(resolver: RefResolver) -> ContextRef: + context = resolver.current_context + if context is None: + raise ValueError("no open tab") + return context + + +def build_click(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Click( + _target(args.get("target"), resolver), + button=str(args.get("button") or "left"), + modifiers=_strings(args, "modifiers"), + count=2 if args.get("doubleClick") else 1, + ) + + +def build_type(args: Mapping[str, Any], resolver: RefResolver) -> Action: + slowly = bool(args.get("slowly")) + return TypeText( + _target(args.get("target"), resolver), + str(_required(args, "text")), + submit=bool(args.get("submit")), + clear=not slowly, + slowly=slowly, + ) + + +def build_press_key(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return PressKey(str(_required(args, "key"))) + + +def build_hover(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Hover(_target(args.get("target"), resolver)) + + +def build_drag(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Drag(_target(args.get("startTarget"), resolver), _target(args.get("endTarget"), resolver)) + + +def build_select_option(args: Mapping[str, Any], resolver: RefResolver) -> Action: + values = _strings(args, "values") + if not values: + raise ValueError("'values' must name at least one option") + return SelectOption(_target(args.get("target"), resolver), values) + + +def build_fill_form(args: Mapping[str, Any], resolver: RefResolver) -> Action: + raw_fields = args.get("fields") + if not isinstance(raw_fields, list) or not raw_fields: + raise ValueError("'fields' must be a non-empty list") + fields = [] + for index, raw in enumerate(raw_fields): + if not isinstance(raw, Mapping): + raise ValueError(f"fields[{index}] must be an object") + kind = _FIELD_KINDS.get(str(raw.get("type") or "")) + if kind is None: + raise ValueError(f"fields[{index}] unsupported type={raw.get('type')!r}") + fields.append(FormField(_target(raw.get("target"), resolver), kind, str(_required(raw, "value")))) + return FillForm(tuple(fields)) + + +def build_drop(args: Mapping[str, Any], resolver: RefResolver) -> Action: + data = args.get("data") or {} + if not isinstance(data, Mapping): + raise ValueError("'data' must map MIME types to strings") + pairs = tuple((str(mime), str(value)) for mime, value in data.items()) + return Drop(_target(args.get("target"), resolver), paths=_strings(args, "paths"), data=pairs) + + +def build_file_upload(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return UploadFile(None, _strings(args, "paths")) + + +def build_handle_dialog(args: Mapping[str, Any], resolver: RefResolver) -> Action: + prompt = args.get("promptText") + return HandleDialog(bool(_required(args, "accept")), None if prompt is None else str(prompt)) + + +def build_evaluate(args: Mapping[str, Any], resolver: RefResolver) -> Action: + target = _target(args["target"], resolver) if args.get("target") else None + return RunScript(str(_required(args, "function")), target=target) + + +def build_navigate(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Navigate(str(_required(args, "url"))) + + +def build_navigate_back(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Back() + + +def build_snapshot(args: Mapping[str, Any], resolver: RefResolver) -> Action: + target = args.get("target") + return Observe( + depth=None if args.get("depth") is None else _number(args, "depth"), + root=None if not target else _target(target, resolver), + ) + + +def build_take_screenshot(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Observe(include_screenshot=True, full_page=bool(args.get("fullPage"))) + + +def build_tabs(args: Mapping[str, Any], resolver: RefResolver) -> Action: + action = str(_required(args, "action")) + if action == "list": + return Observe() + if action == "new": + url = args.get("url") + return OpenContext(None if url is None else str(url)) + if action not in ("select", "close"): + raise ValueError(f"action must be one of: list, new, close, select (got {action!r})") + if args.get("index") is None: + if action == "select": + raise ValueError("action=select requires index") + return CloseContext(_current_context(resolver)) + index, contexts = _number(args, "index"), resolver.contexts + if not 0 <= index < len(contexts): + raise ValueError(f"tab index {index} is out of range: {len(contexts)} tab(s) open") + return SwitchContext(contexts[index]) if action == "select" else CloseContext(contexts[index]) + + +def build_close(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return CloseContext(_current_context(resolver)) + + +def build_mouse_click_xy(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Click( + _point(args, "x", "y", resolver), button=str(args.get("button") or "left"), count=_number(args, "clickCount", 1) + ) + + +def build_mouse_move_xy(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Hover(_point(args, "x", "y", resolver)) + + +def build_mouse_drag_xy(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Drag(_point(args, "startX", "startY", resolver), _point(args, "endX", "endY", resolver)) + + +def build_mouse_wheel(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Scroll(None, dx=_number(args, "deltaX", 0), dy=_number(args, "deltaY", 0)) + + +def build_pdf_save(args: Mapping[str, Any], resolver: RefResolver) -> Action: + filename = args.get("filename") + return GetPdf(None if filename is None else str(filename)) + + +def build_console_messages(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return GetConsole(level=str(args.get("level") or "info"), all=bool(args.get("all"))) + + +def build_network_requests(args: Mapping[str, Any], resolver: RefResolver) -> Action: + url_filter = args.get("filter") + return GetNetwork(static=bool(args.get("static")), filter=None if url_filter is None else str(url_filter)) + + +DECODERS: Mapping[str, Builder] = MappingProxyType( + { + "browser_click": build_click, + "browser_type": build_type, + "browser_press_key": build_press_key, + "browser_hover": build_hover, + "browser_drag": build_drag, + "browser_select_option": build_select_option, + "browser_fill_form": build_fill_form, + "browser_drop": build_drop, + "browser_file_upload": build_file_upload, + "browser_handle_dialog": build_handle_dialog, + "browser_evaluate": build_evaluate, + "browser_navigate": build_navigate, + "browser_navigate_back": build_navigate_back, + "browser_snapshot": build_snapshot, + "browser_take_screenshot": build_take_screenshot, + "browser_tabs": build_tabs, + "browser_close": build_close, + "browser_mouse_click_xy": build_mouse_click_xy, + "browser_mouse_move_xy": build_mouse_move_xy, + "browser_mouse_drag_xy": build_mouse_drag_xy, + "browser_mouse_wheel": build_mouse_wheel, + "browser_pdf_save": build_pdf_save, + "browser_console_messages": build_console_messages, + "browser_network_requests": build_network_requests, + } +) + +__all__ = ["Builder", "DECODERS"] diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/__init__.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/__init__.py deleted file mode 100644 index e65ad9e4f7..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/__init__.py +++ /dev/null @@ -1,78 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Playwright runtime package bootstrap.""" - -from __future__ import annotations - -import sys -from importlib import import_module -from pathlib import Path -from typing import Any - -# SRC_ROOT is the browser_move/ directory that contains controllers/, utils/, etc. -# Adding it to sys.path makes bare module imports (e.g. `from controllers.base import ...`) -# work both when running standalone (MCP server) and when loaded as part of the -# openjiuwen package. -_HERE = Path(__file__).resolve().parent -SRC_ROOT = _HERE.parent -# Walk up four levels (browser_move -> tools -> harness -> openjiuwen -> repo root) -REPO_ROOT = SRC_ROOT.parent.parent.parent.parent -if str(SRC_ROOT) not in sys.path: - sys.path.append(str(SRC_ROOT)) -sys.modules.setdefault("playwright_runtime", sys.modules[__name__]) - -__all__ = [ - "REPO_ROOT", - "SRC_ROOT", - "BrowserStateContextProcessor", - "BrowserStateContextProcessorConfig", - "BrowserWorkingContextProcessor", - "BrowserWorkingContextProcessorConfig", - "BrowserWorkingContextRail", - "build_browser_runtime_mcp_config", - "browser_tools", - "controller", - "register_browser_runtime_mcp_server", - "reset_active_browser_runtimes", - "reset_managed_browser_runtime", - "restart_local_browser_runtime_server", - "service", - "stop_local_browser_runtime_server", -] - - -def __getattr__(name: str) -> Any: - if name in {"BrowserStateContextProcessor", "BrowserStateContextProcessorConfig"}: - module = import_module( - "openjiuwen.harness.tools.browser_move.playwright_runtime.browser_state_context_processor" - ) - return getattr(module, name) - if name in { - "BrowserWorkingContextProcessor", - "BrowserWorkingContextProcessorConfig", - }: - module = import_module( - "openjiuwen.harness.tools.browser_move.playwright_runtime.browser_working_context_processor" - ) - return getattr(module, name) - if name == "BrowserWorkingContextRail": - module = import_module("openjiuwen.harness.tools.browser_move.playwright_runtime.browser_working_context_rail") - return getattr(module, name) - if name == "controller": - return import_module("openjiuwen.harness.tools.browser_move.controllers") - if name == "browser_tools": - return import_module("openjiuwen.harness.tools.browser_move.playwright_runtime.browser_tools") - if name == "service": - return import_module("openjiuwen.harness.tools.browser_move.playwright_runtime.service") - if name in {"reset_active_browser_runtimes", "reset_managed_browser_runtime"}: - module = import_module("openjiuwen.harness.tools.browser_move.playwright_runtime.runtime") - return getattr(module, name) - if name in { - "build_browser_runtime_mcp_config", - "register_browser_runtime_mcp_server", - "restart_local_browser_runtime_server", - "stop_local_browser_runtime_server", - }: - module = import_module("openjiuwen.harness.tools.browser_move.playwright_runtime.browser_tools") - return getattr(module, name) - raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/agents.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/agents.py deleted file mode 100644 index 10153c38dc..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/agents.py +++ /dev/null @@ -1,397 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - -"""Agent builders for runtime and browser worker.""" - -from __future__ import annotations - -import inspect -import json -import os -from typing import Any, Awaitable, Callable, Iterable, Optional - -import anyio - -from openjiuwen.core.common.logging import logger -from openjiuwen.core.common.utils.schema_utils import SchemaUtils -from openjiuwen.core.context_engine import DialogueCompressorConfig -from openjiuwen.core.foundation.llm import ModelClientConfig, ModelRequestConfig -from openjiuwen.core.foundation.tool import McpServerConfig -from openjiuwen.core.single_agent.agents.react_agent import ReActAgent, ReActAgentConfig -from openjiuwen.core.single_agent.schema.agent_card import AgentCard - -from .browser_capabilities import CORE_BROWSER_TOOL_NAMES - -ToolResultObserver = Callable[[str, Any], Awaitable[None]] - - -def _build_dialogue_compressor_config( - provider: str, - api_key: str, - api_base: str, - model_name: str, -) -> Optional[DialogueCompressorConfig]: - compressor_model = (model_name or "").strip() - if not compressor_model: - logger.warning("DialogueCompressor disabled: model_name is empty.") - return None - return DialogueCompressorConfig( - messages_to_keep=10, - keep_last_round=True, - model_client=ModelClientConfig( - client_provider=provider, - api_key=api_key, - api_base=api_base, - verify_ssl=False, - ), - # Use alias field `model` for compatibility with strict alias parsing. - model=ModelRequestConfig(model=compressor_model), - ) - - -def _resolve_tool_timeout_s(default_s: float = 240.0) -> float: - raw = ( - os.getenv("PLAYWRIGHT_TOOL_TIMEOUT_S") - or os.getenv("PLAYWRIGHT_MCP_TIMEOUT_S") - or os.getenv("BROWSER_TIMEOUT_S") - or str(default_s) - ) - try: - parsed = float(raw) - if parsed > 0: - return parsed - except (TypeError, ValueError): - pass - return default_s - - -def _resolve_sampling_value( - keys: tuple[str, ...], - *, - default: float, - min_value: float, - max_value: float, -) -> float: - for key in keys: - raw = (os.getenv(key) or "").strip() - if not raw: - continue - try: - value = float(raw) - except (TypeError, ValueError): - continue - if min_value <= value <= max_value: - return value - return default - - -def _resolve_sampling_params( - *, - temperature_keys: tuple[str, ...], - top_p_keys: tuple[str, ...], - default_temperature: float, - default_top_p: float, -) -> tuple[float, float]: - temperature = _resolve_sampling_value( - temperature_keys, - default=default_temperature, - min_value=0.0, - max_value=2.0, - ) - top_p = _resolve_sampling_value( - top_p_keys, - default=default_top_p, - min_value=0.0, - max_value=1.0, - ) - return temperature, top_p - - -def _format_tool_names(tool_call: Any) -> str: - if isinstance(tool_call, list): - names = [getattr(item, "name", "") for item in tool_call] - names = [name for name in names if name] - return ", ".join(names) if names else "<unknown>" - name = getattr(tool_call, "name", "") - return name or "<unknown>" - - -def _looks_like_tool_call(value: Any) -> bool: - if isinstance(value, list): - return all(hasattr(item, "name") for item in value) - return hasattr(value, "name") - - -def _drop_none_tool_arguments(tool_call: Any) -> None: - calls = tool_call if isinstance(tool_call, list) else [tool_call] - for current_call in calls: - raw_arguments = getattr(current_call, "arguments", None) - if not isinstance(raw_arguments, str): - continue - try: - parsed_arguments = json.loads(raw_arguments) - except json.JSONDecodeError: - continue - - cleaned_arguments = SchemaUtils.remove_none_values(parsed_arguments) - if cleaned_arguments is None: - cleaned_arguments = {} - if cleaned_arguments != parsed_arguments: - current_call.arguments = json.dumps(cleaned_arguments, ensure_ascii=False) - - -def _normalize_execute_call(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[Any, Any, Any, Any, dict[str, Any]]: - """Accept both legacy and current ability_manager.execute call shapes.""" - ctx = kwargs.get("ctx") - tool_call = kwargs.get("tool_call") - session = kwargs.get("session") - tag = kwargs.get("tag") - extra_kwargs = {k: v for k, v in kwargs.items() if k not in {"ctx", "tool_call", "session", "tag"}} - positional = list(args) - - if tool_call is None and positional: - if len(positional) >= 3 and not _looks_like_tool_call(positional[0]): - if ctx is None: - ctx = positional.pop(0) - elif len(positional) >= 4 and ctx is None: - ctx = positional.pop(0) - - if tool_call is None and positional: - tool_call = positional.pop(0) - if session is None and positional: - session = positional.pop(0) - if tag is None and positional: - tag = positional.pop(0) - - return ctx, tool_call, session, tag, extra_kwargs - - -def build_browser_worker_system_prompt( - screenshot_subdir: str = "screenshots", - artifacts_subdir: str = "artifacts", -) -> str: - return ( - "You are a browser worker agent.\n" - "Execute browser tasks step-by-step with Playwright MCP tools and approved runtime helper tools only.\n" - "Before interacting, ensure page or selector readiness.\n" - "Keep actions targeted and avoid unnecessary page snapshots.\n" - "Before broad page snapshots, full-body scans, or generic DOM scraping, " - "choose the smallest compact probe that matches the task. " - "Use browser_probe_interactives for buttons, links, inputs, forms, navigation controls, " - "login controls, pagination controls, menus, and other visible interactive elements. " - "When using browser_probe_interactives only for page-level controls, prefer max_items around 20-30 " - "unless the task explicitly requires a larger inventory. " - "On product pages, marketplace pages, search-result pages, catalog pages, article-list pages, " - "or any page with repeated visible cards/listings, call browser_probe_cards before broad extraction. " - "Use browser_probe_cards to identify compact repeated structures such as product cards, result cards, " - "book cards, article cards, listing rows, title/author/source/summary, " - "title/price/rating/review/availability fields, primary links, " - "visible controls, generation-scoped target IDs, and recurring structure signatures. " - "For product/listing/item-data tasks, prefer browser_probe_cards first; call browser_probe_interactives " - "only if you also need page-level navigation, filters, forms, or controls outside the cards. " - "If browser_probe_cards returns the fields needed for the task, including article/result-row " - "title, link, author/source, or summary fields, use that compact result as the evidence " - "for extraction and final reporting instead of taking screenshots, snapshots, or running extra DOM scans. " - "After a successful card probe, use browser_evaluate, or a browser_run_code tool only when it is " - "explicitly present, for a clearly missing required field or a specific target-based action; " - "do not repeat broad evaluation " - "just to re-read the same cards. Treat page_id/generation_id/url/title/interactives/cards/" - "field_coverage/blockers as one PageState contract. Pass the current generation_id with a " - "probe target_id directly to browser_batch_interact; never translate it into guessed CSS. " - "Explicit selectors are reserved for validated read-only extraction and condition waits. " - "If a card has primary_link " - "or href, navigate to it directly. " - "browser_batch_interact is a standalone runtime helper like browser_probe_interactives and " - "browser_probe_cards, not a browser_custom_action wrapper. Use it directly. " - "For multi-field forms with several known controls, search flows with autocomplete, dropdown/date-picker " - "flows, filter panels, or any short sequence where two or more next click/type/wait/extract steps are " - "already known, call browser_batch_interact before falling back to repeated " - "browser_click/browser_type and separate wait turns. " - "Extract multiple fields from the same page in one batch with named extract_text/extract_value " - "steps, or use one card probe result that already contains every requested field. " - "Do not force browser_batch_interact for a single uncertain click, one simple text field, or a page state " - "that still needs inspection. Prefer generation-scoped target_id in batch steps. " - "Use autocomplete steps for type-then-choose widgets, and condition-based wait_for_selector/wait_for_text " - "steps instead of fixed sleeps. For dynamic pages use wait_for_url, " - "wait_for_first_card_title, wait_for_sort_state, wait_for_result_count, " - "wait_for_dom_text_change, wait_for_stable, or wait_for_tab with a short poll interval and total timeout. " - "When an action opens a new tab, use click plus wait_for_tab in the same batch; wait_for_tab " - "activates the matched tab by default. Do not wait for a new URL on the old tab. " - "Interpret 'fully loaded' or an 'appropriate pause' as condition stability, not a fixed sleep. " - "Do not split click+type+wait, click+wait+click, " - "date open+choose, or search submit+result wait into separate ReAct turns unless browser_batch_interact " - "failed or the next target is genuinely unknown. " - "Keep using browser_fill_form for ordinary visible text fields when that official tool is enough. " - "Use browser_snapshot only when the compact probes are insufficient, when accessibility structure is needed, " - "or when you need exact element references required by a Playwright MCP action. " - "Use browser_evaluate for small exact computations. Use a browser_run_code tool only when it is present " - "in the current tool list and compact probes, evaluate, snapshot, and deterministic actions are " - "insufficient. Never use run-code to dump the entire document body.\n" - "If actions repeatedly fail, stop and report the exact failing action.\n" - "If you use browser_tabs, action MUST be one of: list, new, close, select.\n" - "Use specialized operations only when their explicit primitive tool is present in the current tool list. " - "Do not launch nested browser tasks from the browser worker. If the available tools cannot complete an " - "operation, report the exact blocker instead of inventing a tool name.\n" - "IMPORTANT: Do NOT use browser_take_screenshot unless strictly necessary. " - f"If a screenshot is needed, always save it under '{screenshot_subdir}/'. " - "Use browser_take_screenshot when it is present; do not create screenshots through run-code.\n" - f"If you produce any output files (reports, notes, summaries, markdown, text files, etc.), " - f"write them to the '{artifacts_subdir}/' directory relative to the working directory. " - "Never write output files to the project root or any other location.\n" - "Final output MUST be a single JSON object with keys:\n" - "ok (boolean), final (string), page (object with url and title), " - "screenshot (string|null), error (string|null).\n" - "Also include status (completed|partial|blocked|failed) whenever possible. " - "If the task is not fully complete, include progress as an object with " - "completed_steps, remaining_steps, next_step, completion_evidence, and missing_requirements.\n" - "Set ok=true only when the exact user-visible goal is fully satisfied and you can cite concrete evidence " - "from the page, compact probe results, browser snapshots, screenshots, or generated artifacts. " - "If the task is incomplete or blocked, set ok=false and fill the progress fields so a continuation can " - "resume with minimal repetition.\n" - "Return JSON only, even on failures. " - "Do not output markdown, code fences, or plain text outside the JSON object." - ) - - -async def _notify_tool_results( - tool_call: Any, - results: Any, - observer: ToolResultObserver, -) -> None: - calls = tool_call if isinstance(tool_call, list) else [tool_call] - result_items = results if isinstance(results, list) else [] - for idx, current_call in enumerate(calls): - if idx >= len(result_items): - break - current_result = result_items[idx] - tool_result = current_result[0] if isinstance(current_result, tuple) and current_result else current_result - tool_name = getattr(current_call, "name", "") or "<tool>" - await observer(tool_name, tool_result) - - -def ensure_execute_signature_compat( - agent: ReActAgent, - *, - tool_result_observer: ToolResultObserver | None = None, -) -> None: - """Wrap ability_manager.execute with signature compatibility and timeout.""" - execute_fn = getattr(agent.ability_manager, "execute", None) - if execute_fn is None: - return - if getattr(execute_fn, "_playwright_timeout_wrapped", False): - return - - try: - params = inspect.signature(execute_fn).parameters - except (TypeError, ValueError): - return - - original_execute = execute_fn - supports_ctx = "ctx" in params - supports_tag = "tag" in params - supports_kwargs = any(param.kind == inspect.Parameter.VAR_KEYWORD for param in params.values()) - tool_timeout_s = _resolve_tool_timeout_s() - - async def execute_with_timeout(*args, **kwargs): - ctx, tool_call, session, tag, extra_kwargs = _normalize_execute_call(args, kwargs) - tool_names = _format_tool_names(tool_call) - _drop_none_tool_arguments(tool_call) - call_kwargs: dict[str, Any] = {} - if supports_ctx: - call_kwargs["ctx"] = ctx - if "tool_call" in params: - call_kwargs["tool_call"] = tool_call - if "session" in params: - call_kwargs["session"] = session - if supports_tag: - call_kwargs["tag"] = tag - if supports_kwargs: - call_kwargs.update(extra_kwargs) - try: - with anyio.fail_after(tool_timeout_s): - results = await original_execute(**call_kwargs) - except TimeoutError as exc: - logger.error( - f"Tool execution timed out after {tool_timeout_s:.1f}s; tools={tool_names}" - ) - raise RuntimeError( - f"tool_execution_timeout: tools={tool_names}, timeout_s={tool_timeout_s:.1f}" - ) from exc - if tool_result_observer is not None: - await _notify_tool_results(tool_call, results, tool_result_observer) - return results - - agent.ability_manager.execute = execute_with_timeout - setattr(agent.ability_manager.execute, "_playwright_timeout_wrapped", True) - - -# pylint: disable=too-many-arguments -def build_browser_worker_agent( - provider: str, - api_key: str, - *, - api_base: str, - model_name: str, - mcp_cfg: McpServerConfig, - max_steps: int, - screenshot_subdir: str = "screenshots", - artifacts_subdir: str = "artifacts", - tool_result_observer: ToolResultObserver | None = None, - allowed_tool_names: Optional[Iterable[str]] = None, -) -> ReActAgent: - resolved_allowed_tool_names = ( - CORE_BROWSER_TOOL_NAMES - if allowed_tool_names is None - else tuple(dict.fromkeys(allowed_tool_names)) - ) - logger.info( - "Playwright worker received tool allowlist: %s", - resolved_allowed_tool_names, - ) - screenshot_subdir = ( - (screenshot_subdir or "screenshots").strip().replace("\\", "/").strip("/") or "screenshots" - ) - artifacts_subdir = ( - (artifacts_subdir or "artifacts").strip().replace("\\", "/").strip("/") or "artifacts" - ) - card = AgentCard( - id="agent.playwright.browser_worker", - name="playwright_browser_worker", - description="Browser worker that executes web tasks using Playwright MCP tools.", - input_params={}, - ) - config = ( - ReActAgentConfig() - .configure_model_client( - provider=provider, - api_key=api_key, - api_base=api_base, - model_name=model_name, - ) - .configure_max_iterations(max_steps) - .configure_prompt_template( - [ - { - "role": "system", - "content": build_browser_worker_system_prompt(screenshot_subdir, artifacts_subdir), - } - ] - ) - ) - worker_temperature, worker_top_p = _resolve_sampling_params( - temperature_keys=("BROWSER_WORKER_TEMPERATURE", "BROWSER_MODEL_TEMPERATURE", "MODEL_TEMPERATURE"), - top_p_keys=("BROWSER_WORKER_TOP_P", "BROWSER_MODEL_TOP_P", "MODEL_TOP_P"), - default_temperature=0.2, - default_top_p=0.1, - ) - if config.model_config_obj is not None: - config.model_config_obj.temperature = worker_temperature - config.model_config_obj.top_p = worker_top_p - agent = ReActAgent(card=card).configure(config) - agent.ability_manager.add(mcp_cfg) - agent.ability_manager.set_mcp_tool_allowlist( - mcp_cfg, - resolved_allowed_tool_names, - ) - ensure_execute_signature_compat(agent, tool_result_observer=tool_result_observer) - return agent diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_logging.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_logging.py deleted file mode 100644 index 502be9f913..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_logging.py +++ /dev/null @@ -1,194 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - -from __future__ import annotations - -import gzip -import hashlib -import json -import logging -import os -from pathlib import Path -import re -from threading import Lock -from typing import Any - -from openjiuwen.core.common.logging import logger as common_logger -from openjiuwen.core.common.logging.browser_context import is_browser_agent_log_context - - -_BROWSER_LOGGER_NAME = "openjiuwen.browser_agent" -_BROWSER_HANDLER_MARKER = "_openjiuwen_browser_agent_file_handler" -_BROWSER_LOG_ANNOUNCED_MARKER = "_openjiuwen_browser_agent_file_announced" -_FALSE_VALUES = {"0", "false", "no", "off", ""} -_DISABLE_VALUES = {"0", "false", "no", "off", "none", "null", "-"} -_LOCK = Lock() -_AUDIT_LOCK = Lock() - - -def _env_bool(name: str, default: bool = False) -> bool: - value = os.getenv(name) - if value is None: - return default - return value.strip().lower() not in _FALSE_VALUES - - -def _get_level() -> int: - level_name = os.getenv("OPENJIUWEN_BROWSER_AGENT_LOG_LEVEL", "INFO") - return getattr(logging, level_name.strip().upper(), logging.INFO) - - -def _default_log_path() -> Path: - return Path.cwd() / "logs" / "browser_agent.log" - - -def get_browser_agent_log_path() -> Path | None: - """Return the configured browser-agent log path. - - Browser-agent file logging is enabled by default so users running - jiuwenswarm-start get a separate browser log without extra environment - setup. Set OPENJIUWEN_BROWSER_AGENT_LOG_FILE to one of - 0/false/no/off/none/null/- to disable the dedicated file. - """ - configured = os.getenv("OPENJIUWEN_BROWSER_AGENT_LOG_FILE") - if configured is None: - return _default_log_path().resolve() - - configured = configured.strip() - if configured.lower() in _DISABLE_VALUES: - return None - - return Path(configured).expanduser().resolve() - - -def get_browser_agent_logger() -> logging.Logger: - """Return the dedicated browser-agent logger. - - By default, browser-agent logs are written to ./logs/browser_agent.log in - UTF-8 and are not propagated to the combined application log. Override the - target path with OPENJIUWEN_BROWSER_AGENT_LOG_FILE. Set - OPENJIUWEN_BROWSER_AGENT_LOG_MIRROR_COMMON=1 to also mirror browser logs - to the normal combined logger. - """ - browser_logger = logging.getLogger(_BROWSER_LOGGER_NAME) - browser_logger.setLevel(_get_level()) - - mirror_common = _env_bool( - "OPENJIUWEN_BROWSER_AGENT_LOG_MIRROR_COMMON", - default=False, - ) - browser_logger.propagate = mirror_common - - log_path = get_browser_agent_log_path() - if log_path is None: - return browser_logger - - with _LOCK: - for handler in browser_logger.handlers: - if getattr(handler, _BROWSER_HANDLER_MARKER, False): - if getattr(handler, "baseFilename", None) == str(log_path): - return browser_logger - - log_path.parent.mkdir(parents=True, exist_ok=True) - - file_handler = logging.FileHandler( - log_path, - mode="w", - encoding="utf-8", - ) - setattr(file_handler, _BROWSER_HANDLER_MARKER, True) - file_handler.setLevel(_get_level()) - file_handler.setFormatter( - logging.Formatter( - "%(asctime)s | browser_agent | %(levelname)s | %(message)s" - ) - ) - browser_logger.addHandler(file_handler) - - if not getattr(browser_logger, _BROWSER_LOG_ANNOUNCED_MARKER, False): - common_logger.info( - "[BROWSER_AGENT_LOG] dedicated browser log file enabled: %s", - str(log_path), - ) - setattr(browser_logger, _BROWSER_LOG_ANNOUNCED_MARKER, True) - - return browser_logger - - -def browser_agent_log_info(message: str, *args: Any) -> None: - browser_logger = get_browser_agent_logger() - browser_logger.info(message, *args) - - if get_browser_agent_log_path() is None: - common_logger.info(message, *args) - - -def browser_agent_log_warning(message: str, *args: Any) -> None: - browser_logger = get_browser_agent_logger() - browser_logger.warning(message, *args) - - if get_browser_agent_log_path() is None: - common_logger.warning(message, *args) - - -def browser_agent_log_error(message: str, *args: Any) -> None: - browser_logger = get_browser_agent_logger() - browser_logger.error(message, *args) - - if get_browser_agent_log_path() is None: - common_logger.error(message, *args) - - -def write_browser_agent_audit_artifact(kind: str, raw: Any) -> dict[str, Any]: - """Persist an opted-in, content-addressed raw browser observation.""" - if isinstance(raw, str): - raw_text = raw - else: - raw_text = json.dumps(raw, ensure_ascii=False, default=str) - raw_bytes = raw_text.encode("utf-8", "ignore") - digest = hashlib.sha256(raw_bytes).hexdigest() - audit = { - "raw_size_bytes": len(raw_bytes), - "raw_sha256": digest[:16], - "stored": False, - } - if not is_browser_agent_log_context() or not _env_bool( - "OPENJIUWEN_BROWSER_AGENT_AUDIT_RAW", - default=False, - ): - return audit - - log_path = get_browser_agent_log_path() - if log_path is None: - return audit - safe_kind = re.sub(r"[^a-z0-9_-]+", "_", str(kind or "observation").lower()).strip("_") - safe_kind = safe_kind or "observation" - artifact_name = f"{safe_kind}_{digest[:16]}.json.gz" - artifact_path = log_path.parent / "browser_agent_audit" / artifact_name - try: - with _AUDIT_LOCK: - artifact_path.parent.mkdir(parents=True, exist_ok=True) - if not artifact_path.exists(): - with gzip.open(artifact_path, mode="wt", encoding="utf-8") as stream: - json.dump( - {"kind": safe_kind, "sha256": digest, "raw": raw_text}, - stream, - ensure_ascii=False, - separators=(",", ":"), - ) - audit["stored"] = True - get_browser_agent_logger().info( - "[BROWSER_AUDIT] kind=%s sha256=%s size=%s artifact=%s", - safe_kind, - digest[:16], - len(raw_bytes), - artifact_name, - ) - except OSError as exc: - get_browser_agent_logger().warning( - "[BROWSER_AUDIT_ERROR] kind=%s sha256=%s error=%s", - safe_kind, - digest[:16], - exc, - ) - return audit diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_tools.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_tools.py deleted file mode 100644 index 6e9f01326c..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_tools.py +++ /dev/null @@ -1,628 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Browser MCP integration helpers for playwright_runtime.""" - -from __future__ import annotations - -import json -import os -import shlex -import socket -import subprocess -import sys -import time -from pathlib import Path -from typing import Any -from urllib.parse import urlparse - -from openjiuwen.core.foundation.tool import McpServerConfig -from openjiuwen.core.runner import Runner -_DEFAULT_SERVER_ID = "playwright_runtime_wrapper" -_DEFAULT_SERVER_NAME = "playwright-runtime-wrapper" -_SUPPORTED_CLIENT_TYPES = {"stdio", "sse", "streamable-http", "streamable_http", "http"} -_AUTO_HTTP_FALLBACK = "PLAYWRIGHT_RUNTIME_MCP_AUTO_HTTP_FALLBACK" -_PROXY_BLOCKLIST = {"http://127.0.0.1:9", "http://localhost:9"} -_LOCAL_SERVER_PROCESS: subprocess.Popen[str] | None = None -_LOCAL_SERVER_URL: str | None = None -_OPENJIUWEN_CLIENTS_PATCHED = False -_client_registry: dict[str, Any] = {} -_browser_runtime_stdout_handle = None -_browser_runtime_stderr_handle = None -_BROWSER_RUNTIME_STDOUT_LOG = "browser_runtime_stdout.log" -_BROWSER_RUNTIME_STDERR_LOG = "browser_runtime_stderr.log" - - -def get_registered_client(server_id: str) -> Any: - """Return the MCP client created for *server_id*, or None if not found.""" - return _client_registry.get(server_id) - - -def _env_first(*names: str, default: str = "") -> str: - for name in names: - value = (os.getenv(name) or "").strip() - if value: - return value - return default - - -def _env_bool(*names: str, default: bool = False) -> bool: - value = _env_first(*names) - if not value: - return default - return value.lower() in {"1", "true", "yes", "on"} - - -def _parse_args(raw: str) -> list[str]: - text = (raw or "").strip() - if not text: - return [] - if text.startswith("["): - try: - parsed = json.loads(text) - if isinstance(parsed, list): - return [str(item) for item in parsed] - except Exception: - pass - try: - return shlex.split(text, posix=(os.name != "nt")) - except Exception: - return text.split() - - -def _runtime_logs_dir() -> Path: - configured = _env_first("PLAYWRIGHT_RUNTIME_LOG_DIR", "BROWSER_RUNTIME_LOG_DIR") - if configured: - path = Path(configured).expanduser() - else: - from .config import resolve_playwright_mcp_cwd - - path = Path(resolve_playwright_mcp_cwd()).expanduser() / "logs" - path.mkdir(parents=True, exist_ok=True) - return path - - -def _runtime_cwd() -> Path: - from .config import resolve_playwright_mcp_cwd - - path = Path(resolve_playwright_mcp_cwd()).expanduser() - path.mkdir(parents=True, exist_ok=True) - return path - - -def _browser_runtime_log_paths() -> tuple[Path, Path]: - logs_dir = _runtime_logs_dir() - return ( - logs_dir / _BROWSER_RUNTIME_STDOUT_LOG, - logs_dir / _BROWSER_RUNTIME_STDERR_LOG, - ) - - -def _close_browser_runtime_log_handles() -> None: - global _browser_runtime_stdout_handle - global _browser_runtime_stderr_handle - - for handle_name in ("_browser_runtime_stdout_handle", "_browser_runtime_stderr_handle"): - handle = globals().get(handle_name) - if handle is None: - continue - try: - handle.close() - except Exception: - pass - globals()[handle_name] = None - - -def _server_script() -> Path: - return Path(__file__).resolve().parent.parent / "playwright_runtime_mcp_server.py" - - -def _normalize_client_type(client_type: str) -> str: - value = (client_type or "").strip().lower() - if value in {"http", "streamable_http"}: - return "streamable-http" - return value - - -def _ensure_openjiuwen_client_patch() -> None: - global _OPENJIUWEN_CLIENTS_PATCHED - if _OPENJIUWEN_CLIENTS_PATCHED: - return - - import openjiuwen.core.runner.resources_manager.tool_manager as tool_mgr - - from ..clients.stdio_client import BrowserMoveStdioClient - from ..clients.streamable_http_client import BrowserMoveStreamableHttpClient - - tool_mgr.StdioClient = BrowserMoveStdioClient - original_create_client = tool_mgr.ToolMgr._create_client - - def _patched_create_client(config: McpServerConfig): - normalized = _normalize_client_type(getattr(config, "client_type", "")) - if normalized == "streamable-http": - client = BrowserMoveStreamableHttpClient(config) - elif normalized == "stdio": - client = BrowserMoveStdioClient(config) - else: - client = original_create_client(config) - _client_registry[config.server_id] = client - return client - - tool_mgr.StreamableHttpClient = BrowserMoveStreamableHttpClient - tool_mgr.ToolMgr._create_client = staticmethod(_patched_create_client) - _OPENJIUWEN_CLIENTS_PATCHED = True - - -def ensure_browser_runtime_client_patch() -> None: - """Ensure MCP client creation is patched for browser runtime transports.""" - _ensure_openjiuwen_client_patch() - - -def _build_child_env() -> dict[str, str]: - env = dict(os.environ) - passthrough_keys = [ - "MODEL_NAME", - "MODEL_PROVIDER", - "API_KEY", - "API_BASE", - "OPENROUTER_API_KEY", - "OPENROUTER_BASE_URL", - "OPENAI_API_KEY", - "OPENAI_BASE_URL", - "PLAYWRIGHT_MCP_COMMAND", - "PLAYWRIGHT_MCP_ARGS", - "PLAYWRIGHT_CDP_URL", - "PLAYWRIGHT_CDP_HEADERS", - "PLAYWRIGHT_MCP_CDP_ENDPOINT", - "PLAYWRIGHT_MCP_CDP_TIMEOUT", - "PLAYWRIGHT_MCP_BROWSER", - "PLAYWRIGHT_MCP_DEVICE", - "PLAYWRIGHT_BROWSERS_PATH", - "PLAYWRIGHT_TOOL_TIMEOUT_S", - "PLAYWRIGHT_RUNTIME_MCP_COMMAND", - "PLAYWRIGHT_RUNTIME_MCP_ARGS", - "PLAYWRIGHT_RUNTIME_MCP_SERVER_PATH", - "PLAYWRIGHT_RUNTIME_MCP_TIMEOUT_S", - "BROWSER_TIMEOUT_S", - "HTTP_PROXY", - "HTTPS_PROXY", - "NO_PROXY", - ] - for key in passthrough_keys: - value = os.getenv(key) - if value: - env[key] = value - - api_key = (os.getenv("API_KEY") or "").strip() - api_base = (os.getenv("API_BASE") or "").strip() - model_provider = (os.getenv("MODEL_PROVIDER") or "").strip().lower() - - if api_key and not env.get("OPENROUTER_API_KEY") and "openrouter.ai" in api_base: - env["OPENROUTER_API_KEY"] = api_key - if api_base and not env.get("OPENROUTER_BASE_URL") and "openrouter.ai" in api_base: - env["OPENROUTER_BASE_URL"] = api_base - - if api_key and not env.get("OPENAI_API_KEY") and "openrouter.ai" not in api_base: - env["OPENAI_API_KEY"] = api_key - if api_base and not env.get("OPENAI_BASE_URL") and "openrouter.ai" not in api_base: - env["OPENAI_BASE_URL"] = api_base - - if model_provider == "openrouter": - env["MODEL_PROVIDER"] = "openrouter" - elif model_provider in {"openai", "siliconflow"}: - env["MODEL_PROVIDER"] = model_provider - - for proxy_key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"): - proxy_value = (env.get(proxy_key) or "").strip().lower() - if proxy_value in _PROXY_BLOCKLIST: - env.pop(proxy_key, None) - return env - - -def _runtime_host() -> str: - return _env_first("PLAYWRIGHT_RUNTIME_MCP_HOST", "BROWSER_RUNTIME_MCP_HOST", default="127.0.0.1") - - -def _runtime_port() -> str: - return _env_first("PLAYWRIGHT_RUNTIME_MCP_PORT", "BROWSER_RUNTIME_MCP_PORT", default="8940") - - -def _runtime_path(transport: str) -> str: - default_path = "/mcp" if _normalize_client_type(transport) == "streamable-http" else "/sse" - path = _env_first("PLAYWRIGHT_RUNTIME_MCP_PATH", "BROWSER_RUNTIME_MCP_PATH", default=default_path) - if not path.startswith("/"): - return f"/{path}" - return path - - -def _build_server_url(transport: str) -> str: - return f"http://{_runtime_host()}:{_runtime_port()}{_runtime_path(transport)}" - - -def _is_port_available(host: str, port: int) -> bool: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - sock.bind((host, port)) - except OSError: - return False - return True - - -def _pick_available_port(host: str, preferred_port: int, max_attempts: int = 25) -> int: - if preferred_port > 0 and _is_port_available(host, preferred_port): - return preferred_port - for port in range(preferred_port + 1, preferred_port + max_attempts + 1): - if _is_port_available(host, port): - return port - raise RuntimeError("No available port for local browser runtime MCP server.") - - -def _wait_port_open(host: str, port: int, timeout_s: float = 30.0) -> None: - deadline = time.time() + timeout_s - while time.time() < deadline: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.settimeout(0.8) - if sock.connect_ex((host, port)) == 0: - return - time.sleep(0.2) - raise RuntimeError(f"Browser runtime MCP server did not start in time: {host}:{port}") - - -def _start_local_server(transport: str, host: str, port: int, path: str) -> str: - global _LOCAL_SERVER_PROCESS - global _LOCAL_SERVER_URL - global _browser_runtime_stdout_handle - global _browser_runtime_stderr_handle - - normalized = _normalize_client_type(transport) - if normalized not in {"sse", "streamable-http"}: - raise ValueError(f"Unsupported local server transport: {transport}") - - server_script = _server_script() - if not server_script.exists(): - raise FileNotFoundError(f"playwright runtime MCP server not found: {server_script}") - - command = _env_first( - "PLAYWRIGHT_RUNTIME_MCP_COMMAND", - "BROWSER_RUNTIME_MCP_COMMAND", - default=sys.executable, - ) - cmd = [ - command, - str(server_script), - "--transport", - normalized, - "--host", - host, - "--port", - str(port), - "--path", - path, - "--no-banner", - ] - stdout_log_path, stderr_log_path = _browser_runtime_log_paths() - _close_browser_runtime_log_handles() - stdout_handle = stdout_log_path.open("a", encoding="utf-8") - stderr_handle = stderr_log_path.open("a", encoding="utf-8") - try: - _LOCAL_SERVER_PROCESS = subprocess.Popen( - cmd, - cwd=str(_runtime_cwd()), - env=_build_child_env(), - stdout=stdout_handle, - stderr=stderr_handle, - text=True, - ) - except Exception: - stdout_handle.close() - stderr_handle.close() - raise - _browser_runtime_stdout_handle = stdout_handle - _browser_runtime_stderr_handle = stderr_handle - _wait_port_open(host, port) - _LOCAL_SERVER_URL = f"http://{host}:{port}{path}" - return _LOCAL_SERVER_URL - - -def _parse_local_server_url(server_url: str) -> tuple[str, int, str]: - parsed = urlparse(server_url) - if parsed.scheme not in {"http", "https"} or not parsed.hostname or parsed.port is None: - raise ValueError(f"Invalid browser runtime MCP URL: {server_url}") - return parsed.hostname, int(parsed.port), parsed.path or "/mcp" - - -def stop_local_browser_runtime_server() -> None: - global _LOCAL_SERVER_PROCESS - global _LOCAL_SERVER_URL - - proc = _LOCAL_SERVER_PROCESS - _LOCAL_SERVER_PROCESS = None - _LOCAL_SERVER_URL = None - if proc is None or proc.poll() is not None: - _close_browser_runtime_log_handles() - return - try: - proc.terminate() - proc.wait(timeout=5) - except Exception: - try: - proc.kill() - proc.wait(timeout=2) - except Exception: - pass - finally: - _close_browser_runtime_log_handles() - - -def restart_local_browser_runtime_server() -> str | None: - transport = _normalize_client_type( - _env_first( - "PLAYWRIGHT_RUNTIME_MCP_CLIENT_TYPE", - "BROWSER_RUNTIME_MCP_CLIENT_TYPE", - default="streamable-http", - ) - ) - current_url = _LOCAL_SERVER_URL - current_proc = _LOCAL_SERVER_PROCESS - - if transport not in {"sse", "streamable-http"}: - stop_local_browser_runtime_server() - return None - - host = _runtime_host() - path = _runtime_path(transport) - preferred_port = int(_runtime_port()) - if current_url: - host, preferred_port, path = _parse_local_server_url(current_url) - - had_local_server = current_url is not None or (current_proc is not None and current_proc.poll() is None) - stop_local_browser_runtime_server() - if not had_local_server: - return None - - deadline = time.time() + 10.0 - while time.time() < deadline: - if _is_port_available(host, preferred_port): - return _start_local_server(transport, host, preferred_port, path) - time.sleep(0.3) - raise RuntimeError(f"Browser runtime port is still occupied after shutdown: {host}:{preferred_port}") - - -def _ensure_local_server_started(transport: str) -> str: - global _LOCAL_SERVER_PROCESS - global _LOCAL_SERVER_URL - - normalized = _normalize_client_type(transport) - if normalized not in {"sse", "streamable-http"}: - raise ValueError(f"Unsupported local server transport: {transport}") - if _LOCAL_SERVER_PROCESS is not None and _LOCAL_SERVER_PROCESS.poll() is None and _LOCAL_SERVER_URL: - return _LOCAL_SERVER_URL - - host = _runtime_host() - preferred_port = int(_runtime_port()) - path = _runtime_path(normalized) - port = _pick_available_port(host, preferred_port) - return _start_local_server(normalized, host, port, path) - - -def _build_http_fallback_config(base_cfg: McpServerConfig, server_url: str | None = None) -> McpServerConfig: - return McpServerConfig( - server_id=f"{base_cfg.server_id}_http", - server_name=base_cfg.server_name, - server_path=server_url or _build_server_url("streamable-http"), - client_type="streamable-http", - ) - - -def _build_http_retry_config(base_cfg: McpServerConfig, server_url: str) -> McpServerConfig: - return McpServerConfig( - server_id=base_cfg.server_id, - server_name=base_cfg.server_name, - server_path=server_url, - client_type="streamable-http", - ) - - -def _result_is_ok(result: Any) -> bool: - if result is None: - return True - is_ok = getattr(result, "is_ok", None) - if callable(is_ok): - try: - return bool(is_ok()) - except Exception: - return False - return False - - -def _result_error_text(result: Any) -> str: - if result is None: - return "" - for attr in ("error", "msg"): - method = getattr(result, attr, None) - if callable(method): - try: - value = method() - if value is not None: - return str(value) - except Exception: - pass - raw_error = getattr(result, "_error", None) - if raw_error is not None: - return str(raw_error) - return str(result) - - -def build_browser_runtime_mcp_config() -> McpServerConfig | None: - """Build browser runtime MCP config for OpenJiuWen agent registration.""" - if not _env_bool("PLAYWRIGHT_RUNTIME_MCP_ENABLED", "BROWSER_RUNTIME_MCP_ENABLED", default=False): - return None - - server_id = _env_first( - "PLAYWRIGHT_RUNTIME_MCP_SERVER_ID", - "BROWSER_RUNTIME_MCP_SERVER_ID", - default=_DEFAULT_SERVER_ID, - ) - server_name = _env_first( - "PLAYWRIGHT_RUNTIME_MCP_SERVER_NAME", - "BROWSER_RUNTIME_MCP_SERVER_NAME", - default=_DEFAULT_SERVER_NAME, - ) - client_type = _normalize_client_type( - _env_first( - "PLAYWRIGHT_RUNTIME_MCP_CLIENT_TYPE", - "BROWSER_RUNTIME_MCP_CLIENT_TYPE", - default="streamable-http", - ) - ) - if client_type not in _SUPPORTED_CLIENT_TYPES: - raise ValueError("PLAYWRIGHT_RUNTIME_MCP_CLIENT_TYPE must be stdio, sse, or streamable-http.") - - if client_type == "sse": - server_path = _env_first( - "PLAYWRIGHT_RUNTIME_MCP_SERVER_PATH", - "BROWSER_RUNTIME_MCP_SERVER_PATH", - default=_build_server_url("sse"), - ) - return McpServerConfig( - server_id=server_id, - server_name=server_name, - server_path=server_path, - client_type="sse", - ) - - if client_type == "streamable-http": - server_path = _env_first( - "PLAYWRIGHT_RUNTIME_MCP_SERVER_PATH", - "BROWSER_RUNTIME_MCP_SERVER_PATH", - default=_build_server_url("streamable-http"), - ) - return McpServerConfig( - server_id=server_id, - server_name=server_name, - server_path=server_path, - client_type="streamable-http", - ) - - server_script = _server_script() - if not server_script.exists(): - raise FileNotFoundError(f"playwright runtime MCP server not found: {server_script}") - - command = _env_first( - "PLAYWRIGHT_RUNTIME_MCP_COMMAND", - "BROWSER_RUNTIME_MCP_COMMAND", - default=sys.executable, - ) - args_raw = _env_first("PLAYWRIGHT_RUNTIME_MCP_ARGS", "BROWSER_RUNTIME_MCP_ARGS") - args = ( - _parse_args(args_raw) - if args_raw - else [ - "-m", - "openjiuwen.harness.tools.browser_move.playwright_runtime_mcp_server", - "--transport", - "stdio", - "--no-banner", - "--log-level", - "ERROR", - ] - ) - - params: dict[str, Any] = { - "command": command, - "args": args, - "cwd": str(_runtime_cwd()), - } - timeout_raw = _env_first( - "PLAYWRIGHT_RUNTIME_MCP_TIMEOUT_S", - "BROWSER_RUNTIME_MCP_TIMEOUT_S", - default="300", - ) - try: - timeout_s = int(timeout_raw) - if timeout_s > 0: - params["timeout_s"] = timeout_s - except ValueError: - pass - - child_env = _build_child_env() - if child_env: - params["env"] = child_env - - server_path = _env_first( - "PLAYWRIGHT_RUNTIME_MCP_SERVER_PATH", - "BROWSER_RUNTIME_MCP_SERVER_PATH", - default="stdio://playwright-runtime-wrapper", - ) - return McpServerConfig( - server_id=server_id, - server_name=server_name, - server_path=server_path, - client_type="stdio", - params=params, - ) - - -async def register_browser_runtime_mcp_server(agent: Any, *, tag: str = "agent.main") -> bool: - """Register browser runtime MCP server on an OpenJiuWen agent with fallbacks.""" - _ensure_openjiuwen_client_patch() - cfg = build_browser_runtime_mcp_config() - if cfg is None: - return False - - async def _register_once(target_cfg: McpServerConfig) -> tuple[bool, str]: - result = await Runner.resource_mgr.add_mcp_server(target_cfg, tag=tag) - if _result_is_ok(result): - agent.ability_manager.add(target_cfg) - return True, "" - - error_text = _result_error_text(result) - if "already exist" in error_text.lower(): - agent.ability_manager.add(target_cfg) - return True, error_text - return False, error_text - - http_err = "" - if cfg.client_type == "stdio" and _env_bool(_AUTO_HTTP_FALLBACK, default=True): - http_cfg = _build_http_fallback_config(cfg) - ok, http_err = await _register_once(http_cfg) - if ok: - return True - try: - auto_url = _ensure_local_server_started("streamable-http") - http_cfg = _build_http_fallback_config(cfg, server_url=auto_url) - ok, auto_http_err = await _register_once(http_cfg) - if ok: - return True - http_err = f"{http_err} | {auto_http_err}".strip(" |") - except Exception as exc: - http_err = f"{http_err} | {exc}".strip(" |") - - ok, error_text = await _register_once(cfg) - if ok: - return True - - normalized_type = _normalize_client_type(cfg.client_type) - if normalized_type in {"streamable-http", "sse"}: - try: - auto_url = _ensure_local_server_started(normalized_type) - retry_cfg = ( - _build_http_retry_config(cfg, auto_url) - if normalized_type == "streamable-http" - else McpServerConfig( - server_id=cfg.server_id, - server_name=cfg.server_name, - server_path=auto_url, - client_type="sse", - ) - ) - ok, retry_err = await _register_once(retry_cfg) - if ok: - return True - error_text = f"{error_text} | {retry_err}".strip(" |") - except Exception as exc: - error_text = f"{error_text} | {exc}".strip(" |") - - if http_err: - error_text = f"{error_text} | http={http_err}".strip(" |") - raise RuntimeError(f"Failed to register browser runtime MCP server: {error_text}") diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/config.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/config.py deleted file mode 100644 index 286e474e61..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/config.py +++ /dev/null @@ -1,276 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Runtime configuration helpers.""" - -from __future__ import annotations - -import json -import os -import re -from dataclasses import dataclass -from pathlib import Path -from typing import Any, Dict, Iterable, Optional - -from openjiuwen.core.foundation.tool import McpServerConfig - -from .browser_capabilities import ( - CORE_BROWSER_CAPABILITY_NAME, - DEFAULT_BROWSER_CAPABILITIES, - POLICY_ONLY_BROWSER_CAPABILITY_NAMES, -) -from ..utils.env import ( - DEFAULT_BROWSER_TIMEOUT_S, - DEFAULT_GUARDRAIL_MAX_FAILURES, - DEFAULT_GUARDRAIL_MAX_STEPS, - DEFAULT_GUARDRAIL_RETRY_ONCE, - DEFAULT_MODEL_NAME, # noqa: F401 (re-exported for main.py / mcp_server / tests) - DEFAULT_PLAYWRIGHT_MCP_ARGS, - DEFAULT_PLAYWRIGHT_MCP_COMMAND, - MISSING_API_KEY_MESSAGE, # noqa: F401 (re-exported for main.py / mcp_server) - first_non_empty_env, - is_truthy_env, - load_repo_dotenv, # noqa: F401 (re-exported for main.py / mcp_server) - parse_command_args, - resolve_bool_env, - resolve_browser_timeout_s, - resolve_int_env, - resolve_model_name, - resolve_model_settings, -) - - -_playwright_mcp_capability_names: list[str] = [] -for capability_spec in DEFAULT_BROWSER_CAPABILITIES: - if capability_spec.name == CORE_BROWSER_CAPABILITY_NAME: - continue - if capability_spec.name in POLICY_ONLY_BROWSER_CAPABILITY_NAMES: - continue - _playwright_mcp_capability_names.append(capability_spec.name) -PLAYWRIGHT_MCP_CAPABILITY_NAMES = tuple(_playwright_mcp_capability_names) - - -@dataclass -class BrowserRunGuardrails: - max_steps: int = DEFAULT_GUARDRAIL_MAX_STEPS - max_failures: int = DEFAULT_GUARDRAIL_MAX_FAILURES - timeout_s: int = DEFAULT_BROWSER_TIMEOUT_S - retry_once: bool = True - resume_on_max_iterations: bool = False - - -@dataclass(frozen=True) -class BrowserInstanceConfig: - """Per-instance browser identity, used to isolate one browser per agent. - - All fields default to empty/0, which reproduces the legacy process-global - (env-driven) behavior. When ``key`` is non-empty the browser is isolated: - the Playwright MCP ``server_id`` is suffixed with the key and the managed - profile / port / user-data-dir are derived from it instead of shared env. - Agents sharing the same ``key`` intentionally share one browser. - """ - - key: str = "" - driver_mode: str = "" # "", "managed", "remote", "extension"; "" -> env/default - managed_port: int = 0 # 0 -> auto-allocate (keyed) or env/default (legacy) - user_data_dir: str = "" # "" -> derived from key under mcp_cwd/.browser-profiles - profile_name: str = "" # "" -> key, then env BROWSER_PROFILE_NAME - cdp_url: str = "" # remote mode: explicit CDP endpoint - browser_binary: str = "" # optional Chrome path override - - def sanitized_key(self) -> str: - """Return the key reduced to id-safe characters (``[A-Za-z0-9_-]``).""" - return re.sub(r"[^A-Za-z0-9_-]+", "-", (self.key or "").strip()).strip("-") - - -@dataclass(frozen=True) -class RuntimeSettings: - provider: str - api_key: str - api_base: str - model_name: str - mcp_cfg: McpServerConfig - guardrails: BrowserRunGuardrails - instance: Optional[BrowserInstanceConfig] = None - - -def resolve_playwright_mcp_cwd() -> str: - """Resolve MCP working directory with relocatable defaults.""" - configured = ( - (os.getenv("PLAYWRIGHT_RUNTIME_MCP_CWD") or "").strip() - or (os.getenv("BROWSER_RUNTIME_MCP_CWD") or "").strip() - or (os.getenv("PLAYWRIGHT_RUNTIME_WORKDIR") or "").strip() - or (os.getenv("BROWSER_RUNTIME_WORKDIR") or "").strip() - ) - if configured: - return str(Path(configured).expanduser().resolve()) - return str(Path.cwd().expanduser().resolve()) - - -def _ensure_capabilities( - args: list[str], - required_capabilities: Iterable[str], -) -> list[str]: - """Return Playwright MCP args with the specified capabilities enabled.""" - normalized_args: list[str] = [] - capabilities: list[str] = [] - index = 0 - while index < len(args): - arg = args[index] - if arg == "--caps": - if index + 1 < len(args) and not args[index + 1].startswith("--"): - capabilities.extend(args[index + 1].split(",")) - index += 2 - continue - elif arg.startswith("--caps="): - capabilities.extend(arg.removeprefix("--caps=").split(",")) - index += 1 - continue - normalized_args.append(arg) - index += 1 - - capabilities = list(dict.fromkeys(capability.strip() for capability in capabilities if capability.strip())) - for capability in required_capabilities: - if capability not in capabilities: - capabilities.append(capability) - normalized_args.append(f"--caps={','.join(capabilities)}") - return normalized_args - - -def build_browser_guardrails() -> BrowserRunGuardrails: - return BrowserRunGuardrails( - max_steps=resolve_int_env( - "BROWSER_GUARDRAIL_MAX_STEPS", - default=DEFAULT_GUARDRAIL_MAX_STEPS, - minimum=1, - ), - max_failures=resolve_int_env( - "BROWSER_GUARDRAIL_MAX_FAILURES", - default=DEFAULT_GUARDRAIL_MAX_FAILURES, - minimum=0, - ), - timeout_s=resolve_browser_timeout_s(), - retry_once=resolve_bool_env( - "BROWSER_GUARDRAIL_RETRY_ONCE", - default=DEFAULT_GUARDRAIL_RETRY_ONCE, - ), - resume_on_max_iterations=resolve_bool_env( - "BROWSER_GUARDRAIL_RESUME_ON_MAX_ITERATIONS", - default=False, - ), - ) - - -def build_playwright_mcp_config( - instance: Optional[BrowserInstanceConfig] = None, - required_capability: Optional[list[str]] = None, -) -> McpServerConfig: - command = ( - os.getenv("PLAYWRIGHT_MCP_COMMAND", DEFAULT_PLAYWRIGHT_MCP_COMMAND).strip() or DEFAULT_PLAYWRIGHT_MCP_COMMAND - ) - capabilities_to_enable = ( - *PLAYWRIGHT_MCP_CAPABILITY_NAMES, - *(required_capability or ()), - ) - args = _ensure_capabilities( - parse_command_args(os.getenv("PLAYWRIGHT_MCP_ARGS", DEFAULT_PLAYWRIGHT_MCP_ARGS)), - capabilities_to_enable, - ) - cwd = resolve_playwright_mcp_cwd() - driver_mode = (os.getenv("BROWSER_DRIVER") or "").strip().lower() - extension_mode = driver_mode == "extension" or is_truthy_env(os.getenv("PLAYWRIGHT_MCP_EXTENSION") or "") - timeout_s = resolve_int_env( - "PLAYWRIGHT_MCP_TIMEOUT_S", - "BROWSER_TIMEOUT_S", - default=DEFAULT_BROWSER_TIMEOUT_S, - minimum=1, - ) - - env_map: Dict[str, str] = {} - for key in ( - "PLAYWRIGHT_BROWSERS_PATH", - "HTTP_PROXY", - "HTTPS_PROXY", - "NO_PROXY", - ): - value = os.getenv(key) - if value: - env_map[key] = value - - extra_env_json = (os.getenv("PLAYWRIGHT_MCP_ENV_JSON") or "").strip() - if extra_env_json: - try: - extra = json.loads(extra_env_json) - if isinstance(extra, dict): - for k, v in extra.items(): - env_map[str(k)] = str(v) - except json.JSONDecodeError as exc: - raise ValueError(f"Invalid PLAYWRIGHT_MCP_ENV_JSON: {exc}") from exc - - if extension_mode: - env_map["PLAYWRIGHT_MCP_EXTENSION"] = "true" - extension_token = first_non_empty_env("PLAYWRIGHT_MCP_EXTENSION_TOKEN") - if extension_token: - env_map["PLAYWRIGHT_MCP_EXTENSION_TOKEN"] = extension_token - if "--extension" not in args: - args.append("--extension") - else: - # CDP support for official Playwright MCP server. An explicit per-instance - # cdp_url (remote mode) wins over the shared env endpoint. - instance_cdp = instance.cdp_url.strip() if instance and instance.cdp_url else "" - cdp_endpoint = instance_cdp or first_non_empty_env("PLAYWRIGHT_MCP_CDP_ENDPOINT", "PLAYWRIGHT_CDP_URL") - cdp_headers = first_non_empty_env("PLAYWRIGHT_MCP_CDP_HEADERS", "PLAYWRIGHT_CDP_HEADERS") - cdp_timeout = first_non_empty_env("PLAYWRIGHT_MCP_CDP_TIMEOUT", "PLAYWRIGHT_CDP_TIMEOUT_MS") - browser_name = first_non_empty_env("PLAYWRIGHT_MCP_BROWSER") - device_name = first_non_empty_env("PLAYWRIGHT_MCP_DEVICE") - - if cdp_endpoint: - if device_name: - raise ValueError("PLAYWRIGHT_MCP_DEVICE is not supported with CDP endpoint mode.") - env_map["PLAYWRIGHT_MCP_CDP_ENDPOINT"] = cdp_endpoint - if not browser_name: - # CDP mode is Chromium-only. - env_map["PLAYWRIGHT_MCP_BROWSER"] = "chrome" - if cdp_headers: - env_map["PLAYWRIGHT_MCP_CDP_HEADERS"] = cdp_headers - if cdp_timeout: - env_map["PLAYWRIGHT_MCP_CDP_TIMEOUT"] = cdp_timeout - - params: Dict[str, Any] = { - "command": command, - "args": args, - "cwd": cwd, - } - if timeout_s > 0: - params["timeout_s"] = timeout_s - if env_map: - params["env"] = env_map - - # Isolate the MCP registration per browser identity. Same key -> same - # server_id (intentional sharing); different key -> isolated server. - server_id = "playwright_official_stdio" - server_name = "playwright-official" - instance_key = instance.sanitized_key() if instance else "" - if instance_key: - server_id = f"{server_id}__{instance_key}" - server_name = f"{server_name}-{instance_key}" - - return McpServerConfig( - server_id=server_id, - server_name=server_name, - server_path="stdio://playwright", - client_type="stdio", - params=params, - ) - - -def build_runtime_settings(instance: Optional[BrowserInstanceConfig] = None) -> RuntimeSettings: - provider, api_key, api_base = resolve_model_settings() - return RuntimeSettings( - provider=provider, - api_key=api_key, - api_base=api_base, - model_name=resolve_model_name(), - mcp_cfg=build_playwright_mcp_config(instance), - guardrails=build_browser_guardrails(), - instance=instance, - ) diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/runtime.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/runtime.py deleted file mode 100644 index fef3224428..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/runtime.py +++ /dev/null @@ -1,6279 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. - -"""Runtime wiring for browser tool registration and service lifecycle.""" - -from __future__ import annotations - -import asyncio -import hashlib -import json -import re -import time -from collections import deque -from typing import Any, Dict, Iterable, Optional -from urllib.parse import parse_qs, parse_qsl, unquote, urlencode, urlsplit -from weakref import WeakSet - -from openjiuwen.core.common.exception.codes import StatusCode -from openjiuwen.core.common.exception.errors import BaseError, build_error -from openjiuwen.core.common.logging import logger -from openjiuwen.core.common.logging.browser_context import ( - reset_browser_agent_log_context, - set_browser_agent_log_context, -) -from openjiuwen.core.common.utils.schema_utils import SchemaUtils -from openjiuwen.core.foundation.llm import ToolMessage -from openjiuwen.core.foundation.tool import McpServerConfig -from openjiuwen.core.runner import Runner -from openjiuwen.core.single_agent.prompts.builder import PromptSection -from openjiuwen.core.single_agent.rail.base import AgentCallbackContext, AgentRail -from openjiuwen.harness.rails._multimodal import ( - should_enable_read_image_multimodal, -) - -from ..controllers import ActionController, BaseController, validate_batch_steps -from ..utils.parsing import extract_json_object -from .browser_capabilities import ( - CORE_BROWSER_TOOL_NAMES, -) -from .browser_logging import ( - browser_agent_log_info, - browser_agent_log_warning, - write_browser_agent_audit_artifact, -) -from .browser_tools import ensure_browser_runtime_client_patch -from .browser_working_context import ( - BROWSER_TASK_STATE_KEY, - BrowserWorkingContextStore, - latest_browser_user_request, -) -from .config import BrowserInstanceConfig, BrowserRunGuardrails -from .page_state import CARD_EVIDENCE_FIELDS, BrowserPageState, BrowserTarget -from .probe_semantics import normalize_card_probe_payload -from .probes import ( - build_browser_state_metadata_js, - build_card_probe_js, - build_interactive_probe_js, -) -from .semantic_state import SemanticStateTracker, price_interval_signature -from .service import MAX_ITERATION_MESSAGE, BrowserService, BrowserTaskProgressState -from .site_profiles import ( - get_selector_cache, - infer_profile_evidence_entity, - site_profiles_for_url, -) -from .status_logging import BrowserSubagentStatusLogger, is_browser_subagent_status_log_enabled - -_BROWSER_PROGRESS_STATE_KEY = "__browser_subagent_progress_state__" -_BROWSER_PROGRESS_TASK_KEY = "__browser_subagent_last_task__" -_BROWSER_IMAGE_CAPABILITY_SECTION_NAME = "browser_image_input_capability" -_BROWSER_PHASE_STATE_KEY = BROWSER_TASK_STATE_KEY -_BROWSER_SCREENSHOT_TOOL_NAMES = frozenset({"browser_take_screenshot"}) -_BROWSER_LOG_CONTEXT_TOKEN_KEY = "__browser_agent_log_context_token__" -_BROWSER_TOOL_RUNTIME_STATE_KEY = "__browser_tool_runtime_state__" -_BROWSER_ACTION_GROUP_BY_CALL_KEY = "__browser_action_group_by_call__" -_BROWSER_ACTION_GROUP_RESULTS_KEY = "__browser_action_group_results__" -_BROWSER_SKIP_TOOL_CALLS_KEY = "_skip_tool_calls" -_BROWSER_TASK_DEADLINE_KEY = "__browser_task_deadline__" -_BROWSER_INVOCATION_DEADLINE_KEY = "__browser_invocation_deadline__" -_BROWSER_SIMPLE_TASK_DEADLINE_S = 240.0 -_BROWSER_COMPLEX_TASK_DEADLINE_S = 600.0 -_BROWSER_MODEL_RETRY_LIMIT = 1 -_BROWSER_MODEL_PROTOCOL_RETRY_LIMIT = 1 -_BROWSER_REPLAN_DENIAL_LIMIT = 3 -_BROWSER_READ_ONLY_RECOVERY_LIMIT = 1 -_BROWSER_TASK_RESUME_LIMIT = 1 -_BROWSER_TERMINAL_SYNTHESIS_KEY = "terminal_synthesis_started" -_BROWSER_OBSERVATION_MESSAGE_MAX_CHARS = 12_000 -_BROWSER_BOUNDED_OBSERVATION_TOOL_TOKENS = ( - "browser_probe_cards", - "browser_probe_interactives", - "browser_snapshot", - "browser_find", - "browser_evaluate", -) -_BROWSER_RUNTIME_TOOL_NAMES = frozenset( - { - "browser_batch_interact", - "browser_probe_interactives", - "browser_probe_cards", - "browser_custom_action", - "browser_list_custom_actions", - "browser_cancel_run", - "browser_clear_cancel", - "browser_runtime_health", - } -) -_ACTIVE_BROWSER_RUNTIMES: WeakSet[Any] = WeakSet() -_BROWSER_PROGRESS_TAG_RE = re.compile( - r"<browser_progress>\s*(\{.*?\})\s*</browser_progress>", - re.DOTALL | re.IGNORECASE, -) -_BROWSER_UNFINISHED_TOOL_INTENT_RE = re.compile( - r"(?:" - r"<[^>]*(?:dsml|tool[_ ]?calls?)[^>]*>" - r"|(?:让我|我(?:将|会|需要|要))继续(?:调用|使用|执行).{0,100}" - r"(?:工具|browser_|evaluate|probe|navigate|snapshot|batch)" - r"|(?:let me|i(?:'ll| will| need to))\s+(?:continue|now)\s+" - r"(?:call|calling|use|using|run).{0,100}" - r"(?:tool|browser_|evaluate|probe|navigate|snapshot|batch)" - r")", - re.IGNORECASE | re.DOTALL, -) -_BROWSER_TOOL_ERROR_PREFIX_RE = re.compile( - r"^\s*(?:#{1,6}\s*)?(?:ability execution error|tool execution error|" - r"workflow execution error|agent execution error|error|failed)\s*(?::|\n|$)", - re.IGNORECASE, -) -_BROWSER_TIMEOUT_ERROR_RE = re.compile( - r"\b(?:timed?\s*out|timeout(?:error)?|deadline\s+exceeded)\b", - re.IGNORECASE, -) -_BROWSER_OUTPUT_REQUEST_CUE_RE = re.compile( - r"(?:返回|告诉我|提取|获取|输出|给出|列出|汇报|报告|比较|对比|" - r"return|tell\s+me|extract|retrieve|output|provide|list|report|compare)" - r"[^。；;.!?\n]{0,240}", - re.IGNORECASE, -) -_BROWSER_EXPLICIT_REF_RE = re.compile( - r"""^\s*\[?\s*ref\s*=\s*["']?([^"'\]\s]+)["']?\s*\]?\s*$""", - re.IGNORECASE, -) -_BROWSER_PAGE_STATE_TAG = "browser_page_state" -_BROWSER_SNAPSHOT_REF_RE = re.compile( - r"""\bref\s*=\s*["']?([A-Za-z0-9_.:-]+)""", - re.IGNORECASE, -) -_BROWSER_REF_TARGET_KEYS = frozenset( - { - "target", - "startTarget", - "endTarget", - "ref", - "startRef", - "endRef", - } -) -_BROWSER_REF_ALIAS_KEYS = { - "ref": "target", - "startRef": "startTarget", - "endRef": "endTarget", -} -_BATCH_MODEL_LOCATOR_KEYS = frozenset( - {"target_id", "ref", "selector", "role", "name", "label", "placeholder", "text", "testid"} -) -_BATCH_REGISTERED_TARGET_OPS = frozenset( - { - "click", - "fill", - "type", - "autocomplete", - "select_option", - "set_checked", - "select_visible_text", - } -) -_BATCH_SAFE_READ_SELECTOR_OPS = frozenset({"extract_text", "extract_value"}) -_BATCH_MUTATING_TARGET_OPS = frozenset( - {"click", "fill", "type", "autocomplete", "select_option", "set_checked", "select_visible_text"} -) -_BATCH_EXPLICIT_SELECTOR_OPS = frozenset( - { - "wait_for_selector", - "wait_for_first_card_title", - "wait_for_sort_state", - "wait_for_result_count", - "wait_for_dom_text_change", - "wait_for_stable", - } -) -_BATCH_SAFE_GENERATION_REFRESH_OPS = frozenset( - { - "wait_for_selector", - "wait_for_text", - "wait_for_load_state", - "wait_for_url", - "wait_for_first_card_title", - "wait_for_sort_state", - "wait_for_result_count", - "wait_for_dom_text_change", - "wait_for_stable", - "wait_for_tab", - } -) -_SINGLE_BATCH_CONDITION_OPS = frozenset({"wait_for_text"}) -_BROWSER_TERMINAL_STATUSES = frozenset({"blocked", "partial", "completed"}) -_BROWSER_NON_RETRYABLE_BLOCKER_TOKENS = frozenset( - { - "captcha", - "login_required", - "permission_denied", - "access_denied", - "user_intervention_required", - "payment_required", - "security_verification", - "task_deadline_exhausted", - } -) -_BROWSER_FIELD_ALIASES: Dict[str, tuple[str, ...]] = { - "title": ("title", "name", "标题", "名称", "电影", "商品"), - "url": ("url", "link", "href", "primary link", "primary_link", "链接", "网址"), - "price": ("price", "cost", "价格", "价钱", "费用"), - "rating": ("rating", "score", "评分", "星级"), - "product_rating": ("product rating", "item rating", "商品评分", "宝贝评分"), - "shop_rating": ("shop rating", "seller rating", "store rating", "店铺评分", "卖家评分"), - "author": ("author", "creator", "writer", "作者", "博主", "发布者", "回答者"), - "likes": ("likes", "like count", "upvotes", "点赞", "点赞数", "赞同", "赞同数", "获赞", "获赞数"), - "favorites": ("favorites", "favourites", "bookmarks", "收藏", "收藏数"), - "comments": ("comments", "comment count", "replies", "评论", "评论数", "回复数"), - "shop": ("shop", "store", "merchant", "seller", "店铺", "商家", "卖家"), - "duration": ("duration", "video length", "时长", "视频长度"), - "views": ("views", "view count", "play count", "播放量", "观看数"), - "exchange_rate": ("exchange rate", "conversion rate", "汇率", "兑换率"), - "high_temperature": ( - "high temperature", - "maximum temperature", - "highest temperature", - "最高温", - "最高气温", - "高温", - ), - "low_temperature": ( - "low temperature", - "minimum temperature", - "lowest temperature", - "最低温", - "最低气温", - "低温", - ), - "sort_state": ("sort state", "sort order", "ordering", "排序状态", "排序方式", "排序"), - "date": ("date", "日期", "哪天"), - "time": ("time", "时间", "几点"), - "address": ("address", "location", "地址", "地点"), -} -# Inferred display fields are advisory; these distinctions require typed evidence. -_BROWSER_STRICT_EVIDENCE_FIELDS = frozenset({"rating", "product_rating", "shop_rating", "sort_state"}) -_BROWSER_EVALUATE_FIELD_ALIASES = { - "article_title": "title", - "page_title": "title", - "product_title": "title", - "result_title": "title", - "author_name": "author", - "author_raw": "author", - "creator_name": "author", - "profile_name": "author", - "writer_name": "author", - "like_count": "likes", - "likes_count": "likes", - "like_raw": "likes", - "upvote_count": "likes", - "favorite_count": "favorites", - "favorites_count": "favorites", - "favourite_count": "favorites", - "bookmark_count": "favorites", - "fav_raw": "favorites", - "comment_count": "comments", - "comments_count": "comments", - "reply_count": "comments", - "comment_raw": "comments", - "com_raw": "comments", - "active_tab": "sort_state", - "active_tabs": "sort_state", - "selected_tab": "sort_state", - "selected_tabs": "sort_state", - "sort_selected": "sort_state", -} -_BROWSER_CONTEXTUAL_EVALUATE_FIELD_ALIASES = frozenset( - { - "name", - "profile_name", - "active_tab", - "active_tabs", - "selected_tab", - "selected_tabs", - } -) -_BROWSER_SELECTED_SORT_LABELS = { - "sale": "销量", - "sales": "销量", - "volume": "销量", - "销量": "销量", - "price": "价格", - "价格": "价格", - "latest": "最新", - "newest": "最新", - "最新": "最新", - "comprehensive": "综合", - "relevance": "综合", - "default": "综合", - "综合": "综合", - "rating": "评分", - "score": "评分", - "评分": "评分", -} -_BROWSER_UNKNOWN_VALUE_RE = re.compile( - r"^(?:unknown|n/?a|not available|not found|未找到|未知|暂无|未显示|不可用)$", - re.IGNORECASE, -) -_BROWSER_ZERO_COMMENT_RE = re.compile( - r"^(?:0|no comments?|no replies|还没有评论|暂无评论|没有评论|暂无回复|0\s*条评论)$", - re.IGNORECASE, -) -_BROWSER_IMAGE_CAPABILITY_GUIDANCE = { - True: { - "en": ( - "The current model can inspect image input. Use browser_take_screenshot only when " - "pixel-level visual evidence is required and DOM probes, accessibility snapshots, " - "or targeted evaluation cannot answer the task." - ), - "cn": ( - "当前模型可以理解图片输入。仅当任务需要像素级视觉证据，且 DOM 探测、无障碍快照或" - "定向脚本无法解决时，才使用 browser_take_screenshot。" - ), - }, - False: { - "en": ( - "Image input is unavailable or unverified for this run. Do not request screenshots, " - "including browser_take_screenshot or browser_batch_interact with op=screenshot. " - "Use DOM probes, accessibility snapshots, and targeted evaluation instead." - ), - "cn": ( - "本次运行的图片输入能力不可用或尚未确认。不要请求截图，包括 " - "browser_take_screenshot 或 browser_batch_interact 的 op=screenshot；" - "请改用 DOM 探测、无障碍快照和定向脚本。" - ), - }, -} -_BROWSER_PHASE_DEFINITIONS = { - "navigation": { - "budget": 12, - "completion_condition": "Target URL or expected page identity is observed.", - }, - "form": { - "budget": 24, - "completion_condition": "All required fields are populated and submission is evidenced.", - }, - "filtering": { - "budget": 20, - "completion_condition": "Requested filter/sort state and changed results are observed.", - }, - "extraction": { - "budget": 20, - "completion_condition": "One structured result contains every requested field.", - }, -} - - -def canonicalize_playwright_tool_name(tool_name: str) -> str: - """Normalize the common Playwright MCP server/tool separator spelling.""" - value = str(tool_name or "") - return re.sub( - r"(playwright[-_]official)-browser_", - r"\1_browser_", - value, - flags=re.IGNORECASE, - ) - - -def _contains_any_token(value: str, tokens: Iterable[str]) -> bool: - for token in tokens: - if token in value: - return True - return False - - -def _has_non_empty_mapping_value( - value: Dict[str, Any], - keys: Iterable[str], -) -> bool: - for key in keys: - if value.get(key) not in (None, ""): - return True - return False - - -def _copy_non_none_values( - value: Dict[str, Any], - keys: Iterable[str], -) -> Dict[str, Any]: - copied: Dict[str, Any] = {} - for key in keys: - item = value.get(key) - if item is not None: - copied[key] = item - return copied - - -class BrowserAgentRuntime: - """Runtime kernel for browser lifecycle and deterministic helper actions.""" - - def __init__( - self, - provider: str, - api_key: str, - api_base: str, - model_name: str, - mcp_cfg: McpServerConfig, - guardrails: BrowserRunGuardrails, - instance: Optional[BrowserInstanceConfig] = None, - allowed_tool_names: Optional[Iterable[str]] = None, - ) -> None: - ensure_browser_runtime_client_patch() - self._instance = instance - resolved_allowed_tool_names = ( - CORE_BROWSER_TOOL_NAMES if allowed_tool_names is None else tuple(dict.fromkeys(allowed_tool_names)) - ) - self._service = BrowserService( - provider=provider, - api_key=api_key, - api_base=api_base, - model_name=model_name, - mcp_cfg=mcp_cfg, - guardrails=guardrails, - instance=instance, - allowed_tool_names=resolved_allowed_tool_names, - ) - self._browser_custom_action_tool = None - self._browser_list_actions_tool = None - self._controller: BaseController = ActionController() - self._code_executor = None - self._browser_probe_interactives_tool = None - self._browser_probe_cards_tool = None - self._browser_batch_interact_tool = None - self._page_state = BrowserPageState() - self._page_generation = self._page_state.generation - self._reference_generations = self._page_state.reference_generations - self._selector_primary_links = self._page_state.selector_primary_links - self._last_observed_url = "" - self._semantic_state_tracker = SemanticStateTracker() - _ACTIVE_BROWSER_RUNTIMES.add(self) - - @property - def service(self) -> BrowserService: - return self._service - - @property - def browser_custom_action_tool(self) -> Any: - return self._browser_custom_action_tool - - @property - def browser_list_actions_tool(self) -> Any: - return self._browser_list_actions_tool - - @property - def browser_probe_interactives_tool(self) -> Any: - return self._browser_probe_interactives_tool - - @property - def browser_probe_cards_tool(self) -> Any: - return self._browser_probe_cards_tool - - @property - def browser_batch_interact_tool(self) -> Any: - return self._browser_batch_interact_tool - - @property - def controller(self) -> BaseController: - return self._controller - - @property - def code_executor(self) -> Any: - return self._code_executor - - def _ensure_page_state(self) -> BrowserPageState: - """Create PageState lazily for compatibility with lightweight test runtimes.""" - page_state = getattr(self, "_page_state", None) - if isinstance(page_state, BrowserPageState): - return page_state - page_state = BrowserPageState( - generation=int(getattr(self, "_page_generation", 0) or 0), - ) - page_state.reference_generations.update(getattr(self, "_reference_generations", {}) or {}) - page_state.selector_primary_links.update(getattr(self, "_selector_primary_links", {}) or {}) - self._page_state = page_state - self._reference_generations = page_state.reference_generations - self._selector_primary_links = page_state.selector_primary_links - return page_state - - @property - def page_id(self) -> str: - return self._ensure_page_state().page_id - - @property - def generation_id(self) -> str: - return self._ensure_page_state().generation_id - - def export_page_state(self) -> Dict[str, Any]: - return self._ensure_page_state().export() - - def set_task_requested_fields(self, fields: Iterable[str]) -> None: - """Keep PageState projection scoped to fields requested by this task.""" - - self._ensure_page_state().set_requested_fields(fields) - - def resolve_model_target_id(self, target_id: str) -> BrowserTarget: - """Resolve or safely refresh a runtime-owned model target.""" - - page_state = self._ensure_page_state() - normalized = str(target_id or "").strip() - target = page_state.get_target(normalized) - if target is None: - raise ValueError( - f"Unknown PageState target_id {normalized}; current generation is " - f"{page_state.generation_id}." - ) - if target.generation != page_state.generation: - target = page_state.get_target(page_state.refresh_target_id(normalized)) - if target is None: - raise ValueError( - f"PageState target_id {normalized} could not be refreshed in " - f"{page_state.generation_id}." - ) - return target - - def target_recovery_state(self, target_id: str) -> Dict[str, Any]: - """Return bounded current-generation recovery details for one target.""" - - page_state = self._ensure_page_state() - return { - "current_generation": page_state.generation_id, - "candidate_fresh_targets": page_state.recovery_candidates(target_id, limit=5), - } - - def _advance_page_generation(self) -> None: - page_state = self._ensure_page_state() - page_state.advance() - self._page_generation = page_state.generation - - def _observe_page_url(self, url: Any, *, force_navigation: bool = False) -> None: - normalized = str(url or "").strip() - changed = bool( - normalized - and ( - (self._last_observed_url and normalized != self._last_observed_url) - or (not self._last_observed_url and bool(self._reference_generations)) - ) - ) - if force_navigation or changed: - self._advance_page_generation() - self._ensure_page_state().observe(url=normalized) - if normalized: - self._last_observed_url = normalized - - @classmethod - def extract_result_url(cls, value: Any) -> str: - if isinstance(value, dict): - direct = value.get("url") - if direct: - return str(direct) - page = value.get("page") - if isinstance(page, dict) and page.get("url"): - return str(page["url"]) - for key in ("page", "page_state", "result", "content", "text"): - nested = value.get(key) - resolved = cls.extract_result_url(nested) - if resolved: - return resolved - elif isinstance(value, (list, tuple)): - for nested in value: - resolved = cls.extract_result_url(nested) - if resolved: - return resolved - elif isinstance(value, str): - match = re.search( - r"^\s*(?:-\s+)?Page\s+URL\s*:\s*(https?://[^\s<>\"]+)", - value, - re.IGNORECASE | re.MULTILINE, - ) - if match is not None: - return match.group(1).rstrip(".,;)") - return "" - - @classmethod - def _extract_result_title(cls, value: Any) -> str: - if isinstance(value, dict): - direct = value.get("title") - if direct: - return str(direct) - page = value.get("page") - if isinstance(page, dict) and page.get("title"): - return str(page["title"]) - for key in ("page", "page_state", "result", "content", "text"): - nested = value.get(key) - resolved = cls._extract_result_title(nested) - if resolved: - return resolved - elif isinstance(value, (list, tuple)): - for nested in value: - resolved = cls._extract_result_title(nested) - if resolved: - return resolved - elif isinstance(value, str): - match = re.search( - r"^\s*(?:-\s+)?Page\s+Title\s*:\s*([^\r\n]+)", - value, - re.IGNORECASE | re.MULTILINE, - ) - if match is not None: - return match.group(1).strip() - return "" - - @classmethod - def classify_tool_result(cls, value: Any) -> Dict[str, Any]: - """Return one normalized success/error view for every tool transport.""" - - nested = value - explicit_failure = bool( - getattr(value, "success", None) is False - or getattr(value, "isError", getattr(value, "is_error", None)) is True - ) - denied = False - error_text = str(getattr(value, "error", "") or "").strip() if explicit_failure else "" - data = getattr(value, "data", None) - if data is not None: - nested = data - - if isinstance(nested, dict): - mapping_failure, denied, mapping_error = cls._classify_mapping_tool_result(nested) - explicit_failure = explicit_failure or mapping_failure - error_text = mapping_error or error_text - elif isinstance(nested, str): - stripped = nested.strip() - try: - parsed = json.loads(stripped) - except (TypeError, ValueError): - parsed = None - if isinstance(parsed, dict): - parsed_outcome = cls.classify_tool_result(parsed) - explicit_failure = explicit_failure or not parsed_outcome["success"] - denied = denied or parsed_outcome["denied"] - error_text = error_text or str(parsed_outcome["error"] or "") - elif _BROWSER_TOOL_ERROR_PREFIX_RE.match(stripped): - explicit_failure = True - error_text = stripped - - timed_out = bool(error_text and _BROWSER_TIMEOUT_ERROR_RE.search(error_text)) - return { - "success": not explicit_failure, - "error": error_text[:2_000], - "denied": denied, - "timed_out": timed_out, - } - - @staticmethod - def _classify_mapping_tool_result(value: Dict[str, Any]) -> tuple[bool, bool, str]: - status = str(value.get("status") or "").strip().lower() - denied = value.get("denied") is True or status == "denied" - nested_error = value.get("error") - if isinstance(nested_error, dict): - nested_error = nested_error.get("message") or nested_error.get("code") - error_text = str(nested_error or "").strip() - failure = bool( - value.get("ok") is False - or value.get("success") is False - or value.get("isError") is True - or value.get("is_error") is True - or error_text - or status in {"error", "failed", "failure", "partial", "denied", "timeout", "timed_out"} - ) - for step in value.get("steps") or []: - if not isinstance(step, dict): - continue - step_status = str(step.get("status") or "").strip().lower() - if step.get("ok") is not False and step_status not in { - "error", - "failed", - "failure", - "timeout", - "timed_out", - }: - continue - failure = True - error_text = error_text or str(step.get("error") or f"batch step {step.get('index')} failed").strip() - break - if not failure and isinstance(value.get("content"), list): - for item in value["content"]: - if not isinstance(item, dict) or not isinstance(item.get("text"), str): - continue - if _BROWSER_TOOL_ERROR_PREFIX_RE.match(item["text"]): - return True, denied, item["text"].strip() - return failure, denied, error_text - - @classmethod - def tool_result_succeeded(cls, value: Any) -> bool: - return bool(cls.classify_tool_result(value)["success"]) - - def _register_snapshot_refs(self, value: Any, *, replace: bool = False) -> None: - page_state = self._ensure_page_state() - registered = page_state.replace_ax_snapshot(value) if replace else page_state.register_ax_snapshot(value) - if registered: - return - # Preserve ref-only snapshots that do not follow the normal AX line shape. - text = value if isinstance(value, str) else str(value) - for ref_value in _BROWSER_SNAPSHOT_REF_RE.findall(text): - page_state.reference_generations[str(ref_value)] = page_state.generation - - def validate_reference_values(self, values: Iterable[str]) -> None: - page_state = self._ensure_page_state() - stale_values: set[str] = set() - for ref_value in values: - ref_generation = page_state.reference_generations.get(ref_value) - if ref_generation is not None and ref_generation != page_state.generation: - stale_values.add(ref_value) - stale = sorted(stale_values) - if stale: - raise ValueError( - "Stale browser snapshot reference(s) " - f"{', '.join(stale)} belong to an older page generation. " - "Call browser_snapshot again and use refs from " - f"generation {self.generation_id}." - ) - - def record_tool_reference_state( - self, - *, - tool_name: str, - tool_args: Any, - tool_result: Any, - ) -> None: - normalized_name = str(tool_name or "").strip().lower() - if not self.tool_result_succeeded(tool_result): - return - result_url = self.extract_result_url(tool_result) - result_title = self._extract_result_title(tool_result) - navigation_like = _contains_any_token( - normalized_name, - ("browser_navigate", "browser_navigate_back", "browser_tabs"), - ) - if "browser_tabs" in normalized_name and isinstance(tool_args, dict): - navigation_like = str(tool_args.get("action") or "").lower() in { - "new", - "select", - "close", - } - self._observe_page_url(result_url, force_navigation=navigation_like) - self._ensure_page_state().observe(url=result_url, title=result_title) - if "browser_snapshot" in normalized_name or "browser_find" in normalized_name: - self._register_snapshot_refs(tool_result) - - def _annotate_probe_generation(self, value: Any) -> None: - if isinstance(value, dict): - if "selector_hint" in value or "generation_id" in value: - value["generation_id"] = self.generation_id - for nested in value.values(): - self._annotate_probe_generation(nested) - elif isinstance(value, list): - for nested in value: - self._annotate_probe_generation(nested) - - def register_card_primary_links(self, result: Dict[str, Any]) -> None: - page_state = self._ensure_page_state() - cards = result.get("cards") - if not isinstance(cards, list): - return - for card in cards: - if not isinstance(card, dict): - continue - href = str(card.get("primary_link") or card.get("href") or "").strip() - if not href: - continue - for key in ("selector_hint", "primary_link_selector_hint"): - selector = str(card.get(key) or "").strip() - if selector: - page_state.selector_primary_links[selector] = ( - page_state.generation, - href, - ) - - def resolve_primary_link(self, tool_args: Any) -> str: - if isinstance(tool_args, str): - try: - parsed = json.loads(tool_args) - except ValueError: - return "" - else: - parsed = tool_args - if not isinstance(parsed, dict): - return "" - explicit = str(parsed.get("primary_link") or parsed.get("href") or parsed.get("url") or "").strip() - if explicit: - return explicit - target_id = str(parsed.get("target_id") or parsed.get("target") or "").strip() - if target_id.startswith("t_g"): - try: - target = self.resolve_model_target_id(target_id) - except ValueError: - target = None - if target is not None and target.href: - return target.href - for key in ("selector", "target"): - selector = str(parsed.get(key) or "").strip() - generation_and_href = self._selector_primary_links.get(selector) - if generation_and_href and generation_and_href[0] == self._ensure_page_state().generation: - return generation_and_href[1] - return "" - - @staticmethod - def _register_runtime_tool(tool_obj: Any, *, tool_name: str) -> None: - add_result = Runner.resource_mgr.add_tool(tool_obj, tag="agent.playwright.runtime") - if add_result is None or getattr(add_result, "is_ok", lambda: False)(): - return - error_value = getattr(add_result, "value", add_result) - if "already exist" in str(error_value): - return - raise RuntimeError(f"Failed to register {tool_name} tool: {error_value}") - - async def cancel_run(self, session_id: str, request_id: Optional[str] = None) -> Dict[str, Any]: - await self._service.request_cancel(session_id=session_id, request_id=request_id) - return { - "ok": True, - "session_id": session_id, - "request_id": request_id, - "error": None, - } - - async def clear_cancel(self, session_id: str, request_id: Optional[str] = None) -> Dict[str, Any]: - await self._service.clear_cancel(session_id=session_id, request_id=request_id) - return { - "ok": True, - "session_id": session_id, - "request_id": request_id, - "error": None, - } - - def _playwright_client_lookup_keys(self) -> list[str]: - """Return likely registry keys for the active Playwright MCP client.""" - server_id = str(getattr(self._service.mcp_cfg, "server_id", "") or "").strip() - server_name = str(getattr(self._service.mcp_cfg, "server_name", "") or "").strip() - - candidates = [ - server_id, - server_name, - server_id.replace("-", "_"), - server_id.replace("_", "-"), - server_name.replace("-", "_"), - server_name.replace("_", "-"), - ] - - # Common IDs seen in the direct Playwright runtime path. Skipped for a - # keyed instance so its lookup never resolves the legacy/unkeyed client. - if not (self._instance and self._instance.key): - candidates.extend( - [ - "playwright_official_stdio", - "playwright-official", - "playwright", - ] - ) - - result: list[str] = [] - seen: set[str] = set() - for item in candidates: - if item and item not in seen: - seen.add(item) - result.append(item) - return result - - @staticmethod - def _unwrap_mcp_text_result(raw: Any) -> Any: - """Extract text payload from common MCP tool-result shapes.""" - if isinstance(raw, dict): - if raw.get("__browser_compact_rpc__") is True: - return BrowserAgentRuntime._unwrap_mcp_text_result(raw.get("payload")) - - content = raw.get("content") - if isinstance(content, list): - texts: list[str] = [] - for item in content: - if isinstance(item, dict) and item.get("type") == "text": - texts.append(str(item.get("text") or "")) - if texts: - return "\n".join(texts) - - if "result" in raw: - return raw.get("result") - - if "text" in raw: - return raw.get("text") - - if "data" in raw: - return raw.get("data") - - return raw - - @classmethod - def _compact_run_code_payload(cls, payload: Any) -> Any: - """Keep the JSON result and discard generic run-code page-state text.""" - unwrapped = cls._unwrap_mcp_text_result(payload) - parsed = extract_json_object(unwrapped) - if isinstance(parsed, dict): - return json.dumps( - parsed, - ensure_ascii=False, - separators=(",", ":"), - ) - return payload - - async def _get_playwright_mcp_tool(self, tool_name: str) -> Any: - """Resolve a registered Playwright MCP tool through Runner.resource_mgr.""" - server_id = str(getattr(self._service.mcp_cfg, "server_id", "") or "").strip() - server_name = str(getattr(self._service.mcp_cfg, "server_name", "") or "").strip() - - # When this runtime is bound to a specific browser identity, the cfg - # server_id is already unique; the generic fallbacks below could resolve - # the legacy/unkeyed server, so they are skipped to keep isolation. - keyed = bool(self._instance and self._instance.key) - - server_id_candidates = [ - server_id, - server_id.replace("-", "_"), - server_id.replace("_", "-"), - ] - if not keyed: - server_id_candidates += [ - "playwright_official_stdio", - "playwright-official-stdio", - "playwright", - ] - - server_name_candidates = [ - server_name, - server_name.replace("-", "_"), - server_name.replace("_", "-"), - ] - if not keyed: - server_name_candidates += [ - "playwright-official", - "playwright_official", - "playwright", - ] - - def _first_tool(value: Any) -> Any: - if isinstance(value, list): - return next((item for item in value if item is not None), None) - return value - - tried: list[str] = [] - - for candidate in server_id_candidates: - if not candidate: - continue - - tried.append(f"server_id={candidate}") - - tool = None - try: - tool = await Runner.resource_mgr.get_mcp_tool( - name=tool_name, - server_id=candidate, - skip_if_tag_not_exists=True, - ignore_exception=True, - ) - tool = _first_tool(tool) - except Exception: - logger.debug( - "Failed to resolve MCP tool %s using server_id=%s", - tool_name, - candidate, - exc_info=True, - ) - - if tool is not None: - return tool - - for candidate in server_name_candidates: - if not candidate: - continue - - tried.append(f"server_name={candidate}") - - tool = None - try: - tool = await Runner.resource_mgr.get_mcp_tool( - name=tool_name, - server_name=candidate, - skip_if_tag_not_exists=True, - ignore_exception=True, - ) - tool = _first_tool(tool) - except Exception: - logger.debug( - "Failed to resolve MCP tool %s using server_name=%s", - tool_name, - candidate, - exc_info=True, - ) - - if tool is not None: - return tool - - raise RuntimeError(f"Registered Playwright MCP tool not found: {tool_name}. Tried {', '.join(tried)}") - - async def _get_playwright_run_code_tool(self) -> tuple[Any, str]: - """Resolve browser_run_code_unsafe, with browser_run_code as compatibility fallback.""" - try: - return await self._get_playwright_mcp_tool("browser_run_code_unsafe"), "browser_run_code_unsafe" - except RuntimeError: - logger.debug( - "browser_run_code_unsafe is unavailable; falling back to browser_run_code", - exc_info=True, - ) - - return await self._get_playwright_mcp_tool("browser_run_code"), "browser_run_code" - - async def _call_playwright_run_code_unsafe(self, js_code: str) -> Any: - """Execute a compact runtime RPC over the registered Playwright transport.""" - total_started_at = time.perf_counter() - resolution_started_at = time.perf_counter() - tool, tool_name = await self._get_playwright_run_code_tool() - resolution_elapsed_ms = int(max(0.0, (time.perf_counter() - resolution_started_at) * 1000)) - - invoke_started_at = time.perf_counter() - result = await tool.invoke({"code": js_code}) - invoke_elapsed_ms = int(max(0.0, (time.perf_counter() - invoke_started_at) * 1000)) - - success = getattr(result, "success", None) - if success is False: - error = str(getattr(result, "error", "") or "").strip() - raise RuntimeError(error or f"{tool_name} failed") - - data = getattr(result, "data", None) - if data is not None: - payload = data - else: - payload = result - transport_response_size_bytes = len(str(payload).encode("utf-8", "ignore")) - compact_payload = self._compact_run_code_payload(payload) - return { - "__browser_compact_rpc__": True, - "payload": compact_payload, - "rpc_metrics": { - "tool_name": tool_name, - "tool_resolution_elapsed_ms": resolution_elapsed_ms, - "transport_invoke_elapsed_ms": invoke_elapsed_ms, - "rpc_total_elapsed_ms": int(max(0.0, (time.perf_counter() - total_started_at) * 1000)), - "script_size_bytes": len(js_code.encode("utf-8", "ignore")), - "transport_response_size_bytes": transport_response_size_bytes, - "response_size_bytes": len(str(compact_payload).encode("utf-8", "ignore")), - }, - } - - async def _call_playwright_tool(self, tool_name: str, inputs: Dict[str, Any]) -> Any: - """Invoke one registered Playwright MCP tool and unwrap its result data.""" - tool = await self._get_playwright_mcp_tool(tool_name) - result = await tool.invoke(inputs) - - success = getattr(result, "success", None) - if success is False: - error = str(getattr(result, "error", "") or "").strip() - raise RuntimeError(error or f"{tool_name} failed") - - data = getattr(result, "data", None) - if data is not None: - return data - return result - - async def ensure_runtime_ready(self) -> None: - _ACTIVE_BROWSER_RUNTIMES.add(self) - await self._service.ensure_runtime_ready() - if self._code_executor is not None: - return - - async def _direct_code_executor(js_code: str): - return await self._call_playwright_run_code_unsafe(js_code) - - self._code_executor = _direct_code_executor - self._controller.bind_code_executor(_direct_code_executor) - self._controller.register_builtin_actions() - - async def capture_browser_state(self, *, action_group_id: str = "") -> Dict[str, Any]: - """Capture a fresh, non-cached browser observation for the next model call.""" - await self.ensure_runtime_ready() - - dom = "" - dom_error = None - snapshot_captured = False - snapshot_audit: Dict[str, Any] = {} - try: - raw_snapshot = await self._call_playwright_tool("browser_snapshot", {}) - raw_snapshot = self._unwrap_mcp_text_result(raw_snapshot) - snapshot_audit = write_browser_agent_audit_artifact("ax_snapshot", raw_snapshot) - snapshot_captured = True - if isinstance(raw_snapshot, str): - dom = raw_snapshot - elif raw_snapshot is not None: - dom = json.dumps(raw_snapshot, ensure_ascii=False) - except Exception as exc: - dom_error = f"browser_snapshot failed: {exc}" - logger.warning( - "[BrowserAgentRuntime] current DOM snapshot capture failed: %s", - exc, - exc_info=True, - ) - - metadata, metadata_error = await self._capture_browser_metadata() - - self._observe_page_url(metadata.get("url")) - self._ensure_page_state().observe(title=metadata.get("title")) - if snapshot_captured: - self._register_snapshot_refs(dom, replace=True) - page_state = self.export_page_state() - - errors = [error for error in (dom_error, metadata_error) if error] - semantic_state = metadata.get("semantic_state") - if not isinstance(semantic_state, dict): - semantic_state = {} - semantic_state.update( - { - "url": metadata.get("url") or "", - "field_coverage": sorted(self._ensure_page_state().field_coverage), - } - ) - semantic_tracker = self._ensure_semantic_state_tracker() - if metadata_error: - semantic_progress = semantic_tracker.latest - semantic_progress.update( - { - "progress": "unknown", - "observable_progress": False, - "capture_error": metadata_error, - } - ) - else: - semantic_progress = semantic_tracker.observe( - semantic_state, - action_group_id=action_group_id, - ) - semantic_state = self._with_semantic_provenance( - semantic_progress, - fallback_state=semantic_state, - ) - return { - "ok": not errors, - "error": "; ".join(errors) or None, - "url": metadata.get("url") or "", - "title": metadata.get("title") or "", - "tabs": metadata.get("tabs") or [], - "page_position": metadata.get("page_position") or {}, - "semantic_state": semantic_state, - "semantic_progress": semantic_progress, - "field_coverage": semantic_state.get("field_coverage") or [], - "page_state": page_state, - "dom": "", - "dom_error": dom_error, - "audit": {"ax_snapshot": snapshot_audit} if snapshot_audit else {}, - } - - async def capture_reconciliation_browser_state(self, *, action_group_id: str) -> Dict[str, Any]: - """Reconcile an ambiguous mutation without capturing a full AX snapshot.""" - - await self.ensure_runtime_ready() - metadata, metadata_error = await self._capture_browser_metadata() - self._observe_page_url(metadata.get("url")) - self._ensure_page_state().observe(title=metadata.get("title")) - page_state = self.export_page_state() - semantic_state = metadata.get("semantic_state") - if not isinstance(semantic_state, dict): - semantic_state = {} - semantic_state.update( - { - "url": metadata.get("url") or page_state.get("url") or "", - "field_coverage": page_state.get("field_coverage") or [], - } - ) - tracker = self._ensure_semantic_state_tracker() - if metadata_error: - semantic_progress = tracker.latest - semantic_progress.update( - { - "progress": "unknown", - "observable_progress": False, - "capture_error": metadata_error, - } - ) - else: - semantic_progress = tracker.observe(semantic_state, action_group_id=action_group_id) - semantic_state = self._with_semantic_provenance( - semantic_progress, - fallback_state=semantic_state, - ) - return { - "ok": not metadata_error, - "error": metadata_error, - "url": metadata.get("url") or page_state.get("url") or "", - "title": metadata.get("title") or page_state.get("title") or "", - "tabs": metadata.get("tabs") or [], - "page_position": metadata.get("page_position") or {}, - "semantic_state": semantic_state, - "semantic_progress": semantic_progress, - "field_coverage": semantic_state.get("field_coverage") or [], - "page_state": page_state, - "dom": "", - "dom_error": None, - "reconciliation_only": True, - } - - async def capture_compact_browser_state(self, *, action_group_id: str) -> Dict[str, Any]: - """Merge completed read-only observations without another browser round trip.""" - page_state = self.export_page_state() - semantic_tracker = self._ensure_semantic_state_tracker() - semantic_state = semantic_tracker.current_state - semantic_state.update( - { - "url": page_state.get("url") or semantic_state.get("url") or "", - "field_coverage": page_state.get("field_coverage") or semantic_state.get("field_coverage") or [], - } - ) - semantic_progress = semantic_tracker.observe( - semantic_state, - action_group_id=action_group_id, - ) - semantic_state = self._with_semantic_provenance( - semantic_progress, - fallback_state=semantic_state, - ) - return { - "ok": True, - "error": None, - "url": page_state.get("url") or "", - "title": page_state.get("title") or "", - "tabs": [], - "page_position": {}, - "semantic_state": semantic_state, - "semantic_progress": semantic_progress, - "field_coverage": semantic_state["field_coverage"], - "page_state": page_state, - "dom": "", - "dom_error": None, - } - - def _with_semantic_provenance( - self, - semantic_progress: Dict[str, Any], - *, - fallback_state: Dict[str, Any], - ) -> Dict[str, Any]: - tracked_state = semantic_progress.get("semantic_state") - semantic_state = dict(tracked_state) if isinstance(tracked_state, dict) else dict(fallback_state) - semantic_state["generation_id"] = self.generation_id - semantic_progress["semantic_state"] = semantic_state - return semantic_state - - async def _capture_browser_metadata(self) -> tuple[Dict[str, Any], Optional[str]]: - """Capture the bounded metadata used for semantic reconciliation.""" - - try: - raw_metadata = await self._call_playwright_run_code_unsafe(build_browser_state_metadata_js()) - raw_metadata = self._unwrap_mcp_text_result(raw_metadata) - metadata = extract_json_object(raw_metadata) - if not metadata: - return {}, "Could not parse browser state metadata result JSON" - if metadata.get("ok") is False: - return metadata, str(metadata.get("error") or "browser state metadata capture failed") - return metadata, None - except Exception as exc: - logger.warning( - "[BrowserAgentRuntime] current browser metadata capture failed: %s", - exc, - exc_info=True, - ) - return {}, f"browser state metadata capture failed: {exc}" - - async def acquire_task_resources(self) -> None: - """Acquire one task reference before invoking a reusable subagent.""" - _ACTIVE_BROWSER_RUNTIMES.add(self) - self._service.acquire_task_binding() - - async def ensure_started(self) -> None: - await self.ensure_runtime_ready() - await self._service.ensure_started() - if self._browser_custom_action_tool is not None: - return - from .runtime_tools import ( - BrowserBatchInteractTool, - BrowserCustomActionTool, - BrowserListActionsTool, - BrowserProbeCardsTool, - BrowserProbeInteractivesTool, - ) - - self._browser_probe_interactives_tool = BrowserProbeInteractivesTool(self, language="en") - self._browser_probe_cards_tool = BrowserProbeCardsTool(self, language="en") - self._browser_batch_interact_tool = BrowserBatchInteractTool(self, language="en") - self._browser_custom_action_tool = BrowserCustomActionTool(self, language="en") - self._browser_list_actions_tool = BrowserListActionsTool(self, language="en") - self._register_runtime_tool( - self._browser_probe_interactives_tool, - tool_name="browser_probe_interactives", - ) - self._register_runtime_tool( - self._browser_probe_cards_tool, - tool_name="browser_probe_cards", - ) - self._register_runtime_tool( - self._browser_batch_interact_tool, - tool_name="browser_batch_interact", - ) - self._register_runtime_tool( - self._browser_custom_action_tool, - tool_name="browser_custom_action", - ) - self._register_runtime_tool( - self._browser_list_actions_tool, - tool_name="browser_list_custom_actions", - ) - - # Legacy browser_run_task compatibility: let the worker agent call - # deterministic controller helpers without introducing another planner. - if self._service.browser_agent is not None: - # Add compact standalone helpers first so the browser worker sees the - # same first-class affordances as the probe tools before MCP primitives. - self._service.browser_agent.ability_manager.add(self._browser_probe_interactives_tool.card) - self._service.browser_agent.ability_manager.add(self._browser_probe_cards_tool.card) - self._service.browser_agent.ability_manager.add(self._browser_batch_interact_tool.card) - - async def run_browser_task( - self, - task: str, - session_id: Optional[str] = None, - request_id: Optional[str] = None, - timeout_s: Optional[int] = None, - ) -> Dict[str, Any]: - await self.ensure_started() - return await self._service.run_task( - task=task, - session_id=session_id, - request_id=request_id, - timeout_s=timeout_s, - ) - - async def _materialize_ax_target(self, target: BrowserTarget) -> BrowserTarget: - """Convert an MCP AX ref into a temporary DOM marker without model translation.""" - if target.locator.get("selector"): - return target - ref_value = str(target.ref or target.locator.get("ref") or "").strip() - if not ref_value: - raise ValueError(f"PageState target has no executable locator: {target.target_id}") - - attribute = "data-openjiuwen-target-id" - marker_value = target.target_id - function = ( - f"(element) => {{element.setAttribute({json.dumps(attribute)}, {json.dumps(marker_value)});return true;}}" - ) - tool = await self._get_playwright_mcp_tool("browser_evaluate") - result = await tool.invoke( - { - "target": ref_value, - "element": f"PageState target {target.target_id}", - "function": function, - } - ) - if getattr(result, "success", None) is False: - error = str(getattr(result, "error", "") or "").strip() - raise ValueError(error or f"Failed to resolve AX ref {ref_value} for batch execution") - selector = f'[{attribute}="{marker_value}"]' - self._ensure_page_state().update_target_locator( - target.target_id, - {"selector": selector}, - ) - return target - - async def _resolve_batch_target( - self, - step: Dict[str, Any], - *, - generation_id: str, - option: bool = False, - allow_navigation_rewrite: bool = False, - ) -> None: - page_state = self._ensure_page_state() - prefix = "option_" if option else "" - target_id = str( - step.get(f"{prefix}target_id") or (step.get("choose_target_id") if option else "") or "" - ).strip() - ref_value = str(step.get(f"{prefix}ref") or (step.get("choose_ref") if option else "") or "").strip() - selector = str(step.get(f"{prefix}selector") or (step.get("choose_selector") if option else "") or "").strip() - op = str(step.get("op") or "").strip().lower() - - if not target_id and not ref_value and not selector: - return - require_registered_selector = op in _BATCH_REGISTERED_TARGET_OPS - if op in _BATCH_EXPLICIT_SELECTOR_OPS or op in _BATCH_SAFE_READ_SELECTOR_OPS: - require_registered_selector = False - target = page_state.resolve_target( - generation_id=generation_id, - target_id=target_id, - ref=ref_value, - selector=selector, - require_registered_selector=require_registered_selector, - ) - if target is None: - return - if target.href and target.source in {"card", "card_primary_link"} and op == "click": - if allow_navigation_rewrite: - for key in _BATCH_MODEL_LOCATOR_KEYS: - step.pop(key, None) - step["resolved_target_id"] = target.target_id - step["_navigate_url"] = target.href - return - raise ValueError( - f"Target {target.target_id} has primary_link={target.href}; " - "call browser_navigate directly instead of clicking it in a batch" - ) - requires_actionability = op in _BATCH_MUTATING_TARGET_OPS and target.source != "ax" - if requires_actionability: - is_unavailable = not target.visible or not target.enabled or not target.actionable - if is_unavailable: - raise ValueError(f"PageState target {target.target_id} is not actionable in {target.generation_id}") - if target.locator.get("ref"): - target = await self._materialize_ax_target(target) - - locator = dict(target.locator) - if option: - for key in ( - "option_target_id", - "choose_target_id", - "option_ref", - "choose_ref", - "option_selector", - "choose_selector", - "option_role", - "choose_role", - "option_name", - "choose_name", - "option_text", - "choose_text", - "text_to_choose", - ): - step.pop(key, None) - if locator.get("selector"): - step["option_selector"] = locator["selector"] - elif locator.get("role"): - step["option_role"] = locator["role"] - if locator.get("name"): - step["option_name"] = locator["name"] - elif locator.get("text"): - step["option_text"] = locator["text"] - else: - raise ValueError(f"PageState option target is not executable: {target.target_id}") - step["resolved_option_target_id"] = target.target_id - return - - for key in _BATCH_MODEL_LOCATOR_KEYS: - step.pop(key, None) - step.update(locator) - step["resolved_target_id"] = target.target_id - - async def _resolve_batch_steps( - self, - steps: Any, - *, - generation_id: str, - allow_navigation_rewrite: bool = False, - ) -> list[Dict[str, Any]]: - resolved_steps = [dict(step) for step in steps] - for step in resolved_steps: - await self._resolve_batch_target( - step, - generation_id=generation_id, - allow_navigation_rewrite=allow_navigation_rewrite, - ) - option_target_keys = ( - "option_target_id", - "choose_target_id", - "option_ref", - "choose_ref", - "option_selector", - "choose_selector", - ) - if _has_non_empty_mapping_value(step, option_target_keys): - await self._resolve_batch_target( - step, - generation_id=generation_id, - option=True, - ) - return resolved_steps - - async def _validate_safe_read_locators(self, steps: list[Dict[str, Any]]) -> None: - """Validate model-provided read locators once before batch execution.""" - checks: list[Dict[str, Any]] = [] - for index, step in enumerate(steps): - operation = str(step.get("op") or "").strip().lower() - selector = str(step.get("selector") or "").strip() - resolved_target_id = str(step.get("resolved_target_id") or "").strip() - if operation not in _BATCH_SAFE_READ_SELECTOR_OPS: - continue - if not selector or resolved_target_id: - continue - checks.append({"index": index, "selector": selector}) - if not checks: - return - script = f"""() => {{ - const checks = {json.dumps(checks, ensure_ascii=False)}; - const results = checks.map((check) => {{ - let nodes = []; - try {{ nodes = Array.from(document.querySelectorAll(check.selector)); }} - catch (error) {{ - return {{...check, match_count: 0, visible: false, error: String(error)}}; - }} - const element = nodes[0]; - const visible = Boolean(element && element.isConnected && - element.getClientRects().length && getComputedStyle(element).visibility !== 'hidden'); - return {{...check, match_count: nodes.length, visible}}; - }}); - return JSON.stringify({{ok: true, results}}); - }}""" - raw = await self._call_playwright_run_code_unsafe(script) - parsed = extract_json_object(self._unwrap_mcp_text_result(raw)) - results = parsed.get("results") if isinstance(parsed, dict) else None - if not isinstance(results, list): - raise ValueError("Could not validate safe batch read locators") - invalid = [ - item - for item in results - if not isinstance(item, dict) or item.get("match_count") != 1 or item.get("visible") is not True - ] - if invalid: - first = invalid[0] if isinstance(invalid[0], dict) else {} - raise ValueError( - "Read-only batch selector must match exactly one visible element: " - f"step={first.get('index')}, selector={first.get('selector')}, " - f"match_count={first.get('match_count')}" - ) - - async def _refresh_stale_batch_targets( - self, - steps: Any, - *, - generation_id: str, - ) -> tuple[list[Dict[str, Any]], str]: - """Refresh stale Probe target IDs without reviving refs or CSS.""" - - page_state = self._ensure_page_state() - if str(generation_id or "").strip() == page_state.generation_id: - return [dict(step) for step in steps], "" - - refreshed_steps = [dict(step) for step in steps] - refreshed_any = False - target_alias_groups = ( - ("target_id",), - ("option_target_id", "choose_target_id"), - ) - for step in refreshed_steps: - step_refreshed = await self._refresh_stale_batch_step( - step, - page_state=page_state, - target_alias_groups=target_alias_groups, - ) - refreshed_any = refreshed_any or step_refreshed - - condition_only = all( - str(step.get("op") or "").strip().lower() in _BATCH_SAFE_GENERATION_REFRESH_OPS for step in refreshed_steps - ) - if not refreshed_any and not condition_only: - raise ValueError( - f"Stale PageState generation {generation_id}; current generation is " - f"{page_state.generation_id}. Probe or snapshot the current page again." - ) - return refreshed_steps, str(generation_id or "").strip() - - async def _refresh_stale_batch_step( - self, - step: Dict[str, Any], - *, - page_state: BrowserPageState, - target_alias_groups: tuple[tuple[str, ...], ...], - ) -> bool: - op = str(step.get("op") or "").strip().lower() - refreshed = False - for aliases in target_alias_groups: - populated_alias = next( - (alias for alias in aliases if str(step.get(alias) or "").strip()), - "", - ) - if not populated_alias: - continue - stale_target_id = str(step[populated_alias]) - try: - refreshed_target_id = page_state.refresh_target_id(stale_target_id) - except ValueError: - refreshed_target_id = await self._refresh_runtime_owned_target( - stale_target_id, - require_actionable=op in _BATCH_MUTATING_TARGET_OPS, - ) - step[populated_alias] = refreshed_target_id - refreshed = True - - if _has_non_empty_mapping_value(step, ("ref", "option_ref", "choose_ref")): - raise ValueError( - "Native AX refs cannot be refreshed across PageState generations; capture the current snapshot again" - ) - has_model_selector = _has_non_empty_mapping_value( - step, - ("selector", "option_selector", "choose_selector"), - ) - if has_model_selector and not refreshed and op not in _BATCH_SAFE_GENERATION_REFRESH_OPS: - raise ValueError( - "Model-authored selectors cannot refresh a stale PageState generation; " - "use a current generation-scoped target_id" - ) - return refreshed - - async def _refresh_runtime_owned_target( - self, - target_id: str, - *, - require_actionable: bool, - ) -> str: - """Refresh a stable Probe locator after observation-only generation churn.""" - page_state = self._ensure_page_state() - stale = page_state.get_target(target_id) - if stale is None: - raise ValueError(f"Unknown PageState target_id: {target_id}") - if stale.source == "ax": - raise ValueError(f"Stale AX target_id {target_id} cannot cross generations; capture a current snapshot") - if stale.href and not stale.selector: - return page_state.rebind_target( - target_id, - visible=True, - enabled=True, - actionable=True, - ) - selector = str(stale.selector or stale.locator.get("selector") or "").strip() - if not selector: - raise ValueError(f"Stale target_id {target_id} has no runtime-owned selector to refresh") - await self.ensure_runtime_ready() - script = f"""() => {{ - const selector = {json.dumps(selector)}; - let nodes = []; - try {{ nodes = Array.from(document.querySelectorAll(selector)); }} - catch (error) {{ return JSON.stringify({{ok: false, error: String(error)}}); }} - const element = nodes[0]; - if (!element) return JSON.stringify({{ok: true, match_count: nodes.length, visible: false}}); - const style = getComputedStyle(element); - const rect = element.getBoundingClientRect(); - const visible = Boolean(element.isConnected && rect.width > 0 && rect.height > 0 && - style.display !== 'none' && style.visibility !== 'hidden'); - const enabled = !element.disabled && element.getAttribute('aria-disabled') !== 'true'; - const hit = visible ? document.elementFromPoint( - Math.max(0, Math.min(innerWidth - 1, rect.left + rect.width / 2)), - Math.max(0, Math.min(innerHeight - 1, rect.top + rect.height / 2)) - ) : null; - const actionable = Boolean(visible && enabled && hit && (hit === element || element.contains(hit))); - return JSON.stringify({{ok: true, match_count: nodes.length, visible, enabled, actionable}}); - }}""" - raw = await self._call_playwright_run_code_unsafe(script) - validation = extract_json_object(self._unwrap_mcp_text_result(raw)) - valid = ( - ( - validation.get("ok") is True - and validation.get("match_count") == 1 - and validation.get("visible") is True - and (not require_actionable or validation.get("actionable") is True) - ) - if isinstance(validation, dict) - else False - ) - if not valid: - raise ValueError( - f"Stale target_id {target_id} could not be refreshed in {page_state.generation_id}: " - f"{validation or 'validation unavailable'}" - ) - return page_state.rebind_target( - target_id, - visible=bool(validation.get("visible")), - enabled=bool(validation.get("enabled")), - actionable=bool(validation.get("actionable")), - ) - - def _batch_recovery_candidates(self, steps: Any) -> list[Dict[str, Any]]: - page_state = self._ensure_page_state() - candidates: list[Dict[str, Any]] = [] - seen: set[str] = set() - for step in steps if isinstance(steps, list) else []: - if not isinstance(step, dict): - continue - for key in ("target_id", "option_target_id", "choose_target_id"): - target_id = str(step.get(key) or "").strip() - if not target_id: - continue - for candidate in page_state.recovery_candidates(target_id): - candidate_id = str(candidate.get("target_id") or "") - if candidate_id and candidate_id not in seen: - candidates.append(candidate) - seen.add(candidate_id) - if len(candidates) >= 5: - return candidates - return candidates - - @staticmethod - def _single_batch_primitive_spec( - step: Dict[str, Any], - ) -> Optional[tuple[str, Dict[str, Any]]]: - """Map a semantically equivalent one-step batch to an MCP primitive.""" - op = str(step.get("op") or "").strip().lower() - selector = str(step.get("selector") or "").strip() - element = str(step.get("description") or step.get("resolved_target_id") or selector or op) - target_args = {"target": selector, "element": element} if selector else {} - - navigation_url = str(step.get("_navigate_url") or "").strip() - if op == "click" and navigation_url: - return "browser_navigate", {"url": navigation_url} - if op == "click" and target_args: - return "browser_click", target_args - if op in {"fill", "type"} and target_args: - try: - typing_delay_ms = int(step.get("delay_ms") or 0) - except (TypeError, ValueError): - typing_delay_ms = 0 - return "browser_type", { - **target_args, - "text": str(step.get("value") or ""), - "slowly": op == "type" or typing_delay_ms > 0, - } - if op == "select_option" and target_args: - values = step.get("values") - if values is None: - values = step.get("option_value", step.get("value")) - if values is not None: - normalized_values = values if isinstance(values, list) else [values] - return "browser_select_option", { - **target_args, - "values": [str(value) for value in normalized_values], - } - if op == "press" and not any(step.get(key) for key in _BATCH_MODEL_LOCATOR_KEYS): - return "browser_press_key", {"key": str(step.get("key") or "Enter")} - if op == "wait_for_text": - return "browser_wait_for", {"text": str(step.get("text") or "")} - if op == "sleep": - wait_ms = max(0, int(step.get("ms", step.get("time_ms", 0)) or 0)) - return "browser_wait_for", {"time": wait_ms / 1000} - if op == "screenshot": - args: Dict[str, Any] = { - "type": "png", - "fullPage": bool(step.get("full_page", False)), - } - if step.get("path"): - args["filename"] = str(step["path"]) - return "browser_take_screenshot", args - return None - - async def _run_single_batch_primitive( - self, - step: Dict[str, Any], - ) -> Optional[Dict[str, Any]]: - spec = self._single_batch_primitive_spec(step) - if spec is None: - return None - - tool_name, tool_args = spec - op = str(step.get("op") or "").strip().lower() - started_at = time.perf_counter() - try: - tool = await self._get_playwright_mcp_tool(tool_name) - except RuntimeError: - logger.debug( - "Single-step primitive %s is unavailable; using compact runtime RPC", - tool_name, - exc_info=True, - ) - return None - try: - tool_result = await tool.invoke(tool_args) - success = self.tool_result_succeeded(tool_result) - error = str(getattr(tool_result, "error", "") or "").strip() - payload = getattr(tool_result, "data", None) - if payload is None: - payload = tool_result - except Exception as exc: - success = False - error = str(exc) - payload = None - - elapsed_ms = int(max(0.0, (time.perf_counter() - started_at) * 1000)) - if success: - recorded_payload = payload - if tool_name == "browser_navigate": - target_url = str(tool_args.get("url") or "") - if isinstance(payload, dict): - recorded_payload = dict(payload) - recorded_payload.setdefault("url", target_url) - else: - recorded_payload = {"url": target_url, "result": payload} - self.record_tool_reference_state( - tool_name=tool_name, - tool_args=tool_args, - tool_result=recorded_payload, - ) - - runtime_url = self.extract_result_url(payload) - if tool_name == "browser_navigate" and success: - runtime_url = runtime_url or str(tool_args.get("url") or "") - runtime_title = self._extract_result_title(payload) - step_result = { - "index": 0, - "op": op, - "ok": success, - "status": "completed" if success else "failed", - "elapsed_ms": elapsed_ms, - } - if error: - step_result["error"] = error - conditions = [] - if op in _SINGLE_BATCH_CONDITION_OPS: - condition = dict(step_result) - condition["observed"] = {"text": str(step.get("text") or "")} - conditions.append(condition) - return { - "ok": success, - "status": "completed" if success else "failed", - "error": error or None, - "action": "browser_batch_interact", - "execution_mode": "primitive", - "generation_id": self.generation_id, - "steps": [step_result], - "extracted": {}, - "conditions": conditions, - "metrics": { - "tool_name": tool_name, - "executor_elapsed_ms": elapsed_ms, - "response_size_bytes": len(str(payload).encode("utf-8", "ignore")), - }, - "_runtime_page": {"url": runtime_url, "title": runtime_title}, - } - - async def batch_interact( - self, - *, - steps: Any, - generation_id: str, - timeout_ms: Any = None, - condition_timeout_ms: Any = None, - wait_after_each_ms: Any = None, - continue_on_error: bool = False, - global_timeout_ms: Any = None, - session_id: str = "", - request_id: str = "", - ) -> Dict[str, Any]: - page_state = self._ensure_page_state() - validation_errors = validate_batch_steps(steps) - validation_errors.extend(self._validate_batch_target_contract(steps)) - if validation_errors: - return { - "ok": False, - "status": "failed", - "error": f"batch_validation_failed: {validation_errors[0]}", - "validation_errors": validation_errors, - "page_state": page_state.export(), - } - recovered_generation = "" - try: - refreshed_steps, recovered_generation = await self._refresh_stale_batch_targets( - steps, - generation_id=generation_id, - ) - effective_generation_id = page_state.generation_id - page_state.validate_generation(effective_generation_id) - await self.ensure_runtime_ready() - resolved_steps = await self._resolve_batch_steps( - refreshed_steps, - generation_id=effective_generation_id, - allow_navigation_rewrite=len(refreshed_steps) == 1, - ) - await self._validate_safe_read_locators(resolved_steps) - except ValueError as exc: - return { - "ok": False, - "status": "failed", - "error": f"batch_target_validation_failed: {exc}", - "generation_id": page_state.generation_id, - "current_generation": page_state.generation_id, - "candidate_fresh_targets": self._batch_recovery_candidates(steps), - "page_state": page_state.export(), - } - result = None - if len(resolved_steps) == 1: - result = await self._run_single_batch_primitive(resolved_steps[0]) - if result is None: - self._controller.bind_runtime(self) - if self._code_executor is not None: - self._controller.bind_code_executor(self._code_executor) - result = await self._controller.run_action( - action="browser_batch_interact", - session_id=session_id, - request_id=request_id, - steps=resolved_steps, - timeout_ms=timeout_ms, - condition_timeout_ms=condition_timeout_ms, - wait_after_each_ms=wait_after_each_ms, - continue_on_error=continue_on_error, - global_timeout_ms=global_timeout_ms, - generation_id=effective_generation_id, - ) - if isinstance(result, dict): - runtime_page = result.pop("_runtime_page", {}) - runtime_url = runtime_page.get("url") if isinstance(runtime_page, dict) else "" - runtime_title = runtime_page.get("title") if isinstance(runtime_page, dict) else "" - if result.get("execution_mode") != "primitive": - self._observe_page_url(runtime_url) - self._ensure_page_state().observe( - url=runtime_url, - title=runtime_title, - ) - extracted = result.get("extracted") - if isinstance(extracted, dict): - self._ensure_page_state().add_field_coverage(extracted.keys()) - result["generation_id"] = self.generation_id - if recovered_generation: - result["generation_recovered_from"] = recovered_generation - result["page_state"] = self.export_page_state() - return result - - @staticmethod - def _validate_batch_target_contract(steps: Any) -> list[str]: - """Keep mutating Batch actions on one generation-scoped target contract.""" - errors: list[str] = [] - for index, step in enumerate(steps if isinstance(steps, list) else []): - if not isinstance(step, dict): - continue - op = str(step.get("op") or "").strip().lower() - if op in _BATCH_MUTATING_TARGET_OPS and not str(step.get("target_id") or "").strip(): - errors.append(f"steps[{index}] op={op} requires target_id from the current PageState") - if op not in _BATCH_SAFE_READ_SELECTOR_OPS: - continue - read_target_keys = [key for key in ("target_id", "selector") if str(step.get(key) or "").strip()] - if len(read_target_keys) != 1: - errors.append(f"steps[{index}] op={op} requires exactly one of target_id or selector") - return errors - - async def run_custom_action( - self, - *, - action: str, - session_id: str = "", - request_id: str = "", - params: Optional[Dict[str, Any]] = None, - ) -> Dict[str, Any]: - await self.ensure_runtime_ready() - self._controller.bind_runtime(self) - if self._code_executor is not None: - self._controller.bind_code_executor(self._code_executor) - return await self._controller.run_action( - action=action, - session_id=session_id, - request_id=request_id, - **(params or {}), - ) - - async def _execute_probe_json( - self, - js_code: str, - *, - artifact_kind: str, - ) -> tuple[Dict[str, Any], Any, int]: - """Execute a probe and recover one malformed JSON response internally.""" - - last_raw: Any = "" - for attempt in range(2): - last_raw = await self._code_executor(js_code) - if not self.tool_result_succeeded(last_raw): - return ( - {"ok": False, "error": str(self._unwrap_mcp_text_result(last_raw))[:1_000]}, - write_browser_agent_audit_artifact(artifact_kind, last_raw), - attempt, - ) - last_raw = self._unwrap_mcp_text_result(last_raw) - parsed = extract_json_object(last_raw) - if parsed: - return ( - parsed, - write_browser_agent_audit_artifact(artifact_kind, last_raw), - attempt, - ) - return ( - {}, - write_browser_agent_audit_artifact(artifact_kind, last_raw), - 1, - ) - - async def probe_interactives( - self, - *, - max_items: int = 20, - viewport_only: bool = True, - query: str = "", - ) -> Dict[str, Any]: - """Return compact visible/high-value interactive elements from the current page.""" - await self.ensure_runtime_ready() - - if self._code_executor is None: - return { - "ok": False, - "error": "browser_code_executor_not_ready", - "elements": [], - "page_state": self.export_page_state(), - } - - effective_max_items = min(30, max(1, int(max_items))) - js_code = build_interactive_probe_js( - max_items=effective_max_items, - viewport_only=viewport_only, - query=query, - site_profiles=site_profiles_for_url(self._ensure_page_state().url), - generation_id=self.generation_id, - ) - - try: - parsed, raw_audit, parse_retry_count = await self._execute_probe_json( - js_code, - artifact_kind="interactive_probe", - ) - except Exception as exc: - return { - "ok": False, - "error": f"browser_probe_interactives failed: {exc}", - "elements": [], - "page_state": self.export_page_state(), - } - - if not parsed or parsed.get("ok") is False: - return { - "ok": False, - "error": parsed.get("error") or "Could not parse browser_probe_interactives result JSON", - "audit": raw_audit, - "elements": [], - "page_state": self._ensure_page_state().export_summary(), - } - - parsed.setdefault("ok", True) - parsed.setdefault("error", None) - parsed.setdefault("elements", []) - self._observe_page_url(parsed.get("url")) - self._annotate_probe_generation(parsed) - page_state = self._ensure_page_state() - page_state.register_interactives(parsed) - exported = page_state.export() - return { - "ok": bool(parsed.get("ok")), - "error": parsed.get("error"), - "url": parsed.get("url") or exported.get("url"), - "title": parsed.get("title") or exported.get("title"), - "generation_id": exported["generation_id"], - "count": len(exported["interactives"]), - "elements": exported["interactives"], - "page_state": page_state.export_summary(), - "diagnostics": { - "query": str(query or "")[:160], - "query_widened": bool(parsed.get("query_widened")), - "total_candidates": int(parsed.get("total_candidates") or 0), - "parse_retry_count": parse_retry_count, - }, - "audit": raw_audit, - } - - async def probe_cards( - self, - *, - max_cards: int = 8, - viewport_only: bool = True, - include_buttons: bool = True, - query: str = "", - ) -> Dict[str, Any]: - """Return compact repeated card/listing structures from the current page.""" - await self.ensure_runtime_ready() - - if self._code_executor is None: - return { - "ok": False, - "error": "browser_code_executor_not_ready", - "cards": [], - "page_state": self.export_page_state(), - } - - site_profiles = site_profiles_for_url(self._ensure_page_state().url) - selector_cache = get_selector_cache() - selector_cache_records = selector_cache.export_for_probe() - - effective_max_cards = min(12, max(1, int(max_cards))) - js_code = build_card_probe_js( - max_cards=effective_max_cards, - viewport_only=viewport_only, - include_buttons=include_buttons, - query=query, - site_profiles=site_profiles, - selector_cache_records=selector_cache_records, - generation_id=self.generation_id, - ) - - try: - parsed, raw_audit, parse_retry_count = await self._execute_probe_json( - js_code, - artifact_kind="card_probe", - ) - except Exception as exc: - return { - "ok": False, - "error": f"browser_probe_cards failed: {exc}", - "cards": [], - "page_state": self.export_page_state(), - } - - if not parsed or parsed.get("ok") is False: - return { - "ok": False, - "error": parsed.get("error") or "Could not parse browser_probe_cards result JSON", - "audit": raw_audit, - "cards": [], - "page_state": self._ensure_page_state().export_summary(), - } - - parsed.setdefault("ok", True) - parsed.setdefault("error", None) - parsed.setdefault("cards", []) - self._observe_page_url(parsed.get("url")) - self._annotate_probe_generation(parsed) - normalize_card_probe_payload(parsed) - page_state = self._ensure_page_state() - page_state.register_cards(parsed) - self.register_card_primary_links(parsed) - parsed["page_state"] = page_state.export() - - if parsed.get("ok"): - try: - selector_cache.record_card_probe_cache_rejection(parsed) - except Exception: - logger.debug( - "Failed to record rejected card-probe selector cache attempt", - exc_info=True, - ) - - if parsed.get("ok") and parsed.get("cards"): - try: - selector_cache.record_card_probe_result(parsed) - except Exception: - logger.debug( - "Failed to record card probe result in selector cache", - exc_info=True, - ) - - exported = page_state.export() - return { - "ok": bool(parsed.get("ok")), - "error": parsed.get("error"), - "url": parsed.get("url") or exported.get("url"), - "title": parsed.get("title") or exported.get("title"), - "generation_id": exported["generation_id"], - "count": len(exported["cards"]), - "observed_count": int(parsed.get("observed_count") or 0), - "cards": exported["cards"], - "page_state": page_state.export_summary(), - "diagnostics": { - **(parsed.get("diagnostics") or parsed.get("cache_diagnostics") or {}), - "parse_retry_count": parse_retry_count, - }, - "audit": raw_audit, - } - - async def list_actions(self) -> Dict[str, Any]: - return { - "ok": True, - "actions": self._controller.list_actions(), - "details": self._controller.describe_actions(), - } - - async def runtime_health(self) -> Dict[str, Any]: - return { - "ok": bool(self._service.connection_healthy), - "started": bool(self._service.started), - "last_heartbeat_ok": self._service.last_heartbeat_ok, - "provider": self._service.provider, - "api_base": self._service.api_base, - "model_name": self._service.model_name, - } - - async def shutdown(self) -> None: - try: - await self._service.shutdown() - finally: - _ACTIVE_BROWSER_RUNTIMES.discard(self) - - async def release_task_resources(self) -> None: - """Release task bindings while preserving Chrome and its profile.""" - fully_released = await self._service.release_task_binding() - if fully_released: - self._advance_page_generation() - self._last_observed_url = "" - self._ensure_semantic_state_tracker().reset() - - async def reset(self) -> None: - """Release the current browser and restart lazily on the next task.""" - try: - await self._service.reset() - finally: - self._advance_page_generation() - self._last_observed_url = "" - self._ensure_semantic_state_tracker().reset() - - @property - def semantic_progress(self) -> Dict[str, Any]: - """Return the latest task-local semantic progress observation.""" - return self._ensure_semantic_state_tracker().latest - - def acknowledge_semantic_replan(self) -> None: - """Acknowledge one materially different action selected after replanning.""" - self._ensure_semantic_state_tracker().acknowledge_replan() - - def reset_semantic_task(self) -> None: - """Start semantic loop tracking for a new user task without resetting Chrome.""" - self._ensure_semantic_state_tracker().reset() - - def _ensure_semantic_state_tracker(self) -> SemanticStateTracker: - tracker = getattr(self, "_semantic_state_tracker", None) - if not isinstance(tracker, SemanticStateTracker): - tracker = SemanticStateTracker() - self._semantic_state_tracker = tracker - return tracker - - -async def reset_active_browser_runtimes() -> int: - """Reset every live browser runtime owned by this process.""" - runtimes = list(_ACTIVE_BROWSER_RUNTIMES) - if not runtimes: - return 0 - - results = await asyncio.gather( - *(runtime.reset() for runtime in runtimes), - return_exceptions=True, - ) - reset_count = 0 - for runtime, result in zip(runtimes, results): - if isinstance(result, BaseException): - logger.warning( - "Failed to reset active browser runtime %s: %s", - id(runtime), - result, - ) - continue - reset_count += 1 - return reset_count - - -async def reset_managed_browser_runtime( - *, - browser_key: str = "", - profile_name: str = "jiuwenclaw", - display_mode: str, - browser_binary: str = "", -) -> int: - """Reset only the configured managed browser, including its idle handle.""" - normalized_binary = BrowserService.normalize_browser_binary(browser_binary) - normalized_key = str(browser_key or "").strip() - normalized_profile = str(profile_name or "").strip() - normalized_mode = str(display_mode or "").strip().lower() - - reset_count = 0 - for runtime in list(_ACTIVE_BROWSER_RUNTIMES): - identity = runtime.service.lifecycle_identity - matches_browser = identity.browser_key == normalized_key - matches_profile = identity.profile_name == normalized_profile - matches_mode = identity.display_mode == normalized_mode - matches_binary = identity.browser_binary == normalized_binary - if identity.driver_mode != "managed": - continue - if not matches_browser or not matches_profile: - continue - if not matches_mode or not matches_binary: - continue - try: - await runtime.reset() - except Exception as exc: - logger.warning( - "Failed to reset managed browser runtime %s: %s", - id(runtime), - exc, - ) - else: - reset_count += 1 - - reset_count += await BrowserService.reset_registered_managed_browser( - browser_key=normalized_key, - profile_name=normalized_profile, - display_mode=normalized_mode, - browser_binary=normalized_binary, - ) - return reset_count - - -class BrowserRuntimeRail(AgentRail): - """Rail that makes direct browser sessions resumable and completion-aware.""" - - def __init__(self, runtime: BrowserAgentRuntime) -> None: - super().__init__() - self._runtime = runtime - self._status_logger = BrowserSubagentStatusLogger() if is_browser_subagent_status_log_enabled() else None - browser_agent_log_info( - "[BROWSER_SUBAGENT_BOOT] enabled=%s logger=%s", - self._status_logger is not None, - type(self._status_logger).__name__ if self._status_logger is not None else None, - ) - - def _emit_status(self, method_name: str, ctx: AgentCallbackContext) -> None: - if self._status_logger is None: - return - method = getattr(self._status_logger, method_name, None) - if not callable(method): - browser_agent_log_warning( - "[BROWSER_SUBAGENT_ERROR] missing_status_method=%s", - method_name, - ) - return - try: - method(ctx) - except Exception: - browser_agent_log_warning( - "[BROWSER_SUBAGENT_ERROR] method=%s", - method_name, - exc_info=True, - ) - - async def before_invoke(self, ctx: AgentCallbackContext) -> None: - if isinstance(getattr(ctx, "extra", None), dict): - ctx.extra[_BROWSER_LOG_CONTEXT_TOKEN_KEY] = set_browser_agent_log_context(True) - try: - timeout_s = max(1, int(self._runtime.service.guardrails.timeout_s)) - except (TypeError, ValueError, AttributeError): - timeout_s = 600 - ctx.extra.setdefault(_BROWSER_TASK_DEADLINE_KEY, time.monotonic() + timeout_s) - run_context = self._browser_run_context(ctx) - if run_context.get("browser_resume") is True: - ctx.extra["_browser_resume_requested"] = True - if ctx.steering_queue is None: - ctx.bind_steering_queue(asyncio.Queue()) - self._emit_status("before_invoke", ctx) - await self._runtime.ensure_runtime_ready() - await self._ensure_browser_mcp_ability(ctx) - session = getattr(ctx, "session", None) - if session is None: - return - self._hydrate_service_progress_from_session(session) - query = getattr(getattr(ctx, "inputs", None), "query", None) - task_text = str(query or "").strip() - if task_text: - state = self._ensure_task_state( - session, - task_text, - resume=bool(ctx.extra.get("_browser_resume_requested")), - ) - self._bind_shared_task_deadline(ctx, session, state) - - async def before_model_call(self, ctx: AgentCallbackContext) -> None: - self._emit_status("before_model_call", ctx) - if self._browser_run_context(ctx).get("browser_resume") is True: - ctx.extra["_browser_resume_requested"] = True - session = getattr(ctx, "session", None) - state: Dict[str, Any] = {} - if session is not None: - messages = getattr(getattr(ctx, "inputs", None), "messages", None) or [] - task_text = latest_browser_user_request(messages) - if task_text: - state = self._ensure_task_state( - session, - task_text, - resume=bool(ctx.extra.get("_browser_resume_requested")), - ) - self._bind_shared_task_deadline(ctx, session, state) - self._sync_semantic_progress(session) - loaded_state = session.get_state(_BROWSER_PHASE_STATE_KEY) - state = loaded_state if isinstance(loaded_state, dict) else {} - - if self._finish_if_task_deadline_exhausted(ctx, session, state): - return - - if self._prepare_terminal_synthesis(ctx, session, state): - return - - builder = getattr(ctx.agent, "system_prompt_builder", None) - if builder is None: - return - if str(state.get("status") or "").strip().lower() not in _BROWSER_TERMINAL_STATUSES: - builder.remove_section("browser_terminal_synthesis") - - image_input_supported = self._image_input_supported(ctx.agent) - builder.add_section( - PromptSection( - name=_BROWSER_IMAGE_CAPABILITY_SECTION_NAME, - content=_BROWSER_IMAGE_CAPABILITY_GUIDANCE[image_input_supported], - priority=85, - ) - ) - - async def after_model_call(self, ctx: AgentCallbackContext) -> None: - self._emit_status("after_model_call", ctx) - response = getattr(getattr(ctx, "inputs", None), "response", None) - tool_calls = list(getattr(response, "tool_calls", None) or []) - session = getattr(ctx, "session", None) - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - if ( - tool_calls - and isinstance(state, dict) - and str(state.get("status") or "").strip().lower() in _BROWSER_TERMINAL_STATUSES - ): - ctx.request_force_finish(self._structured_terminal_result(state)) - return - response_text = str(getattr(response, "content", "") or "") - if self._is_unfinished_model_protocol_response(state, tool_calls, response_text): - if self._request_model_protocol_recovery(ctx, session, state): - return - self._finish_model_protocol_failure(ctx, session, state) - return - call_ids = [str(tool_call.id) for tool_call in tool_calls if getattr(tool_call, "id", None)] - extra = getattr(ctx, "extra", None) - if not call_ids or not isinstance(extra, dict): - return - action_group_id = hashlib.sha256("\x1f".join(call_ids).encode("utf-8")).hexdigest()[:16] - group_by_call = extra.setdefault(_BROWSER_ACTION_GROUP_BY_CALL_KEY, {}) - for call_id in call_ids: - group_by_call[call_id] = action_group_id - extra.setdefault(_BROWSER_ACTION_GROUP_RESULTS_KEY, {})[action_group_id] = { - "expected": call_ids, - "completed": [], - "read_only": all( - self._is_read_only_recovery(call.name, call.arguments) for call in tool_calls - ), - } - - @staticmethod - def _has_unfinished_tool_intent(response_text: str) -> bool: - return _BROWSER_UNFINISHED_TOOL_INTENT_RE.search(str(response_text or "")) is not None - - @classmethod - def _is_unfinished_model_protocol_response( - cls, - state: Any, - tool_calls: list[Any], - response_text: str, - ) -> bool: - if tool_calls or not isinstance(state, dict): - return False - status = str(state.get("status") or "in_progress").strip().lower() - if status in _BROWSER_TERMINAL_STATUSES: - return False - return cls._has_unfinished_tool_intent(response_text) - - @staticmethod - def _request_model_protocol_recovery( - ctx: AgentCallbackContext, - session: Any, - state: Dict[str, Any], - ) -> bool: - retry_count = int(state.get("model_protocol_retry_count") or 0) - extra = getattr(ctx, "extra", None) - deadline = float(extra.get(_BROWSER_TASK_DEADLINE_KEY) or 0.0) if isinstance(extra, dict) else 0.0 - deadline_exhausted = bool(deadline and deadline - time.monotonic() <= 5.0) - if retry_count >= _BROWSER_MODEL_PROTOCOL_RETRY_LIMIT or deadline_exhausted: - return False - state["model_protocol_retry_count"] = retry_count + 1 - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - if ctx.steering_queue is None: - ctx.bind_steering_queue(asyncio.Queue()) - ctx.push_steering( - "Your previous response described another browser tool call but did not emit a real tool call. " - "Continue in this same task: emit the actual tool call, or answer concisely using only evidence " - "already collected." - ) - ctx.request_model_continue() - return True - - @classmethod - def _finish_model_protocol_failure( - cls, - ctx: AgentCallbackContext, - session: Any, - state: Dict[str, Any], - ) -> None: - evidence_available = cls._has_task_evidence(state) - state["status"] = "partial" if evidence_available else "blocked" - state["next_action_class"] = "finish" - state["terminal_reason"] = "model_tool_protocol_error" - blockers = list(state.get("blockers") or []) - if "model_tool_protocol_error" not in blockers: - blockers.append("model_tool_protocol_error") - state["blockers"] = blockers[:10] - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - ctx.request_force_finish(cls._structured_terminal_result(state)) - - def _prepare_terminal_synthesis( - self, - ctx: AgentCallbackContext, - session: Any, - state: Dict[str, Any], - ) -> bool: - """Allow one tool-disabled prose pass, then force the runtime result.""" - - status = str(state.get("status") or "").strip().lower() - if status not in _BROWSER_TERMINAL_STATUSES: - return False - if state.get(_BROWSER_TERMINAL_SYNTHESIS_KEY): - ctx.request_force_finish(self._structured_terminal_result(state)) - return True - - state[_BROWSER_TERMINAL_SYNTHESIS_KEY] = True - state["next_action_class"] = "finish" - if session is not None: - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - inputs = getattr(ctx, "inputs", None) - if inputs is not None: - inputs.tools = [] - builder = getattr(ctx.agent, "system_prompt_builder", None) - if builder is not None: - authoritative = self._authoritative_terminal_payload(state) - builder.add_section( - PromptSection( - name="browser_terminal_synthesis", - content={ - "en": ( - "Browser execution has ended. Do not call tools. Summarize the runtime-owned " - f"result without changing status, blockers, missing fields, or evidence: {authoritative}" - ), - "cn": ( - "浏览器执行已结束。不要调用工具。请说明以下由运行时确定的结果，不得修改状态、" - f"阻断项、缺失字段或证据：{authoritative}" - ), - }, - priority=100, - ) - ) - return False - - async def on_model_exception(self, ctx: AgentCallbackContext) -> None: - self._emit_status("on_model_exception", ctx) - exception = getattr(ctx, "exception", None) - if not self._is_retryable_model_exception(exception): - return - extra = getattr(ctx, "extra", None) - deadline = float(extra.get(_BROWSER_TASK_DEADLINE_KEY) or 0.0) if isinstance(extra, dict) else 0.0 - remaining_s = deadline - time.monotonic() if deadline else 0.0 - if ctx.retry_attempt < _BROWSER_MODEL_RETRY_LIMIT and remaining_s > 5.0: - ctx.request_retry(delay_seconds=min(0.5, max(0.0, remaining_s - 5.0))) - return - ctx.request_force_finish(self._structured_model_failure(ctx, exception)) - - @staticmethod - def _is_retryable_model_exception(exception: Optional[Exception]) -> bool: - if isinstance(exception, (asyncio.TimeoutError, TimeoutError, ConnectionError)): - return True - message = str(exception or "").lower() - retryable_tokens = ( - "timeout", - "timed out", - "429", - "500", - "502", - "503", - "504", - "upstream", - "rate limit", - "service unavailable", - ) - return _contains_any_token(message, retryable_tokens) - - def _structured_model_failure( - self, - ctx: AgentCallbackContext, - exception: Optional[Exception], - ) -> Dict[str, Any]: - session = getattr(ctx, "session", None) - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - state = state if isinstance(state, dict) else {} - evidence_available = self._has_task_evidence(state) - status = "partial" if evidence_available else "failed" - if state: - state["status"] = "partial" if evidence_available else "blocked" - state["next_action_class"] = "finish" - state["terminal_reason"] = "model_provider_unavailable" - blockers = list(state.get("blockers") or []) - if "model_provider_unavailable" not in blockers: - blockers.append("model_provider_unavailable") - state["blockers"] = blockers[:10] - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - payload = { - "ok": False, - "status": status, - "error": { - "code": "model_provider_unavailable", - "message": str(exception or "model provider failed")[:500], - "retry_attempts": int(ctx.retry_attempt) + 1, - }, - "progress": { - "task_id": state.get("task_id"), - "status": state.get("status"), - "current_phase": state.get("current_phase"), - "required_fields": list(state.get("required_fields") or [])[:32], - "field_coverage": list(state.get("field_coverage") or [])[:32], - "required_evidence_slots": list(state.get("required_evidence_slots") or [])[:12], - "evidence_slots": list(state.get("evidence_slots") or [])[-12:], - "blockers": list(state.get("blockers") or [])[:10], - } - if state - else {}, - } - return { - "output": json.dumps(payload, ensure_ascii=False, separators=(",", ":")), - "result_type": "error", - "error": "model_provider_unavailable", - "progress_state": self._load_progress_state(session).to_dict() if session is not None else {}, - } - - @classmethod - def _authoritative_terminal_payload(cls, state: Dict[str, Any]) -> Dict[str, Any]: - status = str(state.get("status") or "partial").strip().lower() - missing = cls._missing_completion_requirements(state) - unverified = cls._advisory_missing_fields(state) if status == "completed" else set() - missing = [field for field in missing if field not in unverified] - blockers = [str(item) for item in state.get("blockers") or [] if str(item).strip()] - missing_slots = [slot for slot in cls._missing_evidence_slots(state) if slot["field"] not in unverified] - unavailable_slots = cls._unavailable_evidence_slots(state) - retryable = cls._terminal_result_retryable(state, missing) - deadline_started_at = float(state.get("deadline_started_at") or 0.0) - deadline_at = float(state.get("deadline_at") or 0.0) - now = time.time() - return { - "status": status, - "task_id": state.get("task_id"), - "current_phase": state.get("current_phase"), - "missing_fields": missing[:32], - "unverified_fields": sorted(unverified)[:32], - "missing_slots": missing_slots[:12], - "unavailable_slots": unavailable_slots[:12], - "requested_slots": [ - dict(slot) - for slot in (state.get("required_evidence_slots") or [])[:12] - if isinstance(slot, dict) - ], - "blockers": blockers[:10], - "field_coverage": list(state.get("field_coverage") or [])[:32], - "evidence": list(state.get("evidence_slots") or [])[-12:], - "observations": cls._task_observations(state)[-3:], - "current_page": dict(state.get("last_page") or {}), - "requested_result_count": int(state.get("requested_result_count") or 0), - "observed_result_count": int(state.get("observed_result_count") or 0), - "terminal_reason": state.get("terminal_reason"), - "retryable": retryable, - "recommended_recovery": cls._recommended_recovery(state, missing), - "resume_count": int(state.get("resume_count") or 0), - "deadline": { - "budget_s": float(state.get("deadline_budget_s") or 0.0), - "elapsed_s": round(max(0.0, now - deadline_started_at), 3) if deadline_started_at else 0.0, - "remaining_s": round(max(0.0, deadline_at - now), 3) if deadline_at else 0.0, - "invocation_remaining_s": float(state.get("invocation_remaining_s") or 0.0), - }, - } - - @classmethod - def _terminal_result_retryable( - cls, - state: Dict[str, Any], - missing: Optional[list[str]] = None, - ) -> bool: - status = str(state.get("status") or "").strip().lower() - if status == "completed" or int(state.get("resume_count") or 0) >= _BROWSER_TASK_RESUME_LIMIT: - return False - blockers = " ".join(str(item or "").strip().lower() for item in state.get("blockers") or []) - if any(token in blockers for token in _BROWSER_NON_RETRYABLE_BLOCKER_TOKENS): - return False - if "phase_budget_exhausted" in blockers: - return False - requirements = missing if missing is not None else cls._missing_completion_requirements(state) - unresolved_slots = cls._missing_evidence_slots(state) - if requirements and not unresolved_slots and cls._unavailable_evidence_slots(state): - return False - return bool( - status in {"partial", "blocked"} - and ( - requirements - or cls._has_task_evidence(state) - or state.get("terminal_reason") in { - "model_provider_unavailable", - "model_tool_protocol_error", - "runtime_completion_requirements_missing", - "runtime_blocked", - "semantic_replan_denial_budget_exhausted", - "task_invocation_slice_exhausted", - } - ) - ) - - @staticmethod - def _recommended_recovery(state: Dict[str, Any], missing: list[str]) -> str: - reported = str(state.get("worker_reported_next_action") or "").strip() - if reported: - return reported[:200] - reason = str(state.get("terminal_reason") or "").strip().lower() - if reason in { - "model_provider_unavailable", - "model_tool_protocol_error", - "task_invocation_slice_exhausted", - }: - return "retry_model_step_from_current_page" - if missing and not BrowserRuntimeRail._missing_evidence_slots(state): - if BrowserRuntimeRail._unavailable_evidence_slots(state): - return "finish_with_explicit_unknown_fields" - if missing: - return "collect_missing_evidence_from_current_page" - return "replan_from_current_page" - - @classmethod - def _structured_terminal_result(cls, state: Dict[str, Any]) -> Dict[str, Any]: - payload = cls._authoritative_terminal_payload(state) - status = str(payload.get("status") or "partial") - return { - "output": json.dumps( - {"browser_result": payload}, - ensure_ascii=False, - separators=(",", ":"), - ), - "result_type": "answer" if status == "completed" else "error", - "error": None if status == "completed" else "browser_task_incomplete", - "authoritative_browser_result": payload, - } - - @classmethod - def _render_authoritative_terminal_output( - cls, - state: Dict[str, Any], - model_summary: str, - ) -> tuple[str, Dict[str, Any]]: - payload = cls._authoritative_terminal_payload(state) - if str(model_summary or "").lstrip().startswith("{"): - try: - previous = json.loads(model_summary).get("browser_result") - except (ValueError, AttributeError): - previous = None - if isinstance(previous, dict): - model_summary = str(previous.get("summary") or state.get("last_worker_final") or "") - if str(model_summary or "").strip(): - payload["summary"] = str(model_summary).strip()[:8_000] - return ( - json.dumps({"browser_result": payload}, ensure_ascii=False, separators=(",", ":")), - payload, - ) - - async def before_tool_call(self, ctx: AgentCallbackContext) -> None: - self._emit_status("before_tool_call", ctx) - if self._handle_progress_tool_alias(ctx): - return - session = getattr(ctx, "session", None) - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - state = state if isinstance(state, dict) else {} - self._bind_shared_task_deadline(ctx, session, state) - if self._finish_if_task_deadline_exhausted(ctx, session, state): - return - try: - self._prepare_tool_call(ctx) - except (ValueError, BaseError) as exc: - self._deny_tool_call(ctx, exc) - - def _handle_progress_tool_alias(self, ctx: AgentCallbackContext) -> bool: - inputs = getattr(ctx, "inputs", None) - tool_name = str(getattr(inputs, "tool_name", "") or "").strip().lower() - if tool_name != "browser_progress" and not tool_name.endswith("_browser_progress"): - return False - session = getattr(ctx, "session", None) - if session is None: - self._deny_tool_call(ctx, ValueError("browser_progress requires an active browser task session")) - return True - payload = self._coerce_tool_args(getattr(inputs, "tool_args", None)) - state = session.get_state(_BROWSER_PHASE_STATE_KEY) - state = state if isinstance(state, dict) else {} - reported_status = str(payload.get("status") or "").strip().lower() - if not reported_status: - missing = self._missing_completion_requirements(state) - blockers = [str(item) for item in state.get("blockers") or [] if str(item).strip()] - runtime_ready = bool( - state.get("structured_evidence") - or state.get("evidence_slots") - or state.get("field_coverage") - ) - if not blockers and not missing and runtime_ready: - payload["status"] = "completed" - reported_status = "completed" - elif missing and not self._missing_evidence_slots(state) and self._unavailable_evidence_slots(state): - payload["status"] = "partial" - reported_status = "partial" - else: - self._deny_tool_call( - ctx, - ValueError( - "browser_progress did not include a terminal status and runtime evidence is " - "not yet complete. Continue this browser task and collect only the unresolved " - "requirements before finishing." - ), - ) - return True - if reported_status not in _BROWSER_TERMINAL_STATUSES: - self._deny_tool_call( - ctx, - ValueError( - "<browser_progress> is a final text protocol. Continue with a real browser tool " - "until status is completed, partial, or blocked." - ), - ) - return True - self._apply_worker_progress_to_task_state(session, payload, "") - state = session.get_state(_BROWSER_PHASE_STATE_KEY) - state = state if isinstance(state, dict) else {} - ctx.request_force_finish(self._structured_terminal_result(state)) - return True - - def _prepare_tool_call(self, ctx: AgentCallbackContext) -> None: - inputs = getattr(ctx, "inputs", None) - tool_name = str(getattr(inputs, "tool_name", "") or "") - if tool_name.strip().lower() == "browser_progress": - raise ValueError( - "<browser_progress> is a text protocol appended to the final response, not a callable tool." - ) - canonical_tool_name = self._canonicalize_tool_name(tool_name) - if canonical_tool_name != tool_name: - inputs.tool_name = canonical_tool_name - tool_name = canonical_tool_name - tool_args = getattr(inputs, "tool_args", None) - tool_name, tool_args = self._rewrite_primary_link_click(inputs, tool_name, tool_args) - evidence_fields = self._runtime_evidence_fields(tool_args) - normalized_args = self._normalize_playwright_ref_args(tool_name, tool_args) - if normalized_args is not tool_args: - inputs.tool_args = normalized_args - if "browser_batch_interact" in tool_name.strip().lower() and not self._image_input_supported( - getattr(ctx, "agent", None) - ): - batch_args = self._coerce_tool_args(normalized_args) - steps = batch_args.get("steps") - if any( - isinstance(step, dict) and str(step.get("op") or "").strip().lower() == "screenshot" - for step in (steps if isinstance(steps, list) else []) - ): - raise ValueError( - "Screenshot input is unavailable for this browser agent. " - "Remove the screenshot batch step and use DOM probes or " - "structured extraction." - ) - if "playwright" in tool_name.strip().lower() and "browser_" in tool_name.strip().lower(): - self._runtime.validate_reference_values(self._extract_playwright_ref_values(normalized_args)) - self._validate_model_tool_args(ctx, tool_name, normalized_args) - session = getattr(ctx, "session", None) - self._sync_semantic_progress(session) - group = self._tool_action_group(ctx) - action_class = self._consume_phase_budget( - session, - tool_name, - normalized_args, - current_page_state=self._runtime.export_page_state(), - replan_group_admitted=bool(group.get("read_only") and group.get("trial_admitted")), - ) - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - if isinstance(state, dict) and state.get("replan_trial_pending") and group.get("read_only"): - group["trial_admitted"] = True - extra = getattr(ctx, "extra", None) - if isinstance(extra, dict): - tool_call_id = self._tool_call_id(inputs) - tool_runtime_state = extra.setdefault(_BROWSER_TOOL_RUNTIME_STATE_KEY, {}) - tool_runtime_state[tool_call_id] = { - "started_at": time.perf_counter(), - "action_class": action_class, - "evidence_fields": evidence_fields, - } - - @staticmethod - def _tool_action_group(ctx: AgentCallbackContext) -> Dict[str, Any]: - extra = getattr(ctx, "extra", None) or {} - call_id = BrowserRuntimeRail._tool_call_id(getattr(ctx, "inputs", None)) - group_id = (extra.get(_BROWSER_ACTION_GROUP_BY_CALL_KEY) or {}).get(call_id) - return (extra.get(_BROWSER_ACTION_GROUP_RESULTS_KEY) or {}).get(group_id, {}) - - @classmethod - def _validate_model_tool_args(cls, ctx: AgentCallbackContext, tool_name: str, tool_args: Any) -> None: - """Validate locally before reserving a browser strategy trial.""" - manager = getattr(getattr(ctx, "agent", None), "ability_manager", None) - get_tool = getattr(manager, "get", None) - tool = get_tool(tool_name) if callable(get_tool) else None - if callable(get_tool) and tool is None: - raise build_error(StatusCode.AGENT_TOOL_NOT_FOUND, error_msg=f"Tool {tool_name} is not registered") - schema = getattr(tool, "input_params", None) - if isinstance(schema, dict): - SchemaUtils.validate_with_schema(cls._coerce_tool_args(tool_args), schema) - if tool_name == "browser_batch_interact": - args = cls._coerce_tool_args(tool_args) - errors = validate_batch_steps(args.get("steps")) - if errors: - raise build_error(StatusCode.SCHEMA_VALIDATE_INVALID, reason="; ".join(errors), data=args) - - def _runtime_evidence_fields(self, tool_args: Any) -> list[str]: - args = self._coerce_tool_args(tool_args) - fields: set[str] = set() - for value in (args.get("field"), args.get("fields")): - values = value if isinstance(value, list) else [value] - for item in values: - if isinstance(item, str): - canonical = self._canonical_field_name(item) - if canonical in _BROWSER_FIELD_ALIASES: - fields.add(canonical) - - target_id = str(args.get("target_id") or args.get("target") or "").strip() - if target_id.startswith("t_g"): - try: - target = self._runtime.resolve_model_target_id(target_id) - except ValueError: - target = None - if target is not None and target.field_name in _BROWSER_FIELD_ALIASES: - fields.add(target.field_name) - return sorted(fields) - - def _rewrite_primary_link_click( - self, - inputs: Any, - tool_name: str, - tool_args: Any, - ) -> tuple[str, Any]: - normalized_tool_name = tool_name.strip().lower() - if "playwright" not in normalized_tool_name or not normalized_tool_name.endswith("browser_click"): - return tool_name, tool_args - primary_link = self._runtime.resolve_primary_link(tool_args) - if not isinstance(primary_link, str) or not primary_link: - return tool_name, tool_args - inputs.tool_name = re.sub( - r"browser_click$", - "browser_navigate", - tool_name, - flags=re.IGNORECASE, - ) - inputs.tool_args = {"url": primary_link} - return inputs.tool_name, inputs.tool_args - - async def on_tool_exception(self, ctx: AgentCallbackContext) -> None: - self._emit_status("on_tool_exception", ctx) - - async def after_tool_call(self, ctx: AgentCallbackContext) -> None: - self._emit_status("after_tool_call", ctx) - inputs = getattr(ctx, "inputs", None) - tool_name = str(getattr(inputs, "tool_name", "") or "").strip() - tool_result = self._normalize_tool_result(getattr(inputs, "tool_result", None)) - inputs.tool_result = tool_result - outcome = BrowserAgentRuntime.classify_tool_result(tool_result) - if outcome["denied"] or (isinstance(tool_result, dict) and tool_result.get("executed") is False): - self._set_tool_message_outcome( - inputs, - success=False, - executed=False, - state_changed=False, - denied=True, - ) - return - - session = getattr(ctx, "session", None) - if outcome["success"] and isinstance(tool_result, dict): - self._enrich_probe_result_contract(session, inputs, tool_name, tool_result) - state_changed = self._result_may_have_changed_browser_state(tool_name, tool_result, outcome) - self._set_tool_message_outcome( - inputs, - success=bool(outcome["success"]), - executed=True, - state_changed=state_changed, - denied=False, - ) - if outcome["success"]: - self._runtime.record_tool_reference_state( - tool_name=tool_name, - tool_args=getattr(inputs, "tool_args", None), - tool_result=tool_result, - ) - if outcome["success"] or state_changed: - self._attach_page_state(inputs, tool_name, tool_result) - if session is None: - self._compact_large_observation_message(inputs, tool_name, tool_result) - self._mark_action_group_call_completed(ctx, {}) - return - evidence_args = self._tool_evidence_args(ctx, inputs) - evidence_result = tool_result - if isinstance(tool_result, str): - evidence_result = { - "ok": bool(outcome["success"]), - "error": outcome["error"] or None, - "result": tool_result, - "page_state": self._runtime.export_page_state(), - } - progress_delta = self._record_phase_result( - session, - tool_name, - evidence_args, - evidence_result, - ) - progress_delta.update( - { - "executed": True, - "denied": False, - "state_changed": state_changed, - "ambiguous": bool(not outcome["success"] and state_changed), - } - ) - action_class, elapsed_ms = self._consume_tool_call_timing(ctx, inputs) - recovered = self._record_recent_action( - session, - tool_name=tool_name, - tool_args=getattr(inputs, "tool_args", None), - tool_result=evidence_result, - action_class=str(action_class or ""), - elapsed_ms=elapsed_ms, - progress_delta=progress_delta, - ) - if recovered: - self._runtime.acknowledge_semantic_replan() - self._compact_large_observation_message(inputs, tool_name, tool_result) - if not self._is_browser_progress_tool(tool_name): - self._mark_action_group_call_completed(ctx, progress_delta) - return - self._persist_service_progress_to_session(session) - self._mark_action_group_call_completed(ctx, progress_delta) - - def _tool_evidence_args(self, ctx: AgentCallbackContext, inputs: Any) -> Dict[str, Any]: - args = dict(self._coerce_tool_args(getattr(inputs, "tool_args", None))) - extra = getattr(ctx, "extra", None) - runtime_states = extra.get(_BROWSER_TOOL_RUNTIME_STATE_KEY) if isinstance(extra, dict) else None - call_state = runtime_states.get(self._tool_call_id(inputs)) if isinstance(runtime_states, dict) else None - if isinstance(call_state, dict) and call_state.get("evidence_fields"): - args["_runtime_evidence_fields"] = list(call_state["evidence_fields"]) - return args - - @classmethod - def _enrich_probe_result_contract( - cls, - session: Any, - inputs: Any, - tool_name: str, - tool_result: Dict[str, Any], - ) -> None: - if "probe_cards" not in str(tool_name or "").strip().lower(): - return - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - state = state if isinstance(state, dict) else {} - requested_count = int(state.get("requested_result_count") or 0) - observed_count = int(tool_result.get("observed_count") or 0) - diagnostics = tool_result.get("diagnostics") - diagnostics = dict(diagnostics) if isinstance(diagnostics, dict) else {} - diagnostics.update( - { - "requested_count": requested_count, - "observed_count": observed_count, - } - ) - - current_classes = cls._probe_card_classifications(tool_result.get("cards")) - conflicts = cls._probe_classification_conflicts(state, current_classes) - has_conflict = bool(conflicts or diagnostics.get("classification_conflict")) - if has_conflict: - diagnostics["classification_conflict"] = True - if conflicts: - diagnostics["classification_conflicts"] = conflicts - fallback_count = int(state.get("probe_classification_fallback_count") or 0) - if fallback_count < 1: - diagnostics["recommended_fallback"] = "one_precise_probe" - state["probe_classification_fallback_count"] = fallback_count + 1 - else: - diagnostics["recommended_fallback"] = "retain_unknown_classification" - diagnostics["precise_fallback_remaining"] = max( - 0, - 1 - int(state.get("probe_classification_fallback_count") or 0), - ) - tool_result["requested_count"] = requested_count - tool_result["observed_count"] = observed_count - tool_result["diagnostics"] = diagnostics - page_state = tool_result.get("page_state") - if isinstance(page_state, dict): - page_state["requested_count"] = requested_count - page_state["observed_count"] = observed_count - if has_conflict: - page_state["classification_conflict"] = True - state["last_card_probe"] = { - "url": str(tool_result.get("url") or "")[:500], - "generation_id": str(tool_result.get("generation_id") or ""), - "classifications": current_classes, - } - if session is not None: - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - cls._rewrite_tool_message_contract(inputs, tool_result) - - @staticmethod - def _probe_classification_conflicts( - state: Dict[str, Any], - current_classes: Dict[str, str], - ) -> list[str]: - previous_probe = state.get("last_card_probe") - previous_probe = previous_probe if isinstance(previous_probe, dict) else {} - previous_classes = previous_probe.get("classifications") - previous_classes = previous_classes if isinstance(previous_classes, dict) else {} - return [ - identity - for identity, classification in current_classes.items() - if identity in previous_classes and previous_classes[identity] != classification - ][:5] - - @staticmethod - def _probe_card_classifications(cards: Any) -> Dict[str, str]: - classifications: Dict[str, str] = {} - for card in cards if isinstance(cards, list) else []: - if not isinstance(card, dict): - continue - identity = " ".join( - str(card.get("primary_link") or card.get("href") or card.get("title") or "").split() - ).lower()[:300] - if not identity: - continue - classifications[identity] = ":".join( - ( - str(card.get("region") or ""), - str(card.get("kind") or ""), - str(bool(card.get("is_ad"))).lower(), - ) - ) - return classifications - - @staticmethod - def _rewrite_tool_message_contract(inputs: Any, tool_result: Dict[str, Any]) -> None: - tool_msg = getattr(inputs, "tool_msg", None) - content = getattr(tool_msg, "content", None) - if tool_msg is None or not isinstance(content, str): - return - try: - parsed = json.loads(content) - except (TypeError, ValueError): - return - if not isinstance(parsed, dict): - return - for key in ("requested_count", "observed_count", "diagnostics"): - parsed[key] = tool_result.get(key) - tool_msg.content = json.dumps(parsed, ensure_ascii=False, separators=(",", ":")) - - def _consume_tool_call_timing(self, ctx: AgentCallbackContext, inputs: Any) -> tuple[str, int]: - extra = getattr(ctx, "extra", None) - tool_runtime_state = extra.get(_BROWSER_TOOL_RUNTIME_STATE_KEY, {}) if isinstance(extra, dict) else {} - call_state = ( - tool_runtime_state.pop(self._tool_call_id(inputs), {}) - if isinstance(tool_runtime_state, dict) - else {} - ) - if isinstance(extra, dict) and not tool_runtime_state: - extra.pop(_BROWSER_TOOL_RUNTIME_STATE_KEY, None) - started_at = call_state.get("started_at") if isinstance(call_state, dict) else None - action_class = str(call_state.get("action_class") or "") if isinstance(call_state, dict) else "" - elapsed_ms = int(max(0.0, (time.perf_counter() - started_at) * 1000)) if started_at else 0 - return action_class, elapsed_ms - - @staticmethod - def _set_tool_message_outcome( - inputs: Any, - *, - success: bool, - executed: bool, - state_changed: bool, - denied: bool, - ) -> None: - tool_msg = getattr(inputs, "tool_msg", None) - if tool_msg is None: - return - metadata = dict(tool_msg.metadata) if isinstance(tool_msg.metadata, dict) else {} - metadata.update( - { - "success": success, - "executed": executed, - "state_changed": state_changed, - "denied": denied, - } - ) - tool_msg.metadata = metadata - - @staticmethod - def _compact_large_observation_message(inputs: Any, tool_name: str, tool_result: Any) -> None: - """Bound one model-visible raw observation after runtime evidence extraction.""" - - tool_msg = getattr(inputs, "tool_msg", None) - content = getattr(tool_msg, "content", None) - if tool_msg is None or not isinstance(content, str): - return - if not _contains_any_token( - str(tool_name or "").strip().lower(), - _BROWSER_BOUNDED_OBSERVATION_TOOL_TOKENS, - ): - return - if len(content) <= _BROWSER_OBSERVATION_MESSAGE_MAX_CHARS: - return - - audit = write_browser_agent_audit_artifact("large_browser_observation", tool_result) - payload = { - "ok": BrowserAgentRuntime.tool_result_succeeded(tool_result), - "observation": "bounded_browser_tool_result", - "tool": str(tool_name or "").rsplit("_", 1)[-1], - "truncated": True, - "original_chars": len(content), - "preview_head": content[:6_000], - "preview_tail": content[-2_000:], - "note": ( - "Runtime evidence and targets were recorded before compaction. " - "Use browser_working_context and browser_state for subsequent actions." - ), - } - if audit: - payload["audit"] = audit - rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) - if len(rendered) > _BROWSER_OBSERVATION_MESSAGE_MAX_CHARS: - payload["preview_head"] = content[:4_000] - payload["preview_tail"] = content[-1_000:] - rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) - if len(rendered) > _BROWSER_OBSERVATION_MESSAGE_MAX_CHARS: - payload["preview_head"] = content[:2_000] - payload.pop("preview_tail", None) - rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) - if len(rendered) > _BROWSER_OBSERVATION_MESSAGE_MAX_CHARS: - payload.pop("preview_head", None) - rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) - tool_msg.content = rendered - - @classmethod - def _result_may_have_changed_browser_state( - cls, - tool_name: str, - tool_result: Any, - outcome: Dict[str, Any], - ) -> bool: - if not cls._tool_may_change_browser_state(tool_name): - return False - if outcome.get("success"): - return True - if outcome.get("timed_out"): - return True - if not isinstance(tool_result, dict): - return False - if str(tool_result.get("status") or "").strip().lower() != "partial": - return False - for step in tool_result.get("steps") or []: - if not isinstance(step, dict) or step.get("ok") is not True: - continue - if str(step.get("op") or "").strip().lower() in _BATCH_MUTATING_TARGET_OPS: - return True - return False - - @staticmethod - def _tool_call_id(inputs: Any) -> str: - tool_call = getattr(inputs, "tool_call", None) - call_id = str(getattr(tool_call, "id", "") or "").strip() - return call_id or f"tool-context-{id(inputs)}" - - @classmethod - def _denial_code(cls, message: str) -> str: - normalized = str(message or "").lower() - if "already" in normalized and "browser task" in normalized: - return "browser_task_terminal" - if "semantic" in normalized or "replan" in normalized: - return "browser_replan_required" - if "budget exhausted" in normalized: - return "browser_phase_budget_exhausted" - if "browser_progress" in normalized: - return "browser_progress_is_text_protocol" - if "screenshot input" in normalized: - return "browser_image_input_unavailable" - if "known url" in normalized: - return "browser_direct_navigation_required" - return "browser_action_denied" - - def _deny_tool_call(self, ctx: AgentCallbackContext, exc: Exception) -> None: - inputs = getattr(ctx, "inputs", None) - session = getattr(ctx, "session", None) - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - state = state if isinstance(state, dict) else {} - status = str(state.get("status") or "in_progress").strip().lower() - payload = { - "ok": False, - "status": "denied", - "executed": False, - "state_changed": False, - "denied": True, - "error": { - "code": self._denial_code(str(exc)), - "message": str(exc), - "terminal": status in _BROWSER_TERMINAL_STATUSES, - "replan_required": bool(state.get("replan_required")), - }, - "runtime_state": { - "task_status": status, - "current_phase": state.get("current_phase"), - "next_action_class": state.get("next_action_class"), - "blockers": list(state.get("blockers") or [])[:10], - }, - } - target_match = re.search(r"\bt_g\d+_\d+\b", str(exc)) - if target_match is not None: - payload["runtime_state"].update( - self._runtime.target_recovery_state(target_match.group(0)) - ) - call_id = self._tool_call_id(inputs) - inputs.tool_result = payload - inputs.tool_msg = ToolMessage( - content=json.dumps(payload, ensure_ascii=False, separators=(",", ":")), - tool_call_id=call_id, - metadata={ - "executed": False, - "state_changed": False, - "denied": True, - }, - ) - extra = getattr(ctx, "extra", None) - if isinstance(extra, dict): - extra.setdefault(_BROWSER_SKIP_TOOL_CALLS_KEY, {})[call_id] = True - self._mark_action_group_call_completed( - ctx, - { - "executed": False, - "state_changed": False, - "denied": True, - "new_evidence_fields": [], - }, - ) - - @staticmethod - def _tool_may_change_browser_state(tool_name: str) -> bool: - normalized = str(tool_name or "").strip().lower() - observation_tokens = ( - "browser_probe_cards", - "browser_probe_interactives", - "browser_snapshot", - "browser_find", - ) - return not _contains_any_token(normalized, observation_tokens) - - def _canonicalize_tool_name(self, tool_name: str) -> str: - canonical = canonicalize_playwright_tool_name(tool_name) - normalized = canonical.strip().lower() - for prefix in ("mcp_playwright-official_", "playwright-official_"): - local_name = normalized.removeprefix(prefix) - if local_name != normalized and local_name in _BROWSER_RUNTIME_TOOL_NAMES: - return local_name - if not normalized.startswith("browser_") or normalized in _BROWSER_RUNTIME_TOOL_NAMES: - return canonical - configured = set(self._runtime.service.allowed_tool_names or CORE_BROWSER_TOOL_NAMES) - if normalized not in configured: - return canonical - server_name = str(getattr(self._runtime.service.mcp_cfg, "server_name", "") or "").strip() - if not server_name: - return canonical - return f"mcp_{server_name}_{normalized}" - - def _mark_action_group_call_completed( - self, - ctx: AgentCallbackContext, - progress_delta: Dict[str, Any], - ) -> None: - extra = getattr(ctx, "extra", None) - if not isinstance(extra, dict): - return - call_id = self._tool_call_id(getattr(ctx, "inputs", None)) - group_by_call = extra.get(_BROWSER_ACTION_GROUP_BY_CALL_KEY) - group_id = group_by_call.pop(call_id, "") if isinstance(group_by_call, dict) else "" - groups = extra.get(_BROWSER_ACTION_GROUP_RESULTS_KEY) - group = groups.get(group_id) if isinstance(groups, dict) and group_id else None - if isinstance(group, dict): - completed = group.setdefault("completed", []) - if call_id not in completed: - completed.append(call_id) - if progress_delta.get("executed") is False: - denied = group.setdefault("denied", []) - if call_id not in denied: - denied.append(call_id) - elif progress_delta.get("executed") is True: - executed = group.setdefault("executed", []) - if call_id not in executed: - executed.append(call_id) - if progress_delta.get("state_changed") is True: - group["state_changed"] = True - evidence_fields = group.setdefault("evidence_fields", []) - for field_name in progress_delta.get("new_evidence_fields") or []: - if field_name not in evidence_fields: - evidence_fields.append(field_name) - if set(completed) >= set(group.get("expected") or []): - group["ready"] = True - session = getattr(ctx, "session", None) - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - if isinstance(state, dict): - state["last_action_group"] = { - "action_group_id": group_id, - "call_count": len(group.get("expected") or []), - "executed_count": len(group.get("executed") or []), - "denied_count": len(group.get("denied") or []), - "state_changed": bool(group.get("state_changed")), - "evidence_fields": evidence_fields[:32], - } - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - groups.pop(group_id, None) - if not groups: - extra.pop(_BROWSER_ACTION_GROUP_RESULTS_KEY, None) - if isinstance(group_by_call, dict) and not group_by_call: - extra.pop(_BROWSER_ACTION_GROUP_BY_CALL_KEY, None) - - def _attach_page_state( - self, - inputs: Any, - tool_name: str, - tool_result: Any, - ) -> None: - normalized_name = str(tool_name or "").strip().lower() - page_state_tokens = ( - "browser_navigate", - "browser_tabs", - "browser_snapshot", - "browser_find", - "browser_probe_", - "browser_batch_interact", - ) - if not _contains_any_token(normalized_name, page_state_tokens): - return - - page_state = self._runtime.export_page_state() - already_embedded = isinstance(tool_result, dict) and isinstance( - tool_result.get("page_state"), - dict, - ) - if isinstance(tool_result, dict) and not already_embedded: - tool_result["page_state"] = page_state - - if already_embedded: - return - tool_msg = getattr(inputs, "tool_msg", None) - content = getattr(tool_msg, "content", None) - if tool_msg is None or not isinstance(content, str): - return - if _contains_any_token(normalized_name, ("browser_snapshot", "browser_find")): - compact_observation = { - "ok": True, - "observation": "compact_page_state", - "page_state": page_state, - "audit": write_browser_agent_audit_artifact("direct_ax_observation", tool_result), - } - tool_msg.content = json.dumps( - compact_observation, - ensure_ascii=False, - separators=(",", ":"), - ) - return - marker = ( - f"\n<{_BROWSER_PAGE_STATE_TAG}>" - f"{json.dumps(page_state, ensure_ascii=False, separators=(',', ':'))}" - f"</{_BROWSER_PAGE_STATE_TAG}>" - ) - if f"<{_BROWSER_PAGE_STATE_TAG}>" not in content: - tool_msg.content = content.rstrip() + marker - - async def after_invoke(self, ctx: AgentCallbackContext) -> None: - try: - session = getattr(ctx, "session", None) - result = getattr(getattr(ctx, "inputs", None), "result", None) - if session is None or not isinstance(result, dict): - return - - session_id = session.get_session_id() - self._hydrate_service_progress_from_session(session) - output_text = str(result.get("output") or "") - clean_output, progress_payload = self._extract_progress_payload(output_text) - if clean_output != output_text: - result["output"] = clean_output - - if progress_payload is not None: - self._apply_worker_progress_to_task_state(session, progress_payload, clean_output) - if self._finalize_terminal_invoke(session, result, clean_output): - return - progress_state = self._load_progress_state(session) - parsed_progress = self._build_progress_result(progress_state, clean_output) - self._runtime.service.set_progress_state(session_id, progress_state) - exported = progress_state.to_dict() - if self._runtime.service.should_treat_as_completed(parsed_progress) and clean_output.strip(): - result["result_type"] = "answer" - result["progress_state"] = exported - self._clear_progress_state(session) - return - - failure_summary = self._runtime.service.build_failure_summary( - task=self._load_task_text(session), - error=str(parsed_progress.get("error") or "browser_task_incomplete"), - page_url=progress_state.last_page_url if progress_state is not None else "", - page_title=progress_state.last_page_title if progress_state is not None else "", - final=clean_output, - screenshot=progress_state.last_screenshot if progress_state is not None else None, - attempt=1, - progress_state=progress_state, - ) - result["result_type"] = "error" - result["failure_summary"] = failure_summary - result["progress_state"] = exported - result["output"] = failure_summary if not clean_output else f"{clean_output}\n\n{failure_summary}" - self._persist_service_progress_to_session(session) - return - - if self._is_max_iteration_result(result): - if self._terminalize_invoke_failure(session, "max_iterations_reached"): - if self._finalize_terminal_invoke(session, result, clean_output): - return - progress_state = self._load_progress_state(session) - self._runtime.service.set_progress_state(session_id, progress_state) - failure_summary = self._runtime.service.build_failure_summary( - task=self._load_task_text(session), - error="max_iterations_reached", - page_url=progress_state.last_page_url if progress_state is not None else "", - page_title=progress_state.last_page_title if progress_state is not None else "", - final=clean_output or output_text, - screenshot=progress_state.last_screenshot if progress_state is not None else None, - attempt=1, - progress_state=progress_state, - ) - result["failure_summary"] = failure_summary - result["progress_state"] = progress_state.to_dict() - result["output"] = failure_summary - self._persist_service_progress_to_session(session) - return - - if str(result.get("result_type", "")).lower() == "answer": - task_state = session.get_state(_BROWSER_PHASE_STATE_KEY) - if not isinstance(task_state, dict) or not task_state: - self._clear_progress_state(session) - return - self._apply_worker_progress_to_task_state( - session, - {"status": "completed"}, - clean_output, - ) - if self._finalize_terminal_invoke(session, result, clean_output): - return - progress_state = self._load_progress_state(session) - exported = progress_state.to_dict() - parsed_progress = self._build_progress_result(progress_state, clean_output) - if self._runtime.service.should_treat_as_completed(parsed_progress) and clean_output.strip(): - result["progress_state"] = exported - self._clear_progress_state(session) - return - failure_summary = self._runtime.service.build_failure_summary( - task=self._load_task_text(session), - error="empty_or_unverified_browser_result", - page_url=progress_state.last_page_url, - page_title=progress_state.last_page_title, - final=clean_output, - screenshot=progress_state.last_screenshot, - attempt=1, - progress_state=progress_state, - ) - result["result_type"] = "error" - result["failure_summary"] = failure_summary - result["progress_state"] = exported - result["output"] = failure_summary if not clean_output else f"{clean_output}\n\n{failure_summary}" - self._persist_service_progress_to_session(session) - return - - progress_state = self._load_progress_state(session) - if str(result.get("result_type") or "").lower() == "error": - self._terminalize_invoke_failure(session, "browser_execution_error") - if self._finalize_terminal_invoke(session, result, clean_output): - return - exported = progress_state.to_dict() if not progress_state.is_empty() else None - if exported: - self._runtime.service.set_progress_state(session_id, progress_state) - result["progress_state"] = exported - self._persist_service_progress_to_session(session) - - finally: - self._emit_status("after_invoke", ctx) - extra = getattr(ctx, "extra", None) - if isinstance(extra, dict): - token = extra.pop(_BROWSER_LOG_CONTEXT_TOKEN_KEY, None) - if token is not None: - reset_browser_agent_log_context(token) - - def _finalize_terminal_invoke( - self, - session: Any, - result: Dict[str, Any], - model_summary: str, - ) -> bool: - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - if not isinstance(state, dict): - return False - status = str(state.get("status") or "").strip().lower() - if status not in _BROWSER_TERMINAL_STATUSES: - return False - output, payload = self._render_authoritative_terminal_output(state, model_summary) - progress_state = self._load_progress_state(session) - result["output"] = output - result["result_type"] = "answer" if status == "completed" else "error" - result["error"] = None if status == "completed" else "browser_task_incomplete" - result["progress_state"] = progress_state.to_dict() - result["authoritative_browser_result"] = payload - if status == "completed": - self._clear_progress_state(session) - else: - result["failure_summary"] = self._runtime.service.build_failure_summary( - task=self._load_task_text(session), - error="browser_task_incomplete", - page_url=progress_state.last_page_url, - page_title=progress_state.last_page_title, - final="", - screenshot=progress_state.last_screenshot, - attempt=1, - progress_state=progress_state, - ) - self._persist_service_progress_to_session(session) - return True - - @staticmethod - def _normalize_tool_result(tool_result: Any) -> Any: - outcome = BrowserAgentRuntime.classify_tool_result(tool_result) - data = getattr(tool_result, "data", None) - normalized = data if data is not None else tool_result - if outcome["success"]: - return normalized - if isinstance(normalized, dict): - result = dict(normalized) - else: - result = {"raw_result": str(normalized or "")[:2_000]} - result["ok"] = False - result.setdefault("status", "denied" if outcome["denied"] else "failed") - if not result.get("error"): - result["error"] = outcome["error"] or "browser tool failed" - if outcome["denied"]: - result.setdefault("denied", True) - result.setdefault("executed", False) - result.setdefault("state_changed", False) - if outcome["timed_out"]: - result["timed_out"] = True - return result - - def _normalize_playwright_ref_args(self, tool_name: str, tool_args: Any) -> Any: - """Rewrite aliases and resolve PageState targets for direct MCP tools.""" - normalized_name = str(tool_name or "").strip().lower() - if "playwright" not in normalized_name or "browser_" not in normalized_name: - return tool_args - - original_is_json = isinstance(tool_args, str) - if original_is_json: - try: - parsed = json.loads(tool_args) - except ValueError: - return tool_args - elif isinstance(tool_args, dict): - parsed = dict(tool_args) - else: - return tool_args - if not isinstance(parsed, dict): - return tool_args - - parsed, changed = self._normalize_playwright_target_payload(parsed) - if normalized_name.endswith("browser_snapshot") and "element" in parsed: - parsed.pop("element") - changed = True - if normalized_name.endswith("browser_evaluate"): - if "field" in parsed: - parsed.pop("field", None) - changed = True - fields = parsed.get("fields") - if isinstance(fields, list) and all(isinstance(field_name, str) for field_name in fields): - parsed.pop("fields", None) - changed = True - if not changed: - normalized_args = tool_args - elif original_is_json: - normalized_args = json.dumps(parsed, ensure_ascii=False) - else: - normalized_args = parsed - return normalized_args - - def _normalize_playwright_target_payload( - self, - tool_args: Dict[str, Any], - ) -> tuple[Dict[str, Any], bool]: - parsed = dict(tool_args) - changed = self._normalize_playwright_target_aliases(parsed) - for key in _BROWSER_REF_TARGET_KEYS: - value = str(parsed.get(key) or "").strip() - if not value.startswith("t_g"): - continue - target = self._runtime.resolve_model_target_id(value) - executable_target = str(target.ref or target.selector or "").strip() - if not executable_target: - raise ValueError(f"PageState target_id {target.target_id} has no executable locator.") - parsed[key] = executable_target - element_key = { - "target": "element", - "startTarget": "startElement", - "endTarget": "endElement", - }.get(key) - if element_key and not str(parsed.get(element_key) or "").strip(): - parsed[element_key] = str(target.text or target.kind or target.target_id)[:200] - changed = True - - fields = parsed.get("fields") - if isinstance(fields, list): - normalized_fields = [] - for field in fields: - if not isinstance(field, dict): - normalized_fields.append(field) - continue - nested, nested_changed = self._normalize_playwright_target_payload(field) - normalized_fields.append(nested) - changed = changed or nested_changed - parsed["fields"] = normalized_fields - return parsed, changed - - @staticmethod - def _normalize_playwright_target_aliases(parsed: Dict[str, Any]) -> bool: - changed = False - - if "target_id" in parsed: - parsed["target"] = parsed.pop("target_id") - changed = True - if "generation_id" in parsed: - parsed.pop("generation_id", None) - changed = True - - for alias, target_key in _BROWSER_REF_ALIAS_KEYS.items(): - if alias not in parsed: - continue - if target_key not in parsed: - parsed[target_key] = parsed[alias] - parsed.pop(alias, None) - changed = True - - for key in _BROWSER_REF_TARGET_KEYS: - value = parsed.get(key) - if not isinstance(value, str): - continue - match = _BROWSER_EXPLICIT_REF_RE.fullmatch(value) - if match is None: - continue - parsed[key] = match.group(1) - changed = True - return changed - - @staticmethod - def _extract_playwright_ref_values(tool_args: Any) -> tuple[str, ...]: - if isinstance(tool_args, str): - try: - parsed = json.loads(tool_args) - except ValueError: - return () - else: - parsed = tool_args - if not isinstance(parsed, dict): - return () - values: list[str] = [] - for key in _BROWSER_REF_TARGET_KEYS: - value = parsed.get(key) - if not isinstance(value, str): - continue - normalized = value.strip() - if normalized: - values.append(normalized) - return tuple(values) - - @staticmethod - def _is_browser_progress_tool(tool_name: str) -> bool: - name = (tool_name or "").strip().lower() - if not name: - return False - if name in { - "browser_cancel_run", - "browser_clear_cancel", - "browser_list_custom_actions", - "browser_runtime_health", - }: - return False - return name.startswith("browser_") or ".browser_" in name - - @staticmethod - def _extract_progress_payload(output_text: str) -> tuple[str, Optional[Dict[str, Any]]]: - text = str(output_text or "") - match = _BROWSER_PROGRESS_TAG_RE.search(text) - if match is None: - return text, None - payload_text = match.group(1).strip() - try: - payload = json.loads(payload_text) - except json.JSONDecodeError: - return text, None - cleaned = _BROWSER_PROGRESS_TAG_RE.sub("", text, count=1).strip() - return cleaned, payload if isinstance(payload, dict) else None - - @staticmethod - def _build_progress_result( - progress_state: BrowserTaskProgressState, - clean_output: str, - ) -> Dict[str, Any]: - """Build completion output from runtime-owned progress only.""" - - status = str(progress_state.status or "").strip().lower() or "partial" - progress = progress_state.to_dict() - return { - "ok": status == "completed", - "status": status, - "progress": progress, - "final": clean_output, - "error": None if status == "completed" else "browser_task_incomplete", - } - - @classmethod - def _apply_worker_progress_to_task_state( - cls, - session: Any, - payload: Dict[str, Any], - final: str, - ) -> None: - """Validate a worker completion request against authoritative runtime state.""" - - state = session.get_state(_BROWSER_PHASE_STATE_KEY) - if not isinstance(state, dict): - return - reported_status = str(payload.get("status") or "partial").strip().lower() - state["worker_reported_status"] = reported_status - reported_blockers = payload.get("blockers") or payload.get("missing_requirements") or [] - if isinstance(reported_blockers, str): - reported_blockers = [reported_blockers] - state["worker_reported_blockers"] = [str(item)[:300] for item in reported_blockers if str(item).strip()][:10] - next_action = payload.get("next_action") or payload.get("next_step") - if next_action: - state["worker_reported_next_action"] = str(next_action)[:200] - if final: - state["last_worker_final"] = str(final)[:8_000] - - current_status = str(state.get("status") or "in_progress").strip().lower() - if current_status in _BROWSER_TERMINAL_STATUSES: - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - return - - runtime_blockers = [str(item) for item in state.get("blockers") or [] if str(item).strip()] - advisory_fields = cls._advisory_missing_fields(state) if final.strip() else set() - missing_fields = [ - field for field in cls._missing_completion_requirements(state) if field not in advisory_fields - ] - unavailable_slots = cls._unavailable_evidence_slots(state) - phases = state.get("phases") if isinstance(state.get("phases"), dict) else {} - completed_phase_count = sum( - 1 for details in phases.values() if isinstance(details, dict) and details.get("status") == "completed" - ) - evidence_available = cls._has_task_evidence(state) - runtime_ready = bool(completed_phase_count or evidence_available) - if state.get("replan_required"): - runtime_blockers = list(dict.fromkeys([*runtime_blockers, "semantic_replan_required"])) - - completion_requirements_met = ( - bool(final.strip() or evidence_available) - and not runtime_blockers - and not missing_fields - and not unavailable_slots - and runtime_ready - ) - if reported_status == "completed" and completion_requirements_met: - state["status"] = "completed" - state["next_action_class"] = "finish" - state["terminal_reason"] = "runtime_completion_validated" - elif reported_status in _BROWSER_TERMINAL_STATUSES: - state["status"] = "blocked" if runtime_blockers else "partial" - state["next_action_class"] = "finish" - state["terminal_reason"] = ( - "runtime_blocked" if runtime_blockers else "runtime_completion_requirements_missing" - ) - if runtime_blockers: - state["blockers"] = runtime_blockers[:10] - elif missing_fields: - state["blockers"] = [f"missing_required_field:{field}" for field in missing_fields[:10]] - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - - @staticmethod - def _is_max_iteration_result(result: Dict[str, Any]) -> bool: - output = str(result.get("output") or "").strip() - result_type = str(result.get("result_type") or "").strip().lower() - return result_type == "error" and MAX_ITERATION_MESSAGE.lower() in output.lower() - - @staticmethod - def _load_task_text(session: Any) -> str: - if session is None: - return "" - return str(session.get_state(_BROWSER_PROGRESS_TASK_KEY) or "").strip() - - @staticmethod - def _load_progress_state(session: Any) -> BrowserTaskProgressState: - if session is None: - return BrowserTaskProgressState() - task_state = session.get_state(_BROWSER_PHASE_STATE_KEY) - if isinstance(task_state, dict) and task_state: - return BrowserTaskProgressState.from_task_state(task_state) - return BrowserTaskProgressState.from_dict(session.get_state(_BROWSER_PROGRESS_STATE_KEY)) - - def _hydrate_service_progress_from_session(self, session: Any) -> BrowserTaskProgressState: - session_id = session.get_session_id() - progress_state = self._load_progress_state(session) - if progress_state.is_empty(): - self._runtime.service.clear_progress_state(session_id) - return progress_state - self._runtime.service.set_progress_state(session_id, progress_state) - return progress_state - - def _persist_service_progress_to_session(self, session: Any) -> None: - if session is None: - return - session_id = session.get_session_id() - progress_state = self._load_progress_state(session) - if progress_state.is_empty(): - self._runtime.service.clear_progress_state(session_id) - exported: Dict[str, Any] = {} - else: - self._runtime.service.set_progress_state(session_id, progress_state) - exported = progress_state.to_dict() - session.update_state( - { - _BROWSER_PROGRESS_STATE_KEY: exported, - } - ) - - def _ensure_task_state(self, session: Any, task: str, *, resume: bool = False) -> Dict[str, Any]: - normalized_task = str(task or "").strip() - state = session.get_state(_BROWSER_PHASE_STATE_KEY) - if isinstance(state, dict) and str(state.get("task") or "").strip() == normalized_task: - self._runtime.set_task_requested_fields(state.get("required_fields") or []) - return state - if isinstance(state, dict) and resume: - resumed = self._resume_task_state(session, state, normalized_task) - self._runtime.set_task_requested_fields(resumed.get("required_fields") or []) - return resumed - self._runtime.reset_semantic_task() - self._runtime.service.clear_progress_state(session.get_session_id()) - state = self._build_phase_state(normalized_task) - self._runtime.set_task_requested_fields(state.get("required_fields") or []) - session.update_state( - { - _BROWSER_PROGRESS_TASK_KEY: normalized_task, - _BROWSER_PROGRESS_STATE_KEY: {}, - _BROWSER_PHASE_STATE_KEY: state, - } - ) - return state - - @staticmethod - def _browser_run_context(ctx: AgentCallbackContext) -> Dict[str, Any]: - context = getattr(getattr(ctx, "inputs", None), "run_context", None) - if context is None: - context = (getattr(ctx, "extra", None) or {}).get("run_context") - if isinstance(context, dict): - return context.get("extra", context) - extra = getattr(context, "extra", None) - return extra if isinstance(extra, dict) else {} - - @staticmethod - def _bind_shared_task_deadline( - ctx: AgentCallbackContext, - session: Any, - state: Dict[str, Any], - ) -> float: - if session is None or not isinstance(state, dict) or not state: - return 0.0 - extra = getattr(ctx, "extra", None) - shared_context = BrowserRuntimeRail._browser_run_context(ctx) - context_budget_s = float(shared_context.get("browser_query_budget_s") or 0.0) - budget_s = float( - state.get("deadline_budget_s") - or context_budget_s - or ( - _BROWSER_COMPLEX_TASK_DEADLINE_S - if str(state.get("task_type") or "") == "complex" - else _BROWSER_SIMPLE_TASK_DEADLINE_S - ) - ) - now = time.time() - context_started_at = float(shared_context.get("browser_query_started_at") or 0.0) - context_deadline_at = float(shared_context.get("browser_query_deadline_at") or 0.0) - started_at = float(state.get("deadline_started_at") or context_started_at or now) - deadline_at = float(state.get("deadline_at") or context_deadline_at or started_at + budget_s) - if context_deadline_at: - deadline_at = min(deadline_at, context_deadline_at) - query_id = str(shared_context.get("browser_query_id") or state.get("query_id") or "").strip() - if query_id: - state["query_id"] = query_id[:128] - state["deadline_budget_s"] = budget_s - state["deadline_started_at"] = started_at - state["deadline_at"] = deadline_at - remaining_s = max(0.0, deadline_at - now) - state["deadline_remaining_s"] = round(remaining_s, 3) - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - if isinstance(extra, dict): - invocation_deadline = float(extra.get(_BROWSER_INVOCATION_DEADLINE_KEY) or 0.0) - if not invocation_deadline: - invocation_deadline = time.monotonic() + remaining_s - extra[_BROWSER_INVOCATION_DEADLINE_KEY] = invocation_deadline - effective_remaining_s = min( - remaining_s, - max(0.0, invocation_deadline - time.monotonic()), - ) - extra[_BROWSER_TASK_DEADLINE_KEY] = time.monotonic() + effective_remaining_s - state["invocation_remaining_s"] = round(effective_remaining_s, 3) - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - return effective_remaining_s - return remaining_s - - @classmethod - def _finish_if_task_deadline_exhausted( - cls, - ctx: AgentCallbackContext, - session: Any, - state: Dict[str, Any], - ) -> bool: - if session is None or not isinstance(state, dict) or not state: - return False - deadline_at = float(state.get("deadline_at") or 0.0) - shared_exhausted = bool(deadline_at and time.time() >= deadline_at) - extra = getattr(ctx, "extra", None) - invocation_deadline = ( - float(extra.get(_BROWSER_INVOCATION_DEADLINE_KEY) or 0.0) - if isinstance(extra, dict) - else 0.0 - ) - invocation_exhausted = bool(invocation_deadline and time.monotonic() >= invocation_deadline) - if not shared_exhausted and not invocation_exhausted: - return False - status = str(state.get("status") or "in_progress").strip().lower() - if status not in _BROWSER_TERMINAL_STATUSES: - missing = cls._missing_completion_requirements(state) - evidence_available = cls._has_task_evidence(state) - if evidence_available and not missing and not state.get("blockers"): - state["status"] = "completed" - state["terminal_reason"] = "task_deadline_completed_from_evidence" - else: - state["status"] = "partial" if evidence_available else "blocked" - blockers = list(state.get("blockers") or []) - blocker = "task_deadline_exhausted" if shared_exhausted else "task_invocation_slice_exhausted" - if blocker not in blockers: - blockers.append(blocker) - state["blockers"] = blockers[:10] - state["terminal_reason"] = blocker - state["next_action_class"] = "finish" - state["deadline_remaining_s"] = ( - 0.0 if shared_exhausted else round(max(0.0, deadline_at - time.time()), 3) - ) - state["invocation_remaining_s"] = 0.0 - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - ctx.request_force_finish(cls._structured_terminal_result(state)) - return True - - @staticmethod - def _terminalize_invoke_failure(session: Any, reason: str) -> bool: - state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None - if not isinstance(state, dict) or not state: - return False - status = str(state.get("status") or "in_progress").strip().lower() - if status not in _BROWSER_TERMINAL_STATUSES: - evidence_available = BrowserRuntimeRail._has_task_evidence(state) - state["status"] = "partial" if evidence_available else "blocked" - state["next_action_class"] = "finish" - state["terminal_reason"] = reason - blockers = list(state.get("blockers") or []) - if reason not in blockers: - blockers.append(reason) - state["blockers"] = blockers[:10] - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - return True - - def _resume_task_state( - self, - session: Any, - state: Dict[str, Any], - resume_instruction: str, - ) -> Dict[str, Any]: - if str(state.get("resume_instruction") or "").strip() == resume_instruction: - return state - missing = self._missing_completion_requirements(state) - if not self._terminal_result_retryable(state, missing): - return state - - state["resume_count"] = int(state.get("resume_count") or 0) + 1 - state["resume_instruction"] = resume_instruction[:2_000] - state["status"] = "in_progress" - state["next_action_class"] = ( - "collect_missing_evidence" if missing else "materially_different_strategy" - ) - state["terminal_reason"] = "" - state.pop(_BROWSER_TERMINAL_SYNTHESIS_KEY, None) - state["replan_required"] = False - state["replan_trial_pending"] = False - state["replan_count"] = 0 - state["replan_denial_count"] = 0 - state["blocked_strategy"] = "" - state["trial_strategy"] = "" - state["model_protocol_retry_count"] = 0 - recoverable_prefixes = ( - "semantic_", - "missing_required_field:", - "missing_evidence_slot:", - "model_provider_unavailable", - "model_tool_protocol_error", - "task_invocation_slice_exhausted", - ) - state["blockers"] = [ - blocker - for blocker in state.get("blockers") or [] - if not str(blocker).strip().lower().startswith(recoverable_prefixes) - ][:10] - known_urls = list(state.get("known_urls") or []) - for url in re.findall(r"https?://[^\s<>\"]+", resume_instruction): - if url not in known_urls: - known_urls.append(url) - state["known_urls"] = known_urls[:4] - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - self._runtime.reset_semantic_task() - return state - - @staticmethod - def _build_phase_state(task: str) -> Dict[str, Any]: - normalized = str(task or "").lower() - complex_tokens = ( - "form", - "filter", - "sort", - "compare", - "book", - "checkout", - "register", - "login", - "purchase", - "choose", - "select", - "fill", - "submit", - "switch", - "cart", - "basket", - "add to cart", - "\u8868\u5355", - "\u7b5b\u9009", - "\u6392\u5e8f", - "\u6bd4\u8f83", - "\u5bf9\u6bd4", - "\u9009\u62e9", - "\u586b\u5199", - "\u63d0\u4ea4", - "\u5207\u6362", - "\u8bbe\u7f6e", - "\u65e5\u671f", - "\u9884\u8ba2", - "\u767b\u5f55", - "\u6ce8\u518c", - "\u7ed3\u8d26", - "\u8d2d\u4e70", - "\u8d2d\u7269\u8f66", - "\u52a0\u8d2d", - "\u52a0\u5165\u8d2d\u7269\u8f66", - ) - task_type = "complex" if any(token in normalized for token in complex_tokens) else "simple" - phase_names = tuple(_BROWSER_PHASE_DEFINITIONS) if task_type == "complex" else ("navigation", "extraction") - phases = { - name: { - "status": "pending", - "attempts": 0, - "budget": _BROWSER_PHASE_DEFINITIONS[name]["budget"], - "completion_condition": _BROWSER_PHASE_DEFINITIONS[name]["completion_condition"], - "blocked_signature": "", - "visited_price_intervals": [], - } - for name in phase_names - } - required_slots = BrowserRuntimeRail._infer_required_evidence_slots(task) - required_fields = list( - dict.fromkeys( - str(slot.get("field") or "") - for slot in required_slots - if str(slot.get("field") or "").strip() - ) - ) - return { - "task_id": hashlib.sha256(str(task).encode("utf-8")).hexdigest()[:16], - "task": task, - "goal": task, - "requirements_source": "inferred", - "task_type": task_type, - "status": "in_progress", - "phases": phases, - "current_phase": phase_names[0], - "known_urls": re.findall(r"https?://[^\s<>\"]+", task)[:4], - "constraints": [], - "required_fields": required_fields, - "required_evidence_slots": required_slots, - "requested_result_count": BrowserRuntimeRail._infer_requested_result_count(task), - "observed_result_count": 0, - "replan_count": 0, - "replan_denial_count": 0, - "resume_count": 0, - "direct_navigation_guidance_count": 0, - "read_only_recovery_counts": {}, - "model_protocol_retry_count": 0, - "replan_required": False, - "replan_trial_pending": False, - "blocked_strategy": "", - "failed_strategies": [], - "trial_strategy": "", - "last_strategy_fingerprint": "", - "next_action_class": "navigation", - "semantic_revision": 0, - "structured_evidence": [], - "evidence_slots": [], - "field_coverage": [], - "blockers": [], - "recent_actions": [], - "action_sequence": 0, - "last_page": {}, - } - - @classmethod - def _infer_required_fields(cls, task: str) -> list[str]: - normalized = str(task or "").lower() - request_scopes = _BROWSER_OUTPUT_REQUEST_CUE_RE.findall(normalized) - output_scope = " ".join(request_scopes).strip() or normalized - inferred = [ - field - for field, aliases in _BROWSER_FIELD_ALIASES.items() - if any(cls._contains_field_alias(output_scope, alias) for alias in (field, *aliases)) - ] - if "product_rating" in inferred or "shop_rating" in inferred: - inferred = [field_name for field_name in inferred if field_name != "rating"] - if "product_rating" in inferred and not any( - cls._contains_field_alias(normalized, alias) for alias in ("title", "name", "标题", "名称", "电影", "结果") - ): - inferred = [field_name for field_name in inferred if field_name != "title"] - if "shop_rating" in inferred: - without_shop_rating = re.sub( - r"shop\s+rating|seller\s+rating|store\s+rating|店铺评分|卖家评分", - "", - normalized, - flags=re.IGNORECASE, - ) - if not any( - cls._contains_field_alias(without_shop_rating, alias) for alias in _BROWSER_FIELD_ALIASES["shop"] - ): - inferred = [field_name for field_name in inferred if field_name != "shop"] - return inferred - - @classmethod - def _infer_required_evidence_slots(cls, task: str) -> list[Dict[str, str]]: - """Create one provenance-bearing evidence slot per requested field.""" - normalized = str(task or "").lower() - required_fields = cls._infer_required_fields(task) - entity = cls._infer_evidence_entity(normalized) - comprehensive_requested = any( - token in normalized for token in ("comprehensive", "relevance", "综合", "默认排序") - ) - latest_requested = any(token in normalized for token in ("latest", "newest", "最新", "最近发布")) - split_title_variants = bool( - "title" in required_fields and comprehensive_requested and latest_requested - ) - slots = [ - {"entity": entity, "variant": "default", "field": field_name} - for field_name in required_fields - if not (field_name == "title" and split_title_variants) - ] - if split_title_variants: - slots.extend( - [ - {"entity": entity, "variant": "comprehensive", "field": "title"}, - {"entity": entity, "variant": "latest", "field": "title"}, - ] - ) - return slots - - @staticmethod - def _infer_evidence_entity(normalized_task: str) -> str: - profile_entity = infer_profile_evidence_entity(normalized_task) - if profile_entity: - return profile_entity - entities = ( - (("商品",), "product"), - (("回答",), "article_or_answer"), - (("article", "文章"), "article"), - (("电影",), "movie"), - (("weather", "天气", "气温"), "weather"), - (("hotel", "酒店"), "hotel"), - (("video", "视频"), "video"), - ) - for aliases, entity in entities: - if any(alias in normalized_task for alias in aliases): - return entity - return "task_result" - - @staticmethod - def _contains_field_alias(text: str, alias: str) -> bool: - normalized_alias = str(alias or "").strip().lower() - if not normalized_alias: - return False - if any("\u4e00" <= character <= "\u9fff" for character in normalized_alias): - return normalized_alias in text - return re.search(rf"(?<![a-z0-9_]){re.escape(normalized_alias)}(?![a-z0-9_])", text) is not None - - @staticmethod - def _infer_requested_result_count(task: str) -> int: - normalized = str(task or "").lower() - patterns = ( - r"(?:top|first)\s*(\d{1,2})(?:\s*(?:results?|items?|articles?|links?))?", - r"(?:前|最前)\s*(\d{1,2})\s*(?:条|个|篇|项|部|名|则)?", - r"(\d{1,2})\s*(?:条|个|篇|项|部)\s*(?:结果|记录|文章|商品|视频|链接)", - ) - for pattern in patterns: - match = re.search(pattern, normalized, re.IGNORECASE) - if match is not None: - return min(20, max(1, int(match.group(1)))) - return 0 - - @staticmethod - def _snake_case_field_name(value: Any) -> str: - text = str(value or "").strip() - text = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", text) - text = re.sub(r"[^a-zA-Z0-9\u4e00-\u9fff]+", "_", text) - return text.strip("_").lower() - - @classmethod - def _canonical_field_name(cls, value: Any) -> str: - snake_name = cls._snake_case_field_name(value) - normalized = " ".join(snake_name.replace("_", " ").split()) - if not normalized: - return "" - for field_name, aliases in _BROWSER_FIELD_ALIASES.items(): - candidates = (field_name.replace("_", " "), *aliases) - if any(normalized == candidate.lower() for candidate in candidates): - return field_name - return str(value or "").strip()[:80] - - @classmethod - def _canonical_evaluate_field_name(cls, value: Any) -> str: - snake_name = cls._snake_case_field_name(value) - if not snake_name: - return "" - explicit = _BROWSER_EVALUATE_FIELD_ALIASES.get(snake_name) - if explicit: - return explicit - canonical = cls._canonical_field_name(snake_name) - if canonical != snake_name: - return canonical - for suffix in ("_raw", "_text", "_value"): - if not snake_name.endswith(suffix): - continue - base_name = snake_name[: -len(suffix)] - canonical_base = cls._canonical_field_name(base_name) - if canonical_base in _BROWSER_FIELD_ALIASES: - return canonical_base - return snake_name[:80] - - @classmethod - def _field_mentioned(cls, field_name: str, text: str) -> bool: - aliases = _BROWSER_FIELD_ALIASES.get(str(field_name), (str(field_name),)) - return any(cls._contains_field_alias(text, alias) for alias in aliases) - - def _sync_semantic_progress(self, session: Any) -> None: - if session is None: - return - progress = self._runtime.semantic_progress - if BrowserWorkingContextStore.sync_semantic_progress(session, progress): - self._runtime.acknowledge_semantic_replan() - - @staticmethod - def _coerce_tool_args(tool_args: Any) -> Dict[str, Any]: - if isinstance(tool_args, dict): - return tool_args - if isinstance(tool_args, str): - try: - parsed = json.loads(tool_args) - except ValueError: - return {} - return parsed if isinstance(parsed, dict) else {} - return {} - - @classmethod - def _classify_tool_phase( - cls, - tool_name: str, - tool_args: Any, - state: Dict[str, Any], - ) -> str: - name = str(tool_name or "").strip().lower() - args = cls._coerce_tool_args(tool_args) - serialized = json.dumps(args, ensure_ascii=False).lower() - if any(token in name for token in ("navigate", "navigate_back", "tabs")): - return "navigation" - filter_terms = ( - "filter", - "sort", - "price", - "rating", - "category", - "\u7b5b\u9009", - "\u6392\u5e8f", - "\u4ef7\u683c", - "\u8bc4\u5206", - "\u5206\u7c7b", - ) - filter_terms_present = _contains_any_token(serialized, filter_terms) - script_tool = any(token in name for token in ("evaluate", "run_code")) - filter_intent = bool( - filter_terms_present - and ( - not script_tool - or cls._operation_intent(name, args) == "script_mutation" - ) - ) - if "browser_batch_interact" in name: - steps = args.get("steps") - operations = {str(step.get("op") or "").lower() for step in steps or [] if isinstance(step, dict)} - extraction_operations = {"extract_text", "extract_value"} - mutating_operations = _BATCH_MUTATING_TARGET_OPS | {"press", "navigate", "navigate_back"} - if operations & extraction_operations and not operations & mutating_operations: - return "extraction" - if filter_intent: - return "filtering" - if operations & { - "fill", - "type", - "autocomplete", - "select_option", - "select_visible_text", - "set_checked", - }: - return "form" - if operations & {"extract_text", "extract_value"}: - return "extraction" - if filter_intent: - return "filtering" - if any(token in name for token in ("fill", "type", "select", "press", "file_upload")): - return "form" - extraction_tokens = ( - "probe", - "snapshot", - "evaluate", - "run_code", - "console", - "network_request", - ) - if _contains_any_token(name, extraction_tokens): - return "extraction" - current = str(state.get("current_phase") or "navigation") - return current if current in _BROWSER_PHASE_DEFINITIONS else "navigation" - - @classmethod - def _phase_action_signature( - cls, - tool_name: str, - tool_args: Any, - ) -> str: - args = cls._coerce_tool_args(tool_args) - semantic_keys = ( - "url", - "href", - "name", - "text", - "query", - "value", - "values", - "text_value", - "label_value", - "option_value", - "option_label", - "option_text", - "choose_text", - "checked", - "key", - "field", - "action", - "element", - ) - actions: list[Dict[str, Any]] = [] - steps = args.get("steps") - if isinstance(steps, list): - for step in steps: - if isinstance(step, dict): - compact = _copy_non_none_values(step, semantic_keys) - compact["op"] = str(step.get("op") or "").strip().lower() - actions.append(compact) - else: - compact = _copy_non_none_values(args, semantic_keys) - normalized_name = str(tool_name or "").strip().lower() - operation = normalized_name.rsplit("browser_", 1)[-1] - compact["op"] = operation - actions.append(compact) - return json.dumps( - actions, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - )[:600] - - @classmethod - def _strategy_fingerprint( - cls, - state: Dict[str, Any], - tool_name: str, - tool_args: Any, - action_class: str, - current_page_state: Optional[Dict[str, Any]] = None, - ) -> str: - """Identify a material strategy without selector or generation noise.""" - - args = cls._coerce_tool_args(tool_args) - fields = cls._requested_strategy_fields(state, args) - intent = cls._operation_intent(tool_name, args) - descriptor = { - "action_class": action_class, - "page": cls._normalize_navigation_url(cls._current_page_url(state, current_page_state)), - "target": cls._strategy_target(args, current_page_state), - "fields": fields, - "intent": intent, - } - serialized = json.dumps( - descriptor, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ) - field_label = ",".join(fields[:4]) or "none" - return (f"{action_class}:{intent}:{field_label}:{hashlib.sha256(serialized.encode('utf-8')).hexdigest()[:16]}")[ - :240 - ] - - @staticmethod - def _current_page_url( - state: Dict[str, Any], - current_page_state: Optional[Dict[str, Any]], - ) -> str: - page_state = current_page_state if isinstance(current_page_state, dict) else {} - last_page = state.get("last_page") if isinstance(state.get("last_page"), dict) else {} - return str(page_state.get("url") or last_page.get("url") or "") - - @classmethod - def _strategy_target( - cls, - args: Dict[str, Any], - current_page_state: Optional[Dict[str, Any]] = None, - ) -> str: - url = str(args.get("url") or args.get("href") or "").strip() - if url: - return cls._normalize_navigation_url(url) - target_id = str(args.get("target_id") or "").strip() - if target_id: - return cls._target_semantic_identity(target_id, current_page_state) - for key in ("name", "label", "placeholder", "text", "element"): - value = " ".join(str(args.get(key) or "").split()).lower() - if value: - return f"semantic:{value[:120]}" - steps = args.get("steps") - if isinstance(steps, list): - labels = [] - for step in steps[:12]: - if not isinstance(step, dict): - continue - step_target_id = str(step.get("target_id") or "").strip() - if step_target_id: - labels.append(cls._target_semantic_identity(step_target_id, current_page_state)) - continue - label = next( - ( - " ".join(str(step.get(key) or "").split()).lower() - for key in ("name", "label", "placeholder", "text", "element") - if str(step.get(key) or "").strip() - ), - "", - ) - labels.append(label[:80] or "page_target") - if labels: - return "batch:" + "|".join(labels) - return ( - "page_target" if _has_non_empty_mapping_value(args, ("target_id", "target", "ref", "selector")) else "page" - ) - - @staticmethod - def _target_semantic_identity( - target_id: str, - current_page_state: Optional[Dict[str, Any]], - ) -> str: - page_state = current_page_state if isinstance(current_page_state, dict) else {} - - def find_target(value: Any) -> Optional[Dict[str, Any]]: - if isinstance(value, dict): - if str(value.get("target_id") or "") == target_id: - return value - for nested in value.values(): - match = find_target(nested) - if match is not None: - return match - elif isinstance(value, list): - for nested in value: - match = find_target(nested) - if match is not None: - return match - return None - - target = find_target(page_state) - if target is None: - return "registered_target" - identity = { - key: " ".join(str(target.get(key) or "").split()).lower()[:160] - for key in ("href", "field", "role", "kind", "region", "text") - if str(target.get(key) or "").strip() - } - return json.dumps(identity, ensure_ascii=False, sort_keys=True, separators=(",", ":"))[:400] - - @classmethod - def _requested_strategy_fields(cls, state: Dict[str, Any], args: Dict[str, Any]) -> list[str]: - fields: set[str] = set() - for value in (args.get("field"), args.get("fields")): - values = value if isinstance(value, list) else [value] - for item in values: - canonical = cls._canonical_field_name(item) - if canonical: - fields.add(canonical) - steps = args.get("steps") - if isinstance(steps, list): - for step in steps: - if not isinstance(step, dict): - continue - canonical = cls._canonical_field_name(step.get("field")) - if canonical: - fields.add(canonical) - serialized = json.dumps(args, ensure_ascii=False, default=str).lower() - for field_name in state.get("required_fields") or []: - if cls._field_mentioned(str(field_name), serialized): - fields.add(str(field_name)) - return sorted(fields) - - @staticmethod - def _operation_intent(tool_name: str, args: Dict[str, Any]) -> str: - steps = args.get("steps") - if isinstance(steps, list): - operations = sorted( - { - str(step.get("op") or "").strip().lower() - for step in steps - if isinstance(step, dict) and str(step.get("op") or "").strip() - } - ) - return "+".join(operations)[:120] or "batch" - name = str(tool_name or "").strip().lower().rsplit("browser_", 1)[-1] - if name in {"evaluate", "run_code", "run_code_unsafe"}: - expression = str( - args.get("function") or args.get("expression") or args.get("script") or args.get("code") or "" - ).lower() - if re.search(r"\.click\s*\(|dispatchEvent|\.value\s*=|setAttribute\s*\(", expression): - return "script_mutation" - if re.search(r"textContent|innerText|return|querySelector|document\.title", expression): - return "script_extraction" - return "script_inspection" - return name or "other" - - @staticmethod - def _price_interval_signature(tool_name: str, tool_args: Any) -> str: - """Delegate cross-site filter normalization to semantic-state logic.""" - return price_interval_signature(tool_name, tool_args) - - @classmethod - def _classify_action_class(cls, tool_name: str, tool_args: Any, state: Dict[str, Any]) -> str: - name = str(tool_name or "").strip().lower() - args = cls._coerce_tool_args(tool_args) - serialized = json.dumps(args, ensure_ascii=False).lower() - if any(token in name for token in ("navigate", "navigate_back", "tabs")): - return "navigation" - if any(token in name for token in ("mouse_wheel", "scroll")): - return "viewport_exploration" - if any(token in name for token in ("evaluate", "run_code")): - return "script_exploration" - if "probe_cards" in name: - return "structured_extraction" - if any(token in name for token in ("probe_interactives", "snapshot", "find")): - return "target_discovery" - phase = cls._classify_tool_phase(tool_name, tool_args, state) - if phase == "extraction": - return "structured_extraction" - if phase in {"form", "filtering"}: - return phase - if any(token in name for token in ("click", "hover", "press", "drag", "drop")): - return "interaction" - if any(token in serialized for token in ("filter", "sort", "price", "rating", "筛选", "排序", "价格")): - return "filtering" - return "other" - - @classmethod - def _consume_phase_budget( - cls, - session: Any, - tool_name: str, - tool_args: Any, - *, - current_page_state: Optional[Dict[str, Any]] = None, - replan_group_admitted: bool = False, - ) -> str: - if session is None: - return "other" - state = session.get_state(_BROWSER_PHASE_STATE_KEY) - if not isinstance(state, dict): - return "other" - cls._reject_terminal_state(state) - if cls._is_replan_exempt_tool(tool_name): - return cls._classify_action_class(tool_name, tool_args, state) - phases = state.setdefault("phases", {}) - phase = cls._classify_tool_phase(tool_name, tool_args, state) - details = cls._phase_details(phases, phase) - signature = cls._phase_action_signature(tool_name, tool_args) - action_class = cls._classify_action_class(tool_name, tool_args, state) - strategy_fingerprint = cls._strategy_fingerprint( - state, - tool_name, - tool_args, - action_class, - current_page_state, - ) - - attempts = int(details.get("attempts") or 0) - budget = max(1, int(details.get("budget") or 1)) - if attempts >= budget: - details["status"] = "replan_required" - details["blocked_signature"] = signature - state["current_phase"] = phase - state["status"] = "partial" - state["blockers"] = [f"{phase}_phase_budget_exhausted"] - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - raise ValueError( - f"Browser {phase} phase budget exhausted ({attempts}/{budget}). " - "Finish with available evidence or return partial/blocked." - ) - - cls._reject_revisited_price_interval(details, phase, tool_name, tool_args) - - if cls._direct_navigation_is_required(state, phases, phase, current_page_state): - state["direct_navigation_guidance_count"] = int(state.get("direct_navigation_guidance_count") or 0) + 1 - recommended_url = str((state.get("known_urls") or [""])[0]) - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - raise ValueError( - "The task already contains a known URL. Navigate to it directly before selector exploration: " - f"{recommended_url}" - ) - - try: - if not replan_group_admitted: - cls._consume_replan_gate( - state, - tool_name, - tool_args, - action_class, - strategy_fingerprint, - ) - except ValueError: - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - raise - - details["attempts"] = attempts + 1 - details["status"] = "in_progress" - details["last_signature"] = signature - details["last_semantic_signature"] = signature - state["current_phase"] = phase - state["last_action_class"] = action_class - state["last_strategy_fingerprint"] = strategy_fingerprint - state["next_action_class"] = "" - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - return action_class - - @classmethod - def _direct_navigation_is_required( - cls, - state: Dict[str, Any], - phases: Dict[str, Any], - phase: str, - current_page_state: Optional[Dict[str, Any]], - ) -> bool: - if ( - not state.get("known_urls") - or phase == "navigation" - or int(state.get("direct_navigation_guidance_count") or 0) >= 1 - ): - return False - total_attempts = sum(int(item.get("attempts") or 0) for item in phases.values() if isinstance(item, dict)) - return total_attempts == 0 and not cls._known_url_navigation_satisfied( - state, - current_page_state, - ) - - @classmethod - def _known_url_navigation_satisfied( - cls, - state: Dict[str, Any], - current_page_state: Optional[Dict[str, Any]], - ) -> bool: - page_state = current_page_state if isinstance(current_page_state, dict) else {} - last_page = state.get("last_page") if isinstance(state.get("last_page"), dict) else {} - current_url = str(page_state.get("url") or last_page.get("url") or "").strip() - if not current_url: - return False - current_normalized = cls._normalize_navigation_url(current_url) - current_document = cls._normalize_navigation_url(current_url, include_query=False) - try: - current_parts = urlsplit(current_url) - except ValueError: - current_parts = None - for known_url in state.get("known_urls") or []: - known_normalized = cls._normalize_navigation_url(known_url) - if known_normalized and known_normalized == current_normalized: - return True - known_document = cls._normalize_navigation_url(known_url, include_query=False) - if known_document and known_document == current_document: - return True - try: - known_parts = urlsplit(str(known_url)) - except ValueError: - known_parts = None - if current_parts is None or known_parts is None: - continue - if current_parts.netloc.lower() != known_parts.netloc.lower(): - continue - known_path = (known_parts.path or "/").rstrip("/") or "/" - current_path = (current_parts.path or "/").rstrip("/") or "/" - if known_path == "/" and (current_path != "/" or bool(current_parts.query)): - return True - if known_path != "/" and current_path.startswith(f"{known_path}/"): - return True - return False - - @staticmethod - def _normalize_navigation_url(value: Any, *, include_query: bool = True) -> str: - raw = str(value or "").strip().rstrip(".,;，。；)") - if not raw: - return "" - try: - parsed = urlsplit(raw) - except ValueError: - return raw.lower() - if not parsed.netloc: - return raw.lower().rstrip("/") - path = unquote(parsed.path or "/").rstrip("/") or "/" - query = "" - if include_query: - query_items = [ - (key, item) - for key, item in parse_qsl(parsed.query, keep_blank_values=True) - if not key.lower().startswith("utm_") - ] - query = urlencode(sorted(query_items)) - normalized = f"{parsed.netloc.lower()}{path}" - return f"{normalized}?{query}" if query else normalized - - @staticmethod - def _reject_terminal_state(state: Dict[str, Any]) -> None: - status = str(state.get("status") or "in_progress").strip().lower() - if status in _BROWSER_TERMINAL_STATUSES: - raise ValueError( - f"Browser task is already {status}; do not issue another browser action. " - "Finish with the runtime evidence and blockers." - ) - - @staticmethod - def _phase_details(phases: Dict[str, Any], phase: str) -> Dict[str, Any]: - return phases.setdefault( - phase, - { - "status": "pending", - "attempts": 0, - "budget": _BROWSER_PHASE_DEFINITIONS[phase]["budget"], - "completion_condition": _BROWSER_PHASE_DEFINITIONS[phase]["completion_condition"], - "blocked_signature": "", - "visited_price_intervals": [], - }, - ) - - @classmethod - def _consume_replan_gate( - cls, - state: Dict[str, Any], - tool_name: str, - tool_args: Any, - action_class: str, - strategy_fingerprint: str = "", - ) -> None: - cls._reject_terminal_state(state) - if not state.get("replan_required"): - return - if cls._is_replan_exempt_tool(tool_name): - return - if state.get("replan_trial_pending"): - cls._consume_replan_denial(state, "replan_trial_pending") - raise ValueError( - "A replan trial already ran without verified semantic progress. " - "Wait for the runtime observation, or finish partial/blocked instead of trying " - "another selector, generation, tool, or phase." - ) - replan_count = int(state.get("replan_count") or 0) - if replan_count >= 2: - state["status"] = "partial" if cls._has_task_evidence(state) else "blocked" - state["blockers"] = ["semantic_replan_budget_exhausted"] - state["terminal_reason"] = "semantic_replan_budget_exhausted" - state["next_action_class"] = "finish" - raise ValueError( - "Semantic progress remained blocked after two replan trials. " - "Return blocked or partial with the available structured evidence." - ) - blocked_strategy = str( - state.get("blocked_strategy") - or state.get("last_strategy_fingerprint") - or state.get("last_action_class") - or "" - ) - failed_strategies = {str(item) for item in state.get("failed_strategies") or [] if str(item).strip()} - if blocked_strategy: - failed_strategies.add(blocked_strategy) - current_strategy = strategy_fingerprint or action_class - if current_strategy in failed_strategies or action_class in failed_strategies: - if cls._allow_read_only_recovery(state, tool_name, tool_args, current_strategy): - state["replan_count"] = replan_count + 1 - state["replan_trial_pending"] = True - state["trial_strategy"] = current_strategy - state["status"] = "replan_trial" - return - cls._consume_replan_denial(state, "repeated_strategy") - raise ValueError( - "Semantic loop detected. Changing selector, generation, tool name, or phase does not " - f"change the {action_class} target/field/intent strategy. Re-plan materially or finish " - "partial/blocked." - ) - state["replan_count"] = replan_count + 1 - state["replan_trial_pending"] = True - state["trial_strategy"] = current_strategy - state["status"] = "replan_trial" - - @staticmethod - def _is_replan_exempt_tool(tool_name: str) -> bool: - normalized_name = str(tool_name or "").strip().lower() - exempt_tokens = ( - "browser_recall_offload", - "browser_runtime_health", - "browser_list_custom_actions", - ) - return _contains_any_token(normalized_name, exempt_tokens) - - @classmethod - def _allow_read_only_recovery( - cls, - state: Dict[str, Any], - tool_name: str, - tool_args: Any, - strategy_fingerprint: str, - ) -> bool: - if not cls._is_read_only_recovery(tool_name, tool_args): - return False - counts = state.setdefault("read_only_recovery_counts", {}) - count = int(counts.get(strategy_fingerprint) or 0) - if count >= _BROWSER_READ_ONLY_RECOVERY_LIMIT: - return False - counts[strategy_fingerprint] = count + 1 - return True - - @classmethod - def _is_read_only_recovery(cls, tool_name: str, tool_args: Any) -> bool: - normalized_name = str(tool_name or "").strip().lower() - args = cls._coerce_tool_args(tool_args) - if any(token in normalized_name for token in ("probe", "find", "snapshot", "screenshot")): - return True - if "browser_tabs" in normalized_name: - return str(args.get("action") or "list").strip().lower() in {"list", "select"} - if any(token in normalized_name for token in ("evaluate", "run_code")): - return cls._operation_intent(normalized_name, args) != "script_mutation" - if "browser_batch_interact" not in normalized_name: - return False - steps = args.get("steps") - if not isinstance(steps, list) or not steps: - return False - read_only_ops = _BATCH_SAFE_READ_SELECTOR_OPS | _BATCH_EXPLICIT_SELECTOR_OPS | { - "wait_for_text", - "wait_for_load_state", - "wait_for_url", - "wait_for_tab", - } - return all( - isinstance(step, dict) and str(step.get("op") or "").strip().lower() in read_only_ops - for step in steps - ) - - @staticmethod - def _consume_replan_denial(state: Dict[str, Any], reason: str) -> None: - denial_count = int(state.get("replan_denial_count") or 0) + 1 - state["replan_denial_count"] = denial_count - state["last_denial_reason"] = str(reason or "")[:120] - if denial_count < _BROWSER_REPLAN_DENIAL_LIMIT: - return - blockers = list(state.get("blockers") or []) - if "semantic_replan_denial_budget_exhausted" not in blockers: - blockers.append("semantic_replan_denial_budget_exhausted") - state["blockers"] = blockers[:10] - state["status"] = "blocked" - state["next_action_class"] = "finish" - state["terminal_reason"] = "semantic_replan_denial_budget_exhausted" - - @classmethod - def _reject_revisited_price_interval( - cls, - details: Dict[str, Any], - phase: str, - tool_name: str, - tool_args: Any, - ) -> None: - price_interval = cls._price_interval_signature(tool_name, tool_args) - visited_intervals = details.setdefault("visited_price_intervals", []) - if phase == "filtering" and price_interval and price_interval in visited_intervals: - raise ValueError( - f"Price interval {price_interval} was already visited in this task. " - "Use the existing evidence or choose a different interval." - ) - - @classmethod - def _record_phase_result( - cls, - session: Any, - tool_name: str, - tool_args: Any, - tool_result: Any, - ) -> Dict[str, Any]: - state = session.get_state(_BROWSER_PHASE_STATE_KEY) - if not isinstance(state, dict): - return {} - if str(state.get("status") or "").strip().lower() in _BROWSER_TERMINAL_STATUSES: - return { - "phase": state.get("current_phase") or "unknown", - "success": BrowserAgentRuntime.tool_result_succeeded(tool_result), - "new_evidence_fields": [], - "evidence_added": False, - "recovered": False, - } - phase = cls._classify_tool_phase(tool_name, tool_args, state) - phases = state.get("phases") - if not isinstance(phases, dict): - return {} - details = phases.get(phase) - if not isinstance(details, dict): - return {} - succeeded = BrowserAgentRuntime.tool_result_succeeded(tool_result) - if not succeeded: - args = cls._coerce_tool_args(tool_args) - result = tool_result if isinstance(tool_result, dict) else {"result": tool_result} - evidence_delta = cls._record_tool_evidence( - state, - result, - tool_name=tool_name, - tool_args=args, - ) - completion_evidence = cls._phase_completion_evidence( - phase, - str(tool_name or "").strip().lower(), - args, - result, - ) - cls._record_failed_phase_result(state, details, tool_result) - missing_fields = cls._missing_completion_requirements(state) if phase == "extraction" else [] - if completion_evidence and not missing_fields: - cls._complete_phase(state, phases, phase, details, completion_evidence) - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - return {"phase": phase, "success": False, **evidence_delta} - - details["successes"] = int(details.get("successes") or 0) + 1 - details["last_error"] = "" - args = cls._coerce_tool_args(tool_args) - result = tool_result if isinstance(tool_result, dict) else {"result": tool_result} - cls._record_price_interval(details, phase, tool_name, args) - evidence_delta = cls._record_tool_evidence(state, result, tool_name=tool_name, tool_args=args) - if evidence_delta.get("recovered"): - BrowserWorkingContextStore.mark_replan_recovered(state) - completion_evidence = cls._phase_completion_evidence( - phase, - str(tool_name or "").strip().lower(), - args, - result, - ) - missing_fields = cls._missing_completion_requirements(state) if phase == "extraction" else [] - if completion_evidence and not missing_fields: - cls._complete_phase(state, phases, phase, details, completion_evidence) - elif details.get("status") != "replan_required": - details["status"] = "in_progress" - details["missing_fields"] = missing_fields - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - return {"phase": phase, "success": True, **evidence_delta} - - @staticmethod - def _missing_required_fields(state: Dict[str, Any]) -> list[str]: - required = {str(item) for item in state.get("required_fields") or [] if str(item).strip()} - coverage = {str(item) for item in state.get("field_coverage") or [] if str(item).strip()} - return sorted(required - coverage) - - @staticmethod - def _has_task_evidence(state: Dict[str, Any]) -> bool: - """Use evidence records, not compatibility coverage, as task truth.""" - - return bool(state.get("evidence_slots") or state.get("structured_evidence")) - - @staticmethod - def _task_observations(state: Dict[str, Any]) -> list[Dict[str, Any]]: - return [ - item for item in state.get("structured_evidence") or [] - if isinstance(item, dict) and item.get("kind") == "page_observation" - ] - - @classmethod - def _advisory_missing_fields(cls, state: Dict[str, Any]) -> set[str]: - """Do not mistake an incomplete field adapter for an unsuccessful page read.""" - if state.get("requirements_source") != "inferred" or not cls._task_observations(state): - return set() - slots = state.get("required_evidence_slots") or [] - # Comparison variants and typed rating/sort evidence remain strict. - if any(isinstance(slot, dict) and slot.get("variant", "default") != "default" for slot in slots): - return set() - return set(cls._missing_required_fields(state)) - _BROWSER_STRICT_EVIDENCE_FIELDS - - @classmethod - def _missing_completion_requirements(cls, state: Dict[str, Any]) -> list[str]: - missing = cls._missing_required_fields(state) - missing_field_names = set(missing) - missing_slots = cls._missing_evidence_slots(state) - missing.extend( - f"evidence_slot:{slot['entity']}:{slot['variant']}:{slot['field']}" - for slot in missing_slots - if not (slot["variant"] == "default" and slot["field"] in missing_field_names) - ) - requested_count = int(state.get("requested_result_count") or 0) - observed_count = int(state.get("observed_result_count") or 0) - if requested_count and observed_count < requested_count: - missing.append(f"result_count:{observed_count}/{requested_count}") - return missing - - @classmethod - def _missing_evidence_slots(cls, state: Dict[str, Any]) -> list[Dict[str, str]]: - required_slots = set() - for slot in state.get("required_evidence_slots") or []: - if isinstance(slot, dict): - required_slots.add(cls._evidence_slot_key(slot)) - - covered_slots = set() - for slot in state.get("evidence_slots") or []: - if not isinstance(slot, dict): - continue - status = str(slot.get("status") or "").strip().lower() - if slot.get("value") not in (None, "") or status in {"missing", "unknown"}: - covered_slots.add(cls._evidence_slot_key(slot)) - return [ - {"entity": entity, "variant": variant, "field": field_name} - for entity, variant, field_name in sorted(required_slots - covered_slots) - ] - - @classmethod - def _unavailable_evidence_slots(cls, state: Dict[str, Any]) -> list[Dict[str, str]]: - required_slots = set() - for slot in state.get("required_evidence_slots") or []: - if isinstance(slot, dict): - required_slots.add(cls._evidence_slot_key(slot)) - - unavailable = [] - for slot in state.get("evidence_slots") or []: - if not isinstance(slot, dict): - continue - key = cls._evidence_slot_key(slot) - status = str(slot.get("status") or "").strip().lower() - if key not in required_slots or status not in {"missing", "unknown"}: - continue - unavailable.append( - { - "entity": key[0], - "variant": key[1], - "field": key[2], - "status": status or "unknown", - } - ) - return unavailable - - @staticmethod - def _evidence_slot_key(slot: Dict[str, Any]) -> tuple[str, str, str]: - return ( - str(slot.get("entity") or "").strip().lower(), - str(slot.get("variant") or "").strip().lower(), - str(slot.get("field") or "").strip().lower(), - ) - - @staticmethod - def _record_failed_phase_result(state: Dict[str, Any], details: Dict[str, Any], tool_result: Any) -> None: - if str(state.get("status") or "").strip().lower() in _BROWSER_TERMINAL_STATUSES: - return - details["status"] = "pending" - if isinstance(tool_result, dict): - details["last_error"] = str(tool_result.get("error") or "")[:300] - if not state.get("replan_trial_pending"): - return - trial_strategy = str(state.get("trial_strategy") or "") - state["replan_trial_pending"] = False - state["replan_required"] = True - state["blocked_strategy"] = trial_strategy - BrowserWorkingContextStore.record_failed_strategy(state, trial_strategy) - state["status"] = "replan_required" - - @classmethod - def _record_tool_evidence( - cls, - state: Dict[str, Any], - result: Dict[str, Any], - *, - tool_name: str, - tool_args: Dict[str, Any], - ) -> Dict[str, Any]: - previous_evidence = { - cls._evidence_signature(item) for item in state.get("structured_evidence") or [] - if isinstance(item, dict) - } - previous_coverage = set(state.get("field_coverage") or []) - cls._record_structured_evidence(state, result, tool_name=tool_name, tool_args=tool_args) - evidence_added = any( - cls._evidence_signature(item) not in previous_evidence - for item in state.get("structured_evidence") or [] if isinstance(item, dict) - ) - new_evidence_fields = sorted(set(state.get("field_coverage") or []) - previous_coverage) - return { - "new_evidence_fields": new_evidence_fields, - "evidence_added": evidence_added, - "recovered": bool(evidence_added or new_evidence_fields), - } - - @classmethod - def _record_price_interval( - cls, - details: Dict[str, Any], - phase: str, - tool_name: str, - args: Dict[str, Any], - ) -> None: - price_interval = cls._price_interval_signature(tool_name, args) - if phase != "filtering" or not price_interval: - return - visited_intervals = details.setdefault("visited_price_intervals", []) - if price_interval not in visited_intervals: - visited_intervals.append(price_interval) - - @classmethod - def _record_structured_evidence( - cls, - state: Dict[str, Any], - result: Dict[str, Any], - *, - tool_name: str, - tool_args: Dict[str, Any], - ) -> None: - evidence = cls._structured_evidence(result) - if not evidence and "evaluate" in str(tool_name or "").lower(): - evidence = cls._evaluate_evidence( - result, - tool_args, - required_fields=state.get("required_fields") or [], - ) - if not evidence and "probe_interactives" in str(tool_name or "").lower(): - evidence = cls._interactive_probe_evidence(result) - stored_evidence = state.setdefault("structured_evidence", []) - known_signatures = {cls._evidence_signature(item) for item in stored_evidence if isinstance(item, dict)} - observation = cls._page_observation_evidence(state, result, tool_name, tool_args) - for item in (evidence, observation): - if item and cls._evidence_signature(item) not in known_signatures: - stored_evidence.append(item) - del stored_evidence[:-20] - if not evidence: - return - if evidence.get("kind") == "card_probe": - state["observed_result_count"] = max( - int(state.get("observed_result_count") or 0), - int(evidence.get("observed_count") or 0), - ) - cls._record_evidence_slots( - state, - evidence, - result=result, - tool_name=tool_name, - tool_args=tool_args, - ) - BrowserWorkingContextStore.refresh_field_coverage(state) - - @classmethod - def _page_observation_evidence( - cls, - state: Dict[str, Any], - result: Dict[str, Any], - tool_name: str, - tool_args: Dict[str, Any], - ) -> Dict[str, Any]: - """Retain source text without inventing field coverage or another memory store.""" - if not BrowserAgentRuntime.tool_result_succeeded(result): - return {} - if not any(token in tool_name for token in ("browser_evaluate", "browser_find", "browser_snapshot")): - return {} - if not cls._is_read_only_recovery(tool_name, tool_args): - return {} - value = cls._evaluate_result_value(result) - if value in (None, "", [], {}) or isinstance(value, (bool, int, float)): - return {} - if not BrowserAgentRuntime.tool_result_succeeded(value): - return {} - raw_text = value if isinstance(value, str) else cls._evidence_signature(value) - raw_text = raw_text.split("### Ran Playwright code", 1)[0].strip() - # AX references and selector spelling are identity metadata, not new facts. - normalized = re.sub(r"\s*\[(?:ref|target_id|generation_id)=[^\]]+\]", "", raw_text) - normalized = " ".join(normalized.split()) - if len(normalized) < 16: - return {} - _, source = cls._evidence_variant_and_url(state, result, tool_args) - if not source: - return {} - page = result.get("page_state") or {} - generation = str(result.get("generation_id") or page.get("generation_id") or "") - target = str(tool_args.get("target") or tool_args.get("selector") or "")[:240] - return { - "kind": "page_observation", - "source": source, - "generation_id": generation, - "raw_text": normalized[:4_000], - "content_hash": hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:20], - "provenance": { - "tool": tool_name, - **cls._evaluate_execution_provenance(tool_args, target), - }, - } - - @classmethod - def _record_evidence_slots( - cls, - state: Dict[str, Any], - evidence: Dict[str, Any], - *, - result: Dict[str, Any], - tool_name: str, - tool_args: Dict[str, Any], - ) -> None: - if not state.get("required_evidence_slots"): - return - variant, source_url = cls._evidence_variant_and_url(state, result, tool_args) - values, generation, provenance, field_status = cls._evidence_values(evidence) - if not values and not field_status: - return - slots = state.setdefault("evidence_slots", []) - covered = { - cls._evidence_slot_key(slot): index - for index, slot in enumerate(slots) - if isinstance(slot, dict) - } - for required in state.get("required_evidence_slots") or []: - if not isinstance(required, dict): - continue - key = cls._evidence_slot_key(required) - if key[1] != "default" and key[1] != variant: - continue - slot = cls._build_evidence_slot( - key, - values=values, - provenance=provenance, - field_status=field_status, - generation=generation, - source_url=source_url, - tool_name=tool_name, - ) - if slot is None: - continue - existing_index = covered.get(key) - if existing_index is not None: - existing = slots[existing_index] - existing_status = str(existing.get("status") or "present").strip().lower() - if existing_status == "present" or slot.get("status") != "present": - continue - slots[existing_index] = slot - else: - slots.append(slot) - covered[key] = len(slots) - 1 - del state["evidence_slots"][:-20] - - @staticmethod - def _build_evidence_slot( - key: tuple[str, str, str], - *, - values: Dict[str, Any], - provenance: Dict[str, Any], - field_status: Dict[str, Any], - generation: str, - source_url: str, - tool_name: str, - ) -> Optional[Dict[str, Any]]: - value = values.get(key[2]) - status = str(field_status.get(key[2]) or ("present" if value not in (None, "") else "")).lower() - if status not in {"present", "missing", "unknown"}: - return None - if status == "present" and value in (None, ""): - return None - item = provenance.get(key[2]) if isinstance(provenance, dict) else None - item = item if isinstance(item, dict) else {} - slot = { - "entity": key[0], - "variant": key[1], - "field": key[2], - "status": status, - "source": str(item.get("source") or source_url or tool_name or "")[:500], - "generation": str(item.get("generation_id") or generation or ""), - } - if status == "present": - slot["value"] = str(value)[:500] - for provenance_key in ("selector", "raw_text"): - provenance_value = str(item.get(provenance_key) or "").strip() - if provenance_value: - slot[provenance_key] = provenance_value[:600] - return slot - - @staticmethod - def _evidence_values( - evidence: Dict[str, Any], - ) -> tuple[Dict[str, Any], str, Dict[str, Any], Dict[str, str]]: - values = evidence.get("values") - if isinstance(values, dict): - provenance = evidence.get("provenance") - statuses = evidence.get("field_status") - return ( - values, - str(evidence.get("generation_id") or ""), - provenance if isinstance(provenance, dict) else {}, - dict(statuses) if isinstance(statuses, dict) else {}, - ) - cards = evidence.get("cards") - if not isinstance(cards, list): - return {}, str(evidence.get("generation_id") or ""), {}, {} - first = next((card for card in cards if isinstance(card, dict)), {}) - if not first: - return {}, "", {}, {} - provenance = first.get("provenance") - statuses = first.get("field_status") - return ( - first, - str(first.get("generation_id") or evidence.get("generation_id") or ""), - provenance if isinstance(provenance, dict) else {}, - dict(statuses) if isinstance(statuses, dict) else {}, - ) - - @staticmethod - def _evidence_variant_and_url( - state: Dict[str, Any], - result: Dict[str, Any], - tool_args: Dict[str, Any], - ) -> tuple[str, str]: - page_state = result.get("page_state") if isinstance(result.get("page_state"), dict) else {} - last_page = state.get("last_page") if isinstance(state.get("last_page"), dict) else {} - source_url = str( - BrowserAgentRuntime.extract_result_url(result) - or page_state.get("url") - or tool_args.get("url") - or last_page.get("url") - or "" - ).strip() - query: Dict[str, list[str]] = {} - try: - query = parse_qs(urlsplit(source_url).query) - except ValueError: - pass - order = " ".join(query.get("order", []) + query.get("sort", [])).lower() - serialized_args = json.dumps(tool_args, ensure_ascii=False, default=str).lower() - if any(token in order for token in ("pubdate", "latest", "newest", "recent")) or any( - token in serialized_args for token in ("最新", "latest", "newest", "pubdate") - ): - return "latest", source_url - if any(token in order for token in ("totalrank", "relevance", "default", "comprehensive")) or any( - token in serialized_args for token in ("综合", "comprehensive", "relevance", "totalrank") - ): - return "comprehensive", source_url - required_variants = { - str(slot.get("variant") or "") - for slot in state.get("required_evidence_slots") or [] - if isinstance(slot, dict) - } - if required_variants == {"comprehensive", "latest"} and source_url and not order: - return "comprehensive", source_url - return "default", source_url - - @classmethod - def _evidence_signature(cls, evidence: Dict[str, Any]) -> str: - if isinstance(evidence, dict) and evidence.get("kind") == "page_observation": - return json.dumps([evidence.get("source"), evidence.get("content_hash")], ensure_ascii=False) - ignored_keys = { - "generation", - "generation_id", - "sel", - "selector", - "selector_hint", - "ref", - "target_id", - "target", - "provenance", - "execution_provenance", - } - - def semantic_value(value: Any) -> Any: - if isinstance(value, dict): - return {str(key): semantic_value(item) for key, item in value.items() if str(key) not in ignored_keys} - if isinstance(value, list): - return [semantic_value(item) for item in value] - return value - - return json.dumps( - semantic_value(evidence), - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - default=str, - ) - - @classmethod - def _evaluate_evidence( - cls, - result: Dict[str, Any], - tool_args: Dict[str, Any], - *, - required_fields: Iterable[str] = (), - ) -> Dict[str, Any]: - value = cls._evaluate_result_value(result) - if value is None: - return {} - trusted_fields, allowed_fields = cls._evaluate_field_scope(tool_args, required_fields) - compact_values, fields, field_status, raw_values = cls._compact_evaluate_values( - value, - allowed_fields=allowed_fields, - trusted_fields=trusted_fields, - ) - if not compact_values and not field_status: - return {} - raw_preview = json.dumps(value, ensure_ascii=False, default=str) if not isinstance(value, str) else value - generation_id = str(result.get("generation_id") or tool_args.get("generation_id") or "") - target = str(tool_args.get("target") or tool_args.get("element") or "")[:240] - return { - "kind": "targeted_evaluate", - "generation_id": generation_id, - "fields": fields, - "values": compact_values, - "field_status": field_status, - "preview": " ".join(raw_preview.split())[:800], - "provenance": cls._evaluate_field_provenance( - compact_values, - field_status=field_status, - raw_values=raw_values, - target=target, - generation_id=generation_id, - ), - "execution_provenance": cls._evaluate_execution_provenance(tool_args, target), - } - - @staticmethod - def _evaluate_result_value(result: Dict[str, Any]) -> Any: - value = next( - (result.get(key) for key in ("result", "value", "data") if result.get(key) is not None), - None, - ) - if not isinstance(value, str): - return value - try: - return json.loads(value) - except (TypeError, ValueError): - return extract_json_object(value) or value - - @classmethod - def _evaluate_field_scope( - cls, - tool_args: Dict[str, Any], - required_fields: Iterable[str], - ) -> tuple[set[str], set[str]]: - trusted_fields = { - cls._canonical_field_name(field_name) - for field_name in tool_args.get("_runtime_evidence_fields") or [] - } - trusted_fields.discard("") - allowed_fields = { - cls._canonical_field_name(field_name) - for field_name in required_fields - } - allowed_fields.discard("") - allowed_fields.update(trusted_fields) - return trusted_fields, allowed_fields - - @staticmethod - def _evaluate_execution_provenance(tool_args: Dict[str, Any], target: str) -> Dict[str, str]: - expression = str( - tool_args.get("function") - or tool_args.get("expression") - or tool_args.get("script") - or tool_args.get("code") - or "" - ) - return { - "target": target, - "expression_sha256": hashlib.sha256(expression.encode("utf-8")).hexdigest()[:16] if expression else "", - } - - @classmethod - def _compact_evaluate_values( - cls, - value: Any, - *, - allowed_fields: set[str], - trusted_fields: set[str], - ) -> tuple[Dict[str, str], list[str], Dict[str, str], Dict[str, str]]: - if isinstance(value, dict): - return cls._compact_evaluate_mapping( - value, - allowed_fields=allowed_fields, - trusted_fields=trusted_fields, - ) - if isinstance(value, list) and value and all(isinstance(item, dict) for item in value[:5]): - return cls._compact_evaluate_list( - value, - allowed_fields=allowed_fields, - trusted_fields=trusted_fields, - ) - return {}, [], {}, {} - - @classmethod - def _compact_evaluate_mapping( - cls, - value: Dict[str, Any], - *, - allowed_fields: set[str], - trusted_fields: set[str], - ) -> tuple[Dict[str, str], list[str], Dict[str, str], Dict[str, str]]: - compact_values: Dict[str, str] = {} - field_status: Dict[str, str] = {} - raw_values: Dict[str, str] = {} - for key, item in list(value.items())[:20]: - normalized = cls._normalize_evaluate_mapping_item( - key, - item, - allowed_fields=allowed_fields, - trusted_fields=trusted_fields, - ) - if normalized is None: - continue - field_name, field_value, status, raw_value = normalized - field_status[field_name] = status - raw_values[field_name] = raw_value - if status == "present": - compact_values[field_name] = field_value - return compact_values, sorted(compact_values), field_status, raw_values - - @classmethod - def _normalize_evaluate_mapping_item( - cls, - key: Any, - item: Any, - *, - allowed_fields: set[str], - trusted_fields: set[str], - ) -> tuple[str, str, str, str] | None: - selected_sort = cls._selected_sort_value(key, item) - normalized: tuple[str, str, str, str] | None = None - if selected_sort: - if not allowed_fields or "sort_state" in allowed_fields: - normalized = ("sort_state", selected_sort, "present", f"{key}={item}"[:600]) - else: - snake_name = cls._snake_case_field_name(key) - canonical = cls._canonical_evaluate_field_name(key) - field_allowed = canonical in _BROWSER_FIELD_ALIASES and ( - not allowed_fields or canonical in allowed_fields - ) - contextual_alias_allowed = ( - snake_name not in _BROWSER_CONTEXTUAL_EVALUATE_FIELD_ALIASES - or canonical in trusted_fields - ) - if field_allowed and contextual_alias_allowed and not ( - canonical == "sort_state" and isinstance(item, bool) - ): - normalized_value, status = cls._normalize_evaluate_field_value(canonical, item) - normalized = (canonical, normalized_value, status, str(item)[:600]) - return normalized - - @staticmethod - def _merge_first_values(target: Dict[str, str], source: Dict[str, str]) -> None: - for field_name, field_value in source.items(): - target.setdefault(field_name, field_value) - - @classmethod - def _compact_evaluate_list( - cls, - value: list[Dict[str, Any]], - *, - allowed_fields: set[str], - trusted_fields: set[str], - ) -> tuple[Dict[str, str], list[str], Dict[str, str], Dict[str, str]]: - compact_values: Dict[str, str] = {} - field_status: Dict[str, str] = {} - raw_values: Dict[str, str] = {} - fields: set[str] = set() - for item in value[:5]: - item_values, item_fields, item_status, item_raw = cls._compact_evaluate_mapping( - item, - allowed_fields=allowed_fields, - trusted_fields=trusted_fields, - ) - fields.update(item_fields) - cls._merge_first_values(compact_values, item_values) - cls._merge_first_values(field_status, item_status) - cls._merge_first_values(raw_values, item_raw) - if not fields and not field_status: - return {}, [], {}, {} - compact_values["items"] = json.dumps(value[:3], ensure_ascii=False, default=str)[:800] - return compact_values, sorted(fields), field_status, raw_values - - @classmethod - def _selected_sort_value(cls, key: Any, value: Any) -> str: - snake_name = cls._snake_case_field_name(key) - if snake_name in {"active_tab", "active_tabs", "selected_tab", "selected_tabs"}: - if isinstance(value, bool): - return "" - text = " ".join(str(value or "").split()) - return text[:120] if text else "" - if not snake_name.endswith(("_selected", "_tab_selected")) or value is not True: - return "" - base_name = re.sub(r"(?:_tab)?_selected$", "", snake_name) - for token, label in _BROWSER_SELECTED_SORT_LABELS.items(): - if token in base_name: - return label - return "" - - @staticmethod - def _normalize_evaluate_field_value(field_name: str, value: Any) -> tuple[str, str]: - if value in (None, "", [], {}): - return "", "missing" - text = " ".join(str(value).split())[:300] - if field_name == "comments" and _BROWSER_ZERO_COMMENT_RE.fullmatch(text): - return "0", "present" - if _BROWSER_UNKNOWN_VALUE_RE.fullmatch(text): - return "", "unknown" - return text, "present" - - @staticmethod - def _evaluate_field_provenance( - values: Dict[str, str], - *, - field_status: Dict[str, str], - raw_values: Dict[str, str], - target: str, - generation_id: str, - ) -> Dict[str, Dict[str, str]]: - return { - field_name: { - "source": "browser_evaluate", - "selector": target, - "raw_text": str(raw_values.get(field_name) or values.get(field_name) or "")[:600], - "generation_id": generation_id, - } - for field_name in set(values) | set(field_status) - } - - @classmethod - def _interactive_probe_evidence(cls, result: Dict[str, Any]) -> Dict[str, Any]: - elements = result.get("elements") - if not isinstance(elements, list) or not elements: - return {} - targets = [ - { - "target_id": str(item.get("target_id") or ""), - "generation_id": str(item.get("generation_id") or result.get("generation_id") or ""), - "kind": str(item.get("kind") or ""), - "region": str(item.get("region") or ""), - "name": str(item.get("accessible_name") or item.get("text") or "")[:160], - "actionable": bool(item.get("actionable")), - "selected": bool(item.get("selected")), - "selected_source": str(item.get("selected_source") or "")[:40], - } - for item in elements[:12] - if isinstance(item, dict) - ] - selected_sort = next( - ( - target - for target in targets - if target["selected"] and cls._is_sort_interactive_target(target) - ), - None, - ) - evidence = { - "kind": "interactive_probe", - "generation_id": str(result.get("generation_id") or ""), - "fields": [], - "target_count": len(targets), - "targets": targets, - } - if selected_sort and selected_sort["name"]: - evidence["fields"] = ["sort_state"] - evidence["values"] = {"sort_state": selected_sort["name"]} - evidence["provenance"] = { - "sort_state": { - "source": "browser_probe_interactives", - "selector": selected_sort["target_id"], - "raw_text": selected_sort["name"], - "generation_id": selected_sort["generation_id"], - "selection_source": selected_sort["selected_source"], - } - } - return evidence - - @staticmethod - def _is_sort_interactive_target(target: Dict[str, Any]) -> bool: - if str(target.get("kind") or "") in {"sort", "sort_tab", "sort_option"}: - return True - name = str(target.get("name") or "").strip().lower() - region = str(target.get("region") or "").strip().lower() - return bool( - region in {"sort", "filtering", "search_controls"} - or any(token in name for token in _BROWSER_SELECTED_SORT_LABELS) - ) - - @classmethod - def _record_recent_action( - cls, - session: Any, - *, - tool_name: str, - tool_args: Any, - tool_result: Any, - action_class: str, - elapsed_ms: int, - progress_delta: Dict[str, Any], - ) -> bool: - state = session.get_state(_BROWSER_PHASE_STATE_KEY) - if not isinstance(state, dict): - return False - state["action_sequence"] = int(state.get("action_sequence") or 0) + 1 - recent_actions = deque(state.get("recent_actions") or [], maxlen=6) - recent_actions.append( - cls._build_action_record( - state, - tool_name=tool_name, - tool_args=tool_args, - tool_result=tool_result, - action_class=action_class, - elapsed_ms=elapsed_ms, - progress_delta=progress_delta, - ) - ) - state["recent_actions"] = list(recent_actions) - cls._update_last_page(state, tool_result) - session.update_state({_BROWSER_PHASE_STATE_KEY: state}) - return bool(progress_delta.get("recovered")) - - @classmethod - def _build_action_record( - cls, - state: Dict[str, Any], - *, - tool_name: str, - tool_args: Any, - tool_result: Any, - action_class: str, - elapsed_ms: int, - progress_delta: Dict[str, Any], - ) -> Dict[str, Any]: - succeeded = bool(progress_delta.get("success")) - error = str(tool_result.get("error") or "").strip() if isinstance(tool_result, dict) else "" - semantic_delta = "evidence_added" if progress_delta.get("evidence_added") else "pending" - outcome_status = "success" if succeeded else "failed" - if progress_delta.get("ambiguous"): - outcome_status = "ambiguous" - semantic_delta = "awaiting_observation" - elif not succeeded: - semantic_delta = "error" - return { - "seq": state["action_sequence"], - "phase": progress_delta.get("phase") or state.get("current_phase"), - "action_class": action_class or cls._classify_action_class(tool_name, tool_args, state), - "target_summary": cls._target_summary(tool_name, tool_args), - "outcome": "success" if succeeded else f"error: {error[:180] or 'tool_failed'}", - "outcome_status": outcome_status, - "semantic_delta": semantic_delta, - "new_evidence_fields": list(progress_delta.get("new_evidence_fields") or []), - "elapsed_ms": max(0, int(elapsed_ms)), - } - - @staticmethod - def _update_last_page(state: Dict[str, Any], tool_result: Any) -> None: - result = tool_result if isinstance(tool_result, dict) else {} - page_state = result.get("page_state") if isinstance(result.get("page_state"), dict) else {} - url = BrowserAgentRuntime.extract_result_url(result) or str(page_state.get("url") or "") - title = str(result.get("title") or page_state.get("title") or "") - if url or title: - last_page = state.setdefault("last_page", {}) - if url: - last_page["url"] = url - if title: - last_page["title"] = title - - @classmethod - def _target_summary(cls, tool_name: str, tool_args: Any) -> str: - args = cls._coerce_tool_args(tool_args) - name = str(tool_name or "").strip().rsplit(".", 1)[-1] - summary: Dict[str, Any] = {"tool": name} - for key in ("url", "target_id", "target", "ref", "element", "name", "label", "text", "value"): - value = args.get(key) - if value not in (None, "", [], {}): - summary[key] = str(value)[:160] - steps = args.get("steps") - if isinstance(steps, list): - summary["ops"] = [str(step.get("op") or "") for step in steps[:12] if isinstance(step, dict)] - expression = str(args.get("function") or args.get("expression") or args.get("script") or args.get("code") or "") - if expression: - summary["expression_sha256"] = hashlib.sha256(expression.encode("utf-8")).hexdigest()[:16] - summary["expression_chars"] = len(expression) - return json.dumps(summary, ensure_ascii=False, separators=(",", ":"))[:600] - - @staticmethod - def _phase_completion_evidence( - phase: str, - tool_name: str, - args: Dict[str, Any], - result: Dict[str, Any], - ) -> str: - successful_condition_operations = { - str(condition.get("op") or "").strip().lower() - for condition in (result.get("conditions") if isinstance(result.get("conditions"), list) else []) - if isinstance(condition, dict) and condition.get("ok") is True - } - if phase == "navigation": - result_url = BrowserAgentRuntime.extract_result_url(result) - if result_url or "navigate" in tool_name: - return result_url or "navigation tool succeeded" - if phase == "form" and successful_condition_operations & { - "wait_for_url", - "wait_for_selector", - "wait_for_text", - "wait_for_first_card_title", - "wait_for_result_count", - "wait_for_dom_text_change", - "wait_for_stable", - "wait_for_tab", - }: - return "form batch completed with an observable condition" - if phase == "filtering" and successful_condition_operations & { - "wait_for_sort_state", - "wait_for_result_count", - "wait_for_dom_text_change", - "wait_for_first_card_title", - }: - return "requested state and changed results were observed" - if phase == "extraction": - extracted = result.get("extracted") - cards = result.get("cards") - if isinstance(extracted, dict) and extracted: - return f"structured extraction returned {len(extracted)} field(s)" - if isinstance(cards, list) and cards: - excluded_regions = {"hot_search", "sidebar", "account", "chat"} - excluded_kinds = { - "hot_search", - "paid_column", - "promotion", - "activity", - "account", - "shop", - "chat", - } - natural_cards = [] - for card in cards: - if not isinstance(card, dict) or card.get("is_ad") is True: - continue - if str(card.get("region") or "") in excluded_regions: - continue - if str(card.get("kind") or "") in excluded_kinds: - continue - natural_cards.append(card) - if natural_cards: - return f"card probe returned {len(natural_cards)} structured result(s)" - return "" - - @staticmethod - def _complete_phase( - state: Dict[str, Any], - phases: Dict[str, Any], - phase: str, - details: Dict[str, Any], - completion_evidence: str, - ) -> None: - if str(state.get("status") or "").strip().lower() in _BROWSER_TERMINAL_STATUSES: - return - details["status"] = "completed" - details["completion_evidence"] = completion_evidence[:300] - phase_order = list(phases) - try: - remaining_phases = phase_order[phase_order.index(phase) + 1:] - except ValueError: - remaining_phases = [] - next_phase = next( - ( - candidate - for candidate in remaining_phases - if isinstance(phases.get(candidate), dict) and phases[candidate].get("status") != "completed" - ), - phase, - ) - state["current_phase"] = next_phase - if all(isinstance(item, dict) and item.get("status") == "completed" for item in phases.values()): - state["status"] = "completed" - state["next_action_class"] = "finish" - else: - state["next_action_class"] = next_phase - - @classmethod - def _structured_evidence(cls, result: Dict[str, Any]) -> Dict[str, Any]: - extracted = result.get("extracted") - if isinstance(extracted, dict) and extracted: - return cls._structured_extraction_evidence(result, extracted) - condition_evidence = cls._condition_evidence(result) - if condition_evidence: - return condition_evidence - return cls._card_probe_evidence(result) - - @classmethod - def _structured_extraction_evidence( - cls, - result: Dict[str, Any], - extracted: Dict[str, Any], - ) -> Dict[str, Any]: - provenance = result.get("field_provenance") - compact_values: Dict[str, str] = {} - compact_provenance: Dict[str, Any] = {} - for key, value in list(extracted.items())[:20]: - if value in (None, "", [], {}): - continue - canonical = cls._canonical_field_name(key) or str(key) - compact_values[canonical] = str(value)[:300] - if isinstance(provenance, dict) and isinstance(provenance.get(key), dict): - compact_provenance[canonical] = dict(provenance[key]) - condition_values, condition_provenance = cls._successful_condition_values(result) - for key, value in condition_values.items(): - compact_values.setdefault(key, value) - if key in condition_provenance: - compact_provenance.setdefault(key, condition_provenance[key]) - return { - "kind": "structured_extraction", - "generation_id": str(result.get("generation_id") or ""), - "fields": sorted(compact_values), - "values": compact_values, - "provenance": compact_provenance, - } - - @classmethod - def _condition_evidence(cls, result: Dict[str, Any]) -> Dict[str, Any]: - values, provenance = cls._successful_condition_values(result) - if not values: - return {} - return { - "kind": "condition_observation", - "generation_id": str(result.get("generation_id") or ""), - "fields": sorted(values), - "values": values, - "provenance": provenance, - } - - @staticmethod - def _successful_condition_values( - result: Dict[str, Any], - ) -> tuple[Dict[str, str], Dict[str, Dict[str, str]]]: - field_by_op = { - "wait_for_sort_state": "sort_state", - "wait_for_result_count": "result_count", - "wait_for_first_card_title": "title", - "wait_for_url": "url", - "wait_for_dom_text_change": "dom_text", - "wait_for_tab": "tab_state", - } - values: Dict[str, str] = {} - provenance: Dict[str, Dict[str, str]] = {} - generation_id = str(result.get("generation_id") or "") - conditions = result.get("conditions") - for item in conditions if isinstance(conditions, list) else []: - if not isinstance(item, dict) or item.get("ok") is not True: - continue - field_name = field_by_op.get(str(item.get("op") or "").strip().lower()) - if not field_name: - continue - observed = item.get("observed") - if isinstance(observed, dict): - value = next( - ( - observed[key] - for key in ("value", "text", "url", "count", "tabs", "stable") - if observed.get(key) not in (None, "", [], {}) - ), - None, - ) - else: - value = observed - if value in (None, "", [], {}): - continue - rendered = ( - json.dumps(value, ensure_ascii=False, default=str) - if isinstance(value, (dict, list)) - else str(value) - ) - values[field_name] = rendered[:500] - provenance[field_name] = { - "source": "browser_batch_interact", - "raw_text": rendered[:600], - "generation_id": generation_id, - } - return values, provenance - - @classmethod - def _card_probe_evidence(cls, result: Dict[str, Any]) -> Dict[str, Any]: - cards = result.get("cards") - if not isinstance(cards, list) or not cards: - return {} - compact_cards: list[Dict[str, Any]] = [] - fields: set[str] = set() - for card in cards: - compact = cls._compact_evidence_card(card, result) - if compact is None: - continue - metadata_fields = { - "generation_id", - "result_index", - "kind", - "provenance", - "field_status", - } - field_status = compact.get("field_status", {}) - for field_name in compact: - if field_name in metadata_fields: - continue - if field_status.get(field_name, "present") == "present": - fields.add(field_name) - compact_cards.append(compact) - if len(compact_cards) >= 5: - break - if not compact_cards: - return {} - return { - "kind": "card_probe", - "fields": sorted(fields), - "cards": compact_cards, - "observed_count": int(result.get("observed_count") or len(compact_cards)), - } - - @classmethod - def _compact_evidence_card( - cls, - card: Any, - result: Dict[str, Any], - ) -> Optional[Dict[str, Any]]: - if not isinstance(card, dict) or not cls._is_natural_evidence_card(card): - return None - compact: Dict[str, Any] = { - "generation_id": str(card.get("generation_id") or result.get("generation_id") or ""), - "result_index": card.get("result_index"), - "kind": card.get("kind"), - } - for field_name in CARD_EVIDENCE_FIELDS: - value = card.get(field_name) - if value in (None, "", [], {}): - continue - compact.setdefault(cls._canonical_field_name(field_name) or field_name, str(value)[:300]) - provenance = card.get("field_provenance") - if isinstance(provenance, dict): - compact["provenance"] = { - cls._canonical_field_name(field_name) or str(field_name): dict(item) - for field_name, item in list(provenance.items())[:20] - if isinstance(item, dict) - } - statuses = card.get("field_status") - if isinstance(statuses, dict): - compact["field_status"] = { - cls._canonical_field_name(field_name) or str(field_name): str(status) - for field_name, status in list(statuses.items())[:20] - if str(status).strip().lower() in {"present", "missing", "unknown"} - } - return compact - - @staticmethod - def _is_natural_evidence_card(card: Dict[str, Any]) -> bool: - region = str(card.get("region") or "") - kind = str(card.get("kind") or "") - if not (region or kind or "is_ad" in card): - return True - return ( - card.get("is_ad") is not True - and region - not in { - "hot_search", - "sidebar", - "account", - "chat", - "activity", - "sponsored_result", - "resource", - "commercial_module", - "ai_overview", - "question_module", - "related", - } - and kind - not in { - "hot_search", - "paid_column", - "promotion", - "activity", - "account", - "shop", - "chat", - "download_resource", - "commercial_module", - "ai_overview", - "people_also_ask", - "related_links", - "knowledge_panel", - } - ) - - def _clear_progress_state(self, session: Any) -> None: - if session is None: - return - session_id = session.get_session_id() - self._runtime.service.clear_progress_state(session_id) - session.update_state( - { - _BROWSER_PROGRESS_STATE_KEY: {}, - _BROWSER_PROGRESS_TASK_KEY: "", - _BROWSER_PHASE_STATE_KEY: {}, - } - ) - - @staticmethod - def _image_input_supported(agent: Any) -> bool: - return should_enable_read_image_multimodal(agent) - - async def _effective_browser_tool_allowlist( - self, - agent: Any, - mcp_cfg: McpServerConfig, - ) -> tuple[str, ...]: - del mcp_cfg - configured = self._runtime.service.allowed_tool_names or CORE_BROWSER_TOOL_NAMES - if self._image_input_supported(agent): - return tuple(configured) - - return tuple(tool_name for tool_name in configured if tool_name not in _BROWSER_SCREENSHOT_TOOL_NAMES) - - async def _ensure_browser_mcp_ability(self, ctx: AgentCallbackContext) -> None: - agent = getattr(ctx, "agent", None) - ability_manager = getattr(agent, "ability_manager", None) - if ability_manager is None: - return - mcp_cfg = self._runtime.service.mcp_cfg - ability_manager.add(mcp_cfg) - allowed_tool_names = await self._effective_browser_tool_allowlist( - agent, - mcp_cfg, - ) - if allowed_tool_names is not None: - ability_manager.set_mcp_tool_allowlist(mcp_cfg, allowed_tool_names) diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/runtime_tools.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/runtime_tools.py deleted file mode 100644 index 3bd80d5efa..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/runtime_tools.py +++ /dev/null @@ -1,850 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""DeepAgent Tool wrappers around BrowserAgentRuntime.""" - -from __future__ import annotations - -import contextvars -from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, List - -from openjiuwen.core.foundation.tool import Tool, ToolCard -from openjiuwen.harness.tools.base_tool import ToolOutput - -if TYPE_CHECKING: - from .runtime import BrowserAgentRuntime - -_ctx_parent_session_id: contextvars.ContextVar[str] = contextvars.ContextVar( - "playwright_runtime_parent_session_id", - default="", -) -_ctx_parent_request_id: contextvars.ContextVar[str] = contextvars.ContextVar( - "playwright_runtime_parent_request_id", - default="", -) - -_CANCEL_DESC = ( - "Cancel an in-progress browser task by session_id. " - "Optionally pass request_id to target a specific request within the session. " - "Returns JSON with ok/session_id/request_id/error." -) -_CANCEL_PARAMS: Dict[str, Any] = { - "type": "object", - "properties": { - "session_id": {"type": "string", "description": "Session ID of the task to cancel"}, - "request_id": {"type": "string", "description": "Optional: specific request ID to cancel"}, - }, - "required": ["session_id"], -} - -_CLEAR_CANCEL_DESC = ( - "Clear the cancellation flag for a browser session or request. " - "Returns JSON with ok/session_id/request_id/error." -) -_CLEAR_CANCEL_PARAMS: Dict[str, Any] = { - "type": "object", - "properties": { - "session_id": {"type": "string", "description": "Session ID to clear"}, - "request_id": {"type": "string", "description": "Optional: specific request ID to clear"}, - }, - "required": ["session_id"], -} - -_CUSTOM_ACTION_DESC = ( - "Run a registered custom browser action by name. " - "Use for deterministic helpers such as drag-and-drop or coordinate resolution " - "alongside the direct Playwright MCP browser tools. " - "Call browser_list_custom_actions first to discover available actions and parameters. " - "Aliases source/target and source_x/source_y/target_x/target_y are accepted." -) -_CUSTOM_ACTION_PARAMS: Dict[str, Any] = { - "type": "object", - "properties": { - "action": {"type": "string", "description": "Name of the custom action to run"}, - "session_id": {"type": "string", "description": "Session ID (optional)"}, - "request_id": {"type": "string", "description": "Request ID (optional)"}, - "params": { - "type": "object", - "description": "Extra key-value parameters forwarded to the action", - "properties": {}, - "required": [], - }, - }, - "required": ["action"], -} - -_LIST_ACTIONS_DESC = ( - "List available custom browser actions and detailed parameter guidance " - "for browser_custom_action." -) -_LIST_ACTIONS_PARAMS: Dict[str, Any] = { - "type": "object", - "properties": {}, - "required": [], -} - -_RUNTIME_HEALTH_DESC = ( - "Return runtime readiness, heartbeat status, and selected provider/model configuration." -) -_RUNTIME_HEALTH_PARAMS: Dict[str, Any] = { - "type": "object", - "properties": {}, - "required": [], -} - -_PROBE_INTERACTIVES_DESC = ( - "Return a compact list of visible, high-value interactive elements on the current page. " - "Use this for page-level controls such as buttons, links, inputs, forms, navigation, login, " - "pagination, menus, calendar dates, sort tabs, rating filters, and visible actions. " - "Dynamic controls are returned as generation-scoped targets with region/kind semantics. " - "The optional query filter is alias-aware for common " - "search/input terms, including placeholders, aria labels, input type/name/id/class, and Chinese " - "search text such as 搜索/关键词. Prefer max_items around 20-30 unless a larger inventory " - "is needed. For product/search/listing card data, prefer browser_probe_cards first. " - "The result includes compact PageState and generation-scoped target_id values that can be " - "passed directly to browser_batch_interact without rebuilding CSS. It also includes " - "role/text/region/kind plus match_count/visible/enabled/actionable. The model-facing " - "result does not expose Probe-local ids or internal selectors." -) -_PROBE_INTERACTIVES_PARAMS: Dict[str, Any] = { - "type": "object", - "properties": { - "max_items": { - "type": "integer", - "description": "Maximum number of elements to return. Default 30, hard-capped at 40.", - }, - "viewport_only": { - "type": "boolean", - "description": "When true, only return elements currently visible in the viewport. Default true.", - }, - "query": { - "type": "string", - "description": "Optional text filter, e.g. 'cart', 'search', 'next', or 'login'.", - }, - }, - "required": [], -} - -_PROBE_CARDS_DESC = ( - "Return compact repeated card/listing structures from the current page. " - "Use this first on product pages, marketplace pages, search-result pages, catalog pages, " - "article-list pages, table/list-row result pages, or any page with repeated visible cards/listings. " - "The result includes compact PageState with generation-scoped card/control target_id values, " - "candidate card title, author/source, summary/snippet, price, rating, " - "review count, availability, primary link, and visible controls, " - "region/kind/result_index/is_ad semantics that distinguish main results, accounts, sidebars, " - "hot searches, shops, chats, and product links, " - "match_count/visible/enabled/clickable/generation_id, recurring structure signatures, and " - "cache diagnostics. Navigate primary_link/href directly instead of clicking a card hint. " - "If this returns the fields needed " - "for the task, including article/search-result title/link/author/source/summary fields, " - "use the compact card result directly instead of taking screenshots/snapshots or running " - "broad DOM evaluation. Only evaluate again when a required field is missing." -) -_PROBE_CARDS_PARAMS: Dict[str, Any] = { - "type": "object", - "properties": { - "max_cards": { - "type": "integer", - "description": "Maximum number of cards to return. Default 12, hard-capped at 20.", - }, - "viewport_only": { - "type": "boolean", - "description": "When true, only inspect cards visible in the current viewport. Default true.", - }, - "include_buttons": { - "type": "boolean", - "description": "When true, include visible buttons/links inside each card. Default true.", - }, - "query": { - "type": "string", - "description": "Optional text filter, e.g. 'mouse', 'book', 'laptop', or 'cart'.", - }, - }, - "required": [], -} - -_BATCH_INTERACT_DESC = ( - "Execute deterministic browser interactions in one standalone runtime tool call. " - "Use after browser_probe_interactives/browser_probe_cards when a page-level flow has multiple " - "known targets, such as three or more form fields, click+type+choose autocomplete, dropdown or " - "date-picker selection, filter panels, search submit plus result wait, or compact extraction. " - "This is a first-class helper like the probe tools; do not route this through browser_custom_action. " - "Pass the current PageState generation_id and use target_id from probes for actions. " - "Only read-only extraction and explicit wait operations accept a validated selector. " - "Locator strategies are mutually exclusive. " - "A one-step call is rewritten by the runtime to an equivalent official browser primitive when " - "one exists, so it does not fail and require another model turn. Multi-step calls are preflighted " - "before side effects. Results use status=completed/partial/failed and contain only compact step " - "status, extracted fields, condition observations, metrics, and PageState." -) -_BATCH_INTERACT_PARAMS: Dict[str, Any] = { - "type": "object", - "properties": { - "steps": { - "type": "array", - "description": ( - "Ordered browser steps to execute in one batch. Supported ops: click, fill, type, " - "autocomplete, select_visible_text, press, select_option, set_checked, " - "wait_for_selector, wait_for_text, wait_for_load_state, wait_for_url, " - "wait_for_first_card_title, wait_for_sort_state, wait_for_result_count, " - "wait_for_dom_text_change, wait_for_stable, wait_for_tab, extract_text, " - "extract_value, screenshot." - ), - "minItems": 1, - "maxItems": 25, - "items": { - "type": "object", - "properties": { - "op": { - "type": "string", - "description": "Step operation name.", - "enum": [ - "click", - "fill", - "type", - "autocomplete", - "select_visible_text", - "press", - "select_option", - "set_checked", - "wait_for_selector", - "wait_for_text", - "wait_for_load_state", - "wait_for_url", - "wait_for_first_card_title", - "wait_for_sort_state", - "wait_for_result_count", - "wait_for_dom_text_change", - "wait_for_stable", - "wait_for_tab", - "extract_text", - "extract_value", - "screenshot", - ], - }, - "target_id": { - "type": "string", - "description": ( - "Generation-scoped target_id returned by the current PageState/probe. " - "Preferred for actions and mutually exclusive with ref/selector/role/" - "label/placeholder/text/testid." - ), - }, - "ref": { - "type": "string", - "description": ( - "Native AX ref returned by browser_snapshot/browser_find in the current generation. " - "The runtime resolves it without model-generated CSS." - ), - }, - "selector": { - "type": "string", - "description": ( - "Current-generation selector_hint from a probe, or an explicit selector only " - "for a wait condition. " - "Do not combine with another locator strategy." - ), - }, - "role": { - "type": "string", - "description": "ARIA role to locate, e.g. button/textbox/link.", - }, - "name": { - "type": "string", - "description": "Accessible name for role-based locating.", - }, - "label": { - "type": "string", - "description": "Label text for labeled form controls.", - }, - "placeholder": { - "type": "string", - "description": "Placeholder text for form controls.", - }, - "testid": { - "type": "string", - "description": "Exact data-testid value for a target.", - }, - "text": { - "type": "string", - "description": "Visible text for locate/click/wait_for_text.", - }, - "value": { - "type": "string", - "description": "Value to fill/type/select, or autocomplete query.", - }, - "option_text": { - "type": "string", - "description": ( - "Visible option text for autocomplete/select_visible_text/native select label." - ), - }, - "choose_text": { - "type": "string", - "description": "Alias for option_text in autocomplete flows.", - }, - "option_selector": { - "type": "string", - "description": "CSS selector for an autocomplete/dropdown option to choose.", - }, - "choose_selector": { - "type": "string", - "description": "Alias for option_selector.", - }, - "option_role": { - "type": "string", - "description": ("ARIA role for an autocomplete/dropdown option, e.g. option/menuitem."), - }, - "choose_role": { - "type": "string", - "description": "Alias for option_role.", - }, - "option_name": { - "type": "string", - "description": "Accessible name for option_role.", - }, - "choose_name": { - "type": "string", - "description": "Alias for option_name.", - }, - "option_value": { - "type": "string", - "description": "Native select option value. Alias for value when op=select_option.", - }, - "values": { - "type": "array", - "items": {"type": "string"}, - "description": ( - "MCP/Playwright-style native select values list. Also accepted for single-select fields." - ), - }, - "option_label": { - "type": "string", - "description": ("Native select visible label. Alias for option_text when op=select_option."), - }, - "label_value": { - "type": "string", - "description": "Native select visible label alias.", - }, - "index": { - "type": "integer", - "description": "Native select option index.", - }, - "key": { - "type": "string", - "description": "Keyboard key for press, e.g. Enter or Escape.", - }, - "checked": { - "type": "boolean", - "description": "Desired checked state for set_checked.", - }, - "state": { - "type": "string", - "description": ( - "Wait state for wait_for_selector/load_state, e.g. visible/attached/domcontentloaded." - ), - }, - "option_target_id": { - "type": "string", - "description": "PageState target_id for an autocomplete/dropdown option.", - }, - "option_ref": { - "type": "string", - "description": "Current-generation AX ref for an autocomplete/dropdown option.", - }, - "url": { - "type": "string", - "description": "Exact URL expected by wait_for_url.", - }, - "expected_url": { - "type": "string", - "description": "Alias for the exact URL expected by wait_for_url.", - }, - "url_contains": { - "type": "string", - "description": "URL substring expected by wait_for_url.", - }, - "url_pattern": { - "type": "string", - "description": "Regular expression expected to match the current URL.", - }, - "title_contains": { - "type": "string", - "description": "Title substring expected on a newly opened tab.", - }, - "min_tabs": { - "type": "integer", - "description": ( - "Minimum context tab count for wait_for_tab. By default the runtime waits " - "for one more tab than existed when the batch started." - ), - }, - "activate": { - "type": "boolean", - "description": ( - "For wait_for_tab, activate the matched new tab for subsequent steps. Default true." - ), - }, - "expected_text": { - "type": "string", - "description": "Expected first-card title text.", - }, - "attribute": { - "type": "string", - "description": "Attribute inspected by wait_for_sort_state; defaults to aria-sort.", - }, - "expected_value": { - "type": "string", - "description": "Expected attribute or text value for a wait condition.", - }, - "count": { - "type": "integer", - "description": "Exact result count expected.", - }, - "min_count": { - "type": "integer", - "description": "Minimum result count expected.", - }, - "max_count": { - "type": "integer", - "description": "Maximum result count expected.", - }, - "previous_text": { - "type": "string", - "description": "Previous DOM text used by wait_for_dom_text_change.", - }, - "poll_interval_ms": { - "type": "integer", - "description": ( - "Dynamic-condition polling interval, clamped to 50..1000ms; default 100ms. " - "Polling always shares the step's total timeout." - ), - }, - "stable_ms": { - "type": "integer", - "description": "How long a matched value must remain unchanged.", - }, - "field": { - "type": "string", - "description": "Structured output field name for extraction steps.", - }, - "exact": { - "type": "boolean", - "description": "Use exact matching for role/label/text locators.", - }, - "optional": { - "type": "boolean", - "description": "When true, failure records the step but continues.", - }, - "timeout_ms": { - "type": "integer", - "description": "Optional per-step timeout override.", - }, - "delay_ms": { - "type": "integer", - "description": "Optional typing delay in milliseconds.", - }, - "max_chars": { - "type": "integer", - "description": "Maximum characters for extract_text.", - }, - "path": { - "type": "string", - "description": "Screenshot path for op=screenshot.", - }, - "full_page": { - "type": "boolean", - "description": "When true, screenshot the full page.", - }, - "description": { - "type": "string", - "description": "Human-readable purpose for logs/debugging.", - }, - }, - "required": ["op"], - }, - }, - "timeout_ms": { - "type": "integer", - "description": ("Default per-step timeout in milliseconds. Default 2500, clamped to 250..30000."), - }, - "generation_id": { - "type": "string", - "pattern": "^g[0-9]+$", - "description": ( - "Current PageState generation_id returned by navigate, probe, snapshot, or the previous batch. " - "Targets from older generations are rejected before browser execution." - ), - }, - "condition_timeout_ms": { - "type": "integer", - "description": ( - "Default timeout for condition waits in milliseconds. " - "Default 10000, clamped to the action timeout..30000." - ), - }, - "continue_on_error": { - "type": "boolean", - "description": "When true, continue after failed steps and return per-step errors.", - }, - "global_timeout_ms": { - "type": "integer", - "description": ( - "Hard timeout for the whole batch. Default is computed from step count, capped at " - "120000; explicit values are capped at 180000." - ), - }, - "session_id": { - "type": "string", - "description": "Optional browser task session id.", - }, - "request_id": { - "type": "string", - "description": "Optional browser task request id.", - }, - }, - "required": ["steps", "generation_id"], -} - - -class BrowserCancelTool(Tool): - """Cancel an in-progress browser task.""" - - def __init__(self, runtime: "BrowserAgentRuntime", language: str = "cn") -> None: - del language - super().__init__( - ToolCard( - name="browser_cancel_run", - description=_CANCEL_DESC, - input_params=_CANCEL_PARAMS, - ) - ) - self._runtime = runtime - - async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: - del kwargs - await self._runtime.ensure_runtime_ready() - session_id = inputs.get("session_id", "") - request_id = inputs.get("request_id") or None - try: - result = await self._runtime.cancel_run(session_id=session_id, request_id=request_id) - return ToolOutput( - success=bool(result.get("ok", True)), - data=result, - error=result.get("error"), - ) - except Exception as exc: - return ToolOutput(success=False, error=str(exc)) - - async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: - del inputs, kwargs - if False: - yield None - - -class BrowserClearCancelTool(Tool): - """Clear a cancellation flag for a browser task.""" - - def __init__(self, runtime: "BrowserAgentRuntime", language: str = "cn") -> None: - del language - super().__init__( - ToolCard( - name="browser_clear_cancel", - description=_CLEAR_CANCEL_DESC, - input_params=_CLEAR_CANCEL_PARAMS, - ) - ) - self._runtime = runtime - - async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: - del kwargs - await self._runtime.ensure_runtime_ready() - session_id = inputs.get("session_id", "") - request_id = inputs.get("request_id") or None - try: - result = await self._runtime.clear_cancel(session_id=session_id, request_id=request_id) - return ToolOutput( - success=bool(result.get("ok", True)), - data=result, - error=result.get("error"), - ) - except Exception as exc: - return ToolOutput(success=False, error=str(exc)) - - async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: - del inputs, kwargs - if False: - yield None - - -class BrowserCustomActionTool(Tool): - """Run a registered custom browser action.""" - - def __init__(self, runtime: "BrowserAgentRuntime", language: str = "cn") -> None: - del language - super().__init__( - ToolCard( - name="browser_custom_action", - description=_CUSTOM_ACTION_DESC, - input_params=_CUSTOM_ACTION_PARAMS, - ) - ) - self._runtime = runtime - - async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: - del kwargs - action = inputs.get("action", "") - session_id = (inputs.get("session_id") or "").strip() or _ctx_parent_session_id.get() - request_id = (inputs.get("request_id") or "").strip() or _ctx_parent_request_id.get() - params: Dict[str, Any] = inputs.get("params") or {} - try: - result = await self._runtime.run_custom_action( - action=action, - session_id=session_id, - request_id=request_id, - params=params, - ) - return ToolOutput( - success=bool(result.get("ok", True)), - data=result, - error=result.get("error"), - ) - except Exception as exc: - return ToolOutput(success=False, error=str(exc)) - - async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: - del inputs, kwargs - if False: - yield None - - -class BrowserListActionsTool(Tool): - """List available custom browser actions.""" - - def __init__(self, runtime: "BrowserAgentRuntime", language: str = "cn") -> None: - del language - super().__init__( - ToolCard( - name="browser_list_custom_actions", - description=_LIST_ACTIONS_DESC, - input_params=_LIST_ACTIONS_PARAMS, - ) - ) - self._runtime = runtime - - async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: - del inputs, kwargs - try: - data = await self._runtime.list_actions() - return ToolOutput(success=True, data=data) - except Exception as exc: - return ToolOutput(success=False, error=str(exc)) - - async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: - del inputs, kwargs - if False: - yield None - - -class BrowserProbeInteractivesTool(Tool): - """Compact visible-interactive-element probe.""" - - def __init__(self, runtime: "BrowserAgentRuntime", language: str = "cn") -> None: - del language - super().__init__( - ToolCard( - name="browser_probe_interactives", - description=_PROBE_INTERACTIVES_DESC, - input_params=_PROBE_INTERACTIVES_PARAMS, - ) - ) - self._runtime = runtime - - async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: - del kwargs - - try: - max_items = int(inputs.get("max_items", 30)) - except (TypeError, ValueError): - max_items = 30 - max_items = max(1, min(40, max_items)) - - viewport_only_raw = inputs.get("viewport_only", True) - if isinstance(viewport_only_raw, str): - viewport_only = viewport_only_raw.strip().lower() not in {"0", "false", "no"} - else: - viewport_only = bool(viewport_only_raw) - - query = str(inputs.get("query") or "").strip() - - try: - data = await self._runtime.probe_interactives( - max_items=max_items, - viewport_only=viewport_only, - query=query, - ) - return ToolOutput( - success=bool(data.get("ok", True)), - data=data, - error=data.get("error"), - ) - except Exception as exc: - return ToolOutput(success=False, error=str(exc)) - - async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: - del inputs, kwargs - if False: - yield None - - -class BrowserProbeCardsTool(Tool): - """Compact repeated-card/listing probe.""" - - def __init__(self, runtime: "BrowserAgentRuntime", language: str = "cn") -> None: - del language - super().__init__( - ToolCard( - name="browser_probe_cards", - description=_PROBE_CARDS_DESC, - input_params=_PROBE_CARDS_PARAMS, - ) - ) - self._runtime = runtime - - async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: - del kwargs - - try: - max_cards = int(inputs.get("max_cards", 12)) - except (TypeError, ValueError): - max_cards = 12 - max_cards = max(1, min(20, max_cards)) - - viewport_only_raw = inputs.get("viewport_only", True) - if isinstance(viewport_only_raw, str): - viewport_only = viewport_only_raw.strip().lower() not in {"0", "false", "no"} - else: - viewport_only = bool(viewport_only_raw) - - include_buttons_raw = inputs.get("include_buttons", True) - if isinstance(include_buttons_raw, str): - include_buttons = include_buttons_raw.strip().lower() not in { - "0", - "false", - "no", - } - else: - include_buttons = bool(include_buttons_raw) - - query = str(inputs.get("query") or "").strip() - - try: - data = await self._runtime.probe_cards( - max_cards=max_cards, - viewport_only=viewport_only, - include_buttons=include_buttons, - query=query, - ) - return ToolOutput( - success=bool(data.get("ok", True)), - data=data, - error=data.get("error"), - ) - except Exception as exc: - return ToolOutput(success=False, error=str(exc)) - - async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: - del inputs, kwargs - if False: - yield None - - -class BrowserBatchInteractTool(Tool): - def __init__(self, runtime: "BrowserAgentRuntime", language: str = "cn") -> None: - del language - super().__init__( - ToolCard( - name="browser_batch_interact", - description=_BATCH_INTERACT_DESC, - input_params=_BATCH_INTERACT_PARAMS, - ) - ) - self._runtime = runtime - - async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: - del kwargs - - steps = inputs.get("steps") - session_id = (inputs.get("session_id") or "").strip() or _ctx_parent_session_id.get() - request_id = (inputs.get("request_id") or "").strip() or _ctx_parent_request_id.get() - - try: - result = await self._runtime.batch_interact( - steps=steps, - generation_id=str(inputs.get("generation_id") or ""), - timeout_ms=inputs.get("timeout_ms"), - condition_timeout_ms=inputs.get("condition_timeout_ms"), - wait_after_each_ms=inputs.get("wait_after_each_ms"), - continue_on_error=bool(inputs.get("continue_on_error", False)), - global_timeout_ms=inputs.get("global_timeout_ms"), - session_id=session_id, - request_id=request_id, - ) - return ToolOutput( - success=bool(result.get("ok", True)), - data=result, - error=result.get("error"), - ) - except Exception as exc: - return ToolOutput(success=False, error=str(exc)) - - async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: - del inputs, kwargs - if False: - yield None - - -class BrowserRuntimeHealthTool(Tool): - """Return runtime readiness and heartbeat metadata.""" - - def __init__(self, runtime: "BrowserAgentRuntime", language: str = "cn") -> None: - del language - super().__init__( - ToolCard( - name="browser_runtime_health", - description=_RUNTIME_HEALTH_DESC, - input_params=_RUNTIME_HEALTH_PARAMS, - ) - ) - self._runtime = runtime - - async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: - del inputs, kwargs - try: - data = await self._runtime.runtime_health() - return ToolOutput(success=True, data=data) - except Exception as exc: - return ToolOutput(success=False, error=str(exc)) - - async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: - del inputs, kwargs - if False: - yield None - - -def build_browser_runtime_tools( - runtime: "BrowserAgentRuntime", - language: str = "cn", -) -> List[Tool]: - """Build browser helper tools backed by ``BrowserAgentRuntime``. - - By default this returns deterministic helper tools only. The browser subagent - continues to use Playwright MCP primitive tools directly for low-level browser - actions. - """ - - return [ - BrowserProbeInteractivesTool(runtime, language), - BrowserProbeCardsTool(runtime, language), - BrowserBatchInteractTool(runtime, language), - ] diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/service.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/service.py deleted file mode 100644 index 613753f73d..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/service.py +++ /dev/null @@ -1,1918 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. - -"""Browser backend service with sticky sessions and guardrails.""" - -from __future__ import annotations - -import asyncio -import base64 -import contextvars -import json -import mimetypes -import os -import shutil -import socket -import uuid -from dataclasses import dataclass, field -from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional -from urllib.request import urlopen - -from openjiuwen.core.foundation.store.base_kv_store import BaseKVStore -from openjiuwen.core.foundation.store.kv.in_memory_kv_store import InMemoryKVStore -from openjiuwen.core.foundation.tool import McpServerConfig -from openjiuwen.core.runner import Runner -from openjiuwen.core.single_agent.agents.react_agent import ReActAgent - -from ..drivers.managed_browser import ManagedBrowserDriver, _default_chrome_user_data_dir -from ..utils.parsing import extract_json_object -from .agents import build_browser_worker_agent -from .config import BrowserInstanceConfig, BrowserRunGuardrails, parse_command_args, resolve_playwright_mcp_cwd -from .profiles import BrowserProfile, BrowserProfileStore -from .service_registry import BROWSER_SERVICE_REGISTRY, BrowserServiceIdentity - -MAX_ITERATION_MESSAGE = "Max iterations reached without completion" -_ctx_observer_session_id: contextvars.ContextVar[str] = contextvars.ContextVar( - "playwright_runtime_observer_session_id", - default="", -) -_ctx_observer_request_id: contextvars.ContextVar[str] = contextvars.ContextVar( - "playwright_runtime_observer_request_id", - default="", -) - - -@dataclass -class BrowserTaskProgressState: - request_id: str = "" - status: str = "unknown" - completed_steps: list[str] = field(default_factory=list) - remaining_steps: list[str] = field(default_factory=list) - next_step: str = "" - completion_evidence: list[str] = field(default_factory=list) - missing_requirements: list[str] = field(default_factory=list) - recent_tool_steps: list[str] = field(default_factory=list) - last_page_url: str = "" - last_page_title: str = "" - last_screenshot: Any = None - last_worker_final: str = "" - - @classmethod - def from_dict(cls, data: Optional[Dict[str, Any]]) -> "BrowserTaskProgressState": - if not isinstance(data, dict): - return cls() - last_page = data.get("last_page") if isinstance(data.get("last_page"), dict) else {} - return cls( - request_id=str(data.get("request_id") or "").strip(), - status=str(data.get("status") or "unknown").strip() or "unknown", - completed_steps=[str(item).strip() for item in data.get("completed_steps") or [] if str(item).strip()], - remaining_steps=[str(item).strip() for item in data.get("remaining_steps") or [] if str(item).strip()], - next_step=str(data.get("next_step") or "").strip(), - completion_evidence=[ - str(item).strip() for item in data.get("completion_evidence") or [] if str(item).strip() - ], - missing_requirements=[ - str(item).strip() for item in data.get("missing_requirements") or [] if str(item).strip() - ], - recent_tool_steps=[str(item).strip() for item in data.get("recent_tool_steps") or [] if str(item).strip()], - last_page_url=str(last_page.get("url") or "").strip(), - last_page_title=str(last_page.get("title") or "").strip(), - last_screenshot=data.get("last_screenshot"), - last_worker_final=str(data.get("last_worker_final") or "").strip(), - ) - - @classmethod - def from_task_state(cls, data: Optional[Dict[str, Any]]) -> "BrowserTaskProgressState": - """Build the legacy progress DTO from runtime-owned task state.""" - - if not isinstance(data, dict) or not data: - return cls() - completed_steps, remaining_steps, completion_evidence = cls._phase_progress(data) - completion_evidence.extend(cls._evidence_summaries(data)) - recent_steps = cls._recent_action_summaries(data) - current_phase = str(data.get("current_phase") or "").strip() - next_action = str(data.get("next_action_class") or current_phase).strip() - last_page = data.get("last_page") if isinstance(data.get("last_page"), dict) else {} - return cls( - request_id=str(data.get("task_id") or "").strip(), - status=str(data.get("status") or "unknown").strip() or "unknown", - completed_steps=completed_steps, - remaining_steps=remaining_steps, - next_step=next_action, - completion_evidence=completion_evidence[-10:], - missing_requirements=[str(item) for item in data.get("blockers") or [] if str(item).strip()], - recent_tool_steps=recent_steps, - last_page_url=str(last_page.get("url") or "").strip(), - last_page_title=str(last_page.get("title") or "").strip(), - last_worker_final=str(data.get("last_worker_final") or "").strip(), - ) - - @staticmethod - def _phase_progress(data: Dict[str, Any]) -> tuple[list[str], list[str], list[str]]: - phases = data.get("phases") if isinstance(data.get("phases"), dict) else {} - completed_steps: list[str] = [] - remaining_steps: list[str] = [] - completion_evidence: list[str] = [] - for phase_name, details in phases.items(): - if not isinstance(details, dict): - continue - phase_label = str(phase_name).strip() - if details.get("status") == "completed": - completed_steps.append(phase_label) - evidence = str(details.get("completion_evidence") or "").strip() - if evidence: - completion_evidence.append(f"{phase_label}: {evidence}") - else: - remaining_steps.append(phase_label) - return completed_steps, remaining_steps, completion_evidence - - @staticmethod - def _evidence_summaries(data: Dict[str, Any]) -> list[str]: - summaries: list[str] = [] - evidence_records = data.get("structured_evidence") - if isinstance(evidence_records, list): - for record in evidence_records[-3:]: - if not isinstance(record, dict): - continue - fields = ", ".join(str(item) for item in record.get("fields") or []) - if fields: - summaries.append(f"{record.get('kind') or 'evidence'}: {fields}") - evidence_slots = data.get("evidence_slots") - if isinstance(evidence_slots, list): - for slot in evidence_slots[-4:]: - if not isinstance(slot, dict): - continue - label = "/".join( - str(slot.get(key) or "") for key in ("entity", "variant", "field") - ).strip("/") - value = str(slot.get("value") or "").strip() - if label and value: - summaries.append(f"{label}: {value[:160]}") - return summaries - - @staticmethod - def _recent_action_summaries(data: Dict[str, Any]) -> list[str]: - recent_steps: list[str] = [] - recent_actions = data.get("recent_actions") - if isinstance(recent_actions, list): - for action in recent_actions[-6:]: - if not isinstance(action, dict): - continue - summary_parts = ( - str(action.get("action_class") or "").strip(), - str(action.get("target_summary") or "").strip(), - str(action.get("outcome") or "").strip(), - str(action.get("semantic_delta") or "").strip(), - ) - summary = " ".join(filter(None, summary_parts)) - if summary: - recent_steps.append(summary[:300]) - return recent_steps - - def is_empty(self) -> bool: - return ( - self.status == "unknown" - and not self.completed_steps - and not self.remaining_steps - and not self.next_step - and not self.completion_evidence - and not self.missing_requirements - and not self.recent_tool_steps - and not self.last_page_url - and not self.last_page_title - and not self.last_worker_final - and self.last_screenshot in (None, "") - ) - - def to_dict(self) -> Dict[str, Any]: - return { - "status": self.status, - "completed_steps": list(self.completed_steps), - "remaining_steps": list(self.remaining_steps), - "next_step": self.next_step or None, - "completion_evidence": list(self.completion_evidence), - "missing_requirements": list(self.missing_requirements), - "recent_tool_steps": list(self.recent_tool_steps), - "last_page": { - "url": self.last_page_url, - "title": self.last_page_title, - }, - "last_screenshot": self.last_screenshot, - "last_worker_final": self.last_worker_final or None, - "request_id": self.request_id or None, - } - - -class BrowserService: - """Backend browser service with sticky logical sessions.""" - - def __init__( - self, - provider: str, - api_key: str, - api_base: str, - model_name: str, - mcp_cfg: McpServerConfig, - guardrails: BrowserRunGuardrails, - cancel_store: Optional[BaseKVStore] = None, - instance: Optional[BrowserInstanceConfig] = None, - allowed_tool_names: Optional[Iterable[str]] = None, - ) -> None: - self.provider = provider - self.api_key = api_key - self.api_base = api_base - self.model_name = model_name - self.mcp_cfg = mcp_cfg - self.guardrails = guardrails - self._instance = instance or BrowserInstanceConfig() - self._allowed_tool_names = ( - None - if allowed_tool_names is None - else tuple(dict.fromkeys(allowed_tool_names)) - ) - self._cancel_store: BaseKVStore = cancel_store or InMemoryKVStore() - - self.started = False - self._browser_agent: Optional[ReActAgent] = None - self._locks: Dict[str, asyncio.Lock] = {} - self._sessions: set[str] = set() - self._inflight_tasks: Dict[str, set[asyncio.Task[Any]]] = {} - self._screenshot_subdir = "screenshots" - self._artifacts_subdir = "artifacts" - self._mcp_cwd = self._resolve_mcp_cwd() - self._screenshots_dir = self._mcp_cwd / self._screenshot_subdir - self._artifacts_dir = self._mcp_cwd / self._artifacts_subdir - self._profile_store = BrowserProfileStore(self._resolve_profile_store_path()) - self._profile_name = self._resolve_profile_name() - self._driver_mode = self._resolve_driver_mode() - self._active_profile: Optional[BrowserProfile] = None - self._managed_driver: Optional[ManagedBrowserDriver] = None - self._registered_cdp_endpoint: str = "" - self._failure_context_by_session: Dict[str, str] = {} - self._progress_by_session: Dict[str, BrowserTaskProgressState] = {} - self._heartbeat_task: Optional[asyncio.Task[Any]] = None - self._heartbeat_interval: float = 30.0 - self._connection_healthy: bool = False - self._last_heartbeat_ok: Optional[float] = None - self._lifecycle_identity = self._build_lifecycle_identity() - self._registry_acquired = False - self._task_binding_ref_count = 0 - - @property - def browser_agent(self) -> Optional[ReActAgent]: - return self._browser_agent - - @browser_agent.setter - def browser_agent(self, value: Optional[ReActAgent]) -> None: - self._browser_agent = value - - @property - def allowed_tool_names(self) -> Optional[tuple[str, ...]]: - """Return the normalized task-scoped Playwright tool allowlist.""" - return self._allowed_tool_names - - @property - def artifacts_subdir(self) -> str: - return self._artifacts_subdir - - @property - def connection_healthy(self) -> bool: - return self._connection_healthy - - @property - def last_heartbeat_ok(self) -> Optional[float]: - return self._last_heartbeat_ok - - def _resolve_profile_store_path(self) -> Path: - configured = (os.getenv("BROWSER_PROFILE_STORE_PATH") or "").strip() - if configured: - return Path(configured).expanduser() - return self._mcp_cwd / ".browser" / "profiles.json" - - @staticmethod - def _legacy_profile_store_path() -> Path: - return Path.home() / ".jiuwenclaw" / "browser-move" / ".browser" / "profiles.json" - - def _iter_profile_store_paths(self) -> list[Path]: - primary = self._resolve_profile_store_path() - paths = [primary] - configured = (os.getenv("BROWSER_PROFILE_STORE_PATH") or "").strip() - if not configured: - legacy = self._legacy_profile_store_path() - if legacy.resolve() != primary.resolve(): - paths.append(legacy) - return paths - - def _resolve_profile_name(self) -> str: - """Resolve the active profile name (per-instance key wins over env). - - Distinct browser keys map to distinct profile names so the managed - user-data-dir and persisted port never collide across agents. - """ - instance = self._instance - name = "" - if instance: - name = (instance.profile_name or instance.sanitized_key() or "").strip() - if not name: - name = (os.getenv("BROWSER_PROFILE_NAME") or "").strip() - return name or "jiuwenclaw" - - def _resolve_driver_mode(self) -> str: - explicit = (self._instance.driver_mode or "").strip().lower() if self._instance else "" - if not explicit: - explicit = (os.getenv("BROWSER_DRIVER") or "").strip().lower() - if explicit: - if explicit not in {"remote", "managed", "extension"}: - raise ValueError("BROWSER_DRIVER must be one of: remote, managed, extension") - return explicit - return "remote" - - def _build_lifecycle_identity(self) -> BrowserServiceIdentity: - """Build the process resource identity for this browser instance.""" - instance = self._instance - browser_key = instance.sanitized_key() if instance else "" - managed_args = tuple(parse_command_args(os.getenv("BROWSER_MANAGED_ARGS") or "")) - headless = any( - arg == "--headless" or arg.startswith("--headless=") - for arg in managed_args - ) - if self._driver_mode == "managed": - display_mode = "headless" if headless else "headed" - else: - display_mode = self._driver_mode - - explicit_user_data_dir = str( - (instance.user_data_dir if instance else "") - or os.getenv("BROWSER_MANAGED_USER_DATA_DIR") - or "" - ).strip() - normalized_user_data_dir = ( - os.path.normcase( - os.path.normpath(str(Path(explicit_user_data_dir).expanduser())) - ) - if explicit_user_data_dir - else f"profile:{self._profile_name}" - ) - server_id = (self.mcp_cfg.server_id or "").strip() or self.mcp_cfg.server_name - return BrowserServiceIdentity( - browser_key=browser_key, - profile_name=self._profile_name, - driver_mode=self._driver_mode, - display_mode=display_mode, - managed_args=managed_args, - browser_binary=self.normalize_browser_binary( - self._configured_browser_binary() - ), - user_data_dir=normalized_user_data_dir, - cdp_endpoint=self._configured_cdp_endpoint(), - server_id=server_id, - ) - - @property - def lifecycle_identity(self) -> BrowserServiceIdentity: - return self._lifecycle_identity - - def _acquire_registry_lease(self) -> None: - if self._registry_acquired: - return - BROWSER_SERVICE_REGISTRY.acquire(self._lifecycle_identity, self) - self._registry_acquired = True - - def acquire_task_binding(self) -> None: - """Acquire one task reference to the shared browser process resources.""" - self._task_binding_ref_count += 1 - self._acquire_registry_lease() - - @staticmethod - def _allocate_free_port() -> int: - """Reserve an OS-assigned free TCP port for a managed Chrome launch. - - Uses bind-to-0 so concurrent BrowserService instances never pick the - same port. The socket is closed immediately and the port handed to the - managed driver, which launches Chrome on it right away; the driver's - endpoint-ready check absorbs the small TOCTOU window. - """ - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.bind(("127.0.0.1", 0)) - return int(sock.getsockname()[1]) - - @staticmethod - def _cancel_key(session_id: str, request_id: Optional[str] = None) -> str: - rid = (request_id or "").strip() or "*" - return f"playwright_runtime:cancel:{session_id}:{rid}" - - @staticmethod - def _inflight_key(session_id: str, request_id: Optional[str] = None) -> str: - rid = (request_id or "").strip() - return f"{session_id}:{rid}" if rid else session_id - - def _register_inflight_task(self, session_id: str, request_id: str, task: asyncio.Task[Any]) -> None: - keys = (self._inflight_key(session_id), self._inflight_key(session_id, request_id)) - for key in keys: - self._inflight_tasks.setdefault(key, set()).add(task) - - def _unregister_inflight_task(self, session_id: str, request_id: str, task: asyncio.Task[Any]) -> None: - keys = (self._inflight_key(session_id), self._inflight_key(session_id, request_id)) - for key in keys: - tasks = self._inflight_tasks.get(key) - if not tasks: - continue - tasks.discard(task) - if not tasks: - self._inflight_tasks.pop(key, None) - - def _resolve_mcp_cwd(self) -> Path: - params = getattr(self.mcp_cfg, "params", {}) or {} - raw = str(params.get("cwd", "")).strip() - if raw: - return Path(raw).expanduser() - return Path(resolve_playwright_mcp_cwd()).expanduser() - - @staticmethod - def _is_cdp_endpoint_ready(endpoint: str) -> bool: - base = str(endpoint or "").strip().rstrip("/") - if not base: - return False - try: - with urlopen(f"{base}/json/version", timeout=1.5) as response: # nosec B310 - payload = json.loads(response.read().decode("utf-8", errors="ignore")) - if isinstance(payload, dict): - return bool(payload.get("webSocketDebuggerUrl") or payload.get("Browser")) - except (OSError, ValueError): - return False - return False - - def _resolve_existing_cdp_profile(self) -> Optional[BrowserProfile]: - # A keyed instance must only ever reuse a browser whose profile name - # matches its own key. The shared store tracks a single global - # selected_profile; adopting it would let whoever starts second attach - # to another key's live Chrome, collapsing the per-key isolation. - keyed = bool(self._instance and self._instance.key) - candidates: list[BrowserProfile] = [] - for store_path in self._iter_profile_store_paths(): - store = BrowserProfileStore(store_path) - if not keyed: - selected = store.selected_profile() - if selected is not None: - candidates.append(selected) - named = store.get_profile(self._profile_name) - if named is not None and all(named.name != item.name for item in candidates): - candidates.append(named) - - _expected_extra_args = parse_command_args(os.getenv("BROWSER_MANAGED_ARGS") or "") - expected_binary = self.normalize_browser_binary(self._configured_browser_binary()) - for profile in candidates: - endpoint = str(profile.cdp_url or "").strip() - if not endpoint or not self._is_cdp_endpoint_ready(endpoint): - continue - # Don't reuse a live Chrome whose launch args differ from the current - # BROWSER_MANAGED_ARGS — e.g. a headed Chrome when headless is now on. - if profile.extra_args != _expected_extra_args: - continue - if self.normalize_browser_binary(profile.browser_binary) != expected_binary: - continue - try: - self._profile_store.upsert_profile(profile, select=True) - except Exception: - pass - return profile - return None - - def _should_replace_managed_driver(self, profile: BrowserProfile) -> bool: - if self._managed_driver is None: - return False - if profile.driver_type != "managed": - return True - if self._active_profile is None: - return True - current_endpoint = str(self._active_profile.cdp_url or "").strip() - target_endpoint = str(profile.cdp_url or "").strip() - return current_endpoint != target_endpoint - - def _configured_cdp_endpoint(self) -> str: - params = getattr(self.mcp_cfg, "params", {}) or {} - env_map = dict(params.get("env", {}) or {}) - return str(env_map.get("PLAYWRIGHT_MCP_CDP_ENDPOINT") or "").strip() - - def _configured_browser_binary(self) -> str: - instance_binary = self._instance.browser_binary if self._instance else "" - return str(instance_binary or os.getenv("BROWSER_MANAGED_BINARY") or "").strip() - - @staticmethod - def normalize_browser_binary(value: str) -> str: - raw = str(value or "").strip() - if not raw: - return "" - return os.path.normcase(os.path.normpath(str(Path(raw).expanduser()))) - - def _resolve_managed_port(self) -> int: - """Resolve the managed Chrome debug port. - - Precedence: explicit per-instance port > (keyed: auto-allocate a free - port; legacy/unkeyed: env BROWSER_MANAGED_PORT, else 9333). A global env - port cannot serve multiple keys, so keyed instances ignore it in favor of - auto-allocation to avoid silently colliding on one browser. - """ - instance = self._instance - if instance and instance.managed_port > 0: - return instance.managed_port - if instance and instance.key: - return self._allocate_free_port() - port_raw = (os.getenv("BROWSER_MANAGED_PORT") or "9333").strip() - try: - port = int(port_raw) - if port <= 0: - raise ValueError - except ValueError as exc: - raise ValueError(f"Invalid BROWSER_MANAGED_PORT: {port_raw}") from exc - return port - - def _build_managed_profile(self) -> BrowserProfile: - instance = self._instance - host = (os.getenv("BROWSER_MANAGED_HOST") or "127.0.0.1").strip() or "127.0.0.1" - port = self._resolve_managed_port() - - kill_existing_raw = (os.getenv("BROWSER_MANAGED_KILL_EXISTING") or "").strip().lower() - kill_existing = kill_existing_raw in {"1", "true", "yes", "on"} - explicit_user_data_dir = ( - (instance.user_data_dir if instance else "") or (os.getenv("BROWSER_MANAGED_USER_DATA_DIR") or "") - ).strip() - if explicit_user_data_dir: - user_data_dir = explicit_user_data_dir - elif kill_existing: - user_data_dir = _default_chrome_user_data_dir() - else: - user_data_dir = str(self._mcp_cwd / ".browser-profiles" / self._profile_name) - browser_binary = self._configured_browser_binary() - extra_args = parse_command_args(os.getenv("BROWSER_MANAGED_ARGS") or "") - cdp_url = f"http://{host}:{port}" - return BrowserProfile( - name=self._profile_name, - driver_type="managed", - cdp_url=cdp_url, - browser_binary=browser_binary, - user_data_dir=user_data_dir, - debug_port=port, - host=host, - extra_args=extra_args, - ) - - def _inject_cdp_endpoint(self, endpoint: str) -> None: - params = dict(getattr(self.mcp_cfg, "params", {}) or {}) - env_map = dict(params.get("env", {}) or {}) - env_map["PLAYWRIGHT_MCP_CDP_ENDPOINT"] = endpoint - env_map.setdefault("PLAYWRIGHT_MCP_BROWSER", "chrome") - env_map.pop("PLAYWRIGHT_MCP_DEVICE", None) - params["env"] = env_map - self.mcp_cfg.params = params - - async def _ensure_managed_driver_started(self) -> bool: - if self._driver_mode != "managed": - return False - previous_endpoint = self._configured_cdp_endpoint() - reusable_profile = self._resolve_existing_cdp_profile() - if reusable_profile is not None: - replaced_driver = False - if self._should_replace_managed_driver(reusable_profile): - await self._stop_managed_driver() - replaced_driver = True - self._active_profile = reusable_profile - endpoint = str(reusable_profile.cdp_url or "").strip() - self._inject_cdp_endpoint(endpoint) - return replaced_driver or endpoint != previous_endpoint - - if self._managed_driver is not None: - ready = await asyncio.to_thread(self._managed_driver.is_endpoint_ready) - if ready: - return False - await self._stop_managed_driver() - - profile = self._profile_store.get_profile(self._profile_name) - _expected_extra_args = parse_command_args(os.getenv("BROWSER_MANAGED_ARGS") or "") - if ( - profile is None - or profile.driver_type != "managed" - or profile.debug_port <= 0 - or not str(profile.user_data_dir).strip() - or profile.extra_args != _expected_extra_args - ): - profile = self._build_managed_profile() - configured_binary = self._configured_browser_binary() - profile.browser_binary = configured_binary - self._profile_store.upsert_profile(profile, select=True) - self._active_profile = profile - - kill_existing_raw = (os.getenv("BROWSER_MANAGED_KILL_EXISTING") or "").strip().lower() - kill_existing = kill_existing_raw in {"1", "true", "yes", "on"} - - driver = ManagedBrowserDriver(profile=profile) - endpoint = await asyncio.to_thread(driver.start, 30.0, kill_existing) - self._inject_cdp_endpoint(endpoint) - profile.cdp_url = endpoint - self._profile_store.upsert_profile(profile, select=True) - self._managed_driver = driver - BROWSER_SERVICE_REGISTRY.register_managed_driver( - self._lifecycle_identity, - self, - driver, - ) - return True - - async def _stop_managed_driver(self) -> None: - if self._managed_driver is None: - return - driver = self._managed_driver - self._managed_driver = None - try: - await asyncio.to_thread(driver.stop) - finally: - BROWSER_SERVICE_REGISTRY.unregister_managed_driver( - self._lifecycle_identity, - driver, - ) - - def _ensure_screenshots_dir(self) -> None: - self._screenshots_dir.mkdir(parents=True, exist_ok=True) - - def _resolve_local_screenshot_path(self, raw_path: str) -> Path: - path = Path(raw_path).expanduser() - candidates: List[Path] = [] - if path.is_absolute(): - candidates.append(path) - else: - candidates.extend( - [ - self._mcp_cwd / path, - Path.cwd() / path, - path, - ] - ) - - for candidate in candidates: - if candidate.exists() and candidate.is_file(): - return candidate - return path - - def _ensure_screenshot_in_folder(self, source_path: Path) -> Path: - if not source_path.exists() or not source_path.is_file(): - return source_path - - self._ensure_screenshots_dir() - try: - source_resolved = source_path.resolve() - target_dir_resolved = self._screenshots_dir.resolve() - try: - source_resolved.relative_to(target_dir_resolved) - return source_resolved - except ValueError: - pass - - target_path = self._screenshots_dir / source_path.name - if target_path.exists(): - target_resolved = target_path.resolve() - if target_resolved != source_resolved: - target_path = self._screenshots_dir / ( - f"{source_path.stem}-{uuid.uuid4().hex[:8]}{source_path.suffix}" - ) - - shutil.copy2(source_path, target_path) - return target_path - except Exception: - return source_path - - def _normalize_screenshot_value(self, screenshot: Any) -> Any: - """Normalize screenshot for downstream multimodal APIs. - - - Keep remote URLs and existing data URLs as-is. - - Ensure local screenshots are copied into screenshots/ folder. - - Convert local image file paths to data URLs. - """ - if screenshot is None or not isinstance(screenshot, str): - return screenshot - - raw = screenshot.strip() - if not raw: - return None - - lowered = raw.lower() - if lowered.startswith(("http://", "https://", "data:image/")): - return raw - - local_path_str = raw[7:] if lowered.startswith("file://") else raw - local_path = self._resolve_local_screenshot_path(local_path_str) - if not local_path.exists() or not local_path.is_file(): - return raw - local_path = self._ensure_screenshot_in_folder(local_path) - - mime_type, _ = mimetypes.guess_type(str(local_path)) - if not mime_type or not mime_type.startswith("image/"): - return raw - - try: - encoded = base64.b64encode(local_path.read_bytes()).decode("ascii") - except Exception: - return raw - return f"data:{mime_type};base64,{encoded}" - - @staticmethod - def _is_retryable_transport_message(text: str) -> bool: - lowered = str(text or "").lower() - markers = ( - "session terminated", - "not connected", - "endofstream", - "closedresourceerror", - "brokenresourceerror", - "stream closed", - "connection closed", - "broken pipe", - "remoteprotocolerror", - "readerror", - "writeerror", - ) - return any(marker in lowered for marker in markers) - - @classmethod - def _is_retryable_transport_error(cls, exc: Exception) -> bool: - name = type(exc).__name__.lower() - text = str(exc).lower() - return cls._is_retryable_transport_message(name) or cls._is_retryable_transport_message(text) - - @staticmethod - def _is_retryable_runtime_result(parsed: Dict[str, Any]) -> bool: - if not isinstance(parsed, dict) or bool(parsed.get("ok", False)): - return False - text = (f"{parsed.get('error', '')}\n{parsed.get('final', '')}").lower() - markers = ( - "frame has been detached", - "execution context was destroyed", - "target page, context or browser has been closed", - "target closed", - "navigation failed because browser has disconnected", - "context closed", - "page crashed", - "net::err_network_changed", - "net::err_internet_disconnected", - ) - return any(marker in text for marker in markers) - - @staticmethod - def _should_restart_after_runtime_result(parsed: Dict[str, Any]) -> bool: - if not isinstance(parsed, dict): - return False - text = (f"{parsed.get('error', '')}\n{parsed.get('final', '')}").lower() - restart_markers = ( - "frame has been detached", - "target page, context or browser has been closed", - "target closed", - "context closed", - "page crashed", - ) - return any(marker in text for marker in restart_markers) - - async def request_cancel(self, session_id: str, request_id: Optional[str] = None) -> None: - sid = (session_id or "").strip() - if not sid: - raise ValueError("session_id is required for cancellation") - await self._cancel_store.set(self._cancel_key(sid, request_id), "1") - - request_id_clean = (request_id or "").strip() - if request_id_clean: - keys = [self._inflight_key(sid, request_id_clean)] - else: - keys = [self._inflight_key(sid)] - - for key in keys: - for task in list(self._inflight_tasks.get(key, set())): - if not task.done(): - task.cancel() - - async def clear_cancel(self, session_id: str, request_id: Optional[str] = None) -> None: - sid = (session_id or "").strip() - if not sid: - return - if request_id: - await self._cancel_store.delete(self._cancel_key(sid, request_id)) - return - await self._cancel_store.delete(self._cancel_key(sid, "*")) - - async def is_cancelled(self, session_id: str, request_id: Optional[str] = None) -> bool: - sid = (session_id or "").strip() - if not sid: - return False - if request_id: - exact = await self._cancel_store.get(self._cancel_key(sid, request_id)) - if exact is not None: - return True - wildcard = await self._cancel_store.get(self._cancel_key(sid, "*")) - return wildcard is not None - - def session_new(self, session_id: Optional[str] = None) -> str: - sid = (session_id or "").strip() or f"browser-{uuid.uuid4().hex}" - self._sessions.add(sid) - if sid not in self._locks: - self._locks[sid] = asyncio.Lock() - return sid - - def _validate_mcp_command(self) -> str: - command = str(self.mcp_cfg.params.get("command") or "").strip() - if not command: - raise RuntimeError( - "Playwright MCP command is empty. Configure PLAYWRIGHT_MCP_COMMAND " - "or restore the bundled runtime settings." - ) - command_path = Path(command).expanduser() - if command_path.is_absolute(): - resolved = str(command_path) - else: - resolved = shutil.which(command) - if resolved is None: - raise RuntimeError( - f"Playwright MCP command '{command}' was not found in PATH. " - "Install Node.js 20+ or configure an absolute executable path." - ) - resolved_path = Path(os.path.abspath(resolved)) - if not resolved_path.is_file() or not os.access(resolved_path, os.X_OK): - raise RuntimeError( - f"Playwright MCP command is not an executable file: {resolved_path}" - ) - return str(resolved_path) - - async def ensure_runtime_ready(self) -> None: - if self._task_binding_ref_count <= 0: - # Standalone BrowserAgentRuntime usage has no outer TaskTool hook. - self.acquire_task_binding() - self._acquire_registry_lease() - if self.started: - browser_rebound = await self._ensure_managed_driver_started() - configured_endpoint = self._configured_cdp_endpoint() - if browser_rebound or configured_endpoint != self._registered_cdp_endpoint: - await self._refresh_mcp_server_binding() - self._browser_agent = None - return - - self.mcp_cfg.params["command"] = self._validate_mcp_command() - - from .browser_tools import ensure_browser_runtime_client_patch - - ensure_browser_runtime_client_patch() - - await self._ensure_managed_driver_started() - self._ensure_screenshots_dir() - await Runner.start() - - register_result = await Runner.resource_mgr.add_mcp_server(self.mcp_cfg, tag="browser.service") - if register_result is not None and not getattr(register_result, "is_ok", lambda: False)(): - if hasattr(register_result, "error") and callable(register_result.error): - error_value = register_result.error() - elif hasattr(register_result, "msg") and callable(register_result.msg): - error_value = register_result.msg() - else: - error_value = getattr(register_result, "value", register_result) - if "already exist" not in str(error_value): - raise RuntimeError(f"Failed to register Playwright MCP server: {error_value}") - - self._registered_cdp_endpoint = self._configured_cdp_endpoint() - self.started = True - if BROWSER_SERVICE_REGISTRY.activate_binding( - self._lifecycle_identity, - self, - ): - self._start_heartbeat() - - async def ensure_started(self) -> None: - await self.ensure_runtime_ready() - if self._browser_agent is not None: - return - - self._browser_agent = build_browser_worker_agent( - provider=self.provider, - api_key=self.api_key, - api_base=self.api_base, - model_name=self.model_name, - mcp_cfg=self.mcp_cfg, - max_steps=self.guardrails.max_steps, - screenshot_subdir=self._screenshot_subdir, - artifacts_subdir=self._artifacts_subdir, - tool_result_observer=self._observe_worker_tool_result, - allowed_tool_names=self._allowed_tool_names, - ) - - def _start_heartbeat(self) -> None: - if self._heartbeat_task is None or self._heartbeat_task.done(): - self._heartbeat_task = asyncio.create_task(self._heartbeat_loop(), name="browser-heartbeat") - - async def _check_connection(self) -> None: - from .browser_tools import get_registered_client - - if self._managed_driver is not None: - driver_dict = getattr(self._managed_driver, "__dict__", {}) - endpoint_ready_fn = driver_dict.get("is_endpoint_ready") - if endpoint_ready_fn is None: - endpoint_ready_fn = driver_dict.get("_is_endpoint_ready") - if endpoint_ready_fn is None: - endpoint_ready_fn = getattr(self._managed_driver, "is_endpoint_ready", None) - if endpoint_ready_fn is None: - endpoint_ready_fn = getattr(self._managed_driver, "_is_endpoint_ready", None) - ready = await asyncio.to_thread(endpoint_ready_fn) - if not ready: - if self._managed_driver: - self._managed_driver.clear() - raise RuntimeError("Chrome CDP endpoint not responding") - server_id = (self.mcp_cfg.server_id or "").strip() or self.mcp_cfg.server_name - client = get_registered_client(server_id) - if client is None: - raise RuntimeError("Playwright MCP client not found in registry") - if not await client.ping(): - raise RuntimeError("Playwright MCP subprocess not responding") - - async def _heartbeat_loop(self) -> None: - from openjiuwen.core.common.logging import logger as _logger - - while True: - await asyncio.sleep(self._heartbeat_interval) - try: - await self._check_connection() - self._connection_healthy = True - self._last_heartbeat_ok = asyncio.get_event_loop().time() - _logger.info("BrowserService heartbeat: connection healthy") - except Exception as exc: - self._connection_healthy = False - _logger.warning("BrowserService heartbeat: connection unhealthy — %s", exc) - _logger.info( - "BrowserService heartbeat: restart deferred until the next browser task " - "to avoid reviving manually closed browsers during idle periods" - ) - - async def _restart(self) -> None: - """Tear down and reinitialize the browser service (e.g. after stdio subprocess dies).""" - await self._restart_browser_runtime() - - async def _remove_registered_mcp_server(self) -> None: - server_resource_id = (self.mcp_cfg.server_id or "").strip() or self.mcp_cfg.server_name - await Runner.resource_mgr.remove_mcp_server( - server_id=server_resource_id, - ignore_exception=True, - ) - - async def _refresh_mcp_server_binding(self) -> None: - await self._remove_registered_mcp_server() - register_result = await Runner.resource_mgr.add_mcp_server(self.mcp_cfg, tag="browser.service") - if register_result is not None and not getattr(register_result, "is_ok", lambda: False)(): - if hasattr(register_result, "error") and callable(register_result.error): - error_value = register_result.error() - elif hasattr(register_result, "msg") and callable(register_result.msg): - error_value = register_result.msg() - else: - error_value = getattr(register_result, "value", register_result) - if "already exist" not in str(error_value): - raise RuntimeError(f"Failed to refresh Playwright MCP server binding: {error_value}") - self._registered_cdp_endpoint = self._configured_cdp_endpoint() - - async def _reset_browser_runtime(self) -> None: - try: - if self.started: - await self._remove_registered_mcp_server() - except Exception: - pass - self.started = False - self._registered_cdp_endpoint = "" - self._browser_agent = None - await self._stop_managed_driver() - - async def _stop_heartbeat(self) -> None: - heartbeat = self._heartbeat_task - self._heartbeat_task = None - if heartbeat is None or heartbeat.done(): - return - heartbeat.cancel() - try: - await heartbeat - except asyncio.CancelledError: - pass - - def _clear_task_scoped_state(self) -> None: - """Drop observers and task/session state without touching the profile.""" - self._browser_agent = None - for tasks in self._inflight_tasks.values(): - for task in tuple(tasks): - if not task.done(): - task.cancel() - self._inflight_tasks.clear() - self._locks.clear() - self._sessions.clear() - self._failure_context_by_session.clear() - self._progress_by_session.clear() - - async def release_task_binding(self) -> bool: - """Release task-owned MCP/observer resources and preserve Chrome state.""" - if self._task_binding_ref_count > 0: - self._task_binding_ref_count -= 1 - if self._task_binding_ref_count > 0: - return False - if not self._registry_acquired: - await self._stop_heartbeat() - self._clear_task_scoped_state() - self.started = False - self._registered_cdp_endpoint = "" - self._connection_healthy = False - self._last_heartbeat_ok = None - return True - - release = BROWSER_SERVICE_REGISTRY.release( - self._lifecycle_identity, - self, - ) - self._registry_acquired = False - await self._stop_heartbeat() - - if release.close_mcp_binding: - try: - await self._remove_registered_mcp_server() - except Exception: - pass - - self.started = False - self._registered_cdp_endpoint = "" - self._connection_healthy = False - self._last_heartbeat_ok = None - self._clear_task_scoped_state() - - next_owner = release.next_heartbeat_owner - if next_owner is not None and next_owner.started: - next_owner.start_heartbeat_if_started() - return True - - @property - def managed_driver(self) -> Optional[ManagedBrowserDriver]: - """Return the managed driver coordinated by the process registry.""" - return self._managed_driver - - def start_heartbeat_if_started(self) -> None: - """Start heartbeat after registry ownership is transferred.""" - if self.started: - self._start_heartbeat() - - async def detach_lifecycle_resources(self) -> Optional[ManagedBrowserDriver]: - """Detach this task binding while preserving the driver for reset.""" - driver = self._managed_driver - await self._stop_heartbeat() - self.started = False - self._registered_cdp_endpoint = "" - self._browser_agent = None - self._connection_healthy = False - self._last_heartbeat_ok = None - return driver - - async def remove_registered_mcp_binding(self) -> None: - """Remove this service's MCP binding during coordinated reset.""" - await self._remove_registered_mcp_server() - - def clear_managed_browser_resources(self) -> None: - """Forget stopped managed resources after a coordinated reset.""" - self._managed_driver = None - self._active_profile = None - - @classmethod - async def _reset_lifecycle_resources( - cls, - identity: BrowserServiceIdentity, - fallback_service: Optional["BrowserService"] = None, - ) -> bool: - """Stop all process resources associated with one browser identity.""" - snapshot = BROWSER_SERVICE_REGISTRY.begin_reset(identity) - services = snapshot.services - if not services and fallback_service is not None: - services = (fallback_service,) - drivers = list(snapshot.managed_drivers) - for service in services: - driver = await service.detach_lifecycle_resources() - if driver is not None and all(existing is not driver for existing in drivers): - drivers.append(driver) - - representative = services[0] if services else fallback_service - if representative is not None: - try: - await representative.remove_registered_mcp_binding() - except Exception: - pass - - for driver in drivers: - try: - await asyncio.to_thread(driver.stop) - except Exception: - pass - finally: - BROWSER_SERVICE_REGISTRY.unregister_managed_driver( - identity, - driver, - ) - for service in services: - service.clear_managed_browser_resources() - return bool(services or drivers) - - async def reset(self) -> None: - """Reset one shared browser identity and restart lazily on next use.""" - await self._reset_lifecycle_resources( - self._lifecycle_identity, - fallback_service=self, - ) - - @classmethod - async def reset_registered_managed_browser( - cls, - *, - browser_key: str, - profile_name: str, - display_mode: str, - browser_binary: str, - ) -> int: - """Stop a preserved managed browser matching one logical identity.""" - identities = BROWSER_SERVICE_REGISTRY.find_managed_driver_identities( - browser_key=str(browser_key or "").strip(), - profile_name=str(profile_name or "").strip(), - display_mode=str(display_mode or "").strip().lower(), - browser_binary=cls.normalize_browser_binary(browser_binary), - ) - reset_count = 0 - for identity in identities: - if await cls._reset_lifecycle_resources(identity): - reset_count += 1 - return reset_count - - async def _restart_browser_runtime(self) -> None: - await self.reset() - await self.ensure_started() - - @staticmethod - def _build_worker_conversation_id(session_id: str, request_id: str) -> str: - sid = (session_id or "").strip() or "browser-session" - rid = (request_id or "").strip() or "request" - return f"{sid}:worker:{rid}:{uuid.uuid4().hex}" - - @classmethod - def _clean_progress_items(cls, value: Any, *, limit: int) -> list[str]: - if value is None: - return [] - if isinstance(value, (list, tuple, set)): - candidates = list(value) - else: - candidates = [value] - cleaned: list[str] = [] - seen: set[str] = set() - for item in candidates: - text_value = " ".join(str(item or "").split()).strip() - if not text_value: - continue - lowered = text_value.lower() - if lowered in seen: - continue - seen.add(lowered) - cleaned.append(cls._trim_text(text_value, 220)) - if len(cleaned) >= limit: - break - return cleaned - - @staticmethod - def _normalize_progress_status(value: Any) -> str: - normalized = str(value or "").strip().lower() - aliases = { - "complete": "completed", - "completed": "completed", - "done": "completed", - "partial": "partial", - "in_progress": "partial", - "in-progress": "partial", - "blocked": "blocked", - "failed": "failed", - } - return aliases.get(normalized, "") - - @staticmethod - def _push_recent_tool_step(existing: list[str], step: str, *, limit: int = 8) -> list[str]: - normalized = " ".join(str(step or "").split()).strip() - if not normalized: - return list(existing) - updated = [item for item in existing if item != normalized] - updated.append(normalized) - return updated[-limit:] - - @classmethod - def _extract_page_snapshot(cls, value: Any) -> tuple[str, str]: - if not isinstance(value, dict): - return "", "" - page = value.get("page") if isinstance(value.get("page"), dict) else {} - url = str(value.get("url") or page.get("url") or "").strip() - title = str(value.get("title") or page.get("title") or "").strip() - return cls._trim_text(url, 240), cls._trim_text(title, 120) - - @staticmethod - def _extract_screenshot_snapshot(value: Any) -> Any: - if not isinstance(value, dict): - return None - screenshot = value.get("screenshot") - if screenshot in (None, ""): - return None - return screenshot - - @classmethod - def _summarize_observation_payload(cls, value: Any) -> str: - if isinstance(value, dict): - parts: list[str] = [] - error_text = str(value.get("error") or "").strip() - if error_text: - parts.append(f"error={cls._trim_text(error_text, 140)}") - for key in ("message", "text", "result", "output", "value", "selector"): - candidate = value.get(key) - if candidate in (None, "") or isinstance(candidate, (dict, list, tuple, set)): - continue - parts.append(cls._trim_text(str(candidate), 160)) - break - url, title = cls._extract_page_snapshot(value) - if url: - parts.append(f"url={url}") - if title: - parts.append(f"title={title}") - ok_value = value.get("ok") - if ok_value is True and not parts: - parts.append("ok") - if not parts and value: - parts.append(", ".join(list(value.keys())[:4])) - return "; ".join(parts) - if isinstance(value, (list, tuple)): - nested = [cls._summarize_observation_payload(item) for item in list(value)[:2]] - nested = [item for item in nested if item] - return " | ".join(nested) - text_value = " ".join(str(value or "").split()).strip() - return cls._trim_text(text_value, 160) - - @classmethod - def _summarize_tool_result(cls, tool_name: str, tool_result: Any) -> str: - payload_summary = cls._summarize_observation_payload(tool_result) or "completed" - if tool_name: - return f"{tool_name}: {payload_summary}" - return payload_summary - - def _get_progress_state(self, session_id: str) -> BrowserTaskProgressState: - return self._progress_by_session.setdefault(session_id, BrowserTaskProgressState()) - - def _update_progress_from_tool_observation( - self, - *, - session_id: str, - request_id: str, - tool_name: str, - tool_result: Any, - ) -> None: - sid = (session_id or "").strip() - if not sid: - return - progress_state = self._get_progress_state(sid) - if request_id: - progress_state.request_id = request_id - progress_state.recent_tool_steps = self._push_recent_tool_step( - progress_state.recent_tool_steps, - self._summarize_tool_result(tool_name, tool_result), - ) - url, title = self._extract_page_snapshot(tool_result) - if url: - progress_state.last_page_url = url - if title: - progress_state.last_page_title = title - screenshot = self._extract_screenshot_snapshot(tool_result) - if screenshot is not None: - progress_state.last_screenshot = screenshot - if progress_state.status == "unknown": - progress_state.status = "partial" - - async def _observe_worker_tool_result(self, tool_name: str, tool_result: Any) -> None: - session_id = _ctx_observer_session_id.get().strip() - if not session_id: - return - request_id = _ctx_observer_request_id.get().strip() - self._update_progress_from_tool_observation( - session_id=session_id, - request_id=request_id, - tool_name=tool_name, - tool_result=tool_result, - ) - - @classmethod - def _should_treat_as_completed(cls, parsed: Dict[str, Any]) -> bool: - status = cls._normalize_progress_status(parsed.get("status") or parsed.get("task_status")) - if status != "completed": - return False - progress = parsed.get("progress") if isinstance(parsed.get("progress"), dict) else {} - missing = cls._clean_progress_items( - progress.get("missing_requirements") or parsed.get("missing_requirements"), - limit=4, - ) - evidence = cls._clean_progress_items( - progress.get("completion_evidence") or parsed.get("completion_evidence"), - limit=4, - ) - final_text = str(parsed.get("final") or "").strip() - return not missing and bool(evidence or final_text) - - def _update_progress_from_worker_result( - self, - *, - session_id: str, - request_id: str, - parsed: Dict[str, Any], - ) -> None: - sid = (session_id or "").strip() - if not sid or not isinstance(parsed, dict): - return - progress_state = self._get_progress_state(sid) - if request_id: - progress_state.request_id = request_id - - progress_payload = parsed.get("progress") if isinstance(parsed.get("progress"), dict) else {} - status = self._normalize_progress_status(parsed.get("status") or parsed.get("task_status")) - if not status: - if bool(parsed.get("ok", False)): - status = "completed" - elif self._is_max_iteration_result(parsed): - status = "partial" - elif str(parsed.get("error") or "").strip(): - status = "failed" - else: - status = "partial" - progress_state.status = status - - completed_steps = self._clean_progress_items( - progress_payload.get("completed_steps") or parsed.get("completed_steps"), - limit=8, - ) - if completed_steps: - progress_state.completed_steps = completed_steps - - remaining_steps = self._clean_progress_items( - progress_payload.get("remaining_steps") or parsed.get("remaining_steps"), - limit=8, - ) - if remaining_steps: - progress_state.remaining_steps = remaining_steps - - next_step = str(progress_payload.get("next_step") or parsed.get("next_step") or "").strip() - if next_step: - progress_state.next_step = self._trim_text(next_step, 220) - - completion_evidence = self._clean_progress_items( - progress_payload.get("completion_evidence") or parsed.get("completion_evidence"), - limit=6, - ) - if completion_evidence: - progress_state.completion_evidence = completion_evidence - - missing_requirements = self._clean_progress_items( - progress_payload.get("missing_requirements") or parsed.get("missing_requirements"), - limit=6, - ) - if missing_requirements: - progress_state.missing_requirements = missing_requirements - - url, title = self._extract_page_snapshot(parsed) - if url: - progress_state.last_page_url = url - if title: - progress_state.last_page_title = title - screenshot = self._extract_screenshot_snapshot(parsed) - if screenshot is not None: - progress_state.last_screenshot = screenshot - - final_text = str(parsed.get("final") or "").strip() - if final_text: - progress_state.last_worker_final = self._trim_text(final_text, 1200) - if bool(parsed.get("ok", False)) and not progress_state.completion_evidence: - progress_state.completion_evidence = [self._trim_text(final_text, 220)] - - @classmethod - def _build_progress_context(cls, progress_state: Optional[BrowserTaskProgressState]) -> str: - if progress_state is None: - return "" - lines: list[str] = [] - if progress_state.status and progress_state.status != "unknown": - lines.append(f"- Known progress status: {progress_state.status}") - if progress_state.completed_steps: - lines.append(f"- Completed steps: {' | '.join(progress_state.completed_steps)}") - if progress_state.remaining_steps: - lines.append(f"- Remaining steps: {' | '.join(progress_state.remaining_steps)}") - if progress_state.next_step: - lines.append(f"- Next step to try: {progress_state.next_step}") - if progress_state.completion_evidence: - lines.append(f"- Completion evidence observed: {' | '.join(progress_state.completion_evidence)}") - if progress_state.missing_requirements: - lines.append(f"- Missing requirements / blockers: {' | '.join(progress_state.missing_requirements)}") - if progress_state.recent_tool_steps: - lines.append("- Recent browser tool activity:") - lines.extend(f" - {step}" for step in progress_state.recent_tool_steps[-6:]) - if not lines: - return "" - return "Known progress for continuation:\n" + "\n".join(lines) - - @staticmethod - def _is_empty_progress_state(progress_state: BrowserTaskProgressState) -> bool: - return ( - progress_state.status == "unknown" - and not any( - ( - progress_state.completed_steps, - progress_state.remaining_steps, - progress_state.next_step, - progress_state.completion_evidence, - progress_state.missing_requirements, - progress_state.recent_tool_steps, - progress_state.last_page_url, - progress_state.last_page_title, - progress_state.last_worker_final, - ) - ) - and progress_state.last_screenshot in (None, "") - ) - - def _export_progress_state(self, session_id: str) -> Optional[Dict[str, Any]]: - progress_state = self._progress_by_session.get((session_id or "").strip()) - if progress_state is None: - return None - if self._is_empty_progress_state(progress_state): - return None - return progress_state.to_dict() - - @classmethod - def build_progress_context(cls, progress_state: Optional[BrowserTaskProgressState]) -> str: - return cls._build_progress_context(progress_state) - - def record_tool_progress( - self, - *, - session_id: str, - request_id: str, - tool_name: str, - tool_result: Any, - ) -> None: - self._update_progress_from_tool_observation( - session_id=session_id, - request_id=request_id, - tool_name=tool_name, - tool_result=tool_result, - ) - - @classmethod - def should_treat_as_completed(cls, parsed: Dict[str, Any]) -> bool: - return cls._should_treat_as_completed(parsed) - - def record_worker_progress( - self, - *, - session_id: str, - request_id: str, - parsed: Dict[str, Any], - ) -> None: - self._update_progress_from_worker_result( - session_id=session_id, - request_id=request_id, - parsed=parsed, - ) - - def get_progress_state(self, session_id: str) -> Optional[BrowserTaskProgressState]: - return self._progress_by_session.get((session_id or "").strip()) - - def export_progress_state(self, session_id: str) -> Optional[Dict[str, Any]]: - return self._export_progress_state(session_id) - - def build_failure_summary( - self, - *, - task: str, - error: str, - page_url: str, - page_title: str, - final: str, - screenshot: Optional[str], - attempt: int, - progress_state: Optional[BrowserTaskProgressState], - ) -> str: - return self._build_failure_summary( - task=task, - error=error, - page_url=page_url, - page_title=page_title, - final=final, - screenshot=screenshot, - attempt=attempt, - progress_state=progress_state, - ) - - def set_progress_state(self, session_id: str, progress_state: BrowserTaskProgressState) -> None: - sid = (session_id or "").strip() - if not sid: - return - if progress_state.is_empty(): - self._progress_by_session.pop(sid, None) - return - self._progress_by_session[sid] = progress_state - - def clear_progress_state(self, session_id: str) -> None: - sid = (session_id or "").strip() - if not sid: - return - self._progress_by_session.pop(sid, None) - - async def _run_task_once(self, task: str, session_id: str, request_id: str) -> Dict[str, Any]: - if self._browser_agent is None: - raise RuntimeError("BrowserService is not started") - - task_prompt = ( - f"Session id: {session_id}\n" - f"Request id: {request_id}\n" - f"Max steps: {self.guardrails.max_steps}\n" - f"Max failures: {self.guardrails.max_failures}\n\n" - f"Task:\n{task}\n\n" - "Perform the task in the current logical browser session/tab for this session id." - ) - worker_conversation_id = self._build_worker_conversation_id(session_id, request_id) - token_session = _ctx_observer_session_id.set(session_id) - token_request = _ctx_observer_request_id.set(request_id) - try: - result = await Runner.run_agent( - self._browser_agent, - {"query": task_prompt, "conversation_id": worker_conversation_id, "request_id": request_id}, - ) - finally: - _ctx_observer_session_id.reset(token_session) - _ctx_observer_request_id.reset(token_request) - output_text = result.get("output") if isinstance(result, dict) else result - parsed = extract_json_object(output_text) - if parsed: - self._update_progress_from_worker_result( - session_id=session_id, - request_id=request_id, - parsed=parsed, - ) - return parsed - - output_str = str(output_text) if output_text is not None else "" - output_lower = output_str.lower() - if MAX_ITERATION_MESSAGE.lower() in output_lower: - parsed = { - "ok": False, - "final": output_str, - "page": {"url": "", "title": ""}, - "screenshot": None, - "error": "max_iterations_reached", - "status": "partial", - } - self._update_progress_from_worker_result( - session_id=session_id, - request_id=request_id, - parsed=parsed, - ) - return parsed - - parsed = { - "ok": False, - "final": output_str, - "page": {"url": "", "title": ""}, - "screenshot": None, - "error": "Browser worker did not return valid JSON output", - "status": "failed", - } - self._update_progress_from_worker_result( - session_id=session_id, - request_id=request_id, - parsed=parsed, - ) - return parsed - - async def run_task_once(self, task: str, session_id: str, request_id: str) -> Dict[str, Any]: - return await self._run_task_once(task=task, session_id=session_id, request_id=request_id) - - @staticmethod - def _is_max_iteration_result(parsed: Dict[str, Any]) -> bool: - if not isinstance(parsed, dict): - return False - if str(parsed.get("error", "")).strip().lower() == "max_iterations_reached": - return True - marker = MAX_ITERATION_MESSAGE.lower() - for key in ("final", "error"): - value = parsed.get(key) - if value is None: - continue - if marker in str(value).lower(): - return True - return False - - @staticmethod - def _build_resume_task(task: str, previous_final: str, progress_context: str = "") -> str: - base = (task or "").strip() - previous = (previous_final or "").strip() - if len(previous) > 1200: - previous = previous[:1200] + "...[truncated]" - context_parts: list[str] = [ - "Continuation context:", - "- The previous run reached max iterations before completion.", - "- Continue from the current browser state in this same session.", - "- Avoid repeating already completed steps unless needed for recovery.", - ] - if progress_context: - context_parts.append(progress_context) - if previous: - context_parts.extend( - [ - "- Previous partial status (may be incomplete):", - previous, - ] - ) - return f"{base}\n\n" + "\n".join(context_parts) - - @staticmethod - def _trim_text(value: Any, limit: int) -> str: - text = str(value or "").strip() - if len(text) > limit: - return text[:limit] + "...[truncated]" - return text - - @classmethod - def _build_failure_summary( - cls, - *, - task: str, - error: str, - page_url: str, - page_title: str, - final: str, - screenshot: Any, - attempt: int, - progress_state: Optional[BrowserTaskProgressState] = None, - ) -> str: - lines = [ - "Failure summary for continuation:", - f"- Original task: {cls._trim_text(task, 400) or '(empty)'}", - f"- Failed attempt: {attempt}", - f"- Error: {cls._trim_text(error, 300) or '(unknown)'}", - ] - if page_url or page_title: - lines.append( - f"- Last page: url={cls._trim_text(page_url, 240) or '(unknown)'}, " - f"title={cls._trim_text(page_title, 120) or '(unknown)'}" - ) - screenshot_text = cls._trim_text(screenshot, 200) - if screenshot_text: - lines.append(f"- Last screenshot: {screenshot_text}") - progress_context = cls._build_progress_context(progress_state) - if progress_context: - lines.append(progress_context) - final_excerpt = cls._trim_text(final, 1200) - if final_excerpt: - lines.append("- Partial output excerpt:") - lines.append(final_excerpt) - return "\n".join(lines) - - @staticmethod - def _build_task_with_failure_context(task: str, failure_summary: str) -> str: - base = (task or "").strip() - summary = (failure_summary or "").strip() - if not summary: - return base - return ( - f"{base}\n\n" - "Previous failed attempt context:\n" - f"{summary}\n\n" - "Continuation instructions:\n" - "- Continue from the current browser state in this same session.\n" - "- Do not repeat completed steps unless required for recovery.\n" - "- Prioritize resolving the listed failure." - ) - - async def run_task( - self, - task: str, - session_id: Optional[str] = None, - request_id: Optional[str] = None, - timeout_s: Optional[int] = None, - ) -> Dict[str, Any]: - await self.ensure_started() - sid = self.session_new(session_id) - rid = (request_id or "").strip() or uuid.uuid4().hex - effective_timeout = int(timeout_s) if (timeout_s is not None and timeout_s > 0) else self.guardrails.timeout_s - timeout_deadline = ( - asyncio.get_running_loop().time() + float(effective_timeout) - ) - attempts = 2 if self.guardrails.retry_once else 1 - base_task = (task or "").strip() - previous_failure_summary = self._failure_context_by_session.get(sid, "") - - async with self._locks[sid]: - current_task = asyncio.current_task() - if current_task is not None: - self._register_inflight_task(sid, rid, current_task) - try: - if await self.is_cancelled(sid, rid): - await self.clear_cancel(sid, rid) - await self.clear_cancel(sid, None) - result = { - "ok": False, - "session_id": sid, - "request_id": rid, - "final": "", - "page": {"url": "", "title": ""}, - "screenshot": None, - "error": "cancelled_by_frontend", - "attempt": 0, - "failure_summary": None, - "progress_state": None, - } - return result - last_error: Optional[str] = None - used_max_iteration_resume = False - next_task = self._build_task_with_failure_context(base_task, previous_failure_summary) - attempt_idx = 0 - max_attempts = attempts + (1 if self.guardrails.resume_on_max_iterations else 0) - last_failure_final = "" - last_failure_page: Dict[str, Any] = {} - last_failure_screenshot: Any = None - while attempt_idx < max_attempts: - remaining_timeout = ( - timeout_deadline - asyncio.get_running_loop().time() - ) - if remaining_timeout <= 0: - last_error = f"task_timeout: exceeded {effective_timeout}s" - break - try: - parsed = await asyncio.wait_for( - self.run_task_once(task=next_task, session_id=sid, request_id=rid), - timeout=remaining_timeout, - ) - attempt_idx += 1 - self._update_progress_from_worker_result( - session_id=sid, - request_id=rid, - parsed=parsed, - ) - parsed_ok = bool(parsed.get("ok", False)) - if not parsed_ok and self._should_treat_as_completed(parsed): - parsed = dict(parsed) - parsed["ok"] = True - parsed["error"] = None - parsed_ok = True - should_resume_max_iter = ( - (not parsed_ok) - and self._is_max_iteration_result(parsed) - and self.guardrails.resume_on_max_iterations - and not used_max_iteration_resume - ) - if not parsed_ok: - last_error = str(parsed.get("error") or "") - last_failure_final = str(parsed.get("final", "")) - last_failure_page = parsed.get("page") if isinstance(parsed.get("page"), dict) else {} - last_failure_screenshot = parsed.get("screenshot") - - if should_resume_max_iter: - used_max_iteration_resume = True - next_task = self._build_resume_task( - next_task, - str(parsed.get("final", "")), - progress_context=self._build_progress_context(self._progress_by_session.get(sid)), - ) - last_error = str(parsed.get("error") or MAX_ITERATION_MESSAGE) - continue - - if not parsed_ok and attempt_idx < attempts and self._is_retryable_runtime_result(parsed): - page = parsed.get("page") if isinstance(parsed.get("page"), dict) else {} - failure_summary = self._build_failure_summary( - task=base_task, - error=str(parsed.get("error") or ""), - page_url=str(page.get("url", "")), - page_title=str(page.get("title", "")), - final=str(parsed.get("final", "")), - screenshot=parsed.get("screenshot"), - attempt=attempt_idx, - progress_state=self._progress_by_session.get(sid), - ) - should_restart = self._is_retryable_transport_message( - failure_summary - ) or self._should_restart_after_runtime_result(parsed) - if should_restart: - try: - await self._restart() - except Exception as restart_exc: - last_error = f"restart_failed: {restart_exc!r}" - break - next_task = self._build_task_with_failure_context(base_task, failure_summary) - continue - - page = parsed.get("page") if isinstance(parsed.get("page"), dict) else {} - screenshot = self._normalize_screenshot_value(parsed.get("screenshot")) - response = { - "ok": parsed_ok, - "session_id": sid, - "request_id": rid, - "final": str(parsed.get("final", "")), - "page": { - "url": str(page.get("url", "")), - "title": str(page.get("title", "")), - }, - "screenshot": screenshot, - "error": parsed.get("error"), - "attempt": attempt_idx, - "progress_state": None, - } - if parsed_ok: - self._failure_context_by_session.pop(sid, None) - self._progress_by_session.pop(sid, None) - response["failure_summary"] = None - return response - - failure_summary = self._build_failure_summary( - task=base_task, - error=str(parsed.get("error") or ""), - page_url=str(page.get("url", "")), - page_title=str(page.get("title", "")), - final=str(parsed.get("final", "")), - screenshot=parsed.get("screenshot"), - attempt=attempt_idx, - progress_state=self._progress_by_session.get(sid), - ) - self._failure_context_by_session[sid] = failure_summary - response["failure_summary"] = failure_summary - response["progress_state"] = self._export_progress_state(sid) - return response - except TimeoutError: - attempt_idx += 1 - last_error = f"task_timeout: exceeded {effective_timeout}s" - if ( - attempt_idx >= attempts - or asyncio.get_running_loop().time() >= timeout_deadline - ): - break - except asyncio.CancelledError: - await self.clear_cancel(sid, rid) - await self.clear_cancel(sid, None) - result = { - "ok": False, - "session_id": sid, - "request_id": rid, - "final": "", - "page": {"url": "", "title": ""}, - "screenshot": None, - "error": "cancelled_by_frontend", - "attempt": attempt_idx + 1, - "failure_summary": None, - "progress_state": None, - } - return result - except Exception as exc: - attempt_idx += 1 - last_error = str(exc) or repr(exc) - if attempt_idx >= attempts: - break - # Restart before retry on known transport/session failures. - if (not str(exc)) or self._is_retryable_transport_error(exc): - try: - await self._restart() - except Exception as restart_exc: - last_error = f"restart_failed: {restart_exc!r}" - break - - await self.clear_cancel(sid, rid) - await self.clear_cancel(sid, None) - page_url = str(last_failure_page.get("url", "")) if isinstance(last_failure_page, dict) else "" - page_title = str(last_failure_page.get("title", "")) if isinstance(last_failure_page, dict) else "" - failure_summary = self._build_failure_summary( - task=base_task, - error=last_error or "unknown browser execution error", - page_url=page_url, - page_title=page_title, - final=last_failure_final, - screenshot=last_failure_screenshot, - attempt=min(attempt_idx, max_attempts), - progress_state=self._progress_by_session.get(sid), - ) - self._failure_context_by_session[sid] = failure_summary - result = { - "ok": False, - "session_id": sid, - "request_id": rid, - "final": "", - "page": {"url": "", "title": ""}, - "screenshot": None, - "error": last_error or "unknown browser execution error", - "attempt": min(attempt_idx, max_attempts), - "failure_summary": failure_summary, - "progress_state": self._export_progress_state(sid), - } - return result - finally: - if current_task is not None: - self._unregister_inflight_task(sid, rid, current_task) - - async def shutdown(self) -> None: - try: - await self.reset() - finally: - self._task_binding_ref_count = 0 - await self.release_task_binding() - BROWSER_SERVICE_REGISTRY.discard_idle(self._lifecycle_identity) diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/service_registry.py b/openjiuwen/harness/tools/browser_move/playwright_runtime/service_registry.py deleted file mode 100644 index 10861be434..0000000000 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/service_registry.py +++ /dev/null @@ -1,257 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Process-level ownership registry for browser service resources.""" - -from __future__ import annotations - -import threading -import weakref -from dataclasses import dataclass, field -from typing import Any, Optional - - -@dataclass(frozen=True) -class BrowserServiceIdentity: - """Stable process-level identity for one logical browser instance.""" - - browser_key: str - profile_name: str - driver_mode: str - display_mode: str - managed_args: tuple[str, ...] - browser_binary: str - user_data_dir: str - cdp_endpoint: str - server_id: str - - -@dataclass(frozen=True) -class BrowserServiceRelease: - """Ownership changes produced when one task-scoped service releases.""" - - close_mcp_binding: bool - next_heartbeat_owner: Optional[Any] - - -@dataclass(frozen=True) -class BrowserServiceResetSnapshot: - """Resources that must be stopped for an explicit identity reset.""" - - services: tuple[Any, ...] - managed_drivers: tuple[Any, ...] - - -@dataclass -class _BrowserServiceEntry: - services: weakref.WeakSet[Any] = field(default_factory=weakref.WeakSet) - binding_services: weakref.WeakSet[Any] = field(default_factory=weakref.WeakSet) - heartbeat_owner: Optional[weakref.ReferenceType[Any]] = None - managed_drivers: list[Any] = field(default_factory=list) - - -class BrowserServiceRegistry: - """Coordinate process resources without sharing task/model state. - - BrowserService objects remain task-scoped. Registry entries only coordinate - the MCP binding, managed Chrome process handles, and heartbeat ownership for - services that address the same browser identity. - """ - - def __init__(self) -> None: - self._entries: dict[BrowserServiceIdentity, _BrowserServiceEntry] = {} - self._lock = threading.RLock() - - def acquire(self, identity: BrowserServiceIdentity, service: Any) -> None: - with self._lock: - entry = self._entries.setdefault(identity, _BrowserServiceEntry()) - entry.services.add(service) - - def activate_binding(self, identity: BrowserServiceIdentity, service: Any) -> bool: - """Mark an MCP binding active and return whether service owns heartbeat.""" - with self._lock: - entry = self._entries.setdefault(identity, _BrowserServiceEntry()) - entry.services.add(service) - entry.binding_services.add(service) - owner = self._live_owner(entry) - if owner is None: - entry.heartbeat_owner = weakref.ref(service) - owner = service - return owner is service - - def release( - self, - identity: BrowserServiceIdentity, - service: Any, - ) -> BrowserServiceRelease: - """Release a task binding and transfer heartbeat ownership if needed.""" - with self._lock: - entry = self._entries.get(identity) - if entry is None: - return BrowserServiceRelease( - close_mcp_binding=False, - next_heartbeat_owner=None, - ) - - was_binding_service = service in entry.binding_services - was_owner = self._live_owner(entry) is service - entry.binding_services.discard(service) - entry.services.discard(service) - - next_owner = None - if was_owner: - next_owner = next(iter(entry.binding_services), None) - entry.heartbeat_owner = ( - weakref.ref(next_owner) if next_owner is not None else None - ) - - close_binding = was_binding_service and not entry.binding_services - if not entry.services: - # Chrome intentionally outlives the last task binding. Retain - # only its process handle so an explicit identity-scoped - # configuration restart can stop it after task collection. - if not entry.managed_drivers: - self._entries.pop(identity, None) - - return BrowserServiceRelease( - close_mcp_binding=close_binding, - next_heartbeat_owner=next_owner, - ) - - def register_managed_driver( - self, - identity: BrowserServiceIdentity, - service: Any, - driver: Any, - ) -> None: - with self._lock: - entry = self._entries.setdefault(identity, _BrowserServiceEntry()) - entry.services.add(service) - if getattr(driver, "owns_process", False): - # A new owner means prior handles for this exact identity are - # stale (for example, the user closed the previous window). - entry.managed_drivers = [driver] - return - if any( - getattr(existing, "owns_process", False) - for existing in entry.managed_drivers - ): - return - if all(existing is not driver for existing in entry.managed_drivers): - entry.managed_drivers.append(driver) - - def unregister_managed_driver( - self, - identity: BrowserServiceIdentity, - driver: Any, - ) -> None: - with self._lock: - entry = self._entries.get(identity) - if entry is None: - return - entry.managed_drivers = [ - existing - for existing in entry.managed_drivers - if existing is not driver - ] - if not entry.services and not entry.managed_drivers: - self._entries.pop(identity, None) - - def begin_reset( - self, - identity: BrowserServiceIdentity, - ) -> BrowserServiceResetSnapshot: - """Detach shared process resources for one explicit reset operation.""" - with self._lock: - entry = self._entries.get(identity) - if entry is None: - return BrowserServiceResetSnapshot(services=(), managed_drivers=()) - - services = tuple(entry.services) - drivers = list(entry.managed_drivers) - for service in services: - driver = getattr(service, "managed_driver", None) - if driver is not None and all(existing is not driver for existing in drivers): - drivers.append(driver) - - entry.binding_services.clear() - entry.heartbeat_owner = None - entry.managed_drivers.clear() - if not entry.services: - self._entries.pop(identity, None) - - return BrowserServiceResetSnapshot( - services=services, - managed_drivers=tuple(drivers), - ) - - def is_heartbeat_owner( - self, - identity: BrowserServiceIdentity, - service: Any, - ) -> bool: - with self._lock: - entry = self._entries.get(identity) - return entry is not None and self._live_owner(entry) is service - - def discard_idle(self, identity: BrowserServiceIdentity) -> None: - """Discard registry metadata after an explicit full shutdown.""" - with self._lock: - entry = self._entries.get(identity) - if entry is None: - return - if not entry.services and not entry.binding_services and not entry.managed_drivers: - self._entries.pop(identity, None) - - def find_managed_driver_identities( - self, - *, - browser_key: str, - profile_name: str, - display_mode: str, - browser_binary: str, - ) -> tuple[BrowserServiceIdentity, ...]: - """Find preserved drivers for one exact logical browser identity.""" - with self._lock: - matches: list[BrowserServiceIdentity] = [] - for identity, entry in self._entries.items(): - if not entry.managed_drivers: - continue - if not isinstance(identity, BrowserServiceIdentity): - continue - if identity.browser_key != browser_key: - continue - if identity.profile_name != profile_name: - continue - if identity.driver_mode != "managed": - continue - if identity.display_mode != display_mode: - continue - if identity.browser_binary != browser_binary: - continue - matches.append(identity) - return tuple(matches) - - def clear(self) -> None: - """Clear metadata for isolated tests after their resources are stopped.""" - with self._lock: - self._entries.clear() - - @staticmethod - def _live_owner(entry: _BrowserServiceEntry) -> Optional[Any]: - owner = entry.heartbeat_owner() if entry.heartbeat_owner is not None else None - if owner is None or owner not in entry.binding_services: - entry.heartbeat_owner = None - return None - return owner - - -BROWSER_SERVICE_REGISTRY = BrowserServiceRegistry() - - -__all__ = [ - "BROWSER_SERVICE_REGISTRY", - "BrowserServiceIdentity", - "BrowserServiceRegistry", - "BrowserServiceRelease", - "BrowserServiceResetSnapshot", -] diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime_mcp_server.py b/openjiuwen/harness/tools/browser_move/playwright_runtime_mcp_server.py index f072ff095b..2b344e9f30 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime_mcp_server.py +++ b/openjiuwen/harness/tools/browser_move/playwright_runtime_mcp_server.py @@ -1,67 +1,56 @@ # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""MCP server wrapper for Playwright browser runtime. - -Usage (from repo root): - uv run python src/playwright_runtime_mcp_server.py - uv run python src/playwright_runtime_mcp_server.py --transport streamable-http --host 127.0.0.1 --port 8940 -""" +"""MCP server entry point for the browser runtime; run it with ``python -m`` and this module path.""" from __future__ import annotations import argparse import asyncio -import os -import sys from contextlib import asynccontextmanager from pathlib import Path from typing import TYPE_CHECKING, Any, Dict, Optional -# Ensure repo and src packages are importable when running as a script. -_HERE = Path(__file__).resolve().parent -_SRC_ROOT = _HERE -_REPO_ROOT = _SRC_ROOT.parent -for _p in (str(_REPO_ROOT), str(_SRC_ROOT)): - if _p not in sys.path: - sys.path.append(_p) - from fastmcp import Context, FastMCP -from clients.stdio_client import BrowserMoveStdioClient -from clients.streamable_http_client import BrowserMoveStreamableHttpClient -from playwright_runtime.config import ( - DEFAULT_BROWSER_TIMEOUT_S, + +from openjiuwen.harness.tools.browser_move.contract import SurfaceSettings +from openjiuwen.harness.tools.browser_move.runtime.config import ( MISSING_API_KEY_MESSAGE, - build_runtime_settings, + CoreSettings, + RuntimeServerSettings, load_repo_dotenv, + process_environ, + runtime_server_environ, ) -from controllers.action import ( +from openjiuwen.harness.tools.browser_move.surfaces.browser.helpers import register_custom_action_specs +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.actions import ( bind_runtime, clear_runtime_runner, + get_default_controller, list_actions, register_builtin_actions, run_action, ) - -# Use browser_move client subclasses for MCP transport retry and timeout handling. -import openjiuwen.core.runner.resources_manager.tool_manager as _tool_mgr_mod +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings import ( + browser_surface_settings, + build_runtime_settings, +) if TYPE_CHECKING: - from playwright_runtime.runtime import BrowserAgentRuntime - -_tool_mgr_mod.StdioClient = BrowserMoveStdioClient -_tool_mgr_mod.StreamableHttpClient = BrowserMoveStreamableHttpClient + from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade import BrowserAgentRuntime _runtime: Optional["BrowserAgentRuntime"] = None _runtime_lock = asyncio.Lock() +_core_settings: Optional[CoreSettings] = None +_browser_settings: Optional[SurfaceSettings] = None load_repo_dotenv() def _build_runtime() -> "BrowserAgentRuntime": - from playwright_runtime.runtime import BrowserAgentRuntime + from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade import BrowserAgentRuntime - settings = build_runtime_settings() + settings = build_runtime_settings(core_settings=_core_settings, browser_settings=_browser_settings) if not settings.api_key: raise RuntimeError(MISSING_API_KEY_MESSAGE) return BrowserAgentRuntime( @@ -71,6 +60,8 @@ def _build_runtime() -> "BrowserAgentRuntime": model_name=settings.model_name, mcp_cfg=settings.mcp_cfg, guardrails=settings.guardrails, + core_settings=_core_settings, + browser_settings=_browser_settings, ) @@ -126,6 +117,7 @@ def _resolve_request_id(explicit_request_id: str, ctx: Context | None = None) -> async def _lifespan(_server: FastMCP): try: register_builtin_actions() + register_custom_action_specs(get_default_controller(), upload_root=None) yield {} finally: await _shutdown_runtime() @@ -168,8 +160,6 @@ async def browser_run_task( request_id=effective_request_id or None, timeout_s=timeout_s if timeout_s > 0 else None, ) - # Strip base64 screenshot from MCP response — it becomes a ToolMessage in - # calling agent context and can exceed the LLM context window on its own. screenshot = result.get("screenshot") if isinstance(screenshot, str) and screenshot.startswith("data:"): result = {**result, "screenshot": "[screenshot saved]"} @@ -222,7 +212,7 @@ async def browser_clear_cancel( "Or use coord_source_x+coord_source_y+coord_target_x+coord_target_y. " "Aliases source/target and source_x/source_y/target_x/target_y are accepted. " "Use list_upload_files to discover files available at BROWSER_UPLOAD_ROOT before uploading. " - "Register your own via controllers.action.register_action." + "Register your own via surfaces.browser.runner.actions.register_action." ), ) async def browser_custom_action( @@ -265,45 +255,46 @@ async def browser_list_custom_actions() -> dict[str, Any]: description="Return runtime readiness and selected model/provider config.", ) async def browser_runtime_health() -> dict[str, Any]: - settings = build_runtime_settings() - svc = _runtime.service if _runtime is not None else None + settings = build_runtime_settings(core_settings=_core_settings, browser_settings=_browser_settings) + health = await _runtime.runtime_health() if _runtime is not None else {} return { - "ok": svc.connection_healthy if svc is not None else False, + "ok": bool(health.get("ok", False)), "started": _runtime is not None, - "last_heartbeat_ok": svc.last_heartbeat_ok if svc is not None else None, + "detail": health.get("detail"), "provider": settings.provider, "api_base": settings.api_base, "model_name": settings.model_name, } -def parse_args() -> argparse.Namespace: +def parse_args(settings: RuntimeServerSettings | None = None) -> argparse.Namespace: + settings = settings or RuntimeServerSettings.server_from_env(process_environ()) parser = argparse.ArgumentParser(description="Playwright runtime MCP server") parser.add_argument( "--transport", choices=["stdio", "sse", "streamable-http", "http"], - default=(os.getenv("PLAYWRIGHT_RUNTIME_MCP_TRANSPORT") or "stdio").strip().lower(), + default=settings.transport, help="MCP transport mode. Use stdio for agent-launched server; sse/http for standalone server.", ) parser.add_argument( "--host", - default=(os.getenv("PLAYWRIGHT_RUNTIME_MCP_HOST") or "127.0.0.1").strip(), + default=settings.host, help="Host for sse/http transports.", ) parser.add_argument( "--port", type=int, - default=int((os.getenv("PLAYWRIGHT_RUNTIME_MCP_PORT") or "8940").strip()), + default=int(settings.port), help="Port for sse/http transports.", ) parser.add_argument( "--path", - default=(os.getenv("PLAYWRIGHT_RUNTIME_MCP_PATH") or "").strip(), + default=settings.path, help="Optional custom endpoint path for sse/http transports.", ) parser.add_argument( "--log-level", - default=(os.getenv("PLAYWRIGHT_RUNTIME_MCP_LOG_LEVEL") or "INFO").strip(), + default=settings.log_level, help="Server log level.", ) parser.add_argument( @@ -322,40 +313,24 @@ def parse_args() -> argparse.Namespace: return parser.parse_args() -def _apply_timeout_defaults() -> None: - """Set runtime timeout defaults based on transport when not explicitly provided.""" - timeout_text = str(DEFAULT_BROWSER_TIMEOUT_S) - os.environ.setdefault("BROWSER_TIMEOUT_S", timeout_text) - os.environ.setdefault("PLAYWRIGHT_TOOL_TIMEOUT_S", os.getenv("BROWSER_TIMEOUT_S", timeout_text)) - - -def _resolve_stateless_http(args: argparse.Namespace) -> bool: +def _resolve_stateless_http(args: argparse.Namespace, settings: RuntimeServerSettings) -> bool: """Resolve whether to run HTTP transports in stateless mode.""" if args.stateless_http: return True + if settings.stateless_http is not None: + return settings.stateless_http - env_raw = (os.getenv("PLAYWRIGHT_RUNTIME_MCP_STATELESS_HTTP") or "").strip().lower() - if env_raw in {"1", "true", "yes", "on"}: - return True - if env_raw in {"0", "false", "no", "off"}: - return False - - # Default to stateless for streamable/http to avoid stateful session crashes. return args.transport in {"streamable-http", "http"} -def _configure_stdio_logging() -> None: +def _configure_stdio_logging(log_dir: Path | None) -> None: """Prevent application logs from polluting stdio JSON-RPC output.""" try: from openjiuwen.core.common.logging.default.log_config import log_config from openjiuwen.core.common.logging.manager import LogManager - configured = ( - (os.getenv("PLAYWRIGHT_RUNTIME_LOG_DIR") or "").strip() - or (os.getenv("BROWSER_RUNTIME_LOG_DIR") or "").strip() - ) - if configured: - log_path = str(Path(configured).expanduser().resolve()) + if log_dir is not None: + log_path = str(log_dir.resolve()) else: log_path = str((Path.cwd().expanduser().resolve() / "logs").resolve()) log_config._log_config["log_path"] = log_path @@ -376,11 +351,16 @@ def _configure_stdio_logging() -> None: def main() -> None: + global _core_settings, _browser_settings + load_repo_dotenv() - args = parse_args() + environ = runtime_server_environ(process_environ()) + server_settings = RuntimeServerSettings.server_from_env(environ) + args = parse_args(server_settings) if args.transport == "stdio": - _configure_stdio_logging() - _apply_timeout_defaults() + _configure_stdio_logging(server_settings.log_dir) + _core_settings = CoreSettings.from_env(environ) + _browser_settings = browser_surface_settings(environ) kwargs: Dict[str, Any] = { "show_banner": not args.no_banner, "log_level": args.log_level, @@ -390,7 +370,7 @@ def main() -> None: kwargs["port"] = args.port if args.path: kwargs["path"] = args.path - if args.transport in {"streamable-http", "http"} and _resolve_stateless_http(args): + if args.transport in {"streamable-http", "http"} and _resolve_stateless_http(args, server_settings): kwargs["stateless_http"] = True mcp.run(transport=args.transport, **kwargs) diff --git a/openjiuwen/harness/tools/browser_move/provisioning/__init__.py b/openjiuwen/harness/tools/browser_move/provisioning/__init__.py new file mode 100644 index 0000000000..e1962d1c63 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/provisioning/__init__.py @@ -0,0 +1,28 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Provisioning: start or attach the device a driver connects to.""" + +from __future__ import annotations + +from typing import Any, Protocol + +from ..contract.device import ProvisionedDevice +from ..contract.registry import PROVISIONERS, register_provisioner + + +class Provisioner(Protocol): + """Makes one device ready from the surface settings and releases it.""" + + def provision(self, surface_settings: Any) -> ProvisionedDevice: ... + + def release(self, device: ProvisionedDevice) -> None: ... + + def is_alive(self, device: ProvisionedDevice) -> bool: + """Whether the device still answers; a dead one is released and provisioned again.""" + + +for _name in ("managed", "remote", "extension"): + register_provisioner(_name, f"{__name__}.managed_browser:ManagedBrowserProvisioner") + + +__all__ = ["PROVISIONERS", "ProvisionedDevice", "Provisioner"] diff --git a/openjiuwen/harness/tools/browser_move/drivers/managed_browser.py b/openjiuwen/harness/tools/browser_move/provisioning/managed_browser.py similarity index 57% rename from openjiuwen/harness/tools/browser_move/drivers/managed_browser.py rename to openjiuwen/harness/tools/browser_move/provisioning/managed_browser.py index ea5897ae0b..687bfcf2ac 100644 --- a/openjiuwen/harness/tools/browser_move/drivers/managed_browser.py +++ b/openjiuwen/harness/tools/browser_move/provisioning/managed_browser.py @@ -8,24 +8,62 @@ import json import os import shutil +import socket import subprocess import sys import time from pathlib import Path -from typing import Optional +from types import MappingProxyType +from typing import Optional, Protocol from urllib.request import urlopen -from ..playwright_runtime.profiles import BrowserProfile +from ..contract.device import ProvisionedDevice +from .profiles import BrowserProfile, BrowserProfileStore -def _default_chrome_user_data_dir() -> str: - """Return the platform-standard Chrome user data directory path. +class ManagedBrowserSettings(Protocol): + """The settings fields the managed browser provisioner reads; the browser surface settings have them.""" - Does not guarantee the directory exists 鈥?only that this is where Chrome - stores its default profile on the current OS. - """ + @property + def provisioning(self) -> str: ... + @property + def cdp_url(self) -> str: ... + @property + def cdp_headers(self) -> str: ... + @property + def cdp_timeout(self) -> str: ... + @property + def launch_args(self) -> tuple[str, ...]: ... + @property + def mcp_extension_token(self) -> str: ... + @property + def mcp_cwd(self) -> Path: ... + @property + def profile_store_path(self) -> Path | None: ... + @property + def profile_name(self) -> str: ... + @property + def managed_args(self) -> tuple[str, ...]: ... + @property + def browser_binary(self) -> str: ... + @property + def browser_binary_candidates(self) -> tuple[str, ...]: ... + @property + def managed_kill_existing(self) -> bool: ... + @property + def managed_port(self) -> int: ... + @property + def managed_host(self) -> str: ... + @property + def user_data_dir(self) -> str: ... + @property + def local_app_data(self) -> str: ... + + +def _default_chrome_user_data_dir(local_app_data: str) -> str: + """Return the platform's default Chrome user data directory (it may not exist).""" if os.name == "nt": - local_app_data = os.getenv("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local") + local_app_data = local_app_data or str(Path.home() / "AppData" / "Local") return str(Path(local_app_data) / "Google" / "Chrome" / "User Data") if sys.platform == "darwin": return str(Path.home() / "Library" / "Application Support" / "Google" / "Chrome") @@ -33,11 +71,7 @@ def _default_chrome_user_data_dir() -> str: def _kill_chrome_by_user_data_dir(user_data_dir: str) -> int: - """Kill Chrome processes whose command line references user_data_dir. - - Returns the number of PIDs sent a kill signal. Failures are silently - swallowed so a misconfigured environment never blocks startup. - """ + """Kill Chrome processes started with ``user_data_dir``; returns how many were signalled.""" normalized = str(Path(user_data_dir).expanduser().resolve()).lower().replace("\\", "/") killed = 0 @@ -110,7 +144,7 @@ def _cleanup_chrome_singleton_files(user_data_dir: str) -> None: pass -def _candidate_chrome_binaries() -> list[str]: +def _candidate_chrome_binaries(env_install_paths: tuple[str, ...]) -> list[str]: names = [ "chrome", "google-chrome", @@ -120,18 +154,10 @@ def _candidate_chrome_binaries() -> list[str]: binaries = [item for item in resolved if item] if os.name == "nt": - windows_roots = [ - os.getenv("LOCALAPPDATA"), - os.getenv("ProgramFiles"), - os.getenv("ProgramFiles(x86)"), - os.getenv("ProgramW6432"), - str(Path.home() / "AppData" / "Local"), + install_paths = [ + *env_install_paths, + str(Path.home() / "AppData" / "Local" / "Google" / "Chrome" / "Application" / "chrome.exe"), ] - install_paths = [] - for root in windows_roots: - if not root: - continue - install_paths.append(str(Path(root) / "Google" / "Chrome" / "Application" / "chrome.exe")) elif sys.platform == "darwin": install_paths = [ "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/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", @@ -148,7 +174,6 @@ def _candidate_chrome_binaries() -> list[str]: if Path(path).exists(): binaries.append(path) - # Preserve order and remove duplicates. unique: list[str] = [] seen: set[str] = set() for candidate in binaries: @@ -159,6 +184,12 @@ def _candidate_chrome_binaries() -> list[str]: return unique +def _allocate_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + def _is_chrome_identifier(value: str) -> bool: lowered = (value or "").strip().replace("\\", "/").lower() if not lowered: @@ -166,14 +197,99 @@ def _is_chrome_identifier(value: str) -> bool: return "chrome" in Path(lowered).name -class ManagedBrowserDriver: - """Launch and manage a dedicated local Chrome process.""" +def _answers_cdp(endpoint: str) -> bool: + try: + with urlopen(f"{endpoint}/json/version", timeout=1.5) as response: # nosec + payload = json.loads(response.read().decode("utf-8", errors="ignore")) + if isinstance(payload, dict): + return bool(payload.get("webSocketDebuggerUrl") or payload.get("Browser")) + except (OSError, ValueError): + return False + return False + + +class ManagedBrowserProvisioner: + """Provisions Chrome for the managed, remote and extension modes; a managed instance owns the Chrome it starts.""" - def __init__(self, profile: BrowserProfile) -> None: + def __init__( + self, + profile: Optional[BrowserProfile] = None, + profile_store: Optional[BrowserProfileStore] = None, + ) -> None: self.profile = profile + self._profile_store = profile_store + self._binary_candidates: tuple[str, ...] = () self._process: Optional[subprocess.Popen] = None self._owns_process: bool = False + def provision(self, surface_settings: ManagedBrowserSettings) -> ProvisionedDevice: + settings = surface_settings + if settings.provisioning == "remote": + return ProvisionedDevice( + kind="chrome", + endpoint=settings.cdp_url, + extras=MappingProxyType( + { + "cdp_headers": settings.cdp_headers, + "cdp_timeout": settings.cdp_timeout, + "launch_args": settings.launch_args, + } + ), + ) + if settings.provisioning == "extension": + return ProvisionedDevice( + kind="chrome", + endpoint="", + extras=MappingProxyType( + { + "extension": True, + "extension_token": settings.mcp_extension_token, + "launch_args": settings.launch_args, + } + ), + ) + store = self._profile_store or BrowserProfileStore( + settings.profile_store_path or settings.mcp_cwd / ".browser" / "profiles.json" + ) + profile = store.get_profile(settings.profile_name) + if ( + profile is None + or profile.driver_type != "managed" + or profile.debug_port <= 0 + or not str(profile.user_data_dir).strip() + or profile.extra_args != list(settings.managed_args) + ): + profile = _build_profile(settings) + profile.browser_binary = settings.browser_binary + store.upsert_profile(profile, select=True) + self.profile = profile + self._binary_candidates = settings.browser_binary_candidates + + endpoint = self.start(30.0, settings.managed_kill_existing) + profile.cdp_url = endpoint + store.upsert_profile(profile, select=True) + return ProvisionedDevice( + kind="chrome", + endpoint=endpoint, + extras=MappingProxyType( + { + "profile": profile, + "port": profile.debug_port, + "user_data_dir": profile.user_data_dir, + "managed_args": tuple(profile.extra_args), + "launch_args": settings.launch_args, + } + ), + cleanup=self.stop, + ) + + def release(self, device: ProvisionedDevice) -> None: + if device.cleanup is not None: + device.cleanup() + + def is_alive(self, device: ProvisionedDevice) -> bool: + return not device.endpoint or _answers_cdp(device.endpoint) + @property def cdp_endpoint(self) -> str: host = self.profile.host or "127.0.0.1" @@ -197,16 +313,11 @@ def _resolve_binary(self) -> str: if resolved: return resolved raise RuntimeError(f"Configured Chrome binary not found: {explicit}") - raise RuntimeError( - "Managed mode supports Chrome only. " - "Set BROWSER_MANAGED_BINARY to a Chrome executable." - ) + raise RuntimeError("Managed mode supports Chrome only. Set BROWSER_MANAGED_BINARY to a Chrome executable.") - candidates = _candidate_chrome_binaries() + candidates = _candidate_chrome_binaries(self._binary_candidates) if not candidates: - raise RuntimeError( - "Chrome needs to be installed. No Chrome binary was found on this machine." - ) + raise RuntimeError("Chrome needs to be installed. No Chrome binary was found on this machine.") return candidates[0] def _build_args(self, binary: str) -> list[str]: @@ -234,15 +345,7 @@ def _build_args(self, binary: str) -> list[str]: return args def _is_endpoint_ready(self) -> bool: - endpoint = f"{self.cdp_endpoint}/json/version" - try: - with urlopen(endpoint, timeout=1.5) as response: # nosec B310 - payload = json.loads(response.read().decode("utf-8", errors="ignore")) - if isinstance(payload, dict): - return bool(payload.get("webSocketDebuggerUrl") or payload.get("Browser")) - except (OSError, ValueError): - return False - return False + return _answers_cdp(self.cdp_endpoint) def start(self, timeout_s: float = 30.0, kill_existing: bool = False) -> str: if self._process is not None and self._process.poll() is None: @@ -275,9 +378,7 @@ def start(self, timeout_s: float = 30.0, kill_existing: bool = False) -> str: while time.time() < deadline: if self._process.poll() is not None: self._owns_process = False - raise RuntimeError( - f"Managed browser process exited early with code {self._process.returncode}" - ) + raise RuntimeError(f"Managed browser process exited early with code {self._process.returncode}") if self._is_endpoint_ready(): return self.cdp_endpoint time.sleep(0.25) @@ -307,8 +408,6 @@ def stop(self, wait_timeout_s: float = 5.0) -> None: pass def clear(self): - # Reap a Chrome child that exited (e.g. user closed the window) but - # whose Popen handle still sits on the driver. if self._process is None: return process = self._process @@ -317,5 +416,27 @@ def clear(self): if process is None: return from openjiuwen.core.common.logging import logger as _logger + _logger.info("ManagedBrowser: clear handle process") process.poll() + + +def _build_profile(settings: ManagedBrowserSettings) -> BrowserProfile: + port = settings.managed_port or _allocate_free_port() + if settings.user_data_dir: + user_data_dir = settings.user_data_dir + elif settings.managed_kill_existing: + user_data_dir = _default_chrome_user_data_dir(settings.local_app_data) + else: + user_data_dir = str(settings.mcp_cwd / ".browser-profiles" / settings.profile_name) + host = settings.managed_host + return BrowserProfile( + name=settings.profile_name, + driver_type="managed", + cdp_url=f"http://{host}:{port}", + browser_binary=settings.browser_binary, + user_data_dir=user_data_dir, + debug_port=port, + host=host, + extra_args=list(settings.managed_args), + ) diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/profiles.py b/openjiuwen/harness/tools/browser_move/provisioning/profiles.py similarity index 100% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/profiles.py rename to openjiuwen/harness/tools/browser_move/provisioning/profiles.py diff --git a/openjiuwen/harness/tools/browser_move/runtime/__init__.py b/openjiuwen/harness/tools/browser_move/runtime/__init__.py new file mode 100644 index 0000000000..f38257c8a6 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/__init__.py @@ -0,0 +1,13 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The core runtime package: the composition root, the session, the service and the topologies.""" + +from __future__ import annotations + +from pathlib import Path + +_HERE = Path(__file__).resolve().parent +SRC_ROOT = _HERE.parent +REPO_ROOT = SRC_ROOT.parent.parent.parent.parent + +__all__ = ["REPO_ROOT", "SRC_ROOT"] diff --git a/openjiuwen/harness/tools/browser_move/runtime/agents.py b/openjiuwen/harness/tools/browser_move/runtime/agents.py new file mode 100644 index 0000000000..914d7c73b7 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/agents.py @@ -0,0 +1,296 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Agent builders for the runtime and the worker.""" + +from __future__ import annotations + +import inspect +import json +from typing import Any, Awaitable, Callable, Iterable + +import anyio + +from openjiuwen.core.common.logging import logger +from openjiuwen.core.common.utils.schema_utils import SchemaUtils +from openjiuwen.core.foundation.tool import Tool +from openjiuwen.core.single_agent.agents.react_agent import ReActAgent, ReActAgentConfig +from openjiuwen.core.single_agent.schema.agent_card import AgentCard + +from ..contract import DriverInfo, SurfaceBundle +from .assembly import build_agent_parts, core_rails +from .config import CoreSettings +from .driver_session import DriverSession +from .rails.final_answer_recovery import FinalAnswerRecoveryRail + +WORKER_LANGUAGE = "en" +ToolResultObserver = Callable[[str, Any], Awaitable[None]] + + +def _format_tool_names(tool_call: Any) -> str: + if isinstance(tool_call, list): + names = [getattr(item, "name", "") for item in tool_call] + names = [name for name in names if name] + return ", ".join(names) if names else "<unknown>" + name = getattr(tool_call, "name", "") + return name or "<unknown>" + + +def _looks_like_tool_call(value: Any) -> bool: + if isinstance(value, list): + return all(hasattr(item, "name") for item in value) + return hasattr(value, "name") + + +def _drop_none_tool_arguments(tool_call: Any) -> None: + calls = tool_call if isinstance(tool_call, list) else [tool_call] + for current_call in calls: + raw_arguments = getattr(current_call, "arguments", None) + if not isinstance(raw_arguments, str): + continue + try: + parsed_arguments = json.loads(raw_arguments) + except json.JSONDecodeError: + continue + + cleaned_arguments = SchemaUtils.remove_none_values(parsed_arguments) + if cleaned_arguments is None: + cleaned_arguments = {} + if cleaned_arguments != parsed_arguments: + current_call.arguments = json.dumps(cleaned_arguments, ensure_ascii=False) + + +def _normalize_execute_call(args: tuple[Any, ...], kwargs: dict[str, Any]) -> tuple[Any, Any, Any, Any, dict[str, Any]]: + """Accept both legacy and current ability_manager.execute call shapes.""" + ctx = kwargs.get("ctx") + tool_call = kwargs.get("tool_call") + session = kwargs.get("session") + tag = kwargs.get("tag") + extra_kwargs = {k: v for k, v in kwargs.items() if k not in {"ctx", "tool_call", "session", "tag"}} + positional = list(args) + + if tool_call is None and positional: + if len(positional) >= 3 and not _looks_like_tool_call(positional[0]): + if ctx is None: + ctx = positional.pop(0) + elif len(positional) >= 4 and ctx is None: + ctx = positional.pop(0) + + if tool_call is None and positional: + tool_call = positional.pop(0) + if session is None and positional: + session = positional.pop(0) + if tag is None and positional: + tag = positional.pop(0) + + return ctx, tool_call, session, tag, extra_kwargs + + +async def _notify_tool_results( + tool_call: Any, + results: Any, + observer: ToolResultObserver, +) -> None: + calls = tool_call if isinstance(tool_call, list) else [tool_call] + result_items = results if isinstance(results, list) else [] + for idx, current_call in enumerate(calls): + if idx >= len(result_items): + break + current_result = result_items[idx] + tool_result = current_result[0] if isinstance(current_result, tuple) and current_result else current_result + tool_name = getattr(current_call, "name", "") or "<tool>" + await observer(tool_name, tool_result) + + +def ensure_execute_signature_compat( + agent: ReActAgent, + *, + tool_result_observer: ToolResultObserver | None = None, + tool_timeout_s: float | None = None, +) -> None: + """Wrap ability_manager.execute with signature compatibility and timeout.""" + execute_fn = getattr(agent.ability_manager, "execute", None) + if execute_fn is None: + return + if getattr(execute_fn, "_playwright_timeout_wrapped", False): + return + + try: + params = inspect.signature(execute_fn).parameters + except (TypeError, ValueError): + return + + original_execute = execute_fn + supports_ctx = "ctx" in params + supports_tag = "tag" in params + supports_kwargs = any(param.kind == inspect.Parameter.VAR_KEYWORD for param in params.values()) + if tool_timeout_s is None: + tool_timeout_s = CoreSettings.from_process_env().tool_timeout_s + + async def execute_with_timeout(*args, **kwargs): + ctx, tool_call, session, tag, extra_kwargs = _normalize_execute_call(args, kwargs) + tool_names = _format_tool_names(tool_call) + _drop_none_tool_arguments(tool_call) + call_kwargs: dict[str, Any] = {} + if supports_ctx: + call_kwargs["ctx"] = ctx + if "tool_call" in params: + call_kwargs["tool_call"] = tool_call + if "session" in params: + call_kwargs["session"] = session + if supports_tag: + call_kwargs["tag"] = tag + if supports_kwargs: + call_kwargs.update(extra_kwargs) + try: + with anyio.fail_after(tool_timeout_s): + results = await original_execute(**call_kwargs) + except TimeoutError as exc: + logger.error(f"Tool execution timed out after {tool_timeout_s:.1f}s; tools={tool_names}") + raise RuntimeError(f"tool_execution_timeout: tools={tool_names}, timeout_s={tool_timeout_s:.1f}") from exc + if tool_result_observer is not None: + await _notify_tool_results(tool_call, results, tool_result_observer) + return results + + agent.ability_manager.execute = execute_with_timeout + setattr(agent.ability_manager.execute, "_playwright_timeout_wrapped", True) + + +def _normalize_output_subdir(subdir: str, default: str) -> str: + return (subdir or default).strip().replace("\\", "/").strip("/") or default + + +def worker_driver_name(info: DriverInfo) -> str: + return info.display_name or info.name + + +def default_worker_prompt( + screenshot_subdir: str, + artifacts_subdir: str, + *, + tool_names: tuple[str, ...] | None = None, + driver_name: str, +) -> str: + """The worker's system prompt for a surface that brings none.""" + tools = f" ({', '.join(tool_names)})" if tool_names else "" + return ( + f"You are a worker agent. Complete the task with the {driver_name} tools{tools} only. " + f"Save screenshots under {screenshot_subdir}/ and other files under {artifacts_subdir}/. " + "When the task is done, answer with the result." + ) + + +# pylint: disable=too-many-arguments +def build_browser_worker_agent( + provider: str, + api_key: str, + *, + api_base: str, + model_name: str, + max_steps: int, + tools: Iterable[Tool] = (), + screenshot_subdir: str = "screenshots", + artifacts_subdir: str = "artifacts", + tool_result_observer: ToolResultObserver | None = None, + settings: CoreSettings | None = None, + worker_prompt: Callable[..., str], + driver_name: str, +) -> ReActAgent: + """The worker ReActAgent with ``tools`` as its abilities; ``worker_prompt`` renders its system prompt.""" + screenshot_subdir = _normalize_output_subdir(screenshot_subdir, "screenshots") + artifacts_subdir = _normalize_output_subdir(artifacts_subdir, "artifacts") + card = AgentCard( + id="agent.playwright.browser_worker", + name="playwright_browser_worker", + description=f"Worker that executes tasks using {driver_name} tools.", + input_params={}, + ) + config = ( + ReActAgentConfig() + .configure_model_client( + provider=provider, + api_key=api_key, + api_base=api_base, + model_name=model_name, + ) + .configure_max_iterations(max_steps) + .configure_prompt_template( + [ + { + "role": "system", + "content": worker_prompt(screenshot_subdir, artifacts_subdir, driver_name=driver_name), + } + ] + ) + ) + settings = settings or CoreSettings.from_process_env() + if config.model_config_obj is not None: + config.model_config_obj.temperature = settings.worker_temperature + config.model_config_obj.top_p = settings.worker_top_p + agent = ReActAgent(card=card).configure(config) + for tool in tools: + agent.ability_manager.add_ability(tool.card, tool) + ensure_execute_signature_compat( + agent, tool_result_observer=tool_result_observer, tool_timeout_s=settings.tool_timeout_s + ) + return agent + + +async def build_worker( + session: DriverSession, + bundle: SurfaceBundle, + *, + provider: str, + api_key: str, + api_base: str, + model_name: str, + max_steps: int, + core: CoreSettings, + capabilities: Iterable[str], + tool_result_observer: ToolResultObserver, + screenshot_subdir: str, + artifacts_subdir: str, + image_support: bool = False, +) -> ReActAgent: + """The worker over a started session: the factory's worker tools, the core rails, the final-answer recovery.""" + parts = build_agent_parts( + session, bundle, capabilities=capabilities, image_support=image_support, language=WORKER_LANGUAGE, worker=True + ) + worker = build_browser_worker_agent( + provider=provider, + api_key=api_key, + api_base=api_base, + model_name=model_name, + max_steps=max_steps, + tools=parts.tools, + screenshot_subdir=screenshot_subdir, + artifacts_subdir=artifacts_subdir, + tool_result_observer=tool_result_observer, + settings=core, + worker_prompt=bundle.profile.worker_prompt or default_worker_prompt, + driver_name=worker_driver_name(session.assembly.stack.info), + ) + rails = core_rails(bundle.profile, parts.builders, session.page_state, lambda: session) + for rail in (*rails, *parts.rails, FinalAnswerRecoveryRail()): + await worker.register_rail(rail) + return worker + + +async def refresh_browser_worker_system_prompt( + agent: ReActAgent, + *, + worker_prompt: Callable[..., str], + screenshot_subdir: str = "screenshots", + artifacts_subdir: str = "artifacts", + driver_name: str, +) -> None: + """Re-render the worker system prompt for the tools its next invoke sends to the model.""" + tool_names = tuple(tool.name for tool in await agent.ability_manager.list_tool_info()) + logger.info("Worker prompt rendered for tools: %s", tool_names) + prompt = worker_prompt( + _normalize_output_subdir(screenshot_subdir, "screenshots"), + _normalize_output_subdir(artifacts_subdir, "artifacts"), + tool_names=tool_names, + driver_name=driver_name, + ) + agent.config.configure_prompt_template([{"role": "system", "content": prompt}]) diff --git a/openjiuwen/harness/tools/browser_move/runtime/assembly.py b/openjiuwen/harness/tools/browser_move/runtime/assembly.py new file mode 100644 index 0000000000..dd58aac722 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/assembly.py @@ -0,0 +1,268 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The composition root: builds the driver stack and the agent from registry names only.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from typing import Any + +from openjiuwen.core.common.logging import logger +from openjiuwen.core.foundation.llm import Model +from openjiuwen.core.foundation.tool import Tool +from openjiuwen.core.single_agent.rail.base import AgentRail +from openjiuwen.harness.deep_agent import DeepAgent +from openjiuwen.harness.factory import create_deep_agent + +from ..contract import ConnectionLost, Driver, Registry, SurfaceBundle, SurfaceProfile +from ..drivers import DRIVERS +from ..drivers.middleware import RetryOnStale, Timeouts, Tracing +from ..provisioning import PROVISIONERS, ProvisionedDevice, Provisioner +from ..surfaces import SURFACES +from .config import CoreSettings +from .driver_session import DriverSession +from .page_state import PageState +from .rails.action_policy import ActionPolicy +from .rails.connection_lost import ConnectionLostRail +from .rails.intercepted_action import InterceptedActionHintRail +from .rails.repeated_call_guard import RepeatedCallGuardRail +from .tool_factory import Builder, ToolContext, ToolFactory + + +@dataclass(frozen=True) +class Registries: + drivers: Registry[Driver] + surfaces: Registry[SurfaceBundle] + provisioners: Registry[Provisioner] + + +@dataclass(frozen=True) +class DriverAssembly: + """A started driver stack, its device and surface; ``rebuild`` makes a stack for a new device.""" + + stack: Driver + device: ProvisionedDevice + provisioner: Provisioner + rebuild: Callable[[ProvisionedDevice], Driver] | None = field(default=None, compare=False) + surface: SurfaceBundle | None = None + + async def close(self) -> None: + try: + await self.stack.stop() + finally: + await asyncio.to_thread(self.provisioner.release, self.device) + + +@dataclass(frozen=True) +class AgentParts: + tools: tuple[Tool, ...] + prompt_sections: tuple[str, ...] + rails: tuple[AgentRail, ...] + builders: Mapping[str, Builder] = field(default_factory=dict) + + +def _log_trace(record: dict[str, Any]) -> None: + logger.debug("driver trace: %s", record) + + +def _stack_builder(core: CoreSettings, surface: SurfaceBundle, drivers: Registry[Driver]): + def build(device: ProvisionedDevice) -> Driver: + driver = drivers.create(core.driver, device, surface.settings) + if core.surface not in driver.info.surfaces: + served = ", ".join(sorted(driver.info.surfaces)) or "none" + raise ValueError(f"driver {core.driver!r} does not serve surface {core.surface!r} (it serves: {served})") + per_effect = {effect: min(seconds, core.tool_timeout_s) for effect, seconds in surface.profile.timeouts.items()} + return Tracing(RetryOnStale(Timeouts(driver, per_effect)), _log_trace) + + return build + + +def assemble_surface( + core: CoreSettings, + environ: Mapping[str, str], + *, + instance: Any = None, + settings: Any = None, + registries: Registries | None = None, +) -> SurfaceBundle: + """The surface by name from ``core``; ``instance`` is its per-agent identity, ``settings`` replaces ``environ``.""" + registries = registries or Registries(DRIVERS, SURFACES, PROVISIONERS) + return registries.surfaces.create(core.surface, environ, instance, settings) + + +async def assemble_driver( + core: CoreSettings, + environ: Mapping[str, str], + *, + instance: Any = None, + registries: Registries | None = None, + settings: Any = None, +) -> DriverAssembly: + """Build the surface, provisioner, driver and middleware named in ``core`` and start the stack.""" + registries = registries or Registries(DRIVERS, SURFACES, PROVISIONERS) + surface = assemble_surface(core, environ, instance=instance, settings=settings, registries=registries) + provisioner = registries.provisioners.create(surface.settings.provisioning) + build_stack = _stack_builder(core, surface, registries.drivers) + device = await asyncio.to_thread(provisioner.provision, surface.settings) + try: + stack = build_stack(device) + except BaseException: + await asyncio.to_thread(provisioner.release, device) + raise + assembly = DriverAssembly(stack, device, provisioner, build_stack, surface) + try: + await stack.start() + except BaseException: + await assembly.close() + raise + return assembly + + +async def open_session( + core: CoreSettings, + environ: Mapping[str, str], + *, + page_state: PageState, + instance: Any = None, + settings: Any = None, + registries: Registries | None = None, +) -> DriverSession: + """A session over a started stack; a first start that loses its connection is provisioned once more.""" + try: + assembly = await assemble_driver(core, environ, instance=instance, registries=registries, settings=settings) + except ConnectionLost: + assembly = await assemble_driver(core, environ, instance=instance, registries=registries, settings=settings) + return DriverSession(assembly, page_state, assembly.surface.profile.messages) + + +def new_page_state(profile: SurfaceProfile) -> PageState: + """The surface's ``PageState`` subclass when its profile brings one, else the core's own.""" + return PageState() if profile.page_state is None else profile.page_state() + + +def core_rails( + profile: SurfaceProfile, + builders: Mapping[str, Builder], + page_state: PageState, + session_of: Callable[[], DriverSession | None], +) -> tuple[AgentRail, ...]: + """The policy, the repeated-call guard, the intercepted-action hint and the lost-connection stop. + + ``builders`` is the live mapping and ``session_of`` the live lookup: both may fill after the rails are built. + """ + return ( + ActionPolicy(builders, profile.policy_rules, page_state), + RepeatedCallGuardRail(), + InterceptedActionHintRail(profile.messages["retry"]), + ConnectionLostRail(session_of), + ) + + +def build_agent_parts( + session: DriverSession, + bundle: SurfaceBundle, + *, + capabilities: Iterable[str], + image_support: bool, + language: str, + workspace: Any = None, + worker: bool = False, +) -> AgentParts: + """Tools and prompt sections from the profile and the driver, and the profile's rails.""" + profile = bundle.profile + info = session.assembly.stack.info + tools = ToolFactory(profile, info).build( + capabilities, + image_support=image_support, + session=session, + page_state=session.page_state, + workspace=workspace, + language=language, + worker=worker, + ) + context = ToolContext( + session, session.page_state, profile.renderer, info, profile.settings, image_support, workspace, language + ) + rails = tuple(factory(context) for factory in profile.rails) + return AgentParts(tools.tools, tools.prompt_sections, rails, tools.builders) + + +def agent_over( + parts: AgentParts, + rails: Iterable[AgentRail], + model: Model, + *, + language: str, + workspace: Any = None, + system_prompt: str = "", + **agent_options: Any, +) -> DeepAgent: + """A deep agent with the parts' tools and prompt sections; ``rails`` go before the profile's own.""" + return create_deep_agent( + model, + system_prompt="\n\n".join(text for text in (system_prompt, *parts.prompt_sections) if text), + tools=list(parts.tools), + rails=[*rails, *parts.rails], + workspace=workspace, + language=language, + **agent_options, + ) + + +async def build_agent( + core: CoreSettings, + environ: Mapping[str, str], + *, + model: Model, + capabilities: Iterable[str], + image_support: bool, + language: str, + system_prompt: str = "", + workspace: Any = None, + instance: Any = None, + registries: Registries | None = None, + **agent_options: Any, +) -> tuple[DeepAgent, DriverSession]: + """A deep agent over a started driver session; the caller closes the session when the agent is done.""" + assembly = await assemble_driver(core, environ, instance=instance, registries=registries) + session = DriverSession(assembly, new_page_state(assembly.surface.profile), assembly.surface.profile.messages) + try: + parts = build_agent_parts( + session, + assembly.surface, + capabilities=capabilities, + image_support=image_support, + language=language, + workspace=workspace, + ) + agent = agent_over( + parts, + core_rails(assembly.surface.profile, parts.builders, session.page_state, lambda: session), + model, + language=language, + workspace=workspace, + system_prompt=system_prompt, + **agent_options, + ) + except BaseException: + await session.close() + raise + return agent, session + + +__all__ = [ + "AgentParts", + "DriverAssembly", + "Registries", + "agent_over", + "assemble_driver", + "assemble_surface", + "build_agent", + "build_agent_parts", + "core_rails", + "new_page_state", + "open_session", +] diff --git a/openjiuwen/harness/tools/browser_move/runtime/config.py b/openjiuwen/harness/tools/browser_move/runtime/config.py new file mode 100644 index 0000000000..2056e42fad --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/config.py @@ -0,0 +1,396 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The core settings: driver, surface, model, guardrails, logging and the runtime server, read from one mapping.""" + +from __future__ import annotations + +import logging +import os +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Mapping + +from ..surfaces import DEFAULT_SURFACE +from . import REPO_ROOT +from .utils.env import ( + DEFAULT_BROWSER_TIMEOUT_S, + DEFAULT_GUARDRAIL_MAX_FAILURES, + DEFAULT_GUARDRAIL_MAX_STEPS, + DEFAULT_GUARDRAIL_RETRY_ONCE, + DEFAULT_MODEL_NAME, + FALSY_ENV_VALUES, + MISSING_API_KEY_MESSAGE, # noqa: F401 + first_value, + infer_provider_from_api_base, + int_value, + is_falsy_env, + is_truthy_env, + normalize_provider, +) + + +@dataclass +class RunGuardrails: + max_steps: int = DEFAULT_GUARDRAIL_MAX_STEPS + max_failures: int = DEFAULT_GUARDRAIL_MAX_FAILURES + timeout_s: int = DEFAULT_BROWSER_TIMEOUT_S + retry_once: bool = True + resume_on_max_iterations: bool = False + + +DEFAULT_DRIVER = "playwright_mcp" +TOPOLOGIES = ("direct", "delegated_worker") +DEFAULT_TOOL_TIMEOUT_S = 240.0 +DEFAULT_WORKER_TEMPERATURE = 0.2 +DEFAULT_WORKER_TOP_P = 0.1 +DEFAULT_LOG_FILE = Path("logs") / "browser_agent.log" +_LEGACY_DRIVER_VALUES = frozenset({"managed", "remote", "extension"}) +_LOG_FLAG_OFF = FALSY_ENV_VALUES | {""} +_LOG_FILE_OFF = FALSY_ENV_VALUES | {"none", "null", "-"} +_BASE_URL_KEYS = ( + "OPENROUTER_BASE_URL", + "OPENROUTER_API_BASE", + "SILICONFLOW_BASE_URL", + "SILICONFLOW_API_BASE", + "DASHSCOPE_BASE_URL", + "DASHSCOPE_API_BASE", + "OPENAI_BASE_URL", + "OPENAI_API_BASE", +) +_PROVIDER_ENDPOINTS: dict[str, tuple[tuple[str, ...], tuple[str, ...], str]] = { + "openrouter": ( + ("OPENROUTER_API_KEY", "OPENAI_API_KEY"), + ("OPENROUTER_BASE_URL", "OPENROUTER_API_BASE"), + "https://openrouter.ai/api/v1", + ), + "siliconflow": ( + ("SILICONFLOW_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY"), + ("SILICONFLOW_BASE_URL", "SILICONFLOW_API_BASE"), + "https://api.siliconflow.cn/v1", + ), + "dashscope": ( + ("DASHSCOPE_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY"), + ("DASHSCOPE_BASE_URL", "DASHSCOPE_API_BASE"), + "https://dashscope.aliyuncs.com/compatible-mode/v1", + ), + "openai": ( + ("OPENAI_API_KEY", "OPENROUTER_API_KEY"), + ("OPENAI_BASE_URL", "OPENAI_API_BASE"), + "https://api.openai.com/v1", + ), +} + + +def _bool(environ: Mapping[str, str], key: str, *, default: bool) -> bool: + value = environ.get(key) or "" + if is_truthy_env(value): + return True + if is_falsy_env(value): + return False + return default + + +def _float(environ: Mapping[str, str], keys: tuple[str, ...], *, default: float, maximum: float) -> float: + for key in keys: + try: + value = float((environ.get(key) or "").strip()) + except ValueError: + continue + if 0.0 <= value <= maximum: + return value + return default + + +def _driver(environ: Mapping[str, str]) -> str: + value = (environ.get("BROWSER_DRIVER") or "").strip().lower() + return DEFAULT_DRIVER if not value or value in _LEGACY_DRIVER_VALUES else value + + +def _topology(environ: Mapping[str, str]) -> str: + value = (environ.get("BROWSER_TOPOLOGY") or "").strip().lower() + if value and value not in TOPOLOGIES: + raise ValueError(f"BROWSER_TOPOLOGY must be one of: {', '.join(TOPOLOGIES)} (got {value!r})") + return value + + +def _provider(environ: Mapping[str, str]) -> str: + raw = first_value(environ, "MODEL_PROVIDER", "MODEL_CLIENT_PROVIDER") + if raw: + return normalize_provider(raw) + api_base = first_value(environ, "API_BASE", "MODEL_API_BASE") or first_value(environ, *_BASE_URL_KEYS) + inferred = infer_provider_from_api_base(api_base) + if inferred: + return inferred + for provider, key in ( + ("openrouter", "OPENROUTER_API_KEY"), + ("siliconflow", "SILICONFLOW_API_KEY"), + ("dashscope", "DASHSCOPE_API_KEY"), + ): + if first_value(environ, key): + return provider + return "openai" + + +def _tool_timeout_s(environ: Mapping[str, str]) -> float: + raw = ( + environ.get("PLAYWRIGHT_TOOL_TIMEOUT_S") + or environ.get("PLAYWRIGHT_MCP_TIMEOUT_S") + or environ.get("BROWSER_TIMEOUT_S") + ) + try: + value = float(raw or DEFAULT_TOOL_TIMEOUT_S) + except ValueError: + return DEFAULT_TOOL_TIMEOUT_S + return value if value > 0 else DEFAULT_TOOL_TIMEOUT_S + + +def _guardrails(environ: Mapping[str, str]) -> RunGuardrails: + return RunGuardrails( + max_steps=int_value(environ, "BROWSER_GUARDRAIL_MAX_STEPS", default=DEFAULT_GUARDRAIL_MAX_STEPS, minimum=1), + max_failures=int_value( + environ, "BROWSER_GUARDRAIL_MAX_FAILURES", default=DEFAULT_GUARDRAIL_MAX_FAILURES, minimum=0 + ), + timeout_s=int_value( + environ, "BROWSER_TIMEOUT_S", "PLAYWRIGHT_TOOL_TIMEOUT_S", default=DEFAULT_BROWSER_TIMEOUT_S, minimum=1 + ), + retry_once=_bool(environ, "BROWSER_GUARDRAIL_RETRY_ONCE", default=DEFAULT_GUARDRAIL_RETRY_ONCE), + resume_on_max_iterations=_bool(environ, "BROWSER_GUARDRAIL_RESUME_ON_MAX_ITERATIONS", default=False), + ) + + +def _log_flag(environ: Mapping[str, str], key: str) -> bool: + value = environ.get(key) + return value is not None and value.strip().lower() not in _LOG_FLAG_OFF + + +def _log_file(environ: Mapping[str, str]) -> Path | None: + configured = environ.get("OPENJIUWEN_BROWSER_AGENT_LOG_FILE") + if configured is None: + return DEFAULT_LOG_FILE + configured = configured.strip() + if configured.lower() in _LOG_FILE_OFF: + return None + return Path(configured).expanduser() + + +def _log_level(environ: Mapping[str, str]) -> int: + name = environ.get("OPENJIUWEN_BROWSER_AGENT_LOG_LEVEL", "INFO").strip().upper() + level = getattr(logging, name, logging.INFO) + return level if isinstance(level, int) else logging.INFO + + +def _timeout_or_zero(raw: str) -> int: + try: + return int(raw) + except ValueError: + return 0 + + +def _optional_path(raw: str) -> Path | None: + return Path(raw).expanduser() if raw else None + + +@dataclass(frozen=True) +class RuntimeServerSettings: + """The runtime MCP server (``playwright_runtime_mcp_server``): how the agent reaches it, how it listens.""" + + enabled: bool = False + server_id: str = "playwright_runtime_wrapper" + server_name: str = "playwright-runtime-wrapper" + client_type: str = "streamable-http" + server_path: str = "" + command: str = sys.executable + args: str = "" + timeout_s: int = 300 + host: str = "127.0.0.1" + port: str = "8940" + path: str = "" + log_dir: Path | None = None + auto_http_fallback: bool = True + transport: str = "stdio" + log_level: str = "INFO" + stateless_http: bool | None = None + + @classmethod + def from_env(cls, environ: Mapping[str, str]) -> RuntimeServerSettings: + """Client side: ``PLAYWRIGHT_RUNTIME_*`` first, then the ``BROWSER_RUNTIME_*`` aliases.""" + auto_http_fallback = first_value(environ, "PLAYWRIGHT_RUNTIME_MCP_AUTO_HTTP_FALLBACK") + return cls( + enabled=is_truthy_env( + first_value(environ, "PLAYWRIGHT_RUNTIME_MCP_ENABLED", "BROWSER_RUNTIME_MCP_ENABLED") + ), + server_id=first_value( + environ, "PLAYWRIGHT_RUNTIME_MCP_SERVER_ID", "BROWSER_RUNTIME_MCP_SERVER_ID", default=cls.server_id + ), + server_name=first_value( + environ, + "PLAYWRIGHT_RUNTIME_MCP_SERVER_NAME", + "BROWSER_RUNTIME_MCP_SERVER_NAME", + default=cls.server_name, + ), + client_type=first_value( + environ, + "PLAYWRIGHT_RUNTIME_MCP_CLIENT_TYPE", + "BROWSER_RUNTIME_MCP_CLIENT_TYPE", + default=cls.client_type, + ), + server_path=first_value(environ, "PLAYWRIGHT_RUNTIME_MCP_SERVER_PATH", "BROWSER_RUNTIME_MCP_SERVER_PATH"), + command=first_value( + environ, "PLAYWRIGHT_RUNTIME_MCP_COMMAND", "BROWSER_RUNTIME_MCP_COMMAND", default=sys.executable + ), + args=first_value(environ, "PLAYWRIGHT_RUNTIME_MCP_ARGS", "BROWSER_RUNTIME_MCP_ARGS"), + timeout_s=_timeout_or_zero( + first_value(environ, "PLAYWRIGHT_RUNTIME_MCP_TIMEOUT_S", "BROWSER_RUNTIME_MCP_TIMEOUT_S", default="300") + ), + host=first_value(environ, "PLAYWRIGHT_RUNTIME_MCP_HOST", "BROWSER_RUNTIME_MCP_HOST", default=cls.host), + port=first_value(environ, "PLAYWRIGHT_RUNTIME_MCP_PORT", "BROWSER_RUNTIME_MCP_PORT", default=cls.port), + path=first_value(environ, "PLAYWRIGHT_RUNTIME_MCP_PATH", "BROWSER_RUNTIME_MCP_PATH"), + log_dir=_optional_path(first_value(environ, "PLAYWRIGHT_RUNTIME_LOG_DIR", "BROWSER_RUNTIME_LOG_DIR")), + auto_http_fallback=is_truthy_env(auto_http_fallback) if auto_http_fallback else True, + ) + + @classmethod + def server_from_env(cls, environ: Mapping[str, str]) -> RuntimeServerSettings: + """Server entry point: ``PLAYWRIGHT_RUNTIME_*`` only (the log dir also takes its alias), its own defaults.""" + stateless_http = environ.get("PLAYWRIGHT_RUNTIME_MCP_STATELESS_HTTP") or "" + return cls( + transport=(environ.get("PLAYWRIGHT_RUNTIME_MCP_TRANSPORT") or "stdio").strip().lower(), + host=(environ.get("PLAYWRIGHT_RUNTIME_MCP_HOST") or "127.0.0.1").strip(), + port=(environ.get("PLAYWRIGHT_RUNTIME_MCP_PORT") or "8940").strip(), + path=(environ.get("PLAYWRIGHT_RUNTIME_MCP_PATH") or "").strip(), + log_level=(environ.get("PLAYWRIGHT_RUNTIME_MCP_LOG_LEVEL") or "INFO").strip(), + log_dir=_optional_path(first_value(environ, "PLAYWRIGHT_RUNTIME_LOG_DIR", "BROWSER_RUNTIME_LOG_DIR")), + stateless_http=True if is_truthy_env(stateless_http) else False if is_falsy_env(stateless_http) else None, + ) + + +@dataclass(frozen=True) +class CoreSettings: + """Settings of the agent core: driver, surface, model, guardrails and logging; nothing surface-specific.""" + + driver: str = DEFAULT_DRIVER + surface: str = DEFAULT_SURFACE + topology: str = "" + provider: str = "openai" + api_key: str = "" + api_base: str = _PROVIDER_ENDPOINTS["openai"][2] + model_name: str = DEFAULT_MODEL_NAME + worker_temperature: float = DEFAULT_WORKER_TEMPERATURE + worker_top_p: float = DEFAULT_WORKER_TOP_P + tool_timeout_s: float = DEFAULT_TOOL_TIMEOUT_S + guardrails: RunGuardrails = field(default_factory=RunGuardrails) + log_file: Path | None = DEFAULT_LOG_FILE + log_level: int = logging.INFO + log_mirror_common: bool = False + log_audit_raw: bool = False + status_log: bool = True + runtime_server: RuntimeServerSettings = field(default_factory=RuntimeServerSettings) + + @classmethod + def from_env(cls, environ: Mapping[str, str]) -> CoreSettings: + provider = _provider(environ) + key_names, base_names, default_base = _PROVIDER_ENDPOINTS.get(provider, _PROVIDER_ENDPOINTS["openai"]) + return cls( + driver=_driver(environ), + topology=_topology(environ), + provider=provider, + api_key=first_value(environ, "API_KEY", "MODEL_API_KEY", *key_names), + api_base=first_value(environ, "API_BASE", "MODEL_API_BASE", *base_names) or default_base, + model_name=first_value(environ, "MODEL_NAME") or DEFAULT_MODEL_NAME, + worker_temperature=_float( + environ, + ("BROWSER_WORKER_TEMPERATURE", "BROWSER_MODEL_TEMPERATURE", "MODEL_TEMPERATURE"), + default=DEFAULT_WORKER_TEMPERATURE, + maximum=2.0, + ), + worker_top_p=_float( + environ, + ("BROWSER_WORKER_TOP_P", "BROWSER_MODEL_TOP_P", "MODEL_TOP_P"), + default=DEFAULT_WORKER_TOP_P, + maximum=1.0, + ), + tool_timeout_s=_tool_timeout_s(environ), + guardrails=_guardrails(environ), + log_file=_log_file(environ), + log_level=_log_level(environ), + log_mirror_common=_log_flag(environ, "OPENJIUWEN_BROWSER_AGENT_LOG_MIRROR_COMMON"), + log_audit_raw=_log_flag(environ, "OPENJIUWEN_BROWSER_AGENT_AUDIT_RAW"), + status_log=not is_falsy_env(environ.get("BROWSER_SUBAGENT_STATUS_LOG") or ""), + runtime_server=RuntimeServerSettings.from_env(environ), + ) + + @classmethod + def from_process_env(cls) -> CoreSettings: + return cls.from_env(os.environ) + + @classmethod + def logging_from_process_env(cls) -> CoreSettings: + """Default settings with only the log fields read, so a bad unrelated variable cannot break logging.""" + environ = os.environ + return cls( + log_file=_log_file(environ), + log_level=_log_level(environ), + log_mirror_common=_log_flag(environ, "OPENJIUWEN_BROWSER_AGENT_LOG_MIRROR_COMMON"), + log_audit_raw=_log_flag(environ, "OPENJIUWEN_BROWSER_AGENT_AUDIT_RAW"), + ) + + +def process_environ() -> dict[str, str]: + """A snapshot of the process environment, for entry points and child-process environments.""" + return dict(os.environ) + + +def resolve_repo_dotenv_path() -> Path: + return Path(REPO_ROOT).expanduser().resolve() / ".env" + + +def load_repo_dotenv(*, override: bool = False) -> bool: + env_path = resolve_repo_dotenv_path() + if not env_path.exists(): + return False + try: + from dotenv import load_dotenv + except ImportError: + return False + return bool(load_dotenv(dotenv_path=env_path, override=override)) + + +def runtime_server_environ(environ: Mapping[str, str]) -> dict[str, str]: + """``environ`` with the runtime MCP server's tool timeout defaults filled in.""" + env = dict(environ) + env.setdefault("BROWSER_TIMEOUT_S", str(DEFAULT_BROWSER_TIMEOUT_S)) + env.setdefault("PLAYWRIGHT_TOOL_TIMEOUT_S", env["BROWSER_TIMEOUT_S"]) + return env + + +_PROXY_BLOCKLIST = frozenset({"http://127.0.0.1:9", "http://localhost:9"}) + + +def runtime_server_child_env(environ: Mapping[str, str]) -> dict[str, str]: + """Environment of the runtime MCP server child: ``environ`` with API_KEY / API_BASE under provider names.""" + env = dict(environ) + api_key = (env.get("API_KEY") or "").strip() + api_base = (env.get("API_BASE") or "").strip() + model_provider = (env.get("MODEL_PROVIDER") or "").strip().lower() + + if api_key and not env.get("OPENROUTER_API_KEY") and "openrouter.ai" in api_base: + env["OPENROUTER_API_KEY"] = api_key + if api_base and not env.get("OPENROUTER_BASE_URL") and "openrouter.ai" in api_base: + env["OPENROUTER_BASE_URL"] = api_base + + if api_key and not env.get("OPENAI_API_KEY") and "openrouter.ai" not in api_base: + env["OPENAI_API_KEY"] = api_key + if api_base and not env.get("OPENAI_BASE_URL") and "openrouter.ai" not in api_base: + env["OPENAI_BASE_URL"] = api_base + + if model_provider == "openrouter": + env["MODEL_PROVIDER"] = "openrouter" + elif model_provider in {"openai", "siliconflow"}: + env["MODEL_PROVIDER"] = model_provider + + for proxy_key in ("HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY"): + proxy_value = (env.get(proxy_key) or "").strip().lower() + if proxy_value in _PROXY_BLOCKLIST: + env.pop(proxy_key, None) + return env diff --git a/openjiuwen/harness/tools/browser_move/runtime/driver_session.py b/openjiuwen/harness/tools/browser_move/runtime/driver_session.py new file mode 100644 index 0000000000..45dd789826 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/driver_session.py @@ -0,0 +1,114 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Runs commands on the assembled driver stack and recovers from one lost connection per task.""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +from collections.abc import Mapping +from typing import TYPE_CHECKING, NoReturn + +from openjiuwen.core.common.exception.codes import StatusCode +from openjiuwen.core.common.exception.errors import BaseError, build_error + +from ..contract import Action, ActionResult, ConnectionLost, DriverError, Health, Observation, Observe +from .page_state import PageState + +if TYPE_CHECKING: + from .assembly import DriverAssembly + + +class DriverSession: + """Runs commands on the stack; on ``ConnectionLost`` it restarts once per task and never re-sends.""" + + def __init__(self, assembly: DriverAssembly, page_state: PageState, messages: Mapping[str, str]) -> None: + self.assembly = assembly + self._page_state = page_state + self._restarted = messages["driver_restarted"] + self._lost = messages["driver_lost"] + self._losses = 0 + self._ended = False + + @property + def page_state(self) -> PageState: + return self._page_state + + @property + def lost(self) -> str | None: + """The ``driver_lost`` message once a loss ended the task, ``None`` while the stack can still serve it.""" + return self._lost if self._ended else None + + def begin_task(self) -> None: + self._losses = 0 + self._ended = False + + async def execute(self, action: Action) -> ActionResult: + try: + return await self.assembly.stack.execute(action) + except ConnectionLost as error: + await self._recover(error) + + async def observe(self, command: Observe) -> Observation: + try: + return await self.assembly.stack.observe(command) + except ConnectionLost as error: + await self._recover(error) + + async def stop(self) -> None: + """Stop the stack and keep the device, between tasks.""" + await self.assembly.stack.stop() + + async def start(self) -> None: + """Start the stack again on a live device; a dead one is released and provisioned again first.""" + try: + await self._start() + except ConnectionLost as error: + await self._restart(error) + + async def _start(self) -> None: + assembly = self.assembly + if await asyncio.to_thread(assembly.provisioner.is_alive, assembly.device): + await assembly.stack.start() + return + provisioner = assembly.provisioner + await asyncio.to_thread(provisioner.release, assembly.device) + device = await asyncio.to_thread(provisioner.provision, assembly.surface.settings) + try: + stack = assembly.rebuild(device) + except BaseException: + await asyncio.to_thread(provisioner.release, device) + raise + self.assembly = dataclasses.replace(assembly, stack=stack, device=device) + await stack.start() + + async def health(self) -> Health: + return await self.assembly.stack.health() + + async def close(self) -> None: + await self.assembly.close() + + async def _recover(self, error: ConnectionLost) -> NoReturn: + self._losses += 1 + if self._losses > 1: + raise self._end(error) from error + await self._restart(error) + raise ConnectionLost(self._restarted, dispatched=error.dispatched) from error + + async def _restart(self, error: ConnectionLost) -> None: + try: + with contextlib.suppress(DriverError): + await self.assembly.stack.stop() + await self._start() + except Exception as restart_error: + raise self._end(restart_error) from error + self._page_state.reset() + + def _end(self, cause: Exception) -> BaseError: + self._ended = True + return build_error(StatusCode.DEEPAGENT_RUNTIME_ERROR, error_msg=self._lost, cause=cause) + + +__all__ = ["DriverSession"] diff --git a/openjiuwen/harness/tools/browser_move/runtime/page_state.py b/openjiuwen/harness/tools/browser_move/runtime/page_state.py new file mode 100644 index 0000000000..ad524085b2 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/page_state.py @@ -0,0 +1,439 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Generations of the surface state and the generation-scoped target registry.""" + +from __future__ import annotations + +import dataclasses +import re +import uuid +from collections.abc import Iterable +from dataclasses import dataclass, field +from typing import Any + +from ..contract import ( + Action, + ActionResult, + ContextRef, + DriverRef, + Locator, + Observation, + Observe, + Ref, + StaleRef, + SurfaceContext, + iter_refs, +) + +_MAX_PUBLIC_INTERACTIVES = 20 +_MAX_TARGET_HISTORY = 1024 + + +def compact_text(value: Any, limit: int = 120) -> str: + return " ".join(str(value or "").split())[:limit] + + +def _generation_number(generation_id: str) -> int: + normalized = str(generation_id or "").strip().lower() + if not re.fullmatch(r"g\d+", normalized): + raise ValueError("generation_id must use the PageState generation format, for example g3") + return int(normalized[1:]) + + +def _element_change(before: Observation, after: Observation) -> float: + """Jaccard distance between the ``(ref, role)`` sets of two observations.""" + old = {(element.ref.ref, element.role) for element in before.elements} + new = {(element.ref.ref, element.role) for element in after.elements} + union = old | new + return 1 - len(old & new) / len(union) if union else 0.0 + + +def _stale_message(ref: Ref, current: str) -> str: + if isinstance(ref, DriverRef): + return f"Stale AX ref {ref.ref} belongs to {ref.generation}; current generation is {current}." + if isinstance(ref, Locator) and ref.css: + return f"Stale selector belongs to {ref.generation}; current generation is {current}: {ref.css}" + return f"Stale {type(ref).__name__} belongs to {ref.generation}; current generation is {current}." + + +@dataclass +class Target: + """One model-visible target bound to a single state generation.""" + + target_id: str + page_id: str + generation: int + source: str + locator: dict[str, str] = field(default_factory=dict) + ref: str = "" + selector: str = "" + role: str = "" + name: str = "" + text: str = "" + visible: bool = True + enabled: bool = True + actionable: bool = False + clickable: bool = False + + @property + def generation_id(self) -> str: + return f"g{self.generation}" + + def compact_index(self) -> dict[str, Any]: + """The small index entry shown to the model.""" + result: dict[str, Any] = {"target_id": self.target_id, "generation_id": self.generation_id} + if self.role: + result["role"] = self.role + if self.text: + result["text"] = compact_text(self.text, 80) + result["match_count"] = 1 + result["visible"] = self.visible + result["enabled"] = self.enabled + result["actionable"] = self.actionable + if self.clickable: + result["clickable"] = True + return result + + +class PageState: + """Generations, the last observation and the targets minted from it; a surface subclasses it for its own state.""" + + target_type: type[Target] = Target + id_prefix = "state" + observe_again = "Observe again." + snapshot_again = "Observe again." + + def __init__( + self, + *, + page_id: str | None = None, + generation: int = 0, + element_change_threshold: float = 0.5, + ) -> None: + self.page_id = page_id or f"{self.id_prefix}-{uuid.uuid4().hex[:12]}" + self.generation = max(0, int(generation)) + self.element_change_threshold = element_change_threshold + self.identity = "" + self.last_observation: Observation | None = None + self.screenshot_generation = "" + self._dispatched_since_observation = False + self.surface_context: SurfaceContext | None = None + self.contexts: tuple[ContextRef, ...] = () + self.current_context: ContextRef | None = None + self.free_observations = 0 + self.reference_generations: dict[str, int] = {} + self.selector_generations: dict[str, int] = {} + self._targets: dict[str, Target] = {} + self._ref_targets: dict[str, str] = {} + self._selector_targets: dict[tuple[int, str], str] = {} + self._interactive_target_ids: list[str] = [] + self._target_counter = 0 + + @property + def generation_id(self) -> str: + return f"g{self.generation}" + + @property + def fresh_observation(self) -> Observation | None: + """The last observation while it is current: of this generation, with no command dispatched after it.""" + observation = self.last_observation + if observation is None or observation.generation != self.generation_id or self._dispatched_since_observation: + return None + return observation + + def note_dispatched(self) -> None: + """A command reached the surface but failed, so there is no result to record.""" + self._dispatched_since_observation = True + + def advance(self) -> None: + """Start a new generation while retaining stale-target history.""" + self.generation += 1 + self.identity = "" + self._interactive_target_ids.clear() + self._on_advance() + self._trim_target_history() + + def validate_generation(self, generation_id: str) -> None: + """Reject callers that are not using the current generation.""" + requested = _generation_number(generation_id) + if requested != self.generation: + raise ValueError( + f"Stale PageState generation {generation_id}; current generation is " + f"{self.generation_id}. {self.observe_again}" + ) + + def record(self, action: Action, outcome: ActionResult | Observation) -> ActionResult | Observation: + """Advance the generation when ``outcome`` shows a new context or document; return it stamped.""" + if isinstance(outcome, Observation) and isinstance(action, Observe) and action.scoped: + return self._register_observation(outcome, complete=False) + result = outcome if isinstance(outcome, ActionResult) else ActionResult(ok=True, observation=outcome) + observation = result.observation + if observation is not None and not isinstance(action, Observe): + self.free_observations += 1 + surface = (observation.surface if observation is not None else None) or result.surface + identity = surface.identity if surface is not None else "" + moved = bool(identity) and self.identity not in ("", identity) + if moved or self._changes_context(result): + self.advance() + self._forget_contexts() + if identity: + self.identity = identity + if surface is not None: + self.surface_context = surface + self._note_surface(surface) + if observation is not None: + observation = self._register_observation(observation) + self.current_context = observation.context + self.last_observation = observation + if observation.screenshot is not None: + self.screenshot_generation = self.generation_id + self._dispatched_since_observation = observation is None + if isinstance(outcome, Observation): + return observation + return dataclasses.replace(result, generation=self.generation_id, observation=observation) + + def check(self, action: Action) -> None: + """Raise ``StaleRef(dispatched=False)`` if any ref in ``action`` belongs to an older generation.""" + stale = [ + _stale_message(ref, self.generation_id) + for ref in iter_refs(action) + if ref.generation and _generation_number(ref.generation) < self.generation + ] + if stale: + raise StaleRef(" ".join(stale), dispatched=False) + + def generation_of(self, ref: str) -> str: + """The generation ``ref`` was last registered in; the current one for a ref never seen.""" + generation = self.reference_generations.get(ref) + return self.generation_id if generation is None else f"g{generation}" + + def resolve(self, model_id: str, generation: str) -> Ref: + """The contract ref behind a ``target_id``; ``ValueError`` when it is stale, unknown or has no locator.""" + target = self.resolve_target(generation_id=generation, target_id=model_id) + if target.ref: + return DriverRef(target.ref, target.generation_id) + if target.selector: + return Locator(css=target.selector, text=None, generation=target.generation_id) + raise ValueError(f"PageState target_id {target.target_id} has no element locator") + + def reset(self) -> None: + """Start a new generation with no contexts and no last observation, as after a driver restart.""" + self.advance() + self._forget_contexts() + + def resolve_target( + self, + *, + generation_id: str, + target_id: str = "", + ref: str = "", + selector: str = "", + require_registered_selector: bool = True, + ) -> Target | None: + """Resolve one target and reject every stale generation before execution.""" + self.validate_generation(generation_id) + normalized_target_id = str(target_id or "").strip() + normalized_ref = str(ref or "").strip() + normalized_selector = str(selector or "").strip() + + if normalized_target_id: + target = self._targets.get(normalized_target_id) + if target is None: + raise ValueError(f"Unknown PageState target_id: {normalized_target_id}") + self._validate_target_generation(target) + return target + + if normalized_ref: + ref_generation = self.reference_generations.get(normalized_ref) + if ref_generation is None: + raise ValueError(f"Unknown AX ref: {normalized_ref}. {self.snapshot_again}") + if ref_generation != self.generation: + raise ValueError( + f"Stale AX ref {normalized_ref} belongs to g{ref_generation}; " + f"current generation is {self.generation_id}." + ) + resolved_id = self._ref_targets.get(normalized_ref) + target = self._targets.get(resolved_id or "") + if target is None: + raise ValueError(f"AX ref is not registered as a target: {normalized_ref}") + self._validate_target_generation(target) + return target + + if normalized_selector: + selector_generation = self.selector_generations.get(normalized_selector) + if selector_generation is not None and selector_generation != self.generation: + raise ValueError( + f"Stale selector belongs to g{selector_generation}; current generation is " + f"{self.generation_id}: {normalized_selector}" + ) + resolved_id = self._selector_targets.get((self.generation, normalized_selector)) + target = self._targets.get(resolved_id or "") + if target is not None: + return target + if require_registered_selector: + raise ValueError( + "Unregistered action selector. Use target_id/ref from the current " + f"PageState instead of constructing CSS: {normalized_selector}" + ) + self.selector_generations[normalized_selector] = self.generation + return None + + def get_target(self, target_id: str) -> Target | None: + """An internal target, stale history included, for recovery.""" + return self._targets.get(str(target_id or "").strip()) + + def export(self) -> dict[str, Any]: + """A bounded index suitable for every result.""" + return { + "page_id": self.page_id, + "generation_id": self.generation_id, + "interactives": self._export_targets(self._interactive_target_ids, _MAX_PUBLIC_INTERACTIVES), + } + + def _note_surface(self, surface: SurfaceContext) -> None: + """A surface subclass reads its own context type here.""" + + def _on_advance(self) -> None: + """A surface subclass clears its per-generation state here.""" + + def _on_observation(self, observation: Observation) -> None: + """A surface subclass reads a complete or partial observation here.""" + + def _forget_target(self, target: Target) -> None: + """A surface subclass drops what it keeps for a target trimmed from the history.""" + + def _changes_context(self, result: ActionResult) -> bool: + if result.context_changed: + return True + observation = result.observation + if observation is None: + return False + if self.current_context is not None and observation.context.id != self.current_context.id: + return True + return ( + self.last_observation is not None + and _element_change(self.last_observation, observation) > self.element_change_threshold + ) + + def _forget_contexts(self) -> None: + self.last_observation = None + self.surface_context = None + self.contexts = () + self.current_context = None + + def _register_observation(self, observation: Observation, *, complete: bool = True) -> Observation: + elements = tuple( + dataclasses.replace(element, ref=DriverRef(element.ref.ref, self.generation_id)) + for element in observation.elements + ) + refs = {element.ref.ref for element in elements} + missing_refs = [ + ref_value + for ref_value, generation in self.reference_generations.items() + if complete and generation == self.generation and ref_value not in refs + ] + for ref_value in missing_refs: + self._remove_current_ax_ref(ref_value) + for element in elements: + self._register_ax_ref(element.ref.ref, role=element.role, name=element.name) + stamped = dataclasses.replace(observation, generation=self.generation_id, elements=elements) + self._on_observation(stamped) + return stamped + + def _register_ax_ref(self, ref_value: str, *, role: str, name: str) -> None: + self.reference_generations[ref_value] = self.generation + existing_id = self._ref_targets.get(ref_value) + existing_target = self._targets.get(existing_id or "") + if existing_target is not None and existing_target.generation == self.generation: + target = existing_target + else: + target = self._new_target( + source="ax", + locator={"ref": ref_value}, + ref=ref_value, + role=role, + name=name, + text=name, + ) + self._ref_targets[ref_value] = target.target_id + if target.target_id not in self._interactive_target_ids: + self._interactive_target_ids.append(target.target_id) + + def _new_target(self, *, source: str, locator: dict[str, str], **kwargs: Any) -> Target: + self._target_counter += 1 + target_id = f"t_g{self.generation}_{self._target_counter}" + target = self.target_type( + target_id=target_id, + page_id=self.page_id, + generation=self.generation, + source=source, + locator=dict(locator), + **kwargs, + ) + self._targets[target_id] = target + return target + + def _validate_target_generation(self, target: Target) -> None: + if target.page_id != self.page_id or target.generation != self.generation: + raise ValueError( + f"Stale target_id {target.target_id} belongs to {target.generation_id}; " + f"current generation is {self.generation_id}." + ) + + def _remove_current_ax_ref(self, ref_value: str) -> None: + target_id = self._ref_targets.get(ref_value) + target = self._targets.get(target_id or "") + if target is not None and target.generation == self.generation and target.source == "ax": + self._targets.pop(target.target_id, None) + self._interactive_target_ids = [ + current_target_id + for current_target_id in self._interactive_target_ids + if current_target_id != target.target_id + ] + if self._ref_targets.get(ref_value) == target.target_id: + self._ref_targets.pop(ref_value, None) + if self.reference_generations.get(ref_value) == self.generation: + self.reference_generations.pop(ref_value, None) + + def _export_targets(self, target_ids: Iterable[str], limit: int) -> list[dict[str, Any]]: + exported: list[dict[str, Any]] = [] + seen: set[str] = set() + for target_id in target_ids: + if target_id in seen: + continue + seen.add(target_id) + target = self._targets.get(target_id) + if target is None or target.generation != self.generation: + continue + exported.append(target.compact_index()) + if len(exported) >= limit: + break + return exported + + def _trim_target_history(self) -> None: + overflow = len(self._targets) - _MAX_TARGET_HISTORY + if overflow <= 0: + return + stale_ids = [target_id for target_id, target in self._targets.items() if target.generation != self.generation][ + :overflow + ] + for target_id in stale_ids: + target = self._targets.pop(target_id, None) + if target is None: + continue + if target.ref and self._ref_targets.get(target.ref) == target_id: + self._ref_targets.pop(target.ref, None) + if self.reference_generations.get(target.ref) == target.generation: + self.reference_generations.pop(target.ref, None) + if target.selector: + selector_key = (target.generation, target.selector) + if self._selector_targets.get(selector_key) == target_id: + self._selector_targets.pop(selector_key, None) + if self.selector_generations.get(target.selector) == target.generation: + self.selector_generations.pop(target.selector, None) + self._forget_target(target) + + +__all__ = ["PageState", "Target", "compact_text"] diff --git a/openjiuwen/harness/tools/browser_move/runtime/rails/__init__.py b/openjiuwen/harness/tools/browser_move/runtime/rails/__init__.py new file mode 100644 index 0000000000..be3d893c17 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/rails/__init__.py @@ -0,0 +1,4 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Surface-neutral rails: the action policy, the repeated-call guard, the hint, the recovery, the lost connection.""" diff --git a/openjiuwen/harness/tools/browser_move/runtime/rails/action_policy.py b/openjiuwen/harness/tools/browser_move/runtime/rails/action_policy.py new file mode 100644 index 0000000000..922f7e54b4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/rails/action_policy.py @@ -0,0 +1,67 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The security rail that checks each command against the surface's policy rules before it reaches the driver.""" + +from __future__ import annotations + +import json +from collections.abc import Callable, Iterable, Mapping +from typing import Any + +from openjiuwen.core.foundation.llm import ToolMessage +from openjiuwen.core.single_agent.rail.base import AgentCallbackEvent +from openjiuwen.harness.rails.security.base_security_rail import ( + BaseSecurityRail, + SecurityCheckContext, + SecurityDecision, + SecurityReject, +) + +from ...contract import Action, PolicyDenied, PolicyRule, RefResolver +from ..page_state import PageState + +Builder = Callable[[Mapping[str, Any], RefResolver], Action] + + +class ActionPolicy(BaseSecurityRail): + """Denies a tool call whose arguments do not decode or that a ``PolicyRule`` rejects.""" + + supported_events = {AgentCallbackEvent.BEFORE_TOOL_CALL} + + def __init__(self, builders: Mapping[str, Builder], rules: Iterable[PolicyRule], page_state: PageState): + super().__init__() + self._builders = builders + self._rules = tuple(rules) + self._page_state = page_state + + async def run_security_check(self, security_ctx: SecurityCheckContext) -> SecurityDecision: + inputs = security_ctx.callback_ctx.inputs + build = self._builders.get(inputs.tool_name) + if build is None: + return self.allow() + try: + args = json.loads(inputs.tool_args) if isinstance(inputs.tool_args, str) else inputs.tool_args + action = build(args or {}, self._page_state) + except ValueError as error: + return self.reject(str(error), result=PolicyDenied(str(error))) + for rule in self._rules: + if isinstance(action, rule.command_type): + reason = rule.check(action, self._page_state.surface_context) + if reason: + return self.reject(reason, result=PolicyDenied(reason)) + return self.allow() + + def _apply_reject(self, security_ctx: SecurityCheckContext, decision: SecurityReject) -> None: + ctx = security_ctx.callback_ctx + call_id = security_ctx.subject_id + ctx.inputs.tool_result = decision.result + ctx.inputs.tool_msg = ToolMessage( + content=f"### Error\n{decision.message}", + tool_call_id=call_id, + metadata={"executed": False, "state_changed": False, "denied": True}, + ) + ctx.extra.setdefault("_skip_tool_calls", {})[call_id] = True + + +__all__ = ["ActionPolicy"] diff --git a/openjiuwen/harness/tools/browser_move/runtime/rails/connection_lost.py b/openjiuwen/harness/tools/browser_move/runtime/rails/connection_lost.py new file mode 100644 index 0000000000..b4b3b942f1 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/rails/connection_lost.py @@ -0,0 +1,30 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Ends the run once the driver session has lost its connection for good: no further model call.""" + +from __future__ import annotations + +from collections.abc import Callable + +from openjiuwen.core.single_agent.rail.base import AgentCallbackContext, AgentRail + +from ..driver_session import DriverSession + + +class ConnectionLostRail(AgentRail): + """Before each model call, finish the run with the session's loss message when the session reports one.""" + + def __init__(self, session_of: Callable[[], DriverSession | None]) -> None: + super().__init__() + self._session_of = session_of + + async def before_model_call(self, ctx: AgentCallbackContext) -> None: + session = self._session_of() + lost = None if session is None else session.lost + if lost is None: + return + ctx.request_force_finish({"output": lost, "result_type": "error", "error": lost}) + + +__all__ = ["ConnectionLostRail"] diff --git a/openjiuwen/harness/tools/browser_move/runtime/rails/final_answer_recovery.py b/openjiuwen/harness/tools/browser_move/runtime/rails/final_answer_recovery.py new file mode 100644 index 0000000000..aae0cb5ba5 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/rails/final_answer_recovery.py @@ -0,0 +1,64 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""One model call that asks the worker for its final JSON after it ended without one; no tool runs.""" + +from __future__ import annotations + +import contextvars +from collections.abc import Iterator +from contextlib import contextmanager + +from openjiuwen.core.single_agent.rail.base import AgentCallbackContext, AgentRail + +EMPTY_OUTPUT = "empty_output" +NON_JSON_OUTPUT = "non_json_output" +MAX_ITERATIONS_REACHED = "max_iterations_reached" + +STEP_LIMIT_SENTENCE = "The step limit has been reached." +_JSON_REPLY = "the final JSON object required by your instructions and nothing else.\n" +RECOVERY_MESSAGE = ( + "Tools are disabled now. Using only what you have already seen in this conversation, reply with " + f"{_JSON_REPLY}" + "If the task is not fully complete, set ok=false and say in final what you found and what is missing." +) +RESTATE_MESSAGE = ( + f"Tools are disabled now. Your previous reply already answered the task. Restate that answer as {_JSON_REPLY}" + "Put the answer in final as you gave it: do not re-check it, change it or add caveats. Set ok=false only if your " + "previous reply said the task could not be completed." +) + +_RECOVERY_ACTIVE: contextvars.ContextVar[bool] = contextvars.ContextVar("browser_final_answer_recovery", default=False) + + +def recovery_reason(output: str, *, max_iterations_reached: bool) -> str: + if max_iterations_reached: + return MAX_ITERATIONS_REACHED + return EMPTY_OUTPUT if not output.strip() else NON_JSON_OUTPUT + + +def recovery_message(reason: str) -> str: + if reason == MAX_ITERATIONS_REACHED: + return f"{STEP_LIMIT_SENTENCE} {RECOVERY_MESSAGE}" + return RESTATE_MESSAGE if reason == NON_JSON_OUTPUT else RECOVERY_MESSAGE + + +@contextmanager +def final_answer_recovery() -> Iterator[None]: + """Mark the worker invoke run inside this block as the recovery call.""" + token = _RECOVERY_ACTIVE.set(True) + try: + yield + finally: + _RECOVERY_ACTIVE.reset(token) + + +class FinalAnswerRecoveryRail(AgentRail): + """During recovery: stop after one model call and run none of its tool calls.""" + + async def after_model_call(self, ctx: AgentCallbackContext) -> None: + if not _RECOVERY_ACTIVE.get(): + return + response = getattr(ctx.inputs, "response", None) + if getattr(response, "tool_calls", None): + ctx.request_force_finish({"output": getattr(response, "content", "") or "", "result_type": "answer"}) diff --git a/openjiuwen/harness/tools/browser_move/runtime/rails/intercepted_action.py b/openjiuwen/harness/tools/browser_move/runtime/rails/intercepted_action.py new file mode 100644 index 0000000000..f678908793 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/rails/intercepted_action.py @@ -0,0 +1,69 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Compact tool errors for actions another element intercepted.""" + +from __future__ import annotations + +import re + +from openjiuwen.core.single_agent.rail.base import AgentCallbackContext, AgentRail + +from ...contract import ActionIntercepted, DriverRef, Locator, PointRef +from ..tool_factory import CommandRecord, command_record + +_CONSENT = re.compile( + r"onetrust|cookie|consent|cmp|didomi|usercentrics|cookiebot|gdpr|truste|qc-cmp|sp_message", + re.IGNORECASE, +) +_MAX_LOCATOR_CHARS = 60 + +_CONSENT_HINT = ( + "This is a cookie-consent dialog: close it first (prefer 'Reject all' / 'Close'; " + "if its settings panel is open, confirm or close that panel)." +) +_GENERIC_HINT = ( + "If it is a dialog, banner or overlay, dismiss or close it first. If it is part of the same control " + "(an icon or custom styling over an input), click the control's visible label or text instead." +) + + +def _target(record: CommandRecord) -> str: + target = getattr(record.action, "target", None) + if isinstance(target, DriverRef): + return target.ref + if isinstance(target, Locator): + return (target.css or target.text or "the target")[:_MAX_LOCATOR_CHARS] + if isinstance(target, PointRef): + return f"({target.x}, {target.y})" + return "the target" + + +def explain_blocked_command(record: CommandRecord, retry: str) -> str | None: + """Compact error for a command another element intercepted, from its record; else None.""" + if record.error != ActionIntercepted.__name__ or not record.blocked_by: + return None + hint = _CONSENT_HINT if _CONSENT.search(record.blocked_by) else _GENERIC_HINT + return ( + "### Error\n" + f"{record.command.__name__} not delivered: {_target(record)} is covered by {record.blocked_by}.\n" + f"{hint}\n" + f"{retry}" + ) + + +class InterceptedActionHintRail(AgentRail): + """Rewrite the model-facing message of an intercepted action; ``retry`` is the surface's retry sentence.""" + + def __init__(self, retry: str) -> None: + super().__init__() + self._retry = retry + + async def after_tool_call(self, ctx: AgentCallbackContext) -> None: + inputs = ctx.inputs + tool_msg = getattr(inputs, "tool_msg", None) + record = command_record(getattr(inputs, "tool_result", None)) + readable = isinstance(getattr(tool_msg, "content", None), str) + explained = explain_blocked_command(record, self._retry) if record is not None and readable else None + if explained is not None: + inputs.tool_msg = tool_msg.model_copy(update={"content": explained}) diff --git a/openjiuwen/harness/tools/browser_move/runtime/rails/repeated_call_guard.py b/openjiuwen/harness/tools/browser_move/runtime/rails/repeated_call_guard.py new file mode 100644 index 0000000000..243efff093 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/rails/repeated_call_guard.py @@ -0,0 +1,164 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Stop the worker from looping on tool calls that make no progress.""" + +from __future__ import annotations + +import dataclasses +import json +import re +from dataclasses import dataclass +from typing import Any + +from openjiuwen.core.common.logging import logger +from openjiuwen.core.single_agent.prompts.builder import PromptSection +from openjiuwen.core.single_agent.rail.base import AgentCallbackContext, AgentRail + +from ...contract import RunScript +from ..tool_factory import CommandRecord, command_record + +REPEAT_NOTE_THRESHOLD = 3 +REPEAT_WITHHOLD_TOOLS_THRESHOLD = 5 + +_STATE_KEY = "_browser_repeated_call_guard" +_FINISH_SECTION = "browser_repeated_call_finish" +_NOTE_HEADER = "### Repeated call" + +_WHITESPACE = re.compile(r"\s+") +_DIGITS = re.compile(r"\d+") + +_NOOP_ATOM = ( + r"(?:-?\d+(?:\.\d+)?|'[^'\\]*'|\"[^\"\\]*\"|`[^`$\\]*`|true|false|null|undefined" + r"|document\.title|document\.URL|(?:window\.)?location\.href)" +) +_NOOP_KEY = r"(?:[A-Za-z_$][\w$]*|'[^'\\]*'|\"[^\"\\]*\")" +_NOOP_OBJECT = rf"\{{(?:{_NOOP_KEY}:{_NOOP_ATOM}(?:,{_NOOP_KEY}:{_NOOP_ATOM})*,?)?\}}" +_NOOP_ARRAY = rf"\[(?:{_NOOP_ATOM}(?:,{_NOOP_ATOM})*,?)?\]" +_NOOP_VALUE = rf"(?:{_NOOP_ATOM}|{_NOOP_ARRAY}|\({_NOOP_OBJECT}\)|\({_NOOP_ATOM}\)|\({_NOOP_ARRAY}\))" +_NOOP_BLOCK_VALUE = rf"(?:{_NOOP_VALUE}|{_NOOP_OBJECT})" +_NOOP_FUNCTION = re.compile(rf"(?:async)?\(\)=>(?:{_NOOP_VALUE}|\{{return{_NOOP_BLOCK_VALUE};?\}});?") + + +@dataclass +class _RepeatState: + last_key: Any = None + last_fingerprint: Any = None + repeat_count: int = 0 + noop_count: int = 0 + withhold_tools: bool = False + + +def _call_key(tool_name: str, arguments: Any) -> str: + if isinstance(arguments, str): + try: + arguments = json.loads(arguments) + except ValueError: + return json.dumps([tool_name, arguments.strip()]) + return json.dumps([tool_name, arguments], sort_keys=True, ensure_ascii=False, default=str) + + +def _record_identity(record: CommandRecord) -> tuple[Any, Any]: + """(masked command, result); a script compares without whitespace and with its digits masked.""" + action, value = record.action, record.value + if isinstance(action, RunScript): + action = dataclasses.replace(action, code=_DIGITS.sub("#", _WHITESPACE.sub("", action.code))) + value = _DIGITS.sub("#", json.dumps(value, sort_keys=True, ensure_ascii=False, default=str)) + return action, (record.ok, record.error, value, record.surface) + + +def _is_noop_script(record: CommandRecord | None) -> bool: + action = record.action if record is not None else None + return isinstance(action, RunScript) and _NOOP_FUNCTION.fullmatch(_WHITESPACE.sub("", action.code)) is not None + + +def _repeat_note(repeat_count: int) -> str: + return ( + f"{_NOTE_HEADER}\n" + f"This exact call has returned the same result {repeat_count} times in a row; " + "repeating it will not change anything.\n" + "If you already have what the task needs, return the final JSON answer now. " + "Otherwise take a different action." + ) + + +def _noop_note(noop_count: int) -> str: + return ( + f"{_NOTE_HEADER}\n" + f"The last {noop_count} calls only returned constants or the document's title or address; " + "they do not change anything or give new information.\n" + "If you already have what the task needs, return the final JSON answer now. " + "Otherwise take a different action." + ) + + +class RepeatedCallGuardRail(AgentRail): + """Warn on the 3rd identical or no-op call in a row; withhold tools for the next model call after the 5th.""" + + async def before_invoke(self, ctx: AgentCallbackContext) -> None: + self._start(ctx) + + @staticmethod + def _start(ctx: AgentCallbackContext) -> _RepeatState: + state = ctx.extra[_STATE_KEY] = _RepeatState() + builder = getattr(ctx.agent, "system_prompt_builder", None) + if builder is not None: + builder.remove_section(_FINISH_SECTION) + return state + + async def after_tool_call(self, ctx: AgentCallbackContext) -> None: + state = ctx.extra.get(_STATE_KEY) + inputs = ctx.inputs + tool_msg = getattr(inputs, "tool_msg", None) + if not isinstance(state, _RepeatState) or tool_msg is None: + return + record = command_record(getattr(inputs, "tool_result", None)) + if record is None: + key = _call_key(getattr(inputs, "tool_name", ""), getattr(inputs, "tool_args", None)) + fingerprint = None + else: + key, fingerprint = _record_identity(record) + if key == state.last_key and fingerprint == state.last_fingerprint: + state.repeat_count += 1 + else: + state.last_key, state.last_fingerprint, state.repeat_count = key, fingerprint, 1 + state.noop_count = state.noop_count + 1 if _is_noop_script(record) else 0 + + count = max(state.repeat_count, state.noop_count) + if count < REPEAT_NOTE_THRESHOLD: + return + if count >= REPEAT_WITHHOLD_TOOLS_THRESHOLD: + state.withhold_tools = True + if isinstance(tool_msg.content, str) and _NOTE_HEADER not in tool_msg.content: + if state.noop_count > state.repeat_count: + note = _noop_note(state.noop_count) + else: + note = _repeat_note(state.repeat_count) + inputs.tool_msg = tool_msg.model_copy(update={"content": f"{tool_msg.content.rstrip()}\n{note}"}) + + async def before_model_call(self, ctx: AgentCallbackContext) -> None: + state = ctx.extra.get(_STATE_KEY) + if not isinstance(state, _RepeatState): + state = self._start(ctx) + if not state.withhold_tools: + return + count = max(state.repeat_count, state.noop_count) + logger.info( + "Worker repeated the same tool call %d times; withholding tools for this model call.", + count, + ) + ctx.inputs.tools = [] + builder = getattr(ctx.agent, "system_prompt_builder", None) + if builder is not None: + text = ( + f"The same tool call returned the same result {count} times in a row. " + "Tools are disabled for this reply. Return the final JSON answer now from what you " + "already have; if the task is not complete, set ok=false and fill the progress fields." + ) + builder.add_section( + PromptSection( + name=_FINISH_SECTION, + content={"en": text, "cn": text}, + priority=100, + ) + ) diff --git a/openjiuwen/harness/tools/browser_move/runtime/service.py b/openjiuwen/harness/tools/browser_move/runtime/service.py new file mode 100644 index 0000000000..a3d6bdfda5 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/service.py @@ -0,0 +1,358 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The backend service: the driver session's lifecycle, with sticky sessions and guardrails.""" + +from __future__ import annotations + +import asyncio +import base64 +import shutil +import uuid +from pathlib import Path +from typing import Any, Dict, Iterable, List, Optional + +from openjiuwen.core.foundation.store.base_kv_store import BaseKVStore +from openjiuwen.core.foundation.store.kv.in_memory_kv_store import InMemoryKVStore +from openjiuwen.core.single_agent.agents.react_agent import ReActAgent + +from ..contract import SurfaceBundle +from .assembly import new_page_state, open_session +from .config import CoreSettings, RunGuardrails, process_environ +from .driver_session import DriverSession +from .page_state import PageState +from .service_registry import SERVICE_REGISTRY +from .worker import TaskProgressState, _WorkerLoop + + +def _image_mime_type(data: bytes) -> str | None: + if data.startswith(b"\x89PNG\r\n\x1a\n"): + return "image/png" + if data.startswith(b"\xff\xd8\xff"): + return "image/jpeg" + if data.startswith((b"GIF87a", b"GIF89a")): + return "image/gif" + if data[:4] == b"RIFF" and data[8:12] == b"WEBP": + return "image/webp" + return None + + +class AgentService(_WorkerLoop): + """Backend service with sticky logical sessions; holds the driver session the worker runs on.""" + + def __init__( + self, + core: CoreSettings, + surface: SurfaceBundle, + *, + provider: str, + api_key: str, + api_base: str, + model_name: str, + guardrails: RunGuardrails, + allowed_tool_names: Optional[Iterable[str]] = None, + output_dir: Path, + identity: str, + cancel_store: Optional[BaseKVStore] = None, + page_state: Optional[PageState] = None, + ) -> None: + self.core = core + self.surface = surface + self.provider = provider + self.api_key = api_key + self.api_base = api_base + self.model_name = model_name + self.guardrails = guardrails + self.identity = identity + self._allowed_tool_names = None if allowed_tool_names is None else tuple(dict.fromkeys(allowed_tool_names)) + self._cancel_store: BaseKVStore = cancel_store or InMemoryKVStore() + self.page_state: PageState = page_state if page_state is not None else new_page_state(surface.profile) + self._session: Optional[DriverSession] = None + + self.started = False + self.agent: Optional[ReActAgent] = None + self._locks: Dict[str, asyncio.Lock] = {} + self._sessions: set[str] = set() + self._inflight_tasks: Dict[str, set[asyncio.Task[Any]]] = {} + self._screenshot_subdir = "screenshots" + self._artifacts_subdir = "artifacts" + self._output_dir = output_dir + self._screenshots_dir = output_dir / self._screenshot_subdir + self._artifacts_dir = output_dir / self._artifacts_subdir + self._failure_context_by_session: Dict[str, str] = {} + self._progress_by_session: Dict[str, TaskProgressState] = {} + self._registry_acquired = False + self._task_binding_ref_count = 0 + + @property + def allowed_tool_names(self) -> Optional[tuple[str, ...]]: + return self._allowed_tool_names + + @property + def screenshot_subdir(self) -> str: + return self._screenshot_subdir + + @property + def artifacts_subdir(self) -> str: + return self._artifacts_subdir + + @property + def session(self) -> Optional[DriverSession]: + return self._session + + def _acquire_registry_lease(self) -> None: + if self._registry_acquired: + return + SERVICE_REGISTRY.acquire(self.identity, self) + self._registry_acquired = True + + def acquire_task_binding(self) -> None: + """Acquire one task reference to the shared process resources.""" + self._task_binding_ref_count += 1 + self._acquire_registry_lease() + if self._session is not None: + self._session.begin_task() + + @staticmethod + def _cancel_key(session_id: str, request_id: Optional[str] = None) -> str: + rid = (request_id or "").strip() or "*" + return f"playwright_runtime:cancel:{session_id}:{rid}" + + @staticmethod + def _inflight_key(session_id: str, request_id: Optional[str] = None) -> str: + rid = (request_id or "").strip() + return f"{session_id}:{rid}" if rid else session_id + + def _register_inflight_task(self, session_id: str, request_id: str, task: asyncio.Task[Any]) -> None: + keys = (self._inflight_key(session_id), self._inflight_key(session_id, request_id)) + for key in keys: + self._inflight_tasks.setdefault(key, set()).add(task) + + def _unregister_inflight_task(self, session_id: str, request_id: str, task: asyncio.Task[Any]) -> None: + keys = (self._inflight_key(session_id), self._inflight_key(session_id, request_id)) + for key in keys: + tasks = self._inflight_tasks.get(key) + if not tasks: + continue + tasks.discard(task) + if not tasks: + self._inflight_tasks.pop(key, None) + + def _ensure_screenshots_dir(self) -> None: + self._screenshots_dir.mkdir(parents=True, exist_ok=True) + + def _resolve_local_screenshot_path(self, raw_path: str) -> Path: + path = Path(raw_path).expanduser() + candidates: List[Path] = [] + if path.is_absolute(): + candidates.append(path) + else: + candidates.extend( + [ + self._output_dir / path, + Path.cwd() / path, + path, + ] + ) + + for candidate in candidates: + if candidate.exists() and candidate.is_file(): + return candidate + return path + + def _ensure_screenshot_in_folder(self, source_path: Path) -> Path: + if not source_path.exists() or not source_path.is_file(): + return source_path + + self._ensure_screenshots_dir() + try: + source_resolved = source_path.resolve() + target_dir_resolved = self._screenshots_dir.resolve() + try: + source_resolved.relative_to(target_dir_resolved) + return source_resolved + except ValueError: + pass + + target_path = self._screenshots_dir / source_path.name + if target_path.exists(): + target_resolved = target_path.resolve() + if target_resolved != source_resolved: + target_path = self._screenshots_dir / ( + f"{source_path.stem}-{uuid.uuid4().hex[:8]}{source_path.suffix}" + ) + + shutil.copy2(source_path, target_path) + return target_path + except Exception: + return source_path + + def _normalize_screenshot_value(self, screenshot: Any) -> str | None: + """The answer's screenshot as an http(s) link or an image data link, else ``None``.""" + if not isinstance(screenshot, str) or not screenshot.strip(): + return None + raw = screenshot.strip() + lowered = raw.lower() + if lowered.startswith(("http://", "https://")): + return raw + if lowered.startswith("data:image/"): + _, _, encoded = raw.partition(",") + try: + data = base64.b64decode(encoded, validate=True) + except ValueError: + return None + return raw if _image_mime_type(data) else None + + local_path = self._resolve_local_screenshot_path(raw[7:] if lowered.startswith("file://") else raw) + if not local_path.is_file(): + return None + local_path = self._ensure_screenshot_in_folder(local_path) + try: + data = local_path.read_bytes() + except OSError: + return None + mime_type = _image_mime_type(data) + return f"data:{mime_type};base64,{base64.b64encode(data).decode('ascii')}" if mime_type else None + + async def request_cancel(self, session_id: str, request_id: Optional[str] = None) -> None: + sid = (session_id or "").strip() + if not sid: + raise ValueError("session_id is required for cancellation") + await self._cancel_store.set(self._cancel_key(sid, request_id), "1") + + request_id_clean = (request_id or "").strip() + if request_id_clean: + keys = [self._inflight_key(sid, request_id_clean)] + else: + keys = [self._inflight_key(sid)] + + for key in keys: + for task in list(self._inflight_tasks.get(key, set())): + if not task.done(): + task.cancel() + + async def clear_cancel(self, session_id: str, request_id: Optional[str] = None) -> None: + sid = (session_id or "").strip() + if not sid: + return + if request_id: + await self._cancel_store.delete(self._cancel_key(sid, request_id)) + return + await self._cancel_store.delete(self._cancel_key(sid, "*")) + + async def is_cancelled(self, session_id: str, request_id: Optional[str] = None) -> bool: + sid = (session_id or "").strip() + if not sid: + return False + if request_id: + exact = await self._cancel_store.get(self._cancel_key(sid, request_id)) + if exact is not None: + return True + wildcard = await self._cancel_store.get(self._cancel_key(sid, "*")) + return wildcard is not None + + def session_new(self, session_id: Optional[str] = None) -> str: + sid = (session_id or "").strip() or f"session-{uuid.uuid4().hex}" + self._sessions.add(sid) + if sid not in self._locks: + self._locks[sid] = asyncio.Lock() + return sid + + async def _open_session(self) -> DriverSession: + session = await open_session( + self.core, process_environ(), page_state=self.page_state, settings=self.surface.settings + ) + SERVICE_REGISTRY.register_managed_driver(self.identity, self, session.assembly.provisioner) + return session + + async def ensure_runtime_ready(self) -> None: + """Open the driver session once; after a task release, start its stack again on the kept device.""" + if self._task_binding_ref_count <= 0: + self.acquire_task_binding() + self._acquire_registry_lease() + if self.started: + return + if self._session is None: + self._session = await self._open_session() + else: + await self._session.start() + self._session.begin_task() + self._ensure_screenshots_dir() + self.started = True + SERVICE_REGISTRY.activate_binding(self.identity, self) + + async def _restart(self) -> None: + """Release the session after a dead transport; the next task's worker call rebuilds on a fresh one.""" + await self.reset() + + def _clear_task_scoped_state(self) -> None: + """Drop observers and task/session state without touching the profile.""" + self.agent = None + for tasks in self._inflight_tasks.values(): + for task in tuple(tasks): + if not task.done(): + task.cancel() + self._inflight_tasks.clear() + self._locks.clear() + self._sessions.clear() + self._failure_context_by_session.clear() + self._progress_by_session.clear() + + async def release_task_binding(self) -> bool: + """Release the task's resources: the last binding stops the driver stack and keeps the device.""" + if self._task_binding_ref_count > 0: + self._task_binding_ref_count -= 1 + if self._task_binding_ref_count > 0: + return False + if self._registry_acquired: + SERVICE_REGISTRY.release(self.identity, self) + self._registry_acquired = False + if self.started and self._session is not None: + await self._session.stop() + self.started = False + self._clear_task_scoped_state() + return True + + async def detach_lifecycle_resources(self) -> Optional[Any]: + """Close this service's session during a coordinated reset; returns the provisioner of its device.""" + session, self._session = self._session, None + self.started = False + self.agent = None + if session is None: + return None + await session.close() + return session.assembly.provisioner + + @classmethod + async def reset_lifecycle_resources(cls, identity: str, fallback_service: Optional["AgentService"] = None) -> bool: + """Stop all process resources associated with one identity.""" + snapshot = SERVICE_REGISTRY.begin_reset(identity) + services = snapshot.services + if not services and fallback_service is not None: + services = (fallback_service,) + drivers = list(snapshot.managed_drivers) + for service in services: + driver = await service.detach_lifecycle_resources() + if driver is not None and all(existing is not driver for existing in drivers): + drivers.append(driver) + + for driver in drivers: + try: + await asyncio.to_thread(driver.stop) + except Exception: + pass + finally: + SERVICE_REGISTRY.unregister_managed_driver(identity, driver) + return bool(services or drivers) + + async def reset(self) -> None: + """Reset one shared identity and restart lazily on next use.""" + await self.reset_lifecycle_resources(self.identity, fallback_service=self) + + async def shutdown(self) -> None: + try: + await self.reset() + finally: + self._task_binding_ref_count = 0 + await self.release_task_binding() + SERVICE_REGISTRY.discard_idle(self.identity) diff --git a/openjiuwen/harness/tools/browser_move/runtime/service_registry.py b/openjiuwen/harness/tools/browser_move/runtime/service_registry.py new file mode 100644 index 0000000000..1874fc972a --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/service_registry.py @@ -0,0 +1,123 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Process-level ownership registry for service resources, keyed by the service's identity string.""" + +from __future__ import annotations + +import threading +import weakref +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class ServiceResetSnapshot: + """Resources that must be stopped for an explicit identity reset.""" + + services: tuple[Any, ...] + managed_drivers: tuple[Any, ...] + + +@dataclass +class _ServiceEntry: + services: weakref.WeakSet[Any] = field(default_factory=weakref.WeakSet) + binding_services: weakref.WeakSet[Any] = field(default_factory=weakref.WeakSet) + managed_drivers: list[Any] = field(default_factory=list) + + +class ServiceRegistry: + """Shares device handles and task bindings between the services of one identity.""" + + def __init__(self) -> None: + self._entries: dict[str, _ServiceEntry] = {} + self._lock = threading.RLock() + + def acquire(self, identity: str, service: Any) -> None: + with self._lock: + entry = self._entries.setdefault(identity, _ServiceEntry()) + entry.services.add(service) + + def activate_binding(self, identity: str, service: Any) -> None: + """Mark the service's task binding active.""" + with self._lock: + entry = self._entries.setdefault(identity, _ServiceEntry()) + entry.services.add(service) + entry.binding_services.add(service) + + def release(self, identity: str, service: Any) -> None: + """Release a task binding.""" + with self._lock: + entry = self._entries.get(identity) + if entry is None: + return + entry.binding_services.discard(service) + entry.services.discard(service) + if not entry.services and not entry.managed_drivers: + self._entries.pop(identity, None) + + def register_managed_driver(self, identity: str, service: Any, driver: Any) -> None: + with self._lock: + entry = self._entries.setdefault(identity, _ServiceEntry()) + entry.services.add(service) + if getattr(driver, "owns_process", False): + entry.managed_drivers = [driver] + return + if any(getattr(existing, "owns_process", False) for existing in entry.managed_drivers): + return + if all(existing is not driver for existing in entry.managed_drivers): + entry.managed_drivers.append(driver) + + def unregister_managed_driver(self, identity: str, driver: Any) -> None: + with self._lock: + entry = self._entries.get(identity) + if entry is None: + return + entry.managed_drivers = [existing for existing in entry.managed_drivers if existing is not driver] + if not entry.services and not entry.managed_drivers: + self._entries.pop(identity, None) + + def begin_reset(self, identity: str) -> ServiceResetSnapshot: + """Detach shared process resources for one explicit reset operation.""" + with self._lock: + entry = self._entries.get(identity) + if entry is None: + return ServiceResetSnapshot(services=(), managed_drivers=()) + + services = tuple(entry.services) + drivers = list(entry.managed_drivers) + for service in services: + driver = getattr(service, "managed_driver", None) + if driver is not None and all(existing is not driver for existing in drivers): + drivers.append(driver) + + entry.binding_services.clear() + entry.managed_drivers.clear() + if not entry.services: + self._entries.pop(identity, None) + + return ServiceResetSnapshot(services=services, managed_drivers=tuple(drivers)) + + def discard_idle(self, identity: str) -> None: + """Discard registry metadata after an explicit full shutdown.""" + with self._lock: + entry = self._entries.get(identity) + if entry is None: + return + if not entry.services and not entry.binding_services and not entry.managed_drivers: + self._entries.pop(identity, None) + + def managed_driver_identities(self) -> tuple[str, ...]: + """The identities that hold a preserved managed driver.""" + with self._lock: + return tuple(identity for identity, entry in self._entries.items() if entry.managed_drivers) + + def clear(self) -> None: + """Clear metadata for isolated tests after their resources are stopped.""" + with self._lock: + self._entries.clear() + + +SERVICE_REGISTRY = ServiceRegistry() + + +__all__ = ["SERVICE_REGISTRY", "ServiceRegistry", "ServiceResetSnapshot"] diff --git a/openjiuwen/harness/tools/browser_move/runtime/session.py b/openjiuwen/harness/tools/browser_move/runtime/session.py new file mode 100644 index 0000000000..ef4974c9af --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/session.py @@ -0,0 +1,359 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. + +"""The runtime over one driver session: the service, the factory tools and the topology that runs a task.""" + +from __future__ import annotations + +import json +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, Iterable, Mapping, Optional + +from openjiuwen.core.foundation.llm.model_clients.openai_model_client import OpenAIModelClient +from openjiuwen.core.foundation.store.base_kv_store import BaseKVStore +from openjiuwen.core.single_agent.rail.base import AgentRail + +from ..contract import Action, Effect, Health, SurfaceBundle +from .assembly import AgentParts, build_agent_parts, core_rails +from .config import CoreSettings, RunGuardrails +from .driver_session import DriverSession +from .page_state import PageState +from .service import AgentService +from .tool_factory import Builder, ToolContext, ToolFlags +from .topology import Topology, topology_for + + +@dataclass(frozen=True) +class ToolKind: + """What the rail knows of one tool call: the command decoded from its arguments, or a helper's declared flags.""" + + action: Action | None = None + effect: Effect = Effect.ACT + exempt_from_budget: bool = False + progress: bool = False + known: bool = False + helper: bool = False + fields: frozenset[str] = frozenset() + + @property + def command(self) -> type[Action] | None: + return None if self.action is None else type(self.action) + + @property + def read_only(self) -> bool: + return self.effect is Effect.READ + + +def _tool_args(value: Any) -> Dict[str, Any]: + if isinstance(value, dict): + return value + if isinstance(value, str): + try: + parsed = json.loads(value) + except ValueError: + return {} + return parsed if isinstance(parsed, dict) else {} + return {} + + +class AgentRuntime: + """One session, the parts built on it, the rails' live builders, and the topology that runs each task.""" + + def __init__( + self, + core: CoreSettings, + surface: SurfaceBundle, + *, + provider: str, + api_key: str, + api_base: str, + model_name: str, + guardrails: RunGuardrails, + allowed_tool_names: Optional[Iterable[str]] = None, + workspace: Any = None, + language: str = "cn", + output_dir: Path, + identity: str, + cancel_store: Optional[BaseKVStore] = None, + page_state: Optional[PageState] = None, + ) -> None: + self._core = core + self._surface = surface + self._workspace = workspace + self._language = language + self._allowed_tool_names = None if allowed_tool_names is None else tuple(dict.fromkeys(allowed_tool_names)) + self._capabilities: tuple[str, ...] | None = None + self._service = AgentService( + core, + surface, + provider=provider, + api_key=api_key, + api_base=api_base, + model_name=model_name, + guardrails=guardrails, + allowed_tool_names=self._allowed_tool_names, + output_dir=output_dir, + identity=identity, + cancel_store=cancel_store, + page_state=page_state, + ) + self._topology: Topology | None = None + self._dedicated_log = True + self._session: DriverSession | None = None + self._context: ToolContext | None = None + self._parts: dict[tuple[bool, str], AgentParts] = {} + self._builders: dict[str, Builder] = {} + self._fields: dict[str, frozenset[str]] = {} + self._flags: dict[str, ToolFlags] = {} + + @property + def service(self) -> AgentService: + return self._service + + @property + def core(self) -> CoreSettings: + return self._core + + @property + def surface(self) -> SurfaceBundle: + return self._surface + + @property + def dedicated_log(self) -> bool: + """Whether the agent's invokes write to the dedicated log file; the runner's own agent uses the process log.""" + return self._dedicated_log + + @dedicated_log.setter + def dedicated_log(self, value: bool) -> None: + self._dedicated_log = value + + @property + def workspace(self) -> Any: + return self._workspace + + @property + def language(self) -> str: + return self._language + + @property + def capabilities(self) -> tuple[str, ...]: + """The catalog entries whose tools are all allowed; the default entry alone when no allowlist was given.""" + if self._capabilities is None: + catalog = self._surface.profile.capabilities + allowed = set(catalog[0].tool_names if self._allowed_tool_names is None else self._allowed_tool_names) + self._capabilities = tuple(entry.name for entry in catalog if set(entry.tool_names) <= allowed) + return self._capabilities + + @property + def builders(self) -> Mapping[str, Builder]: + """The decoders of every factory tool built so far; the policy rail reads it live.""" + return self._builders + + def core_rails(self) -> tuple[AgentRail, ...]: + return core_rails(self._surface.profile, self._builders, self.page_state, lambda: self._session) + + @property + def page_state(self) -> PageState: + return self._service.page_state + + @property + def session(self) -> DriverSession | None: + return self._session + + @property + def context(self) -> ToolContext | None: + """The tool context of the started session, for the runtime's own calls.""" + return self._context + + @property + def topology(self) -> Topology | None: + return self._topology + + async def metadata(self) -> tuple[Mapping[str, Any], str | None]: + """The profile's metadata of the current context, ``({}, None)`` for a surface without the hook.""" + hook = self._surface.profile.page_metadata + return ({}, None) if hook is None else await hook(self._context) + + def tool_kind(self, tool_name: str, tool_args: Any) -> ToolKind: + """The command ``tool_args`` decode to on a factory tool, or a helper's flags; unknown tools act.""" + build = self._builders.get(tool_name) + if build is not None: + fields = self._fields.get(tool_name, frozenset()) + try: + action = build(_tool_args(tool_args), self.page_state) + except ValueError: + return ToolKind(progress=True, known=True, fields=fields) + return ToolKind(action=action, effect=action.effect, progress=True, known=True, fields=fields) + flags = self._flags.get(tool_name) + if flags is None: + return ToolKind() + return ToolKind( + effect=flags.effect, + exempt_from_budget=flags.exempt_from_budget, + progress=flags.progress, + known=True, + helper=True, + ) + + async def cancel_run(self, session_id: str, request_id: Optional[str] = None) -> Dict[str, Any]: + await self._service.request_cancel(session_id=session_id, request_id=request_id) + return { + "ok": True, + "session_id": session_id, + "request_id": request_id, + "error": None, + } + + async def clear_cancel(self, session_id: str, request_id: Optional[str] = None) -> Dict[str, Any]: + await self._service.clear_cancel(session_id=session_id, request_id=request_id) + return { + "ok": True, + "session_id": session_id, + "request_id": request_id, + "error": None, + } + + async def ensure_runtime_ready(self) -> None: + """Make sure the service's driver session runs; a new session gets a fresh tool context and parts.""" + await self._service.ensure_runtime_ready() + session = self._service.session + if session is self._session: + return + self._drop_session() + profile = session.assembly.surface.profile + self._context = ToolContext( + session, + self.page_state, + profile.renderer, + session.assembly.stack.info, + profile.settings, + False, + self._workspace, + self._language, + ) + self._session = session + + def agent_parts(self, *, image_support: bool, language: str) -> AgentParts: + """The factory tools of the started session in the receiving agent's language, once per key.""" + parts = self._parts.get((image_support, language)) + if parts is None: + session = self._session + parts = build_agent_parts( + session, + session.assembly.surface, + capabilities=self.capabilities, + image_support=image_support, + language=language, + workspace=self._workspace, + ) + self._parts[(image_support, language)] = parts + self._builders.update(parts.builders) + for tool in parts.tools: + flags = getattr(tool, "flags", None) + if isinstance(flags, ToolFlags): + self._flags[tool.card.name] = flags + elif tool.card.name in parts.builders: + properties = (tool.card.input_params or {}).get("properties") or {} + self._fields[tool.card.name] = frozenset(properties) + return parts + + def _drop_session(self) -> None: + self._session = None + self._context = None + self._parts = {} + + async def acquire_task_resources(self) -> None: + """Acquire one task reference before invoking a reusable subagent.""" + self._service.acquire_task_binding() + + async def ensure_started(self) -> None: + """The runner's agent, by the topology: the worker through the service, or the main agent on this runtime.""" + if self._topology is None: + self._topology = topology_for(self._core) + await self._topology.start(self) + + async def run_task( + self, + task: str, + session_id: Optional[str] = None, + request_id: Optional[str] = None, + timeout_s: Optional[int] = None, + ) -> Dict[str, Any]: + """Run one task through the topology; the answer carries the context the task ended in.""" + await self.ensure_started() + answer = await self._topology.run(self, task, session_id, request_id, timeout_s) + answer["context"] = await self._answer_context(bool(answer.get("ok"))) + return answer + + async def _answer_context(self, ok: bool) -> Dict[str, Any]: + """After a successful run the profile's metadata, then the surface identity of the current context.""" + metadata: Mapping[str, Any] = {} + if ok and self._context is not None: + metadata, _ = await self.metadata() + context = self.page_state.surface_context + return {"identity": "" if context is None else context.identity, "metadata": dict(metadata)} + + def main_agent_answer( + self, session_id: str, request_id: str, *, result: Optional[Dict[str, Any]] = None, error: Optional[str] = None + ) -> Dict[str, Any]: + """The runner's answer dict from a main-agent run: the output, and status when the rail reported one.""" + result = result or {} + payload = result.get("authoritative_browser_result") + progress = result.get("progress_state") + error = error if error is not None else result.get("error") + answer: Dict[str, Any] = { + "ok": not error and result.get("result_type") != "error", + "session_id": session_id, + "request_id": request_id, + "final": str(result.get("output") or ""), + "screenshot": None, + "error": error or None, + } + status = (payload or {}).get("status") or (progress or {}).get("status") + if status: + answer["status"] = status + if progress: + answer["progress"] = progress + return answer + + async def runtime_health(self) -> Dict[str, Any]: + session = self._service.session + if session is not None and self._service.started: + health = await session.health() + else: + health = Health(ok=False, detail="not started") + return { + "ok": health.ok, + "started": bool(self._service.started), + "detail": health.detail, + "provider": self._service.provider, + "api_base": self._service.api_base, + "model_name": self._service.model_name, + } + + async def shutdown(self) -> None: + try: + await self._service.shutdown() + finally: + self._drop_session() + if self._topology is not None: + self._topology.drop() + self._topology = None + await OpenAIModelClient.aclose() + + async def release_task_resources(self) -> bool: + """Release one task binding; the last one stops the driver stack, keeps the device and resets the state.""" + fully_released = await self._service.release_task_binding() + if fully_released: + self.page_state.reset() + return fully_released + + async def reset(self) -> None: + """Release the current session and restart lazily on the next task.""" + try: + await self._service.reset() + finally: + self._drop_session() + self.page_state.reset() diff --git a/openjiuwen/harness/tools/browser_move/runtime/tool_factory.py b/openjiuwen/harness/tools/browser_move/runtime/tool_factory.py new file mode 100644 index 0000000000..1e4d96faae --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/tool_factory.py @@ -0,0 +1,436 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Agent tools built from the surface profile and the driver: one ``CommandTool`` per selected tool spec.""" + +from __future__ import annotations + +import base64 +from collections.abc import Callable, Iterable, Mapping +from dataclasses import dataclass, field +from types import MappingProxyType +from typing import Any, AsyncIterator + +from openjiuwen.core.common.logging import tool_logger +from openjiuwen.core.foundation.tool import Tool, ToolCard +from openjiuwen.core.foundation.tool.schema import ToolOutput + +from ..contract import ( + Action, + ActionIntercepted, + ActionResult, + ActionTimeout, + ContentBlock, + ContextRef, + DriverError, + DriverInfo, + DriverRef, + Effect, + Locator, + Observation, + Observe, + PointRef, + Ref, + RefResolver, + Renderer, + Screenshot, + StaleRef, + SurfaceContext, + SurfaceProfile, + ToolSpec, +) +from ..contract.masking import masked +from .driver_session import DriverSession +from .page_state import PageState + +Builder = Callable[[Mapping[str, Any], RefResolver], Action] + +_ERROR = "### Error" + + +@dataclass(frozen=True) +class ToolContext: + """What the profile's ``extra_tools`` and ``rails`` factories receive; ``driver`` is the session.""" + + driver: DriverSession + page_state: PageState + renderer: Renderer + driver_info: DriverInfo + settings: Any + image_support: bool + workspace: Any = None + language: str = "cn" + + async def execute(self, action: Action) -> ActionResult: + """Run ``action`` on the driver session and record it in ``page_state``.""" + try: + result = await self.driver.execute(action) + except DriverError as error: + if error.dispatched: + self.page_state.note_dispatched() + raise + return self.page_state.record(action, result) + + async def observe(self, command: Observe) -> Observation: + """Observe through the driver session and record the observation in ``page_state``.""" + return self.page_state.record(command, await self.driver.observe(command)) + + +@dataclass(frozen=True) +class ToolFlags: + """What an extra tool declares; ``requires`` lists the commands the driver must handle.""" + + effect: Effect = Effect.READ + exempt_from_budget: bool = False + progress: bool = True + worker: bool = True + requires: tuple[type[Action], ...] = () + + +@dataclass(frozen=True) +class CommandRecord: + """What one tool call did, for the rails; ``action`` is masked, an ``Observe`` keeps its elements in ``value``.""" + + command: type[Action] + effect: Effect + ok: bool + dispatched: bool + context_changed: bool = False + error: str | None = None + blocked_by: str | None = None + surface: SurfaceContext | None = None + value: Any = field(default=None, hash=False) + generation: str = "" + action: Action | None = None + + @classmethod + def of( + cls, action: Action, result: ActionResult, *, effect: Effect | None = None, value: Any = None + ) -> CommandRecord: + """The record of a command the driver ran; ``effect`` overrides the command's own.""" + return cls( + command=type(action), + effect=action.effect if effect is None else effect, + ok=result.ok, + dispatched=True, + context_changed=result.context_changed, + surface=result.surface, + value=value, + generation=result.generation, + action=masked(action), + ) + + @classmethod + def failed( + cls, action: Action, error: DriverError, page_state: PageState, *, effect: Effect | None = None + ) -> CommandRecord: + """The record of a command the driver refused or failed.""" + return cls( + command=type(action), + effect=action.effect if effect is None else effect, + ok=False, + dispatched=error.dispatched, + error=type(error).__name__, + blocked_by=error.by if isinstance(error, ActionIntercepted) else None, + surface=page_state.surface_context, + generation=page_state.generation_id, + action=masked(action), + ) + + @property + def message_metadata(self) -> dict[str, bool]: + """The tool message metadata the context processors read.""" + mutating = self.effect is not Effect.READ and (self.ok or self.error == ActionTimeout.__name__) + return { + "success": self.ok, + "executed": self.ok if self.effect is Effect.READ else self.dispatched, + "state_changed": self.context_changed or mutating, + "denied": False, + } + + +@dataclass(frozen=True) +class AgentTools: + """``builders`` maps each command tool's model-facing name to its decoder, for the policy rail.""" + + tools: tuple[Tool, ...] + prompt_sections: tuple[str, ...] + builders: Mapping[str, Builder] + + +class CommandTool(Tool): + """One model-facing tool: its arguments become a command run on the driver session, its outcome rendered text.""" + + def __init__( + self, card: ToolCard, build: Builder, context: ToolContext, *, renders_observation: bool, image_source: str + ) -> None: + super().__init__(card) + self._build = build + self._context = context + self._renders_observation = renders_observation + self._image_source = image_source + + async def invoke(self, inputs: Mapping[str, Any], **kwargs: Any) -> ToolOutput: + del kwargs + page_state = self._context.page_state + try: + action = self._build(inputs or {}, page_state) + except ValueError as error: + return ToolOutput(success=False, error=f"{_ERROR}\n{error}") + try: + try: + page_state.check(action) + except StaleRef as stale: + return self._failure(action, stale, await self._context.observe(Observe())) + if isinstance(action, Observe): + return self._observed(action, await self._context.observe(action)) + result = await self._context.execute(action) + except StaleRef as stale: + fresh = stale.observation + return self._failure( + action, stale, None if fresh is None else self._context.page_state.record(Observe(), fresh) + ) + except DriverError as error: + return self._failure(action, error, None) + return self._output(action, result, result.value, self._context.renderer.result(action, result)) + + async def stream(self, inputs: Mapping[str, Any], **kwargs: Any) -> AsyncIterator[ToolOutput]: + yield await self.invoke(inputs, **kwargs) + + def _observed(self, action: Observe, observation: Observation) -> ToolOutput: + result = ActionResult( + ok=True, + value=observation.screenshot, + generation=observation.generation, + observation=observation, + surface=observation.surface, + ) + renderer = self._context.renderer + blocks = renderer.observation(observation) if self._renders_observation else renderer.result(action, result) + return self._output(action, result, observation.elements, blocks) + + def _output(self, action: Action, result: ActionResult, value: Any, blocks: tuple[ContentBlock, ...]) -> ToolOutput: + data = self._data(blocks, CommandRecord.of(action, result, value=value)) + return ToolOutput(success=result.ok, data=data, error=None if result.ok else data["content"]) + + def _failure(self, action: Action, error: DriverError, observation: Observation | None) -> ToolOutput: + record = CommandRecord.failed(action, error, self._context.page_state) + data = self._data(self._context.renderer.error(action, error, observation), record) + return ToolOutput(success=False, data=data, error=data["content"]) + + def _data(self, blocks: tuple[ContentBlock, ...], record: CommandRecord) -> dict[str, Any]: + text = "\n".join(block for block in blocks if isinstance(block, str)) + images = [block for block in blocks if isinstance(block, Screenshot)] if self._context.image_support else [] + if not images: + return {"content": text, "command": record} + note = f"{len(images)} image(s) attached as multimodal input." + return { + "content": f"{text}\n\n{note}" if text else note, + "multimodal": [self._image_item(image) for image in images], + "command": record, + } + + def _image_item(self, image: Screenshot) -> dict[str, str]: + encoded = base64.b64encode(image.png).decode("ascii") + return { + "type": "image", + "source": self._image_source, + "source_path": self.card.name, + "mime_type": "image/png", + "data_url": f"data:image/png;base64,{encoded}", + } + + +def command_record(tool_result: Any) -> CommandRecord | None: + """The record a ``CommandTool`` left in its output; ``None`` for a call that decoded to no command.""" + data = getattr(tool_result, "data", None) + record = data.get("command") if isinstance(data, Mapping) else None + return record if isinstance(record, CommandRecord) else None + + +class ToolFactory: + """Selects tool specs by capability and emits each as the driver's native tool or the profile's own form.""" + + def __init__(self, profile: SurfaceProfile, driver_info: DriverInfo) -> None: + self._profile = profile + self._info = driver_info + self._specs = {spec.name: spec for spec in profile.tool_specs} + self._reported: set[str] = set() + + def build( + self, + capabilities: Iterable[str], + *, + image_support: bool, + session: DriverSession, + page_state: PageState, + workspace: Any = None, + language: str = "cn", + worker: bool = False, + ) -> AgentTools: + context = ToolContext( + session, + page_state, + self._profile.renderer, + self._info, + self._profile.settings, + image_support, + workspace, + language, + ) + tools: list[Tool] = [] + sections: list[str] = [] + builders: dict[str, Builder] = {} + for spec, kinds in self._select(capabilities): + emitted = self._emit(spec, kinds) + if emitted is None: + continue + card, build = emitted + screenshot_only = spec.commands == (Observe,) and self._builds_screenshot(build) + if screenshot_only and not image_support: + continue + renders_observation = spec.commands == (Observe,) and not screenshot_only + tools.append( + CommandTool( + card, build, context, renders_observation=renders_observation, image_source=self._profile.name + ) + ) + builders[card.name] = build + if spec.prompt_section: + sections.append(spec.prompt_section) + for factory in self._profile.extra_tools: + flags = _flags(factory) + if worker and not flags.worker: + continue + missing = [command.__name__ for command in flags.requires if command not in self._info.actions] + if missing: + name = getattr(factory, "__name__", repr(factory)) + self._report(name, f"{self._info.name} does not handle {missing}") + continue + tools.append(factory(context)) + names = {tool.card.name for tool in tools} + for name in self._unserved(capabilities): + if name not in names and self._info.tool_name_prefix + name not in names: + self._report(name, "no tool spec and no extra tool serves it") + return AgentTools(tuple(tools), tuple(sections), MappingProxyType(builders)) + + def _select(self, capabilities: Iterable[str]) -> Iterable[tuple[ToolSpec, frozenset[type]]]: + seen: set[str] = set() + for entry in capabilities: + names = self._profile.capability_tools.get(entry, ()) + if not names: + self._report(entry, "the capability selects no tool") + pairs = self._profile.capability_catalog.get(entry, frozenset()) + for name in names: + spec = self._specs.get(name) + if spec is None or name in seen: + continue + seen.add(name) + kinds = self._handled_kinds(spec, pairs) + if kinds is None: + self._report(name, f"{self._info.name} does not handle its commands") + continue + yield spec, kinds + + def _unserved(self, capabilities: Iterable[str]) -> list[str]: + names = (name for entry in capabilities for name in self._profile.capability_tools.get(entry, ())) + return [name for name in names if name not in self._specs] + + def _handled_kinds(self, spec: ToolSpec, pairs: frozenset[tuple[type[Action], type | None]]) -> frozenset | None: + usable: set[type] = set() + for command in spec.commands: + if command is Observe: + continue + accepted = self._info.actions.get(command) + if accepted is None: + return None + needed = {kind for paired, kind in pairs if paired is command} + if needed and None not in needed and not needed & accepted: + return None + usable |= {kind for kind in needed if kind is not None} & accepted + return frozenset(usable) + + def _emit(self, spec: ToolSpec, kinds: frozenset) -> tuple[ToolCard, Builder] | None: + native = self._info.native_tools.get(spec.name) + if native is not None: + description, schema, build = native.description, native.schema, native.build + elif spec.schema is not None and spec.build is not None: + description, schema, build = self._description(spec), spec.schema, spec.build + elif len(spec.commands) == 1 and kinds: + description, schema, build = self._description(spec), _generated_schema(kinds), _generated_build(spec) + else: + self._report(spec.name, "no schema, and no single command with ref kinds to generate one from") + return None + name = self._info.tool_name_prefix + spec.name + return ToolCard(id=name, name=name, description=description, input_params=dict(schema)), build + + def _description(self, spec: ToolSpec) -> str: + if spec.description: + return spec.description + hints = [self._info.tool_hints[command] for command in spec.commands if command in self._info.tool_hints] + return hints[0] if hints else spec.commands[0].__doc__ or spec.commands[0].__name__ + + @staticmethod + def _builds_screenshot(build: Builder) -> bool: + try: + action = build({}, _NO_REFS) + except ValueError: + return False + return isinstance(action, Observe) and action.include_screenshot + + def _report(self, name: str, reason: str) -> None: + if name not in self._reported: + self._reported.add(name) + tool_logger.info("tool %s skipped: %s", name, reason) + + +def _flags(factory: Any) -> ToolFlags: + flags = getattr(factory, "flags", None) + return flags if isinstance(flags, ToolFlags) else ToolFlags() + + +class _NoRefs: + contexts: tuple[ContextRef, ...] = () + current_context: ContextRef | None = None + screenshot_generation = "" + + def resolve(self, model_id: str, generation: str) -> Ref: + raise ValueError(model_id) + + def generation_of(self, ref: str) -> str: + return "" + + +_NO_REFS = _NoRefs() + + +def _generated_schema(kinds: frozenset) -> dict[str, Any]: + properties: dict[str, Any] = {} + if DriverRef in kinds: + properties["ref"] = {"type": "string"} + if PointRef in kinds: + properties["x"] = {"type": "integer"} + properties["y"] = {"type": "integer"} + if Locator in kinds: + properties["css"] = {"type": "string"} + return {"type": "object", "properties": properties} + + +def _generated_build(spec: ToolSpec) -> Builder: + command = spec.commands[0] + + def build(args: Mapping[str, Any], resolver: RefResolver) -> Action: + if args.get("ref"): + ref = str(args["ref"]) + return command(DriverRef(ref, resolver.generation_of(ref))) + if args.get("x") is not None and args.get("y") is not None: + return command(PointRef(int(args["x"]), int(args["y"]), resolver.screenshot_generation)) + if args.get("css"): + css = str(args["css"]) + return command(Locator(css=css, text=None, generation=resolver.generation_of(css))) + return command() + + return build + + +__all__ = ["AgentTools", "CommandRecord", "CommandTool", "ToolContext", "ToolFactory", "ToolFlags", "command_record"] diff --git a/openjiuwen/harness/tools/browser_move/runtime/topology.py b/openjiuwen/harness/tools/browser_move/runtime/topology.py new file mode 100644 index 0000000000..0d9f43edd4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/topology.py @@ -0,0 +1,199 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Who runs a task behind the runtime: the main agent on the runtime itself, or a worker through the service.""" + +from __future__ import annotations + +import asyncio +import uuid +from typing import TYPE_CHECKING, Any, Protocol + +from openjiuwen.core.foundation.llm import Model, ModelClientConfig, ModelRequestConfig +from openjiuwen.core.runner import Runner +from openjiuwen.harness.image_modality_probe import probe_image_support + +from ..contract.registry import default_topology +from .agents import WORKER_LANGUAGE, build_worker +from .assembly import agent_over +from .config import CoreSettings + +if TYPE_CHECKING: + from .session import AgentRuntime + + +class Topology(Protocol): + name: str + + async def start(self, runtime: AgentRuntime) -> None: ... + + async def run( + self, + runtime: AgentRuntime, + task: str, + session_id: str | None, + request_id: str | None, + timeout_s: int | None, + ) -> dict[str, Any]: ... + + def drop(self) -> None: ... + + +def _model(service: Any) -> Model: + core = service.core + return Model( + model_client_config=ModelClientConfig( + client_provider=service.provider, api_key=service.api_key, api_base=service.api_base, verify_ssl=False + ), + model_config=ModelRequestConfig( + model=service.model_name, temperature=core.worker_temperature, top_p=core.worker_top_p + ), + ) + + +class Direct: + """The main agent, built on the runtime, calls the tools itself; one run per task under the guardrails' timeout.""" + + name = "direct" + + def __init__(self) -> None: + self._agent: Any = None + + async def start(self, runtime: AgentRuntime) -> None: + """The profile's main agent once; a profile without one gets the session now and the agent after the probe.""" + if self._agent is not None: + return + runtime.dedicated_log = False + main_agent = runtime.surface.profile.main_agent + if main_agent is None: + await runtime.ensure_runtime_ready() + return + service = runtime.service + self._agent = main_agent( + runtime, + _model(service), + language=WORKER_LANGUAGE, + max_iterations=service.guardrails.max_steps, + workspace=runtime.workspace, + capabilities=runtime.capabilities, + ) + + async def run( + self, + runtime: AgentRuntime, + task: str, + session_id: str | None, + request_id: str | None, + timeout_s: int | None, + ) -> dict[str, Any]: + service = runtime.service + sid = service.session_new(session_id) + rid = (request_id or "").strip() or uuid.uuid4().hex + timeout = int(timeout_s) if timeout_s else service.guardrails.timeout_s + inputs = {"query": task, "conversation_id": f"{sid}.direct.{rid}.{uuid.uuid4().hex}", "request_id": rid} + images = await probe_image_support(_model(service)) + if self._agent is None: + parts = runtime.agent_parts(image_support=bool(images), language=WORKER_LANGUAGE) + self._agent = agent_over( + parts, + runtime.core_rails(), + _model(service), + language=WORKER_LANGUAGE, + workspace=runtime.workspace, + max_iterations=service.guardrails.max_steps, + enable_read_image_multimodal=bool(images), + ) + try: + result = await asyncio.wait_for(Runner.run_agent(self._agent, inputs), timeout=float(timeout)) + except TimeoutError: + return runtime.main_agent_answer(sid, rid, error=f"task_timeout: exceeded {timeout}s") + except Exception as exc: # noqa: BLE001 + return runtime.main_agent_answer(sid, rid, error=str(exc) or repr(exc)) + return runtime.main_agent_answer(sid, rid, result=result if isinstance(result, dict) else {"output": result}) + + def drop(self) -> None: + self._agent = None + + +class DelegatedWorker: + """The runtime's caller hands the task to the service, which runs the worker over the session.""" + + name = "delegated_worker" + + def __init__(self) -> None: + self._image_support = False + self._offered: frozenset[str] = frozenset() + self._dropped: frozenset[str] = frozenset() + + async def start(self, runtime: AgentRuntime) -> None: + await runtime.ensure_runtime_ready() + service = runtime.service + if service.agent is None: + session = service.session + service.agent = await build_worker( + session, + session.assembly.surface, + provider=service.provider, + api_key=service.api_key, + api_base=service.api_base, + model_name=service.model_name, + max_steps=service.guardrails.max_steps, + core=service.core, + capabilities=runtime.capabilities, + tool_result_observer=service.observe_worker_tool_result, + screenshot_subdir=service.screenshot_subdir, + artifacts_subdir=service.artifacts_subdir, + image_support=self._image_support, + ) + if self._dropped: + service.agent.ability_manager.remove_ability(sorted(self._dropped)) + self._dropped = frozenset() + self._offered = _tool_names(service.agent) + + async def run( + self, + runtime: AgentRuntime, + task: str, + session_id: str | None, + request_id: str | None, + timeout_s: int | None, + ) -> dict[str, Any]: + if await probe_image_support(_model(runtime.service)) and not self._image_support: + self._image_support = True + started = runtime.service.agent + if started is not None: + self._dropped = self._offered - _tool_names(started) + runtime.service.agent = None + + async def worker() -> Any: + await self.start(runtime) + return runtime.service.agent + + return await runtime.service.run_task( + task=task, session_id=session_id, request_id=request_id, timeout_s=timeout_s, worker=worker + ) + + def drop(self) -> None: + """The worker lives in the service's slot, which the service clears on reset and shutdown.""" + + +def _tool_names(agent: Any) -> frozenset[str]: + return frozenset(ability.name for ability in agent.ability_manager.list()) + + +_TOPOLOGIES: dict[str, type[Direct] | type[DelegatedWorker]] = { + Direct.name: Direct, + DelegatedWorker.name: DelegatedWorker, +} + + +def topology_for(core: CoreSettings) -> Topology: + """The strategy named by ``core.topology``, else the one the driver's registration declares as its default.""" + name = core.topology or default_topology(core.driver) + if name not in _TOPOLOGIES: + known = ", ".join(sorted(_TOPOLOGIES)) + raise ValueError(f"driver {core.driver!r} declares the topology {name!r}, which is none of: {known}") + return _TOPOLOGIES[name]() + + +__all__ = ["WORKER_LANGUAGE", "DelegatedWorker", "Direct", "Topology", "topology_for"] diff --git a/openjiuwen/harness/tools/browser_move/utils/__init__.py b/openjiuwen/harness/tools/browser_move/runtime/utils/__init__.py similarity index 56% rename from openjiuwen/harness/tools/browser_move/utils/__init__.py rename to openjiuwen/harness/tools/browser_move/runtime/utils/__init__.py index d87bb58992..87eb34e80d 100644 --- a/openjiuwen/harness/tools/browser_move/utils/__init__.py +++ b/openjiuwen/harness/tools/browser_move/runtime/utils/__init__.py @@ -8,17 +8,8 @@ DEFAULT_GUARDRAIL_MAX_STEPS, DEFAULT_GUARDRAIL_RETRY_ONCE, DEFAULT_MODEL_NAME, - DEFAULT_PLAYWRIGHT_MCP_ARGS, - DEFAULT_PLAYWRIGHT_MCP_COMMAND, MISSING_API_KEY_MESSAGE, - load_repo_dotenv, parse_command_args, - resolve_bool_env, - resolve_browser_timeout_s, - resolve_int_env, - resolve_model_name, - resolve_model_settings, - resolve_repo_dotenv_path, ) from .parsing import extract_json_object @@ -28,16 +19,7 @@ "DEFAULT_GUARDRAIL_MAX_STEPS", "DEFAULT_GUARDRAIL_RETRY_ONCE", "DEFAULT_MODEL_NAME", - "DEFAULT_PLAYWRIGHT_MCP_ARGS", - "DEFAULT_PLAYWRIGHT_MCP_COMMAND", "MISSING_API_KEY_MESSAGE", "extract_json_object", - "load_repo_dotenv", "parse_command_args", - "resolve_bool_env", - "resolve_browser_timeout_s", - "resolve_int_env", - "resolve_model_name", - "resolve_model_settings", - "resolve_repo_dotenv_path", ] diff --git a/openjiuwen/harness/tools/browser_move/runtime/utils/env.py b/openjiuwen/harness/tools/browser_move/runtime/utils/env.py new file mode 100644 index 0000000000..9a751da854 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/utils/env.py @@ -0,0 +1,99 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Pure parsing helpers for the settings modules; nothing here reads the process environment.""" + +from __future__ import annotations + +import json +import shlex +from typing import Mapping + +SUPPORTED_MODEL_PROVIDERS = frozenset({"openai", "openrouter", "siliconflow", "dashscope"}) +TRUTHY_ENV_VALUES = frozenset({"1", "true", "yes", "on"}) +FALSY_ENV_VALUES = frozenset({"0", "false", "no", "off"}) +DEFAULT_MODEL_NAME = "anthropic/claude-sonnet-4.5" +DEFAULT_BROWSER_TIMEOUT_S = 600 +DEFAULT_GUARDRAIL_MAX_STEPS = 100 +DEFAULT_GUARDRAIL_MAX_FAILURES = 4 +DEFAULT_GUARDRAIL_RETRY_ONCE = True +DEFAULT_BROWSER_UPLOAD_ROOT = "" +MISSING_API_KEY_MESSAGE = ( + "Missing API key. Set API_KEY (or OPENROUTER_API_KEY / SILICONFLOW_API_KEY / OPENAI_API_KEY / DASHSCOPE_API_KEY)." +) +_PROVIDER_ALIASES = { + "alibaba": "dashscope", + "aliyun": "dashscope", + "silicon-flow": "siliconflow", + "silicon_flow": "siliconflow", +} + + +def parse_command_args(value: str) -> list[str]: + value = (value or "").strip() + if not value: + return [] + if value.startswith("["): + try: + parsed = json.loads(value) + if isinstance(parsed, list): + return [str(x) for x in parsed] + except json.JSONDecodeError: + pass + return shlex.split(value) + + +def env_value(environ: Mapping[str, str], key: str) -> str | None: + value = environ.get(key) + return environ.get(key.upper()) if value is None else value + + +def first_value(environ: Mapping[str, str], *keys: str, default: str = "") -> str: + """The first non-blank value among ``keys``, stripped.""" + for key in keys: + value = (environ.get(key) or "").strip() + if value: + return value + return default + + +def int_value(environ: Mapping[str, str], *keys: str, default: int, minimum: int) -> int: + """The first value among ``keys`` that parses as an int ``>= minimum``.""" + for key in keys: + try: + value = int((environ.get(key) or "").strip()) + except ValueError: + continue + if value >= minimum: + return value + return default + + +def normalize_provider(provider: str) -> str: + raw = (provider or "").strip() + lowered = raw.lower() + if lowered in SUPPORTED_MODEL_PROVIDERS: + return lowered + return _PROVIDER_ALIASES.get(lowered, raw) + + +def is_truthy_env(value: str) -> bool: + lowered = (value or "").strip().lower() + return lowered in TRUTHY_ENV_VALUES + + +def is_falsy_env(value: str) -> bool: + lowered = (value or "").strip().lower() + return lowered in FALSY_ENV_VALUES + + +def infer_provider_from_api_base(api_base: str) -> str: + base = (api_base or "").strip().lower() + if not base: + return "" + if "openrouter.ai" in base: + return "openrouter" + if "siliconflow" in base: + return "siliconflow" + if "dashscope" in base: + return "dashscope" + return "openai" diff --git a/openjiuwen/harness/tools/browser_move/runtime/utils/parsing.py b/openjiuwen/harness/tools/browser_move/runtime/utils/parsing.py new file mode 100644 index 0000000000..f5923d97cb --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/utils/parsing.py @@ -0,0 +1,65 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Shared parsing helpers for runtime/controller responses.""" + +from __future__ import annotations + +import json +from typing import Any + + +def extract_json_object(text: Any) -> dict[str, Any]: + """Best-effort JSON extraction from model or tool text.""" + if isinstance(text, dict): + return text + if text is None: + return {} + + raw = str(text).strip() + if not raw: + return {} + + marker_result = "### Result" + marker_ran = "### Ran Playwright code" + if marker_result in raw and marker_ran in raw: + start = raw.find(marker_result) + len(marker_result) + end = raw.find(marker_ran, start) + if end > start: + raw = raw[start:end].strip() + + for _ in range(2): + try: + parsed = json.loads(raw) + except json.JSONDecodeError: + break + if isinstance(parsed, dict): + return parsed + if isinstance(parsed, str): + raw = parsed.strip() + continue + break + + if "```json" in raw: + start = raw.find("```json") + len("```json") + end = raw.find("```", start) + if end > start: + block = raw[start:end].strip() + try: + parsed = json.loads(block) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict): + return parsed + + first = raw.find("{") + last = raw.rfind("}") + if first >= 0 and last > first: + snippet = raw[first : last + 1] + try: + parsed = json.loads(snippet) + except json.JSONDecodeError: + parsed = None + if isinstance(parsed, dict): + return parsed + + return {} diff --git a/openjiuwen/harness/tools/browser_move/runtime/worker.py b/openjiuwen/harness/tools/browser_move/runtime/worker.py new file mode 100644 index 0000000000..d9020a7e5e --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/runtime/worker.py @@ -0,0 +1,1034 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The worker loop and its progress state: attempts, restarts, the failure context and the answer of one task.""" + +from __future__ import annotations + +import asyncio +import contextvars +import uuid +from dataclasses import dataclass, field +from typing import Any, Awaitable, Callable, Dict, Optional + +from openjiuwen.core.common.logging import logger +from openjiuwen.core.runner import Runner +from openjiuwen.core.single_agent.agents.react_agent import ReActAgent + +from .agents import default_worker_prompt, refresh_browser_worker_system_prompt, worker_driver_name +from .rails.final_answer_recovery import ( + final_answer_recovery, + recovery_message, + recovery_reason, +) +from .utils.parsing import extract_json_object + +MAX_ITERATION_MESSAGE = "Max iterations reached without completion" +_ctx_observer_session_id: contextvars.ContextVar[str] = contextvars.ContextVar( + "playwright_runtime_observer_session_id", + default="", +) +_ctx_observer_request_id: contextvars.ContextVar[str] = contextvars.ContextVar( + "playwright_runtime_observer_request_id", + default="", +) + + +@dataclass +class TaskProgressState: + request_id: str = "" + status: str = "unknown" + completed_steps: list[str] = field(default_factory=list) + remaining_steps: list[str] = field(default_factory=list) + next_step: str = "" + completion_evidence: list[str] = field(default_factory=list) + missing_requirements: list[str] = field(default_factory=list) + recent_tool_steps: list[str] = field(default_factory=list) + last_page_url: str = "" + last_page_title: str = "" + last_screenshot: Any = None + last_worker_final: str = "" + + @classmethod + def from_dict(cls, data: Optional[Dict[str, Any]]) -> "TaskProgressState": + if not isinstance(data, dict): + return cls() + last_page = data.get("last_page") if isinstance(data.get("last_page"), dict) else {} + return cls( + request_id=str(data.get("request_id") or "").strip(), + status=str(data.get("status") or "unknown").strip() or "unknown", + completed_steps=[str(item).strip() for item in data.get("completed_steps") or [] if str(item).strip()], + remaining_steps=[str(item).strip() for item in data.get("remaining_steps") or [] if str(item).strip()], + next_step=str(data.get("next_step") or "").strip(), + completion_evidence=[ + str(item).strip() for item in data.get("completion_evidence") or [] if str(item).strip() + ], + missing_requirements=[ + str(item).strip() for item in data.get("missing_requirements") or [] if str(item).strip() + ], + recent_tool_steps=[str(item).strip() for item in data.get("recent_tool_steps") or [] if str(item).strip()], + last_page_url=str(last_page.get("url") or "").strip(), + last_page_title=str(last_page.get("title") or "").strip(), + last_screenshot=data.get("last_screenshot"), + last_worker_final=str(data.get("last_worker_final") or "").strip(), + ) + + @classmethod + def from_task_state(cls, data: Optional[Dict[str, Any]]) -> "TaskProgressState": + """Build the legacy progress DTO from runtime-owned task state.""" + + if not isinstance(data, dict) or not data: + return cls() + completed_steps, remaining_steps, completion_evidence = cls._phase_progress(data) + completion_evidence.extend(cls._evidence_summaries(data)) + recent_steps = cls._recent_action_summaries(data) + current_phase = str(data.get("current_phase") or "").strip() + next_action = str(data.get("next_action_class") or current_phase).strip() + last_page = data.get("last_page") if isinstance(data.get("last_page"), dict) else {} + return cls( + request_id=str(data.get("task_id") or "").strip(), + status=str(data.get("status") or "unknown").strip() or "unknown", + completed_steps=completed_steps, + remaining_steps=remaining_steps, + next_step=next_action, + completion_evidence=completion_evidence[-10:], + missing_requirements=[str(item) for item in data.get("blockers") or [] if str(item).strip()], + recent_tool_steps=recent_steps, + last_page_url=str(last_page.get("url") or "").strip(), + last_page_title=str(last_page.get("title") or "").strip(), + last_worker_final=str(data.get("last_worker_final") or "").strip(), + ) + + @staticmethod + def _phase_progress(data: Dict[str, Any]) -> tuple[list[str], list[str], list[str]]: + phases = data.get("phases") if isinstance(data.get("phases"), dict) else {} + completed_steps: list[str] = [] + remaining_steps: list[str] = [] + completion_evidence: list[str] = [] + for phase_name, details in phases.items(): + if not isinstance(details, dict): + continue + phase_label = str(phase_name).strip() + if details.get("status") == "completed": + completed_steps.append(phase_label) + evidence = str(details.get("completion_evidence") or "").strip() + if evidence: + completion_evidence.append(f"{phase_label}: {evidence}") + else: + remaining_steps.append(phase_label) + return completed_steps, remaining_steps, completion_evidence + + @staticmethod + def _evidence_summaries(data: Dict[str, Any]) -> list[str]: + summaries: list[str] = [] + evidence_records = data.get("structured_evidence") + if isinstance(evidence_records, list): + for record in evidence_records[-3:]: + if not isinstance(record, dict): + continue + fields = ", ".join(str(item) for item in record.get("fields") or []) + if fields: + summaries.append(f"{record.get('kind') or 'evidence'}: {fields}") + evidence_slots = data.get("evidence_slots") + if isinstance(evidence_slots, list): + for slot in evidence_slots[-4:]: + if not isinstance(slot, dict): + continue + label = "/".join(str(slot.get(key) or "") for key in ("entity", "variant", "field")).strip("/") + value = str(slot.get("value") or "").strip() + if label and value: + summaries.append(f"{label}: {value[:160]}") + return summaries + + @staticmethod + def _recent_action_summaries(data: Dict[str, Any]) -> list[str]: + recent_steps: list[str] = [] + recent_actions = data.get("recent_actions") + if isinstance(recent_actions, list): + for action in recent_actions[-6:]: + if not isinstance(action, dict): + continue + summary_parts = ( + str(action.get("action_class") or "").strip(), + str(action.get("target_summary") or "").strip(), + str(action.get("outcome") or "").strip(), + str(action.get("semantic_delta") or "").strip(), + ) + summary = " ".join(filter(None, summary_parts)) + if summary: + recent_steps.append(summary[:300]) + return recent_steps + + def is_empty(self) -> bool: + return ( + self.status == "unknown" + and not self.completed_steps + and not self.remaining_steps + and not self.next_step + and not self.completion_evidence + and not self.missing_requirements + and not self.recent_tool_steps + and not self.last_page_url + and not self.last_page_title + and not self.last_worker_final + and self.last_screenshot in (None, "") + ) + + def to_dict(self) -> Dict[str, Any]: + return { + "status": self.status, + "completed_steps": list(self.completed_steps), + "remaining_steps": list(self.remaining_steps), + "next_step": self.next_step or None, + "completion_evidence": list(self.completion_evidence), + "missing_requirements": list(self.missing_requirements), + "recent_tool_steps": list(self.recent_tool_steps), + "last_page": { + "url": self.last_page_url, + "title": self.last_page_title, + }, + "last_screenshot": self.last_screenshot, + "last_worker_final": self.last_worker_final or None, + "request_id": self.request_id or None, + } + + +class _WorkerLoop: + @staticmethod + def _is_retryable_transport_message(text: str) -> bool: + lowered = str(text or "").lower() + markers = ( + "session terminated", + "not connected", + "endofstream", + "closedresourceerror", + "brokenresourceerror", + "stream closed", + "connection closed", + "broken pipe", + "remoteprotocolerror", + "readerror", + "writeerror", + ) + return any(marker in lowered for marker in markers) + + @classmethod + def _is_retryable_transport_error(cls, exc: Exception) -> bool: + name = type(exc).__name__.lower() + text = str(exc).lower() + return cls._is_retryable_transport_message(name) or cls._is_retryable_transport_message(text) + + @staticmethod + def _is_retryable_runtime_result(parsed: Dict[str, Any]) -> bool: + if not isinstance(parsed, dict) or bool(parsed.get("ok", False)): + return False + text = (f"{parsed.get('error', '')}\n{parsed.get('final', '')}").lower() + markers = ( + "frame has been detached", + "execution context was destroyed", + "target page, context or browser has been closed", + "target closed", + "navigation failed because browser has disconnected", + "context closed", + "page crashed", + "net::err_network_changed", + "net::err_internet_disconnected", + ) + return any(marker in text for marker in markers) + + @staticmethod + def _should_restart_after_runtime_result(parsed: Dict[str, Any]) -> bool: + if not isinstance(parsed, dict): + return False + text = (f"{parsed.get('error', '')}\n{parsed.get('final', '')}").lower() + restart_markers = ( + "frame has been detached", + "target page, context or browser has been closed", + "target closed", + "context closed", + "page crashed", + ) + return any(marker in text for marker in restart_markers) + + @staticmethod + def _build_worker_conversation_id(session_id: str, request_id: str) -> str: + sid = (session_id or "").strip() or "session" + rid = (request_id or "").strip() or "request" + return f"{sid}:worker:{rid}:{uuid.uuid4().hex}" + + @classmethod + def _clean_progress_items(cls, value: Any, *, limit: int) -> list[str]: + if value is None: + return [] + if isinstance(value, (list, tuple, set)): + candidates = list(value) + else: + candidates = [value] + cleaned: list[str] = [] + seen: set[str] = set() + for item in candidates: + text_value = " ".join(str(item or "").split()).strip() + if not text_value: + continue + lowered = text_value.lower() + if lowered in seen: + continue + seen.add(lowered) + cleaned.append(cls._trim_text(text_value, 220)) + if len(cleaned) >= limit: + break + return cleaned + + @staticmethod + def _normalize_progress_status(value: Any) -> str: + normalized = str(value or "").strip().lower() + aliases = { + "complete": "completed", + "completed": "completed", + "done": "completed", + "partial": "partial", + "in_progress": "partial", + "in-progress": "partial", + "blocked": "blocked", + "failed": "failed", + } + return aliases.get(normalized, "") + + @staticmethod + def _push_recent_tool_step(existing: list[str], step: str, *, limit: int = 8) -> list[str]: + normalized = " ".join(str(step or "").split()).strip() + if not normalized: + return list(existing) + updated = [item for item in existing if item != normalized] + updated.append(normalized) + return updated[-limit:] + + @classmethod + def _extract_page_snapshot(cls, value: Any) -> tuple[str, str]: + if not isinstance(value, dict): + return "", "" + location = value.get("page") if isinstance(value.get("page"), dict) else {} + address = str(value.get("url") or location.get("url") or "").strip() + title = str(value.get("title") or location.get("title") or "").strip() + return cls._trim_text(address, 240), cls._trim_text(title, 120) + + @staticmethod + def _extract_screenshot_snapshot(value: Any) -> Any: + if not isinstance(value, dict): + return None + screenshot = value.get("screenshot") + if screenshot in (None, ""): + return None + return screenshot + + @classmethod + def _summarize_observation_payload(cls, value: Any) -> str: + if isinstance(value, dict): + parts: list[str] = [] + error_text = str(value.get("error") or "").strip() + if error_text: + parts.append(f"error={cls._trim_text(error_text, 140)}") + for key in ("message", "text", "result", "output", "value", "selector"): + candidate = value.get(key) + if candidate in (None, "") or isinstance(candidate, (dict, list, tuple, set)): + continue + parts.append(cls._trim_text(str(candidate), 160)) + break + address, title = cls._extract_page_snapshot(value) + if address: + parts.append(f"url={address}") + if title: + parts.append(f"title={title}") + ok_value = value.get("ok") + if ok_value is True and not parts: + parts.append("ok") + if not parts and value: + parts.append(", ".join(list(value.keys())[:4])) + return "; ".join(parts) + if isinstance(value, (list, tuple)): + nested = [cls._summarize_observation_payload(item) for item in list(value)[:2]] + nested = [item for item in nested if item] + return " | ".join(nested) + text_value = " ".join(str(value or "").split()).strip() + return cls._trim_text(text_value, 160) + + @classmethod + def _summarize_tool_result(cls, tool_name: str, tool_result: Any) -> str: + payload_summary = cls._summarize_observation_payload(tool_result) or "completed" + if tool_name: + return f"{tool_name}: {payload_summary}" + return payload_summary + + def _get_progress_state(self, session_id: str) -> TaskProgressState: + return self._progress_by_session.setdefault(session_id, TaskProgressState()) + + def _update_progress_from_tool_observation( + self, + *, + session_id: str, + request_id: str, + tool_name: str, + tool_result: Any, + ) -> None: + sid = (session_id or "").strip() + if not sid: + return + progress_state = self._get_progress_state(sid) + if request_id: + progress_state.request_id = request_id + progress_state.recent_tool_steps = self._push_recent_tool_step( + progress_state.recent_tool_steps, + self._summarize_tool_result(tool_name, tool_result), + ) + address, title = self._extract_page_snapshot(tool_result) + if address: + progress_state.last_page_url = address + if title: + progress_state.last_page_title = title + screenshot = self._extract_screenshot_snapshot(tool_result) + if screenshot is not None: + progress_state.last_screenshot = screenshot + if progress_state.status == "unknown": + progress_state.status = "partial" + + async def observe_worker_tool_result(self, tool_name: str, tool_result: Any) -> None: + session_id = _ctx_observer_session_id.get().strip() + if not session_id: + return + request_id = _ctx_observer_request_id.get().strip() + self._update_progress_from_tool_observation( + session_id=session_id, + request_id=request_id, + tool_name=tool_name, + tool_result=tool_result, + ) + + @classmethod + def _should_treat_as_completed(cls, parsed: Dict[str, Any]) -> bool: + status = cls._normalize_progress_status(parsed.get("status") or parsed.get("task_status")) + if status != "completed": + return False + progress = parsed.get("progress") if isinstance(parsed.get("progress"), dict) else {} + missing = cls._clean_progress_items( + progress.get("missing_requirements") or parsed.get("missing_requirements"), + limit=4, + ) + evidence = cls._clean_progress_items( + progress.get("completion_evidence") or parsed.get("completion_evidence"), + limit=4, + ) + final_text = str(parsed.get("final") or "").strip() + return not missing and bool(evidence or final_text) + + def _update_progress_from_worker_result( + self, + *, + session_id: str, + request_id: str, + parsed: Dict[str, Any], + ) -> None: + sid = (session_id or "").strip() + if not sid or not isinstance(parsed, dict): + return + progress_state = self._get_progress_state(sid) + if request_id: + progress_state.request_id = request_id + + progress_payload = parsed.get("progress") if isinstance(parsed.get("progress"), dict) else {} + status = self._normalize_progress_status(parsed.get("status") or parsed.get("task_status")) + if not status: + if bool(parsed.get("ok", False)): + status = "completed" + elif self._is_max_iteration_result(parsed): + status = "partial" + elif str(parsed.get("error") or "").strip(): + status = "failed" + else: + status = "partial" + progress_state.status = status + + completed_steps = self._clean_progress_items( + progress_payload.get("completed_steps") or parsed.get("completed_steps"), + limit=8, + ) + if completed_steps: + progress_state.completed_steps = completed_steps + + remaining_steps = self._clean_progress_items( + progress_payload.get("remaining_steps") or parsed.get("remaining_steps"), + limit=8, + ) + if remaining_steps: + progress_state.remaining_steps = remaining_steps + + next_step = str(progress_payload.get("next_step") or parsed.get("next_step") or "").strip() + if next_step: + progress_state.next_step = self._trim_text(next_step, 220) + + completion_evidence = self._clean_progress_items( + progress_payload.get("completion_evidence") or parsed.get("completion_evidence"), + limit=6, + ) + if completion_evidence: + progress_state.completion_evidence = completion_evidence + + missing_requirements = self._clean_progress_items( + progress_payload.get("missing_requirements") or parsed.get("missing_requirements"), + limit=6, + ) + if missing_requirements: + progress_state.missing_requirements = missing_requirements + + address, title = self._extract_page_snapshot(parsed) + if address: + progress_state.last_page_url = address + if title: + progress_state.last_page_title = title + screenshot = self._extract_screenshot_snapshot(parsed) + if screenshot is not None: + progress_state.last_screenshot = screenshot + + final_text = str(parsed.get("final") or "").strip() + if final_text: + progress_state.last_worker_final = self._trim_text(final_text, 1200) + if bool(parsed.get("ok", False)) and not progress_state.completion_evidence: + progress_state.completion_evidence = [self._trim_text(final_text, 220)] + + @classmethod + def _build_progress_context(cls, progress_state: Optional[TaskProgressState]) -> str: + if progress_state is None: + return "" + lines: list[str] = [] + if progress_state.status and progress_state.status != "unknown": + lines.append(f"- Known progress status: {progress_state.status}") + if progress_state.completed_steps: + lines.append(f"- Completed steps: {' | '.join(progress_state.completed_steps)}") + if progress_state.remaining_steps: + lines.append(f"- Remaining steps: {' | '.join(progress_state.remaining_steps)}") + if progress_state.next_step: + lines.append(f"- Next step to try: {progress_state.next_step}") + if progress_state.completion_evidence: + lines.append(f"- Completion evidence observed: {' | '.join(progress_state.completion_evidence)}") + if progress_state.missing_requirements: + lines.append(f"- Missing requirements / blockers: {' | '.join(progress_state.missing_requirements)}") + if progress_state.recent_tool_steps: + lines.append("- Recent tool activity:") + lines.extend(f" - {step}" for step in progress_state.recent_tool_steps[-6:]) + if not lines: + return "" + return "Known progress for continuation:\n" + "\n".join(lines) + + @staticmethod + def _is_empty_progress_state(progress_state: TaskProgressState) -> bool: + return ( + progress_state.status == "unknown" + and not any( + ( + progress_state.completed_steps, + progress_state.remaining_steps, + progress_state.next_step, + progress_state.completion_evidence, + progress_state.missing_requirements, + progress_state.recent_tool_steps, + progress_state.last_page_url, + progress_state.last_page_title, + progress_state.last_worker_final, + ) + ) + and progress_state.last_screenshot in (None, "") + ) + + def _export_progress_state(self, session_id: str) -> Optional[Dict[str, Any]]: + progress_state = self._progress_by_session.get((session_id or "").strip()) + if progress_state is None: + return None + if self._is_empty_progress_state(progress_state): + return None + return progress_state.to_dict() + + @classmethod + def should_treat_as_completed(cls, parsed: Dict[str, Any]) -> bool: + return cls._should_treat_as_completed(parsed) + + def build_failure_summary( + self, + *, + task: str, + error: str, + page_url: str, + page_title: str, + final: str, + screenshot: Optional[str], + attempt: int, + progress_state: Optional[TaskProgressState], + ) -> str: + return self._build_failure_summary( + task=task, + error=error, + page_url=page_url, + page_title=page_title, + final=final, + screenshot=screenshot, + attempt=attempt, + progress_state=progress_state, + ) + + def set_progress_state(self, session_id: str, progress_state: TaskProgressState) -> None: + sid = (session_id or "").strip() + if not sid: + return + if progress_state.is_empty(): + self._progress_by_session.pop(sid, None) + return + self._progress_by_session[sid] = progress_state + + def clear_progress_state(self, session_id: str) -> None: + sid = (session_id or "").strip() + if not sid: + return + self._progress_by_session.pop(sid, None) + + async def _run_task_once( + self, task: str, session_id: str, request_id: str, worker: Callable[[], Awaitable[ReActAgent]] + ) -> Dict[str, Any]: + agent = await worker() + task_prompt = ( + f"Session id: {session_id}\n" + f"Request id: {request_id}\n" + f"Max steps: {self.guardrails.max_steps}\n" + f"Max failures: {self.guardrails.max_failures}\n\n" + f"Task:\n{task}\n\n" + "Perform the task in the current logical session for this session id." + ) + if isinstance(agent, ReActAgent): + await refresh_browser_worker_system_prompt( + agent, + worker_prompt=self._session.assembly.surface.profile.worker_prompt or default_worker_prompt, + screenshot_subdir=self._screenshot_subdir, + artifacts_subdir=self._artifacts_subdir, + driver_name=worker_driver_name(self._session.assembly.stack.info), + ) + worker_conversation_id = self._build_worker_conversation_id(session_id, request_id) + token_session = _ctx_observer_session_id.set(session_id) + token_request = _ctx_observer_request_id.set(request_id) + try: + result = await Runner.run_agent( + agent, + {"query": task_prompt, "conversation_id": worker_conversation_id, "request_id": request_id}, + ) + finally: + _ctx_observer_session_id.reset(token_session) + _ctx_observer_request_id.reset(token_request) + output_text = result.get("output") if isinstance(result, dict) else result + parsed = extract_json_object(output_text) or self._ended_by_runtime(result) + if parsed: + self._update_progress_from_worker_result( + session_id=session_id, + request_id=request_id, + parsed=parsed, + ) + return parsed + + output_str = str(output_text) if output_text is not None else "" + output_lower = output_str.lower() + max_iterations_reached = MAX_ITERATION_MESSAGE.lower() in output_lower + reason = recovery_reason(output_str, max_iterations_reached=max_iterations_reached) + recovery_text = await self._recover_final_answer(agent, worker_conversation_id, request_id, reason) + recovered = extract_json_object(recovery_text) + logger.info( + "Worker final-answer recovery: reason=%s produced_json=%s output_chars=%d", + reason, + bool(recovered), + len(recovery_text), + ) + if recovered: + recovered["recovered_from"] = reason + self._update_progress_from_worker_result( + session_id=session_id, + request_id=request_id, + parsed=recovered, + ) + return recovered + if recovery_text.strip() and (max_iterations_reached or not output_str.strip()): + output_str = recovery_text + + if max_iterations_reached: + parsed = { + "ok": False, + "final": output_str, + "page": {"url": "", "title": ""}, + "screenshot": None, + "error": "max_iterations_reached", + "status": "partial", + } + self._update_progress_from_worker_result( + session_id=session_id, + request_id=request_id, + parsed=parsed, + ) + return parsed + + parsed = { + "ok": False, + "final": output_str, + "page": {"url": "", "title": ""}, + "screenshot": None, + "error": "Worker did not return valid JSON output", + "status": "failed", + } + self._update_progress_from_worker_result( + session_id=session_id, + request_id=request_id, + parsed=parsed, + ) + return parsed + + @staticmethod + def _ended_by_runtime(result: Any) -> Optional[Dict[str, Any]]: + """The answer of a run a rail ended with an error of its own (no JSON to recover): the task is blocked.""" + if not isinstance(result, dict) or result.get("result_type") != "error": + return None + error = str(result.get("error") or "").strip() + if not error: + return None + return {"ok": False, "final": str(result.get("output") or error), "error": error, "status": "blocked"} + + async def _recover_final_answer( + self, agent: ReActAgent, worker_conversation_id: str, request_id: str, reason: str + ) -> str: + """Ask the worker once, without tools and in the same conversation, for its final JSON.""" + try: + with final_answer_recovery(): + result = await Runner.run_agent( + agent, + { + "query": recovery_message(reason), + "conversation_id": worker_conversation_id, + "request_id": request_id, + }, + ) + except Exception: # noqa: BLE001 + logger.warning("Worker final-answer recovery failed: reason=%s", reason, exc_info=True) + return "" + output = result.get("output") if isinstance(result, dict) else result + return str(output) if output is not None else "" + + async def run_task_once( + self, task: str, session_id: str, request_id: str, *, worker: Callable[[], Awaitable[ReActAgent]] + ) -> Dict[str, Any]: + return await self._run_task_once(task=task, session_id=session_id, request_id=request_id, worker=worker) + + @staticmethod + def _is_max_iteration_result(parsed: Dict[str, Any]) -> bool: + if not isinstance(parsed, dict): + return False + if str(parsed.get("error", "")).strip().lower() == "max_iterations_reached": + return True + marker = MAX_ITERATION_MESSAGE.lower() + for key in ("final", "error"): + value = parsed.get(key) + if value is None: + continue + if marker in str(value).lower(): + return True + return False + + @staticmethod + def _build_resume_task(task: str, previous_final: str, progress_context: str = "") -> str: + base = (task or "").strip() + previous = (previous_final or "").strip() + if len(previous) > 1200: + previous = previous[:1200] + "...[truncated]" + context_parts: list[str] = [ + "Continuation context:", + "- The previous run reached max iterations before completion.", + "- Continue from the current state in this same session.", + "- Avoid repeating already completed steps unless needed for recovery.", + ] + if progress_context: + context_parts.append(progress_context) + if previous: + context_parts.extend( + [ + "- Previous partial status (may be incomplete):", + previous, + ] + ) + return f"{base}\n\n" + "\n".join(context_parts) + + @staticmethod + def _trim_text(value: Any, limit: int) -> str: + text = str(value or "").strip() + if len(text) > limit: + return text[:limit] + "...[truncated]" + return text + + @classmethod + def _build_failure_summary( + cls, + *, + task: str, + error: str, + page_url: str, + page_title: str, + final: str, + screenshot: Any, + attempt: int, + progress_state: Optional[TaskProgressState] = None, + ) -> str: + lines = [ + "Failure summary for continuation:", + f"- Original task: {cls._trim_text(task, 400) or '(empty)'}", + f"- Failed attempt: {attempt}", + f"- Error: {cls._trim_text(error, 300) or '(unknown)'}", + ] + if page_url or page_title: + lines.append( + f"- Last location: {cls._trim_text(page_url, 240) or '(unknown)'}, " + f"title={cls._trim_text(page_title, 120) or '(unknown)'}" + ) + screenshot_text = cls._trim_text(screenshot, 200) + if screenshot_text: + lines.append(f"- Last screenshot: {screenshot_text}") + progress_context = cls._build_progress_context(progress_state) + if progress_context: + lines.append(progress_context) + final_excerpt = cls._trim_text(final, 1200) + if final_excerpt: + lines.append("- Partial output excerpt:") + lines.append(final_excerpt) + return "\n".join(lines) + + @staticmethod + def _build_task_with_failure_context(task: str, failure_summary: str) -> str: + base = (task or "").strip() + summary = (failure_summary or "").strip() + if not summary: + return base + return ( + f"{base}\n\n" + "Previous failed attempt context:\n" + f"{summary}\n\n" + "Continuation instructions:\n" + "- Continue from the current state in this same session.\n" + "- Do not repeat completed steps unless required for recovery.\n" + "- Prioritize resolving the listed failure." + ) + + async def run_task( + self, + task: str, + session_id: Optional[str] = None, + request_id: Optional[str] = None, + timeout_s: Optional[int] = None, + *, + worker: Callable[[], Awaitable[ReActAgent]], + ) -> Dict[str, Any]: + """Run ``task`` on the worker ``worker`` returns; after a restart the next call to it rebuilds the worker.""" + sid = self.session_new(session_id) + rid = (request_id or "").strip() or uuid.uuid4().hex + effective_timeout = int(timeout_s) if (timeout_s is not None and timeout_s > 0) else self.guardrails.timeout_s + timeout_deadline = asyncio.get_running_loop().time() + float(effective_timeout) + attempts = 2 if self.guardrails.retry_once else 1 + base_task = (task or "").strip() + previous_failure_summary = self._failure_context_by_session.get(sid, "") + + async with self._locks[sid]: + current_task = asyncio.current_task() + if current_task is not None: + self._register_inflight_task(sid, rid, current_task) + try: + if await self.is_cancelled(sid, rid): + await self.clear_cancel(sid, rid) + await self.clear_cancel(sid, None) + result = { + "ok": False, + "session_id": sid, + "request_id": rid, + "final": "", + "page": {"url": "", "title": ""}, + "screenshot": None, + "error": "cancelled_by_frontend", + "attempt": 0, + "failure_summary": None, + "progress_state": None, + } + return result + last_error: Optional[str] = None + used_max_iteration_resume = False + next_task = self._build_task_with_failure_context(base_task, previous_failure_summary) + attempt_idx = 0 + max_attempts = attempts + (1 if self.guardrails.resume_on_max_iterations else 0) + last_failure_final = "" + last_failure_page: Dict[str, Any] = {} + last_failure_screenshot: Any = None + while attempt_idx < max_attempts: + remaining_timeout = timeout_deadline - asyncio.get_running_loop().time() + if remaining_timeout <= 0: + last_error = f"task_timeout: exceeded {effective_timeout}s" + break + try: + parsed = await asyncio.wait_for( + self.run_task_once(task=next_task, session_id=sid, request_id=rid, worker=worker), + timeout=remaining_timeout, + ) + attempt_idx += 1 + self._update_progress_from_worker_result( + session_id=sid, + request_id=rid, + parsed=parsed, + ) + parsed_ok = bool(parsed.get("ok", False)) + if not parsed_ok and self._should_treat_as_completed(parsed): + parsed = dict(parsed) + parsed["ok"] = True + parsed["error"] = None + parsed_ok = True + should_resume_max_iter = ( + (not parsed_ok) + and self._is_max_iteration_result(parsed) + and self.guardrails.resume_on_max_iterations + and not used_max_iteration_resume + ) + if not parsed_ok: + last_error = str(parsed.get("error") or "") + last_failure_final = str(parsed.get("final", "")) + last_failure_page = parsed.get("page") if isinstance(parsed.get("page"), dict) else {} + last_failure_screenshot = parsed.get("screenshot") + + if should_resume_max_iter: + used_max_iteration_resume = True + next_task = self._build_resume_task( + next_task, + str(parsed.get("final", "")), + progress_context=self._build_progress_context(self._progress_by_session.get(sid)), + ) + last_error = str(parsed.get("error") or MAX_ITERATION_MESSAGE) + continue + + if not parsed_ok and attempt_idx < attempts and self._is_retryable_runtime_result(parsed): + location = parsed.get("page") if isinstance(parsed.get("page"), dict) else {} + failure_summary = self._build_failure_summary( + task=base_task, + error=str(parsed.get("error") or ""), + page_url=str(location.get("url", "")), + page_title=str(location.get("title", "")), + final=str(parsed.get("final", "")), + screenshot=parsed.get("screenshot"), + attempt=attempt_idx, + progress_state=self._progress_by_session.get(sid), + ) + should_restart = self._is_retryable_transport_message( + failure_summary + ) or self._should_restart_after_runtime_result(parsed) + if should_restart: + try: + await self._restart() + except Exception as restart_exc: + last_error = f"restart_failed: {restart_exc!r}" + break + next_task = self._build_task_with_failure_context(base_task, failure_summary) + continue + + location = parsed.get("page") if isinstance(parsed.get("page"), dict) else {} + screenshot = self._normalize_screenshot_value(parsed.get("screenshot")) + response = { + "ok": parsed_ok, + "session_id": sid, + "request_id": rid, + "final": str(parsed.get("final", "")), + "page": { + "url": str(location.get("url", "")), + "title": str(location.get("title", "")), + }, + "screenshot": screenshot, + "error": parsed.get("error"), + "attempt": attempt_idx, + "progress_state": None, + } + if parsed.get("recovered_from"): + response["recovered_from"] = parsed["recovered_from"] + if parsed_ok: + self._failure_context_by_session.pop(sid, None) + self._progress_by_session.pop(sid, None) + response["failure_summary"] = None + return response + + failure_summary = self._build_failure_summary( + task=base_task, + error=str(parsed.get("error") or ""), + page_url=str(location.get("url", "")), + page_title=str(location.get("title", "")), + final=str(parsed.get("final", "")), + screenshot=parsed.get("screenshot"), + attempt=attempt_idx, + progress_state=self._progress_by_session.get(sid), + ) + self._failure_context_by_session[sid] = failure_summary + response["failure_summary"] = failure_summary + response["progress_state"] = self._export_progress_state(sid) + return response + except TimeoutError: + attempt_idx += 1 + last_error = f"task_timeout: exceeded {effective_timeout}s" + if attempt_idx >= attempts or asyncio.get_running_loop().time() >= timeout_deadline: + break + except asyncio.CancelledError: + await self.clear_cancel(sid, rid) + await self.clear_cancel(sid, None) + result = { + "ok": False, + "session_id": sid, + "request_id": rid, + "final": "", + "page": {"url": "", "title": ""}, + "screenshot": None, + "error": "cancelled_by_frontend", + "attempt": attempt_idx + 1, + "failure_summary": None, + "progress_state": None, + } + return result + except Exception as exc: + attempt_idx += 1 + last_error = str(exc) or repr(exc) + if attempt_idx >= attempts: + break + if (not str(exc)) or self._is_retryable_transport_error(exc): + try: + await self._restart() + except Exception as restart_exc: + last_error = f"restart_failed: {restart_exc!r}" + break + + await self.clear_cancel(sid, rid) + await self.clear_cancel(sid, None) + page_url = str(last_failure_page.get("url", "")) if isinstance(last_failure_page, dict) else "" + page_title = str(last_failure_page.get("title", "")) if isinstance(last_failure_page, dict) else "" + failure_summary = self._build_failure_summary( + task=base_task, + error=last_error or "unknown execution error", + page_url=page_url, + page_title=page_title, + final=last_failure_final, + screenshot=last_failure_screenshot, + attempt=min(attempt_idx, max_attempts), + progress_state=self._progress_by_session.get(sid), + ) + self._failure_context_by_session[sid] = failure_summary + result = { + "ok": False, + "session_id": sid, + "request_id": rid, + "final": "", + "page": {"url": "", "title": ""}, + "screenshot": None, + "error": last_error or "unknown execution error", + "attempt": min(attempt_idx, max_attempts), + "failure_summary": failure_summary, + "progress_state": self._export_progress_state(sid), + } + return result + finally: + if current_task is not None: + self._unregister_inflight_task(sid, rid, current_task) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/__init__.py new file mode 100644 index 0000000000..cc66ce949f --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/__init__.py @@ -0,0 +1,12 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Surface packages, one per kind of screen; the built-in ones are registered by name with lazy import strings.""" + +from ..contract.registry import SURFACES, register_surface + +DEFAULT_SURFACE = "browser" + +register_surface(DEFAULT_SURFACE, f"{__name__}.browser.profile:create_browser_surface") + +__all__ = ["DEFAULT_SURFACE", "SURFACES"] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/__init__.py new file mode 100644 index 0000000000..74cb5947ac --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/__init__.py @@ -0,0 +1,4 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The browser surface: browser-owned commands and the browser context.""" diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_capabilities.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/capabilities.py similarity index 78% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/browser_capabilities.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/capabilities.py index 8126aee470..ccf32109dc 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_capabilities.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/capabilities.py @@ -4,14 +4,15 @@ from __future__ import annotations +from collections.abc import Mapping from dataclasses import dataclass from typing import Iterable +from openjiuwen.harness.tools.browser_move.contract import Action, Capability, DriverRef, Locator, PointRef +from openjiuwen.harness.tools.browser_move.surfaces.browser.tools import BROWSER_TOOL_SPECS CORE_BROWSER_CAPABILITY_NAME = "core" -# Keep this list explicit. Prefix-based matching could accidentally expose a -# newly introduced Playwright tool before its capability policy is reviewed. CORE_BROWSER_TOOL_NAMES: tuple[str, ...] = ( "browser_click", "browser_evaluate", @@ -26,6 +27,7 @@ "browser_select_option", "browser_snapshot", "browser_tabs", + "browser_take_screenshot", "browser_type", ) @@ -39,17 +41,9 @@ UNSAFE_DEV_BROWSER_TOOL_NAMES: tuple[str, ...] = ("browser_run_code_unsafe",) -# These are model-facing policy categories, not Playwright MCP --caps values. -POLICY_ONLY_BROWSER_CAPABILITY_NAMES: tuple[str, ...] = ( - "advanced_code", - "extended_interaction", - "unsafe_dev", -) - PDF_BROWSER_TOOL_NAMES: tuple[str, ...] = ("browser_pdf_save",) VISION_BROWSER_TOOL_NAMES: tuple[str, ...] = ( - "browser_take_screenshot", "browser_mouse_click_xy", "browser_mouse_down", "browser_mouse_drag_xy", @@ -114,13 +108,7 @@ ) -@dataclass(frozen=True) -class BrowserCapability: - """An approved browser capability exposed to the main agent.""" - - name: str - description: str - tool_names: tuple[str, ...] +BrowserCapability = Capability @dataclass(frozen=True) @@ -146,11 +134,13 @@ class ResolvedBrowserCapabilities: "compact probes, evaluate, snapshot, and batch are insufficient." ), tool_names=ADVANCED_CODE_BROWSER_TOOL_NAMES, + policy_only=True, ), BrowserCapability( name="extended_interaction", description="Close pages or perform uncommon drag-and-drop interactions.", tool_names=EXTENDED_INTERACTION_BROWSER_TOOL_NAMES, + policy_only=True, ), BrowserCapability( name="unsafe_dev", @@ -159,6 +149,7 @@ class ResolvedBrowserCapabilities: "development or debugging; this replaces advanced_code when both are requested." ), tool_names=UNSAFE_DEV_BROWSER_TOOL_NAMES, + policy_only=True, ), BrowserCapability( name="pdf", @@ -199,6 +190,10 @@ class ResolvedBrowserCapabilities: ), ) +POLICY_ONLY_BROWSER_CAPABILITY_NAMES: tuple[str, ...] = tuple( + entry.name for entry in DEFAULT_BROWSER_CAPABILITIES if entry.policy_only +) + def _stable_unique(values: Iterable[str]) -> tuple[str, ...]: """Normalize names and preserve their first-seen order.""" @@ -234,13 +229,7 @@ def resolve_browser_capabilities( requested_names: Iterable[str] | None, available_capabilities: Iterable[BrowserCapability] = DEFAULT_BROWSER_CAPABILITIES, ) -> ResolvedBrowserCapabilities: - """Validate a main-agent selection and expand it to allowed tool names. - - The main agent is responsible for selecting capabilities from their - descriptions. This resolver performs no task interpretation: it always adds - the core capability, rejects unknown names, and expands approved capability - names into a deterministic task-scoped tool allowlist. - """ + """Validate a capability selection and expand it, with ``core``, to the allowed tool names.""" catalog = _build_capability_index(available_capabilities) requested = _stable_unique(requested_names or ()) @@ -268,8 +257,57 @@ def resolve_browser_capabilities( ) +def capabilities_of( + tool_names: Iterable[str], available_capabilities: Iterable[BrowserCapability] = DEFAULT_BROWSER_CAPABILITIES +) -> tuple[str, ...]: + """The capabilities whose tools are all in ``tool_names`` (the inverse of ``resolve_browser_capabilities``).""" + allowed = set(tool_names) + return tuple(entry.name for entry in available_capabilities if set(entry.tool_names) <= allowed) + + +_SPECS = {spec.name: spec for spec in BROWSER_TOOL_SPECS} +_FIND = "browser_find" +_ELEMENT = (DriverRef, Locator) + +_REF_KINDS: Mapping[str, tuple[type | None, ...]] = { + "browser_click": _ELEMENT, + "browser_type": _ELEMENT, + "browser_hover": _ELEMENT, + "browser_select_option": _ELEMENT, + "browser_drag": _ELEMENT, + "browser_drop": _ELEMENT, + "browser_fill_form": _ELEMENT, + "browser_file_upload": (None, *_ELEMENT), + "browser_evaluate": (None, *_ELEMENT), + "browser_mouse_click_xy": (PointRef,), + "browser_mouse_move_xy": (PointRef,), + "browser_mouse_drag_xy": (PointRef,), +} + +CAPABILITY_TOOLS: Mapping[str, tuple[str, ...]] = { + capability.name: tuple(name for name in capability.tool_names if name in _SPECS or name == _FIND) + for capability in DEFAULT_BROWSER_CAPABILITIES +} + + +def _pairs(names: tuple[str, ...]) -> frozenset[tuple[type[Action], type | None]]: + return frozenset( + (command, kind) + for name in names + if name in _SPECS + for command in _SPECS[name].commands + for kind in _REF_KINDS.get(name, (None,)) + ) + + +CAPABILITY_CATALOG: Mapping[str, frozenset[tuple[type[Action], type | None]]] = { + entry: _pairs(names) for entry, names in CAPABILITY_TOOLS.items() +} + __all__ = [ "ADVANCED_CODE_BROWSER_TOOL_NAMES", + "CAPABILITY_CATALOG", + "CAPABILITY_TOOLS", "CONFIG_BROWSER_TOOL_NAMES", "CORE_BROWSER_CAPABILITY_NAME", "CORE_BROWSER_TOOL_NAMES", @@ -285,5 +323,6 @@ def resolve_browser_capabilities( "VISION_BROWSER_TOOL_NAMES", "BrowserCapability", "ResolvedBrowserCapabilities", + "capabilities_of", "resolve_browser_capabilities", ] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/commands.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/commands.py new file mode 100644 index 0000000000..91811748f4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/commands.py @@ -0,0 +1,152 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Browser-only commands. Drivers handle them with ``@handles`` exactly like the shared ones in ``contract``.""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import ClassVar + +from openjiuwen.harness.tools.browser_move.contract import Action, Effect, Ref + + +@dataclass(frozen=True) +class Navigate(Action): + effect: ClassVar[Effect] = Effect.NAVIGATE + + url: str + + +@dataclass(frozen=True) +class Back(Action): + effect: ClassVar[Effect] = Effect.NAVIGATE + + +@dataclass(frozen=True) +class OpenContext(Action): + """Open a new tab, at ``url`` when given.""" + + effect: ClassVar[Effect] = Effect.NAVIGATE + + url: str | None = None + + +@dataclass(frozen=True) +class SelectOption(Action): + """Select options of a ``<select>``; each value matches an option's value or label.""" + + effect: ClassVar[Effect] = Effect.ACT + + target: Ref + values: tuple[str, ...] + + +@dataclass(frozen=True) +class UploadFile(Action): + """Set files on ``target``, or answer an open file chooser (``target`` ``None``); empty ``paths`` cancels.""" + + effect: ClassVar[Effect] = Effect.ACT + + target: Ref | None + paths: tuple[str, ...] = () + + +def upload_paths(paths: tuple[str, ...], upload_root: Path | None) -> tuple[Path, ...]: + """The resolved absolute path of every entry; a relative name is a file under ``upload_root`` and stays in it.""" + root = Path(upload_root).expanduser().resolve() if upload_root is not None else None + resolved = [] + for raw in paths: + text = raw.strip() + if len(text) >= 2 and text[0] == text[-1] and text[0] in "'\"": + text = text[1:-1].strip() + path = Path(text).expanduser() + if path.is_absolute(): + resolved.append(path.resolve()) + continue + if root is None: + raise ValueError( + f"{raw!r} is a relative path and no upload folder is configured: give an absolute path, or set " + "BROWSER_UPLOAD_ROOT and name a file under it (list_upload_files)" + ) + candidate = (root / path).resolve() + if candidate != root and root not in candidate.parents: + raise ValueError( + f"{raw!r} resolves to {candidate}, outside the upload folder {root}: name a file under it " + "(list_upload_files)" + ) + resolved.append(candidate) + return tuple(resolved) + + +class FieldKind(str, Enum): + TEXTBOX = "textbox" + CHECKBOX = "checkbox" + RADIO = "radio" + COMBOBOX = "combobox" + SLIDER = "slider" + + +@dataclass(frozen=True) +class FormField: + """One field of ``FillForm``. ``value`` is ``"true"``/``"false"`` for a checkbox, the option text for a combobox.""" + + __sensitive_fields__: ClassVar[tuple[str, ...]] = ("value",) + + target: Ref + kind: FieldKind + value: str + sensitive: bool = False + + +@dataclass(frozen=True) +class FillForm(Action): + effect: ClassVar[Effect] = Effect.ACT + + fields: tuple[FormField, ...] + + +@dataclass(frozen=True) +class Drop(Action): + """Drop files (``paths``) and/or data (``(mime type, value)`` pairs) onto ``target``. At least one is set.""" + + effect: ClassVar[Effect] = Effect.ACT + + target: Ref + paths: tuple[str, ...] = () + data: tuple[tuple[str, str], ...] = () + + def __post_init__(self) -> None: + if not self.paths and not self.data: + raise ValueError("Drop needs paths or data") + + +@dataclass(frozen=True) +class GetPdf(Action): + """Save the page as PDF; ``ActionResult.value`` is the saved path.""" + + effect: ClassVar[Effect] = Effect.READ + + filename: str | None = None + + +@dataclass(frozen=True) +class GetConsole(Action): + """Console messages at ``level`` and more severe; ``all`` includes those before the last navigation.""" + + effect: ClassVar[Effect] = Effect.READ + + level: str = "info" + all: bool = False + + +@dataclass(frozen=True) +class GetNetwork(Action): + """Network requests; ``static`` includes successful static resources, ``filter`` is a URL regexp.""" + + effect: ClassVar[Effect] = Effect.READ + + static: bool = False + filter: str | None = None diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/config.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/config.py new file mode 100644 index 0000000000..8dfb8f9921 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/config.py @@ -0,0 +1,197 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Browser surface settings, parsed from the ``BROWSER_*`` and ``PLAYWRIGHT_*`` environment variables.""" + +from __future__ import annotations + +import json +import os +import warnings +from dataclasses import dataclass, field, replace +from pathlib import Path +from types import MappingProxyType +from typing import TYPE_CHECKING, Mapping + +from openjiuwen.core.foundation.tool import McpServerConfig + +from ...runtime.utils.env import ( + env_value, + first_value, + int_value, + is_truthy_env, + parse_command_args, +) + +if TYPE_CHECKING: + from .runner.settings import BrowserInstanceConfig + +ENV_PREFIXES = ("BROWSER_", "PLAYWRIGHT_") +PROVISIONINGS = ("remote", "managed", "extension") +DEFAULT_MCP_TIMEOUT_S = 600 +DEFAULT_PLAYWRIGHT_MCP_COMMAND = "npx" +DEFAULT_PLAYWRIGHT_MCP_ARGS = "-y @playwright/mcp@0.0.78" +DEFAULT_PROFILE_NAME = "jiuwenclaw" +_MCP_PASSTHROUGH_ENV = ("PLAYWRIGHT_BROWSERS_PATH", "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY") + + +def _provisioning(environ: Mapping[str, str]) -> str: + explicit = (environ.get("BROWSER_PROVISIONING") or "").strip().lower() + legacy = (environ.get("BROWSER_DRIVER") or "").strip().lower() + if legacy in PROVISIONINGS: + warnings.warn( + f"BROWSER_DRIVER={legacy} is deprecated: set BROWSER_PROVISIONING={legacy}. " + "BROWSER_DRIVER now selects the driver (playwright_mcp or browser_use).", + DeprecationWarning, + stacklevel=3, + ) + else: + legacy = "" + implied = "extension" if is_truthy_env(environ.get("PLAYWRIGHT_MCP_EXTENSION", "")) else "remote" + value = explicit or legacy or implied + if value not in PROVISIONINGS: + raise ValueError(f"BROWSER_PROVISIONING must be one of: {', '.join(PROVISIONINGS)} (got {explicit!r})") + return value + + +def _managed_port(environ: Mapping[str, str]) -> int: + raw = (environ.get("BROWSER_MANAGED_PORT") or "9333").strip() + try: + port = int(raw) + except ValueError: + port = 0 + if port <= 0: + raise ValueError(f"Invalid BROWSER_MANAGED_PORT: {raw}") + return port + + +def _mcp_env(environ: Mapping[str, str]) -> Mapping[str, str]: + env = {key: environ[key] for key in _MCP_PASSTHROUGH_ENV if environ.get(key)} + extra_json = (environ.get("PLAYWRIGHT_MCP_ENV_JSON") or "").strip() + if extra_json: + try: + extra = json.loads(extra_json) + except json.JSONDecodeError as exc: + raise ValueError(f"Invalid PLAYWRIGHT_MCP_ENV_JSON: {exc}") from exc + if isinstance(extra, dict): + env.update((str(k), str(v)) for k, v in extra.items()) + return MappingProxyType(env) + + +def _optional_path(raw: str) -> Path | None: + return Path(raw).expanduser() if raw else None + + +def _chrome_install_paths(environ: Mapping[str, str]) -> tuple[str, ...]: + if os.name != "nt": + return () + keys = ("LOCALAPPDATA", "ProgramFiles", "ProgramFiles(x86)", "ProgramW6432") + roots = tuple(env_value(environ, key) for key in keys) + return tuple(str(Path(root) / "Google" / "Chrome" / "Application" / "chrome.exe") for root in roots if root) + + +@dataclass(frozen=True) +class BrowserSettings: + """Where the browser comes from and how Playwright MCP and browser-use reach it.""" + + provisioning: str = "remote" + cdp_url: str = "" + cdp_headers: str = "" + cdp_timeout: str = "" + managed_host: str = "127.0.0.1" + managed_port: int = 9333 + managed_kill_existing: bool = False + managed_args: tuple[str, ...] = () + user_data_dir: str = "" + profile_name: str = DEFAULT_PROFILE_NAME + profile_store_path: Path | None = None + browser_binary: str = "" + browser_binary_candidates: tuple[str, ...] = () + local_app_data: str = "" + launch_args: tuple[str, ...] = tuple(parse_command_args(DEFAULT_PLAYWRIGHT_MCP_ARGS)) + mcp_command: str = DEFAULT_PLAYWRIGHT_MCP_COMMAND + mcp_timeout_s: int = DEFAULT_MCP_TIMEOUT_S + mcp_cwd: Path = field(default_factory=lambda: Path.cwd().resolve()) + mcp_browser: str = "" + mcp_device: str = "" + mcp_extension: bool = False + mcp_extension_token: str = "" + mcp_env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + mcp_server: McpServerConfig | None = None + upload_root: Path | None = None + workspace_root: Path | None = None + sidecar_python: Path | None = None + selector_cache_path: Path | None = None + process_env: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({}), repr=False, compare=False) + + @classmethod + def from_env(cls, environ: Mapping[str, str]) -> BrowserSettings: + mcp_cwd = first_value( + environ, + "PLAYWRIGHT_RUNTIME_MCP_CWD", + "BROWSER_RUNTIME_MCP_CWD", + "PLAYWRIGHT_RUNTIME_WORKDIR", + "BROWSER_RUNTIME_WORKDIR", + ) + upload_root = first_value(environ, "BROWSER_UPLOAD_ROOT") + sidecar_python = environ.get("BROWSER_USE_SIDECAR_PYTHON") + return cls( + provisioning=_provisioning(environ), + cdp_url=first_value(environ, "PLAYWRIGHT_MCP_CDP_ENDPOINT", "PLAYWRIGHT_CDP_URL"), + cdp_headers=first_value(environ, "PLAYWRIGHT_MCP_CDP_HEADERS", "PLAYWRIGHT_CDP_HEADERS"), + cdp_timeout=first_value(environ, "PLAYWRIGHT_MCP_CDP_TIMEOUT", "PLAYWRIGHT_CDP_TIMEOUT_MS"), + managed_host=first_value(environ, "BROWSER_MANAGED_HOST", default="127.0.0.1"), + managed_port=_managed_port(environ), + managed_kill_existing=is_truthy_env(environ.get("BROWSER_MANAGED_KILL_EXISTING", "")), + managed_args=tuple(parse_command_args(environ.get("BROWSER_MANAGED_ARGS") or "")), + user_data_dir=first_value(environ, "BROWSER_MANAGED_USER_DATA_DIR"), + profile_name=first_value(environ, "BROWSER_PROFILE_NAME", default=DEFAULT_PROFILE_NAME), + profile_store_path=_optional_path(first_value(environ, "BROWSER_PROFILE_STORE_PATH")), + browser_binary=first_value(environ, "BROWSER_MANAGED_BINARY"), + browser_binary_candidates=_chrome_install_paths(environ), + local_app_data=environ.get("LOCALAPPDATA") or "", + launch_args=tuple(parse_command_args(environ.get("PLAYWRIGHT_MCP_ARGS", DEFAULT_PLAYWRIGHT_MCP_ARGS))), + mcp_command=first_value(environ, "PLAYWRIGHT_MCP_COMMAND", default=DEFAULT_PLAYWRIGHT_MCP_COMMAND), + mcp_timeout_s=int_value( + environ, "PLAYWRIGHT_MCP_TIMEOUT_S", "BROWSER_TIMEOUT_S", default=DEFAULT_MCP_TIMEOUT_S, minimum=1 + ), + mcp_cwd=Path(mcp_cwd or Path.cwd()).expanduser().resolve(), + mcp_browser=first_value(environ, "PLAYWRIGHT_MCP_BROWSER"), + mcp_device=first_value(environ, "PLAYWRIGHT_MCP_DEVICE"), + mcp_extension=is_truthy_env(environ.get("PLAYWRIGHT_MCP_EXTENSION", "")), + mcp_extension_token=first_value(environ, "PLAYWRIGHT_MCP_EXTENSION_TOKEN"), + mcp_env=_mcp_env(environ), + upload_root=Path(upload_root).expanduser().resolve() if upload_root else None, + sidecar_python=Path(sidecar_python) if sidecar_python else None, + selector_cache_path=_optional_path((environ.get("OPENJIUWEN_BROWSER_SELECTOR_CACHE") or "").strip()), + process_env=MappingProxyType(dict(environ)), + ) + + @classmethod + def from_process_env(cls) -> BrowserSettings: + return cls.from_env(os.environ) + + def with_instance(self, instance: BrowserInstanceConfig) -> BrowserSettings: + """These settings for one isolated browser: the instance's non-empty fields win.""" + provisioning = instance.provisioning.strip().lower() or self.provisioning + if provisioning not in PROVISIONINGS: + raise ValueError( + f"BrowserInstanceConfig.provisioning must be one of: {', '.join(PROVISIONINGS)} (got {provisioning!r})" + ) + if instance.managed_port > 0: + managed_port = instance.managed_port + elif instance.key: + managed_port = 0 + else: + managed_port = self.managed_port + return replace( + self, + provisioning=provisioning, + cdp_url=instance.cdp_url.strip() or self.cdp_url, + managed_port=managed_port, + user_data_dir=instance.user_data_dir.strip() or self.user_data_dir, + profile_name=(instance.profile_name or instance.sanitized_key()).strip() or self.profile_name, + browser_binary=instance.browser_binary.strip() or self.browser_binary, + launch_args=tuple(parse_command_args(instance.launch_args)) + if instance.launch_args.strip() + else self.launch_args, + ) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/context.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/context.py new file mode 100644 index 0000000000..554f8e6ff4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/context.py @@ -0,0 +1,30 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The browser's typed surface context, carried in ``Observation.surface`` and ``ActionResult.surface``.""" + +from __future__ import annotations + +from dataclasses import dataclass + +from openjiuwen.harness.tools.browser_move.contract import ContextRef, SurfaceContext + + +@dataclass(frozen=True, kw_only=True) +class BrowserContext(SurfaceContext): + """Page url and title, tabs, PDF flag, scroll position, HTTP status and open dialog.""" + + url: str + title: str = "" + tabs: tuple[ContextRef, ...] = () + tab_urls: tuple[str, ...] = () + current_tab: int | None = None + is_pdf_viewer: bool = False + pixels_above: int = 0 + pixels_below: int = 0 + last_status: int | None = None + dialog: str | None = None + + @property + def identity(self) -> str: + return self.url diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/__init__.py new file mode 100644 index 0000000000..314700dc1b --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/__init__.py @@ -0,0 +1,38 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Browser helpers built on commands: the profile's extra tools, the custom actions and a decision policy's hooks.""" + +from .batch import BatchTool +from .custom import ( + CUSTOM_ACTIONS, + drag_and_drop, + get_element_coordinates, + register_custom_action_specs, + register_custom_actions, + set_input_files, +) +from .find import FindTool +from .page import activate_page, page_metadata +from .probes import ProbeCardsTool, ProbeInteractivesTool, probe_for_policy +from .recall import RecallOffloadTool + +EXTRA_TOOLS = (ProbeInteractivesTool, ProbeCardsTool, BatchTool, FindTool, RecallOffloadTool) + +__all__ = [ + "BatchTool", + "CUSTOM_ACTIONS", + "EXTRA_TOOLS", + "FindTool", + "ProbeCardsTool", + "ProbeInteractivesTool", + "RecallOffloadTool", + "activate_page", + "drag_and_drop", + "get_element_coordinates", + "page_metadata", + "probe_for_policy", + "register_custom_action_specs", + "register_custom_actions", + "set_input_files", +] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/batch.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/batch.py new file mode 100644 index 0000000000..7b7e105e0c --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/batch.py @@ -0,0 +1,1147 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""``browser_batch_interact``: the steps a model plans, each run as commands on the driver session.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +import re +import time +import uuid +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from typing import Any, AsyncIterator + +from openjiuwen.core.foundation.tool import Tool, ToolCard +from openjiuwen.core.foundation.tool.schema import ToolOutput +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + Click, + ConnectionLost, + DriverError, + DriverRef, + Effect, + Locator, + Observation, + Observe, + PressKey, + Ref, + RunScript, + SwitchContext, + TypeText, + Unsupported, + Wait, +) +from openjiuwen.harness.tools.browser_move.contract.masking import masked +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import CommandRecord, ToolContext, ToolFlags +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + FieldKind, + FillForm, + FormField, + Navigate, + SelectOption, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext +from openjiuwen.harness.tools.browser_move.surfaces.browser.helpers.batch_steps import ( + CONDITION_OPS, + DESCRIPTION, + MODEL_LOCATOR_KEYS, + MUTATING_OPS, + OPTION_VALUE_KEYS, + PARAMETERS, + PRIMARY_TARGET_OPS, + READ_OPS, + given, + op_of, + step_errors, + text_of, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.page_state import BrowserPageState, BrowserTarget +from openjiuwen.harness.tools.browser_move.surfaces.browser.policy import policy_denial, policy_rules + +_ACTION = "browser_batch_interact" +_OBSERVED_KEYS = ("url", "text", "value", "count", "stable", "tabs") +_EFFECT_ORDER = (Effect.READ, Effect.SCRIPT, Effect.ACT, Effect.NAVIGATE) +_ERROR_HEADING = "### Error\n" +_POLL_S = 0.1 +_TRANSIENT = re.compile(r"detached|not attached|intercept|not stable|outside of the viewport", re.IGNORECASE) +_MARK = "data-openjiuwen-target-id" +_NOTHING = object() + +_STAMP_JS = "(element, mark) => { element.setAttribute('data-openjiuwen-target-id', mark); return true; }" + +_LOCATE_JS = """(input) => { + const SKIPPED = new Set(['HTML', 'HEAD', 'SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'TITLE']); + const BUTTON_INPUTS = new Set(['button', 'submit', 'reset', 'image']); + const FIELD_TAGS = ['INPUT', 'TEXTAREA', 'SELECT']; + const INPUT_ROLES = {checkbox: 'checkbox', radio: 'radio', range: 'slider', number: 'spinbutton', search: 'searchbox', + hidden: '', file: ''}; + const TAG_ROLES = {BUTTON: 'button', TEXTAREA: 'textbox', OPTION: 'option', IMG: 'img', LI: 'listitem', UL: 'list', + OL: 'list', NAV: 'navigation', MAIN: 'main', FORM: 'form', TABLE: 'table', TR: 'row', TD: 'cell', + TH: 'columnheader', DIALOG: 'dialog', H1: 'heading', H2: 'heading', H3: 'heading', H4: 'heading', H5: 'heading', + H6: 'heading'}; + const norm = (value) => String(value ?? '').replace(/\\s+/g, ' ').trim(); + const matches = (actual, wanted, exact) => exact + ? norm(actual) === norm(wanted) + : norm(actual).toLowerCase().includes(norm(wanted).toLowerCase()); + const shown = (element) => { + const box = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return box.width > 0 && box.height > 0 && style.visibility !== 'hidden' && style.display !== 'none'; + }; + const label = (element) => { + const ids = element.getAttribute('aria-labelledby'); + if (ids) return ids.split(/\\s+/).map((id) => document.getElementById(id)?.textContent || '').join(' '); + return element.getAttribute('aria-label') + || Array.from(element.labels || []).map((item) => item.textContent).join(' '); + }; + const role = (element) => { + const explicit = (element.getAttribute('role') || '').trim().split(/\\s+/)[0]; + if (explicit) return explicit; + if (element.tagName === 'INPUT') { + if (BUTTON_INPUTS.has(element.type)) return 'button'; + if (element.type in INPUT_ROLES) return INPUT_ROLES[element.type]; + return element.hasAttribute('list') ? 'combobox' : 'textbox'; + } + if (element.tagName === 'A') return element.hasAttribute('href') ? 'link' : ''; + if (element.tagName === 'SELECT') return element.multiple || element.size > 1 ? 'listbox' : 'combobox'; + return TAG_ROLES[element.tagName] || ''; + }; + const text = (element) => element.tagName === 'INPUT' && BUTTON_INPUTS.has(element.type) + ? element.value + : element.innerText; + const name = (element) => norm(label(element)) || norm( + FIELD_TAGS.includes(element.tagName) && !BUTTON_INPUTS.has(element.type) + ? element.getAttribute('placeholder') || element.getAttribute('title') + : element.getAttribute('alt') || text(element) || element.getAttribute('title')); + const elements = () => Array.from(document.querySelectorAll('*')).filter((element) => !SKIPPED.has(element.tagName)); + const find = (query) => { + if (query.css) return Array.from(document.querySelectorAll(query.css)); + if (query.testid) return elements().filter((element) => element.getAttribute('data-testid') === query.testid); + if (query.placeholder) { + return elements().filter((element) => element.hasAttribute('placeholder') + && matches(element.getAttribute('placeholder'), query.placeholder, query.exact)); + } + if (query.label) { + return elements().filter((element) => element.tagName !== 'LABEL' + && matches(label(element), query.label, query.exact)); + } + if (query.role) { + return elements().filter((element) => role(element) === query.role && shown(element) + && !element.closest('[aria-hidden="true"]') + && (!query.name || matches(name(element), query.name, query.exact))); + } + const hits = elements().filter((element) => matches(text(element), query.text, query.exact)); + return hits.filter((element) => !hits.some((other) => other !== element && element.contains(other))); + }; + return input.queries.map((query) => { + let found; + try { found = find(query); } catch (error) { return {count: 0, error: String(error)}; } + const element = found[0]; + const visible = Boolean(element) && shown(element); + const enabled = Boolean(element) && !element.disabled && element.getAttribute('aria-disabled') !== 'true'; + const result = {count: found.length, visible, enabled}; + if (found.length !== 1 || !visible || (query.enabled && !enabled)) return result; + if (query.mark) element.setAttribute('data-openjiuwen-target-id', query.mark); + if (query.focus) element.focus(); + if (query.read === 'text') result.text = element.innerText; + if (query.read === 'value') { + if (FIELD_TAGS.includes(element.tagName)) result.value = element.value; + else result.error = 'Not an input, textarea or select element'; + } + if (query.option !== null && query.option !== undefined) { + const option = element.options ? element.options[query.option] : undefined; + if (option) result.value = option.value; + else result.error = 'no option at index ' + query.option; + } + return result; + }); +}""" + +_CONDITION_JS = """(input) => { + const norm = (value, limit) => String(value ?? '').replace(/\\s+/g, ' ').trim().slice(0, limit); + const shown = (element) => { + const box = element.getBoundingClientRect(); + const style = getComputedStyle(element); + return box.width > 0 && box.height > 0 && style.visibility !== 'hidden' && style.display !== 'none'; + }; + const nodes = input.selector ? Array.from(document.querySelectorAll(input.selector)) : []; + if (input.kind === 'selector') { + const visible = nodes.length > 0 && shown(nodes[0]); + const states = {attached: nodes.length > 0, detached: nodes.length === 0, visible, hidden: !visible}; + return {ok: Boolean(states[input.state]), value: input.state}; + } + if (input.kind === 'text') { + const wanted = norm(input.text); + if (!input.exact) { + const body = document.body ? norm(document.body.innerText) : ''; + return {ok: body.toLowerCase().includes(wanted.toLowerCase()), value: wanted}; + } + const found = Array.from(document.querySelectorAll('body *')) + .some((element) => norm(element.innerText) === wanted && shown(element)); + return {ok: found, value: wanted}; + } + if (input.kind === 'load') { + const state = document.readyState; + const ok = ['load', 'networkidle'].includes(input.state) ? state === 'complete' : state !== 'loading'; + return {ok, value: state}; + } + if (input.kind === 'first_card_title') { + if (!nodes.length) return {ok: false, value: ''}; + const title = norm(nodes[0].innerText, 300); + return {ok: Boolean(title) && (!input.expected || title.includes(input.expected)), value: title}; + } + if (input.kind === 'sort_state') { + if (nodes.length !== 1) return {ok: false, value: ''}; + const value = input.attribute === 'text' + ? norm(nodes[0].innerText, 300) + : String(nodes[0].getAttribute(input.attribute) || ''); + return {ok: value === input.expected, value}; + } + if (input.kind === 'result_count') { + const count = nodes.length; + const ok = input.count !== null + ? count === input.count + : (input.min === null || count >= input.min) && (input.max === null || count <= input.max); + return {ok, value: count}; + } + if (input.kind === 'dom_text_change') { + if (nodes.length !== 1) return {ok: false, value: ''}; + const value = norm(nodes[0].innerText, input.max_chars); + return {ok: value !== input.previous, value}; + } + if (input.selector && !nodes.length) return {ok: false, value: ''}; + const source = input.selector ? nodes[0] : document.body; + return {ok: true, value: norm(source ? source.innerText : '', input.max_chars)}; +}""" + +_REFRESH_JS = """(selector) => { + let nodes = []; + try { nodes = Array.from(document.querySelectorAll(selector)); } + catch (error) { return {ok: false, error: String(error)}; } + const element = nodes[0]; + if (!element) return {ok: true, match_count: nodes.length, visible: false}; + const style = getComputedStyle(element); + const rect = element.getBoundingClientRect(); + const visible = Boolean(element.isConnected && rect.width > 0 && rect.height > 0 && + style.display !== 'none' && style.visibility !== 'hidden'); + const enabled = !element.disabled && element.getAttribute('aria-disabled') !== 'true'; + const hit = visible ? document.elementFromPoint( + Math.max(0, Math.min(innerWidth - 1, rect.left + rect.width / 2)), + Math.max(0, Math.min(innerHeight - 1, rect.top + rect.height / 2)) + ) : null; + const actionable = Boolean(visible && enabled && hit && (hit === element || element.contains(hit))); + return {ok: true, match_count: nodes.length, visible, enabled, actionable}; +}""" + + +class _StepError(Exception): + def __init__(self, message: str, position: int = 0) -> None: + super().__init__(message) + self.position = position + + +@dataclass(frozen=True) +class _Aim: + query: Mapping[str, Any] | None = None + ref: Ref | None = None + target_id: str = "" + + +@dataclass(frozen=True) +class _Step: + index: int + op: str + spec: Mapping[str, Any] + primary: _Aim | None = None + option: _Aim | None = None + navigate_to: str = "" + + +@dataclass(frozen=True) +class _Timing: + action_ms: int + condition_ms: int + after_ms: int + total_ms: int + + @classmethod + def of(cls, inputs: Mapping[str, Any], steps: list[_Step]) -> _Timing: + action_ms = max(250, min(30_000, _int(inputs.get("timeout_ms"), 2500))) + condition_ms = max(action_ms, min(30_000, _int(inputs.get("condition_timeout_ms"), 10_000))) + after_ms = max(0, min(5000, _int(inputs.get("wait_after_each_ms"), 0))) + if inputs.get("global_timeout_ms") is None: + conditions = sum(step.op in CONDITION_OPS for step in steps) + computed = (len(steps) - conditions) * (action_ms + after_ms + 250) + computed += conditions * (condition_ms + after_ms + 250) + 5000 + total_ms = min(120_000, max(5000, computed)) + else: + total_ms = max(1000, min(180_000, _int(inputs.get("global_timeout_ms"), 120_000))) + return cls(action_ms, condition_ms, after_ms, total_ms) + + def step_s(self, step: _Step) -> float: + fallback = self.condition_ms if step.op in CONDITION_OPS else self.action_ms + return max(250, _int(step.spec.get("timeout_ms"), fallback)) / 1000 + + +class BatchTool(Tool): + """``browser_batch_interact``: one command per step, after the step targets were checked on the page.""" + + flags = ToolFlags( + effect=Effect.ACT, + requires=(Click, TypeText, PressKey, SelectOption, FillForm, Navigate, Wait, SwitchContext, RunScript), + ) + + def __init__(self, context: ToolContext) -> None: + super().__init__(ToolCard(name=_ACTION, description=DESCRIPTION, input_params=PARAMETERS)) + self._context = context + + async def invoke(self, inputs: Mapping[str, Any], **kwargs: Any) -> ToolOutput: + del kwargs + run = _Run(self._context, inputs or {}) + payload = await run.payload() + record = run.record(payload) + data = {"content": payload} if record is None else {"content": payload, "command": record} + return ToolOutput(success=bool(payload.get("ok")), data=data, error=payload.get("error")) + + async def stream(self, inputs: Mapping[str, Any], **kwargs: Any) -> AsyncIterator[ToolOutput]: + yield await self.invoke(inputs, **kwargs) + + +class _Run: + def __init__(self, context: ToolContext, inputs: Mapping[str, Any]) -> None: + self._context = context + self._inputs = inputs + self._page_state = context.page_state + self._last: Action | None = None + self._effect = Effect.READ + self._dispatched = False + self._context_changed = False + self._commands = 0 + + async def payload(self) -> dict[str, Any]: + page_state = self._page_state + steps = self._inputs.get("steps") + errors = step_errors(steps) + if errors: + return { + "ok": False, + "status": "failed", + "error": f"batch_validation_failed: {errors[0]}", + "validation_errors": errors, + "page_state": page_state.export(), + } + try: + specs, recovered = await self._refreshed(steps, str(self._inputs.get("generation_id") or "")) + planned = [self._planned(index, spec, single=len(specs) == 1) for index, spec in enumerate(specs)] + command = _single_command(planned[0]) if len(planned) == 1 else None + if command is None: + planned = [await self._materialized(step) for step in planned] + except (ValueError, DriverError, _StepError) as error: + return { + "ok": False, + "status": "failed", + "error": f"batch_target_validation_failed: {error}", + "generation_id": page_state.generation_id, + "current_generation": page_state.generation_id, + "candidate_fresh_targets": _candidates(page_state, steps), + "page_state": page_state.export(), + } + payload = await (self._single(planned[0], command) if command is not None else self._steps(planned)) + page_state.add_field_coverage(payload["extracted"].keys()) + payload["generation_id"] = page_state.generation_id + if recovered: + payload["generation_recovered_from"] = recovered + payload["page_state"] = page_state.export() + return payload + + def record(self, payload: Mapping[str, Any]) -> CommandRecord | None: + if self._last is None: + return None + return CommandRecord( + command=type(self._last), + effect=self._effect, + ok=bool(payload.get("ok")), + dispatched=self._dispatched, + context_changed=self._context_changed, + surface=self._page_state.surface_context, + value=payload, + generation=self._page_state.generation_id, + action=masked(self._last), + ) + + async def _execute(self, action: Action) -> ActionResult: + denial = policy_denial(action, self._page_state.surface_context, policy_rules(self._context.settings)) + if denial: + raise _StepError(denial) + self._last = action + try: + result = await self._context.execute(action) + except DriverError as error: + if error.dispatched: + self._ran(action) + raise + self._ran(action) + self._context_changed = self._context_changed or result.context_changed + if not result.ok: + raise _StepError(_message(result.detail)) + return result + + async def _observe(self, command: Observe) -> Observation: + self._last = command + try: + observation = await self._context.observe(command) + except DriverError as error: + if error.dispatched: + self._ran(command) + raise + self._ran(command) + return observation + + async def _tabs(self) -> BrowserContext: + surface = (await self._observe(Observe())).surface + if not isinstance(surface, BrowserContext): + raise _StepError("the observation reported no browser tabs") + return surface + + def _ran(self, action: Action) -> None: + effect = Effect.READ if isinstance(action, RunScript) else action.effect + self._effect = max(self._effect, effect, key=_EFFECT_ORDER.index) + self._dispatched = True + self._commands += 1 + + async def _script(self, code: str, args: Any) -> Any: + value = (await self._execute(RunScript(code=code, args=args))).value + if isinstance(value, str): + try: + return json.loads(value) + except ValueError: + return value + return value + + async def _refreshed(self, steps: list[Any], generation_id: str) -> tuple[list[dict[str, Any]], str]: + specs = [dict(step) for step in steps] + if generation_id.strip() == self._page_state.generation_id: + return specs, "" + refreshed_any = False + for spec in specs: + op = op_of(spec) + refreshed = False + for aliases in (("target_id",), ("option_target_id", "choose_target_id")): + alias = next((key for key in aliases if text_of(spec, key)), "") + if not alias: + continue + try: + spec[alias] = self._page_state.refresh_target_id(str(spec[alias])) + except ValueError: + spec[alias] = await self._rebound(str(spec[alias]), actionable=op in MUTATING_OPS) + refreshed = True + if any(given(spec, key) for key in ("ref", "option_ref", "choose_ref")): + raise ValueError( + "Native AX refs cannot be refreshed across PageState generations; " + "capture the current snapshot again" + ) + selected = any(given(spec, key) for key in ("selector", "option_selector", "choose_selector")) + if selected and not refreshed and op not in CONDITION_OPS: + raise ValueError( + "Model-authored selectors cannot refresh a stale PageState generation; " + "use a current generation-scoped target_id" + ) + refreshed_any = refreshed_any or refreshed + if not refreshed_any and not all(op_of(spec) in CONDITION_OPS for spec in specs): + raise ValueError( + f"Stale PageState generation {generation_id}; current generation is " + f"{self._page_state.generation_id}. Probe or snapshot the current page again." + ) + return specs, generation_id.strip() + + async def _rebound(self, target_id: str, *, actionable: bool) -> str: + page_state = self._page_state + stale = page_state.get_target(target_id) + if stale is None: + raise ValueError(f"Unknown PageState target_id: {target_id}") + if stale.source == "ax": + raise ValueError(f"Stale AX target_id {target_id} cannot cross generations; capture a current snapshot") + if stale.href and not stale.selector: + return page_state.rebind_target(target_id, visible=True, enabled=True, actionable=True) + selector = str(stale.selector or stale.locator.get("selector") or "").strip() + if not selector: + raise ValueError(f"Stale target_id {target_id} has no runtime-owned selector to refresh") + found = await self._script(_REFRESH_JS, selector) + valid = ( + isinstance(found, Mapping) + and found.get("ok") is True + and found.get("match_count") == 1 + and found.get("visible") is True + and (not actionable or found.get("actionable") is True) + ) + if not valid: + raise ValueError( + f"Stale target_id {target_id} could not be refreshed in {page_state.generation_id}: " + f"{found or 'validation unavailable'}" + ) + return page_state.rebind_target( + target_id, + visible=bool(found.get("visible")), + enabled=bool(found.get("enabled")), + actionable=bool(found.get("actionable")), + ) + + def _planned(self, index: int, spec: Mapping[str, Any], *, single: bool) -> _Step: + op = op_of(spec) + primary, navigate_to = self._primary(spec, op, single=single) + return _Step(index, op, spec, primary, self._option(spec, op), navigate_to) + + def _primary(self, spec: Mapping[str, Any], op: str, *, single: bool) -> tuple[_Aim | None, str]: + selector = text_of(spec, "selector") + target = self._target(op, text_of(spec, "target_id"), text_of(spec, "ref"), selector) + if target is not None: + if target.href and target.source in ("card", "card_primary_link") and op == "click": + if single: + return None, target.href + raise ValueError( + f"Target {target.target_id} has primary_link={target.href}; " + "call browser_navigate directly instead of clicking it in a batch" + ) + return _aim_at(_actionable(target, op)), "" + if selector: + return _css_aim(selector, self._page_state.generation_id), "" + query = _primary_query(spec) + return (_Aim(query) if query else None), "" + + def _option(self, spec: Mapping[str, Any], op: str) -> _Aim | None: + target_id = text_of(spec, "option_target_id") or text_of(spec, "choose_target_id") + ref = text_of(spec, "option_ref") or text_of(spec, "choose_ref") + selector = text_of(spec, "option_selector") or text_of(spec, "choose_selector") + target = self._target(op, target_id, ref, selector) + if target is not None: + return _aim_at(_actionable(target, op)) + if selector: + return _css_aim(selector, self._page_state.generation_id) + query = _option_query(spec) + return _Aim(query) if query else None + + def _target(self, op: str, target_id: str, ref: str, selector: str) -> BrowserTarget | None: + if not (target_id or ref or selector): + return None + return self._page_state.resolve_target( + generation_id=self._page_state.generation_id, + target_id=target_id, + ref=ref, + selector=selector, + require_registered_selector=op in MUTATING_OPS, + ) + + async def _materialized(self, step: _Step) -> _Step: + return _Step( + step.index, + step.op, + step.spec, + await self._marked(step.primary), + await self._marked(step.option), + step.navigate_to, + ) + + async def _marked(self, aim: _Aim | None) -> _Aim | None: + if aim is None or aim.query is not None or not isinstance(aim.ref, DriverRef): + return aim + try: + await self._execute(RunScript(code=_STAMP_JS, args=aim.target_id, target=aim.ref)) + except (DriverError, _StepError) as error: + raise ValueError(f"Failed to resolve AX ref {aim.ref.ref} for batch execution: {error}") from error + selector = f'[{_MARK}="{aim.target_id}"]' + self._page_state.update_target_locator(aim.target_id, {"selector": selector}) + return _Aim({"css": selector}, aim.ref, aim.target_id) + + async def _typed(self, action: TypeText) -> ActionResult: + try: + return await self._execute(action) + except Unsupported as error: + if error.dispatched or not action.slowly: + raise + return await self._execute(dataclasses.replace(action, slowly=False, clear=False)) + + async def _single(self, step: _Step, command: Action) -> dict[str, Any]: + started = time.perf_counter() + error = "" + try: + if isinstance(command, Observe): + await self._observe(command) + elif isinstance(command, TypeText): + await self._typed(command) + else: + await self._execute(command) + except (DriverError, _StepError) as failure: + error = str(failure) + elapsed_ms = _elapsed_ms(started) + ok = not error + result: dict[str, Any] = {"index": 0, "op": step.op, "ok": ok, "status": _status(ok), "elapsed_ms": elapsed_ms} + if error: + result["error"] = error + conditions = [] + if step.op == "wait_for_text": + conditions.append({**result, "observed": {"text": str(step.spec.get("text") or "")}}) + return { + "ok": ok, + "status": _status(ok), + "error": error or None, + "action": _ACTION, + "execution_mode": "primitive", + "generation_id": self._page_state.generation_id, + "steps": [result], + "extracted": {}, + "conditions": conditions, + "metrics": {"command": type(command).__name__, "executor_elapsed_ms": elapsed_ms}, + } + + async def _steps(self, steps: list[_Step]) -> dict[str, Any]: + timing = _Timing.of(self._inputs, steps) + generation_id = self._page_state.generation_id + started = time.perf_counter() + checked = [(step, aim) for step in steps for aim in _preflighted(step)] + try: + await self._ready( + [_wanted(aim, enabled=step.op not in READ_OPS) for step, aim in checked], + [time.monotonic() + timing.step_s(step) for step, _ in checked], + ) + except (DriverError, _StepError) as error: + step = checked[getattr(error, "position", 0)][0] + item = {"index": step.index, "op": step.op, "ok": False, "phase": "preflight", "error": str(error)} + item["elapsed_ms"] = _elapsed_ms(started) + failure = f"batch_preflight_failed: steps[{step.index}] {error}" + return self._summary([item], failure, generation_id, started, _elapsed_ms(started), len(checked)) + preflight_ms = _elapsed_ms(started) + initial_tabs = 0 + if any(step.op == "wait_for_tab" and not step.spec.get("min_tabs") for step in steps): + try: + initial_tabs = len((await self._tabs()).tabs) + except (DriverError, _StepError) as error: + return self._summary([], str(error), generation_id, started, preflight_ms, len(checked)) + items: list[dict[str, Any]] = [] + error = None + continue_on_error = bool(self._inputs.get("continue_on_error", False)) + for step in steps: + if _elapsed_ms(started) > timing.total_ms: + error = f"browser_batch_interact timed out after {timing.total_ms}ms" + break + item, failure = await self._item(step, timing, initial_tabs) + items.append(item) + if failure is None: + continue + if isinstance(failure, ConnectionLost) or not (step.spec.get("optional") or continue_on_error): + error = item["error"] + break + return self._summary(items, error, generation_id, started, preflight_ms, len(checked)) + + def _summary( + self, + items: list[dict[str, Any]], + error: str | None, + generation_id: str, + started: float, + preflight_ms: int, + preflight_count: int, + ) -> dict[str, Any]: + ok = error is None and all(item["ok"] for item in items) + status = "completed" if ok else "partial" if any(item["ok"] for item in items) else "failed" + done = [item for item in items if item["ok"] and item.get("field")] + return { + "ok": ok, + "status": status, + "error": None if ok else error or "one or more batch steps failed", + "action": _ACTION, + "execution_mode": "commands", + "session_id": str(self._inputs.get("session_id") or "").strip(), + "request_id": str(self._inputs.get("request_id") or "").strip(), + "generation_id": generation_id, + "steps": [_compact(item) for item in items], + "extracted": {item["field"]: item.get("text", item.get("value")) for item in done}, + "field_provenance": { + item["field"]: { + "selector": item["selector"][:600], + "raw_text": item["raw_text"][:1000], + "generation_id": generation_id, + } + for item in done + }, + "conditions": [_condition_entry(item) for item in items if item["op"] in CONDITION_OPS], + "metrics": { + "executor_elapsed_ms": _elapsed_ms(started), + "internal_steps_elapsed_ms": sum(item["elapsed_ms"] for item in items if "phase" not in item), + "preflight_elapsed_ms": preflight_ms, + "preflight_target_count": preflight_count, + "command_count": self._commands, + }, + } + + async def _item( + self, step: _Step, timing: _Timing, initial_tabs: int + ) -> tuple[dict[str, Any], DriverError | _StepError | None]: + item: dict[str, Any] = {"index": step.index, "op": step.op, "ok": False} + started = time.perf_counter() + failure: DriverError | _StepError | None = None + try: + await self._perform(step, timing.step_s(step), item, initial_tabs) + item["ok"] = True + after = step.spec.get("wait_after_ms") + pause_ms = max(0, _int(after, 0)) if after is not None else timing.after_ms + if pause_ms: + await asyncio.sleep(pause_ms / 1000) + except (DriverError, _StepError) as error: + failure = error + item["error"] = str(error) + item["elapsed_ms"] = _elapsed_ms(started) + return item, failure + + async def _perform(self, step: _Step, timeout_s: float, item: dict[str, Any], initial_tabs: int) -> None: + op, spec = step.op, step.spec + if op == "click": + await self._click(step.primary, timeout_s) + elif op in ("fill", "type"): + locator, _ = await self._locate(step.primary, timeout_s) + value = str(_first(spec, "value", "text_value") or "") + if op == "type" or spec.get("mode") == "type": + await self._type_slowly(locator, value) + else: + await self._execute(TypeText(locator, value)) + elif op == "autocomplete": + locator, _ = await self._locate(step.primary, timeout_s) + value = str(_first(spec, "value", "query", "text_value") or "") + if spec.get("fill_first"): + await self._execute(Click(locator)) + await self._execute(TypeText(locator, value)) + else: + await self._type_slowly(locator, value) + if spec.get("wait_after_type_ms") is not None: + await asyncio.sleep(max(0, _int(spec.get("wait_after_type_ms"), 0)) / 1000) + await self._click(step.option, timeout_s, missing=_NO_OPTION) + elif op == "select_visible_text": + await self._click(step.option, timeout_s, missing=_NO_OPTION) + elif op == "press": + if step.primary is not None: + await self._locate(step.primary, timeout_s, focus=True) + await self._execute(PressKey(str(spec.get("key") or "Enter"))) + elif op == "select_option": + await self._select(step, timeout_s) + elif op == "set_checked": + locator, _ = await self._locate(step.primary, timeout_s) + checked = "false" if spec.get("checked") is not None and not spec.get("checked") else "true" + await self._execute(FillForm((FormField(locator, FieldKind.CHECKBOX, checked),))) + elif op == "wait_for_url": + await self._wait_for_url(spec, timeout_s, item) + elif op == "wait_for_tab": + await self._wait_for_tab(spec, timeout_s, item, initial_tabs) + elif op in CONDITION_OPS: + await self._wait(op, spec, timeout_s, item) + elif op == "sleep": + await self._execute(Wait(seconds=max(0, _int(spec.get("ms") or spec.get("time_ms"), 0)) / 1000)) + elif op in READ_OPS: + read = "text" if op == "extract_text" else "value" + _, found = await self._locate(step.primary, timeout_s, enabled=False, read=read) + raw = str(found.get(read) or "") + if op == "extract_text": + item["text"] = _compact_text(raw, _int(spec.get("max_chars"), 500)) + else: + item["value"] = found.get(read) + item["raw_text"] = raw[:1000] + item["selector"] = str(spec.get("selector") or "") + item["field"] = str( + spec.get("field") or spec.get("name") or spec.get("description") or f"field_{step.index + 1}" + ) + else: + await self._observe(Observe(include_screenshot=True, full_page=bool(spec.get("full_page")))) + + async def _click(self, aim: _Aim | None, timeout_s: float, *, missing: str = "") -> None: + loop = asyncio.get_running_loop() + deadline = loop.time() + timeout_s + for attempt in range(2): + locator, _ = await self._locate(aim, max(0.25, deadline - loop.time()), missing=missing) + try: + await self._execute(Click(locator)) + return + except (DriverError, _StepError) as error: + if attempt or not _TRANSIENT.search(str(error)) or loop.time() >= deadline: + raise + await asyncio.sleep(min(_POLL_S, max(0.0, deadline - loop.time()))) + + async def _type_slowly(self, locator: Locator, value: str) -> None: + await self._execute(Click(locator)) + await self._execute(TypeText(locator, "")) + await self._typed(TypeText(locator, value, clear=False, slowly=True)) + + async def _select(self, step: _Step, timeout_s: float) -> None: + spec = step.spec + values = spec.get("values") + by_index = all(spec.get(key) is None for key in OPTION_VALUE_KEYS if key != "index") + locator, found = await self._locate( + step.primary, timeout_s, option=_int(spec.get("index"), 0) if by_index else None + ) + if values is not None: + chosen = tuple(str(value) for value in values) if isinstance(values, list) else (str(values),) + elif _first(spec, "option_value", "value") is not None: + chosen = (str(_first(spec, "option_value", "value")),) + elif _first(spec, "label_value", "option_label", "option_text", "choose_text") is not None: + chosen = (str(_first(spec, "label_value", "option_label", "option_text", "choose_text")),) + else: + chosen = (str(found.get("value")),) + await self._execute(SelectOption(locator, chosen)) + + async def _wait(self, op: str, spec: Mapping[str, Any], timeout_s: float, item: dict[str, Any]) -> None: + query, stable_ms, field = _condition_query(op, spec) + value = await self._poll(lambda: self._condition(query), timeout_s, _interval_s(spec), stable_ms / 1000) + if field: + item[field] = True if field == "stable" else value + + async def _condition(self, query: Mapping[str, Any]) -> tuple[bool, Any]: + found = await self._script(_CONDITION_JS, dict(query)) + if not isinstance(found, Mapping): + raise _StepError("the page script returned no condition result") + return bool(found.get("ok")), found.get("value") + + async def _wait_for_url(self, spec: Mapping[str, Any], timeout_s: float, item: dict[str, Any]) -> None: + wanted = _url_matcher(spec) + seen: dict[str, Any] = {} + + async def check() -> tuple[bool, Any]: + surface = seen["surface"] = await self._tabs() + for index, url in _tab_candidates(surface): + if wanted(url): + return True, (index, url) + return False, surface.url + + index, url = await self._poll(check, timeout_s, _interval_s(spec)) + surface = seen["surface"] + if index is not None and index != surface.current_tab: + await self._execute(SwitchContext(surface.tabs[index])) + item["tab_switched"] = True + item["url"] = url + + async def _wait_for_tab( + self, spec: Mapping[str, Any], timeout_s: float, item: dict[str, Any], initial_tabs: int + ) -> None: + min_tabs = max(1, _int(spec.get("min_tabs"), 0) or initial_tabs + 1) + wanted = _url_matcher(spec) + title = str(spec.get("title_contains") or "") + seen: dict[str, Any] = {} + + async def check() -> tuple[bool, Any]: + surface = seen["surface"] = await self._tabs() + tabs = [ + {"index": index, "url": _tab_url(surface, index), "title": tab.title or ""} + for index, tab in enumerate(surface.tabs) + ] + matches = [tab["index"] for tab in tabs if wanted(tab["url"]) and title in tab["title"]] + matched = len(tabs) >= min_tabs and bool(matches) + return matched, (tabs, matches[-1] if matched else -1) + + stable_s = _int(spec.get("stable_ms"), 0) / 1000 + tabs, index = await self._poll(check, timeout_s, _interval_s(spec), stable_s) + item["tabs"] = tabs + item["count"] = len(tabs) + if spec.get("activate") is not False and index >= 0: + await self._execute(SwitchContext(seen["surface"].tabs[index])) + item["url"] = tabs[index]["url"] + + async def _poll( + self, check: Callable[[], Awaitable[tuple[bool, Any]]], timeout_s: float, interval_s: float, stable_s: float = 0 + ) -> Any: + loop = asyncio.get_running_loop() + started = loop.time() + last: Any = _NOTHING + since = started + while loop.time() - started <= timeout_s: + ok, value = await check() + now = loop.time() + if ok and stable_s <= 0: + return value + if ok and value == last: + if now - since >= stable_s: + return value + elif ok: + last, since = value, now + else: + last = _NOTHING + await asyncio.sleep(interval_s) + raise _StepError(f"condition timed out after {round(timeout_s * 1000)}ms") + + async def _locate( + self, + aim: _Aim | None, + timeout_s: float, + *, + enabled: bool = True, + focus: bool = False, + read: str | None = None, + option: int | None = None, + missing: str = "", + ) -> tuple[Locator, Mapping[str, Any]]: + if aim is None or aim.query is None: + raise _StepError(missing or "step needs selector, role, label, placeholder, testid, or text") + mark = uuid.uuid4().hex[:12] + wanted = {**_wanted(aim, enabled=enabled), "mark": mark, "focus": focus, "read": read, "option": option} + found = (await self._ready([wanted], [time.monotonic() + timeout_s]))[0] + generation = self._page_state.generation_id + return Locator(css=f'[{_MARK}="{mark}"]', text=None, generation=generation), found + + async def _ready(self, queries: list[Mapping[str, Any]], deadlines: list[float]) -> list[Mapping[str, Any]]: + while queries: + found = await self._script(_LOCATE_JS, {"queries": queries}) + if not isinstance(found, list) or len(found) != len(queries): + raise _StepError("the page script returned no target result") + problems = [_problem(query, item) for query, item in zip(queries, found)] + for position, (problem, final) in enumerate(problems): + if problem and (final or time.monotonic() >= deadlines[position]): + raise _StepError(problem, position) + if not any(problem for problem, _ in problems): + return found + await asyncio.sleep(_POLL_S) + return [] + + +_NO_OPTION = "autocomplete/select_visible_text needs choose_text/option_text or option selector" + + +def _single_command(step: _Step) -> Action | None: + op, spec = step.op, step.spec + ref = step.primary.ref if step.primary is not None else None + if op == "click" and step.navigate_to: + return Navigate(step.navigate_to) + if op == "click" and ref is not None: + return Click(ref) + if op in ("fill", "type") and ref is not None: + slowly = op == "type" or _int(spec.get("delay_ms"), 0) > 0 + return TypeText(ref, str(spec.get("value") or ""), slowly=slowly) + if op == "select_option" and ref is not None: + values = spec.get("values") + if values is None: + values = spec.get("option_value", spec.get("value")) + if values is not None: + return SelectOption(ref, tuple(str(value) for value in (values if isinstance(values, list) else [values]))) + if op == "press" and not any(spec.get(key) for key in MODEL_LOCATOR_KEYS): + return PressKey(str(spec.get("key") or "Enter")) + if op == "wait_for_text": + return Wait(until_text=str(spec.get("text") or "")) + if op == "sleep": + return Wait(seconds=max(0, _int(spec.get("ms", spec.get("time_ms", 0)), 0)) / 1000) + if op == "screenshot": + return Observe(include_screenshot=True, full_page=bool(spec.get("full_page", False))) + return None + + +def _actionable(target: BrowserTarget, op: str) -> BrowserTarget: + available = target.visible and target.enabled and target.actionable + if op in MUTATING_OPS and target.source != "ax" and not available: + raise ValueError(f"PageState target {target.target_id} is not actionable in {target.generation_id}") + return target + + +def _aim_at(target: BrowserTarget) -> _Aim: + if target.ref: + return _Aim(None, DriverRef(target.ref, target.generation_id), target.target_id) + selector = target.selector or target.locator.get("selector", "") + if not selector: + raise ValueError(f"PageState target has no executable locator: {target.target_id}") + return _Aim({"css": selector}, Locator(css=selector, text=None, generation=target.generation_id), target.target_id) + + +def _css_aim(selector: str, generation: str) -> _Aim: + return _Aim({"css": selector}, Locator(css=selector, text=None, generation=generation)) + + +def _primary_query(spec: Mapping[str, Any]) -> dict[str, Any] | None: + exact = bool(spec.get("exact")) + if given(spec, "role"): + return {"role": str(spec["role"]), "name": str(spec.get("name") or ""), "exact": exact} + for key in ("label", "placeholder", "testid", "text"): + if given(spec, key): + return {key: str(spec[key]), "exact": exact} + return None + + +def _option_query(spec: Mapping[str, Any]) -> dict[str, Any] | None: + exact = bool(spec.get("exact")) + role = _first(spec, "option_role", "choose_role") + if role: + name = _first(spec, "option_name", "choose_name", "choose_text", "option_text") + return {"role": str(role), "name": "" if name is None else str(name), "exact": exact} + chosen = _first(spec, "choose_text", "option_text", "text_to_choose", "value") + return {"text": str(chosen), "exact": exact} if chosen not in (None, "") else None + + +def _preflighted(step: _Step) -> list[_Aim]: + aims = [] + if step.op in PRIMARY_TARGET_OPS or (step.op == "press" and step.primary is not None): + aims.append(step.primary) + if step.op == "select_visible_text" or (step.op == "autocomplete" and step.option and step.option.target_id): + aims.append(step.option) + return [aim for aim in aims if aim is not None and aim.query is not None] + + +def _wanted(aim: _Aim, *, enabled: bool) -> dict[str, Any]: + return {**(aim.query or {}), "enabled": enabled} + + +def _problem(query: Mapping[str, Any], found: Any) -> tuple[str, bool]: + if not isinstance(found, Mapping): + return "the page script returned no target result", True + if found.get("error"): + return str(found["error"]), True + count = int(found.get("count") or 0) + if count != 1: + return f"target must match exactly one element; matched {count}", count > 1 + if not found.get("visible"): + return "target is not visible", False + if query.get("enabled") and not found.get("enabled"): + return "target is not enabled", False + return "", False + + +def _condition_query(op: str, spec: Mapping[str, Any]) -> tuple[dict[str, Any], int, str]: + selector = str(spec.get("selector") or "") + stable_ms = _int(spec.get("stable_ms"), 0) + if op == "wait_for_selector": + return {"kind": "selector", "selector": selector, "state": str(spec.get("state") or "visible")}, 0, "" + if op == "wait_for_text": + return {"kind": "text", "text": str(spec.get("text")), "exact": bool(spec.get("exact"))}, 0, "" + if op == "wait_for_load_state": + return {"kind": "load", "state": str(spec.get("state") or "domcontentloaded")}, 0, "" + if op == "wait_for_first_card_title": + expected = str(_first(spec, "expected_text", "text") or "") + return {"kind": "first_card_title", "selector": selector, "expected": expected}, stable_ms, "text" + if op == "wait_for_sort_state": + query = { + "kind": "sort_state", + "selector": selector, + "attribute": str(spec.get("attribute") or "aria-sort"), + "expected": str(_first(spec, "expected_value", "value", "text") or ""), + } + return query, 0, "value" + if op == "wait_for_result_count": + bounds = {"count": "count", "min": "min_count", "max": "max_count"} + query = {"kind": "result_count", "selector": selector} + query.update({name: _number(spec.get(key)) for name, key in bounds.items()}) + return query, stable_ms, "count" + if op == "wait_for_dom_text_change": + query = { + "kind": "dom_text_change", + "selector": selector, + "previous": str(_first(spec, "previous_text", "initial_text") or ""), + "max_chars": _int(spec.get("max_chars"), 1000), + } + return query, stable_ms, "text" + max_chars = _int(spec.get("max_chars"), 1200) if selector else 1200 + return {"kind": "stable", "selector": selector, "max_chars": max_chars}, _int(spec.get("stable_ms"), 500), "stable" + + +def _url_matcher(spec: Mapping[str, Any]) -> Callable[[str], bool]: + exact = str(_first(spec, "url", "expected_url") or "") + contains = str(spec.get("url_contains") or "") + pattern = str(spec.get("url_pattern") or "") + try: + compiled = re.compile(pattern) if pattern else None + except re.error as error: + raise _StepError(f"invalid url_pattern: {error}") from error + + def matches(url: str) -> bool: + return ( + (not exact or url == exact) and contains in url and (compiled is None or compiled.search(url) is not None) + ) + + return matches + + +def _tab_candidates(surface: BrowserContext) -> list[tuple[int | None, str]]: + others = [(index, _tab_url(surface, index)) for index in range(len(surface.tabs)) if index != surface.current_tab] + return [(surface.current_tab, surface.url), *reversed(others)] + + +def _tab_url(surface: BrowserContext, index: int) -> str: + if index < len(surface.tab_urls): + return surface.tab_urls[index] + return surface.url if index == surface.current_tab else "" + + +def _candidates(page_state: BrowserPageState, steps: Any) -> list[dict[str, Any]]: + candidates: list[dict[str, Any]] = [] + seen: set[str] = set() + for step in steps if isinstance(steps, list) else []: + if not isinstance(step, Mapping): + continue + for key in ("target_id", "option_target_id", "choose_target_id"): + target_id = text_of(step, key) + if not target_id: + continue + for candidate in page_state.recovery_candidates(target_id): + candidate_id = str(candidate.get("target_id") or "") + if candidate_id and candidate_id not in seen: + candidates.append(candidate) + seen.add(candidate_id) + if len(candidates) >= 5: + return candidates + return candidates + + +def _compact(item: Mapping[str, Any]) -> dict[str, Any]: + compact = { + "index": item["index"], + "op": item["op"], + "ok": item["ok"], + "status": _status(item["ok"]), + "elapsed_ms": item["elapsed_ms"], + } + compact.update({key: item[key] for key in ("phase", "error") if item.get(key) not in (None, "")}) + return compact + + +def _condition_entry(item: Mapping[str, Any]) -> dict[str, Any]: + entry = _compact(item) + observed = {key: item[key] for key in _OBSERVED_KEYS if key in item} + if observed: + entry["observed"] = observed + return entry + + +def _status(ok: bool) -> str: + return "completed" if ok else "failed" + + +def _message(detail: str | None) -> str: + text = str(detail or "the command failed") + return text[len(_ERROR_HEADING) :] if text.startswith(_ERROR_HEADING) else text + + +def _first(step: Mapping[str, Any], *keys: str) -> Any: + return next((step[key] for key in keys if step.get(key) is not None), None) + + +def _int(value: Any, default: int) -> int: + try: + return int(value or default) + except (TypeError, ValueError): + return default + + +def _number(value: Any) -> float | None: + try: + return None if value is None else float(value) + except (TypeError, ValueError): + return None + + +def _interval_s(spec: Mapping[str, Any]) -> float: + return max(50, min(1000, _int(spec.get("poll_interval_ms"), 100))) / 1000 + + +def _compact_text(value: str, limit: int) -> str: + return " ".join(value.split())[:limit] + + +def _elapsed_ms(started: float) -> int: + return int(max(0.0, (time.perf_counter() - started) * 1000)) + + +__all__ = ["BatchTool"] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/batch_steps.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/batch_steps.py new file mode 100644 index 0000000000..e78dfb356b --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/batch_steps.py @@ -0,0 +1,533 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The ``browser_batch_interact`` card and the step contract checked before any step runs.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +DESCRIPTION = ( + "Execute deterministic browser interactions in one standalone runtime tool call. " + "Use after browser_probe_interactives/browser_probe_cards when a page-level flow has multiple " + "known targets, such as three or more form fields, click+type+choose autocomplete, dropdown or " + "date-picker selection, filter panels, search submit plus result wait, or compact extraction. " + "This is a first-class helper like the probe tools; do not route this through browser_custom_action. " + "Pass the current PageState generation_id and use target_id from probes for actions. " + "Only read-only extraction and explicit wait operations accept a validated selector. " + "Locator strategies are mutually exclusive. " + "A one-step call is rewritten by the runtime to an equivalent official browser primitive when " + "one exists, so it does not fail and require another model turn. Multi-step calls are preflighted " + "before side effects. Results use status=completed/partial/failed and contain only compact step " + "status, extracted fields, condition observations, metrics, and PageState." +) +PARAMETERS: dict[str, Any] = { + "type": "object", + "properties": { + "steps": { + "type": "array", + "description": ( + "Ordered browser steps to execute in one batch. Supported ops: click, fill, type, " + "autocomplete, select_visible_text, press, select_option, set_checked, " + "wait_for_selector, wait_for_text, wait_for_load_state, wait_for_url, " + "wait_for_first_card_title, wait_for_sort_state, wait_for_result_count, " + "wait_for_dom_text_change, wait_for_stable, wait_for_tab, extract_text, " + "extract_value, screenshot." + ), + "minItems": 1, + "maxItems": 25, + "items": { + "type": "object", + "properties": { + "op": { + "type": "string", + "description": "Step operation name.", + "enum": [ + "click", + "fill", + "type", + "autocomplete", + "select_visible_text", + "press", + "select_option", + "set_checked", + "wait_for_selector", + "wait_for_text", + "wait_for_load_state", + "wait_for_url", + "wait_for_first_card_title", + "wait_for_sort_state", + "wait_for_result_count", + "wait_for_dom_text_change", + "wait_for_stable", + "wait_for_tab", + "extract_text", + "extract_value", + "screenshot", + ], + }, + "target_id": { + "type": "string", + "description": ( + "Generation-scoped target_id returned by the current PageState/probe. " + "Preferred for actions and mutually exclusive with ref/selector/role/" + "label/placeholder/text/testid." + ), + }, + "ref": { + "type": "string", + "description": ( + "Native AX ref returned by browser_snapshot/browser_find in the current generation. " + "The runtime resolves it without model-generated CSS." + ), + }, + "selector": { + "type": "string", + "description": ( + "Current-generation selector_hint from a probe, or an explicit selector only " + "for a wait condition. " + "Do not combine with another locator strategy." + ), + }, + "role": { + "type": "string", + "description": "ARIA role to locate, e.g. button/textbox/link.", + }, + "name": { + "type": "string", + "description": "Accessible name for role-based locating.", + }, + "label": { + "type": "string", + "description": "Label text for labeled form controls.", + }, + "placeholder": { + "type": "string", + "description": "Placeholder text for form controls.", + }, + "testid": { + "type": "string", + "description": "Exact data-testid value for a target.", + }, + "text": { + "type": "string", + "description": "Visible text for locate/click/wait_for_text.", + }, + "value": { + "type": "string", + "description": "Value to fill/type/select, or autocomplete query.", + }, + "option_text": { + "type": "string", + "description": ( + "Visible option text for autocomplete/select_visible_text/native select label." + ), + }, + "choose_text": { + "type": "string", + "description": "Alias for option_text in autocomplete flows.", + }, + "option_selector": { + "type": "string", + "description": "CSS selector for an autocomplete/dropdown option to choose.", + }, + "choose_selector": { + "type": "string", + "description": "Alias for option_selector.", + }, + "option_role": { + "type": "string", + "description": ("ARIA role for an autocomplete/dropdown option, e.g. option/menuitem."), + }, + "choose_role": { + "type": "string", + "description": "Alias for option_role.", + }, + "option_name": { + "type": "string", + "description": "Accessible name for option_role.", + }, + "choose_name": { + "type": "string", + "description": "Alias for option_name.", + }, + "option_value": { + "type": "string", + "description": "Native select option value. Alias for value when op=select_option.", + }, + "values": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "MCP/Playwright-style native select values list. Also accepted for single-select fields." + ), + }, + "option_label": { + "type": "string", + "description": ("Native select visible label. Alias for option_text when op=select_option."), + }, + "label_value": { + "type": "string", + "description": "Native select visible label alias.", + }, + "index": { + "type": "integer", + "description": "Native select option index.", + }, + "key": { + "type": "string", + "description": "Keyboard key for press, e.g. Enter or Escape.", + }, + "checked": { + "type": "boolean", + "description": "Desired checked state for set_checked.", + }, + "state": { + "type": "string", + "description": ( + "Wait state for wait_for_selector/load_state, e.g. visible/attached/domcontentloaded." + ), + }, + "option_target_id": { + "type": "string", + "description": "PageState target_id for an autocomplete/dropdown option.", + }, + "option_ref": { + "type": "string", + "description": "Current-generation AX ref for an autocomplete/dropdown option.", + }, + "url": { + "type": "string", + "description": "Exact URL expected by wait_for_url.", + }, + "expected_url": { + "type": "string", + "description": "Alias for the exact URL expected by wait_for_url.", + }, + "url_contains": { + "type": "string", + "description": "URL substring expected by wait_for_url.", + }, + "url_pattern": { + "type": "string", + "description": "Regular expression expected to match the current URL.", + }, + "title_contains": { + "type": "string", + "description": "Title substring expected on a newly opened tab.", + }, + "min_tabs": { + "type": "integer", + "description": ( + "Minimum context tab count for wait_for_tab. By default the runtime waits " + "for one more tab than existed when the batch started." + ), + }, + "activate": { + "type": "boolean", + "description": ( + "For wait_for_tab, activate the matched new tab for subsequent steps. Default true." + ), + }, + "expected_text": { + "type": "string", + "description": "Expected first-card title text.", + }, + "attribute": { + "type": "string", + "description": "Attribute inspected by wait_for_sort_state; defaults to aria-sort.", + }, + "expected_value": { + "type": "string", + "description": "Expected attribute or text value for a wait condition.", + }, + "count": { + "type": "integer", + "description": "Exact result count expected.", + }, + "min_count": { + "type": "integer", + "description": "Minimum result count expected.", + }, + "max_count": { + "type": "integer", + "description": "Maximum result count expected.", + }, + "previous_text": { + "type": "string", + "description": "Previous DOM text used by wait_for_dom_text_change.", + }, + "poll_interval_ms": { + "type": "integer", + "description": ( + "Dynamic-condition polling interval, clamped to 50..1000ms; default 100ms. " + "Polling always shares the step's total timeout." + ), + }, + "stable_ms": { + "type": "integer", + "description": "How long a matched value must remain unchanged.", + }, + "field": { + "type": "string", + "description": "Structured output field name for extraction steps.", + }, + "exact": { + "type": "boolean", + "description": "Use exact matching for role/label/text locators.", + }, + "optional": { + "type": "boolean", + "description": "When true, failure records the step but continues.", + }, + "timeout_ms": { + "type": "integer", + "description": "Optional per-step timeout override.", + }, + "delay_ms": { + "type": "integer", + "description": "Optional typing delay in milliseconds.", + }, + "max_chars": { + "type": "integer", + "description": "Maximum characters for extract_text.", + }, + "path": { + "type": "string", + "description": "Screenshot path for op=screenshot.", + }, + "full_page": { + "type": "boolean", + "description": "When true, screenshot the full page.", + }, + "description": { + "type": "string", + "description": "Human-readable purpose for logs/debugging.", + }, + }, + "required": ["op"], + }, + }, + "timeout_ms": { + "type": "integer", + "description": ("Default per-step timeout in milliseconds. Default 2500, clamped to 250..30000."), + }, + "generation_id": { + "type": "string", + "pattern": "^g[0-9]+$", + "description": ( + "Current PageState generation_id returned by navigate, probe, snapshot, or the previous batch. " + "Targets from older generations are rejected before browser execution." + ), + }, + "condition_timeout_ms": { + "type": "integer", + "description": ( + "Default timeout for condition waits in milliseconds. " + "Default 10000, clamped to the action timeout..30000." + ), + }, + "continue_on_error": { + "type": "boolean", + "description": "When true, continue after failed steps and return per-step errors.", + }, + "global_timeout_ms": { + "type": "integer", + "description": ( + "Hard timeout for the whole batch. Default is computed from step count, capped at " + "120000; explicit values are capped at 180000." + ), + }, + "session_id": { + "type": "string", + "description": "Optional browser task session id.", + }, + "request_id": { + "type": "string", + "description": "Optional browser task request id.", + }, + }, + "required": ["steps", "generation_id"], +} + +_MAX_STEPS = 25 +CONDITION_OPS = frozenset( + { + "wait_for_selector", + "wait_for_text", + "wait_for_load_state", + "wait_for_url", + "wait_for_first_card_title", + "wait_for_sort_state", + "wait_for_result_count", + "wait_for_dom_text_change", + "wait_for_stable", + "wait_for_tab", + } +) +_SELECTOR_CONDITION_OPS = frozenset( + { + "wait_for_selector", + "wait_for_first_card_title", + "wait_for_sort_state", + "wait_for_result_count", + "wait_for_dom_text_change", + "wait_for_stable", + } +) +READ_OPS = frozenset({"extract_text", "extract_value"}) +MUTATING_OPS = frozenset( + {"click", "fill", "type", "autocomplete", "select_option", "set_checked", "select_visible_text"} +) +PRIMARY_TARGET_OPS = (MUTATING_OPS - {"select_visible_text"}) | READ_OPS +_OPTION_TARGET_OPS = frozenset({"autocomplete", "select_visible_text"}) +_SUPPORTED_OPS = PRIMARY_TARGET_OPS | _OPTION_TARGET_OPS | CONDITION_OPS | {"press", "sleep", "screenshot"} +_TEXT_IS_NO_LOCATOR = frozenset({"wait_for_text", "wait_for_sort_state", "wait_for_first_card_title"}) +_LOCATOR_KEYS = ("target_id", "ref", "selector", "role", "label", "placeholder", "testid") +MODEL_LOCATOR_KEYS = (*_LOCATOR_KEYS, "name", "text") +_OPTION_KEYS = ( + ("option_target_id", "choose_target_id"), + ("option_ref", "choose_ref"), + ("option_selector", "choose_selector"), + ("option_role", "choose_role"), + ("option_text", "choose_text", "text_to_choose"), +) +OPTION_VALUE_KEYS = ( + "value", + "values", + "option_value", + "option_text", + "option_label", + "label_value", + "choose_text", + "index", +) +_URL_KEYS = ("url", "expected_url", "url_contains", "url_pattern") + + +def step_errors(steps: Any) -> list[str]: + """Every way ``steps`` breaks the contract, in the order the model should read them; empty when it holds.""" + if not isinstance(steps, list) or not steps: + return ["steps is required and must be a non-empty list"] + if len(steps) > _MAX_STEPS: + return [f"browser_batch_interact accepts at most {_MAX_STEPS} steps"] + errors: list[str] = [] + for index, step in enumerate(steps): + if not isinstance(step, Mapping): + errors.append(f"steps[{index}] must be an object") + elif op_of(step) not in _SUPPORTED_OPS: + errors.append(f"steps[{index}].op is unsupported: {op_of(step) or '<missing>'}") + else: + errors.extend(_requirement_errors(index, op_of(step), step)) + errors.extend(_locator_errors(index, op_of(step), step)) + for index, step in enumerate(steps): + if isinstance(step, Mapping): + errors.extend(_contract_errors(index, op_of(step), step)) + return errors + + +def _requirement_errors(index: int, op: str, step: Mapping[str, Any]) -> list[str]: + where = f"steps[{index}]" + checks = ( + ( + op in ("fill", "type", "autocomplete") and "value" not in step, + f"{where} op={op} requires value" + ("; use value, not text" if "text" in step else ""), + ), + ( + op == "select_option" and all(step.get(key) is None for key in OPTION_VALUE_KEYS), + f"{where} select_option requires an option value", + ), + (op == "press" and not text_of(step, "key"), f"{where} press requires key"), + (op == "wait_for_selector" and not step.get("selector"), f"{where} wait_for_selector requires selector"), + (op == "wait_for_text" and not step.get("text"), f"{where} wait_for_text requires text"), + ( + op == "wait_for_url" and not any(step.get(key) for key in _URL_KEYS), + f"{where} wait_for_url requires a URL condition", + ), + ( + op == "wait_for_first_card_title" and not step.get("selector"), + f"{where} wait_for_first_card_title requires selector", + ), + ( + op == "wait_for_sort_state" + and not (step.get("selector") and _any_set(step, ("expected_value", "value", "text"))), + f"{where} wait_for_sort_state requires selector and expected value", + ), + ( + op == "wait_for_result_count" + and not (step.get("selector") and _any_set(step, ("count", "min_count", "max_count"))), + f"{where} wait_for_result_count requires selector and count", + ), + ( + op == "wait_for_dom_text_change" and not step.get("selector"), + f"{where} wait_for_dom_text_change requires selector", + ), + (op == "sleep" and _sleep_ms(step) < 0, f"{where} sleep requires non-negative ms"), + ) + return [message for failed, message in checks if failed] + + +def _locator_errors(index: int, op: str, step: Mapping[str, Any]) -> list[str]: + where = f"steps[{index}]" + strategies = [key for key in _LOCATOR_KEYS if given(step, key)] + if op not in _TEXT_IS_NO_LOCATOR and given(step, "text"): + strategies.append("text") + received = ", ".join(strategies) + options = [key for aliases in _OPTION_KEYS for key in aliases if given(step, key)] + errors = [] + if given(step, "name") and not given(step, "role"): + errors.append(f"{where}.name requires role") + if op in PRIMARY_TARGET_OPS and not strategies: + errors.append(f"{where} op={op} requires a target") + elif op in PRIMARY_TARGET_OPS and len(strategies) > 1: + errors.append(f"{where} op={op} requires exactly one locator strategy; received {received}") + if op == "press" and len(strategies) > 1: + errors.append(f"{where} op=press accepts at most one locator strategy; received {received}") + if op in _SELECTOR_CONDITION_OPS: + invalid = [key for key in strategies if key != "selector"] + if invalid: + errors.append(f"{where} op={op} accepts only selector; received {', '.join(invalid)}") + elif op not in PRIMARY_TARGET_OPS and op != "press" and strategies: + errors.append(f"{where} op={op} does not accept a primary locator; received {received}") + if op in _OPTION_TARGET_OPS and not options: + errors.append(f"{where} {op} requires an option target") + elif op in _OPTION_TARGET_OPS and len(options) > 1: + errors.append(f"{where} op={op} requires exactly one option locator strategy; received {', '.join(options)}") + if (step.get("option_name") or step.get("choose_name")) not in (None, "") and not ( + step.get("option_role") or step.get("choose_role") + ): + errors.append(f"{where} option_name requires option_role") + return errors + + +def _contract_errors(index: int, op: str, step: Mapping[str, Any]) -> list[str]: + errors = [] + if op in MUTATING_OPS and not text_of(step, "target_id"): + errors.append(f"steps[{index}] op={op} requires target_id from the current PageState") + if op in READ_OPS and len([key for key in ("target_id", "selector") if text_of(step, key)]) != 1: + errors.append(f"steps[{index}] op={op} requires exactly one of target_id or selector") + return errors + + +def op_of(step: Mapping[str, Any]) -> str: + return str(step.get("op") or "").strip().lower() + + +def text_of(step: Mapping[str, Any], key: str) -> str: + return str(step.get(key) or "").strip() + + +def given(step: Mapping[str, Any], key: str) -> bool: + return step.get(key) not in (None, "") + + +def _any_set(step: Mapping[str, Any], keys: tuple[str, ...]) -> bool: + return any(step.get(key) is not None for key in keys) + + +def _sleep_ms(step: Mapping[str, Any]) -> int: + try: + return int(step.get("ms", step.get("time_ms", 0))) + except (TypeError, ValueError): + return -1 diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/custom.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/custom.py new file mode 100644 index 0000000000..198dba1f52 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/custom.py @@ -0,0 +1,406 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The controller's browser custom actions as commands; each takes the tool context and the action's parameters.""" + +from __future__ import annotations + +import functools +import uuid +from collections.abc import Awaitable, Callable, Mapping +from pathlib import Path +from typing import Any + +from openjiuwen.harness.tools.browser_move.contract import ( + ActionResult, + Click, + Drag, + DriverError, + Locator, + PointRef, + Ref, + RunScript, +) +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import ToolContext +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import Navigate, UploadFile +from openjiuwen.harness.tools.browser_move.surfaces.browser.policy import policy_denial, policy_rules +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.actions import ActionController + +_MARK = "data-openjiuwen-target-id" +_ALIASES = "Aliases source/target and source_x/source_y/target_x/target_y are also supported." +_COORDINATES = ("coord_source_x", "coord_source_y", "coord_target_x", "coord_target_y") +_STRICT_HINT = ( + "Multiple file inputs matched. Use a more specific selector (e.g. an id like #file-upload) targeting the " + "visible input." +) + +_POINTS_JS = """(input) => { + const shown = (element) => { + const box = element.getBoundingClientRect(); + return box.width > 0 && box.height > 0; + }; + const byText = (query, role) => { + const term = String(query || '').trim().toLowerCase(); + if (!term) return null; + const all = Array.from(document.querySelectorAll('body *')); + const score = (element) => { + const text = String(element.textContent || '').trim().toLowerCase(); + return !text ? -1 : text === term ? 2 : text.includes(term) ? 1 : -1; + }; + for (const wanted of [2, 1]) { + for (const element of all.filter((candidate) => score(candidate) === wanted)) { + const candidates = role === 'target' && element.parentElement ? [element.parentElement, element] : [element]; + const found = candidates.find(shown); + if (found) return found; + } + } + return null; + }; + const find = (selector, role) => { + let element = null; + try { element = document.querySelector(selector); } catch (error) { element = null; } + if (element && shown(element)) return element; + const hasText = String(selector).match(/:has-text\\s*\\(\\s*['"]([^'"]*)['"]\\s*\\)/); + return byText(hasText ? hasText[1] : selector, role); + }; + const point = (selector, offset, role, mark) => { + const element = selector ? find(selector, role) : null; + if (!element) return null; + if (mark) element.setAttribute('data-openjiuwen-target-id', mark); + const box = element.getBoundingClientRect(); + if (offset) return {x: Math.trunc(box.x + offset.x), y: Math.trunc(box.y + offset.y)}; + return {x: Math.trunc(box.x + box.width / 2), y: Math.trunc(box.y + box.height / 2)}; + }; + return { + source: point(input.source, input.source_offset, 'source', input.source_mark), + target: point(input.target, input.target_offset, 'target', input.target_mark), + }; +}""" + + +async def get_element_coordinates(context: ToolContext, **params: Any) -> dict[str, Any]: + """The centre (or the offset point) of one or two elements found by selector or text, or the given points.""" + ends = _ends(params) + if not (ends["source"] or ends["points"]): + return { + "ok": False, + "error": ( + "Missing location inputs. Provide at least element_source (element_target is optional), or " + f"coord_source_x/coord_source_y/coord_target_x/coord_target_y. {_ALIASES}" + ), + } + try: + refused = await _navigate(context, params) + if refused: + return {"ok": False, "error": refused, **_ids(params)} + if not (ends["source"] or ends["target"]): + sx, sy, tx, ty = ends["points"] + return {"ok": True, "source": {"x": sx, "y": sy}, "target": {"x": tx, "y": ty}, "error": None} + found = await _points(context, ends) + except DriverError as error: + return _failure("browser_get_element_coordinates", error, params) + if not isinstance(found, Mapping): + return {"ok": False, "error": "Could not parse coordinate result JSON from the page script"} + source, target = found.get("source"), found.get("target") + if ends["source"] and source is None: + error = ( + "Failed to determine source coordinates from selector. Use the exact visible text " + '(e.g. "Learn more" not "More information") or a valid CSS selector.' + ) + return {"ok": False, "source": None, "target": None, "error": error} + if ends["target"] and target is None: + error = "Failed to determine target coordinates from selector" + return {"ok": False, "source": source, "target": None, "error": error} + return {"ok": True, "source": source, "target": target, "error": None} + + +async def drag_and_drop(context: ToolContext, **params: Any) -> dict[str, Any]: + """Drag one element onto another (found by selector or text), or from one point to another.""" + ends = _ends(params) + by_elements = bool(ends["source"] and ends["target"]) + if not (by_elements or ends["points"]): + return { + "ok": False, + "error": ( + "Missing drag inputs. Provide either element_source + element_target, or " + f"coord_source_x/coord_source_y/coord_target_x/coord_target_y. {_ALIASES}" + ), + } + shape = {"steps": _int(params.get("steps")), "delay_ms": _int(params.get("delay_ms"))} + try: + refused = await _navigate(context, params) + if refused: + return {"ok": False, "error": refused, **_ids(params)} + if by_elements: + marks = (uuid.uuid4().hex[:12], uuid.uuid4().hex[:12]) + found = await _points(context, ends, marks) + source = found.get("source") if isinstance(found, Mapping) else None + target = found.get("target") if isinstance(found, Mapping) else None + if source is None or target is None: + error = "Failed to determine source or target coordinates from selectors" + return {"ok": False, "error": error, "source": source, "target": target, **shape} + refs: tuple[Ref, Ref] = (_marked(context, marks[0]), _marked(context, marks[1])) + message = f"Dragged element '{ends['source']}' to '{ends['target']}'" + else: + sx, sy, tx, ty = ends["points"] + source, target = {"x": sx, "y": sy}, {"x": tx, "y": ty} + generation = context.page_state.screenshot_generation + refs = (PointRef(sx, sy, generation), PointRef(tx, ty, generation)) + message = f"Dragged from ({sx}, {sy}) to ({tx}, {ty})" + result = await context.execute(Drag(*refs)) + except DriverError as error: + return _failure("browser_drag_and_drop", error, params) + error = _error(result) + return { + "ok": result.ok, + "message": None if error else message, + "source": source, + "target": target, + **shape, + "error": error, + } + + +async def set_input_files(context: ToolContext, **params: Any) -> dict[str, Any]: + """Put files on a file input: directly where the driver sets files on an element, else through its chooser.""" + selector = str(params.get("selector") or "").strip() or 'input[type="file"]' + paths = [str(path) for path in params.get("paths") or [] if path] + if not paths: + return {"ok": False, "error": "paths is required and must be non-empty", **_ids(params)} + target = Locator(css=selector, text=None, generation=context.page_state.generation_id) + shape = {"selector": selector, "paths": paths} + denial = policy_denial(UploadFile(target, tuple(paths)), None, policy_rules(context.settings)) + if denial: + return {"ok": False, "error": denial, **shape} + try: + if Locator in context.driver_info.actions.get(UploadFile, frozenset()): + result = await context.execute(UploadFile(target, tuple(paths))) + else: + result = await context.execute(Click(target)) + if result.ok: + result = await context.execute(UploadFile(None, tuple(paths))) + except DriverError as error: + failure = {"ok": False, "error": str(error), **shape} + if "strict mode violation" in str(error): + failure["hint"] = _STRICT_HINT + return failure + return {"ok": result.ok, **shape, "error": _error(result)} + + +CUSTOM_ACTIONS: Mapping[str, Callable[..., Awaitable[dict[str, Any]]]] = { + "browser_get_element_coordinates": get_element_coordinates, + "browser_drag_and_drop": drag_and_drop, + "browser_set_input_files": set_input_files, +} + + +def register_custom_actions(controller: ActionController, context: ToolContext) -> None: + """Register the custom actions on the controller as handlers over the context, with their specs.""" + register_custom_action_specs(controller, upload_root=context.settings.upload_root) + for name, handler in CUSTOM_ACTIONS.items(): + controller.register_action(name, functools.partial(handler, context)) + + +def register_custom_action_specs(controller: ActionController, *, upload_root: Path | None) -> None: + """The three custom actions' specs and the list_upload_files action over the upload root.""" + + async def list_upload_files( + session_id: str = "", + request_id: str = "", + **kwargs: Any, + ) -> ActionResult: + del kwargs + if upload_root is None: + return { + "ok": False, + "error": ( + "BROWSER_UPLOAD_ROOT is not configured. " + "Set this env var to the directory where uploadable files are stored." + ), + "files": [], + "session_id": session_id, + "request_id": request_id, + } + if not upload_root.exists(): + return { + "ok": False, + "error": f"Upload root directory does not exist: {upload_root}", + "files": [], + "upload_root": str(upload_root), + "session_id": session_id, + "request_id": request_id, + } + files = _list_dir_files(upload_root) + return { + "ok": True, + "upload_root": str(upload_root), + "files": files, + "session_id": session_id, + "request_id": request_id, + } + + controller.register_action_spec( + "browser_get_element_coordinates", + summary="Resolves source/target screen coordinates by selectors or explicit coordinates.", + when_to_use=( + "Use when you need coordinates for one element (element_source only) or two (source + target). " + "element_target is optional." + ), + params={ + "url": "string: optional URL to open before resolving coordinates", + "element_source": "string: source selector/text alias (required for selector mode)", + "element_target": "string: target selector/text alias (optional)", + "coord_source_x": "int: source x coordinate", + "coord_source_y": "int: source y coordinate", + "coord_target_x": "int: target x coordinate", + "coord_target_y": "int: target y coordinate", + "source/target": "string aliases for element_source/element_target", + "source_x/source_y/target_x/target_y": "int aliases for coord_* fields", + }, + ) + controller.register_action_spec( + "browser_drag_and_drop", + summary="Performs drag-and-drop using selectors or explicit coordinates.", + when_to_use=("Use for drag-and-drop tasks instead of generic browser_run_task text-only instructions."), + params={ + "url": "string: optional URL to open before drag-and-drop", + "element_source": "string: source selector/text alias", + "element_target": "string: target selector/text alias", + "coord_source_x": "int: source x coordinate", + "coord_source_y": "int: source y coordinate", + "coord_target_x": "int: target x coordinate", + "coord_target_y": "int: target y coordinate", + "steps": "int: optional drag interpolation steps", + "delay_ms": "int: optional delay between drag steps", + "source/target": "string aliases for element_source/element_target", + "source_x/source_y/target_x/target_y": "int aliases for coord_* fields", + }, + ) + controller.register_action_spec( + "browser_set_input_files", + summary=( + "Sets files on an <input type='file'> element. Requires prior page inspection" + " to select the correct input — do not call this without first reading the page snapshot." + ), + when_to_use=( + "Use for all file upload tasks. Does NOT require a file chooser dialog — sets files directly " + "on the DOM element. Call list_upload_files first to get absolute paths, then call this action. " + "IMPORTANT: pages may have multiple file inputs (e.g. a visible input plus a hidden Dropzone input). " + "Always prefer a specific selector such as '#file-upload' or 'input#id' over the generic " + "'input[type=\"file\"]' to avoid strict mode violations. If you have not yet inspected the page, " + "delegate this action to browser_run_task so the worker can read the page snapshot first." + ), + params={ + "selector": ( + "string: CSS/Playwright selector targeting exactly one file input " + "(default: 'input[type=\"file\"]'). Use a specific ID selector like '#file-upload' " + "when the page has more than one file input element." + ), + "paths": ( + "list[string]: absolute file paths to set on the input (required, non-empty). " + "Parameter name is 'paths' — not 'files', not 'file_paths'." + ), + }, + ) + controller.register_action( + "list_upload_files", + list_upload_files, + overwrite=True, + ) + controller.register_action_spec( + "list_upload_files", + summary="Lists files available for upload from the configured BROWSER_UPLOAD_ROOT directory.", + when_to_use=( + "Call this to discover what files are available and get their exact absolute paths " + "before calling browser_set_input_files to attach them to a file input. " + "Returns a list of {name, path, size_bytes} entries." + ), + ) + + +async def _navigate(context: ToolContext, params: Mapping[str, Any]) -> str | None: + url = str(params.get("url") or "").strip() + return _error(await context.execute(Navigate(url))) if url else None + + +async def _points(context: ToolContext, ends: Mapping[str, Any], marks: tuple[str, str] = ("", "")) -> Any: + args = { + "source": ends["source"], + "target": ends["target"], + "source_offset": ends["source_offset"], + "target_offset": ends["target_offset"], + "source_mark": marks[0], + "target_mark": marks[1], + } + return (await context.execute(RunScript(code=_POINTS_JS, args=args))).value + + +def _marked(context: ToolContext, mark: str) -> Locator: + return Locator(css=f'[{_MARK}="{mark}"]', text=None, generation=context.page_state.generation_id) + + +def _ends(params: Mapping[str, Any]) -> dict[str, Any]: + points = [_int(params.get(name), params.get(name.removeprefix("coord_"))) for name in _COORDINATES] + return { + "source": str(params.get("element_source") or params.get("source") or "").strip(), + "target": str(params.get("element_target") or params.get("target") or "").strip(), + "source_offset": _offset(params.get("element_source_offset")), + "target_offset": _offset(params.get("element_target_offset")), + "points": tuple(points) if None not in points else None, + } + + +def _offset(value: Any) -> dict[str, int] | None: + if not isinstance(value, Mapping): + return None + x, y = _int(value.get("x")), _int(value.get("y")) + return None if x is None or y is None else {"x": x, "y": y} + + +def _int(value: Any, fallback: Any = None) -> int | None: + for candidate in (value, fallback): + try: + return int(candidate) + except (TypeError, ValueError): + continue + return None + + +def _error(result: ActionResult) -> str | None: + if result.ok: + return None + return str(result.detail or "the command failed").removeprefix("### Error\n") + + +def _failure(action: str, error: DriverError, params: Mapping[str, Any]) -> dict[str, Any]: + return {"ok": False, "error": f"{action} failed: {error}", **_ids(params)} + + +def _list_dir_files(root: Path) -> list[dict[str, Any]]: + """Return a flat list of files under *root* with name, path, and size.""" + entries: list[dict[str, Any]] = [] + try: + for item in sorted(root.iterdir()): + if item.is_file(): + try: + size = item.stat().st_size + except OSError: + size = -1 + entries.append({"name": item.name, "path": str(item), "size_bytes": size}) + except OSError: + pass + return entries + + +def _ids(params: Mapping[str, Any]) -> dict[str, str]: + return {"session_id": str(params.get("session_id") or ""), "request_id": str(params.get("request_id") or "")} + + +__all__ = [ + "CUSTOM_ACTIONS", + "drag_and_drop", + "get_element_coordinates", + "register_custom_action_specs", + "register_custom_actions", + "set_input_files", +] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/find.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/find.py new file mode 100644 index 0000000000..d11a33d9db --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/find.py @@ -0,0 +1,179 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""``browser_find``: text or regex search over the snapshot of the current observation.""" + +from __future__ import annotations + +import re +from collections.abc import Callable, Mapping +from typing import Any, AsyncIterator + +from openjiuwen.core.foundation.tool import Tool, ToolCard +from openjiuwen.core.foundation.tool.schema import ToolOutput +from openjiuwen.harness.tools.browser_move.contract import DriverError, Observe +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import CommandRecord, ToolContext, ToolFlags + +_FIND = "browser_find" +_FIND_DESC = ( + "Search the current accessibility / page snapshot for text or a regex. " + "Returns matching nodes with target_id/ref and short context. " + "Requires a prior browser_snapshot (or probe) for the current generation." +) +_FIND_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "Text or regex pattern to search for in the current snapshot.", + }, + "regex": { + "type": "boolean", + "description": "Treat query as a regular expression. Default false.", + }, + "limit": { + "type": "integer", + "description": "Maximum matches to return (1-100). Default 20.", + }, + "generation_id": { + "type": "string", + "description": "Optional PageState generation_id; defaults to current.", + }, + }, + "required": ["query"], +} +_ERROR = "### Error" +_CONTEXT_LINES = 3 +_FENCED_YAML = re.compile(r"```yaml\n(.*?)```", re.DOTALL) +_REGEX_LITERAL = re.compile(r"\A/(.*)/([a-z]*)\Z", re.DOTALL) +_REGEX_FLAGS = {"i": re.IGNORECASE, "m": re.MULTILINE, "s": re.DOTALL} + + +class FindTool(Tool): + """Searches the current observation like the server's own find, observing again when the last one is outdated.""" + + flags = ToolFlags() + + def __init__(self, context: ToolContext) -> None: + native = context.driver_info.native_tools.get(_FIND) + description, schema = (native.description, native.schema) if native is not None else (_FIND_DESC, _FIND_PARAMS) + name = context.driver_info.tool_name_prefix + _FIND + super().__init__(ToolCard(id=name, name=name, description=description, input_params=dict(schema))) + self._context = context + + async def invoke(self, inputs: Mapping[str, Any], **kwargs: Any) -> ToolOutput: + del kwargs + page_state = self._context.page_state + try: + inputs = inputs or {} + if inputs.get("generation_id"): + page_state.validate_generation(str(inputs["generation_id"])) + query, matches, limit = _search(inputs) + except ValueError as error: + return ToolOutput(success=False, error=f"{_ERROR}\n{error}") + observation = page_state.fresh_observation + observed = observation is None + if observation is None: + try: + observation = await self._context.observe(Observe()) + except DriverError as error: + text = f"{_ERROR}\n{error}" + record = CommandRecord.failed(Observe(), error, page_state, effect=self.flags.effect) + return ToolOutput(success=False, data={"content": text, "command": record}, error=text) + text = f"### Result\n{_answer(query, _find(_snapshot(observation.native_text or ''), matches, limit))}" + record = CommandRecord( + command=Observe, + effect=self.flags.effect, + ok=True, + dispatched=observed, + surface=page_state.surface_context, + value=text, + generation=page_state.generation_id, + action=Observe(), + ) + return ToolOutput(success=True, data={"content": text, "command": record}) + + async def stream(self, inputs: Mapping[str, Any], **kwargs: Any) -> AsyncIterator[ToolOutput]: + yield await self.invoke(inputs, **kwargs) + + +def _find(text: str, matches: Callable[[str], bool], limit: int | None) -> tuple[int, list[str]]: + lines = text.split("\n") + indents = [len(line) - len(line.lstrip()) for line in lines] + matched = [index for index, line in enumerate(lines) if matches(line)][:limit] + windows: list[list[int]] = [] + for line in matched: + start, end = max(0, line - _CONTEXT_LINES), min(len(lines) - 1, line + _CONTEXT_LINES) + if windows and start <= windows[-1][1] + 1: + windows[-1][1] = max(windows[-1][1], end) + else: + windows.append([start, end]) + on_path = set(matched) + for line in matched: + on_path.update(_ancestors(lines, indents, line)) + return len(matched), [_snippet(lines, indents, on_path, start, end) for start, end in windows] + + +def _snippet(lines: list[str], indents: list[int], on_path: set[int], start: int, end: int) -> str: + indices = [*_ancestors(lines, indents, start), *range(start, end + 1)] + out = [] + for position, index in enumerate(indices): + previous = indices[position - 1] + if position and index > previous + 1 and index not in on_path and previous not in on_path: + out.append(" " * indents[index] + "...") + out.append(lines[index]) + return "\n".join(out) + + +def _ancestors(lines: list[str], indents: list[int], index: int) -> list[int]: + found = [] + indent = indents[index] + for line in range(index - 1, -1, -1): + if indent <= 0: + break + if lines[line].strip() and indents[line] < indent: + found.append(line) + indent = indents[line] + return found[::-1] + + +def _search(args: Mapping[str, Any]) -> tuple[str, Callable[[str], bool], int | None]: + limit = None + if "query" in args: + limit = max(1, min(100, int(args.get("limit") or 20))) + query = str(args.get("query") or "") + text, pattern = ("", query) if args.get("regex") is True else (query, "") + else: + text, pattern = str(args.get("text") or ""), str(args.get("regex") or "") + if not text and not pattern: + raise ValueError('Provide either "text" or "regex" to search for.') + if text and pattern: + raise ValueError('Provide only one of "text" or "regex", not both.') + if text: + needle = text.lower() + return f'"{text}"', lambda line: needle in line.lower(), limit + compiled, shown = _compile(pattern) + return shown, lambda line: compiled.search(line) is not None, limit + + +def _compile(source: str) -> tuple[re.Pattern[str], str]: + literal = _REGEX_LITERAL.match(source) + pattern, flags = (literal.group(1), literal.group(2).replace("g", "")) if literal else (source, "") + try: + compiled = re.compile(pattern, sum(_REGEX_FLAGS.get(flag, 0) for flag in set(flags))) + except re.error as error: + raise ValueError(f"Invalid regular expression: {error}") from error + return compiled, f"/{pattern}/{''.join(sorted(set(flags)))}" + + +def _snapshot(text: str) -> str: + fenced = _FENCED_YAML.search(text) + return fenced.group(1).rstrip("\n") if fenced else text + + +def _answer(query: str, found: tuple[int, list[str]]) -> str: + count, snippets = found + if not count: + return f"No matches found for {query}." + word = "match" if count == 1 else "matches" + return f"Found {count} {word} for {query}:\n\n" + "\n\n----\n\n".join(snippets) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/page.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/page.py new file mode 100644 index 0000000000..5fe19905f6 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/page.py @@ -0,0 +1,89 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Page metadata from the last browser context plus one page script, and fronting a tab before a policy probes.""" + +from __future__ import annotations + +import re +from typing import TYPE_CHECKING, Any + +from openjiuwen.core.common.logging import tool_logger +from openjiuwen.harness.tools.browser_move.contract import ContextRef, DriverError, Observe, RunScript, SwitchContext +from openjiuwen.harness.tools.browser_move.runtime.utils.parsing import extract_json_object +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext +from openjiuwen.harness.tools.browser_move.surfaces.browser.probes import build_browser_state_metadata_js + +if TYPE_CHECKING: + from openjiuwen.harness.tools.browser_move.runtime.tool_factory import ToolContext + +_PAGE_METADATA = re.compile(r"const pageMetadata = await page\.evaluate\((?P<function>\(\) => \{.*?\n \})\);\n", re.S) + + +def _metadata_function() -> str: + match = _PAGE_METADATA.search(build_browser_state_metadata_js()) + if match is None: + raise ValueError("the page metadata script has no page function") + return match.group("function") + + +_METADATA_FUNCTION = _metadata_function() + + +async def page_metadata(context: ToolContext) -> tuple[dict[str, Any], str | None]: + """URL, title, tabs, page position and semantic state, with the capture error if any.""" + page_state = context.page_state + action = RunScript(code=_METADATA_FUNCTION) + try: + result = await context.execute(action) + except DriverError as error: + tool_logger.warning("page metadata capture failed: %s", error) + return {}, f"browser state metadata capture failed: {error}" + measured = extract_json_object(result.value) + if not result.ok or not measured: + return {}, "Could not parse browser state metadata result JSON" + surface = page_state.surface_context + metadata = { + "ok": True, + "url": page_state.url, + "title": page_state.title, + "tabs": _tabs(surface, page_state.current_context) if isinstance(surface, BrowserContext) else [], + "page_position": measured.get("page_position") or {}, + "semantic_state": measured.get("semantic_state") or {}, + } + return metadata, None + + +async def activate_page(context: ToolContext, url: str) -> bool: + """Focus the tab at ``url``, else the current one, so its timers run and its menus render.""" + page_state = context.page_state + try: + if page_state.current_context is None: + await context.observe(Observe()) + surface = page_state.surface_context + by_address = dict(zip(surface.tab_urls, surface.tabs)) if isinstance(surface, BrowserContext) else {} + wanted = by_address.get(str(url or "")) or page_state.current_context + if wanted is None: + return False + action = SwitchContext(wanted) + result = await context.execute(action) + except DriverError as error: + tool_logger.warning("activate_page failed: %s", error) + return False + return result.ok + + +def _tabs(surface: BrowserContext, current: ContextRef | None) -> list[dict[str, Any]]: + focused = surface.current_tab + if focused is None and current in surface.tabs: + focused = surface.tabs.index(current) + urls = surface.tab_urls + return [ + { + "index": index, + "current": index == focused, + "url": urls[index] if index < len(urls) else surface.url if index == focused else "", + "title": tab.title or "", + } + for index, tab in enumerate(surface.tabs) + ] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/probes.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/probes.py new file mode 100644 index 0000000000..01bf9cb4ca --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/probes.py @@ -0,0 +1,361 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Page probes run as page scripts on the driver session; their targets go into the page state.""" + +from __future__ import annotations + +import json +import re +from collections.abc import Mapping +from typing import Any, AsyncIterator + +from openjiuwen.core.common.logging import tool_logger +from openjiuwen.core.foundation.tool import Tool, ToolCard +from openjiuwen.core.foundation.tool.schema import ToolOutput +from openjiuwen.harness.tools.browser_move.contract import ActionResult, DriverError, RunScript +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import CommandRecord, ToolContext, ToolFlags +from openjiuwen.harness.tools.browser_move.runtime.utils.parsing import extract_json_object +from openjiuwen.harness.tools.browser_move.surfaces.browser.page_state import BrowserPageState +from openjiuwen.harness.tools.browser_move.surfaces.browser.probes import ( + build_card_probe_js, + build_interactive_probe_js, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.probes.semantics import normalize_card_probe_payload +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.logging import ( + write_browser_agent_audit_artifact, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.site_profiles import ( + get_selector_cache, + site_profiles_for_url, +) + +_PROBE_INTERACTIVES_DESC = ( + "Return a compact list of visible, high-value interactive elements on the current page. " + "Use this for page-level controls such as buttons, links, inputs, forms, navigation, login, " + "pagination, menus, calendar dates, sort tabs, rating filters, and visible actions. " + "Dynamic controls are returned as generation-scoped targets with region/kind semantics. " + "The optional query filter is alias-aware for common " + "search/input terms, including placeholders, aria labels, input type/name/id/class, and Chinese " + "search text such as 搜索/关键词. Prefer max_items around 20-30 unless a larger inventory " + "is needed. For product/search/listing card data, prefer browser_probe_cards first. " + "The result includes compact PageState and generation-scoped target_id values that can be " + "passed directly to browser_batch_interact without rebuilding CSS. It also includes " + "role/text/region/kind plus match_count/visible/enabled/actionable. The model-facing " + "result does not expose Probe-local ids or internal selectors." +) +_PROBE_INTERACTIVES_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "max_items": { + "type": "integer", + "description": "Maximum number of elements to return. Default 30, hard-capped at 40.", + }, + "viewport_only": { + "type": "boolean", + "description": "When true, only return elements currently visible in the viewport. Default true.", + }, + "query": { + "type": "string", + "description": "Optional text filter, e.g. 'cart', 'search', 'next', or 'login'.", + }, + }, + "required": [], +} +_PROBE_CARDS_DESC = ( + "Return compact repeated card/listing structures from the current page. " + "Use this first on product pages, marketplace pages, search-result pages, catalog pages, " + "article-list pages, table/list-row result pages, or any page with repeated visible cards/listings. " + "The result includes compact PageState with generation-scoped card/control target_id values, " + "candidate card title, author/source, summary/snippet, price, rating, " + "review count, availability, primary link, and visible controls, " + "region/kind/result_index/is_ad semantics that distinguish main results, accounts, sidebars, " + "hot searches, shops, chats, and product links, " + "match_count/visible/enabled/clickable/generation_id, recurring structure signatures, and " + "cache diagnostics. Navigate primary_link/href directly instead of clicking a card hint. " + "If this returns the fields needed " + "for the task, including article/search-result title/link/author/source/summary fields, " + "use the compact card result directly instead of taking screenshots/snapshots or running " + "broad DOM evaluation. Only evaluate again when a required field is missing." +) +_PROBE_CARDS_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "max_cards": { + "type": "integer", + "description": "Maximum number of cards to return. Default 12, hard-capped at 20.", + }, + "viewport_only": { + "type": "boolean", + "description": "When true, only inspect cards visible in the current viewport. Default true.", + }, + "include_buttons": { + "type": "boolean", + "description": "When true, include visible buttons/links inside each card. Default true.", + }, + "query": { + "type": "string", + "description": "Optional text filter, e.g. 'mouse', 'book', 'laptop', or 'cart'.", + }, + }, + "required": [], +} + +_WRAPPED_PROBE = re.compile( + r"\Aasync \(page\) => \{\n const params = (?P<params>.*?);\n\n" + r" return await page\.evaluate\((?P<function>\(params\) => \{.*\}), params\);\n\}\Z", + re.DOTALL, +) +_FALSE_WORDS = frozenset({"0", "false", "no"}) + + +def _page_script(wrapped: str) -> RunScript: + match = _WRAPPED_PROBE.match(wrapped) + if match is None: + raise ValueError("not a wrapped page probe") + return RunScript(code=match.group("function"), args=json.loads(match.group("params"))) + + +async def _run_probe( + context: ToolContext, action: RunScript, audit_kind: str +) -> tuple[dict[str, Any], dict[str, Any], int, ActionResult]: + for attempt in range(2): + result = await context.execute(action) + audit = write_browser_agent_audit_artifact(audit_kind, result.value) + if not result.ok: + error = str(result.detail or "the page script did not run")[:1_000] + return {"ok": False, "error": error}, audit, attempt, result + parsed = extract_json_object(result.value) + if parsed: + return parsed, audit, attempt, result + return {}, audit, 1, result + + +def _register_probe(page_state: BrowserPageState, payload: dict[str, Any]) -> None: + _follow_address(page_state, payload) + page_state.register_interactives(payload) + + +async def probe_for_policy(context: ToolContext, source: str, params: Mapping[str, Any]) -> dict[str, Any]: + """Run a decision policy's ``(params) => ...`` page function and register its elements as targets.""" + page_state = context.page_state + try: + action = RunScript(code=source, args=dict(params)) + parsed, _audit, _retries, _result = await _run_probe(context, action, "policy_probe") + except DriverError as error: + tool_logger.warning("policy probe failed: %s", error) + return _policy_failure(page_state, f"policy probe failed: {error}") + if not parsed or parsed.get("ok") is False: + return _policy_failure(page_state, str(parsed.get("error") or "policy probe returned no result")) + parsed.setdefault("ok", True) + parsed.setdefault("elements", []) + _register_probe(page_state, parsed) + parsed["generation_id"] = page_state.generation_id + return parsed + + +class ProbeInteractivesTool(Tool): + """``browser_probe_interactives`` as one page script.""" + + flags = ToolFlags(requires=(RunScript,)) + + def __init__(self, context: ToolContext) -> None: + super().__init__( + ToolCard( + name="browser_probe_interactives", + description=_PROBE_INTERACTIVES_DESC, + input_params=_PROBE_INTERACTIVES_PARAMS, + ) + ) + self._context = context + + async def invoke(self, inputs: Mapping[str, Any], **kwargs: Any) -> ToolOutput: + del kwargs + inputs = inputs or {} + page_state = self._context.page_state + query = str(inputs.get("query") or "").strip() + action = _page_script( + build_interactive_probe_js( + max_items=_count(inputs.get("max_items"), default=30, maximum=30), + viewport_only=_flag(inputs.get("viewport_only", True)), + query=query, + site_profiles=site_profiles_for_url(page_state.url), + generation_id=page_state.generation_id, + ) + ) + try: + parsed, audit, retries, result = await _run_probe(self._context, action, "interactive_probe") + except DriverError as error: + failure = _failure(f"browser_probe_interactives failed: {error}", "elements", page_state.export()) + return _output(failure, CommandRecord.failed(action, error, page_state, effect=self.flags.effect)) + record = CommandRecord.of(action, result, effect=self.flags.effect, value=result.value) + if not parsed or parsed.get("ok") is False: + reason = parsed.get("error") or "Could not parse browser_probe_interactives result JSON" + return _output({**_failure(reason, "elements", page_state.export_summary()), "audit": audit}, record) + parsed.setdefault("ok", True) + parsed.setdefault("error", None) + parsed.setdefault("elements", []) + _register_probe(page_state, parsed) + exported = page_state.export() + return _output( + { + "ok": bool(parsed.get("ok")), + "error": parsed.get("error"), + "url": parsed.get("url") or exported.get("url"), + "title": parsed.get("title") or exported.get("title"), + "generation_id": exported["generation_id"], + "count": len(exported["interactives"]), + "elements": exported["interactives"], + "page_state": page_state.export_summary(), + "diagnostics": { + "query": query[:160], + "query_widened": bool(parsed.get("query_widened")), + "total_candidates": int(parsed.get("total_candidates") or 0), + "parse_retry_count": retries, + }, + "audit": audit, + }, + record, + ) + + async def stream(self, inputs: Mapping[str, Any], **kwargs: Any) -> AsyncIterator[ToolOutput]: + yield await self.invoke(inputs, **kwargs) + + +class ProbeCardsTool(Tool): + """``browser_probe_cards`` as one page script; results feed the selector cache as before.""" + + flags = ToolFlags(requires=(RunScript,)) + + def __init__(self, context: ToolContext) -> None: + super().__init__( + ToolCard(name="browser_probe_cards", description=_PROBE_CARDS_DESC, input_params=_PROBE_CARDS_PARAMS) + ) + self._context = context + + async def invoke(self, inputs: Mapping[str, Any], **kwargs: Any) -> ToolOutput: + del kwargs + inputs = inputs or {} + page_state = self._context.page_state + selector_cache = get_selector_cache(self._context.settings.selector_cache_path) + action = _page_script( + build_card_probe_js( + max_cards=_count(inputs.get("max_cards"), default=12, maximum=12), + viewport_only=_flag(inputs.get("viewport_only", True)), + include_buttons=_flag(inputs.get("include_buttons", True)), + query=str(inputs.get("query") or "").strip(), + site_profiles=site_profiles_for_url(page_state.url), + selector_cache_records=selector_cache.export_for_probe(), + generation_id=page_state.generation_id, + ) + ) + try: + parsed, audit, retries, result = await _run_probe(self._context, action, "card_probe") + except DriverError as error: + failure = _failure(f"browser_probe_cards failed: {error}", "cards", page_state.export()) + return _output(failure, CommandRecord.failed(action, error, page_state, effect=self.flags.effect)) + record = CommandRecord.of(action, result, effect=self.flags.effect, value=result.value) + if not parsed or parsed.get("ok") is False: + reason = parsed.get("error") or "Could not parse browser_probe_cards result JSON" + return _output({**_failure(reason, "cards", page_state.export_summary()), "audit": audit}, record) + parsed.setdefault("ok", True) + parsed.setdefault("error", None) + parsed.setdefault("cards", []) + _follow_address(page_state, parsed) + normalize_card_probe_payload(parsed) + page_state.register_cards(parsed) + _register_primary_links(page_state, parsed) + parsed["page_state"] = page_state.export() + _remember_selectors(selector_cache, parsed) + exported = page_state.export() + return _output( + { + "ok": bool(parsed.get("ok")), + "error": parsed.get("error"), + "url": parsed.get("url") or exported.get("url"), + "title": parsed.get("title") or exported.get("title"), + "generation_id": exported["generation_id"], + "count": len(exported["cards"]), + "observed_count": int(parsed.get("observed_count") or 0), + "cards": exported["cards"], + "page_state": page_state.export_summary(), + "diagnostics": { + **(parsed.get("diagnostics") or parsed.get("cache_diagnostics") or {}), + "parse_retry_count": retries, + }, + "audit": audit, + }, + record, + ) + + async def stream(self, inputs: Mapping[str, Any], **kwargs: Any) -> AsyncIterator[ToolOutput]: + yield await self.invoke(inputs, **kwargs) + + +def _follow_address(page_state: BrowserPageState, payload: dict[str, Any]) -> None: + address = str(payload.get("url") or "").strip() + if address and page_state.url not in ("", address): + page_state.reset() + _stamp_generation(payload, page_state.generation_id) + + +def _stamp_generation(value: Any, generation_id: str) -> None: + if isinstance(value, dict): + if "selector_hint" in value or "generation_id" in value: + value["generation_id"] = generation_id + for nested in value.values(): + _stamp_generation(nested, generation_id) + elif isinstance(value, list): + for nested in value: + _stamp_generation(nested, generation_id) + + +def _register_primary_links(page_state: BrowserPageState, payload: dict[str, Any]) -> None: + for card in payload.get("cards") or (): + if not isinstance(card, dict): + continue + href = str(card.get("primary_link") or card.get("href") or "").strip() + if not href: + continue + for key in ("selector_hint", "primary_link_selector_hint"): + selector = str(card.get(key) or "").strip() + if selector: + page_state.selector_primary_links[selector] = (page_state.generation, href) + + +def _remember_selectors(selector_cache: Any, parsed: dict[str, Any]) -> None: + records = [selector_cache.record_card_probe_cache_rejection] + if parsed.get("cards"): + records.append(selector_cache.record_card_probe_result) + for record in records: + try: + record(parsed) + except Exception: # noqa: BLE001 + tool_logger.debug("card probe selector cache not updated", exc_info=True) + + +def _policy_failure(page_state: BrowserPageState, error: str) -> dict[str, Any]: + return {"ok": False, "error": error, "elements": [], "page_state": page_state.export()} + + +def _failure(error: Any, items: str, page_state: dict[str, Any]) -> dict[str, Any]: + return {"ok": False, "error": error, items: [], "page_state": page_state} + + +def _output(payload: dict[str, Any], record: CommandRecord) -> ToolOutput: + data = {"content": payload, "command": record} + return ToolOutput(success=bool(payload.get("ok", True)), data=data, error=payload.get("error")) + + +def _count(value: Any, *, default: int, maximum: int) -> int: + try: + number = int(value if value is not None else default) + except (TypeError, ValueError): + number = default + return max(1, min(maximum, number)) + + +def _flag(value: Any) -> bool: + if isinstance(value, str): + return value.strip().lower() not in _FALSE_WORDS + return bool(value) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/recall.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/recall.py new file mode 100644 index 0000000000..80285cd906 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/helpers/recall.py @@ -0,0 +1,18 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""``browser_recall_offload`` over the workspace and language the agent is built with.""" + +from __future__ import annotations + +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import ToolContext, ToolFlags +from openjiuwen.harness.tools.browser_move.surfaces.browser.offload_recall import BrowserOffloadRecallTool + + +class RecallOffloadTool(BrowserOffloadRecallTool): + """Reads back this session's offloaded browser results; never counts against the run's budgets.""" + + flags = ToolFlags(exempt_from_budget=True, worker=False) + + def __init__(self, context: ToolContext) -> None: + super().__init__(context.workspace, language=context.language) diff --git a/openjiuwen/harness/tools/browser_move/offload_recall.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/offload_recall.py similarity index 85% rename from openjiuwen/harness/tools/browser_move/offload_recall.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/offload_recall.py index 28efb1488b..fd860621f1 100644 --- a/openjiuwen/harness/tools/browser_move/offload_recall.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/offload_recall.py @@ -11,6 +11,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, AsyncIterator, Dict, Optional +from openjiuwen.core.context_engine.context.context_utils import ContextUtils from openjiuwen.core.foundation.tool import Tool, ToolCard from openjiuwen.harness.tools.base_tool import ToolOutput, render_fields @@ -19,7 +20,6 @@ _HANDLE_PATTERN = re.compile(r"^[0-9a-fA-F]{32}$") -_SESSION_PATTERN = re.compile(r"^[A-Za-z0-9_.-]+$") _OFFLOAD_FILE_PREFIX = "ToolResultWindowProcessor" _DEFAULT_LIMIT = 8000 _MAX_LIMIT = 16000 @@ -76,6 +76,14 @@ def _workspace_root(workspace: Optional[str | PathLike[str] | "Workspace"]) -> P return Path(root_path).resolve() +def _offload_session_dirs(session_id: str) -> list[str]: + session_dir = ContextUtils.offload_session_dir_name(session_id) + if session_dir == session_id or "/" in session_id or "\\" in session_id or session_id in {".", ".."}: + return [session_dir] + # Sessions stored before ids were made path-safe sit under the literal id (Linux and macOS). + return [session_dir, session_id] + + class BrowserOffloadRecallTool(Tool): """Read only this browser session's window-processor artifacts.""" @@ -128,11 +136,7 @@ def _validated_session_id(session: Any) -> str: if not callable(get_session_id): raise ValueError("browser offload recall requires the current runtime session") session_id = str(get_session_id() or "").strip() - if ( - not session_id - or not _SESSION_PATTERN.fullmatch(session_id) - or session_id in {".", ".."} - ): + if not session_id: raise ValueError("invalid browser runtime session") return session_id @@ -171,22 +175,21 @@ def _recall( ) def _load_message(self, session_id: str, handle: str) -> Dict[str, Any]: - offload_dir = ( - self._workspace_root - / "context" - / f"{session_id}_context" - / "offload" - ).resolve() - if not offload_dir.is_relative_to(self._workspace_root): - raise ValueError("browser offload directory escapes the configured workspace") - - candidate_path = offload_dir / f"{_OFFLOAD_FILE_PREFIX}_{handle}.json" - if candidate_path.is_symlink(): - raise ValueError("browser offload handle was not found in the current session") - offload_path = candidate_path.resolve() - if offload_path.parent != offload_dir: - raise ValueError("browser offload path is outside the current session") - if not offload_path.is_file(): + offload_path = None + for session_dir in _offload_session_dirs(session_id): + offload_dir = (self._workspace_root / "context" / f"{session_dir}_context" / "offload").resolve() + if not offload_dir.is_relative_to(self._workspace_root): + raise ValueError("browser offload directory escapes the configured workspace") + candidate_path = offload_dir / f"{_OFFLOAD_FILE_PREFIX}_{handle}.json" + if candidate_path.is_symlink(): + raise ValueError("browser offload handle was not found in the current session") + resolved = candidate_path.resolve() + if resolved.parent != offload_dir: + raise ValueError("browser offload path is outside the current session") + if resolved.is_file(): + offload_path = resolved + break + if offload_path is None: raise ValueError("browser offload handle was not found in the current session") if offload_path.stat().st_size > _MAX_OFFLOAD_FILE_SIZE: raise ValueError("browser offload artifact exceeds the safe recall size") @@ -226,7 +229,7 @@ def _build_fragment( } fragment_offset = max(0, match_offset - _QUERY_CONTEXT_CHARS) - fragment = content[fragment_offset:fragment_offset + limit] + fragment = content[fragment_offset : fragment_offset + limit] next_offset = fragment_offset + len(fragment) return { "handle": handle, @@ -244,18 +247,13 @@ def _build_fragment( } def render_for_llm(self, output: ToolOutput) -> str: - """Render the recalled fragment behind a one-line paging header. - - ``next_offset`` is how the model reads the following chunk, so it must - stay visible next to the fragment. - """ + """Render the recalled fragment behind a paging header that shows ``next_offset``.""" if not output.success: return super().render_for_llm(output) data = output.data if not data["found"]: return ( - f'No match for "{data["query"]}" in offloaded result {data["handle"]} ' - f"({data['original_size']} chars)." + f'No match for "{data["query"]}" in offloaded result {data["handle"]} ({data["original_size"]} chars).' ) header = render_fields( { diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/page_state.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/page_state.py similarity index 66% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/page_state.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/page_state.py index c99da6f67f..830fab70b2 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/page_state.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/page_state.py @@ -1,16 +1,17 @@ -#!/usr/bin/env python # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Compact browser page state and generation-scoped target registry.""" +"""The browser's page state: URL and title, probe and card targets, evidence fields and blockers.""" from __future__ import annotations import re -import uuid -from dataclasses import dataclass, field -from typing import Any, Dict, Iterable, Mapping, Optional +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any +from openjiuwen.harness.tools.browser_move.contract import Observation, SurfaceContext +from openjiuwen.harness.tools.browser_move.runtime.page_state import PageState, Target, compact_text _AX_REF_LINE_RE = re.compile( r"^\s*-\s+(?P<role>[A-Za-z][\w-]*)" @@ -73,59 +74,26 @@ "sort_state", } ) -_DEFAULT_CARD_FIELDS = frozenset( - {"title", "price", "rating", "source", "summary", "primary_link", "href"} -) +_DEFAULT_CARD_FIELDS = frozenset({"title", "price", "rating", "source", "summary", "primary_link", "href"}) _MAX_PUBLIC_INTERACTIVES = 20 _MAX_PUBLIC_CARDS = 8 -_MAX_TARGET_HISTORY = 1024 - - -def _compact_text(value: Any, limit: int = 120) -> str: - return " ".join(str(value or "").split())[:limit] - - -def _generation_number(generation_id: str) -> int: - normalized = str(generation_id or "").strip().lower() - if not re.fullmatch(r"g\d+", normalized): - raise ValueError("generation_id must use the PageState generation format, for example g3") - return int(normalized[1:]) @dataclass -class BrowserTarget: - """One model-visible target bound to a single page generation.""" - - target_id: str - page_id: str - generation: int - source: str - locator: Dict[str, str] = field(default_factory=dict) - ref: str = "" - selector: str = "" +class BrowserTarget(Target): + """One model-visible target bound to a single state generation.""" + href: str = "" - role: str = "" - name: str = "" - text: str = "" region: str = "" kind: str = "" - visible: bool = True - enabled: bool = True - actionable: bool = False - clickable: bool = False extractable: bool = False field_name: str = "" selected: bool = False selected_source: str = "" - @property - def generation_id(self) -> str: - """Return the target's generation in the public PageState format.""" - return f"g{self.generation}" - - def compact_index(self) -> Dict[str, Any]: + def compact_index(self) -> dict[str, Any]: """Return the small PageState index entry shown to the model.""" - result: Dict[str, Any] = { + result: dict[str, Any] = { "target_id": self.target_id, "generation_id": self.generation_id, } @@ -134,7 +102,7 @@ def compact_index(self) -> Dict[str, Any]: if self.role: result["role"] = self.role if self.text: - result["text"] = _compact_text(self.text, 80) + result["text"] = compact_text(self.text, 80) if self.region: result["region"] = self.region if self.kind: @@ -157,67 +125,53 @@ def compact_index(self) -> Dict[str, Any]: return result -class BrowserPageState: - """Own page identity, compact representation, and target generations.""" +class BrowserPageState(PageState): + """Own identity, compact representation, and target generations.""" + + target_type = BrowserTarget + id_prefix = "page" + observe_again = "Probe or snapshot the current page again." + snapshot_again = "Capture the current page snapshot again." def __init__( self, *, - page_id: Optional[str] = None, + page_id: str | None = None, generation: int = 0, + element_change_threshold: float = 0.5, ) -> None: - self.page_id = page_id or f"page-{uuid.uuid4().hex[:12]}" - self.generation = max(0, int(generation)) - self.url = "" + super().__init__(page_id=page_id, generation=generation, element_change_threshold=element_change_threshold) self.title = "" self.field_coverage: set[str] = set() self.requested_fields: set[str] = set() self.blockers: set[str] = set() - self.reference_generations: Dict[str, int] = {} - self.selector_generations: Dict[str, int] = {} - self.selector_primary_links: Dict[str, tuple[int, str]] = {} - self._targets: Dict[str, BrowserTarget] = {} - self._ref_targets: Dict[str, str] = {} - self._selector_targets: Dict[tuple[int, str], str] = {} - self._interactive_target_ids: list[str] = [] - self._cards: list[Dict[str, Any]] = [] - self._target_counter = 0 + self.selector_primary_links: dict[str, tuple[int, str]] = {} + self._cards: list[dict[str, Any]] = [] @property - def generation_id(self) -> str: - """Return the current generation in the public PageState format.""" - return f"g{self.generation}" + def url(self) -> str: + return self.identity + + @url.setter + def url(self, value: str) -> None: + self.identity = value def advance(self, *, url: str = "", title: str = "") -> None: """Start a new document generation while retaining stale-target history.""" - self.generation += 1 + super().advance() self.url = str(url or "").strip() - self.title = _compact_text(title, 300) - self.field_coverage.clear() - self.blockers.clear() - self._interactive_target_ids.clear() - self._cards.clear() - self._trim_target_history() + self.title = compact_text(title, 300) def observe(self, *, url: Any = "", title: Any = "") -> None: """Update non-empty document metadata without advancing generation.""" normalized_url = str(url or "").strip() - normalized_title = _compact_text(title, 300) + normalized_title = compact_text(title, 300) if normalized_url: self.url = normalized_url if normalized_title: self.title = normalized_title - def validate_generation(self, generation_id: str) -> None: - """Reject callers that are not using the current PageState generation.""" - requested = _generation_number(generation_id) - if requested != self.generation: - raise ValueError( - f"Stale PageState generation {generation_id}; current generation is " - f"{self.generation_id}. Probe or snapshot the current page again." - ) - - def register_interactives(self, payload: Dict[str, Any]) -> None: + def register_interactives(self, payload: dict[str, Any]) -> None: """Attach safe target IDs to a Probe result and refresh its compact index.""" self.observe(url=payload.get("url"), title=payload.get("title")) elements = payload.get("elements") @@ -234,7 +188,7 @@ def register_interactives(self, payload: Dict[str, Any]) -> None: self._interactive_target_ids.append(target.target_id) self._update_blockers(payload, elements) - def register_cards(self, payload: Dict[str, Any]) -> None: + def register_cards(self, payload: dict[str, Any]) -> None: """Attach target IDs and build compact card records for PageState.""" self.observe(url=payload.get("url"), title=payload.get("title")) cards = payload.get("cards") @@ -244,7 +198,7 @@ def register_cards(self, payload: Dict[str, Any]) -> None: for card in cards: if not isinstance(card, dict): continue - compact_card: Dict[str, Any] = {"generation_id": self.generation_id} + compact_card: dict[str, Any] = {"generation_id": self.generation_id} target = self._register_probe_target(card, source="card") if target is not None: card["target_id"] = target.target_id @@ -261,7 +215,7 @@ def register_cards(self, payload: Dict[str, Any]) -> None: card["field_targets"] = field_targets compact_card["field_targets"] = field_targets - compact_controls: list[Dict[str, Any]] = [] + compact_controls: list[dict[str, Any]] = [] for child_key in ("buttons", "controls", "actions"): children = card.get(child_key) if not isinstance(children, list): @@ -287,11 +241,11 @@ def register_cards(self, payload: Dict[str, Any]) -> None: self._cards.append(compact_card) self._update_blockers(payload, cards) - def _compact_card_fields(self, card: Mapping[str, Any]) -> Dict[str, Any]: + def _compact_card_fields(self, card: Mapping[str, Any]) -> dict[str, Any]: statuses_value = card.get("field_status") statuses = statuses_value if isinstance(statuses_value, Mapping) else {} visible_fields = _DEFAULT_CARD_FIELDS | self.requested_fields - compact: Dict[str, Any] = {} + compact: dict[str, Any] = {} for field_name in _CARD_SEMANTIC_FIELD_NAMES + CARD_EVIDENCE_FIELDS: if field_name in CARD_EVIDENCE_FIELDS and field_name not in visible_fields: continue @@ -301,7 +255,7 @@ def _compact_card_fields(self, card: Mapping[str, Any]) -> Dict[str, Any]: if field_name in CARD_EVIDENCE_FIELDS and statuses.get(field_name, "present") == "present": self.field_coverage.add(field_name) compact[field_name] = ( - field_value if isinstance(field_value, (bool, int, float)) else _compact_text(field_value) + field_value if isinstance(field_value, (bool, int, float)) else compact_text(field_value) ) missing_fields = sorted( field_name @@ -317,7 +271,7 @@ def _compact_card_fields(self, card: Mapping[str, Any]) -> Dict[str, Any]: compact["field_status"] = compact_statuses provenance_value = card.get("field_provenance") if isinstance(provenance_value, Mapping): - compact_provenance: Dict[str, Dict[str, Any]] = {} + compact_provenance: dict[str, dict[str, Any]] = {} for field_name in CARD_EVIDENCE_FIELDS: if field_name not in visible_fields: continue @@ -325,7 +279,7 @@ def _compact_card_fields(self, card: Mapping[str, Any]) -> Dict[str, Any]: if not isinstance(item, Mapping): continue projected = { - key: (_compact_text(value, 240) if isinstance(value, str) else value) + key: (compact_text(value, 240) if isinstance(value, str) else value) for key, value in item.items() if key in {"selector", "raw_text", "generation_id", "source"} and value not in (None, "", [], {}) } @@ -343,25 +297,11 @@ def register_ax_snapshot(self, value: Any) -> tuple[str, ...]: registered: list[str] = [] for match in _AX_REF_LINE_RE.finditer(text): ref_value = match.group("ref") - role = str(match.group("role") or "").strip() - name = str(match.group("name") or "").strip() - self.reference_generations[ref_value] = self.generation - existing_id = self._ref_targets.get(ref_value) - existing_target = self._targets.get(existing_id or "") - if existing_target is not None and existing_target.generation == self.generation: - target = existing_target - else: - target = self._new_target( - source="ax", - locator={"ref": ref_value}, - ref=ref_value, - role=role, - name=name, - text=name, - ) - self._ref_targets[ref_value] = target.target_id - if target.target_id not in self._interactive_target_ids: - self._interactive_target_ids.append(target.target_id) + self._register_ax_ref( + ref_value, + role=str(match.group("role") or "").strip(), + name=str(match.group("name") or "").strip(), + ) registered.append(ref_value) for ref_value in _ANY_REF_RE.findall(text): @@ -404,75 +344,11 @@ def set_requested_fields(self, fields: Iterable[str]) -> None: """Set task-scoped fields used to keep model-facing cards compact.""" self.requested_fields = { - str(field_name or "").strip() - for field_name in fields - if str(field_name or "").strip() + str(field_name or "").strip() for field_name in fields if str(field_name or "").strip() } - def resolve_target( - self, - *, - generation_id: str, - target_id: str = "", - ref: str = "", - selector: str = "", - require_registered_selector: bool = True, - ) -> Optional[BrowserTarget]: - """Resolve one target and reject every stale generation before execution.""" - self.validate_generation(generation_id) - normalized_target_id = str(target_id or "").strip() - normalized_ref = str(ref or "").strip() - normalized_selector = str(selector or "").strip() - - if normalized_target_id: - target = self._targets.get(normalized_target_id) - if target is None: - raise ValueError(f"Unknown PageState target_id: {normalized_target_id}") - self._validate_target_generation(target) - return target - - if normalized_ref: - ref_generation = self.reference_generations.get(normalized_ref) - if ref_generation is None: - raise ValueError(f"Unknown AX ref: {normalized_ref}. Capture the current page snapshot again.") - if ref_generation != self.generation: - raise ValueError( - f"Stale AX ref {normalized_ref} belongs to g{ref_generation}; " - f"current generation is {self.generation_id}." - ) - resolved_id = self._ref_targets.get(normalized_ref) - target = self._targets.get(resolved_id or "") - if target is None: - raise ValueError(f"AX ref is not registered as a target: {normalized_ref}") - self._validate_target_generation(target) - return target - - if normalized_selector: - selector_generation = self.selector_generations.get(normalized_selector) - if selector_generation is not None and selector_generation != self.generation: - raise ValueError( - f"Stale selector belongs to g{selector_generation}; current generation is " - f"{self.generation_id}: {normalized_selector}" - ) - resolved_id = self._selector_targets.get((self.generation, normalized_selector)) - target = self._targets.get(resolved_id or "") - if target is not None: - return target - if require_registered_selector: - raise ValueError( - "Unregistered action selector. Use target_id/ref from the current " - f"PageState instead of constructing CSS: {normalized_selector}" - ) - self.selector_generations[normalized_selector] = self.generation - return None - def refresh_target_id(self, target_id: str) -> str: - """Map a stale Probe target to one unique equivalent current target. - - Native AX refs intentionally remain generation-local. Probe targets may - be refreshed only from runtime-owned identity, never from model-authored - CSS reconstruction. - """ + """Map a stale probe target to its one equivalent current target by runtime-owned identity.""" normalized_target_id = str(target_id or "").strip() target = self._targets.get(normalized_target_id) @@ -500,10 +376,6 @@ def refresh_target_id(self, target_id: str) -> str: ) return candidates[0].target_id - def get_target(self, target_id: str) -> Optional[BrowserTarget]: - """Return an internal target, including stale history, for runtime recovery.""" - return self._targets.get(str(target_id or "").strip()) - def rebind_target( self, target_id: str, @@ -542,7 +414,7 @@ def rebind_target( self._selector_targets[(self.generation, target.selector)] = target.target_id return target.target_id - def recovery_candidates(self, target_id: str, *, limit: int = 5) -> list[Dict[str, Any]]: + def recovery_candidates(self, target_id: str, *, limit: int = 5) -> list[dict[str, Any]]: """Return compact current-generation candidates for a stale target error.""" stale = self.get_target(target_id) if stale is None: @@ -567,12 +439,12 @@ def _equivalent_current_targets( if href_matches: return href_matches - stale_name = _compact_text(stale.name or stale.text, 160).lower() + stale_name = compact_text(stale.name or stale.text, 160).lower() if not stale_name or not (stale.kind or stale.role): return [] matches: list[BrowserTarget] = [] for target in current_targets: - target_name = _compact_text(target.name or target.text, 160).lower() + target_name = compact_text(target.name or target.text, 160).lower() if target_name != stale_name: continue if stale.kind and target.kind != stale.kind: @@ -597,8 +469,8 @@ def update_target_locator(self, target_id: str, locator: Mapping[str, str]) -> N self.selector_generations[selector] = self.generation self._selector_targets[(self.generation, selector)] = target.target_id - def export(self) -> Dict[str, Any]: - """Return a bounded PageState index suitable for every browser result.""" + def export(self) -> dict[str, Any]: + """Return a bounded PageState index suitable for every result.""" interactives = self._export_targets( self._interactive_target_ids, _MAX_PUBLIC_INTERACTIVES, @@ -614,7 +486,7 @@ def export(self) -> Dict[str, Any]: "blockers": sorted(self.blockers), } - def export_summary(self) -> Dict[str, Any]: + def export_summary(self) -> dict[str, Any]: """Return PageState identity without duplicating cards or interactives.""" return { @@ -626,21 +498,38 @@ def export_summary(self) -> Dict[str, Any]: "blockers": sorted(self.blockers), } + def _note_surface(self, surface: SurfaceContext) -> None: + self.observe(title=getattr(surface, "title", "")) + self.contexts = tuple(getattr(surface, "tabs", ())) or self.contexts + + def _on_advance(self) -> None: + self.title = "" + self.field_coverage.clear() + self.blockers.clear() + self._cards.clear() + + def _on_observation(self, observation: Observation) -> None: + names = (element.name for element in observation.elements) + self._update_blockers({}, [observation.context.title or "", *names]) + + def _forget_target(self, target: Target) -> None: + primary_link = self.selector_primary_links.get(target.selector) if target.selector else None + if primary_link and primary_link[0] == target.generation: + self.selector_primary_links.pop(target.selector, None) + def _register_probe_target( self, - item: Dict[str, Any], + item: dict[str, Any], *, source: str, - ) -> Optional[BrowserTarget]: + ) -> BrowserTarget | None: selector = str(item.get("selector_hint") or "").strip() selector_validated = bool(item.get("selector_hint_validated")) match_count = item.get("match_count") if selector and (not selector_validated or match_count != 1): selector = "" - # Probe targets are executable only when the probe proved that its selector - # uniquely identifies the element. AX refs use their own native registry. - locator: Dict[str, str] = {"selector": selector} if selector else {} + locator: dict[str, str] = {"selector": selector} if selector else {} href = str(item.get("primary_link") or item.get("href") or "").strip() if not locator and not href: @@ -682,8 +571,8 @@ def _register_probe_target( def _register_primary_link_target( self, - card: Dict[str, Any], - ) -> Optional[BrowserTarget]: + card: dict[str, Any], + ) -> BrowserTarget | None: href = str(card.get("primary_link") or card.get("href") or "").strip() selector = str(card.get("primary_link_selector_hint") or "").strip() match_count = card.get("primary_link_match_count") @@ -713,14 +602,14 @@ def _register_primary_link_target( self.selector_primary_links[selector] = (self.generation, href) return target - def _register_card_field_targets(self, card: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + def _register_card_field_targets(self, card: dict[str, Any]) -> dict[str, dict[str, Any]]: statuses = card.get("field_status") field_status = statuses if isinstance(statuses, Mapping) else {} provenance = card.get("field_provenance") field_provenance = provenance if isinstance(provenance, Mapping) else {} selector_metadata = card.get("selector_metadata") selector_metadata = selector_metadata if isinstance(selector_metadata, Mapping) else {} - registered: Dict[str, Dict[str, Any]] = {} + registered: dict[str, dict[str, Any]] = {} for field_name in _READ_ONLY_CARD_TARGET_FIELDS: if field_name not in self.requested_fields: continue @@ -760,57 +649,6 @@ def _register_card_field_targets(self, card: Dict[str, Any]) -> Dict[str, Dict[s registered[field_name] = target.compact_index() return registered - def _new_target(self, *, source: str, locator: Dict[str, str], **kwargs: Any) -> BrowserTarget: - self._target_counter += 1 - target_id = f"t_g{self.generation}_{self._target_counter}" - target = BrowserTarget( - target_id=target_id, - page_id=self.page_id, - generation=self.generation, - source=source, - locator=dict(locator), - **kwargs, - ) - self._targets[target_id] = target - return target - - def _validate_target_generation(self, target: BrowserTarget) -> None: - if target.page_id != self.page_id or target.generation != self.generation: - raise ValueError( - f"Stale target_id {target.target_id} belongs to {target.generation_id}; " - f"current generation is {self.generation_id}." - ) - - def _remove_current_ax_ref(self, ref_value: str) -> None: - target_id = self._ref_targets.get(ref_value) - target = self._targets.get(target_id or "") - if target is not None and target.generation == self.generation and target.source == "ax": - self._targets.pop(target.target_id, None) - self._interactive_target_ids = [ - current_target_id - for current_target_id in self._interactive_target_ids - if current_target_id != target.target_id - ] - if self._ref_targets.get(ref_value) == target.target_id: - self._ref_targets.pop(ref_value, None) - if self.reference_generations.get(ref_value) == self.generation: - self.reference_generations.pop(ref_value, None) - - def _export_targets(self, target_ids: Iterable[str], limit: int) -> list[Dict[str, Any]]: - exported: list[Dict[str, Any]] = [] - seen: set[str] = set() - for target_id in target_ids: - if target_id in seen: - continue - seen.add(target_id) - target = self._targets.get(target_id) - if target is None or target.generation != self.generation: - continue - exported.append(target.compact_index()) - if len(exported) >= limit: - break - return exported - def _update_blockers(self, payload: Mapping[str, Any], values: Iterable[Any]) -> None: chunks = [ str(payload.get("error") or ""), @@ -829,30 +667,5 @@ def _update_blockers(self, payload: Mapping[str, Any], values: Iterable[Any]) -> if pattern.search(searchable): self.blockers.add(blocker) - def _trim_target_history(self) -> None: - overflow = len(self._targets) - _MAX_TARGET_HISTORY - if overflow <= 0: - return - stale_ids = [target_id for target_id, target in self._targets.items() if target.generation != self.generation][ - :overflow - ] - for target_id in stale_ids: - target = self._targets.pop(target_id, None) - if target is None: - continue - if target.ref and self._ref_targets.get(target.ref) == target_id: - self._ref_targets.pop(target.ref, None) - if self.reference_generations.get(target.ref) == target.generation: - self.reference_generations.pop(target.ref, None) - if target.selector: - selector_key = (target.generation, target.selector) - if self._selector_targets.get(selector_key) == target_id: - self._selector_targets.pop(selector_key, None) - if self.selector_generations.get(target.selector) == target.generation: - self.selector_generations.pop(target.selector, None) - primary_link = self.selector_primary_links.get(target.selector) - if primary_link and primary_link[0] == target.generation: - self.selector_primary_links.pop(target.selector, None) - - -__all__ = ["BrowserPageState", "BrowserTarget"] + +__all__ = ["CARD_EVIDENCE_FIELDS", "BrowserPageState", "BrowserTarget"] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/policy.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/policy.py new file mode 100644 index 0000000000..4382666fcc --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/policy.py @@ -0,0 +1,92 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The browser surface's policy rules.""" + +from __future__ import annotations + +import re +from collections.abc import Iterable +from pathlib import Path +from typing import Any + +from ...contract import Action, PolicyRule, PressKey, RunScript +from ...contract.surface import SurfaceContext +from .commands import Drop, UploadFile, upload_paths + +CLIPBOARD_DENIED = ( + "Blocked: the clipboard belongs to the host machine, not to this task, so pasting or reading it is not allowed. " + "Read the value from the page instead (its text or an input's value) and type it." +) +_PASTE_MODIFIERS = frozenset({"control", "ctrl", "meta", "cmd", "command", "controlormeta"}) +_CLIPBOARD_READ = re.compile( + r"clipboard['\"\]]*\s*(?:\.|\[\s*['\"])\s*read|execCommand\s*\(\s*['\"`]paste", re.IGNORECASE +) + + +def _is_paste_chord(key: str) -> bool: + *modifiers, final = [part.strip().lower() for part in key.split("+")] + held = set(modifiers) + return (final == "v" and bool(held & _PASTE_MODIFIERS)) or (final == "insert" and "shift" in held) + + +def _deny_paste_key(action: Action, context: SurfaceContext | None) -> str | None: + return CLIPBOARD_DENIED if isinstance(action, PressKey) and _is_paste_chord(action.key) else None + + +def _deny_clipboard_read(action: Action, context: SurfaceContext | None) -> str | None: + return CLIPBOARD_DENIED if isinstance(action, RunScript) and _CLIPBOARD_READ.search(action.code) else None + + +def _inside(path: Path, root: Path) -> bool: + return path == root or root in path.parents + + +def upload_rules(upload_root: Path | None, workspace_root: Path | None) -> tuple[PolicyRule, ...]: + """Deny upload and drop paths outside the upload root and the workspace; with neither configured, every path.""" + + def deny(action: Action, context: SurfaceContext | None) -> str | None: + paths = tuple(str(raw) for raw in getattr(action, "paths", ())) + if not paths: + return None + try: + resolved = upload_paths(paths, upload_root) + except ValueError as exc: + return f"Blocked: {exc}" + roots = tuple(Path(root).expanduser().resolve() for root in (upload_root, workspace_root) if root) + outside = [raw for raw, path in zip(paths, resolved) if not any(_inside(path, root) for root in roots)] + if not outside: + return None + allowed = ", ".join(str(root) for root in roots) or "none configured" + return ( + f"Blocked: {', '.join(repr(raw) for raw in outside)} is outside the folders files may be uploaded from " + f"({allowed}). Upload files from BROWSER_UPLOAD_ROOT (list_upload_files) or the workspace." + ) + + return (PolicyRule(UploadFile, deny), PolicyRule(Drop, deny)) + + +POLICY_RULES: tuple[PolicyRule, ...] = ( + PolicyRule(PressKey, _deny_paste_key), + PolicyRule(RunScript, _deny_clipboard_read), +) + + +def policy_rules(settings: Any) -> tuple[PolicyRule, ...]: + """The browser's rules for ``settings``: the fixed ones and the upload confinement to its folders.""" + return POLICY_RULES + upload_rules(settings.upload_root, settings.workspace_root) + + +def policy_denial( + action: Action, context: SurfaceContext | None, rules: Iterable[PolicyRule] = POLICY_RULES +) -> str | None: + """The first rule's deny reason for ``action``, for helpers that execute commands without the policy rail.""" + for rule in rules: + if isinstance(action, rule.command_type): + reason = rule.check(action, context) + if reason: + return reason + return None + + +__all__ = ["CLIPBOARD_DENIED", "POLICY_RULES", "policy_denial", "policy_rules", "upload_rules"] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/__init__.py new file mode 100644 index 0000000000..101fd89f1d --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/__init__.py @@ -0,0 +1,10 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The page scripts the browser helpers run: metadata, interactive and card probes.""" + +from .cards import build_card_probe_js +from .interactives import build_interactive_probe_js +from .metadata import build_browser_state_metadata_js + +__all__ = ["build_browser_state_metadata_js", "build_card_probe_js", "build_interactive_probe_js"] diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/probes.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/cards.js similarity index 67% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/probes.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/probes/cards.js index 02c92acf8f..a2bf8cdb50 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/probes.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/cards.js @@ -1,897 +1,3 @@ -# coding: utf-8 -"""Compact browser page probes for Playwright runtime.""" - -from __future__ import annotations - -import json -from typing import Any, Dict, List, Optional - - -def _clamp_int(value: Any, *, default: int, minimum: int, maximum: int) -> int: - try: - parsed = int(value) - except (TypeError, ValueError): - parsed = default - return max(minimum, min(maximum, parsed)) - - -def build_browser_state_metadata_js() -> str: - """Build Playwright code for fresh page, tab, and position state.""" - - return r""" -async (page) => { - const pages = page.context().pages(); - - const tabs = await Promise.all( - pages.map(async (tab, index) => { - let title = ''; - try { - title = await tab.title(); - } catch (_error) { - title = ''; - } - return { - index, - current: tab === page, - url: tab.url(), - title, - }; - }) - ); - - const pageMetadata = await page.evaluate(() => { - const root = document.documentElement; - const body = document.body; - const viewportWidth = Math.max(0, window.innerWidth || root?.clientWidth || 0); - const viewportHeight = Math.max(0, window.innerHeight || root?.clientHeight || 0); - const pageWidth = Math.max( - viewportWidth, - root?.scrollWidth || 0, - body?.scrollWidth || 0 - ); - const pageHeight = Math.max( - viewportHeight, - root?.scrollHeight || 0, - body?.scrollHeight || 0 - ); - const scrollX = Math.max(0, window.scrollX || window.pageXOffset || 0); - const scrollY = Math.max(0, window.scrollY || window.pageYOffset || 0); - - const pagePosition = { - viewport_width: viewportWidth, - viewport_height: viewportHeight, - page_width: pageWidth, - page_height: pageHeight, - scroll_x: scrollX, - scroll_y: scrollY, - pixels_above: scrollY, - pixels_below: Math.max(0, pageHeight - scrollY - viewportHeight), - pixels_left: scrollX, - pixels_right: Math.max(0, pageWidth - scrollX - viewportWidth), - }; - - const compact = (value, limit = 160) => String(value || '') - .replace(/\s+/g, ' ') - .trim() - .slice(0, limit); - const visible = (element) => { - if (!element || !element.isConnected) return false; - const rect = element.getBoundingClientRect(); - if (rect.width < 1 || rect.height < 1) return false; - const style = window.getComputedStyle(element); - return style.display !== 'none' && style.visibility !== 'hidden' && Number(style.opacity) !== 0; - }; - const elementKey = (element, index) => compact( - element.getAttribute('name') || - element.getAttribute('id') || - element.getAttribute('aria-label') || - element.getAttribute('placeholder') || - compact(element.innerText || element.textContent || '', 80) || - `${element.tagName.toLowerCase()}:${element.getAttribute('type') || index}`, - 120 - ).toLowerCase(); - - const formValues = []; - const formElements = Array.from(document.querySelectorAll( - 'input:not([type="password"]):not([type="hidden"]),select,textarea,[contenteditable="true"]' - )).slice(0, 80); - formElements.forEach((element, index) => { - if (!visible(element) || element.disabled) return; - const type = String(element.getAttribute('type') || '').toLowerCase(); - let value = ''; - if (type === 'checkbox' || type === 'radio') { - if (!element.checked) return; - value = element.value || true; - } else if (element.tagName.toLowerCase() === 'select') { - value = Array.from(element.selectedOptions || []).map((option) => compact(option.text || option.value)); - } else if (element.isContentEditable) { - value = compact(element.innerText || element.textContent || ''); - } else { - value = compact(element.value || ''); - } - if (value === '' || (Array.isArray(value) && value.length === 0)) return; - formValues.push({ key: elementKey(element, index), value }); - }); - - const selectedFilters = []; - const selectedCandidates = Array.from(document.querySelectorAll( - 'input:checked,select,[aria-selected="true"],[aria-checked="true"],' + - '[aria-current="true"],[data-state="active"],[data-state="selected"],' + - '[role="tab"].active,[role="tab"].selected,' + - '[class*="filter" i] .selected,[class*="filter" i] .active,' + - '[class*="facet" i] .selected,[class*="facet" i] .active,' + - '[class*="sort" i] .selected,[class*="sort" i] .active,' + - '[class*="price" i] .selected,[class*="price" i] .active,' + - '[class*="rating" i] .selected,[class*="rating" i] .active' - )).slice(0, 120); - selectedCandidates.forEach((element, index) => { - if (!visible(element)) return; - const descriptor = compact([ - element.getAttribute('id') || '', - element.getAttribute('class') || '', - element.getAttribute('role') || '', - element.getAttribute('aria-label') || '', - element.getAttribute('data-state') || '', - element.innerText || element.textContent || '', - element.parentElement?.getAttribute('class') || '', - ].join(' '), 300).toLowerCase(); - const nativeSelected = element.matches( - 'input:checked,select,[aria-checked="true"],[aria-selected="true"],[aria-current="true"]' - ); - const filterContext = /(filter|facet|sort|order|price|rating|score|star|category|筛选|排序|价格|评分|星级|分类|销量|综合|最新)/.test(descriptor); - if (!nativeSelected && !filterContext) return; - let value = compact( - element.getAttribute('aria-label') || element.innerText || element.textContent || element.value || '' - ); - if (element.tagName.toLowerCase() === 'select') { - value = Array.from(element.selectedOptions || []).map((option) => compact(option.text || option.value)).join('|'); - } - if (!value) return; - const selectionSource = - element.getAttribute('aria-selected') === 'true' ? 'aria-selected' : - element.getAttribute('aria-checked') === 'true' ? 'aria-checked' : - element.getAttribute('aria-current') === 'true' ? 'aria-current' : - ['active', 'selected'].includes(String(element.getAttribute('data-state') || '').toLowerCase()) - ? 'data-state' : - /(?:^|\s)(?:active|selected)(?:\s|$)/i.test(String(element.getAttribute('class') || '')) - ? 'class' : - element.matches('input:checked,select') ? 'native' : 'semantic-context'; - selectedFilters.push({ key: elementKey(element, index), value, source: selectionSource }); - }); - - let resultCount = null; - const countCandidates = Array.from(document.querySelectorAll( - '[data-total],[data-count],[aria-rowcount],[class*="result-count" i],[class*="total-count" i]' - )).slice(0, 40); - for (const element of countCandidates) { - const raw = element.getAttribute('data-total') || element.getAttribute('data-count') || - element.getAttribute('aria-rowcount') || compact(element.textContent || '', 120); - const match = String(raw || '').replace(/,/g, '').match(/\d+/); - if (!match) continue; - resultCount = Number(match[0]); - break; - } - const resultCandidates = new Set(); - for (const selector of [ - 'main article', 'main [role="article"]', 'main [role="row"]', - '[data-testid*="result" i]', '[data-test*="result" i]', - '[class*="search-result" i]', '[class*="result-item" i]' - ]) { - document.querySelectorAll(selector).forEach((element) => { - if (visible(element)) resultCandidates.add(element); - }); - } - if (resultCount === null && resultCandidates.size > 0) resultCount = resultCandidates.size; - const firstResult = Array.from(resultCandidates)[0]; - const firstResultText = firstResult ? compact(firstResult.innerText || firstResult.textContent || '', 240) : ''; - - const actionFeedback = []; - const feedbackCandidates = Array.from(document.querySelectorAll( - '[role="status"],[role="alert"],[aria-live]:not([aria-live="off"]),[class*="toast" i]' - )).slice(0, 40); - feedbackCandidates.forEach((element, index) => { - if (!visible(element)) return; - const value = compact(element.innerText || element.textContent || element.getAttribute('aria-label') || '', 240); - if (!value || !/(added|cart|basket|success|completed|已加入|购物车|成功|完成)/i.test(value)) return; - actionFeedback.push({ key: `feedback:${index}`, value }); - }); - - const commerceState = []; - const commerceCandidates = Array.from(document.querySelectorAll( - '[aria-label*="cart" i],[aria-label*="basket" i],[title*="cart" i],[title*="购物车" i],' + - '[class*="cart-count" i],[class*="cart-num" i],[data-testid*="cart" i]' - )).slice(0, 40); - commerceCandidates.forEach((element, index) => { - if (!visible(element)) return; - const value = compact( - element.getAttribute('aria-label') || element.getAttribute('title') || element.innerText || element.textContent || '', - 160 - ); - if (!value || !/\d/.test(value)) return; - commerceState.push({ key: elementKey(element, index), value }); - }); - - return { - page_position: pagePosition, - semantic_state: { - form_values: formValues.slice(0, 32), - selected_filters: selectedFilters.slice(0, 32), - result_count: resultCount, - first_result_text: firstResultText, - action_feedback: actionFeedback.slice(0, 4), - commerce_state: commerceState.slice(0, 8), - }, - }; - }); - - let title = ''; - try { - title = await page.title(); - } catch (_error) { - title = ''; - } - - return { - ok: true, - url: page.url(), - title, - tabs, - page_position: pageMetadata.page_position, - semantic_state: pageMetadata.semantic_state, - }; -} -""".strip() - - -def build_interactive_probe_js( - *, - max_items: int = 50, - viewport_only: bool = True, - query: str = "", - site_profiles: Optional[List[Dict[str, Any]]] = None, - generation_id: str = "g0", -) -> str: - """Build browser_run_code JavaScript for compact interactive-element probing.""" - - params: Dict[str, Any] = { - "max_items": _clamp_int(max_items, default=50, minimum=1, maximum=100), - "viewport_only": bool(viewport_only), - "query": str(query or "").strip().lower(), - "site_profiles": site_profiles or [], - "generation_id": str(generation_id or "g0"), - } - params_json = json.dumps(params, ensure_ascii=False) - - return f""" -async (page) => {{ - const params = {params_json}; - - return await page.evaluate((params) => {{ - const maxItems = Math.max(1, Math.min(Number(params.max_items || 50), 100)); - const viewportOnly = params.viewport_only !== false; - const query = String(params.query || '').trim().toLowerCase(); - const generationId = String(params.generation_id || 'g0'); - const host = String(window.location.hostname || '').toLowerCase(); - const activeControlSemantics = Array.isArray(params.site_profiles) - ? params.site_profiles - .filter((profile) => (profile.domains || []).some((domain) => {{ - const value = String(domain || '').toLowerCase(); - return host === value || host.endsWith(`.${{value}}`); - }})) - .map((profile) => profile.semantic_rules?.control_semantics) - .filter(Boolean) - : []; - - const matchesProfilePattern = (value, patterns) => {{ - return Array.isArray(patterns) && patterns.some((pattern) => {{ - try {{ - return new RegExp(String(pattern), 'i').test(String(value || '')); - }} catch (_) {{ - return false; - }} - }}); - }}; - - const selectors = [ - 'button', - 'a[href]', - 'input', - 'select', - 'textarea', - '[role="button"]', - '[role="link"]', - '[role="textbox"]', - '[role="searchbox"]', - '[role="checkbox"]', - '[role="radio"]', - '[role="tab"]', - '[role="option"]', - '[role="gridcell"]', - '[role="menuitem"]', - '[aria-selected]', - '[aria-checked]', - '[aria-sort]', - '[data-date]', - '[data-day]', - '[class*="calendar" i] [class*="day" i]', - '[class*="calendar" i] [class*="date" i]', - '[class*="datepicker" i] [class*="day" i]', - '[class*="date-picker" i] [class*="day" i]', - '[class*="sort" i] [class*="item" i]', - '[class*="sort" i] [class*="option" i]', - '[class*="rating" i] [class*="item" i]', - '[class*="rating" i] [class*="option" i]', - '[contenteditable="true"]', - '[aria-label]', - '[placeholder]', - '[name]', - '[data-testid]', - '[data-test]', - '[data-cy]' - ]; - - const normalize = (value, limit = 140) => {{ - return String(value || '') - .replace(/\\s+/g, ' ') - .trim() - .slice(0, limit); - }}; - - const attrEscape = (value) => {{ - return String(value || '').replace(/\\\\/g, '\\\\\\\\').replace(/"/g, '\\\\"'); - }}; - - const cssEscape = (value) => {{ - const raw = String(value || ''); - if (window.CSS && typeof window.CSS.escape === 'function') {{ - return window.CSS.escape(raw); - }} - return raw.replace(/[^a-zA-Z0-9_-]/g, '\\\\$&'); - }}; - - const roleFromTag = (el) => {{ - const explicit = el.getAttribute('role'); - if (explicit) return explicit; - - const tag = el.tagName.toLowerCase(); - const type = String(el.getAttribute('type') || '').toLowerCase(); - - if (tag === 'button') return 'button'; - if (tag === 'a') return 'link'; - if (el.getAttribute('contenteditable') === 'true') return 'textbox'; - if (tag === 'select') return 'combobox'; - if (tag === 'textarea') return 'textbox'; - if (tag === 'input') {{ - if (type === 'checkbox') return 'checkbox'; - if (type === 'radio') return 'radio'; - if (type === 'submit' || type === 'button') return 'button'; - return 'textbox'; - }} - return ''; - }}; - - const isVisible = (el, rect) => {{ - if (!el || !rect) return false; - if (rect.width < 2 || rect.height < 2) return false; - - const style = window.getComputedStyle(el); - if (!style) return false; - if (style.display === 'none') return false; - if (style.visibility === 'hidden') return false; - if (Number(style.opacity) === 0) return false; - - if (viewportOnly) {{ - if (rect.bottom < 0) return false; - if (rect.right < 0) return false; - if (rect.top > window.innerHeight) return false; - if (rect.left > window.innerWidth) return false; - }} - - return true; - }}; - - const validateSelectorHint = (el, selector) => {{ - if (!el || !selector) return ''; - try {{ - const matches = Array.from(document.querySelectorAll(selector)); - return matches.length === 1 && matches[0] === el ? selector : ''; - }} catch (_) {{ - return ''; - }} - }}; - - const buildSelectorHint = (el) => {{ - const tag = el.tagName.toLowerCase(); - - const testid = - el.getAttribute('data-testid') || - el.getAttribute('data-test') || - el.getAttribute('data-cy'); - if (testid) {{ - let candidate = `[data-cy="${{attrEscape(testid)}}"]`; - if (el.getAttribute('data-testid')) candidate = `[data-testid="${{attrEscape(testid)}}"]`; - if (el.getAttribute('data-test')) candidate = `[data-test="${{attrEscape(testid)}}"]`; - const validated = validateSelectorHint(el, candidate); - if (validated) return validated; - }} - - const id = el.getAttribute('id'); - if (id) {{ - const validated = validateSelectorHint(el, `#${{cssEscape(id)}}`); - if (validated) return validated; - }} - - const aria = el.getAttribute('aria-label'); - if (aria) {{ - const validated = validateSelectorHint( - el, - `${{tag}}[aria-label="${{attrEscape(aria)}}"]` - ); - if (validated) return validated; - }} - - const name = el.getAttribute('name'); - if (name) {{ - const validated = validateSelectorHint(el, `${{tag}}[name="${{attrEscape(name)}}"]`); - if (validated) return validated; - }} - - const placeholder = el.getAttribute('placeholder'); - if (placeholder) {{ - const validated = validateSelectorHint( - el, - `${{tag}}[placeholder="${{attrEscape(placeholder)}}"]` - ); - if (validated) return validated; - }} - - const path = []; - let node = el; - let depth = 0; - - while (node && node.nodeType === Node.ELEMENT_NODE && depth < 4) {{ - const nodeTag = node.tagName.toLowerCase(); - let index = 1; - let prev = node.previousElementSibling; - while (prev) {{ - if (prev.tagName.toLowerCase() === nodeTag) index += 1; - prev = prev.previousElementSibling; - }} - path.unshift(`${{nodeTag}}:nth-of-type(${{index}})`); - node = node.parentElement; - depth += 1; - }} - - return validateSelectorHint(el, path.join(' > ')); - }}; - - const isActionable = (el, rect) => {{ - if (!el || !rect) return false; - if (el.disabled || el.getAttribute('aria-disabled') === 'true') return false; - if (el.hasAttribute('inert')) return false; - const style = window.getComputedStyle(el); - if (!style || style.pointerEvents === 'none') return false; - - const x = Math.max(0, Math.min(window.innerWidth - 1, rect.left + rect.width / 2)); - const y = Math.max(0, Math.min(window.innerHeight - 1, rect.top + rect.height / 2)); - const hit = document.elementFromPoint(x, y); - return !hit || hit === el || el.contains(hit) || hit.contains(el); - }}; - - const elementText = (el) => {{ - const tag = el.tagName.toLowerCase(); - if (tag === 'input' || tag === 'textarea') {{ - return normalize(el.value || el.getAttribute('value') || ''); - }} - return normalize(el.innerText || el.textContent || ''); - }}; - - const accessibleName = (el) => {{ - return normalize( - el.getAttribute('aria-label') || - el.getAttribute('title') || - el.getAttribute('placeholder') || - el.getAttribute('alt') || - el.getAttribute('name') || - '' - ); - }}; - - const semanticContext = (el) => {{ - const parts = []; - let node = el; - let depth = 0; - while (node && node.nodeType === Node.ELEMENT_NODE && depth < 5) {{ - parts.push( - node.tagName || '', - node.getAttribute('id') || '', - node.getAttribute('class') || '', - node.getAttribute('role') || '', - node.getAttribute('aria-label') || '', - node.getAttribute('data-testid') || '', - node.getAttribute('data-date') || '', - node.getAttribute('data-day') || '' - ); - node = node.parentElement; - depth += 1; - }} - return normalize(parts.join(' '), 700).toLowerCase(); - }}; - - const classifyRegion = (el, searchable) => {{ - const semantic = semanticContext(el); - const searchableText = String(searchable || '').toLowerCase(); - const inPageChrome = Boolean(el.closest && el.closest('header,nav,[class*="header" i]')); - for (const profile of activeControlSemantics) {{ - const matchesContext = matchesProfilePattern( - `${{semantic}} ${{searchableText}}`, - profile.context_patterns - ); - if (matchesContext && !inPageChrome) return String(profile.region || 'main'); - if (inPageChrome && matchesProfilePattern(searchableText, profile.global_search_patterns)) {{ - return 'global_search'; - }} - }} - if (el.closest && el.closest('aside,[role="complementary"]')) return 'sidebar'; - const own = normalize([ - el.tagName || '', el.getAttribute('id') || '', el.getAttribute('class') || '', - el.getAttribute('role') || '', el.getAttribute('aria-label') || '', - el.getAttribute('data-testid') || '' - ].join(' '), 360).toLowerCase(); - const context = `${{own}} ${{String(searchable || '').toLowerCase()}}`; - if (/(sidebar|side-bar|aside|right-rail|right-panel|complementary)/.test(context)) return 'sidebar'; - if (/(hot[-_ ]?(search|list|rank|topic)|hotlist|hotrank|toplist|trending)/.test(context) || /\u70ed\u641c|\u70ed\u699c/.test(context)) return 'hot_search'; - if (/(account|profile|user-card|creator-card)/.test(context)) return 'account'; - if (/(shop|store|seller)/.test(context) || /\u5e97\u94fa|\u5356\u5bb6/.test(context)) return 'shop'; - if (/(chat|message|wangwang|aliim|contact-seller)/.test(context) || /\u65fa\u65fa|\u8054\u7cfb\u5356\u5bb6/.test(context)) return 'chat'; - return 'main'; - }}; - - const classifyControlKind = (el, text, name) => {{ - const role = String(el.getAttribute('role') || '').toLowerCase(); - const tag = el.tagName.toLowerCase(); - const type = String(el.getAttribute('type') || '').toLowerCase(); - const own = normalize([ - el.getAttribute('id') || '', el.getAttribute('class') || '', role, - el.getAttribute('aria-label') || '', el.getAttribute('data-testid') || '', - el.getAttribute('data-date') || '', el.getAttribute('data-day') || '', - el.getAttribute('name') || '', el.getAttribute('placeholder') || '', - text || '', name || '' - ].join(' '), 520).toLowerCase(); - const ancestor = semanticContext(el); - for (const profile of activeControlSemantics) {{ - if (!matchesProfilePattern(`${{ancestor}} ${{own}}`, profile.context_patterns)) continue; - for (const candidate of profile.kinds || []) {{ - if (!matchesProfilePattern(own, candidate.patterns)) continue; - const shapeRestrictions = [candidate.tags, candidate.roles, candidate.types] - .some((values) => Array.isArray(values) && values.length); - const shapeMatches = !shapeRestrictions || - (candidate.tags || []).includes(tag) || - (candidate.roles || []).includes(role) || - (candidate.types || []).includes(type); - if (shapeMatches) return String(candidate.kind || ''); - }} - }} - if ( - ['input', 'textarea', 'select'].includes(tag) || - el.isContentEditable || - ['textbox', 'searchbox', 'combobox'].includes(role) || - ['text', 'search', 'email', 'number', 'password'].includes(type) - ) return ''; - const calendarAncestor = /(calendar|date[-_ ]?picker|datepicker)/.test(ancestor) || /\u65e5\u5386|\u9009\u62e9\u65e5\u671f/.test(ancestor); - const directDateEvidence = role === 'gridcell' || el.hasAttribute('data-date') || - el.hasAttribute('data-day') || /(date-cell|day-cell|calendar-day)/.test(own); - if (directDateEvidence && calendarAncestor && /[0-9]/.test(own)) {{ - return 'calendar_date'; - }} - const sortAncestor = /(sort|order|sort-list|sort-tabs)/.test(ancestor) || /\u6392\u5e8f/.test(ancestor); - const sortLabel = /(?:^|\s)(sales?|volume|price|latest|newest|relevance|comprehensive)(?:\s|$)/.test(own) || - /(\u9500\u91cf|\u4ef7\u683c|\u6700\u65b0|\u7efc\u5408|\u8bc4\u5206)/.test(own); - if ((role === 'tab' || /(sort-item|sort-option|sort-tab)/.test(own)) && (sortAncestor || sortLabel)) return 'sort_tab'; - const ratingAncestor = /(rating|score|star|rating-filter)/.test(ancestor) || /\u8bc4\u5206|\u661f\u7ea7/.test(ancestor); - if ((role === 'option' || /(rating-item|rating-option|star-item)/.test(own)) && ratingAncestor) return 'rating_filter'; - if (role === 'tab') return 'tab'; - if (role === 'option') return 'option'; - return ''; - }}; - - const selectionSourceFromUrl = (el, kind) => {{ - if (!['sort', 'sort_tab', 'sort_option', 'rating_filter'].includes(String(kind || ''))) return ''; - let current; - try {{ - current = new URL(window.location.href); - }} catch (_error) {{ - return ''; - }} - const anchor = el.closest && el.closest('a[href]'); - if (anchor) {{ - try {{ - const candidate = new URL(anchor.href, current.href); - if (candidate.origin === current.origin && candidate.pathname === current.pathname) {{ - const candidateEntries = [...candidate.searchParams.entries()]; - if ( - candidateEntries.length > 0 && - candidateEntries.every(([key, value]) => current.searchParams.get(key) === value) - ) return 'url'; - }} - }} catch (_error) {{ - // Fall through to data-value matching. - }} - }} - const candidateValues = [ - el.getAttribute('data-value'), - el.getAttribute('data-sort'), - el.getAttribute('data-order'), - el.getAttribute('value') - ].filter(Boolean).map((value) => String(value).trim().toLowerCase()); - const currentValues = [...current.searchParams.values()] - .map((value) => String(value).trim().toLowerCase()); - return candidateValues.some((value) => currentValues.includes(value)) ? 'url-param' : ''; - }}; - - const queryAliases = (value) => {{ - const raw = String(value || '').trim().toLowerCase(); - if (!raw) return []; - - const aliases = new Set([raw]); - const add = (items) => items.forEach((item) => aliases.add(item)); - - if (['search', 'find', 'query', 'keyword', 'keywords', - '搜索', '搜尋', '查询', '查找', '关键词', '关键字', '搜寻', '検索'] - .some((term) => raw.includes(term))) {{ - add(['search', 'find', 'query', 'keyword', 'keywords', 'searchbox', 'search-box', - 'search_input', 'search-input', - '搜索', '搜', '搜尋', '查询', '查找', '关键词', '关键字', '搜寻', '検索']); - }} - - if (['input', 'textbox', 'text box', 'field', '输入', '輸入', '输入框', '文字框'] - .some((term) => raw.includes(term))) {{ - add(['input', 'textbox', 'text box', 'field', 'textarea', 'keyword', - 'search', 'query', '输入', '輸入', '搜索', '搜尋', '关键词', '关键字']); - }} - - if (['next', 'pagination', 'page', '下一页', '下一頁', '下页', '下頁', '更多'] - .some((term) => raw.includes(term))) {{ - add(['next', 'pagination', 'page', '下一页', '下一頁', '下页', '下頁', - '更多', '加载更多', '載入更多', 'load more']); - }} - - if (['login', 'sign in', 'signin', '登录', '登入', '登陆'] - .some((term) => raw.includes(term))) {{ - add(['login', 'sign in', 'signin', 'log in', '登录', '登入', '登陆']); - }} - - return Array.from(aliases).filter(Boolean); - }}; - - const queryTerms = queryAliases(query); - const exactQuery = String(query || '').trim().toLowerCase(); - - const classifyActionLikelihood = (el, searchable, kind = '') => {{ - const tag = el.tagName.toLowerCase(); - const type = String(el.getAttribute('type') || '').toLowerCase(); - const role = roleFromTag(el); - const text = String(searchable || '').toLowerCase(); - - if (kind === 'calendar_date') return 'date'; - if (kind === 'sort_tab') return 'sort'; - if (kind === 'rating_filter') return 'rating'; - if (['hotel_checkin', 'hotel_checkout'].includes(kind)) return 'date'; - if (kind === 'hotel_destination') return 'input'; - if (kind === 'hotel_search_submit') return 'search'; - if (kind === 'hotel_filter') return 'filter'; - - if ( - type === 'search' || - role === 'searchbox' || - /\b(search|query|keyword|kw|wd)\b/i.test(text) || - /(搜索|搜尋|查询|查找|关键词|关键字|検索)/.test(text) - ) {{ - return 'search'; - }} - - if (['input', 'textarea'].includes(tag) || role === 'textbox') return 'input'; - if (/\b(next|pagination|page)\b/i.test(text) || /(下一页|下一頁|下页|下頁|更多|加载更多|載入更多)/.test(text)) return 'pagination'; - if (/\b(login|sign in|signin|log in)\b/i.test(text) || /(登录|登入|登陆)/.test(text)) return 'login'; - if (/\b(filter|sort|category)\b/i.test(text) || /(筛选|篩選|排序|分类|分類)/.test(text)) return 'filter'; - if (/\b(cart|basket|buy|checkout)\b/i.test(text) || /(购物车|購物車|加入购物车|加入購物車|购买|購買)/.test(text)) return 'commerce'; - - if (tag === 'button') return 'button'; - if (tag === 'a') return 'link'; - return role || tag; - }}; - - const queryMatches = (searchable) => {{ - if (!queryTerms.length) return true; - const haystack = String(searchable || '').toLowerCase(); - return queryTerms.some((term) => haystack.includes(term)); - }}; - - const exactQueryMatches = (searchable) => {{ - if (!exactQuery) return true; - return String(searchable || '').toLowerCase().includes(exactQuery); - }}; - - const scoreElement = (el, rect, text, name, actionLikelihood) => {{ - let score = 0; - const tag = el.tagName.toLowerCase(); - const role = roleFromTag(el); - - if (el.getAttribute('data-testid')) score += 40; - if (el.getAttribute('data-test') || el.getAttribute('data-cy')) score += 30; - if (el.getAttribute('aria-label')) score += 25; - if (tag === 'button') score += 25; - if (tag === 'input' || tag === 'select' || tag === 'textarea') score += 22; - if (el.getAttribute('contenteditable') === 'true') score += 18; - if (tag === 'a') score += 18; - if (role) score += 15; - if (actionLikelihood === 'search') score += 35; - if (['date', 'sort', 'rating'].includes(actionLikelihood)) score += 30; - if (query && queryMatches(`${{actionLikelihood}} ${{tag}} ${{role}}`)) score += 20; - if (text) score += Math.min(20, text.length / 4); - if (name) score += Math.min(15, name.length / 5); - - if (rect.top >= 0 && rect.top <= window.innerHeight) score += 15; - if (rect.left >= 0 && rect.left <= window.innerWidth) score += 5; - - if (el.disabled || el.getAttribute('aria-disabled') === 'true') score -= 50; - - return score; - }}; - - const all = Array.from(document.querySelectorAll(selectors.join(','))); - const seen = new Set(); - const candidates = []; - const widenedCandidates = []; - - for (const el of all) {{ - if (!el || seen.has(el)) continue; - seen.add(el); - - const rect = el.getBoundingClientRect(); - if (!isVisible(el, rect)) continue; - - const tag = el.tagName.toLowerCase(); - const role = roleFromTag(el); - const text = elementText(el); - const name = accessibleName(el); - const testid = - el.getAttribute('data-testid') || - el.getAttribute('data-test') || - el.getAttribute('data-cy') || - ''; - const type = el.getAttribute('type') || ''; - const id = el.getAttribute('id') || ''; - const nameAttr = el.getAttribute('name') || ''; - const placeholder = el.getAttribute('placeholder') || ''; - const className = el.getAttribute('class') || ''; - const title = el.getAttribute('title') || ''; - - const searchable = `${{tag}} ${{role}} ${{type}} ${{id}} ${{nameAttr}} ${{className}} ${{text}} ${{name}} ${{placeholder}} ${{title}} ${{testid}}`.toLowerCase(); - const exactMatch = exactQueryMatches(searchable); - if (!exactMatch && !queryMatches(searchable)) continue; - - const kind = classifyControlKind(el, text, name); - const actionLikelihood = classifyActionLikelihood(el, searchable, kind); - const selectorHint = buildSelectorHint(el); - const actionable = isActionable(el, rect); - const matchCount = selectorHint - ? document.querySelectorAll(selectorHint).length - : 0; - const enabled = !( - el.disabled || - el.getAttribute('aria-disabled') === 'true' || - el.hasAttribute('inert') - ); - const clickable = Boolean( - actionable && enabled && selectorHint && matchCount === 1 - ); - const selectedSource = - el.getAttribute('aria-selected') === 'true' ? 'aria-selected' : - el.getAttribute('aria-checked') === 'true' ? 'aria-checked' : - el.getAttribute('aria-current') === 'true' ? 'aria-current' : - ['active', 'selected', 'checked'].includes(String(el.getAttribute('data-state') || '').toLowerCase()) ? 'data-state' : - /(^|\\s)(active|selected|checked)(\\s|$)/i.test(className) ? 'class' : - selectionSourceFromUrl(el, kind); - - const candidate = {{ - tag, - role, - action_likelihood: actionLikelihood, - region: classifyRegion(el, searchable), - kind: kind || actionLikelihood, - text, - accessible_name: name, - aria_label: normalize(el.getAttribute('aria-label') || ''), - testid: normalize(testid), - input_type: normalize(type), - name: normalize(nameAttr), - placeholder: normalize(placeholder), - href: normalize(el.getAttribute('href') || '', 180), - disabled: Boolean(el.disabled || el.getAttribute('aria-disabled') === 'true'), - selected: Boolean(selectedSource), - selected_source: selectedSource, - visible: true, - enabled, - actionable, - clickable, - match_count: matchCount, - generation_id: generationId, - bbox: [ - Math.round(rect.x), - Math.round(rect.y), - Math.round(rect.width), - Math.round(rect.height) - ], - selector_hint: clickable ? selectorHint : '', - selector_hint_validated: clickable, - score: scoreElement(el, rect, text, name, actionLikelihood) - }}; - (exactMatch ? candidates : widenedCandidates).push(candidate); - }} - - const selectedCandidates = candidates.length ? candidates : widenedCandidates; - selectedCandidates.sort((a, b) => {{ - if (b.score !== a.score) return b.score - a.score; - if (a.bbox[1] !== b.bbox[1]) return a.bbox[1] - b.bbox[1]; - return a.bbox[0] - b.bbox[0]; - }}); - - const elements = selectedCandidates.slice(0, maxItems).map((item, index) => {{ - const copy = {{ ...item }}; - copy.id = `e${{index + 1}}`; - delete copy.score; - return copy; - }}); - - return {{ - ok: true, - url: window.location.href, - title: document.title, - viewport: {{ - width: window.innerWidth, - height: window.innerHeight, - scroll_x: window.scrollX, - scroll_y: window.scrollY - }}, - query, - viewport_only: viewportOnly, - generation_id: generationId, - total_candidates: selectedCandidates.length, - query_widened: Boolean(exactQuery && !candidates.length && widenedCandidates.length), - returned: elements.length, - elements, - error: null - }}; - }}, params); -}} -""".strip() - - -def build_card_probe_js( - *, - max_cards: int = 20, - viewport_only: bool = True, - include_buttons: bool = True, - query: str = "", - site_profiles: Optional[List[Dict[str, Any]]] = None, - selector_cache_records: Optional[List[Dict[str, Any]]] = None, - generation_id: str = "g0", -) -> str: - """Build browser_run_code_unsafe JavaScript for compact repeated-card probing.""" - - params: Dict[str, Any] = { - "max_cards": _clamp_int(max_cards, default=20, minimum=1, maximum=50), - "viewport_only": bool(viewport_only), - "include_buttons": bool(include_buttons), - "query": str(query or "").strip().lower(), - "site_profiles": site_profiles or [], - "selector_cache_records": selector_cache_records or [], - "generation_id": str(generation_id or "g0"), - } - params_json = json.dumps(params, ensure_ascii=False) - - template = r""" async (page) => { const params = __PARAMS__; @@ -1060,6 +166,15 @@ def build_card_probe_js( return true; }; + // Not viewport-clipped: sort controls are often scrolled out of view. + const isRenderedElement = (element) => { + if (!element || !element.isConnected) return false; + const rect = element.getBoundingClientRect(); + if (rect.width < 1 || rect.height < 1) return false; + const style = window.getComputedStyle(element); + return style.display !== 'none' && style.visibility !== 'hidden' && Number(style.opacity) !== 0; + }; + const isEnabled = (el) => { return Boolean( el && @@ -1833,7 +948,7 @@ def build_card_probe_js( ]; const semanticLabel = /(?:sales?|volume|price|latest|newest|relevance|comprehensive|销量|价格|最新|综合|评分)/i; for (const candidate of Array.from(document.querySelectorAll(selectors.join(','))).slice(0, 40)) { - if (!visible(candidate)) continue; + if (!isRenderedElement(candidate)) continue; const value = textOf(candidate, 120) || normalize(candidate.getAttribute('aria-label') || '', 120); const context = normalize([ candidate.getAttribute('class') || '', @@ -2187,9 +1302,13 @@ def build_card_probe_js( 'message' ]; + // "nav" as a token of its own: BEM names such as "mntl-fullscreen-nav__list" carry no "nav-". + const NAV_TOKEN = /(^|[^a-z])nav([^a-z]|$)/; + const NAVIGATION_CONTAINER = 'nav, [role="navigation"], [role="menu"], [role="menubar"]'; + const hasChromeFragment = (value) => { const text = String(value || '').toLowerCase(); - return PAGE_CHROME_FRAGMENTS.some((fragment) => text.includes(fragment)); + return NAV_TOKEN.test(text) || PAGE_CHROME_FRAGMENTS.some((fragment) => text.includes(fragment)); }; const elementChromeText = (el) => { @@ -2883,6 +2002,7 @@ def build_card_probe_js( const selectable = scored.filter((item) => { if (looksLikePageChrome(item.data)) return false; + if (item.el.closest(NAVIGATION_CONTAINER)) return false; // If we already have rich listing-like candidates, remove weak nav-like entries. if (!hasRichCandidates) return true; @@ -2996,6 +2116,3 @@ def build_card_probe_js( }; }, params); } -""".strip() - - return template.replace("__PARAMS__", params_json) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/cards.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/cards.py new file mode 100644 index 0000000000..9ae71d4fff --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/cards.py @@ -0,0 +1,46 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The page script that finds repeated cards; the script body is cards.js, the parameters go in at build time.""" + +from __future__ import annotations + +import json +from importlib.resources import files +from typing import Any, Dict, List, Optional + +_TEMPLATE = files(__package__).joinpath("cards.js").read_text(encoding="utf-8").strip() + + +def _clamp_int(value: Any, *, default: int, minimum: int, maximum: int) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + parsed = default + return max(minimum, min(maximum, parsed)) + + +def build_card_probe_js( + *, + max_cards: int = 20, + viewport_only: bool = True, + include_buttons: bool = True, + query: str = "", + site_profiles: Optional[List[Dict[str, Any]]] = None, + selector_cache_records: Optional[List[Dict[str, Any]]] = None, + generation_id: str = "g0", +) -> str: + """Build the JavaScript for compact repeated-card probing.""" + + params: Dict[str, Any] = { + "max_cards": _clamp_int(max_cards, default=20, minimum=1, maximum=50), + "viewport_only": bool(viewport_only), + "include_buttons": bool(include_buttons), + "query": str(query or "").strip().lower(), + "site_profiles": site_profiles or [], + "selector_cache_records": selector_cache_records or [], + "generation_id": str(generation_id or "g0"), + } + params_json = json.dumps(params, ensure_ascii=False) + + return _TEMPLATE.replace("__PARAMS__", params_json) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/clipboard_guard.js b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/clipboard_guard.js new file mode 100644 index 0000000000..49cdb03f88 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/clipboard_guard.js @@ -0,0 +1,52 @@ +// Runs in every document before the page's own scripts: the managed Chrome shares the host clipboard, +// so a read is refused however it is spelled, not only when a script's text names it. +// One expression without a trailing semicolon: the browser-use sidecar evaluates it through its expression wrapper. +(() => { + const denied = + "Blocked: the clipboard belongs to the host machine, not to this task, so pasting or reading it is not allowed. " + + "Read the value from the page instead (its text or an input's value) and type it."; + const reject = function () { + return Promise.reject(new Error(denied)); + }; + const frozen = (value) => ({ value: value, configurable: false, writable: false }); + const clipboards = []; + if (typeof Clipboard !== "undefined" && Clipboard.prototype) { + clipboards.push(Clipboard.prototype); + } + if (typeof navigator !== "undefined" && navigator.clipboard) { + clipboards.push(navigator.clipboard); + } + for (const clipboard of clipboards) { + for (const name of ["read", "readText"]) { + try { + Object.defineProperty(clipboard, name, frozen(reject)); + } catch (error) { + // already frozen by an earlier install on this document + } + } + } + const documents = []; + if (typeof Document !== "undefined" && Document.prototype) { + documents.push(Document.prototype); + } + if (typeof document !== "undefined") { + documents.push(document); + } + for (const target of documents) { + const execCommand = target.execCommand; + if (typeof execCommand !== "function") { + continue; + } + const guarded = function (command, ...rest) { + if (String(command).toLowerCase() === "paste") { + return false; + } + return execCommand.apply(this, [command, ...rest]); + }; + try { + Object.defineProperty(target, "execCommand", frozen(guarded)); + } catch (error) { + // already frozen by an earlier install on this document + } + } +})() diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/interactives.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/interactives.py new file mode 100644 index 0000000000..25a75c0804 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/interactives.py @@ -0,0 +1,636 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The page script that lists the interactive elements.""" + +from __future__ import annotations + +import json +from typing import Any, Dict, List, Optional + +from .cards import _clamp_int + + +def build_interactive_probe_js( + *, + max_items: int = 50, + viewport_only: bool = True, + query: str = "", + site_profiles: Optional[List[Dict[str, Any]]] = None, + generation_id: str = "g0", +) -> str: + """Build the JavaScript for compact interactive-element probing.""" + + params: Dict[str, Any] = { + "max_items": _clamp_int(max_items, default=50, minimum=1, maximum=100), + "viewport_only": bool(viewport_only), + "query": str(query or "").strip().lower(), + "site_profiles": site_profiles or [], + "generation_id": str(generation_id or "g0"), + } + params_json = json.dumps(params, ensure_ascii=False) + + return f""" +async (page) => {{ + const params = {params_json}; + + return await page.evaluate((params) => {{ + const maxItems = Math.max(1, Math.min(Number(params.max_items || 50), 100)); + const viewportOnly = params.viewport_only !== false; + const query = String(params.query || '').trim().toLowerCase(); + const generationId = String(params.generation_id || 'g0'); + const host = String(window.location.hostname || '').toLowerCase(); + const activeControlSemantics = Array.isArray(params.site_profiles) + ? params.site_profiles + .filter((profile) => (profile.domains || []).some((domain) => {{ + const value = String(domain || '').toLowerCase(); + return host === value || host.endsWith(`.${{value}}`); + }})) + .map((profile) => profile.semantic_rules?.control_semantics) + .filter(Boolean) + : []; + + const matchesProfilePattern = (value, patterns) => {{ + return Array.isArray(patterns) && patterns.some((pattern) => {{ + try {{ + return new RegExp(String(pattern), 'i').test(String(value || '')); + }} catch (_) {{ + return false; + }} + }}); + }}; + + const selectors = [ + 'button', + 'a[href]', + 'input', + 'select', + 'textarea', + '[role="button"]', + '[role="link"]', + '[role="textbox"]', + '[role="searchbox"]', + '[role="checkbox"]', + '[role="radio"]', + '[role="tab"]', + '[role="option"]', + '[role="gridcell"]', + '[role="menuitem"]', + '[aria-selected]', + '[aria-checked]', + '[aria-sort]', + '[data-date]', + '[data-day]', + '[class*="calendar" i] [class*="day" i]', + '[class*="calendar" i] [class*="date" i]', + '[class*="datepicker" i] [class*="day" i]', + '[class*="date-picker" i] [class*="day" i]', + '[class*="sort" i] [class*="item" i]', + '[class*="sort" i] [class*="option" i]', + '[class*="rating" i] [class*="item" i]', + '[class*="rating" i] [class*="option" i]', + '[contenteditable="true"]', + '[aria-label]', + '[placeholder]', + '[name]', + '[data-testid]', + '[data-test]', + '[data-cy]' + ]; + + const normalize = (value, limit = 140) => {{ + return String(value || '') + .replace(/\\s+/g, ' ') + .trim() + .slice(0, limit); + }}; + + const attrEscape = (value) => {{ + return String(value || '').replace(/\\\\/g, '\\\\\\\\').replace(/"/g, '\\\\"'); + }}; + + const cssEscape = (value) => {{ + const raw = String(value || ''); + if (window.CSS && typeof window.CSS.escape === 'function') {{ + return window.CSS.escape(raw); + }} + return raw.replace(/[^a-zA-Z0-9_-]/g, '\\\\$&'); + }}; + + const roleFromTag = (el) => {{ + const explicit = el.getAttribute('role'); + if (explicit) return explicit; + + const tag = el.tagName.toLowerCase(); + const type = String(el.getAttribute('type') || '').toLowerCase(); + + if (tag === 'button') return 'button'; + if (tag === 'a') return 'link'; + if (el.getAttribute('contenteditable') === 'true') return 'textbox'; + if (tag === 'select') return 'combobox'; + if (tag === 'textarea') return 'textbox'; + if (tag === 'input') {{ + if (type === 'checkbox') return 'checkbox'; + if (type === 'radio') return 'radio'; + if (type === 'submit' || type === 'button') return 'button'; + return 'textbox'; + }} + return ''; + }}; + + const isVisible = (el, rect) => {{ + if (!el || !rect) return false; + if (rect.width < 2 || rect.height < 2) return false; + + const style = window.getComputedStyle(el); + if (!style) return false; + if (style.display === 'none') return false; + if (style.visibility === 'hidden') return false; + if (Number(style.opacity) === 0) return false; + + if (viewportOnly) {{ + if (rect.bottom < 0) return false; + if (rect.right < 0) return false; + if (rect.top > window.innerHeight) return false; + if (rect.left > window.innerWidth) return false; + }} + + return true; + }}; + + const validateSelectorHint = (el, selector) => {{ + if (!el || !selector) return ''; + try {{ + const matches = Array.from(document.querySelectorAll(selector)); + return matches.length === 1 && matches[0] === el ? selector : ''; + }} catch (_) {{ + return ''; + }} + }}; + + const buildSelectorHint = (el) => {{ + const tag = el.tagName.toLowerCase(); + + const testid = + el.getAttribute('data-testid') || + el.getAttribute('data-test') || + el.getAttribute('data-cy'); + if (testid) {{ + let candidate = `[data-cy="${{attrEscape(testid)}}"]`; + if (el.getAttribute('data-testid')) candidate = `[data-testid="${{attrEscape(testid)}}"]`; + if (el.getAttribute('data-test')) candidate = `[data-test="${{attrEscape(testid)}}"]`; + const validated = validateSelectorHint(el, candidate); + if (validated) return validated; + }} + + const id = el.getAttribute('id'); + if (id) {{ + const validated = validateSelectorHint(el, `#${{cssEscape(id)}}`); + if (validated) return validated; + }} + + const aria = el.getAttribute('aria-label'); + if (aria) {{ + const validated = validateSelectorHint( + el, + `${{tag}}[aria-label="${{attrEscape(aria)}}"]` + ); + if (validated) return validated; + }} + + const name = el.getAttribute('name'); + if (name) {{ + const validated = validateSelectorHint(el, `${{tag}}[name="${{attrEscape(name)}}"]`); + if (validated) return validated; + }} + + const placeholder = el.getAttribute('placeholder'); + if (placeholder) {{ + const validated = validateSelectorHint( + el, + `${{tag}}[placeholder="${{attrEscape(placeholder)}}"]` + ); + if (validated) return validated; + }} + + const path = []; + let node = el; + let depth = 0; + + while (node && node.nodeType === Node.ELEMENT_NODE && depth < 4) {{ + const nodeTag = node.tagName.toLowerCase(); + let index = 1; + let prev = node.previousElementSibling; + while (prev) {{ + if (prev.tagName.toLowerCase() === nodeTag) index += 1; + prev = prev.previousElementSibling; + }} + path.unshift(`${{nodeTag}}:nth-of-type(${{index}})`); + node = node.parentElement; + depth += 1; + }} + + return validateSelectorHint(el, path.join(' > ')); + }}; + + const isActionable = (el, rect) => {{ + if (!el || !rect) return false; + if (el.disabled || el.getAttribute('aria-disabled') === 'true') return false; + if (el.hasAttribute('inert')) return false; + const style = window.getComputedStyle(el); + if (!style || style.pointerEvents === 'none') return false; + + const x = Math.max(0, Math.min(window.innerWidth - 1, rect.left + rect.width / 2)); + const y = Math.max(0, Math.min(window.innerHeight - 1, rect.top + rect.height / 2)); + const hit = document.elementFromPoint(x, y); + return !hit || hit === el || el.contains(hit) || hit.contains(el); + }}; + + const elementText = (el) => {{ + const tag = el.tagName.toLowerCase(); + if (tag === 'input' || tag === 'textarea') {{ + return normalize(el.value || el.getAttribute('value') || ''); + }} + return normalize(el.innerText || el.textContent || ''); + }}; + + const accessibleName = (el) => {{ + return normalize( + el.getAttribute('aria-label') || + el.getAttribute('title') || + el.getAttribute('placeholder') || + el.getAttribute('alt') || + el.getAttribute('name') || + '' + ); + }}; + + const semanticContext = (el) => {{ + const parts = []; + let node = el; + let depth = 0; + while (node && node.nodeType === Node.ELEMENT_NODE && depth < 5) {{ + parts.push( + node.tagName || '', + node.getAttribute('id') || '', + node.getAttribute('class') || '', + node.getAttribute('role') || '', + node.getAttribute('aria-label') || '', + node.getAttribute('data-testid') || '', + node.getAttribute('data-date') || '', + node.getAttribute('data-day') || '' + ); + node = node.parentElement; + depth += 1; + }} + return normalize(parts.join(' '), 700).toLowerCase(); + }}; + + const classifyRegion = (el, searchable) => {{ + const semantic = semanticContext(el); + const searchableText = String(searchable || '').toLowerCase(); + const inPageChrome = Boolean(el.closest && el.closest('header,nav,[class*="header" i]')); + for (const profile of activeControlSemantics) {{ + const matchesContext = matchesProfilePattern( + `${{semantic}} ${{searchableText}}`, + profile.context_patterns + ); + if (matchesContext && !inPageChrome) return String(profile.region || 'main'); + if (inPageChrome && matchesProfilePattern(searchableText, profile.global_search_patterns)) {{ + return 'global_search'; + }} + }} + if (el.closest && el.closest('aside,[role="complementary"]')) return 'sidebar'; + const own = normalize([ + el.tagName || '', el.getAttribute('id') || '', el.getAttribute('class') || '', + el.getAttribute('role') || '', el.getAttribute('aria-label') || '', + el.getAttribute('data-testid') || '' + ].join(' '), 360).toLowerCase(); + const context = `${{own}} ${{String(searchable || '').toLowerCase()}}`; + if (/(sidebar|side-bar|aside|right-rail|right-panel|complementary)/.test(context)) return 'sidebar'; + if (/(hot[-_ ]?(search|list|rank|topic)|hotlist|hotrank|toplist|trending)/.test(context) || /\u70ed\u641c|\u70ed\u699c/.test(context)) return 'hot_search'; + if (/(account|profile|user-card|creator-card)/.test(context)) return 'account'; + if (/(shop|store|seller)/.test(context) || /\u5e97\u94fa|\u5356\u5bb6/.test(context)) return 'shop'; + if (/(chat|message|wangwang|aliim|contact-seller)/.test(context) || /\u65fa\u65fa|\u8054\u7cfb\u5356\u5bb6/.test(context)) return 'chat'; + return 'main'; + }}; + + const classifyControlKind = (el, text, name) => {{ + const role = String(el.getAttribute('role') || '').toLowerCase(); + const tag = el.tagName.toLowerCase(); + const type = String(el.getAttribute('type') || '').toLowerCase(); + const own = normalize([ + el.getAttribute('id') || '', el.getAttribute('class') || '', role, + el.getAttribute('aria-label') || '', el.getAttribute('data-testid') || '', + el.getAttribute('data-date') || '', el.getAttribute('data-day') || '', + el.getAttribute('name') || '', el.getAttribute('placeholder') || '', + text || '', name || '' + ].join(' '), 520).toLowerCase(); + const ancestor = semanticContext(el); + for (const profile of activeControlSemantics) {{ + if (!matchesProfilePattern(`${{ancestor}} ${{own}}`, profile.context_patterns)) continue; + for (const candidate of profile.kinds || []) {{ + if (!matchesProfilePattern(own, candidate.patterns)) continue; + const shapeRestrictions = [candidate.tags, candidate.roles, candidate.types] + .some((values) => Array.isArray(values) && values.length); + const shapeMatches = !shapeRestrictions || + (candidate.tags || []).includes(tag) || + (candidate.roles || []).includes(role) || + (candidate.types || []).includes(type); + if (shapeMatches) return String(candidate.kind || ''); + }} + }} + if ( + ['input', 'textarea', 'select'].includes(tag) || + el.isContentEditable || + ['textbox', 'searchbox', 'combobox'].includes(role) || + ['text', 'search', 'email', 'number', 'password'].includes(type) + ) return ''; + const calendarAncestor = /(calendar|date[-_ ]?picker|datepicker)/.test(ancestor) || /\u65e5\u5386|\u9009\u62e9\u65e5\u671f/.test(ancestor); + const directDateEvidence = role === 'gridcell' || el.hasAttribute('data-date') || + el.hasAttribute('data-day') || /(date-cell|day-cell|calendar-day)/.test(own); + if (directDateEvidence && calendarAncestor && /[0-9]/.test(own)) {{ + return 'calendar_date'; + }} + const sortAncestor = /(sort|order|sort-list|sort-tabs)/.test(ancestor) || /\u6392\u5e8f/.test(ancestor); + const sortLabel = /(?:^|\s)(sales?|volume|price|latest|newest|relevance|comprehensive)(?:\s|$)/.test(own) || + /(\u9500\u91cf|\u4ef7\u683c|\u6700\u65b0|\u7efc\u5408|\u8bc4\u5206)/.test(own); + if ((role === 'tab' || /(sort-item|sort-option|sort-tab)/.test(own)) && (sortAncestor || sortLabel)) return 'sort_tab'; + const ratingAncestor = /(rating|score|star|rating-filter)/.test(ancestor) || /\u8bc4\u5206|\u661f\u7ea7/.test(ancestor); + if ((role === 'option' || /(rating-item|rating-option|star-item)/.test(own)) && ratingAncestor) return 'rating_filter'; + if (role === 'tab') return 'tab'; + if (role === 'option') return 'option'; + return ''; + }}; + + const selectionSourceFromUrl = (el, kind) => {{ + if (!['sort', 'sort_tab', 'sort_option', 'rating_filter'].includes(String(kind || ''))) return ''; + let current; + try {{ + current = new URL(window.location.href); + }} catch (_error) {{ + return ''; + }} + const anchor = el.closest && el.closest('a[href]'); + if (anchor) {{ + try {{ + const candidate = new URL(anchor.href, current.href); + if (candidate.origin === current.origin && candidate.pathname === current.pathname) {{ + const candidateEntries = [...candidate.searchParams.entries()]; + if ( + candidateEntries.length > 0 && + candidateEntries.every(([key, value]) => current.searchParams.get(key) === value) + ) return 'url'; + }} + }} catch (_error) {{ + // Fall through to data-value matching. + }} + }} + const candidateValues = [ + el.getAttribute('data-value'), + el.getAttribute('data-sort'), + el.getAttribute('data-order'), + el.getAttribute('value') + ].filter(Boolean).map((value) => String(value).trim().toLowerCase()); + const currentValues = [...current.searchParams.values()] + .map((value) => String(value).trim().toLowerCase()); + return candidateValues.some((value) => currentValues.includes(value)) ? 'url-param' : ''; + }}; + + const queryAliases = (value) => {{ + const raw = String(value || '').trim().toLowerCase(); + if (!raw) return []; + + const aliases = new Set([raw]); + const add = (items) => items.forEach((item) => aliases.add(item)); + + if (['search', 'find', 'query', 'keyword', 'keywords', + '搜索', '搜尋', '查询', '查找', '关键词', '关键字', '搜寻', '検索'] + .some((term) => raw.includes(term))) {{ + add(['search', 'find', 'query', 'keyword', 'keywords', 'searchbox', 'search-box', + 'search_input', 'search-input', + '搜索', '搜', '搜尋', '查询', '查找', '关键词', '关键字', '搜寻', '検索']); + }} + + if (['input', 'textbox', 'text box', 'field', '输入', '輸入', '输入框', '文字框'] + .some((term) => raw.includes(term))) {{ + add(['input', 'textbox', 'text box', 'field', 'textarea', 'keyword', + 'search', 'query', '输入', '輸入', '搜索', '搜尋', '关键词', '关键字']); + }} + + if (['next', 'pagination', 'page', '下一页', '下一頁', '下页', '下頁', '更多'] + .some((term) => raw.includes(term))) {{ + add(['next', 'pagination', 'page', '下一页', '下一頁', '下页', '下頁', + '更多', '加载更多', '載入更多', 'load more']); + }} + + if (['login', 'sign in', 'signin', '登录', '登入', '登陆'] + .some((term) => raw.includes(term))) {{ + add(['login', 'sign in', 'signin', 'log in', '登录', '登入', '登陆']); + }} + + return Array.from(aliases).filter(Boolean); + }}; + + const queryTerms = queryAliases(query); + const exactQuery = String(query || '').trim().toLowerCase(); + + const classifyActionLikelihood = (el, searchable, kind = '') => {{ + const tag = el.tagName.toLowerCase(); + const type = String(el.getAttribute('type') || '').toLowerCase(); + const role = roleFromTag(el); + const text = String(searchable || '').toLowerCase(); + + if (kind === 'calendar_date') return 'date'; + if (kind === 'sort_tab') return 'sort'; + if (kind === 'rating_filter') return 'rating'; + if (['hotel_checkin', 'hotel_checkout'].includes(kind)) return 'date'; + if (kind === 'hotel_destination') return 'input'; + if (kind === 'hotel_search_submit') return 'search'; + if (kind === 'hotel_filter') return 'filter'; + + if ( + type === 'search' || + role === 'searchbox' || + /\b(search|query|keyword|kw|wd)\b/i.test(text) || + /(搜索|搜尋|查询|查找|关键词|关键字|検索)/.test(text) + ) {{ + return 'search'; + }} + + if (['input', 'textarea'].includes(tag) || role === 'textbox') return 'input'; + if (/\b(next|pagination|page)\b/i.test(text) || /(下一页|下一頁|下页|下頁|更多|加载更多|載入更多)/.test(text)) return 'pagination'; + if (/\b(login|sign in|signin|log in)\b/i.test(text) || /(登录|登入|登陆)/.test(text)) return 'login'; + if (/\b(filter|sort|category)\b/i.test(text) || /(筛选|篩選|排序|分类|分類)/.test(text)) return 'filter'; + if (/\b(cart|basket|buy|checkout)\b/i.test(text) || /(购物车|購物車|加入购物车|加入購物車|购买|購買)/.test(text)) return 'commerce'; + + if (tag === 'button') return 'button'; + if (tag === 'a') return 'link'; + return role || tag; + }}; + + const queryMatches = (searchable) => {{ + if (!queryTerms.length) return true; + const haystack = String(searchable || '').toLowerCase(); + return queryTerms.some((term) => haystack.includes(term)); + }}; + + const exactQueryMatches = (searchable) => {{ + if (!exactQuery) return true; + return String(searchable || '').toLowerCase().includes(exactQuery); + }}; + + const scoreElement = (el, rect, text, name, actionLikelihood) => {{ + let score = 0; + const tag = el.tagName.toLowerCase(); + const role = roleFromTag(el); + + if (el.getAttribute('data-testid')) score += 40; + if (el.getAttribute('data-test') || el.getAttribute('data-cy')) score += 30; + if (el.getAttribute('aria-label')) score += 25; + if (tag === 'button') score += 25; + if (tag === 'input' || tag === 'select' || tag === 'textarea') score += 22; + if (el.getAttribute('contenteditable') === 'true') score += 18; + if (tag === 'a') score += 18; + if (role) score += 15; + if (actionLikelihood === 'search') score += 35; + if (['date', 'sort', 'rating'].includes(actionLikelihood)) score += 30; + if (query && queryMatches(`${{actionLikelihood}} ${{tag}} ${{role}}`)) score += 20; + if (text) score += Math.min(20, text.length / 4); + if (name) score += Math.min(15, name.length / 5); + + if (rect.top >= 0 && rect.top <= window.innerHeight) score += 15; + if (rect.left >= 0 && rect.left <= window.innerWidth) score += 5; + + if (el.disabled || el.getAttribute('aria-disabled') === 'true') score -= 50; + + return score; + }}; + + const all = Array.from(document.querySelectorAll(selectors.join(','))); + const seen = new Set(); + const candidates = []; + const widenedCandidates = []; + + for (const el of all) {{ + if (!el || seen.has(el)) continue; + seen.add(el); + + const rect = el.getBoundingClientRect(); + if (!isVisible(el, rect)) continue; + + const tag = el.tagName.toLowerCase(); + const role = roleFromTag(el); + const text = elementText(el); + const name = accessibleName(el); + const testid = + el.getAttribute('data-testid') || + el.getAttribute('data-test') || + el.getAttribute('data-cy') || + ''; + const type = el.getAttribute('type') || ''; + const id = el.getAttribute('id') || ''; + const nameAttr = el.getAttribute('name') || ''; + const placeholder = el.getAttribute('placeholder') || ''; + const className = el.getAttribute('class') || ''; + const title = el.getAttribute('title') || ''; + + const searchable = `${{tag}} ${{role}} ${{type}} ${{id}} ${{nameAttr}} ${{className}} ${{text}} ${{name}} ${{placeholder}} ${{title}} ${{testid}}`.toLowerCase(); + const exactMatch = exactQueryMatches(searchable); + if (!exactMatch && !queryMatches(searchable)) continue; + + const kind = classifyControlKind(el, text, name); + const actionLikelihood = classifyActionLikelihood(el, searchable, kind); + const selectorHint = buildSelectorHint(el); + const actionable = isActionable(el, rect); + const matchCount = selectorHint + ? document.querySelectorAll(selectorHint).length + : 0; + const enabled = !( + el.disabled || + el.getAttribute('aria-disabled') === 'true' || + el.hasAttribute('inert') + ); + const clickable = Boolean( + actionable && enabled && selectorHint && matchCount === 1 + ); + const selectedSource = + el.getAttribute('aria-selected') === 'true' ? 'aria-selected' : + el.getAttribute('aria-checked') === 'true' ? 'aria-checked' : + el.getAttribute('aria-current') === 'true' ? 'aria-current' : + ['active', 'selected', 'checked'].includes(String(el.getAttribute('data-state') || '').toLowerCase()) ? 'data-state' : + /(^|\\s)(active|selected|checked)(\\s|$)/i.test(className) ? 'class' : + selectionSourceFromUrl(el, kind); + + const candidate = {{ + tag, + role, + action_likelihood: actionLikelihood, + region: classifyRegion(el, searchable), + kind: kind || actionLikelihood, + text, + accessible_name: name, + aria_label: normalize(el.getAttribute('aria-label') || ''), + testid: normalize(testid), + input_type: normalize(type), + name: normalize(nameAttr), + placeholder: normalize(placeholder), + href: normalize(el.getAttribute('href') || '', 180), + disabled: Boolean(el.disabled || el.getAttribute('aria-disabled') === 'true'), + selected: Boolean(selectedSource), + selected_source: selectedSource, + visible: true, + enabled, + actionable, + clickable, + match_count: matchCount, + generation_id: generationId, + bbox: [ + Math.round(rect.x), + Math.round(rect.y), + Math.round(rect.width), + Math.round(rect.height) + ], + selector_hint: clickable ? selectorHint : '', + selector_hint_validated: clickable, + score: scoreElement(el, rect, text, name, actionLikelihood) + }}; + (exactMatch ? candidates : widenedCandidates).push(candidate); + }} + + const selectedCandidates = candidates.length ? candidates : widenedCandidates; + selectedCandidates.sort((a, b) => {{ + if (b.score !== a.score) return b.score - a.score; + if (a.bbox[1] !== b.bbox[1]) return a.bbox[1] - b.bbox[1]; + return a.bbox[0] - b.bbox[0]; + }}); + + const elements = selectedCandidates.slice(0, maxItems).map((item, index) => {{ + const copy = {{ ...item }}; + copy.id = `e${{index + 1}}`; + delete copy.score; + return copy; + }}); + + return {{ + ok: true, + url: window.location.href, + title: document.title, + viewport: {{ + width: window.innerWidth, + height: window.innerHeight, + scroll_x: window.scrollX, + scroll_y: window.scrollY + }}, + query, + viewport_only: viewportOnly, + generation_id: generationId, + total_candidates: selectedCandidates.length, + query_widened: Boolean(exactQuery && !candidates.length && widenedCandidates.length), + returned: elements.length, + elements, + error: null + }}; + }}, params); +}} +""".strip() diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/metadata.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/metadata.py new file mode 100644 index 0000000000..898c9542e6 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/metadata.py @@ -0,0 +1,234 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The page script that reads the page, tab and scroll position state.""" + +from __future__ import annotations + + +def build_browser_state_metadata_js() -> str: + """Build Playwright code for fresh page, tab, and position state.""" + + return r""" +async (page) => { + const pages = page.context().pages(); + + const tabs = await Promise.all( + pages.map(async (tab, index) => { + let title = ''; + try { + title = await tab.title(); + } catch (_error) { + title = ''; + } + return { + index, + current: tab === page, + url: tab.url(), + title, + }; + }) + ); + + const pageMetadata = await page.evaluate(() => { + const root = document.documentElement; + const body = document.body; + const viewportWidth = Math.max(0, window.innerWidth || root?.clientWidth || 0); + const viewportHeight = Math.max(0, window.innerHeight || root?.clientHeight || 0); + const pageWidth = Math.max( + viewportWidth, + root?.scrollWidth || 0, + body?.scrollWidth || 0 + ); + const pageHeight = Math.max( + viewportHeight, + root?.scrollHeight || 0, + body?.scrollHeight || 0 + ); + const scrollX = Math.max(0, window.scrollX || window.pageXOffset || 0); + const scrollY = Math.max(0, window.scrollY || window.pageYOffset || 0); + + const pagePosition = { + viewport_width: viewportWidth, + viewport_height: viewportHeight, + page_width: pageWidth, + page_height: pageHeight, + scroll_x: scrollX, + scroll_y: scrollY, + pixels_above: scrollY, + pixels_below: Math.max(0, pageHeight - scrollY - viewportHeight), + pixels_left: scrollX, + pixels_right: Math.max(0, pageWidth - scrollX - viewportWidth), + }; + + const compact = (value, limit = 160) => String(value || '') + .replace(/\s+/g, ' ') + .trim() + .slice(0, limit); + const visible = (element) => { + if (!element || !element.isConnected) return false; + const rect = element.getBoundingClientRect(); + if (rect.width < 1 || rect.height < 1) return false; + const style = window.getComputedStyle(element); + return style.display !== 'none' && style.visibility !== 'hidden' && Number(style.opacity) !== 0; + }; + const elementKey = (element, index) => compact( + element.getAttribute('name') || + element.getAttribute('id') || + element.getAttribute('aria-label') || + element.getAttribute('placeholder') || + compact(element.innerText || element.textContent || '', 80) || + `${element.tagName.toLowerCase()}:${element.getAttribute('type') || index}`, + 120 + ).toLowerCase(); + + const formValues = []; + const formElements = Array.from(document.querySelectorAll( + 'input:not([type="password"]):not([type="hidden"]),select,textarea,[contenteditable="true"]' + )).slice(0, 80); + formElements.forEach((element, index) => { + if (!visible(element) || element.disabled) return; + const type = String(element.getAttribute('type') || '').toLowerCase(); + let value = ''; + if (type === 'checkbox' || type === 'radio') { + if (!element.checked) return; + value = element.value || true; + } else if (element.tagName.toLowerCase() === 'select') { + value = Array.from(element.selectedOptions || []).map((option) => compact(option.text || option.value)); + } else if (element.isContentEditable) { + value = compact(element.innerText || element.textContent || ''); + } else { + value = compact(element.value || ''); + } + if (value === '' || (Array.isArray(value) && value.length === 0)) return; + formValues.push({ key: elementKey(element, index), value }); + }); + + const selectedFilters = []; + const selectedCandidates = Array.from(document.querySelectorAll( + 'input:checked,select,[aria-selected="true"],[aria-checked="true"],' + + '[aria-current="true"],[data-state="active"],[data-state="selected"],' + + '[role="tab"].active,[role="tab"].selected,' + + '[class*="filter" i] .selected,[class*="filter" i] .active,' + + '[class*="facet" i] .selected,[class*="facet" i] .active,' + + '[class*="sort" i] .selected,[class*="sort" i] .active,' + + '[class*="price" i] .selected,[class*="price" i] .active,' + + '[class*="rating" i] .selected,[class*="rating" i] .active' + )).slice(0, 120); + selectedCandidates.forEach((element, index) => { + if (!visible(element)) return; + const descriptor = compact([ + element.getAttribute('id') || '', + element.getAttribute('class') || '', + element.getAttribute('role') || '', + element.getAttribute('aria-label') || '', + element.getAttribute('data-state') || '', + element.innerText || element.textContent || '', + element.parentElement?.getAttribute('class') || '', + ].join(' '), 300).toLowerCase(); + const nativeSelected = element.matches( + 'input:checked,select,[aria-checked="true"],[aria-selected="true"],[aria-current="true"]' + ); + const filterContext = /(filter|facet|sort|order|price|rating|score|star|category|筛选|排序|价格|评分|星级|分类|销量|综合|最新)/.test(descriptor); + if (!nativeSelected && !filterContext) return; + let value = compact( + element.getAttribute('aria-label') || element.innerText || element.textContent || element.value || '' + ); + if (element.tagName.toLowerCase() === 'select') { + value = Array.from(element.selectedOptions || []).map((option) => compact(option.text || option.value)).join('|'); + } + if (!value) return; + const selectionSource = + element.getAttribute('aria-selected') === 'true' ? 'aria-selected' : + element.getAttribute('aria-checked') === 'true' ? 'aria-checked' : + element.getAttribute('aria-current') === 'true' ? 'aria-current' : + ['active', 'selected'].includes(String(element.getAttribute('data-state') || '').toLowerCase()) + ? 'data-state' : + /(?:^|\s)(?:active|selected)(?:\s|$)/i.test(String(element.getAttribute('class') || '')) + ? 'class' : + element.matches('input:checked,select') ? 'native' : 'semantic-context'; + selectedFilters.push({ key: elementKey(element, index), value, source: selectionSource }); + }); + + let resultCount = null; + const countCandidates = Array.from(document.querySelectorAll( + '[data-total],[data-count],[aria-rowcount],[class*="result-count" i],[class*="total-count" i]' + )).slice(0, 40); + for (const element of countCandidates) { + const raw = element.getAttribute('data-total') || element.getAttribute('data-count') || + element.getAttribute('aria-rowcount') || compact(element.textContent || '', 120); + const match = String(raw || '').replace(/,/g, '').match(/\d+/); + if (!match) continue; + resultCount = Number(match[0]); + break; + } + const resultCandidates = new Set(); + for (const selector of [ + 'main article', 'main [role="article"]', 'main [role="row"]', + '[data-testid*="result" i]', '[data-test*="result" i]', + '[class*="search-result" i]', '[class*="result-item" i]' + ]) { + document.querySelectorAll(selector).forEach((element) => { + if (visible(element)) resultCandidates.add(element); + }); + } + if (resultCount === null && resultCandidates.size > 0) resultCount = resultCandidates.size; + const firstResult = Array.from(resultCandidates)[0]; + const firstResultText = firstResult ? compact(firstResult.innerText || firstResult.textContent || '', 240) : ''; + + const actionFeedback = []; + const feedbackCandidates = Array.from(document.querySelectorAll( + '[role="status"],[role="alert"],[aria-live]:not([aria-live="off"]),[class*="toast" i]' + )).slice(0, 40); + feedbackCandidates.forEach((element, index) => { + if (!visible(element)) return; + const value = compact(element.innerText || element.textContent || element.getAttribute('aria-label') || '', 240); + if (!value || !/(added|cart|basket|success|completed|已加入|购物车|成功|完成)/i.test(value)) return; + actionFeedback.push({ key: `feedback:${index}`, value }); + }); + + const commerceState = []; + const commerceCandidates = Array.from(document.querySelectorAll( + '[aria-label*="cart" i],[aria-label*="basket" i],[title*="cart" i],[title*="购物车" i],' + + '[class*="cart-count" i],[class*="cart-num" i],[data-testid*="cart" i]' + )).slice(0, 40); + commerceCandidates.forEach((element, index) => { + if (!visible(element)) return; + const value = compact( + element.getAttribute('aria-label') || element.getAttribute('title') || element.innerText || element.textContent || '', + 160 + ); + if (!value || !/\d/.test(value)) return; + commerceState.push({ key: elementKey(element, index), value }); + }); + + return { + page_position: pagePosition, + semantic_state: { + form_values: formValues.slice(0, 32), + selected_filters: selectedFilters.slice(0, 32), + result_count: resultCount, + first_result_text: firstResultText, + action_feedback: actionFeedback.slice(0, 4), + commerce_state: commerceState.slice(0, 8), + }, + }; + }); + + let title = ''; + try { + title = await page.title(); + } catch (_error) { + title = ''; + } + + return { + ok: true, + url: page.url(), + title, + tabs, + page_position: pageMetadata.page_position, + semantic_state: pageMetadata.semantic_state, + }; +} +""".strip() diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/results.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/results.py new file mode 100644 index 0000000000..8fdfcc63df --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/results.py @@ -0,0 +1,274 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Probe result enrichment and the evidence the probes yield.""" + +from __future__ import annotations + +from typing import Any, Dict, Optional + +from openjiuwen.harness.tools.browser_move.surfaces.browser.page_state import CARD_EVIDENCE_FIELDS +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.keys import ( + _BROWSER_PHASE_STATE_KEY, + _BROWSER_SELECTED_SORT_LABELS, +) + + +class _ProbeResultsMixin: + @classmethod + def _enrich_probe_result_contract( + cls, + session: Any, + inputs: Any, + tool_name: str, + tool_result: Dict[str, Any], + ) -> None: + if "probe_cards" not in str(tool_name or "").strip().lower(): + return + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + state = state if isinstance(state, dict) else {} + requested_count = int(state.get("requested_result_count") or 0) + observed_count = int(tool_result.get("observed_count") or 0) + diagnostics = tool_result.get("diagnostics") + diagnostics = dict(diagnostics) if isinstance(diagnostics, dict) else {} + diagnostics.update( + { + "requested_count": requested_count, + "observed_count": observed_count, + } + ) + + current_classes = cls._probe_card_classifications(tool_result.get("cards")) + conflicts = cls._probe_classification_conflicts(state, current_classes) + has_conflict = bool(conflicts or diagnostics.get("classification_conflict")) + if has_conflict: + diagnostics["classification_conflict"] = True + if conflicts: + diagnostics["classification_conflicts"] = conflicts + fallback_count = int(state.get("probe_classification_fallback_count") or 0) + if fallback_count < 1: + diagnostics["recommended_fallback"] = "one_precise_probe" + state["probe_classification_fallback_count"] = fallback_count + 1 + else: + diagnostics["recommended_fallback"] = "retain_unknown_classification" + diagnostics["precise_fallback_remaining"] = max( + 0, + 1 - int(state.get("probe_classification_fallback_count") or 0), + ) + tool_result["requested_count"] = requested_count + tool_result["observed_count"] = observed_count + tool_result["diagnostics"] = diagnostics + page_state = tool_result.get("page_state") + if isinstance(page_state, dict): + page_state["requested_count"] = requested_count + page_state["observed_count"] = observed_count + if has_conflict: + page_state["classification_conflict"] = True + state["last_card_probe"] = { + "url": str(tool_result.get("url") or "")[:500], + "generation_id": str(tool_result.get("generation_id") or ""), + "classifications": current_classes, + } + if session is not None: + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + cls._rewrite_tool_message_contract(inputs, tool_result) + + @staticmethod + def _probe_classification_conflicts( + state: Dict[str, Any], + current_classes: Dict[str, str], + ) -> list[str]: + previous_probe = state.get("last_card_probe") + previous_probe = previous_probe if isinstance(previous_probe, dict) else {} + previous_classes = previous_probe.get("classifications") + previous_classes = previous_classes if isinstance(previous_classes, dict) else {} + return [ + identity + for identity, classification in current_classes.items() + if identity in previous_classes and previous_classes[identity] != classification + ][:5] + + @staticmethod + def _probe_card_classifications(cards: Any) -> Dict[str, str]: + classifications: Dict[str, str] = {} + for card in cards if isinstance(cards, list) else []: + if not isinstance(card, dict): + continue + identity = " ".join( + str(card.get("primary_link") or card.get("href") or card.get("title") or "").split() + ).lower()[:300] + if not identity: + continue + classifications[identity] = ":".join( + ( + str(card.get("region") or ""), + str(card.get("kind") or ""), + str(bool(card.get("is_ad"))).lower(), + ) + ) + return classifications + + @classmethod + def _interactive_probe_evidence(cls, result: Dict[str, Any]) -> Dict[str, Any]: + elements = result.get("elements") + if not isinstance(elements, list) or not elements: + return {} + targets = [ + { + "target_id": str(item.get("target_id") or ""), + "generation_id": str(item.get("generation_id") or result.get("generation_id") or ""), + "kind": str(item.get("kind") or ""), + "region": str(item.get("region") or ""), + "name": str(item.get("accessible_name") or item.get("text") or "")[:160], + "actionable": bool(item.get("actionable")), + "selected": bool(item.get("selected")), + "selected_source": str(item.get("selected_source") or "")[:40], + } + for item in elements[:12] + if isinstance(item, dict) + ] + selected_sort = next( + (target for target in targets if target["selected"] and cls._is_sort_interactive_target(target)), + None, + ) + evidence = { + "kind": "interactive_probe", + "generation_id": str(result.get("generation_id") or ""), + "fields": [], + "target_count": len(targets), + "targets": targets, + } + if selected_sort and selected_sort["name"]: + evidence["fields"] = ["sort_state"] + evidence["values"] = {"sort_state": selected_sort["name"]} + evidence["provenance"] = { + "sort_state": { + "source": "browser_probe_interactives", + "selector": selected_sort["target_id"], + "raw_text": selected_sort["name"], + "generation_id": selected_sort["generation_id"], + "selection_source": selected_sort["selected_source"], + } + } + return evidence + + @staticmethod + def _is_sort_interactive_target(target: Dict[str, Any]) -> bool: + if str(target.get("kind") or "") in {"sort", "sort_tab", "sort_option"}: + return True + name = str(target.get("name") or "").strip().lower() + region = str(target.get("region") or "").strip().lower() + return bool( + region in {"sort", "filtering", "search_controls"} + or any(token in name for token in _BROWSER_SELECTED_SORT_LABELS) + ) + + @classmethod + def _card_probe_evidence(cls, result: Dict[str, Any]) -> Dict[str, Any]: + cards = result.get("cards") + if not isinstance(cards, list) or not cards: + return {} + compact_cards: list[Dict[str, Any]] = [] + fields: set[str] = set() + for card in cards: + compact = cls._compact_evidence_card(card, result) + if compact is None: + continue + metadata_fields = { + "generation_id", + "result_index", + "kind", + "provenance", + "field_status", + } + field_status = compact.get("field_status", {}) + for field_name in compact: + if field_name in metadata_fields: + continue + if field_status.get(field_name, "present") == "present": + fields.add(field_name) + compact_cards.append(compact) + if len(compact_cards) >= 5: + break + if not compact_cards: + return {} + return { + "kind": "card_probe", + "fields": sorted(fields), + "cards": compact_cards, + "observed_count": int(result.get("observed_count") or len(compact_cards)), + } + + @classmethod + def _compact_evidence_card( + cls, + card: Any, + result: Dict[str, Any], + ) -> Optional[Dict[str, Any]]: + if not isinstance(card, dict) or not cls._is_natural_evidence_card(card): + return None + compact: Dict[str, Any] = { + "generation_id": str(card.get("generation_id") or result.get("generation_id") or ""), + "result_index": card.get("result_index"), + "kind": card.get("kind"), + } + for field_name in CARD_EVIDENCE_FIELDS: + value = card.get(field_name) + if value in (None, "", [], {}): + continue + compact.setdefault(cls._canonical_field_name(field_name) or field_name, str(value)[:300]) + provenance = card.get("field_provenance") + if isinstance(provenance, dict): + compact["provenance"] = { + cls._canonical_field_name(field_name) or str(field_name): dict(item) + for field_name, item in list(provenance.items())[:20] + if isinstance(item, dict) + } + statuses = card.get("field_status") + if isinstance(statuses, dict): + compact["field_status"] = { + cls._canonical_field_name(field_name) or str(field_name): str(status) + for field_name, status in list(statuses.items())[:20] + if str(status).strip().lower() in {"present", "missing", "unknown"} + } + return compact + + @staticmethod + def _is_natural_evidence_card(card: Dict[str, Any]) -> bool: + region = str(card.get("region") or "") + kind = str(card.get("kind") or "") + if not (region or kind or "is_ad" in card): + return True + return ( + card.get("is_ad") is not True + and region + not in { + "hot_search", + "sidebar", + "account", + "chat", + "activity", + "sponsored_result", + "resource", + "commercial_module", + "ai_overview", + "question_module", + "related", + } + and kind + not in { + "hot_search", + "paid_column", + "promotion", + "activity", + "account", + "shop", + "chat", + "download_resource", + "commercial_module", + "ai_overview", + "people_also_ask", + "related_links", + "knowledge_panel", + } + ) diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/probe_semantics.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/semantics.py similarity index 97% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/probe_semantics.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/probes/semantics.py index a07ec73305..e6e0ada442 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/probe_semantics.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/probes/semantics.py @@ -10,8 +10,9 @@ from typing import Any, Dict, Mapping from urllib.parse import urlsplit -from .page_state import CARD_EVIDENCE_FIELDS -from .site_profiles import ( +from openjiuwen.harness.tools.browser_move.surfaces.browser.page_state import CARD_EVIDENCE_FIELDS + +from ..site_profiles import ( apply_site_card_semantics, deduplicate_site_cards, normalize_site_card_fields, @@ -153,8 +154,7 @@ def normalize_card_probe_payload(payload: Dict[str, Any]) -> Dict[str, Any]: { "identity": identity[:240], "classifications": [ - {"region": region, "kind": kind, "is_ad": is_ad} - for region, kind, is_ad in sorted(values) + {"region": region, "kind": kind, "is_ad": is_ad} for region, kind, is_ad in sorted(values) ], } for identity, values in classifications.items() diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/profile.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/profile.py new file mode 100644 index 0000000000..be97aa1527 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/profile.py @@ -0,0 +1,77 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""What the browser surface hands to the core.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import TYPE_CHECKING + +from ...contract import Effect +from ...contract.surface import SurfaceBundle, SurfaceProfile +from .capabilities import CAPABILITY_CATALOG, CAPABILITY_TOOLS, DEFAULT_BROWSER_CAPABILITIES +from .commands import ( + Back, + Drop, + FillForm, + GetConsole, + GetNetwork, + GetPdf, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) +from .config import BrowserSettings +from .helpers import EXTRA_TOOLS, activate_page, page_metadata, probe_for_policy, register_custom_actions +from .page_state import BrowserPageState +from .policy import policy_rules +from .renderer import BrowserRenderer +from .runner.agent import main_agent +from .semantic.state import SemanticStateTracker +from .tools import BROWSER_TOOL_SPECS, build_browser_worker_system_prompt + +if TYPE_CHECKING: + from .runner.settings import BrowserInstanceConfig + +_COMMANDS = (Navigate, Back, OpenContext, SelectOption, UploadFile, FillForm, Drop, GetPdf, GetConsole, GetNetwork) +_EFFECT_TIMEOUT_S = 240.0 +MESSAGES: Mapping[str, str] = { + "driver_restarted": "the browser restarted; take a fresh observation", + "driver_lost": "the browser connection was lost again; the task cannot continue", + "context_changed": "Call browser_snapshot again and use refs from generation {generation}.", + "retry": "Then retry the action (take a fresh browser_snapshot if the page changed).", +} + + +def create_browser_surface( + environ: Mapping[str, str], instance: BrowserInstanceConfig | None = None, settings: BrowserSettings | None = None +) -> SurfaceBundle: + """Surface factory (registry signature): ``settings`` replaces parsing ``environ``; ``instance`` is applied last.""" + settings = BrowserSettings.from_env(environ) if settings is None else settings + if instance is not None: + settings = settings.with_instance(instance) + profile = SurfaceProfile( + name="browser", + commands=_COMMANDS, + renderer=BrowserRenderer(), + tool_specs=BROWSER_TOOL_SPECS, + capability_catalog=CAPABILITY_CATALOG, + capability_tools=CAPABILITY_TOOLS, + extra_tools=EXTRA_TOOLS, + policy_rules=policy_rules(settings), + timeouts={effect: _EFFECT_TIMEOUT_S for effect in Effect}, + messages=MESSAGES, + settings=settings, + capabilities=DEFAULT_BROWSER_CAPABILITIES, + custom_actions=register_custom_actions, + page_metadata=page_metadata, + policy_probe=probe_for_policy, + activate=activate_page, + semantic_tracker=SemanticStateTracker, + worker_prompt=build_browser_worker_system_prompt, + page_state=BrowserPageState, + main_agent=main_agent, + ) + return SurfaceBundle(settings=settings, profile=profile) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/__init__.py new file mode 100644 index 0000000000..627c26fc0b --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/__init__.py @@ -0,0 +1,4 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Browser-only rails: anything that reads URLs, tabs or page content.""" diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/keys.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/keys.py new file mode 100644 index 0000000000..9032cbb83a --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/keys.py @@ -0,0 +1,80 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""State keys, statuses and field aliases the browser rail's modules share.""" + +from __future__ import annotations + +from typing import Dict, Iterable + +from openjiuwen.harness.tools.browser_move.surfaces.browser.working_context.store import BROWSER_TASK_STATE_KEY + +_BROWSER_PHASE_STATE_KEY = BROWSER_TASK_STATE_KEY +_BROWSER_TOOL_RUNTIME_STATE_KEY = "__browser_tool_runtime_state__" +_BROWSER_TASK_DEADLINE_KEY = "__browser_task_deadline__" +_BROWSER_TERMINAL_SYNTHESIS_KEY = "terminal_synthesis_started" +_BATCH_MUTATING_TARGET_OPS = frozenset( + {"click", "fill", "type", "autocomplete", "select_option", "set_checked", "select_visible_text"} +) +_BROWSER_TERMINAL_STATUSES = frozenset({"blocked", "partial", "completed"}) +_BROWSER_FIELD_ALIASES: Dict[str, tuple[str, ...]] = { + "title": ("title", "name", "标题", "名称", "电影", "商品"), + "url": ("url", "link", "href", "primary link", "primary_link", "链接", "网址"), + "price": ("price", "cost", "价格", "价钱", "费用"), + "rating": ("rating", "score", "评分", "星级"), + "product_rating": ("product rating", "item rating", "商品评分", "宝贝评分"), + "shop_rating": ("shop rating", "seller rating", "store rating", "店铺评分", "卖家评分"), + "author": ("author", "creator", "writer", "作者", "博主", "发布者", "回答者"), + "likes": ("likes", "like count", "upvotes", "点赞", "点赞数", "赞同", "赞同数", "获赞", "获赞数"), + "favorites": ("favorites", "favourites", "bookmarks", "收藏", "收藏数"), + "comments": ("comments", "comment count", "replies", "评论", "评论数", "回复数"), + "shop": ("shop", "store", "merchant", "seller", "店铺", "商家", "卖家"), + "duration": ("duration", "video length", "时长", "视频长度"), + "views": ("views", "view count", "play count", "播放量", "观看数"), + "exchange_rate": ("exchange rate", "conversion rate", "汇率", "兑换率"), + "high_temperature": ( + "high temperature", + "maximum temperature", + "highest temperature", + "最高温", + "最高气温", + "高温", + ), + "low_temperature": ( + "low temperature", + "minimum temperature", + "lowest temperature", + "最低温", + "最低气温", + "低温", + ), + "sort_state": ("sort state", "sort order", "ordering", "排序状态", "排序方式", "排序"), + "date": ("date", "日期", "哪天"), + "time": ("time", "时间", "几点"), + "address": ("address", "location", "地址", "地点"), +} +_BROWSER_SELECTED_SORT_LABELS = { + "sale": "销量", + "sales": "销量", + "volume": "销量", + "销量": "销量", + "price": "价格", + "价格": "价格", + "latest": "最新", + "newest": "最新", + "最新": "最新", + "comprehensive": "综合", + "relevance": "综合", + "default": "综合", + "综合": "综合", + "rating": "评分", + "score": "评分", + "评分": "评分", +} + + +def _contains_any_token(value: str, tokens: Iterable[str]) -> bool: + for token in tokens: + if token in value: + return True + return False diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/results.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/results.py new file mode 100644 index 0000000000..b36103024d --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/results.py @@ -0,0 +1,204 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Tool-message rewriting: outcome, page state and observation compaction.""" + +from __future__ import annotations + +import json +from typing import Any, Dict + +from openjiuwen.core.single_agent.rail.base import AgentCallbackContext +from openjiuwen.harness.tools.browser_move.contract import Effect, Observe, RunScript +from openjiuwen.harness.tools.browser_move.runtime.session import ToolKind +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import CommandRecord +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.keys import ( + _BATCH_MUTATING_TARGET_OPS, + _BROWSER_TOOL_RUNTIME_STATE_KEY, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.logging import write_browser_agent_audit_artifact + +_BROWSER_OBSERVATION_MESSAGE_MAX_CHARS = 12_000 +_BROWSER_PAGE_STATE_TAG = "browser_page_state" + + +class _ResultsMixin: + def _record_view(self, kind: ToolKind, record: CommandRecord | None, tool_result: Any) -> Dict[str, Any]: + """One dict the evidence machinery reads: a helper's payload as is, a command's record with its text.""" + data = getattr(tool_result, "data", None) + content = data.get("content") if isinstance(data, dict) else None + if isinstance(content, dict): + return content + if record is None: + if isinstance(data, dict): + return data + if isinstance(tool_result, dict): + return tool_result + success = getattr(tool_result, "success", None) + return {"ok": success is not False, "error": getattr(tool_result, "error", None), "result": data} + surface = record.surface + return { + "ok": record.ok, + "error": None if record.ok else (content or record.error), + "result": record.value if isinstance(record.action, RunScript) else content, + "url": str(getattr(surface, "url", "") or ""), + "title": str(getattr(surface, "title", "") or ""), + "generation_id": record.generation, + "page_state": self._runtime.export_page_state(), + } + + @classmethod + def _outcome( + cls, kind: ToolKind, record: CommandRecord | None, tool_result: Any, result: Dict[str, Any] + ) -> tuple[bool, bool, bool]: + if record is not None: + metadata = record.message_metadata + state_changed = bool(metadata["state_changed"]) or cls._partial_batch_mutated(result) + return bool(metadata["success"]), bool(metadata["executed"]), state_changed + success = getattr(tool_result, "success", None) + if success is None: + success = result.get("ok", True) is not False and not result.get("error") + executed = not kind.known + return bool(success), executed, bool(executed and success and not kind.read_only) + + @staticmethod + def _partial_batch_mutated(result: Dict[str, Any]) -> bool: + if str(result.get("status") or "").strip().lower() != "partial": + return False + for step in result.get("steps") or []: + if not isinstance(step, dict) or step.get("ok") is not True: + continue + if str(step.get("op") or "").strip().lower() in _BATCH_MUTATING_TARGET_OPS: + return True + return False + + def _tool_evidence_args(self, ctx: AgentCallbackContext, inputs: Any) -> Dict[str, Any]: + args = dict(self._coerce_tool_args(getattr(inputs, "tool_args", None))) + extra = getattr(ctx, "extra", None) + runtime_states = extra.get(_BROWSER_TOOL_RUNTIME_STATE_KEY) if isinstance(extra, dict) else None + call_state = runtime_states.get(self._tool_call_id(inputs)) if isinstance(runtime_states, dict) else None + if isinstance(call_state, dict) and call_state.get("evidence_fields"): + args["_runtime_evidence_fields"] = list(call_state["evidence_fields"]) + return args + + @staticmethod + def _rewrite_tool_message_contract(inputs: Any, tool_result: Dict[str, Any]) -> None: + tool_msg = getattr(inputs, "tool_msg", None) + content = getattr(tool_msg, "content", None) + if tool_msg is None or not isinstance(content, str): + return + try: + parsed = json.loads(content) + except (TypeError, ValueError): + return + if not isinstance(parsed, dict): + return + for key in ("requested_count", "observed_count", "diagnostics"): + parsed[key] = tool_result.get(key) + tool_msg.content = json.dumps(parsed, ensure_ascii=False, separators=(",", ":")) + + @staticmethod + def _set_tool_message_outcome( + inputs: Any, + *, + success: bool, + executed: bool, + state_changed: bool, + denied: bool, + effect: str | None = None, + ) -> None: + tool_msg = getattr(inputs, "tool_msg", None) + if tool_msg is None: + return + metadata = dict(tool_msg.metadata) if isinstance(tool_msg.metadata, dict) else {} + metadata.update( + { + "success": success, + "executed": executed, + "state_changed": state_changed, + "denied": denied, + } + ) + if effect is not None: + metadata["effect"] = effect + tool_msg.metadata = metadata + + @staticmethod + def _compact_large_observation_message(inputs: Any, kind: ToolKind, tool_name: str, result: Any) -> None: + """Bound one model-visible raw observation after runtime evidence extraction.""" + + tool_msg = getattr(inputs, "tool_msg", None) + content = getattr(tool_msg, "content", None) + if tool_msg is None or not isinstance(content, str): + return + observes = isinstance(kind.action, (Observe, RunScript)) + reads = kind.helper and kind.read_only and not kind.exempt_from_budget + if not (observes or reads): + return + if len(content) <= _BROWSER_OBSERVATION_MESSAGE_MAX_CHARS: + return + + audit = write_browser_agent_audit_artifact("large_browser_observation", result) + payload = { + "ok": bool(result.get("ok")) if isinstance(result, dict) else True, + "observation": "bounded_browser_tool_result", + "tool": str(tool_name or "").rsplit("_", 1)[-1], + "truncated": True, + "original_chars": len(content), + "preview_head": content[:6_000], + "preview_tail": content[-2_000:], + "note": ( + "Runtime evidence and targets were recorded before compaction. " + "Use browser_working_context and browser_state for subsequent actions." + ), + } + if audit: + payload["audit"] = audit + rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + if len(rendered) > _BROWSER_OBSERVATION_MESSAGE_MAX_CHARS: + payload["preview_head"] = content[:4_000] + payload["preview_tail"] = content[-1_000:] + rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + if len(rendered) > _BROWSER_OBSERVATION_MESSAGE_MAX_CHARS: + payload["preview_head"] = content[:2_000] + payload.pop("preview_tail", None) + rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + if len(rendered) > _BROWSER_OBSERVATION_MESSAGE_MAX_CHARS: + payload.pop("preview_head", None) + rendered = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) + tool_msg.content = rendered + + def _attach_page_state(self, inputs: Any, kind: ToolKind, result: Any) -> None: + if kind.helper: + return + observed = isinstance(kind.action, Observe) + if observed and kind.action.include_screenshot: + return + if not observed and kind.effect is not Effect.NAVIGATE: + return + + page_state = self._runtime.export_page_state() + tool_msg = getattr(inputs, "tool_msg", None) + content = getattr(tool_msg, "content", None) + if tool_msg is None or not isinstance(content, str): + return + if observed: + compact_observation = { + "ok": True, + "observation": "compact_page_state", + "page_state": page_state, + "audit": write_browser_agent_audit_artifact("direct_ax_observation", result), + } + tool_msg.content = json.dumps( + compact_observation, + ensure_ascii=False, + separators=(",", ":"), + ) + return + marker = ( + f"\n<{_BROWSER_PAGE_STATE_TAG}>" + f"{json.dumps(page_state, ensure_ascii=False, separators=(',', ':'))}" + f"</{_BROWSER_PAGE_STATE_TAG}>" + ) + if f"<{_BROWSER_PAGE_STATE_TAG}>" not in content: + tool_msg.content = content.rstrip() + marker diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/runtime_rail.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/runtime_rail.py new file mode 100644 index 0000000000..d297dc2a20 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/runtime_rail.py @@ -0,0 +1,1254 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The rail that keeps a browser task on course: the hooks, the lifecycle and the target normalisation.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import hashlib +import json +import re +import time +from typing import Any, Dict, Optional + +from openjiuwen.core.common.exception.codes import StatusCode +from openjiuwen.core.common.exception.errors import BaseError, build_error +from openjiuwen.core.common.logging.browser_context import ( + reset_browser_agent_log_context, + set_browser_agent_log_context, +) +from openjiuwen.core.common.utils.schema_utils import SchemaUtils +from openjiuwen.core.foundation.llm import ToolMessage +from openjiuwen.core.runner.callback.errors import AbortError +from openjiuwen.core.single_agent.prompts.builder import PromptSection +from openjiuwen.core.single_agent.rail.base import AgentCallbackContext, AgentRail +from openjiuwen.harness.prompts import resolve_language +from openjiuwen.harness.rails._multimodal import should_enable_read_image_multimodal +from openjiuwen.harness.tools.browser_move.contract import Click, Effect, PolicyDenied +from openjiuwen.harness.tools.browser_move.runtime.session import ToolKind, _tool_args +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import CommandRecord, command_record +from openjiuwen.harness.tools.browser_move.runtime.worker import TaskProgressState +from openjiuwen.harness.tools.browser_move.surfaces.browser.probes.results import _ProbeResultsMixin +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.keys import ( + _BROWSER_FIELD_ALIASES, + _BROWSER_PHASE_STATE_KEY, + _BROWSER_TASK_DEADLINE_KEY, + _BROWSER_TERMINAL_STATUSES, + _BROWSER_TERMINAL_SYNTHESIS_KEY, + _BROWSER_TOOL_RUNTIME_STATE_KEY, + _contains_any_token, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.results import _ResultsMixin +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.terminal import _TerminalMixin +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade import BrowserAgentRuntime +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.logging import ( + browser_agent_log_info, + browser_agent_log_warning, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.semantic.evidence import _EvidenceMixin +from openjiuwen.harness.tools.browser_move.surfaces.browser.semantic.phases import _PhaseMixin +from openjiuwen.harness.tools.browser_move.surfaces.browser.status_logging import ( + BrowserSubagentStatusLogger, + is_browser_subagent_status_log_enabled, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.tools import ( + _BROWSER_DATE_PICKER_GUIDANCE, + _BROWSER_DATE_PICKER_SECTION_NAME, + _BROWSER_IMAGE_CAPABILITY_GUIDANCE, + _BROWSER_IMAGE_CAPABILITY_SECTION_NAME, + _BROWSER_TASK_RULES_SECTION_NAME, + browser_task_rules, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.working_context.store import latest_browser_user_request + +_BROWSER_PROGRESS_STATE_KEY = "__browser_subagent_progress_state__" +_BROWSER_PROGRESS_TASK_KEY = "__browser_subagent_last_task__" + +_BROWSER_LOG_CONTEXT_TOKEN_KEY = "__browser_agent_log_context_token__" +_BROWSER_ACTION_GROUP_BY_CALL_KEY = "__browser_action_group_by_call__" +_BROWSER_ACTION_GROUP_RESULTS_KEY = "__browser_action_group_results__" +_BROWSER_SKIP_TOOL_CALLS_KEY = "_skip_tool_calls" +_BROWSER_INVOCATION_DEADLINE_KEY = "__browser_invocation_deadline__" +_BROWSER_SIMPLE_TASK_DEADLINE_S = 240.0 +_BROWSER_COMPLEX_TASK_DEADLINE_S = 600.0 +_BROWSER_MODEL_RETRY_LIMIT = 1 +_BROWSER_EXPLICIT_REF_RE = re.compile( + r"""^\s*\[?\s*ref\s*=\s*["']?([^"'\]\s]+)["']?\s*\]?\s*$""", + re.IGNORECASE, +) + +_BROWSER_REF_TARGET_KEYS = frozenset( + { + "target", + "startTarget", + "endTarget", + "ref", + "startRef", + "endRef", + } +) +_BROWSER_REF_ALIAS_KEYS = { + "ref": "target", + "startRef": "startTarget", + "endRef": "endTarget", +} +_BROWSER_TARGET_FIELDS = frozenset(_BROWSER_REF_ALIAS_KEYS.values()) +_BROWSER_RUNTIME_ARG_KEYS = frozenset({"field", "fields", "element"}) + + +class BrowserRuntimeRail(_TerminalMixin, _ResultsMixin, _ProbeResultsMixin, _PhaseMixin, _EvidenceMixin, AgentRail): + """Rail that makes direct browser sessions resumable and completion-aware.""" + + def __init__(self, runtime: BrowserAgentRuntime) -> None: + super().__init__() + self._runtime = runtime + self._agent: Any = None + self._status_logger = ( + BrowserSubagentStatusLogger(command_of=self._command_of) + if is_browser_subagent_status_log_enabled(runtime.core_settings) + else None + ) + browser_agent_log_info( + "[BROWSER_SUBAGENT_BOOT] enabled=%s logger=%s", + self._status_logger is not None, + type(self._status_logger).__name__ if self._status_logger is not None else None, + ) + + def _command_of(self, tool_name: str, tool_args: Any) -> str: + return self._command_name(self._runtime.tool_kind(tool_name, tool_args)) + + def _emit_status(self, method_name: str, ctx: AgentCallbackContext) -> None: + if self._status_logger is None: + return + method = getattr(self._status_logger, method_name, None) + if not callable(method): + browser_agent_log_warning( + "[BROWSER_SUBAGENT_ERROR] missing_status_method=%s", + method_name, + ) + return + try: + method(ctx) + except Exception as exc: + browser_agent_log_warning("[BROWSER_SUBAGENT_ERROR] method=%s error=%s", method_name, exc) + + async def before_invoke(self, ctx: AgentCallbackContext) -> None: + self._agent = ctx.agent + if isinstance(getattr(ctx, "extra", None), dict): + if self._runtime.dedicated_log: + ctx.extra[_BROWSER_LOG_CONTEXT_TOKEN_KEY] = set_browser_agent_log_context(True) + try: + timeout_s = max(1, int(self._runtime.service.guardrails.timeout_s)) + except (TypeError, ValueError, AttributeError): + timeout_s = 600 + ctx.extra.setdefault(_BROWSER_TASK_DEADLINE_KEY, time.monotonic() + timeout_s) + run_context = self._browser_run_context(ctx) + if run_context.get("browser_resume") is True: + ctx.extra["_browser_resume_requested"] = True + if ctx.steering_queue is None: + ctx.bind_steering_queue(asyncio.Queue()) + self._emit_status("before_invoke", ctx) + try: + await self._runtime.ensure_runtime_ready() + except Exception as exc: + raise AbortError(str(exc), cause=exc) from exc + await self._add_agent_parts(ctx) + session = getattr(ctx, "session", None) + if session is None: + return + self._hydrate_service_progress_from_session(session) + query = getattr(getattr(ctx, "inputs", None), "query", None) + task_text = str(query or "").strip() + if task_text: + state = self._ensure_task_state( + session, + task_text, + resume=bool(ctx.extra.get("_browser_resume_requested")), + ) + self._bind_shared_task_deadline(ctx, session, state) + + async def before_model_call(self, ctx: AgentCallbackContext) -> None: + self._emit_status("before_model_call", ctx) + if self._browser_run_context(ctx).get("browser_resume") is True: + ctx.extra["_browser_resume_requested"] = True + session = getattr(ctx, "session", None) + state: Dict[str, Any] = {} + if session is not None: + messages = getattr(getattr(ctx, "inputs", None), "messages", None) or [] + task_text = latest_browser_user_request(messages) + if task_text: + state = self._ensure_task_state( + session, + task_text, + resume=bool(ctx.extra.get("_browser_resume_requested")), + ) + self._bind_shared_task_deadline(ctx, session, state) + self._sync_semantic_progress(session) + loaded_state = session.get_state(_BROWSER_PHASE_STATE_KEY) + state = loaded_state if isinstance(loaded_state, dict) else {} + + if self._finish_if_task_deadline_exhausted(ctx, session, state): + return + + if self._prepare_terminal_synthesis(ctx, session, state): + return + + builder = getattr(ctx.agent, "system_prompt_builder", None) + if builder is None: + return + if str(state.get("status") or "").strip().lower() not in _BROWSER_TERMINAL_STATUSES: + builder.remove_section("browser_terminal_synthesis") + + image_input_supported = self._image_input_supported(ctx.agent) + builder.add_section( + PromptSection( + name=_BROWSER_IMAGE_CAPABILITY_SECTION_NAME, + content=_BROWSER_IMAGE_CAPABILITY_GUIDANCE[image_input_supported], + priority=85, + ) + ) + builder.add_section( + PromptSection( + name=_BROWSER_DATE_PICKER_SECTION_NAME, + content=_BROWSER_DATE_PICKER_GUIDANCE, + priority=85, + ) + ) + builder.add_section( + PromptSection(name=_BROWSER_TASK_RULES_SECTION_NAME, content=browser_task_rules(), priority=85) + ) + + async def after_model_call(self, ctx: AgentCallbackContext) -> None: + self._emit_status("after_model_call", ctx) + response = getattr(getattr(ctx, "inputs", None), "response", None) + tool_calls = list(getattr(response, "tool_calls", None) or []) + session = getattr(ctx, "session", None) + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + if ( + tool_calls + and isinstance(state, dict) + and str(state.get("status") or "").strip().lower() in _BROWSER_TERMINAL_STATUSES + ): + ctx.request_force_finish(self._structured_terminal_result(state)) + return + response_text = str(getattr(response, "content", "") or "") + if self._is_unfinished_model_protocol_response(state, tool_calls, response_text): + if self._request_model_protocol_recovery(ctx, session, state): + return + self._finish_model_protocol_failure(ctx, session, state) + return + call_ids = [str(tool_call.id) for tool_call in tool_calls if getattr(tool_call, "id", None)] + extra = getattr(ctx, "extra", None) + if not call_ids or not isinstance(extra, dict): + return + action_group_id = hashlib.sha256("\x1f".join(call_ids).encode("utf-8")).hexdigest()[:16] + group_by_call = extra.setdefault(_BROWSER_ACTION_GROUP_BY_CALL_KEY, {}) + for call_id in call_ids: + group_by_call[call_id] = action_group_id + extra.setdefault(_BROWSER_ACTION_GROUP_RESULTS_KEY, {})[action_group_id] = { + "expected": call_ids, + "completed": [], + "read_only": all( + self._is_read_only_recovery(self._runtime.tool_kind(call.name, call.arguments), call.arguments) + for call in tool_calls + ), + } + + async def on_model_exception(self, ctx: AgentCallbackContext) -> None: + self._emit_status("on_model_exception", ctx) + exception = getattr(ctx, "exception", None) + if not self._is_retryable_model_exception(exception): + return + extra = getattr(ctx, "extra", None) + deadline = float(extra.get(_BROWSER_TASK_DEADLINE_KEY) or 0.0) if isinstance(extra, dict) else 0.0 + remaining_s = deadline - time.monotonic() if deadline else 0.0 + if ctx.retry_attempt < _BROWSER_MODEL_RETRY_LIMIT and remaining_s > 5.0: + ctx.request_retry(delay_seconds=min(0.5, max(0.0, remaining_s - 5.0))) + return + ctx.request_force_finish(self._structured_model_failure(ctx, exception)) + + @staticmethod + def _is_retryable_model_exception(exception: Optional[Exception]) -> bool: + if isinstance(exception, (asyncio.TimeoutError, TimeoutError, ConnectionError)): + return True + message = str(exception or "").lower() + retryable_tokens = ( + "timeout", + "timed out", + "429", + "500", + "502", + "503", + "504", + "upstream", + "rate limit", + "service unavailable", + ) + return _contains_any_token(message, retryable_tokens) + + async def before_tool_call(self, ctx: AgentCallbackContext) -> None: + self._emit_status("before_tool_call", ctx) + if self._handle_progress_tool_alias(ctx): + return + session = getattr(ctx, "session", None) + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + state = state if isinstance(state, dict) else {} + self._bind_shared_task_deadline(ctx, session, state) + if self._finish_if_task_deadline_exhausted(ctx, session, state): + return + try: + self._prepare_tool_call(ctx) + except (ValueError, BaseError) as exc: + self._deny_tool_call(ctx, exc) + + def _handle_progress_tool_alias(self, ctx: AgentCallbackContext) -> bool: + inputs = getattr(ctx, "inputs", None) + tool_name = str(getattr(inputs, "tool_name", "") or "").strip().lower() + if tool_name != "browser_progress" and not tool_name.endswith("_browser_progress"): + return False + session = getattr(ctx, "session", None) + if session is None: + self._deny_tool_call(ctx, ValueError("browser_progress requires an active browser task session")) + return True + payload = self._coerce_tool_args(getattr(inputs, "tool_args", None)) + state = session.get_state(_BROWSER_PHASE_STATE_KEY) + state = state if isinstance(state, dict) else {} + reported_status = str(payload.get("status") or "").strip().lower() + if not reported_status: + missing = self._missing_completion_requirements(state) + blockers = [str(item) for item in state.get("blockers") or [] if str(item).strip()] + runtime_ready = bool( + state.get("structured_evidence") or state.get("evidence_slots") or state.get("field_coverage") + ) + if not blockers and not missing and runtime_ready: + payload["status"] = "completed" + reported_status = "completed" + elif missing and not self._missing_evidence_slots(state) and self._unavailable_evidence_slots(state): + payload["status"] = "partial" + reported_status = "partial" + else: + self._deny_tool_call( + ctx, + ValueError( + "browser_progress did not include a terminal status and runtime evidence is " + "not yet complete. Continue this browser task and collect only the unresolved " + "requirements before finishing." + ), + ) + return True + if reported_status not in _BROWSER_TERMINAL_STATUSES: + self._deny_tool_call( + ctx, + ValueError( + "<browser_progress> is a final text protocol. Continue with a real browser tool " + "until status is completed, partial, or blocked." + ), + ) + return True + self._apply_worker_progress_to_task_state(session, payload, "") + state = session.get_state(_BROWSER_PHASE_STATE_KEY) + state = state if isinstance(state, dict) else {} + ctx.request_force_finish(self._structured_terminal_result(state)) + return True + + def _prepare_tool_call(self, ctx: AgentCallbackContext) -> None: + inputs = getattr(ctx, "inputs", None) + tool_name = str(getattr(inputs, "tool_name", "") or "") + if tool_name.strip().lower() == "browser_progress": + raise ValueError( + "<browser_progress> is a text protocol appended to the final response, not a callable tool." + ) + tool_args = getattr(inputs, "tool_args", None) + kind = self._runtime.tool_kind(tool_name, tool_args) + if kind.known and not kind.helper and kind.fields & _BROWSER_TARGET_FIELDS: + normalized_args = self._normalize_target_args(tool_args, kind.fields) + if normalized_args is not tool_args: + inputs.tool_args = tool_args = normalized_args + kind = self._runtime.tool_kind(tool_name, tool_args) + tool_name, tool_args, kind = self._rewrite_primary_link_click(inputs, tool_name, tool_args, kind) + self._reject_unresolved_targets(tool_args) + evidence_fields = self._runtime_evidence_fields(tool_args) + if kind.helper and not self._image_input_supported(getattr(ctx, "agent", None)): + steps = _tool_args(tool_args).get("steps") + if any( + isinstance(step, dict) and str(step.get("op") or "").strip().lower() == "screenshot" + for step in (steps if isinstance(steps, list) else []) + ): + raise ValueError( + "Screenshot input is unavailable for this browser agent. " + "Remove the screenshot batch step and use DOM probes or " + "structured extraction." + ) + self._validate_model_tool_args(ctx, tool_name, tool_args) + session = getattr(ctx, "session", None) + self._sync_semantic_progress(session) + group = self._tool_action_group(ctx) + attempts = self._phase_attempts(session) + action_class = self._consume_phase_budget( + session, + kind, + tool_name, + tool_args, + current_page_state=self._runtime.export_page_state(), + replan_group_admitted=bool(group.get("read_only") and group.get("trial_admitted")), + ) + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + if isinstance(state, dict) and state.get("replan_trial_pending") and group.get("read_only"): + group["trial_admitted"] = True + extra = getattr(ctx, "extra", None) + if isinstance(extra, dict): + tool_call_id = self._tool_call_id(inputs) + tool_runtime_state = extra.setdefault(_BROWSER_TOOL_RUNTIME_STATE_KEY, {}) + tool_runtime_state[tool_call_id] = { + "started_at": time.perf_counter(), + "action_class": action_class, + "evidence_fields": evidence_fields, + "generation": self._runtime.generation_id, + "charged": self._phase_attempts(session) > attempts, + } + + async def _add_agent_parts(self, ctx: AgentCallbackContext) -> None: + agent = getattr(ctx, "agent", None) + ability_manager = getattr(agent, "ability_manager", None) + if ability_manager is None: + return + parts = self._runtime.agent_parts( + image_support=self._image_input_supported(agent), language=resolve_language(agent.deep_config.language) + ) + for tool in parts.tools: + ability_manager.add_ability(tool.card, tool) + + @staticmethod + def _tool_action_group(ctx: AgentCallbackContext) -> Dict[str, Any]: + extra = getattr(ctx, "extra", None) or {} + call_id = BrowserRuntimeRail._tool_call_id(getattr(ctx, "inputs", None)) + group_id = (extra.get(_BROWSER_ACTION_GROUP_BY_CALL_KEY) or {}).get(call_id) + return (extra.get(_BROWSER_ACTION_GROUP_RESULTS_KEY) or {}).get(group_id, {}) + + @classmethod + def _validate_model_tool_args(cls, ctx: AgentCallbackContext, tool_name: str, tool_args: Any) -> None: + """Validate locally before reserving a browser strategy trial.""" + manager = getattr(getattr(ctx, "agent", None), "ability_manager", None) + get_tool = getattr(manager, "get", None) + tool = get_tool(tool_name) if callable(get_tool) else None + if callable(get_tool) and tool is None: + raise build_error(StatusCode.AGENT_TOOL_NOT_FOUND, error_msg=f"Tool {tool_name} is not registered") + schema = getattr(tool, "input_params", None) + if isinstance(schema, dict): + SchemaUtils.validate_with_schema(cls._coerce_tool_args(tool_args), schema) + + def _runtime_evidence_fields(self, tool_args: Any) -> list[str]: + args = self._coerce_tool_args(tool_args) + fields: set[str] = set() + for value in (args.get("field"), args.get("fields")): + values = value if isinstance(value, list) else [value] + for item in values: + if isinstance(item, str): + canonical = self._canonical_field_name(item) + if canonical in _BROWSER_FIELD_ALIASES: + fields.add(canonical) + + target_id = str(args.get("target_id") or args.get("target") or "").strip() + if target_id.startswith("t_g"): + try: + target = self._runtime.resolve_model_target_id(target_id) + except ValueError: + target = None + if target is not None and target.field_name in _BROWSER_FIELD_ALIASES: + fields.add(target.field_name) + return sorted(fields) + + def _rewrite_primary_link_click( + self, + inputs: Any, + tool_name: str, + tool_args: Any, + kind: ToolKind, + ) -> tuple[str, Any, ToolKind]: + if not isinstance(kind.action, Click): + return tool_name, tool_args, kind + primary_link = self._runtime.resolve_primary_link(tool_args) + if not isinstance(primary_link, str) or not primary_link: + return tool_name, tool_args, kind + inputs.tool_name = re.sub( + r"browser_click$", + "browser_navigate", + tool_name, + flags=re.IGNORECASE, + ) + inputs.tool_args = {"url": primary_link} + return inputs.tool_name, inputs.tool_args, self._runtime.tool_kind(inputs.tool_name, inputs.tool_args) + + @staticmethod + def _reject_unresolved_targets(tool_args: Any) -> None: + for key in _BROWSER_REF_TARGET_KEYS: + value = str(_tool_args(tool_args).get(key) or "").strip() + if value.startswith("t_g"): + raise ValueError(f"PageState target_id {value} has no executable locator.") + + async def on_tool_exception(self, ctx: AgentCallbackContext) -> None: + self._emit_status("on_tool_exception", ctx) + + async def after_tool_call(self, ctx: AgentCallbackContext) -> None: + self._emit_status("after_tool_call", ctx) + inputs = getattr(ctx, "inputs", None) + tool_name = str(getattr(inputs, "tool_name", "") or "").strip() + tool_args = getattr(inputs, "tool_args", None) + tool_result = getattr(inputs, "tool_result", None) + if self._is_denied(tool_result): + self._set_tool_message_outcome( + inputs, + success=False, + executed=False, + state_changed=False, + denied=True, + ) + return + + record = command_record(tool_result) + kind = self._kind_after(self._runtime.tool_kind(tool_name, tool_args), record) + result = self._record_view(kind, record, tool_result) + success, executed, state_changed = self._outcome(kind, record, tool_result, result) + session = getattr(ctx, "session", None) + if success and "probe_cards" in tool_name: + self._enrich_probe_result_contract(session, inputs, tool_name, result) + self._set_tool_message_outcome( + inputs, + success=success, + executed=executed, + state_changed=state_changed, + denied=False, + effect=kind.effect.value if kind.known and not kind.exempt_from_budget else None, + ) + if success or state_changed: + self._attach_page_state(inputs, kind, result) + if session is None: + self._compact_large_observation_message(inputs, kind, tool_name, result) + self._mark_action_group_call_completed(ctx, {}) + return + evidence_args = self._tool_evidence_args(ctx, inputs) + progress_delta = self._record_phase_result( + session, + kind, + tool_name, + evidence_args, + result, + progressed=self._progressed(ctx, inputs, record), + charged=self._charged(ctx, inputs), + ) + progress_delta.update( + { + "executed": executed, + "denied": False, + "state_changed": state_changed, + "ambiguous": bool(not success and state_changed), + } + ) + action_class, elapsed_ms = self._consume_tool_call_timing(ctx, inputs) + recovered = self._record_recent_action( + session, + kind=kind, + tool_name=tool_name, + tool_args=tool_args, + tool_result=result, + action_class=str(action_class or ""), + elapsed_ms=elapsed_ms, + progress_delta=progress_delta, + ) + if recovered: + self._runtime.acknowledge_semantic_replan() + self._compact_large_observation_message(inputs, kind, tool_name, result) + if not kind.progress: + self._mark_action_group_call_completed(ctx, progress_delta) + return + self._persist_service_progress_to_session(session) + self._mark_action_group_call_completed(ctx, progress_delta) + + @staticmethod + def _is_denied(tool_result: Any) -> bool: + if isinstance(tool_result, PolicyDenied): + return True + return isinstance(tool_result, dict) and ( + tool_result.get("denied") is True or tool_result.get("executed") is False + ) + + @classmethod + def _progressed(cls, ctx: AgentCallbackContext, inputs: Any, record: CommandRecord | None) -> bool: + """An action whose record shows a new context, or a generation other than the one it started in.""" + if record is None or not record.ok or record.effect not in (Effect.ACT, Effect.NAVIGATE): + return False + extra = getattr(ctx, "extra", None) + states = extra.get(_BROWSER_TOOL_RUNTIME_STATE_KEY) if isinstance(extra, dict) else None + call_state = states.get(cls._tool_call_id(inputs)) if isinstance(states, dict) else None + before = call_state.get("generation") if isinstance(call_state, dict) else None + return record.context_changed or (before is not None and record.generation != before) + + @classmethod + def _charged(cls, ctx: AgentCallbackContext, inputs: Any) -> bool: + """Whether the call cost a phase attempt; a repeated page script does not.""" + extra = getattr(ctx, "extra", None) + states = extra.get(_BROWSER_TOOL_RUNTIME_STATE_KEY) if isinstance(extra, dict) else None + call_state = states.get(cls._tool_call_id(inputs)) if isinstance(states, dict) else None + return call_state.get("charged", True) if isinstance(call_state, dict) else True + + @staticmethod + def _kind_after(kind: ToolKind, record: CommandRecord | None) -> ToolKind: + if record is None: + return kind + return dataclasses.replace(kind, action=record.action, effect=record.effect, known=True) + + def _consume_tool_call_timing(self, ctx: AgentCallbackContext, inputs: Any) -> tuple[str, int]: + extra = getattr(ctx, "extra", None) + tool_runtime_state = extra.get(_BROWSER_TOOL_RUNTIME_STATE_KEY, {}) if isinstance(extra, dict) else {} + call_state = ( + tool_runtime_state.pop(self._tool_call_id(inputs), {}) if isinstance(tool_runtime_state, dict) else {} + ) + if isinstance(extra, dict) and not tool_runtime_state: + extra.pop(_BROWSER_TOOL_RUNTIME_STATE_KEY, None) + started_at = call_state.get("started_at") if isinstance(call_state, dict) else None + action_class = str(call_state.get("action_class") or "") if isinstance(call_state, dict) else "" + elapsed_ms = int(max(0.0, (time.perf_counter() - started_at) * 1000)) if started_at else 0 + return action_class, elapsed_ms + + @staticmethod + def _tool_call_id(inputs: Any) -> str: + tool_call = getattr(inputs, "tool_call", None) + call_id = str(getattr(tool_call, "id", "") or "").strip() + return call_id or f"tool-context-{id(inputs)}" + + @classmethod + def _denial_code(cls, message: str) -> str: + normalized = str(message or "").lower() + if "already" in normalized and "browser task" in normalized: + return "browser_task_terminal" + if "semantic" in normalized or "replan" in normalized: + return "browser_replan_required" + if "budget exhausted" in normalized: + return "browser_phase_budget_exhausted" + if "browser_progress" in normalized: + return "browser_progress_is_text_protocol" + if "screenshot input" in normalized: + return "browser_image_input_unavailable" + if "known url" in normalized: + return "browser_direct_navigation_required" + return "browser_action_denied" + + def _deny_tool_call(self, ctx: AgentCallbackContext, exc: Exception) -> None: + inputs = getattr(ctx, "inputs", None) + session = getattr(ctx, "session", None) + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + state = state if isinstance(state, dict) else {} + status = str(state.get("status") or "in_progress").strip().lower() + payload = { + "ok": False, + "status": "denied", + "executed": False, + "state_changed": False, + "denied": True, + "error": { + "code": self._denial_code(str(exc)), + "message": str(exc), + "terminal": status in _BROWSER_TERMINAL_STATUSES, + "replan_required": bool(state.get("replan_required")), + }, + "runtime_state": { + "task_status": status, + "current_phase": state.get("current_phase"), + "next_action_class": state.get("next_action_class"), + "blockers": list(state.get("blockers") or [])[:10], + }, + } + target_match = re.search(r"\bt_g\d+_\d+\b", str(exc)) + if target_match is not None: + payload["runtime_state"].update(self._runtime.target_recovery_state(target_match.group(0))) + call_id = self._tool_call_id(inputs) + inputs.tool_result = payload + inputs.tool_msg = ToolMessage( + content=json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + tool_call_id=call_id, + metadata={ + "executed": False, + "state_changed": False, + "denied": True, + }, + ) + extra = getattr(ctx, "extra", None) + if isinstance(extra, dict): + extra.setdefault(_BROWSER_SKIP_TOOL_CALLS_KEY, {})[call_id] = True + self._mark_action_group_call_completed( + ctx, + { + "executed": False, + "state_changed": False, + "denied": True, + "new_evidence_fields": [], + }, + ) + + def _mark_action_group_call_completed( + self, + ctx: AgentCallbackContext, + progress_delta: Dict[str, Any], + ) -> None: + extra = getattr(ctx, "extra", None) + if not isinstance(extra, dict): + return + call_id = self._tool_call_id(getattr(ctx, "inputs", None)) + group_by_call = extra.get(_BROWSER_ACTION_GROUP_BY_CALL_KEY) + group_id = group_by_call.pop(call_id, "") if isinstance(group_by_call, dict) else "" + groups = extra.get(_BROWSER_ACTION_GROUP_RESULTS_KEY) + group = groups.get(group_id) if isinstance(groups, dict) and group_id else None + if isinstance(group, dict): + completed = group.setdefault("completed", []) + if call_id not in completed: + completed.append(call_id) + if progress_delta.get("executed") is False: + denied = group.setdefault("denied", []) + if call_id not in denied: + denied.append(call_id) + elif progress_delta.get("executed") is True: + executed = group.setdefault("executed", []) + if call_id not in executed: + executed.append(call_id) + if progress_delta.get("state_changed") is True: + group["state_changed"] = True + evidence_fields = group.setdefault("evidence_fields", []) + for field_name in progress_delta.get("new_evidence_fields") or []: + if field_name not in evidence_fields: + evidence_fields.append(field_name) + if set(completed) >= set(group.get("expected") or []): + group["ready"] = True + session = getattr(ctx, "session", None) + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + if isinstance(state, dict): + state["last_action_group"] = { + "action_group_id": group_id, + "call_count": len(group.get("expected") or []), + "executed_count": len(group.get("executed") or []), + "denied_count": len(group.get("denied") or []), + "state_changed": bool(group.get("state_changed")), + "evidence_fields": evidence_fields[:32], + } + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + groups.pop(group_id, None) + if not groups: + extra.pop(_BROWSER_ACTION_GROUP_RESULTS_KEY, None) + if isinstance(group_by_call, dict) and not group_by_call: + extra.pop(_BROWSER_ACTION_GROUP_BY_CALL_KEY, None) + + async def after_invoke(self, ctx: AgentCallbackContext) -> None: + try: + session = getattr(ctx, "session", None) + result = getattr(getattr(ctx, "inputs", None), "result", None) + if session is None or not isinstance(result, dict): + return + + session_id = session.get_session_id() + self._hydrate_service_progress_from_session(session) + output_text = str(result.get("output") or "") + clean_output, progress_payload = self._extract_progress_payload(output_text) + if clean_output != output_text: + result["output"] = clean_output + + if progress_payload is not None: + self._apply_worker_progress_to_task_state(session, progress_payload, clean_output) + if self._finalize_terminal_invoke(session, result, clean_output): + return + progress_state = self._load_progress_state(session) + parsed_progress = self._build_progress_result(progress_state, clean_output) + self._runtime.service.set_progress_state(session_id, progress_state) + exported = progress_state.to_dict() + if self._runtime.service.should_treat_as_completed(parsed_progress) and clean_output.strip(): + result["result_type"] = "answer" + result["progress_state"] = exported + self._clear_progress_state(session) + return + + failure_summary = self._runtime.service.build_failure_summary( + task=self._load_task_text(session), + error=str(parsed_progress.get("error") or "browser_task_incomplete"), + page_url=progress_state.last_page_url if progress_state is not None else "", + page_title=progress_state.last_page_title if progress_state is not None else "", + final=clean_output, + screenshot=progress_state.last_screenshot if progress_state is not None else None, + attempt=1, + progress_state=progress_state, + ) + result["result_type"] = "error" + result["failure_summary"] = failure_summary + result["progress_state"] = exported + result["output"] = failure_summary if not clean_output else f"{clean_output}\n\n{failure_summary}" + self._persist_service_progress_to_session(session) + return + + if self._is_max_iteration_result(result): + if self._terminalize_invoke_failure(session, "max_iterations_reached"): + if self._finalize_terminal_invoke(session, result, clean_output): + return + progress_state = self._load_progress_state(session) + self._runtime.service.set_progress_state(session_id, progress_state) + failure_summary = self._runtime.service.build_failure_summary( + task=self._load_task_text(session), + error="max_iterations_reached", + page_url=progress_state.last_page_url if progress_state is not None else "", + page_title=progress_state.last_page_title if progress_state is not None else "", + final=clean_output or output_text, + screenshot=progress_state.last_screenshot if progress_state is not None else None, + attempt=1, + progress_state=progress_state, + ) + result["failure_summary"] = failure_summary + result["progress_state"] = progress_state.to_dict() + result["output"] = failure_summary + self._persist_service_progress_to_session(session) + return + + if str(result.get("result_type", "")).lower() == "answer": + task_state = session.get_state(_BROWSER_PHASE_STATE_KEY) + if not isinstance(task_state, dict) or not task_state: + self._clear_progress_state(session) + return + self._apply_worker_progress_to_task_state( + session, + {"status": "completed"}, + clean_output, + ) + if self._finalize_terminal_invoke(session, result, clean_output): + return + progress_state = self._load_progress_state(session) + exported = progress_state.to_dict() + parsed_progress = self._build_progress_result(progress_state, clean_output) + if self._runtime.service.should_treat_as_completed(parsed_progress) and clean_output.strip(): + result["progress_state"] = exported + self._clear_progress_state(session) + return + failure_summary = self._runtime.service.build_failure_summary( + task=self._load_task_text(session), + error="empty_or_unverified_browser_result", + page_url=progress_state.last_page_url, + page_title=progress_state.last_page_title, + final=clean_output, + screenshot=progress_state.last_screenshot, + attempt=1, + progress_state=progress_state, + ) + result["result_type"] = "error" + result["failure_summary"] = failure_summary + result["progress_state"] = exported + result["output"] = failure_summary if not clean_output else f"{clean_output}\n\n{failure_summary}" + self._persist_service_progress_to_session(session) + return + + progress_state = self._load_progress_state(session) + if str(result.get("result_type") or "").lower() == "error": + driver_session = self._runtime.session + if driver_session is not None and driver_session.lost is not None: + self._terminalize_invoke_failure(session, "browser_connection_lost", blocked=True) + else: + self._terminalize_invoke_failure(session, "browser_execution_error") + if self._finalize_terminal_invoke(session, result, clean_output): + return + exported = progress_state.to_dict() if not progress_state.is_empty() else None + if exported: + self._runtime.service.set_progress_state(session_id, progress_state) + result["progress_state"] = exported + self._persist_service_progress_to_session(session) + + finally: + self._emit_status("after_invoke", ctx) + extra = getattr(ctx, "extra", None) + if isinstance(extra, dict): + token = extra.pop(_BROWSER_LOG_CONTEXT_TOKEN_KEY, None) + if token is not None: + reset_browser_agent_log_context(token) + + def _finalize_terminal_invoke( + self, + session: Any, + result: Dict[str, Any], + model_summary: str, + ) -> bool: + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + if not isinstance(state, dict): + return False + status = str(state.get("status") or "").strip().lower() + if status not in _BROWSER_TERMINAL_STATUSES: + return False + output, payload = self._render_authoritative_terminal_output(state, model_summary) + progress_state = self._load_progress_state(session) + result["output"] = output + result["result_type"] = "answer" if status == "completed" else "error" + result["error"] = None if status == "completed" else "browser_task_incomplete" + result["progress_state"] = progress_state.to_dict() + result["authoritative_browser_result"] = payload + if status == "completed": + self._clear_progress_state(session) + else: + result["failure_summary"] = self._runtime.service.build_failure_summary( + task=self._load_task_text(session), + error="browser_task_incomplete", + page_url=progress_state.last_page_url, + page_title=progress_state.last_page_title, + final="", + screenshot=progress_state.last_screenshot, + attempt=1, + progress_state=progress_state, + ) + self._persist_service_progress_to_session(session) + return True + + def _normalize_target_args(self, tool_args: Any, fields: frozenset[str] = frozenset()) -> Any: + """Rewrite ref aliases, resolve PageState target ids and drop the runtime's own keys the tool does not take.""" + original_is_json = isinstance(tool_args, str) + if original_is_json: + try: + parsed = json.loads(tool_args) + except ValueError: + return tool_args + elif isinstance(tool_args, dict): + parsed = dict(tool_args) + else: + return tool_args + if not isinstance(parsed, dict): + return tool_args + + parsed, changed = self._normalize_target_payload(parsed) + for key in _BROWSER_RUNTIME_ARG_KEYS - fields if fields else (): + if key in parsed: + parsed.pop(key) + changed = True + if not changed: + return tool_args + return json.dumps(parsed, ensure_ascii=False) if original_is_json else parsed + + def _normalize_target_payload( + self, + tool_args: Dict[str, Any], + ) -> tuple[Dict[str, Any], bool]: + parsed = dict(tool_args) + changed = self._normalize_target_aliases(parsed) + for key in _BROWSER_REF_TARGET_KEYS: + value = str(parsed.get(key) or "").strip() + if not value.startswith("t_g"): + continue + target = self._runtime.resolve_model_target_id(value) + executable_target = str(target.ref or target.selector or "").strip() + if not executable_target: + continue + parsed[key] = executable_target + element_key = { + "target": "element", + "startTarget": "startElement", + "endTarget": "endElement", + }.get(key) + if element_key and not str(parsed.get(element_key) or "").strip(): + parsed[element_key] = str(target.text or target.kind or target.target_id)[:200] + changed = True + + fields = parsed.get("fields") + if isinstance(fields, list): + normalized_fields = [] + for field in fields: + if not isinstance(field, dict): + normalized_fields.append(field) + continue + nested, nested_changed = self._normalize_target_payload(field) + normalized_fields.append(nested) + changed = changed or nested_changed + parsed["fields"] = normalized_fields + return parsed, changed + + @staticmethod + def _normalize_target_aliases(parsed: Dict[str, Any]) -> bool: + changed = False + + if "target_id" in parsed: + parsed["target"] = parsed.pop("target_id") + changed = True + if "generation_id" in parsed: + parsed.pop("generation_id", None) + changed = True + + for alias, target_key in _BROWSER_REF_ALIAS_KEYS.items(): + if alias not in parsed: + continue + if target_key not in parsed: + parsed[target_key] = parsed[alias] + parsed.pop(alias, None) + changed = True + + for key in _BROWSER_REF_TARGET_KEYS: + value = parsed.get(key) + if not isinstance(value, str): + continue + match = _BROWSER_EXPLICIT_REF_RE.fullmatch(value) + if match is None: + continue + parsed[key] = match.group(1) + changed = True + return changed + + @staticmethod + def _load_task_text(session: Any) -> str: + if session is None: + return "" + return str(session.get_state(_BROWSER_PROGRESS_TASK_KEY) or "").strip() + + @staticmethod + def _load_progress_state(session: Any) -> TaskProgressState: + if session is None: + return TaskProgressState() + task_state = session.get_state(_BROWSER_PHASE_STATE_KEY) + if isinstance(task_state, dict) and task_state: + return TaskProgressState.from_task_state(task_state) + return TaskProgressState.from_dict(session.get_state(_BROWSER_PROGRESS_STATE_KEY)) + + def _hydrate_service_progress_from_session(self, session: Any) -> TaskProgressState: + session_id = session.get_session_id() + progress_state = self._load_progress_state(session) + if progress_state.is_empty(): + self._runtime.service.clear_progress_state(session_id) + return progress_state + self._runtime.service.set_progress_state(session_id, progress_state) + return progress_state + + def _persist_service_progress_to_session(self, session: Any) -> None: + if session is None: + return + session_id = session.get_session_id() + progress_state = self._load_progress_state(session) + if progress_state.is_empty(): + self._runtime.service.clear_progress_state(session_id) + exported: Dict[str, Any] = {} + else: + self._runtime.service.set_progress_state(session_id, progress_state) + exported = progress_state.to_dict() + session.update_state( + { + _BROWSER_PROGRESS_STATE_KEY: exported, + } + ) + + def _ensure_task_state(self, session: Any, task: str, *, resume: bool = False) -> Dict[str, Any]: + normalized_task = str(task or "").strip() + state = session.get_state(_BROWSER_PHASE_STATE_KEY) + if isinstance(state, dict) and str(state.get("task") or "").strip() == normalized_task: + self._runtime.set_task_requested_fields(state.get("required_fields") or []) + return state + if isinstance(state, dict) and resume: + resumed = self._resume_task_state(session, state, normalized_task) + self._runtime.set_task_requested_fields(resumed.get("required_fields") or []) + return resumed + self._runtime.reset_semantic_task() + self._runtime.service.clear_progress_state(session.get_session_id()) + state = self._build_phase_state(normalized_task) + self._runtime.set_task_requested_fields(state.get("required_fields") or []) + session.update_state( + { + _BROWSER_PROGRESS_TASK_KEY: normalized_task, + _BROWSER_PROGRESS_STATE_KEY: {}, + _BROWSER_PHASE_STATE_KEY: state, + } + ) + return state + + @staticmethod + def _browser_run_context(ctx: AgentCallbackContext) -> Dict[str, Any]: + context = getattr(getattr(ctx, "inputs", None), "run_context", None) + if context is None: + context = (getattr(ctx, "extra", None) or {}).get("run_context") + if isinstance(context, dict): + return context.get("extra", context) + extra = getattr(context, "extra", None) + return extra if isinstance(extra, dict) else {} + + @staticmethod + def _bind_shared_task_deadline( + ctx: AgentCallbackContext, + session: Any, + state: Dict[str, Any], + ) -> float: + if session is None or not isinstance(state, dict) or not state: + return 0.0 + extra = getattr(ctx, "extra", None) + shared_context = BrowserRuntimeRail._browser_run_context(ctx) + context_budget_s = float(shared_context.get("browser_query_budget_s") or 0.0) + budget_s = float( + state.get("deadline_budget_s") + or context_budget_s + or ( + _BROWSER_COMPLEX_TASK_DEADLINE_S + if str(state.get("task_type") or "") == "complex" + else _BROWSER_SIMPLE_TASK_DEADLINE_S + ) + ) + now = time.time() + context_started_at = float(shared_context.get("browser_query_started_at") or 0.0) + context_deadline_at = float(shared_context.get("browser_query_deadline_at") or 0.0) + started_at = float(state.get("deadline_started_at") or context_started_at or now) + deadline_at = float(state.get("deadline_at") or context_deadline_at or started_at + budget_s) + if context_deadline_at: + deadline_at = min(deadline_at, context_deadline_at) + query_id = str(shared_context.get("browser_query_id") or state.get("query_id") or "").strip() + if query_id: + state["query_id"] = query_id[:128] + state["deadline_budget_s"] = budget_s + state["deadline_started_at"] = started_at + state["deadline_at"] = deadline_at + remaining_s = max(0.0, deadline_at - now) + state["deadline_remaining_s"] = round(remaining_s, 3) + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + if isinstance(extra, dict): + invocation_deadline = float(extra.get(_BROWSER_INVOCATION_DEADLINE_KEY) or 0.0) + if not invocation_deadline: + invocation_deadline = time.monotonic() + remaining_s + extra[_BROWSER_INVOCATION_DEADLINE_KEY] = invocation_deadline + effective_remaining_s = min( + remaining_s, + max(0.0, invocation_deadline - time.monotonic()), + ) + extra[_BROWSER_TASK_DEADLINE_KEY] = time.monotonic() + effective_remaining_s + state["invocation_remaining_s"] = round(effective_remaining_s, 3) + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + return effective_remaining_s + return remaining_s + + @classmethod + def _finish_if_task_deadline_exhausted( + cls, + ctx: AgentCallbackContext, + session: Any, + state: Dict[str, Any], + ) -> bool: + if session is None or not isinstance(state, dict) or not state: + return False + deadline_at = float(state.get("deadline_at") or 0.0) + shared_exhausted = bool(deadline_at and time.time() >= deadline_at) + extra = getattr(ctx, "extra", None) + invocation_deadline = ( + float(extra.get(_BROWSER_INVOCATION_DEADLINE_KEY) or 0.0) if isinstance(extra, dict) else 0.0 + ) + invocation_exhausted = bool(invocation_deadline and time.monotonic() >= invocation_deadline) + if not shared_exhausted and not invocation_exhausted: + return False + status = str(state.get("status") or "in_progress").strip().lower() + if status not in _BROWSER_TERMINAL_STATUSES: + missing = cls._missing_completion_requirements(state) + evidence_available = cls._has_task_evidence(state) + if evidence_available and not missing and not state.get("blockers"): + state["status"] = "completed" + state["terminal_reason"] = "task_deadline_completed_from_evidence" + else: + state["status"] = "partial" if evidence_available else "blocked" + blockers = list(state.get("blockers") or []) + blocker = "task_deadline_exhausted" if shared_exhausted else "task_invocation_slice_exhausted" + if blocker not in blockers: + blockers.append(blocker) + state["blockers"] = blockers[:10] + state["terminal_reason"] = blocker + state["next_action_class"] = "finish" + state["deadline_remaining_s"] = 0.0 if shared_exhausted else round(max(0.0, deadline_at - time.time()), 3) + state["invocation_remaining_s"] = 0.0 + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + ctx.request_force_finish(cls._structured_terminal_result(state)) + return True + + @staticmethod + def _terminalize_invoke_failure(session: Any, reason: str, *, blocked: bool = False) -> bool: + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + if not isinstance(state, dict) or not state: + return False + status = str(state.get("status") or "in_progress").strip().lower() + if status not in _BROWSER_TERMINAL_STATUSES: + evidence_available = BrowserRuntimeRail._has_task_evidence(state) + state["status"] = "partial" if evidence_available and not blocked else "blocked" + state["next_action_class"] = "finish" + state["terminal_reason"] = reason + blockers = list(state.get("blockers") or []) + if reason not in blockers: + blockers.append(reason) + state["blockers"] = blockers[:10] + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + return True + + def _resume_task_state( + self, + session: Any, + state: Dict[str, Any], + resume_instruction: str, + ) -> Dict[str, Any]: + if str(state.get("resume_instruction") or "").strip() == resume_instruction: + return state + missing = self._missing_completion_requirements(state) + if not self._terminal_result_retryable(state, missing): + return state + + state["resume_count"] = int(state.get("resume_count") or 0) + 1 + state["resume_instruction"] = resume_instruction[:2_000] + state["status"] = "in_progress" + state["next_action_class"] = "collect_missing_evidence" if missing else "materially_different_strategy" + state["terminal_reason"] = "" + state.pop(_BROWSER_TERMINAL_SYNTHESIS_KEY, None) + state["replan_required"] = False + state["replan_trial_pending"] = False + state["replan_count"] = 0 + state["replan_denial_count"] = 0 + state["blocked_strategy"] = "" + state["trial_strategy"] = "" + state["model_protocol_retry_count"] = 0 + recoverable_prefixes = ( + "semantic_", + "missing_required_field:", + "missing_evidence_slot:", + "model_provider_unavailable", + "model_tool_protocol_error", + "task_invocation_slice_exhausted", + "browser_connection_lost", + ) + state["blockers"] = [ + blocker + for blocker in state.get("blockers") or [] + if not str(blocker).strip().lower().startswith(recoverable_prefixes) + ][:10] + known_urls = list(state.get("known_urls") or []) + for url in re.findall(r"https?://[^\s<>\"]+", resume_instruction): + if url not in known_urls: + known_urls.append(url) + state["known_urls"] = known_urls[:4] + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + self._runtime.reset_semantic_task() + return state + + @staticmethod + def _coerce_tool_args(tool_args: Any) -> Dict[str, Any]: + if isinstance(tool_args, dict): + return tool_args + if isinstance(tool_args, str): + try: + parsed = json.loads(tool_args) + except ValueError: + return {} + return parsed if isinstance(parsed, dict) else {} + return {} + + @staticmethod + def _command_name(kind: ToolKind) -> str: + return kind.command.__name__ if kind.command is not None else "" + + def _clear_progress_state(self, session: Any) -> None: + if session is None: + return + session_id = session.get_session_id() + self._runtime.service.clear_progress_state(session_id) + session.update_state( + { + _BROWSER_PROGRESS_STATE_KEY: {}, + _BROWSER_PROGRESS_TASK_KEY: "", + _BROWSER_PHASE_STATE_KEY: {}, + } + ) + + def _image_input_supported(self, agent: Any) -> bool: + return should_enable_read_image_multimodal(self._agent if self._agent is not None else agent) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/terminal.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/terminal.py new file mode 100644 index 0000000000..fe34b3fb1c --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/terminal.py @@ -0,0 +1,443 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Model-protocol recovery and the terminal result of a browser task.""" + +from __future__ import annotations + +import asyncio +import json +import re +import time +from typing import Any, Dict, Optional + +from openjiuwen.core.single_agent.prompts.builder import PromptSection +from openjiuwen.core.single_agent.rail.base import AgentCallbackContext +from openjiuwen.harness.tools.browser_move.runtime.worker import MAX_ITERATION_MESSAGE, TaskProgressState +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.keys import ( + _BROWSER_PHASE_STATE_KEY, + _BROWSER_TASK_DEADLINE_KEY, + _BROWSER_TERMINAL_STATUSES, + _BROWSER_TERMINAL_SYNTHESIS_KEY, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.semantic.phases import _PhaseMixin + +_BROWSER_MODEL_PROTOCOL_RETRY_LIMIT = 1 +_BROWSER_TASK_RESUME_LIMIT = 1 +_BROWSER_PROGRESS_TAG_RE = re.compile( + r"<browser_progress>\s*(\{.*?\})\s*</browser_progress>", + re.DOTALL | re.IGNORECASE, +) +_BROWSER_UNFINISHED_TOOL_INTENT_RE = re.compile( + r"(?:" + r"<[^>]*(?:dsml|tool[_ ]?calls?)[^>]*>" + r"|(?:让我|我(?:将|会|需要|要))继续(?:调用|使用|执行).{0,100}" + r"(?:工具|browser_|evaluate|probe|navigate|snapshot|batch)" + r"|(?:let me|i(?:'ll| will| need to))\s+(?:continue|now)\s+" + r"(?:call|calling|use|using|run).{0,100}" + r"(?:tool|browser_|evaluate|probe|navigate|snapshot|batch)" + r")", + re.IGNORECASE | re.DOTALL, +) +_BROWSER_NON_RETRYABLE_BLOCKER_TOKENS = frozenset( + { + "captcha", + "login_required", + "permission_denied", + "access_denied", + "user_intervention_required", + "payment_required", + "security_verification", + "task_deadline_exhausted", + } +) + + +class _TerminalMixin: + @staticmethod + def _has_unfinished_tool_intent(response_text: str) -> bool: + return _BROWSER_UNFINISHED_TOOL_INTENT_RE.search(str(response_text or "")) is not None + + @classmethod + def _is_unfinished_model_protocol_response( + cls, + state: Any, + tool_calls: list[Any], + response_text: str, + ) -> bool: + if tool_calls or not isinstance(state, dict): + return False + status = str(state.get("status") or "in_progress").strip().lower() + if status in _BROWSER_TERMINAL_STATUSES: + return False + return cls._has_unfinished_tool_intent(response_text) + + @staticmethod + def _request_model_protocol_recovery( + ctx: AgentCallbackContext, + session: Any, + state: Dict[str, Any], + ) -> bool: + retry_count = int(state.get("model_protocol_retry_count") or 0) + extra = getattr(ctx, "extra", None) + deadline = float(extra.get(_BROWSER_TASK_DEADLINE_KEY) or 0.0) if isinstance(extra, dict) else 0.0 + deadline_exhausted = bool(deadline and deadline - time.monotonic() <= 5.0) + if retry_count >= _BROWSER_MODEL_PROTOCOL_RETRY_LIMIT or deadline_exhausted: + return False + state["model_protocol_retry_count"] = retry_count + 1 + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + if ctx.steering_queue is None: + ctx.bind_steering_queue(asyncio.Queue()) + ctx.push_steering( + "Your previous response described another browser tool call but did not emit a real tool call. " + "Continue in this same task: emit the actual tool call, or answer concisely using only evidence " + "already collected." + ) + ctx.request_model_continue() + return True + + @classmethod + def _finish_model_protocol_failure( + cls, + ctx: AgentCallbackContext, + session: Any, + state: Dict[str, Any], + ) -> None: + evidence_available = cls._has_task_evidence(state) + state["status"] = "partial" if evidence_available else "blocked" + state["next_action_class"] = "finish" + state["terminal_reason"] = "model_tool_protocol_error" + blockers = list(state.get("blockers") or []) + if "model_tool_protocol_error" not in blockers: + blockers.append("model_tool_protocol_error") + state["blockers"] = blockers[:10] + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + ctx.request_force_finish(cls._structured_terminal_result(state)) + + def _prepare_terminal_synthesis( + self, + ctx: AgentCallbackContext, + session: Any, + state: Dict[str, Any], + ) -> bool: + """Allow one tool-disabled prose pass, then force the runtime result.""" + + status = str(state.get("status") or "").strip().lower() + if status not in _BROWSER_TERMINAL_STATUSES: + return False + if state.get(_BROWSER_TERMINAL_SYNTHESIS_KEY): + ctx.request_force_finish(self._structured_terminal_result(state)) + return True + + state[_BROWSER_TERMINAL_SYNTHESIS_KEY] = True + state["next_action_class"] = "finish" + if session is not None: + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + inputs = getattr(ctx, "inputs", None) + if inputs is not None: + inputs.tools = [] + builder = getattr(ctx.agent, "system_prompt_builder", None) + if builder is not None: + authoritative = self._authoritative_terminal_payload(state) + builder.add_section( + PromptSection( + name="browser_terminal_synthesis", + content={ + "en": ( + "Browser execution has ended. Do not call tools. Summarize the runtime-owned " + f"result without changing status, blockers, missing fields, or evidence: {authoritative}" + ), + "cn": ( + "浏览器执行已结束。不要调用工具。请说明以下由运行时确定的结果，不得修改状态、" + f"阻断项、缺失字段或证据：{authoritative}" + ), + }, + priority=100, + ) + ) + return False + + def _structured_model_failure( + self, + ctx: AgentCallbackContext, + exception: Optional[Exception], + ) -> Dict[str, Any]: + session = getattr(ctx, "session", None) + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + state = state if isinstance(state, dict) else {} + evidence_available = self._has_task_evidence(state) + status = "partial" if evidence_available else "failed" + if state: + state["status"] = "partial" if evidence_available else "blocked" + state["next_action_class"] = "finish" + state["terminal_reason"] = "model_provider_unavailable" + blockers = list(state.get("blockers") or []) + if "model_provider_unavailable" not in blockers: + blockers.append("model_provider_unavailable") + state["blockers"] = blockers[:10] + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + payload = { + "ok": False, + "status": status, + "error": { + "code": "model_provider_unavailable", + "message": str(exception or "model provider failed")[:500], + "retry_attempts": int(ctx.retry_attempt) + 1, + }, + "progress": { + "task_id": state.get("task_id"), + "status": state.get("status"), + "current_phase": state.get("current_phase"), + "required_fields": list(state.get("required_fields") or [])[:32], + "field_coverage": list(state.get("field_coverage") or [])[:32], + "required_evidence_slots": list(state.get("required_evidence_slots") or [])[:12], + "evidence_slots": list(state.get("evidence_slots") or [])[-12:], + "blockers": list(state.get("blockers") or [])[:10], + } + if state + else {}, + } + return { + "output": json.dumps(payload, ensure_ascii=False, separators=(",", ":")), + "result_type": "error", + "error": "model_provider_unavailable", + "progress_state": self._load_progress_state(session).to_dict() if session is not None else {}, + } + + @classmethod + def _authoritative_terminal_payload(cls, state: Dict[str, Any]) -> Dict[str, Any]: + status = str(state.get("status") or "partial").strip().lower() + missing = cls._missing_completion_requirements(state) + unverified = cls._advisory_missing_fields(state) if status == "completed" else set() + missing = [field for field in missing if field not in unverified] + blockers = [str(item) for item in state.get("blockers") or [] if str(item).strip()] + missing_slots = [slot for slot in cls._missing_evidence_slots(state) if slot["field"] not in unverified] + unavailable_slots = cls._unavailable_evidence_slots(state) + retryable = cls._terminal_result_retryable(state, missing) + deadline_started_at = float(state.get("deadline_started_at") or 0.0) + deadline_at = float(state.get("deadline_at") or 0.0) + now = time.time() + return { + "status": status, + "task_id": state.get("task_id"), + "current_phase": state.get("current_phase"), + "missing_fields": missing[:32], + "unverified_fields": sorted(unverified)[:32], + "missing_slots": missing_slots[:12], + "unavailable_slots": unavailable_slots[:12], + "requested_slots": [ + dict(slot) for slot in (state.get("required_evidence_slots") or [])[:12] if isinstance(slot, dict) + ], + "blockers": blockers[:10], + "field_coverage": list(state.get("field_coverage") or [])[:32], + "evidence": list(state.get("evidence_slots") or [])[-12:], + "observations": cls._task_observations(state)[-3:], + "current_page": dict(state.get("last_page") or {}), + "requested_result_count": int(state.get("requested_result_count") or 0), + "observed_result_count": int(state.get("observed_result_count") or 0), + "terminal_reason": state.get("terminal_reason"), + "retryable": retryable, + "recommended_recovery": cls._recommended_recovery(state, missing), + "resume_count": int(state.get("resume_count") or 0), + "deadline": { + "budget_s": float(state.get("deadline_budget_s") or 0.0), + "elapsed_s": round(max(0.0, now - deadline_started_at), 3) if deadline_started_at else 0.0, + "remaining_s": round(max(0.0, deadline_at - now), 3) if deadline_at else 0.0, + "invocation_remaining_s": float(state.get("invocation_remaining_s") or 0.0), + }, + } + + @classmethod + def _terminal_result_retryable( + cls, + state: Dict[str, Any], + missing: Optional[list[str]] = None, + ) -> bool: + status = str(state.get("status") or "").strip().lower() + if status == "completed" or int(state.get("resume_count") or 0) >= _BROWSER_TASK_RESUME_LIMIT: + return False + blockers = " ".join(str(item or "").strip().lower() for item in state.get("blockers") or []) + if any(token in blockers for token in _BROWSER_NON_RETRYABLE_BLOCKER_TOKENS): + return False + if "phase_budget_exhausted" in blockers: + return False + requirements = missing if missing is not None else cls._missing_completion_requirements(state) + unresolved_slots = cls._missing_evidence_slots(state) + if requirements and not unresolved_slots and cls._unavailable_evidence_slots(state): + return False + return bool( + status in {"partial", "blocked"} + and ( + requirements + or cls._has_task_evidence(state) + or state.get("terminal_reason") + in { + "model_provider_unavailable", + "model_tool_protocol_error", + "runtime_completion_requirements_missing", + "runtime_blocked", + "semantic_replan_denial_budget_exhausted", + "task_invocation_slice_exhausted", + "browser_connection_lost", + } + ) + ) + + @staticmethod + def _recommended_recovery(state: Dict[str, Any], missing: list[str]) -> str: + reported = str(state.get("worker_reported_next_action") or "").strip() + if reported: + return reported[:200] + reason = str(state.get("terminal_reason") or "").strip().lower() + if reason in { + "model_provider_unavailable", + "model_tool_protocol_error", + "task_invocation_slice_exhausted", + }: + return "retry_model_step_from_current_page" + if missing and not _PhaseMixin._missing_evidence_slots(state): + if _PhaseMixin._unavailable_evidence_slots(state): + return "finish_with_explicit_unknown_fields" + if missing: + return "collect_missing_evidence_from_current_page" + return "replan_from_current_page" + + @classmethod + def _structured_terminal_result(cls, state: Dict[str, Any]) -> Dict[str, Any]: + payload = cls._authoritative_terminal_payload(state) + status = str(payload.get("status") or "partial") + return { + "output": json.dumps( + {"browser_result": payload}, + ensure_ascii=False, + separators=(",", ":"), + ), + "result_type": "answer" if status == "completed" else "error", + "error": None if status == "completed" else "browser_task_incomplete", + "authoritative_browser_result": payload, + } + + @classmethod + def _render_authoritative_terminal_output( + cls, + state: Dict[str, Any], + model_summary: str, + ) -> tuple[str, Dict[str, Any]]: + payload = cls._authoritative_terminal_payload(state) + if str(model_summary or "").lstrip().startswith("{"): + try: + previous = json.loads(model_summary).get("browser_result") + except (ValueError, AttributeError): + previous = None + if isinstance(previous, dict): + model_summary = str(previous.get("summary") or state.get("last_worker_final") or "") + if str(model_summary or "").strip(): + payload["summary"] = str(model_summary).strip()[:8_000] + return ( + json.dumps({"browser_result": payload}, ensure_ascii=False, separators=(",", ":")), + payload, + ) + + @staticmethod + def _extract_progress_payload(output_text: str) -> tuple[str, Optional[Dict[str, Any]]]: + text = str(output_text or "") + match = _BROWSER_PROGRESS_TAG_RE.search(text) + if match is None: + return text, None + payload_text = match.group(1).strip() + try: + payload = json.loads(payload_text) + except json.JSONDecodeError: + return text, None + cleaned = _BROWSER_PROGRESS_TAG_RE.sub("", text, count=1).strip() + return cleaned, payload if isinstance(payload, dict) else None + + @staticmethod + def _build_progress_result( + progress_state: TaskProgressState, + clean_output: str, + ) -> Dict[str, Any]: + """Build completion output from runtime-owned progress only.""" + + status = str(progress_state.status or "").strip().lower() or "partial" + progress = progress_state.to_dict() + return { + "ok": status == "completed", + "status": status, + "progress": progress, + "final": clean_output, + "error": None if status == "completed" else "browser_task_incomplete", + } + + @classmethod + def _apply_worker_progress_to_task_state( + cls, + session: Any, + payload: Dict[str, Any], + final: str, + ) -> None: + """Validate a worker completion request against authoritative runtime state.""" + + state = session.get_state(_BROWSER_PHASE_STATE_KEY) + if not isinstance(state, dict): + return + reported_status = str(payload.get("status") or "partial").strip().lower() + state["worker_reported_status"] = reported_status + reported_blockers = payload.get("blockers") or payload.get("missing_requirements") or [] + if isinstance(reported_blockers, str): + reported_blockers = [reported_blockers] + state["worker_reported_blockers"] = [str(item)[:300] for item in reported_blockers if str(item).strip()][:10] + next_action = payload.get("next_action") or payload.get("next_step") + if next_action: + state["worker_reported_next_action"] = str(next_action)[:200] + if final: + state["last_worker_final"] = str(final)[:8_000] + + current_status = str(state.get("status") or "in_progress").strip().lower() + if current_status in _BROWSER_TERMINAL_STATUSES: + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + return + + runtime_blockers = [str(item) for item in state.get("blockers") or [] if str(item).strip()] + advisory_fields = cls._advisory_missing_fields(state) if final.strip() else set() + missing_fields = [ + field for field in cls._missing_completion_requirements(state) if field not in advisory_fields + ] + unavailable_slots = cls._unavailable_evidence_slots(state) + phases = state.get("phases") if isinstance(state.get("phases"), dict) else {} + completed_phase_count = sum( + 1 for details in phases.values() if isinstance(details, dict) and details.get("status") == "completed" + ) + evidence_available = cls._has_task_evidence(state) + runtime_ready = bool(completed_phase_count or evidence_available) + if state.get("replan_required"): + runtime_blockers = list(dict.fromkeys([*runtime_blockers, "semantic_replan_required"])) + + completion_requirements_met = ( + bool(final.strip() or evidence_available) + and not runtime_blockers + and not missing_fields + and not unavailable_slots + and runtime_ready + ) + if reported_status == "completed" and completion_requirements_met: + state["status"] = "completed" + state["next_action_class"] = "finish" + state["terminal_reason"] = "runtime_completion_validated" + elif reported_status in _BROWSER_TERMINAL_STATUSES: + state["status"] = "blocked" if runtime_blockers else "partial" + state["next_action_class"] = "finish" + state["terminal_reason"] = ( + "runtime_blocked" if runtime_blockers else "runtime_completion_requirements_missing" + ) + if runtime_blockers: + state["blockers"] = runtime_blockers[:10] + elif missing_fields: + state["blockers"] = [f"missing_required_field:{field}" for field in missing_fields[:10]] + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + + @staticmethod + def _is_max_iteration_result(result: Dict[str, Any]) -> bool: + output = str(result.get("output") or "").strip() + result_type = str(result.get("result_type") or "").strip().lower() + return result_type == "error" and MAX_ITERATION_MESSAGE.lower() in output.lower() diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context_rail.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/working_context.py similarity index 91% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context_rail.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/rails/working_context.py index a4213910a7..65e9fc6950 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context_rail.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/rails/working_context.py @@ -18,8 +18,9 @@ ModelCallInputs, ToolCallInputs, ) +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.logging import browser_agent_log_warning -from .browser_working_context import ( +from ..working_context.store import ( BROWSER_TOOL_MEMORY_METADATA_KEY, BROWSER_WORKING_MEMORY_RECORD_BEGIN, BROWSER_WORKING_MEMORY_RECORD_END, @@ -28,8 +29,6 @@ BrowserWorkingMemory, latest_browser_user_request, ) -from .browser_logging import browser_agent_log_warning - _WORKING_MEMORY_RECORD_RE = re.compile( rf"{re.escape(BROWSER_WORKING_MEMORY_RECORD_BEGIN)}\s*(.*?)\s*" @@ -50,9 +49,6 @@ def __init__(self, config: Optional[BrowserWorkingContextConfig] = None) -> None self._store = BrowserWorkingContextStore(self.config) async def before_invoke(self, ctx: AgentCallbackContext) -> None: - # DeepAgent owns the outer invoke callback but creates a separate inner - # ReAct callback context. Let before_model_call handle that topology so - # an explicitly supplied Session is not counted twice. if getattr(ctx.agent, "react_agent", None) is not None: return inputs = ctx.inputs @@ -63,13 +59,7 @@ async def before_invoke(self, ctx: AgentCallbackContext) -> None: ctx.extra[_REQUEST_STARTED_EXTRA_KEY] = True async def before_model_call(self, ctx: AgentCallbackContext) -> None: - """Begin the request where the inner ReAct Session actually exists. - - DeepAgent routes ``before_invoke`` to its outer lifecycle. Subagents - invoked by conversation id have no Session there; the restorable - Session is created immediately before the inner model loop. This - bridged callback is therefore the authoritative reinvocation boundary. - """ + """Begin the request here, where the inner ReAct session exists.""" if ctx.extra.get(_REQUEST_STARTED_EXTRA_KEY): return diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/renderer.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/renderer.py new file mode 100644 index 0000000000..fe49cba1f4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/renderer.py @@ -0,0 +1,87 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""What the model reads for a browser observation, result or error.""" + +from __future__ import annotations + +import json +from typing import Any + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + CloseContext, + ContentBlock, + DriverError, + Observation, + Observe, + RunScript, + Screenshot, + StaleRef, + SwitchContext, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import OpenContext +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext + +_ERROR = "### Error" +_TAB_COMMANDS = (OpenContext, SwitchContext, CloseContext) + + +class BrowserRenderer: + """The snapshot text verbatim; results as the server's sections, with script values and tab lists rebuilt.""" + + def observation(self, observation: Observation) -> tuple[ContentBlock, ...]: + return _blocks(observation.native_text, observation.screenshot) + + def result(self, action: Action, result: ActionResult) -> tuple[ContentBlock, ...]: + image = result.value if isinstance(result.value, Screenshot) else None + detail = result.detail or "" + if not result.ok: + return _blocks(detail if detail.startswith(_ERROR) else f"{_ERROR}\n{detail}", image) + if isinstance(action, RunScript): + return _blocks(_join(f"### Result\n{_script_value(result.value)}", detail), image) + listing = isinstance(action, Observe) and not action.include_screenshot + if listing or (isinstance(action, _TAB_COMMANDS) and not detail): + tabs = _tab_list(result.surface or (result.observation.surface if result.observation else None)) + return _blocks(_join(tabs and f"### Result\n{tabs}", detail), image) + return _blocks(detail, image) + + def error(self, action: Action, error: DriverError, observation: Observation | None) -> tuple[ContentBlock, ...]: + if observation is None and isinstance(error, StaleRef): + observation = error.observation + blocks = (f"{_ERROR}\n{error}",) + if observation is None: + return blocks + seen = self.observation(observation) + if seen and isinstance(seen[0], str): + return (f"{blocks[0]}\n\n{seen[0]}",) + seen[1:] + return blocks + seen + + +def _blocks(text: str | None, image: Screenshot | None) -> tuple[ContentBlock, ...]: + return tuple(block for block in (text, image) if block) + + +def _join(*parts: str) -> str: + return "\n".join(part for part in parts if part) + + +def _script_value(value: Any) -> str: + if value is None: + return "undefined" + if isinstance(value, str): + return value + return json.dumps(value, indent=2, ensure_ascii=False) + + +def _tab_list(surface: Any) -> str: + if not isinstance(surface, BrowserContext): + return "" + lines = [] + for index, tab in enumerate(surface.tabs): + current = index == surface.current_tab + url = surface.tab_urls[index] if index < len(surface.tab_urls) else surface.url if current else "" + link = f"[{tab.title or ''}]" + (f"({url})" if url else "") + lines.append(f"- {index}: " + ("(current) " if current else "") + link) + return "\n".join(lines) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/__init__.py new file mode 100644 index 0000000000..f3eceaf7d5 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/__init__.py @@ -0,0 +1,20 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The browser runner: the facade and the settings that run a browser task over the core runtime.""" + +from openjiuwen.harness.tools.browser_move.runtime.config import RunGuardrails + +from .facade import BrowserAgentRuntime +from .settings import BrowserInstanceConfig, RuntimeSettings, build_runtime_settings + +BrowserRunGuardrails = RunGuardrails + +__all__ = [ + "BrowserAgentRuntime", + "BrowserInstanceConfig", + "BrowserRunGuardrails", + "RunGuardrails", + "RuntimeSettings", + "build_runtime_settings", +] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/actions.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/actions.py new file mode 100644 index 0000000000..69f1cc6877 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/actions.py @@ -0,0 +1,523 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The custom-action controller: the base contract, the instance registry and the module-level default.""" + +from __future__ import annotations + +import asyncio +import contextlib +import contextvars +import copy +import inspect +from abc import ABC, abstractmethod +from typing import Any, Awaitable, Callable, Protocol + +from openjiuwen.core.common.logging import logger + + +class BaseController(ABC): + """Abstract contract for controller implementations.""" + + @abstractmethod + def bind_runtime(self, runtime: Any) -> None: + """Bind a runtime object used by runtime-backed actions.""" + + @abstractmethod + def bind_runtime_runner(self, runner: Any | None) -> None: + """Bind a runtime runner callable.""" + + @abstractmethod + def clear_runtime_runner(self) -> None: + """Clear any currently bound runtime runner.""" + + @abstractmethod + def bind_code_executor(self, executor: Any | None) -> None: + """Bind ``async (js_code: str) -> Any`` for custom actions that run prebuilt scripts.""" + + @abstractmethod + def clear_code_executor(self) -> None: + """Clear the bound code executor.""" + + @abstractmethod + def register_action(self, name: str, handler: Any, *, overwrite: bool = True) -> None: + """Register an action handler.""" + + @abstractmethod + def register_action_spec( + self, + name: str, + *, + summary: str = "", + when_to_use: str = "", + params: dict[str, str] | None = None, + ) -> None: + """Register metadata for an action.""" + + @abstractmethod + def list_actions(self) -> list[str]: + """List registered action names.""" + + @abstractmethod + def describe_actions(self) -> dict[str, dict[str, Any]]: + """Return metadata for registered actions.""" + + @abstractmethod + async def run_action( + self, + action: str, + session_id: str = "", + request_id: str = "", + **kwargs: Any, + ) -> dict[str, Any]: + """Execute a registered action.""" + + +ActionResult = dict[str, Any] +ActionHandler = Callable[..., Awaitable[Any] | Any] +CodeExecutor = Callable[[str], Awaitable[Any]] + + +class RuntimeRunner(Protocol): + async def __call__( + self, + *, + task: str, + session_id: str | None = None, + request_id: str | None = None, + timeout_s: int | None = None, + ) -> dict[str, Any]: ... + + +_ACTIONS: dict[str, ActionHandler] = {} +_ACTION_SPECS: dict[str, dict[str, Any]] = {} +_RUNTIME_RUNNER: RuntimeRunner | None = None +_CODE_EXECUTOR: CodeExecutor | None = None +_LOCK = asyncio.Lock() +_ctx_browser_worker_action: contextvars.ContextVar[bool] = contextvars.ContextVar( + "playwright_runtime_browser_worker_action", + default=False, +) +_RECURSIVE_BROWSER_ACTIONS = frozenset({"browser_task", "run_browser_task"}) + + +@contextlib.contextmanager +def browser_worker_action_context(): + token = _ctx_browser_worker_action.set(True) + try: + yield + finally: + _ctx_browser_worker_action.reset(token) + + +class ActionController(BaseController): + """Instance-scoped registry and dispatcher for custom actions.""" + + def __init__( + self, + *, + actions: dict[str, ActionHandler] | None = None, + action_specs: dict[str, dict[str, Any]] | None = None, + runtime_runner: RuntimeRunner | None = None, + code_executor: CodeExecutor | None = None, + lock: asyncio.Lock | None = None, + ) -> None: + self._actions: dict[str, ActionHandler] = actions if actions is not None else {} + self._action_specs: dict[str, dict[str, Any]] = action_specs if action_specs is not None else {} + self._runtime_runner: RuntimeRunner | None = runtime_runner + self._code_executor: CodeExecutor | None = code_executor + self._lock: asyncio.Lock = lock if lock is not None else asyncio.Lock() + + @property + def runtime_runner(self) -> RuntimeRunner | None: + return self._runtime_runner + + @property + def code_executor(self) -> CodeExecutor | None: + return self._code_executor + + def bind_runtime(self, runtime: Any) -> None: + run_browser_task = getattr(runtime, "run_browser_task", None) + if run_browser_task is None or not callable(run_browser_task): + raise ValueError("runtime must expose an async run_browser_task(...) method") + + async def _runner( + *, + task: str, + session_id: str | None = None, + request_id: str | None = None, + timeout_s: int | None = None, + ) -> dict[str, Any]: + return await run_browser_task( + task=task, + session_id=session_id, + request_id=request_id, + timeout_s=timeout_s, + ) + + self.bind_runtime_runner(_runner) + + def bind_runtime_runner(self, runner: RuntimeRunner | None) -> None: + self._runtime_runner = runner + + def clear_runtime_runner(self) -> None: + self.bind_runtime_runner(None) + + def bind_code_executor(self, executor: CodeExecutor | None) -> None: + self._code_executor = executor + + def clear_code_executor(self) -> None: + self._code_executor = None + + def register_action(self, name: str, handler: ActionHandler, *, overwrite: bool = True) -> None: + action_name = _normalize_action_name(name) + if not action_name: + raise ValueError("action name must be non-empty") + if not callable(handler): + raise TypeError("handler must be callable") + if not overwrite and action_name in self._actions: + raise ValueError(f"action already exists: {action_name}") + self._actions[action_name] = handler + + def register_action_spec( + self, + name: str, + *, + summary: str = "", + when_to_use: str = "", + params: dict[str, str] | None = None, + ) -> None: + action_name = _normalize_action_name(name) + if not action_name: + raise ValueError("action name must be non-empty") + self._action_specs[action_name] = { + "summary": (summary or "").strip(), + "when_to_use": (when_to_use or "").strip(), + "params": dict(params or {}), + } + + def list_actions(self) -> list[str]: + return sorted(self._actions.keys()) + + def describe_actions(self) -> dict[str, dict[str, Any]]: + result: dict[str, dict[str, Any]] = {} + for name in sorted(self._actions.keys()): + spec = self._action_specs.get(name, {}) + result[name] = { + "summary": str(spec.get("summary", "") or ""), + "when_to_use": str(spec.get("when_to_use", "") or ""), + "params": dict(spec.get("params", {}) or {}), + } + return result + + async def run_action( + self, + action: str, + session_id: str = "", + request_id: str = "", + **kwargs: Any, + ) -> ActionResult: + action_name = _normalize_action_name(action) + sid = (session_id or "").strip() + rid = (request_id or "").strip() + param_keys = ",".join(sorted(str(key) for key in kwargs.keys())) + logger.info( + f"CONTROLLER_ACTION start action={action_name} session_id={sid or '-'} " + f"request_id={rid or '-'} param_keys={param_keys or '-'}" + ) + + if _ctx_browser_worker_action.get() and action_name in _RECURSIVE_BROWSER_ACTIONS: + error = ( + "recursive_browser_task_blocked: browser workers must not invoke " + "browser_task/run_browser_task via browser_custom_action; return a JSON error instead" + ) + logger.warning( + f"CONTROLLER_ACTION blocked action={action_name} session_id={sid or '-'} " + f"request_id={rid or '-'} error={error}" + ) + return { + "ok": False, + "action": action_name, + "session_id": sid, + "request_id": rid, + "error": error, + } + + handler = self._actions.get(action_name) + if handler is None: + logger.warning( + f"CONTROLLER_ACTION unknown action={action_name} session_id={sid or '-'} request_id={rid or '-'}" + ) + return { + "ok": False, + "action": action_name, + "session_id": sid, + "request_id": rid, + "error": f"unknown action: {action_name}", + } + + try: + async with self._lock: + raw = await _maybe_await(handler(session_id=sid, request_id=rid, **kwargs)) + + response = dict(raw) if isinstance(raw, dict) else {"result": raw} + response.setdefault("ok", True) + response.setdefault("action", action_name) + response.setdefault("session_id", sid) + response.setdefault("request_id", rid) + response.setdefault("error", None) + _ok = bool(response.get("ok", False)) + _err = response.get("error") if not _ok else None + logger.info( + f"CONTROLLER_ACTION end action={action_name} session_id={sid or '-'} " + f"request_id={rid or '-'} ok={_ok}" + (f" error={_err!r}" if _err else "") + ) + return response + except Exception as exc: + logger.error( + f"CONTROLLER_ACTION error action={action_name} session_id={sid or '-'} " + f"request_id={rid or '-'} error={exc}" + ) + return { + "ok": False, + "action": action_name, + "session_id": sid, + "request_id": rid, + "error": str(exc), + } + + def register_builtin_actions(self) -> None: + register_builtin_actions(controller=self) + + def register_example_actions(self) -> None: + register_builtin_actions(controller=self) + + def snapshot(self) -> dict[str, Any]: + return { + "actions": dict(self._actions), + "action_specs": copy.deepcopy(self._action_specs), + "runtime_runner": self._runtime_runner, + "code_executor": self._code_executor, + } + + def restore(self, snapshot: dict[str, Any]) -> None: + self._actions.clear() + self._actions.update(dict(snapshot.get("actions", {}))) + self._action_specs.clear() + self._action_specs.update(copy.deepcopy(dict(snapshot.get("action_specs", {})))) + self._runtime_runner = snapshot.get("runtime_runner") + self._code_executor = snapshot.get("code_executor") + + +def _normalize_action_name(name: str) -> str: + return (name or "").strip().lower() + + +async def _maybe_await(value: Any) -> Any: + if inspect.isawaitable(value): + return await value + return value + + +_DEFAULT_CONTROLLER = ActionController( + actions=_ACTIONS, + action_specs=_ACTION_SPECS, + runtime_runner=_RUNTIME_RUNNER, + lock=_LOCK, +) + + +def get_default_controller() -> ActionController: + return _DEFAULT_CONTROLLER + + +def bind_runtime(runtime: Any) -> None: + _DEFAULT_CONTROLLER.bind_runtime(runtime) + global _RUNTIME_RUNNER + _RUNTIME_RUNNER = _DEFAULT_CONTROLLER.runtime_runner + + +def bind_runtime_runner(runner: RuntimeRunner | None) -> None: + _DEFAULT_CONTROLLER.bind_runtime_runner(runner) + global _RUNTIME_RUNNER + _RUNTIME_RUNNER = runner + + +def clear_runtime_runner() -> None: + bind_runtime_runner(None) + + +def bind_code_executor(executor: CodeExecutor | None) -> None: + _DEFAULT_CONTROLLER.bind_code_executor(executor) + global _CODE_EXECUTOR + _CODE_EXECUTOR = executor + + +def clear_code_executor() -> None: + bind_code_executor(None) + + +def register_action(name: str, handler: ActionHandler, *, overwrite: bool = True) -> None: + _DEFAULT_CONTROLLER.register_action(name=name, handler=handler, overwrite=overwrite) + + +def register_action_spec( + name: str, + *, + summary: str = "", + when_to_use: str = "", + params: dict[str, str] | None = None, +) -> None: + _DEFAULT_CONTROLLER.register_action_spec( + name=name, + summary=summary, + when_to_use=when_to_use, + params=params, + ) + + +def list_actions() -> list[str]: + return _DEFAULT_CONTROLLER.list_actions() + + +def describe_actions() -> dict[str, dict[str, Any]]: + return _DEFAULT_CONTROLLER.describe_actions() + + +async def run_action( + action: str, + session_id: str = "", + request_id: str = "", + **kwargs: Any, +) -> ActionResult: + return await _DEFAULT_CONTROLLER.run_action( + action=action, + session_id=session_id, + request_id=request_id, + **kwargs, + ) + + +def register_builtin_actions(controller: ActionController | None = None) -> None: + ctl = controller or _DEFAULT_CONTROLLER + + async def ping(session_id: str = "", request_id: str = "", **kwargs: Any) -> ActionResult: + return { + "ok": True, + "pong": True, + "session_id": session_id, + "request_id": request_id, + "meta": kwargs, + } + + async def echo( + session_id: str = "", + request_id: str = "", + text: str = "", + **kwargs: Any, + ) -> ActionResult: + return { + "ok": True, + "text": text, + "session_id": session_id, + "request_id": request_id, + "meta": kwargs, + } + + async def browser_task( + session_id: str = "", + request_id: str = "", + task: str = "", + timeout_s: int | None = None, + **kwargs: Any, + ) -> ActionResult: + runner = ctl.runtime_runner + if runner is None: + return { + "ok": False, + "error": "runtime_not_bound: call bind_runtime(...) before browser_task", + "session_id": session_id, + "request_id": request_id, + } + + task_text = (task or "").strip() + if not task_text: + return { + "ok": False, + "error": "missing required parameter: task", + "session_id": session_id, + "request_id": request_id, + } + + effective_timeout: int | None = None + if timeout_s is not None: + try: + parsed = int(timeout_s) + if parsed > 0: + effective_timeout = parsed + except Exception: + effective_timeout = None + + return await runner( + task=task_text, + session_id=session_id or None, + request_id=request_id or None, + timeout_s=effective_timeout, + ) + + ctl.register_action("ping", ping, overwrite=True) + ctl.register_action_spec( + "ping", + summary="Health check action.", + when_to_use="Use to verify controller dispatch and session/request threading.", + ) + ctl.register_action("echo", echo, overwrite=True) + ctl.register_action_spec( + "echo", + summary="Echoes provided text and metadata.", + when_to_use="Use for debugging payload passthrough through browser_custom_action.", + params={"text": "string: text to echo back"}, + ) + ctl.register_action("browser_task", browser_task, overwrite=True) + ctl.register_action_spec( + "browser_task", + summary="Runs a free-form browser task through runtime.run_browser_task.", + when_to_use="Use for generic website tasks when no specialized custom action applies.", + params={ + "task": "string: required task prompt for the browser worker", + "timeout_s": "int: optional positive timeout override", + }, + ) + ctl.register_action("run_browser_task", browser_task, overwrite=True) + ctl.register_action_spec( + "run_browser_task", + summary="Alias of browser_task.", + when_to_use="Same behavior as browser_task.", + params={ + "task": "string: required task prompt for the browser worker", + "timeout_s": "int: optional positive timeout override", + }, + ) + + +def register_example_actions(controller: ActionController | None = None) -> None: + register_builtin_actions(controller=controller) + + +__all__ = [ + "BaseController", + "ActionController", + "get_default_controller", + "bind_runtime", + "bind_runtime_runner", + "clear_runtime_runner", + "bind_code_executor", + "clear_code_executor", + "register_action", + "register_action_spec", + "register_builtin_actions", + "register_example_actions", + "browser_worker_action_context", + "list_actions", + "describe_actions", + "run_action", +] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/agent.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/agent.py new file mode 100644 index 0000000000..d4bcd5425e --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/agent.py @@ -0,0 +1,277 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The browser main agent over a facade: its prompts, its rails and context processors, and the profile's hook.""" + +from __future__ import annotations + +import copy +from typing import TYPE_CHECKING, Any, Dict, List, Optional + +from openjiuwen.core.common.logging import logger +from openjiuwen.core.context_engine import ToolResultWindowProcessorConfig +from openjiuwen.core.foundation.llm.model import Model +from openjiuwen.core.foundation.llm.schema.config import ModelRequestConfig +from openjiuwen.core.foundation.tool import McpServerConfig, Tool, ToolCard +from openjiuwen.core.single_agent.rail.base import AgentRail +from openjiuwen.core.single_agent.schema.agent_card import AgentCard +from openjiuwen.core.sys_operation import SysOperation +from openjiuwen.harness.deep_agent import DeepAgent +from openjiuwen.harness.factory import create_deep_agent +from openjiuwen.harness.prompts import resolve_language +from openjiuwen.harness.rails.context_engineer import ContextProcessorRail +from openjiuwen.harness.schema.config import SubAgentConfig +from openjiuwen.harness.schema.decision_policy import DecisionPolicyModel + +from ..rails.runtime_rail import BrowserRuntimeRail +from ..state_processor import BrowserStateContextProcessorConfig +from ..working_context.processor import BrowserWorkingContextProcessorConfig + +if TYPE_CHECKING: + from openjiuwen.harness.workspace.workspace import Workspace + + from .facade import BrowserAgentRuntime + +# Agent checkpoints are namespaced by AgentCard.id inside a conversation. +# Keep the default stable so reconstructing this subagent can restore its +# Session-backed working context on a same-conversation follow-up. +BROWSER_AGENT_CARD_ID = "openjiuwen.browser_agent" +DEFAULT_BROWSER_AGENT_TEMPERATURE = 0.4 +DEFAULT_BROWSER_AGENT_MAX_ITERATIONS = 100 +_BROWSER_MODEL_TEMPERATURE_MARKER = "_browser_agent_temperature" +_BROWSER_PARENT_MODEL_MARKER = "_browser_agent_parent_model" + +DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT_EN = ( + "You are a browser automation agent responsible for executing web tasks directly. " + "Choose the strategy at this agent level and use the available Playwright and runtime tools; " + "the runtime validates targets and outcomes but does not replace your task judgment.\n" + "Every model call includes one runtime-maintained <browser_working_context> followed by the latest " + "<browser_state>. Runtime outcomes and blockers are authoritative; inferred fields are extraction hints, " + "not additional user requirements. Source observations also provide usable evidence. " + "A fresh browser capture occurs initially and after a recognized page mutation; otherwise the cached " + "observation is reused. When the runtime directive requires replanning, change the strategy materially.\n" + "For a simple lookup, prefer a direct search-results URL when the engine and query are known. Use " + "browser_probe_interactives for page controls and browser_probe_cards for repeated results or products. " + "Use the compact PageState target_id with its generation_id directly; never reconstruct guessed CSS from " + "a target ID. When a card has primary_link or href, navigate directly to that URL.\n" + "Use browser_batch_interact when two or more deterministic actions or same-page field extractions are already " + "known. Use a primitive for one uncertain action. Prefer observable condition waits over fixed sleep. Use " + "browser_snapshot only when compact probes are insufficient, and browser_evaluate only for a small exact " + "target or computation. If an older result has a <persisted-output> marker, recall it only when its preview " + "does not contain the needed evidence; recalled targets are not executable after navigation.\n" + "Use canonical fields when convenient, but do not reread a page just to rename already observed facts. " + "Report genuinely unavailable values as unknown. One trustworthy page value or structured result is enough; " + "do not verify the same fact with multiple " + "tools. Stop immediately when the requested outcome is evidenced. The runtime determines final status, so " + "return a concise natural-language result rather than another progress object.\n" + "If an optional capability makes a browser_run_code tool visible, use it only when deterministic tools are " + "insufficient, and never dump the full document. Preserve session continuity and report a concrete blocker " + "when the available browser state or tools cannot complete the task." +) + +DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT_CN = ( + "你是浏览器自动化代理，负责直接完成网页任务。请在当前代理层决定策略并使用可见的 Playwright " + "和 runtime 工具；runtime 负责验证目标和结果，但不代替你的任务判断。\n" + "每次模型调用都会依次提供 runtime 维护的 <browser_working_context> 和最新 <browser_state>。" + "运行结果和阻断项是权威信息；推断字段只是提取提示，不是新增的用户要求。带来源的页面原文也可作为证据。" + "系统仅在初始调用和已识别的" + "页面变更后重新观察；runtime 要求重新规划时，应实质改变策略。\n" + "已知搜索引擎和关键词时，简单查询优先直接构造搜索结果 URL。页面控件使用 " + "browser_probe_interactives，重复结果或商品使用 browser_probe_cards。直接使用 PageState 返回的 " + "target_id 和 generation_id，禁止把 target_id 改写成猜测的 CSS。卡片包含 primary_link 或 href " + "时直接导航该 URL。\n" + "只有两个及以上确定动作或同页多字段提取时使用 browser_batch_interact；单个不确定动作使用基础工具。" + "优先等待可观察条件，不使用固定 sleep。紧凑 Probe 不足时再使用 browser_snapshot；" + "browser_evaluate 仅用于小范围精确目标或计算。旧结果出现 <persisted-output> 且预览不足时才恢复；" + "导航后恢复内容中的目标不可继续操作。\n" + "方便时使用规范字段，但不要仅为改写字段名而重读已经观察到的事实；字段确实不可获取时使用 unknown。" + "一个可信页面值或结构化结果已经足够，不要用多个工具重复验证同一事实。请求结果有证据后立即结束。" + "最终状态由 runtime 决定，只返回简洁自然语言结果，不再维护第二份进度对象。\n" + "只有可选能力明确暴露 browser_run_code 时才使用，并且仅限确定性工具不足的情况；禁止转储完整页面。" + "保持浏览器会话连续；现有页面或工具确实无法完成时，报告具体 blocker。" +) + +DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT: Dict[str, str] = { + "cn": DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT_CN, + "en": DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT_EN, +} + +DEFAULT_BROWSER_AGENT_DESCRIPTION_EN = ( + "Dedicated browser subagent that directly controls the browser with Playwright MCP tools." +) +DEFAULT_BROWSER_AGENT_DESCRIPTION_CN = "专用浏览器子代理，直接使用 Playwright MCP 工具执行网页任务。" +DEFAULT_BROWSER_AGENT_DESCRIPTION: Dict[str, str] = { + "cn": DEFAULT_BROWSER_AGENT_DESCRIPTION_CN, + "en": DEFAULT_BROWSER_AGENT_DESCRIPTION_EN, +} + +_WINDOWED_TOOL_NAMES = [ + "browser_probe_interactives", + "browser_probe_cards", + "browser_snapshot", + "browser_find", + "browser_evaluate", +] + + +def browser_model_with_temperature(model: Model, temperature: float) -> Model: + """Copy a parent model descriptor and override sampling for browser work.""" + resolved_temperature = max(0.0, min(float(temperature), 2.0)) + if getattr(model, _BROWSER_MODEL_TEMPERATURE_MARKER, None) == resolved_temperature: + return model + parent_model = getattr(model, _BROWSER_PARENT_MODEL_MARKER, model) + + model_config = getattr(model, "model_config", None) + if isinstance(model_config, ModelRequestConfig): + browser_model_config = model_config.model_copy( + deep=True, + update={"temperature": resolved_temperature}, + ) + elif model_config is None: + browser_model_config = ModelRequestConfig(temperature=resolved_temperature) + else: + browser_model_config = copy.copy(model_config) + setattr(browser_model_config, "temperature", resolved_temperature) + + model_client_config = getattr(model, "model_client_config", None) + if issubclass(type(model), Model) and model_client_config is not None: + browser_model = Model( + model_client_config=model_client_config, + model_config=browser_model_config, + ) + setattr(browser_model, _BROWSER_MODEL_TEMPERATURE_MARKER, resolved_temperature) + setattr(browser_model, _BROWSER_PARENT_MODEL_MARKER, parent_model) + return browser_model + + # Lightweight test doubles and compatibility model descriptors may not be + # constructible as a concrete Model. Preserve their shape without mutating + # the parent object. + browser_model = copy.copy(model) + browser_model.model_config = browser_model_config + setattr(browser_model, _BROWSER_MODEL_TEMPERATURE_MARKER, resolved_temperature) + setattr(browser_model, _BROWSER_PARENT_MODEL_MARKER, parent_model) + return browser_model + + +def build_main_agent( + runtime: BrowserAgentRuntime, + model: Model, + *, + card: Optional[AgentCard] = None, + system_prompt: Optional[str] = None, + tools: Optional[List[Tool | ToolCard]] = None, + mcps: Optional[List[McpServerConfig]] = None, + subagents: Optional[List[SubAgentConfig | DeepAgent]] = None, + rails: Optional[List[AgentRail]] = None, + enable_task_loop: bool = False, + max_iterations: int = DEFAULT_BROWSER_AGENT_MAX_ITERATIONS, + temperature: float = DEFAULT_BROWSER_AGENT_TEMPERATURE, + workspace: Optional[str | Workspace] = None, + skills: Optional[List[str]] = None, + backend: Optional[Any] = None, + sys_operation: Optional[SysOperation] = None, + language: Optional[str] = None, + prompt_mode: Optional[str] = None, + **config_kwargs: Any, +) -> DeepAgent: + """The browser agent over ``runtime``: the runtime rail, the core rails, the context processors, the prompts.""" + resolved_language = resolve_language(language) + # A decision policy in the model slot is used as is: no temperature copy, no LLM-only context rails. + policy_model = model if isinstance(model, DecisionPolicyModel) else None + browser_model = model if policy_model is not None else browser_model_with_temperature(model, temperature) + if policy_model is not None: + policy_model.bind_runtime(runtime) + + final_card = card or AgentCard( + id=BROWSER_AGENT_CARD_ID, + name="browser_agent", + description=DEFAULT_BROWSER_AGENT_DESCRIPTION.get( + resolved_language, + DEFAULT_BROWSER_AGENT_DESCRIPTION["cn"], + ), + ) + final_prompt = system_prompt or DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT.get( + resolved_language, + DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT["cn"], + ) + + # The browser tools arrive at the first invoke, once the driver has started (BrowserRuntimeRail.before_invoke). + injected_rails: List[AgentRail] = [BrowserRuntimeRail(runtime), *runtime.core_rails()] + processors = [ + ( + "ToolResultWindowProcessor", + ToolResultWindowProcessorConfig( + tool_names=list(_WINDOWED_TOOL_NAMES), + keep_last_k=2, + trim_size=1000, + min_offload_chars=1000, + small_result_trim_size=1000, + ), + ), + ("BrowserStateContextProcessor", BrowserStateContextProcessorConfig(provider=runtime)), + ( + "BrowserWorkingContextProcessor", + BrowserWorkingContextProcessorConfig(language=resolved_language, runtime_projection_only=True), + ), + ] + caller_context_rails = [rail for rail in (rails or []) if isinstance(rail, ContextProcessorRail)] + if policy_model is not None: + config_kwargs.setdefault("enable_model_anomaly_detection_rail", False) + elif caller_context_rails: + for context_rail in caller_context_rails: + context_rail.add_processors(processors) + else: + injected_rails.append(ContextProcessorRail(processors=processors, preset=False)) + + agent = create_deep_agent( + model=browser_model, + card=final_card, + system_prompt=final_prompt, + tools=list(tools or []), + mcps=list(mcps or []), + subagents=subagents, + rails=list(rails or []) + injected_rails, + enable_task_loop=enable_task_loop, + max_iterations=max_iterations, + workspace=workspace, + skills=skills, + backend=backend, + sys_operation=sys_operation, + language=resolved_language, + prompt_mode=prompt_mode, + **config_kwargs, + ) + agent.register_task_resource_cleanup(runtime.release_task_resources, prepare=runtime.acquire_task_resources) + return agent + + +def main_agent( + runtime: BrowserAgentRuntime, + model: Model, + *, + language: str, + max_iterations: int, + workspace: Any, + capabilities: tuple[str, ...], +) -> DeepAgent: + """The profile's main agent: the browser agent with the worker's sampling, over the runtime's capabilities.""" + logger.info("Browser main agent capabilities: %s", ", ".join(capabilities)) + return build_main_agent( + runtime, + model, + language=language, + max_iterations=max_iterations, + temperature=runtime.core.worker_temperature, + workspace=workspace, + ) + + +__all__ = [ + "BROWSER_AGENT_CARD_ID", + "DEFAULT_BROWSER_AGENT_DESCRIPTION", + "DEFAULT_BROWSER_AGENT_MAX_ITERATIONS", + "DEFAULT_BROWSER_AGENT_SYSTEM_PROMPT", + "DEFAULT_BROWSER_AGENT_TEMPERATURE", + "browser_model_with_temperature", + "build_main_agent", + "main_agent", +] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/facade.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/facade.py new file mode 100644 index 0000000000..1dddc04bf6 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/facade.py @@ -0,0 +1,669 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The browser facade over the core runtime: the browser settings, the lifecycle identity and the browser methods.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +import os +from dataclasses import dataclass, replace +from pathlib import Path +from typing import Any, Dict, Iterable, Mapping, Optional +from weakref import WeakSet + +from openjiuwen.core.common.logging import logger +from openjiuwen.core.foundation.tool import McpServerConfig +from openjiuwen.core.runner import Runner +from openjiuwen.harness.tools.browser_move.contract import DriverError, Observe, SurfaceSettings +from openjiuwen.harness.tools.browser_move.runtime.assembly import assemble_surface +from openjiuwen.harness.tools.browser_move.runtime.config import CoreSettings, RunGuardrails, process_environ +from openjiuwen.harness.tools.browser_move.runtime.page_state import Target +from openjiuwen.harness.tools.browser_move.runtime.service import AgentService +from openjiuwen.harness.tools.browser_move.runtime.service_registry import SERVICE_REGISTRY +from openjiuwen.harness.tools.browser_move.runtime.session import AgentRuntime, _tool_args +from openjiuwen.harness.tools.browser_move.runtime.topology import DelegatedWorker + +from .actions import ActionController, BaseController +from .logging import configure_browser_agent_logging, write_browser_agent_audit_artifact +from .settings import BrowserInstanceConfig, browser_surface_settings + +_ACTIVE_BROWSER_RUNTIMES: WeakSet[Any] = WeakSet() +_PROVISIONINGS = ("remote", "managed", "extension") + + +@dataclass(frozen=True) +class BrowserServiceIdentity: + """Stable process-level identity for one logical browser service; ``key`` is the registry's string for it.""" + + browser_key: str + profile_name: str + driver_mode: str + display_mode: str + managed_args: tuple[str, ...] + browser_binary: str + user_data_dir: str + cdp_endpoint: str + server_id: str + + @property + def key(self) -> str: + return json.dumps(dataclasses.asdict(self), sort_keys=True) + + @classmethod + def from_key(cls, key: str) -> BrowserServiceIdentity: + fields = json.loads(key) + return cls(**{**fields, "managed_args": tuple(fields["managed_args"])}) + + +def normalize_browser_binary(value: str) -> str: + raw = str(value or "").strip() + if not raw: + return "" + return os.path.normcase(os.path.normpath(str(Path(raw).expanduser()))) + + +def _with_workspace(settings: Any, workspace: Any) -> Any: + if workspace is None or not hasattr(settings, "workspace_root"): + return settings + root = getattr(workspace, "root_path", workspace) + return replace(settings, workspace_root=Path(root).expanduser().resolve()) + + +def _driver_mode(instance: BrowserInstanceConfig | None, settings: Any) -> str: + explicit = (instance.driver_mode or "").strip().lower() if instance else "" + if not explicit: + return settings.provisioning + if explicit not in _PROVISIONINGS: + raise ValueError( + "BrowserInstanceConfig.provisioning (driver_mode) must be one of: remote, managed, extension " + f"(got {explicit!r}); leave it empty to use BROWSER_PROVISIONING" + ) + return explicit + + +def _output_dir(mcp_cfg: McpServerConfig, settings: Any) -> Path: + params = getattr(mcp_cfg, "params", {}) or {} + raw = str(params.get("cwd", "")).strip() + return Path(raw).expanduser() if raw else settings.mcp_cwd + + +def _identity( + instance: BrowserInstanceConfig | None, settings: Any, mcp_cfg: McpServerConfig, driver_mode: str +) -> BrowserServiceIdentity: + """The identity after the instance is applied to the settings: profile, user data dir and binary are theirs.""" + headless = any(arg == "--headless" or arg.startswith("--headless=") for arg in settings.managed_args) + display_mode = ("headless" if headless else "headed") if driver_mode == "managed" else driver_mode + user_data_dir = str(settings.user_data_dir).strip() + params = getattr(mcp_cfg, "params", {}) or {} + env_map = dict(params.get("env", {}) or {}) + return BrowserServiceIdentity( + browser_key=instance.sanitized_key() if instance else "", + profile_name=settings.profile_name, + driver_mode=driver_mode, + display_mode=display_mode, + managed_args=settings.managed_args, + browser_binary=normalize_browser_binary(settings.browser_binary), + user_data_dir=( + os.path.normcase(os.path.normpath(str(Path(user_data_dir).expanduser()))) + if user_data_dir + else f"profile:{settings.profile_name}" + ), + cdp_endpoint=str(env_map.get("PLAYWRIGHT_MCP_CDP_ENDPOINT") or "").strip(), + server_id=(mcp_cfg.server_id or "").strip() or mcp_cfg.server_name, + ) + + +class BrowserAgentRuntime(AgentRuntime): + """The browser runner: the core runtime over the browser surface, plus the browser views its rails read.""" + + def __init__( + self, + provider: str, + api_key: str, + api_base: str, + model_name: str, + mcp_cfg: McpServerConfig, + guardrails: RunGuardrails, + instance: Optional[BrowserInstanceConfig] = None, + allowed_tool_names: Optional[Iterable[str]] = None, + *, + core_settings: Optional[CoreSettings] = None, + browser_settings: Optional[SurfaceSettings] = None, + workspace: Any = None, + language: str = "cn", + ) -> None: + core = core_settings or CoreSettings.from_process_env() + driver = (instance.driver if instance else "").strip() + if driver: + core = replace(core, driver=driver) + settings = _with_workspace(browser_settings or browser_surface_settings(), workspace) + driver_mode = _driver_mode(instance, settings) + if instance is not None: + settings = settings.with_instance(instance) + output_dir = _output_dir(mcp_cfg, settings) + settings = replace(settings, mcp_server=mcp_cfg, mcp_cwd=output_dir) + self._lifecycle_identity = _identity(instance, settings, mcp_cfg, driver_mode) + configure_browser_agent_logging(core) + super().__init__( + core, + assemble_surface(core, process_environ(), settings=settings), + provider=provider, + api_key=api_key, + api_base=api_base, + model_name=model_name, + guardrails=guardrails, + allowed_tool_names=allowed_tool_names, + workspace=workspace, + language=language, + output_dir=output_dir, + identity=self._lifecycle_identity.key, + ) + self.mcp_cfg = mcp_cfg + self._custom_action_tool = None + self._list_actions_tool = None + self._controller: BaseController = ActionController() + self._bound_session: Any = None + self._tracker: Any = None + self._semantic_mark: tuple[int, Any] | None = None + self.observations_reused = 0 + _ACTIVE_BROWSER_RUNTIMES.add(self) + + @property + def core_settings(self) -> CoreSettings: + return self.core + + @property + def lifecycle_identity(self) -> BrowserServiceIdentity: + return self._lifecycle_identity + + @property + def controller(self) -> BaseController: + return self._controller + + @property + def page_id(self) -> str: + return self.page_state.page_id + + @property + def generation_id(self) -> str: + return self.page_state.generation_id + + def export_page_state(self) -> Dict[str, Any]: + return self.page_state.export() + + @property + def _semantic_state_tracker(self) -> Any: + if self._tracker is None: + factory = self.surface.profile.semantic_tracker + self._tracker = _NoSemanticState() if factory is None else factory() + return self._tracker + + def _semantic_changed(self) -> bool: + """Whether the generation or the last observation moved since the previous semantic observation.""" + mark = (self.page_state.generation, self.page_state.last_observation) + previous, self._semantic_mark = self._semantic_mark, mark + return previous is not None and mark != previous + + def set_task_requested_fields(self, fields: Iterable[str]) -> None: + """Keep PageState projection scoped to fields requested by this task.""" + + self.page_state.set_requested_fields(fields) + + def resolve_model_target_id(self, target_id: str) -> Target: + """Resolve or safely refresh a runtime-owned model target.""" + + page_state = self.page_state + normalized = str(target_id or "").strip() + target = page_state.get_target(normalized) + if target is None: + raise ValueError( + f"Unknown PageState target_id {normalized}; current generation is {page_state.generation_id}." + ) + if target.generation != page_state.generation: + target = page_state.get_target(page_state.refresh_target_id(normalized)) + if target is None: + raise ValueError(f"PageState target_id {normalized} could not be refreshed in {page_state.generation_id}.") + return target + + def target_recovery_state(self, target_id: str) -> Dict[str, Any]: + """Return bounded current-generation recovery details for one target.""" + + page_state = self.page_state + return { + "current_generation": page_state.generation_id, + "candidate_fresh_targets": page_state.recovery_candidates(target_id, limit=5), + } + + def resolve_primary_link(self, tool_args: Any) -> str: + parsed = _tool_args(tool_args) + explicit = str(parsed.get("primary_link") or parsed.get("href") or parsed.get("url") or "").strip() + if explicit: + return explicit + target_id = str(parsed.get("target_id") or parsed.get("target") or "").strip() + if target_id.startswith("t_g"): + try: + target = self.resolve_model_target_id(target_id) + except ValueError: + target = None + if target is not None and target.href: + return target.href + page_state = self.page_state + for key in ("selector", "target"): + selector = str(parsed.get(key) or "").strip() + generation_and_href = page_state.selector_primary_links.get(selector) + if generation_and_href and generation_and_href[0] == page_state.generation: + return generation_and_href[1] + return "" + + def register_runtime_tools(self) -> None: + """The two custom-action tools in ``Runner.resource_mgr``, once.""" + if self._custom_action_tool is not None: + return + from .tools import BrowserCustomActionTool, BrowserListActionsTool + + self._custom_action_tool = BrowserCustomActionTool(self) + self._list_actions_tool = BrowserListActionsTool(self) + self._register_runtime_tool(self._custom_action_tool, tool_name="browser_custom_action") + self._register_runtime_tool(self._list_actions_tool, tool_name="browser_list_custom_actions") + + @staticmethod + def _register_runtime_tool(tool_obj: Any, *, tool_name: str) -> None: + add_result = Runner.resource_mgr.add_tool(tool_obj, tag="agent.playwright.runtime") + if add_result is None or getattr(add_result, "is_ok", lambda: False)(): + return + error_value = getattr(add_result, "value", add_result) + if "already exist" in str(error_value): + return + raise RuntimeError(f"Failed to register {tool_name} tool: {error_value}") + + async def ensure_runtime_ready(self) -> None: + """The core's session, with the custom actions registered on the controller for a new session.""" + _ACTIVE_BROWSER_RUNTIMES.add(self) + await super().ensure_runtime_ready() + if self.session is self._bound_session: + return + self._bound_session = self.session + self._controller.register_builtin_actions() + custom_actions = self.surface.profile.custom_actions + if custom_actions is not None: + custom_actions(self._controller, self.context) + + async def acquire_task_resources(self) -> None: + _ACTIVE_BROWSER_RUNTIMES.add(self) + await super().acquire_task_resources() + + async def ensure_started(self) -> None: + """The core's start; the delegated worker's host gets the two custom-action tools.""" + await super().ensure_started() + if isinstance(self.topology, DelegatedWorker): + self.register_runtime_tools() + + async def run_browser_task( + self, + task: str, + session_id: Optional[str] = None, + request_id: Optional[str] = None, + timeout_s: Optional[int] = None, + ) -> Dict[str, Any]: + return await self.run_task(task, session_id, request_id, timeout_s) + + def main_agent_answer( + self, session_id: str, request_id: str, *, result: Optional[Dict[str, Any]] = None, error: Optional[str] = None + ) -> Dict[str, Any]: + """The core's answer plus ``page``: the location from the browser context of the state.""" + answer = super().main_agent_answer(session_id, request_id, result=result, error=error) + surface = self.page_state.surface_context + answer["page"] = {"url": surface.url if surface else "", "title": surface.title if surface else ""} + return answer + + async def capture_browser_state(self, *, action_group_id: str = "") -> Dict[str, Any]: + """The observation the next model call reads: the last one while it is current, else a fresh one.""" + await self.ensure_runtime_ready() + page_state = self.page_state + observation = page_state.fresh_observation + dom_error = None + snapshot_audit: Dict[str, Any] = {} + if observation is None: + try: + observation = await self.context.observe(Observe()) + except DriverError as exc: + dom_error = f"page observation failed: {exc}" + logger.warning("[BrowserAgentRuntime] observation failed: %s", exc, exc_info=True) + else: + self.observations_reused += 1 + if observation is not None: + snapshot_audit = write_browser_agent_audit_artifact("ax_snapshot", observation.native_text or "") + + metadata, metadata_error = await self.metadata() + exported = page_state.export() + + errors = [error for error in (dom_error, metadata_error) if error] + semantic_state = metadata.get("semantic_state") + if not isinstance(semantic_state, dict): + semantic_state = {} + semantic_state.update( + { + "url": metadata.get("url") or exported["url"], + "field_coverage": sorted(page_state.field_coverage), + } + ) + semantic_tracker = self._semantic_state_tracker + if metadata_error: + semantic_progress = semantic_tracker.latest + semantic_progress.update( + { + "progress": "unknown", + "observable_progress": False, + "capture_error": metadata_error, + } + ) + else: + semantic_progress = semantic_tracker.observe( + semantic_state, + action_group_id=action_group_id, + changed=self._semantic_changed(), + ) + semantic_state = self._with_semantic_provenance( + semantic_progress, + fallback_state=semantic_state, + ) + return { + "ok": not errors, + "error": "; ".join(errors) or None, + "url": metadata.get("url") or exported["url"], + "title": metadata.get("title") or exported["title"], + "tabs": metadata.get("tabs") or [], + "page_position": metadata.get("page_position") or {}, + "semantic_state": semantic_state, + "semantic_progress": semantic_progress, + "field_coverage": semantic_state.get("field_coverage") or [], + "page_state": exported, + "dom": "", + "dom_error": dom_error, + "audit": {"ax_snapshot": snapshot_audit} if snapshot_audit else {}, + } + + async def capture_reconciliation_browser_state(self, *, action_group_id: str) -> Dict[str, Any]: + """Reconcile an ambiguous mutation from the surface metadata alone, without observing.""" + + await self.ensure_runtime_ready() + metadata, metadata_error = await self.metadata() + page_state = self.export_page_state() + semantic_state = metadata.get("semantic_state") + if not isinstance(semantic_state, dict): + semantic_state = {} + semantic_state.update( + { + "url": metadata.get("url") or page_state.get("url") or "", + "field_coverage": page_state.get("field_coverage") or [], + } + ) + tracker = self._semantic_state_tracker + if metadata_error: + semantic_progress = tracker.latest + semantic_progress.update( + { + "progress": "unknown", + "observable_progress": False, + "capture_error": metadata_error, + } + ) + else: + semantic_progress = tracker.observe( + semantic_state, action_group_id=action_group_id, changed=self._semantic_changed() + ) + semantic_state = self._with_semantic_provenance( + semantic_progress, + fallback_state=semantic_state, + ) + return { + "ok": not metadata_error, + "error": metadata_error, + "url": metadata.get("url") or page_state.get("url") or "", + "title": metadata.get("title") or page_state.get("title") or "", + "tabs": metadata.get("tabs") or [], + "page_position": metadata.get("page_position") or {}, + "semantic_state": semantic_state, + "semantic_progress": semantic_progress, + "field_coverage": semantic_state.get("field_coverage") or [], + "page_state": page_state, + "dom": "", + "dom_error": None, + "reconciliation_only": True, + } + + async def capture_compact_browser_state(self, *, action_group_id: str) -> Dict[str, Any]: + """Merge completed read-only observations without another driver round trip.""" + page_state = self.export_page_state() + semantic_tracker = self._semantic_state_tracker + semantic_state = semantic_tracker.current_state + semantic_state.update( + { + "url": page_state.get("url") or semantic_state.get("url") or "", + "field_coverage": page_state.get("field_coverage") or semantic_state.get("field_coverage") or [], + } + ) + semantic_progress = semantic_tracker.observe( + semantic_state, + action_group_id=action_group_id, + changed=self._semantic_changed(), + ) + semantic_state = self._with_semantic_provenance( + semantic_progress, + fallback_state=semantic_state, + ) + return { + "ok": True, + "error": None, + "url": page_state.get("url") or "", + "title": page_state.get("title") or "", + "tabs": [], + "page_position": {}, + "semantic_state": semantic_state, + "semantic_progress": semantic_progress, + "field_coverage": semantic_state["field_coverage"], + "page_state": page_state, + "dom": "", + "dom_error": None, + } + + def _with_semantic_provenance( + self, + semantic_progress: Dict[str, Any], + *, + fallback_state: Dict[str, Any], + ) -> Dict[str, Any]: + tracked_state = semantic_progress.get("semantic_state") + semantic_state = dict(tracked_state) if isinstance(tracked_state, dict) else dict(fallback_state) + semantic_state["generation_id"] = self.generation_id + semantic_progress["semantic_state"] = semantic_state + return semantic_state + + async def run_custom_action( + self, + *, + action: str, + session_id: str = "", + request_id: str = "", + params: Optional[Dict[str, Any]] = None, + ) -> Dict[str, Any]: + await self.ensure_runtime_ready() + self._controller.bind_runtime(self) + return await self._controller.run_action( + action=action, + session_id=session_id, + request_id=request_id, + **(params or {}), + ) + + async def probe_for_policy(self, source: str, params: Dict[str, Any]) -> Dict[str, Any]: + """Run a policy-owned probe script and register its elements as PageState targets.""" + await self.ensure_runtime_ready() + probe = self.surface.profile.policy_probe + if probe is None: + error = f"surface {self.surface.profile.name!r} has no policy probe" + return {"ok": False, "error": error, "elements": [], "page_state": self.export_page_state()} + return await probe(self.context, source, params) + + async def activate_page(self, url: str) -> bool: + """Bring the context at ``url`` (else the current one) to the front so its timers run and its menus render.""" + await self.ensure_runtime_ready() + activate = self.surface.profile.activate + return False if activate is None else await activate(self.context, url) + + async def list_actions(self) -> Dict[str, Any]: + return { + "ok": True, + "actions": self._controller.list_actions(), + "details": self._controller.describe_actions(), + } + + async def shutdown(self) -> None: + try: + await super().shutdown() + finally: + _ACTIVE_BROWSER_RUNTIMES.discard(self) + self._bound_session = None + + async def release_task_resources(self) -> bool: + """Release task bindings while preserving Chrome and its profile.""" + fully_released = await super().release_task_resources() + if fully_released: + self._semantic_state_tracker.reset() + return fully_released + + async def reset(self) -> None: + try: + await super().reset() + finally: + self._bound_session = None + self._semantic_state_tracker.reset() + + @property + def semantic_progress(self) -> Dict[str, Any]: + """Return the latest task-local semantic progress observation.""" + return self._semantic_state_tracker.latest + + def acknowledge_semantic_replan(self) -> None: + """Acknowledge one materially different action selected after replanning.""" + self._semantic_state_tracker.acknowledge_replan() + + def reset_semantic_task(self) -> None: + """Start semantic loop tracking for a new user task without resetting Chrome.""" + self._semantic_state_tracker.reset() + + +class _NoSemanticState: + """The tracker of a surface without semantic state: progress is never observable.""" + + @property + def latest(self) -> Dict[str, Any]: + return {} + + @property + def current_state(self) -> Dict[str, Any]: + return {} + + def observe(self, state: Mapping[str, Any], *, action_group_id: str, changed: bool) -> Dict[str, Any]: + return {"progress": "unknown", "observable_progress": False} + + def reset(self) -> None: + pass + + def acknowledge_replan(self) -> None: + pass + + +async def reset_active_browser_runtimes() -> int: + """Reset every live runtime owned by this process.""" + runtimes = list(_ACTIVE_BROWSER_RUNTIMES) + if not runtimes: + return 0 + + results = await asyncio.gather( + *(runtime.reset() for runtime in runtimes), + return_exceptions=True, + ) + reset_count = 0 + for runtime, result in zip(runtimes, results): + if isinstance(result, BaseException): + logger.warning( + "Failed to reset active runtime %s: %s", + id(runtime), + result, + ) + continue + reset_count += 1 + return reset_count + + +async def reset_registered_managed_browser( + *, + browser_key: str, + profile_name: str, + display_mode: str, + browser_binary: str, +) -> int: + """Stop a preserved managed device matching one logical identity.""" + wanted = ( + str(browser_key or "").strip(), + str(profile_name or "").strip(), + str(display_mode or "").strip().lower(), + normalize_browser_binary(browser_binary), + ) + reset_count = 0 + for key in SERVICE_REGISTRY.managed_driver_identities(): + identity = BrowserServiceIdentity.from_key(key) + actual = (identity.browser_key, identity.profile_name, identity.display_mode, identity.browser_binary) + if identity.driver_mode == "managed" and actual == wanted and await AgentService.reset_lifecycle_resources(key): + reset_count += 1 + return reset_count + + +async def reset_managed_browser_runtime( + *, + browser_key: str = "", + profile_name: str = "jiuwenclaw", + display_mode: str, + browser_binary: str = "", +) -> int: + """Reset only the configured managed device, including its idle handle.""" + normalized_binary = normalize_browser_binary(browser_binary) + normalized_key = str(browser_key or "").strip() + normalized_profile = str(profile_name or "").strip() + normalized_mode = str(display_mode or "").strip().lower() + + reset_count = 0 + for runtime in list(_ACTIVE_BROWSER_RUNTIMES): + identity = runtime.lifecycle_identity + matches_browser = identity.browser_key == normalized_key + matches_profile = identity.profile_name == normalized_profile + matches_mode = identity.display_mode == normalized_mode + matches_binary = identity.browser_binary == normalized_binary + if identity.driver_mode != "managed": + continue + if not matches_browser or not matches_profile: + continue + if not matches_mode or not matches_binary: + continue + try: + await runtime.reset() + except Exception as exc: + logger.warning( + "Failed to reset the managed runtime %s: %s", + id(runtime), + exc, + ) + else: + reset_count += 1 + + reset_count += await reset_registered_managed_browser( + browser_key=normalized_key, + profile_name=normalized_profile, + display_mode=normalized_mode, + browser_binary=normalized_binary, + ) + return reset_count diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/logging.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/logging.py new file mode 100644 index 0000000000..e8846ec6a2 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/logging.py @@ -0,0 +1,161 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The dedicated browser agent log, its audit artifacts and the tool-call logging hook.""" + +from __future__ import annotations + +import gzip +import hashlib +import json +import logging +import re +from pathlib import Path +from threading import Lock +from typing import Any + +from openjiuwen.core.common.logging import logger as common_logger +from openjiuwen.core.common.logging.browser_context import is_browser_agent_log_context +from openjiuwen.core.common.logging.tool_call import register_tool_call_logger +from openjiuwen.harness.tools.browser_move.runtime.config import CoreSettings + +_BROWSER_LOGGER_NAME = "openjiuwen.browser_agent" +_BROWSER_HANDLER_MARKER = "_openjiuwen_browser_agent_file_handler" +_BROWSER_LOG_ANNOUNCED_MARKER = "_openjiuwen_browser_agent_file_announced" +_AUDIT_RAW_ATTR = "_openjiuwen_browser_agent_audit_raw" +_LOCK = Lock() +_AUDIT_LOCK = Lock() + + +def _is_browser_tool_name(tool_name: str) -> bool: + lowered_name = tool_name.lower() + return "browser_" in lowered_name or lowered_name.startswith("mcp_playwright") + + +def configure_browser_agent_logging(settings: CoreSettings) -> logging.Logger: + """Apply the logging part of ``settings`` and send the tool-call lines to the dedicated agent log.""" + browser_logger = _apply_settings(settings) + register_tool_call_logger(_is_browser_tool_name, browser_agent_log_info) + return browser_logger + + +def get_browser_agent_logger() -> logging.Logger: + """Return the dedicated agent logger, configured from the process settings until configured explicitly.""" + browser_logger = logging.getLogger(_BROWSER_LOGGER_NAME) + if not hasattr(browser_logger, _AUDIT_RAW_ATTR): + _apply_settings(CoreSettings.logging_from_process_env()) + return browser_logger + + +def _apply_settings(settings: CoreSettings) -> logging.Logger: + browser_logger = logging.getLogger(_BROWSER_LOGGER_NAME) + log_path = None if settings.log_file is None else settings.log_file.expanduser().resolve() + with _LOCK: + browser_logger.setLevel(settings.log_level) + browser_logger.propagate = settings.log_mirror_common + kept = False + for handler in list(browser_logger.handlers): + if not getattr(handler, _BROWSER_HANDLER_MARKER, False): + continue + if log_path is not None and getattr(handler, "baseFilename", None) == str(log_path): + handler.setLevel(settings.log_level) + kept = True + continue + browser_logger.removeHandler(handler) + handler.close() + if log_path is not None and not kept: + log_path.parent.mkdir(parents=True, exist_ok=True) + file_handler = logging.FileHandler(log_path, mode="w", encoding="utf-8") + setattr(file_handler, _BROWSER_HANDLER_MARKER, True) + file_handler.setLevel(settings.log_level) + file_handler.setFormatter(logging.Formatter("%(asctime)s | browser_agent | %(levelname)s | %(message)s")) + browser_logger.addHandler(file_handler) + if not getattr(browser_logger, _BROWSER_LOG_ANNOUNCED_MARKER, False): + common_logger.info("[BROWSER_AGENT_LOG] dedicated agent log file enabled: %s", str(log_path)) + setattr(browser_logger, _BROWSER_LOG_ANNOUNCED_MARKER, True) + setattr(browser_logger, _AUDIT_RAW_ATTR, settings.log_audit_raw) + return browser_logger + + +def _log_path(browser_logger: logging.Logger) -> Path | None: + for handler in browser_logger.handlers: + if getattr(handler, _BROWSER_HANDLER_MARKER, False): + return Path(handler.baseFilename) + return None + + +def browser_agent_log_info(message: str, *args: Any) -> None: + browser_logger = get_browser_agent_logger() + browser_logger.info(message, *args) + + if _log_path(browser_logger) is None: + common_logger.info(message, *args) + + +def browser_agent_log_warning(message: str, *args: Any) -> None: + browser_logger = get_browser_agent_logger() + browser_logger.warning(message, *args) + + if _log_path(browser_logger) is None: + common_logger.warning(message, *args) + + +def browser_agent_log_error(message: str, *args: Any) -> None: + browser_logger = get_browser_agent_logger() + browser_logger.error(message, *args) + + if _log_path(browser_logger) is None: + common_logger.error(message, *args) + + +def write_browser_agent_audit_artifact(kind: str, raw: Any) -> dict[str, Any]: + """Persist an opted-in, content-addressed raw observation.""" + if isinstance(raw, str): + raw_text = raw + else: + raw_text = json.dumps(raw, ensure_ascii=False, default=str) + raw_bytes = raw_text.encode("utf-8", "ignore") + digest = hashlib.sha256(raw_bytes).hexdigest() + audit = { + "raw_size_bytes": len(raw_bytes), + "raw_sha256": digest[:16], + "stored": False, + } + browser_logger = get_browser_agent_logger() + if not is_browser_agent_log_context() or not getattr(browser_logger, _AUDIT_RAW_ATTR): + return audit + + log_path = _log_path(browser_logger) + if log_path is None: + return audit + safe_kind = re.sub(r"[^a-z0-9_-]+", "_", str(kind or "observation").lower()).strip("_") + safe_kind = safe_kind or "observation" + artifact_name = f"{safe_kind}_{digest[:16]}.json.gz" + artifact_path = log_path.parent / "browser_agent_audit" / artifact_name + try: + with _AUDIT_LOCK: + artifact_path.parent.mkdir(parents=True, exist_ok=True) + if not artifact_path.exists(): + with gzip.open(artifact_path, mode="wt", encoding="utf-8") as stream: + json.dump( + {"kind": safe_kind, "sha256": digest, "raw": raw_text}, + stream, + ensure_ascii=False, + separators=(",", ":"), + ) + audit["stored"] = True + browser_logger.info( + "[BROWSER_AUDIT] kind=%s sha256=%s size=%s artifact=%s", + safe_kind, + digest[:16], + len(raw_bytes), + artifact_name, + ) + except OSError as exc: + browser_logger.warning( + "[BROWSER_AUDIT_ERROR] kind=%s sha256=%s error=%s", + safe_kind, + digest[:16], + exc, + ) + return audit diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/main.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/main.py similarity index 58% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/main.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/runner/main.py index ed12188ba0..86dd2a7edf 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/main.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/main.py @@ -1,35 +1,31 @@ # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""CLI entrypoint for Playwright browser runtime.""" +"""CLI entrypoint for the runtime demo.""" from __future__ import annotations import asyncio import json -import os from openjiuwen.core.common.logging import logger -from openjiuwen.core.runner import Runner - -# Use browser_move client subclasses for MCP transport retry and timeout handling. -import openjiuwen.core.runner.resources_manager.tool_manager as _tool_mgr_mod -from ..clients.stdio_client import BrowserMoveStdioClient -from ..clients.streamable_http_client import BrowserMoveStreamableHttpClient - -_tool_mgr_mod.StdioClient = BrowserMoveStdioClient -_tool_mgr_mod.StreamableHttpClient = BrowserMoveStreamableHttpClient - -from .config import ( +from openjiuwen.harness.tools.browser_move.runtime.config import ( MISSING_API_KEY_MESSAGE, - build_runtime_settings, + CoreSettings, load_repo_dotenv, + process_environ, ) -from .runtime import BrowserAgentRuntime +from openjiuwen.harness.tools.browser_move.runtime.topology import WORKER_LANGUAGE + +from .facade import BrowserAgentRuntime +from .settings import browser_surface_settings, build_runtime_settings, read_demo_inputs async def main() -> None: load_repo_dotenv() - settings = build_runtime_settings() + environ = process_environ() + core_settings = CoreSettings.from_env(environ) + browser_settings = browser_surface_settings(environ) + settings = build_runtime_settings(core_settings=core_settings, browser_settings=browser_settings) if not settings.api_key: raise RuntimeError(MISSING_API_KEY_MESSAGE) @@ -40,25 +36,26 @@ async def main() -> None: model_name=settings.model_name, mcp_cfg=settings.mcp_cfg, guardrails=settings.guardrails, + core_settings=core_settings, + browser_settings=browser_settings, ) - initial_query = (os.getenv("AGENT_QUERY") or "").strip() - session_id = (os.getenv("AGENT_SESSION_ID") or "").strip() or "demo-browser-session" + initial_query, session_id = read_demo_inputs(environ) try: await runtime.ensure_started() - mcp_tools = await Runner.resource_mgr.get_mcp_tool_infos(server_id=settings.mcp_cfg.server_id) or [] + tools = runtime.agent_parts(image_support=False, language=WORKER_LANGUAGE).tools logger.info("=" * 72) - logger.info("Playwright Browser Runtime") + logger.info("Playwright Runtime") logger.info("=" * 72) logger.info(f"Model provider: {settings.provider}") logger.info(f"Model: {settings.model_name}") - logger.info(f"MCP command: {settings.mcp_cfg.params.get('command')}") - logger.info(f"MCP args: {settings.mcp_cfg.params.get('args')}") - logger.info(f"Discovered browser tools: {len(mcp_tools)}") - for item in mcp_tools: - logger.info(f" - {getattr(item, 'name', 'unknown')}") + logger.info(f"Server command: {settings.mcp_cfg.params.get('command')}") + logger.info(f"Server args: {settings.mcp_cfg.params.get('args')}") + logger.info(f"Tools: {len(tools)}") + for tool in tools: + logger.info(f" - {tool.card.name}") logger.info("=" * 72) logger.info(f"Session: {session_id}") logger.info("Continuous mode: enter a task and press Enter.") diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/server_config.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/server_config.py new file mode 100644 index 0000000000..e8f95f1562 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/server_config.py @@ -0,0 +1,130 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The client-side config of browser_move served through its own server entry point.""" + +from __future__ import annotations + +import json +import os +import shlex +from pathlib import Path +from typing import Any, Mapping + +from openjiuwen.core.foundation.tool import McpServerConfig +from openjiuwen.harness.tools.browser_move.runtime.config import ( + RuntimeServerSettings, + process_environ, + runtime_server_child_env, +) + +from .settings import resolve_playwright_mcp_cwd + +_SUPPORTED_CLIENT_TYPES = {"stdio", "sse", "streamable-http", "streamable_http", "http"} + + +def _parse_args(raw: str) -> list[str]: + text = (raw or "").strip() + if not text: + return [] + if text.startswith("["): + try: + parsed = json.loads(text) + if isinstance(parsed, list): + return [str(item) for item in parsed] + except Exception: + pass + try: + return shlex.split(text, posix=(os.name != "nt")) + except Exception: + return text.split() + + +def _runtime_cwd() -> Path: + path = Path(resolve_playwright_mcp_cwd()).expanduser() + path.mkdir(parents=True, exist_ok=True) + return path + + +def _server_script() -> Path: + return Path(__file__).resolve().parents[3] / "playwright_runtime_mcp_server.py" + + +def _normalize_client_type(client_type: str) -> str: + value = (client_type or "").strip().lower() + if value in {"http", "streamable_http"}: + return "streamable-http" + return value + + +def _runtime_path(settings: RuntimeServerSettings, transport: str) -> str: + default_path = "/mcp" if _normalize_client_type(transport) == "streamable-http" else "/sse" + path = settings.path or default_path + if not path.startswith("/"): + return f"/{path}" + return path + + +def _build_server_url(settings: RuntimeServerSettings, transport: str) -> str: + return f"http://{settings.host}:{settings.port}{_runtime_path(settings, transport)}" + + +def build_browser_runtime_mcp_config( + settings: RuntimeServerSettings | None = None, + *, + environ: Mapping[str, str] | None = None, +) -> McpServerConfig | None: + """The server config an OpenJiuWen agent registers to reach the runtime server.""" + environ = process_environ() if environ is None else environ + settings = settings or RuntimeServerSettings.from_env(environ) + if not settings.enabled: + return None + + client_type = _normalize_client_type(settings.client_type) + if client_type not in _SUPPORTED_CLIENT_TYPES: + raise ValueError("PLAYWRIGHT_RUNTIME_MCP_CLIENT_TYPE must be stdio, sse, or streamable-http.") + + if client_type in {"sse", "streamable-http"}: + return McpServerConfig( + server_id=settings.server_id, + server_name=settings.server_name, + server_path=settings.server_path or _build_server_url(settings, client_type), + client_type=client_type, + ) + + server_script = _server_script() + if not server_script.exists(): + raise FileNotFoundError(f"playwright runtime server entry point not found: {server_script}") + + args = ( + _parse_args(settings.args) + if settings.args + else [ + "-m", + "openjiuwen.harness.tools.browser_move.playwright_runtime_mcp_server", + "--transport", + "stdio", + "--no-banner", + "--log-level", + "ERROR", + ] + ) + + params: dict[str, Any] = { + "command": settings.command, + "args": args, + "cwd": str(_runtime_cwd()), + } + if settings.timeout_s > 0: + params["timeout_s"] = settings.timeout_s + + child_env = runtime_server_child_env(environ) + if child_env: + params["env"] = child_env + + return McpServerConfig( + server_id=settings.server_id, + server_name=settings.server_name, + server_path=settings.server_path or "stdio://playwright-runtime-wrapper", + client_type="stdio", + params=params, + ) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/settings.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/settings.py new file mode 100644 index 0000000000..2c1b8376b4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/settings.py @@ -0,0 +1,131 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The browser runner's settings: the instance identity, the runtime settings and the facades over the process env.""" + +from __future__ import annotations + +import os +import re +from dataclasses import dataclass +from typing import Any, Mapping, Optional + +from openjiuwen.core.foundation.tool import McpServerConfig +from openjiuwen.harness.tools.browser_move.contract import SurfaceBundle, SurfaceSettings +from openjiuwen.harness.tools.browser_move.runtime.config import CoreSettings, RunGuardrails +from openjiuwen.harness.tools.browser_move.runtime.utils.env import SUPPORTED_MODEL_PROVIDERS, parse_command_args +from openjiuwen.harness.tools.browser_move.surfaces import SURFACES + + +@dataclass(frozen=True) +class BrowserInstanceConfig: + """Per-agent device identity; agents with the same ``key`` share one device, an empty key shares the default.""" + + key: str = "" + driver_mode: str = "" + managed_port: int = 0 + user_data_dir: str = "" + profile_name: str = "" + cdp_url: str = "" + browser_binary: str = "" + launch_args: str = "" + provisioning: str = "" + driver: str = "" + + def __post_init__(self) -> None: + if self.provisioning and self.driver_mode and self.provisioning != self.driver_mode: + raise ValueError( + f"BrowserInstanceConfig: provisioning={self.provisioning!r} conflicts with " + f"driver_mode={self.driver_mode!r} (driver_mode is an alias of provisioning)" + ) + provisioning = self.provisioning or self.driver_mode + object.__setattr__(self, "provisioning", provisioning) + object.__setattr__(self, "driver_mode", provisioning) + + def sanitized_key(self) -> str: + """Return the key reduced to id-safe characters (``[A-Za-z0-9_-]``).""" + return re.sub(r"[^A-Za-z0-9_-]+", "-", (self.key or "").strip()).strip("-") + + +@dataclass(frozen=True) +class RuntimeSettings: + provider: str + api_key: str + api_base: str + model_name: str + mcp_cfg: McpServerConfig + guardrails: RunGuardrails + instance: Optional[BrowserInstanceConfig] = None + + +def read_demo_inputs(environ: Mapping[str, str]) -> tuple[str, str]: + """``(query, session_id)`` of the demo CLI in ``main.py``.""" + query = (environ.get("AGENT_QUERY") or "").strip() + session_id = (environ.get("AGENT_SESSION_ID") or "").strip() or "demo-browser-session" + return query, session_id + + +def browser_surface(environ: Mapping[str, str] | None = None, settings: Any = None) -> SurfaceBundle: + return SURFACES.create("browser", os.environ if environ is None else environ, None, settings) + + +def browser_surface_settings(environ: Mapping[str, str] | None = None) -> SurfaceSettings: + return browser_surface(environ).settings + + +def resolve_playwright_mcp_cwd() -> str: + """Resolve MCP working directory with relocatable defaults.""" + return str(browser_surface_settings().mcp_cwd) + + +def build_browser_guardrails() -> RunGuardrails: + return CoreSettings.from_process_env().guardrails + + +def build_playwright_mcp_config( + instance: Optional[BrowserInstanceConfig] = None, + required_capability: Optional[list[str]] = None, +) -> McpServerConfig: + return playwright_mcp_config(browser_surface_settings(), instance, required_capability) + + +def playwright_mcp_config( + settings: Any, + instance: Optional[BrowserInstanceConfig] = None, + required_capability: Optional[list[str]] = None, +) -> McpServerConfig: + """The ``@playwright/mcp`` server config from the surface settings; ``instance.cdp_url`` wins over them.""" + from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.config import ( + playwright_mcp_config as server_config, + ) + + if instance is None: + return server_config(settings, required_capability=required_capability or ()) + return server_config( + settings, + cdp_url=instance.cdp_url.strip(), + launch_args=parse_command_args(instance.launch_args) if instance.launch_args.strip() else None, + key=instance.sanitized_key(), + required_capability=required_capability or (), + ) + + +def build_runtime_settings( + instance: Optional[BrowserInstanceConfig] = None, + *, + core_settings: CoreSettings | None = None, + browser_settings: SurfaceSettings | None = None, +) -> RuntimeSettings: + core = core_settings or CoreSettings.from_process_env() + if core.provider not in SUPPORTED_MODEL_PROVIDERS: + raise ValueError( + f"Unsupported MODEL_PROVIDER '{core.provider}'. Supported: openai, openrouter, siliconflow, dashscope." + ) + return RuntimeSettings( + provider=core.provider, + api_key=core.api_key, + api_base=core.api_base, + model_name=core.model_name, + mcp_cfg=playwright_mcp_config(browser_settings or browser_surface_settings(), instance), + guardrails=core.guardrails, + instance=instance, + ) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/tools.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/tools.py new file mode 100644 index 0000000000..ca5f13ebca --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/runner/tools.py @@ -0,0 +1,120 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""DeepAgent Tool wrappers around BrowserAgentRuntime.""" + +from __future__ import annotations + +import contextvars +from typing import TYPE_CHECKING, Any, AsyncIterator, Dict + +from openjiuwen.core.foundation.tool import Tool, ToolCard +from openjiuwen.harness.tools.base_tool import ToolOutput + +if TYPE_CHECKING: + from .facade import BrowserAgentRuntime + +_ctx_parent_session_id: contextvars.ContextVar[str] = contextvars.ContextVar( + "playwright_runtime_parent_session_id", + default="", +) +_ctx_parent_request_id: contextvars.ContextVar[str] = contextvars.ContextVar( + "playwright_runtime_parent_request_id", + default="", +) + +_CUSTOM_ACTION_DESC = ( + "Run a registered custom browser action by name. " + "Use for deterministic helpers such as drag-and-drop or coordinate resolution " + "alongside the direct browser tools. " + "Call browser_list_custom_actions first to discover available actions and parameters. " + "Aliases source/target and source_x/source_y/target_x/target_y are accepted." +) +_CUSTOM_ACTION_PARAMS: Dict[str, Any] = { + "type": "object", + "properties": { + "action": {"type": "string", "description": "Name of the custom action to run"}, + "session_id": {"type": "string", "description": "Session ID (optional)"}, + "request_id": {"type": "string", "description": "Request ID (optional)"}, + "params": { + "type": "object", + "description": "Extra key-value parameters forwarded to the action", + "properties": {}, + "required": [], + }, + }, + "required": ["action"], +} + +_LIST_ACTIONS_DESC = "List available custom browser actions and detailed parameter guidance for browser_custom_action." +_LIST_ACTIONS_PARAMS: Dict[str, Any] = { + "type": "object", + "properties": {}, + "required": [], +} + + +class BrowserCustomActionTool(Tool): + """Run a registered custom action.""" + + def __init__(self, runtime: "BrowserAgentRuntime") -> None: + super().__init__( + ToolCard( + name="browser_custom_action", + description=_CUSTOM_ACTION_DESC, + input_params=_CUSTOM_ACTION_PARAMS, + ) + ) + self._runtime = runtime + + async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: + del kwargs + action = inputs.get("action", "") + session_id = (inputs.get("session_id") or "").strip() or _ctx_parent_session_id.get() + request_id = (inputs.get("request_id") or "").strip() or _ctx_parent_request_id.get() + params: Dict[str, Any] = inputs.get("params") or {} + try: + result = await self._runtime.run_custom_action( + action=action, + session_id=session_id, + request_id=request_id, + params=params, + ) + return ToolOutput( + success=bool(result.get("ok", True)), + data=result, + error=result.get("error"), + ) + except Exception as exc: + return ToolOutput(success=False, error=str(exc)) + + async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: + del inputs, kwargs + if False: + yield None + + +class BrowserListActionsTool(Tool): + """List the registered custom actions.""" + + def __init__(self, runtime: "BrowserAgentRuntime") -> None: + super().__init__( + ToolCard( + name="browser_list_custom_actions", + description=_LIST_ACTIONS_DESC, + input_params=_LIST_ACTIONS_PARAMS, + ) + ) + self._runtime = runtime + + async def invoke(self, inputs: Dict[str, Any], **kwargs: Any) -> ToolOutput: + del inputs, kwargs + try: + data = await self._runtime.list_actions() + return ToolOutput(success=True, data=data) + except Exception as exc: + return ToolOutput(success=False, error=str(exc)) + + async def stream(self, inputs: Dict[str, Any], **kwargs: Any) -> AsyncIterator[Any]: + del inputs, kwargs + if False: + yield None diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/__init__.py new file mode 100644 index 0000000000..8a0a37852a --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/__init__.py @@ -0,0 +1,546 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The browser's conformance scene; every page element reports what happened to it in ``document.title``.""" + +from __future__ import annotations + +import shutil +import tempfile +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + Click, + CloseContext, + ContextRef, + Drag, + DriverRef, + Element, + HandleDialog, + Hover, + IndexRef, + Locator, + Observation, + PointRef, + PressKey, + Ref, + RunScript, + Scene, + SceneCase, + Scroll, + SwitchContext, + TypeText, + Wait, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FieldKind, + FillForm, + FormField, + GetConsole, + GetNetwork, + GetPdf, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext +from openjiuwen.harness.tools.browser_move.surfaces.browser.scene.server import SceneServer + +PAGES = Path(__file__).parent / "pages" +UPLOAD_NAME = "scene-upload.txt" +CLIPBOARD_ALIAS_SCRIPT = "() => { const c = navigator.clipboard; return c.readText(); }" +PASTE_COMMAND_SCRIPT = "() => document.execCommand('paste')" + +Actions = Mapping[type[Action], frozenset[type]] +Check = Callable[[Observation, Observation, ActionResult], bool] + + +@dataclass(frozen=True) +class _Target: + name: str + css_id: str + + +SAVE = _Target("Save order", "save") +NAME = _Target("Full name", "name") +GIFT = _Target("Gift wrap", "gift") +SIZE = _Target("Size", "size") +KEYS = _Target("Key field", "keys") +HOVER = _Target("Hover target", "hover") +CONFIRM = _Target("Show confirm", "confirm") +POPUP = _Target("Open popup", "popup") +DRAG_SOURCE = _Target("Drag source", "drag-source") +DROP_TARGET = _Target("Drop target", "drop-target") +FILE_DROP = _Target("File drop zone", "file-drop") +UPLOAD = _Target("Upload file", "upload") +SCROLL_BOX = _Target("Scroll box", "scroll-box") +COVERED = _Target("Covered button", "covered") +LAST = _Target("Last button", "last") +SECOND_LINK = _Target("Go to second", "second-link") + +_COVER_SCRIPT = "() => { cover(); return true; }" +_ABSENT_TEXT = "scene text that never appears" +_CONFIRM_MESSAGE = "Proceed?" +_TIMEOUT_S = 2.0 + +_ANY_KINDS = (DriverRef, Locator, PointRef) +_ELEMENT_KINDS = (DriverRef, Locator) + + +def _element(observation: Observation, target: _Target) -> Element: + wanted = target.name.casefold() + names = [(element, (element.name or "").strip().casefold()) for element in observation.elements] + exact = [element for element, name in names if name == wanted] + if exact: + return exact[0] + partial = [element for element, name in names if wanted in name] + if len(partial) == 1: + return partial[0] + raise LookupError( + f"no single element named {target.name!r} in the observation; names: {[e.name for e in observation.elements]}" + ) + + +def _ref(observation: Observation, target: _Target, kind: type) -> Ref: + if kind is Locator: + return Locator(css=f"#{target.css_id}", text=None, generation=observation.generation) + element = _element(observation, target) + if kind is DriverRef: + return element.ref + if kind is PointRef: + if element.bounds is None: + raise LookupError(f"the driver reports no bounds for {target.name!r}") + x, y, width, height = element.bounds + scale = observation.screenshot.scale if observation.screenshot is not None else 1.0 + return PointRef( + x=round((x + width / 2) * scale), y=round((y + height / 2) * scale), generation=observation.generation + ) + raise LookupError(f"the scene cannot build a {kind.__name__}") + + +def _kind_for(actions: Actions, command: type[Action], kinds: Sequence[type] = _ANY_KINDS) -> type | str: + accepted = actions.get(command) + if accepted is None: + return f"the driver does not handle {command.__name__}, which this case needs" + for kind in kinds: + if kind in accepted: + return kind + return f"the scene addresses elements for {command.__name__} only as {[k.__name__ for k in kinds]}" + + +def _surface(observation: Observation) -> BrowserContext | None: + return observation.surface if isinstance(observation.surface, BrowserContext) else None + + +def _title(observation: Observation) -> str: + context = _surface(observation) + return context.title if context is not None else observation.context.title or "" + + +def _url(observation: Observation) -> str: + context = _surface(observation) + return context.url if context is not None else "" + + +def _tabs(observation: Observation) -> tuple[ContextRef, ...]: + context = _surface(observation) + return context.tabs if context is not None else () + + +def _other_tab(observation: Observation, not_id: str) -> ContextRef: + for tab in _tabs(observation): + if tab.id != not_id: + return tab + raise LookupError(f"no tab other than {not_id!r} in {_tabs(observation)!r}") + + +def _title_is(expected: str) -> Check: + return lambda before, after, result: _title(after) == expected + + +def _dialog(context: object) -> str | None: + return context.dialog if isinstance(context, BrowserContext) else None + + +def _shows_confirm(before: Observation, after: Observation, result: ActionResult) -> bool: + return _dialog(after.surface) == _dialog(result.surface) == f"confirm: {_CONFIRM_MESSAGE}" + + +def _close_extra_tab(seen: Sequence[Observation]) -> Action: + return CloseContext(context=_other_tab(seen[-1], seen[0].context.id)) + + +class _BrowserScene: + def __init__(self) -> None: + self._server = SceneServer(PAGES) + self._base = "" + self._files: Path | None = None + self._happy: dict[type[Action], Callable[[type | None, Actions], SceneCase | str]] = { + Click: self._click, + TypeText: self._type_text, + PressKey: self._press_key, + Scroll: self._scroll, + Hover: self._hover, + Drag: self._drag, + Wait: self._wait, + SwitchContext: self._switch_context, + CloseContext: self._close_context, + HandleDialog: self._handle_dialog, + RunScript: self._run_script, + Navigate: self._navigate, + Back: self._back, + OpenContext: self._open_context, + SelectOption: self._select_option, + UploadFile: self._upload_file, + FillForm: self._fill_form, + Drop: self._drop, + GetPdf: self._get_pdf, + GetConsole: self._get_console, + GetNetwork: self._get_network, + } + + def start(self) -> None: + self._base = self._server.start() + self._files = Path(tempfile.mkdtemp(prefix="browser-scene-")) + (self._files / UPLOAD_NAME).write_text("conformance upload\n", encoding="utf-8") + + def stop(self) -> None: + self._server.stop() + if self._files is not None: + shutil.rmtree(self._files, ignore_errors=True) + self._files = None + + def url(self, page: str) -> str: + return self._base + page + + @property + def upload_path(self) -> str: + return str(self._files / UPLOAD_NAME) if self._files is not None else UPLOAD_NAME + + def enter(self) -> Sequence[Action]: + return (Navigate(url=self.url("index.html")),) + + def _with_target( + self, kind: type | None, actions: Actions, target: _Target, make: Callable, check: Check + ) -> SceneCase | str: + if kind is not None: + return SceneCase(steps=(lambda seen: make(_ref(seen[-1], target, kind)),), check=check) + click_kind = _kind_for(actions, Click) + if isinstance(click_kind, str): + return click_kind + + def focus(seen: Sequence[Observation]) -> Action: + return Click(target=_ref(seen[-1], target, click_kind)) + + return SceneCase(steps=(focus, lambda seen: make(None)), check=check) + + def happy(self, command: type[Action], kind: type | None, actions: Actions) -> SceneCase | str: + if kind is IndexRef: + return "an IndexRef is internal to a driver; the scene cannot address an element by index" + build = self._happy.get(command) + if build is None: + return f"the scene has no happy path for {command.__name__}" + return build(kind, actions) + + def _click(self, kind: type | None, actions: Actions) -> SceneCase | str: + return self._with_target(kind, actions, SAVE, lambda ref: Click(target=ref), _title_is("clicked")) + + def _type_text(self, kind: type | None, actions: Actions) -> SceneCase | str: + return self._with_target( + kind, actions, NAME, lambda ref: TypeText(target=ref, text="Ada"), _title_is("form:3:false") + ) + + def _press_key(self, kind: type | None, actions: Actions) -> SceneCase | str: + def make(ref: Ref | None) -> Action: + return PressKey(key="Enter", target=ref) + + if kind is None: + return SceneCase(steps=(lambda seen: make(None),), check=_title_is("key:Enter")) + return self._with_target(kind, actions, KEYS, make, _title_is("key:Enter")) + + def _scroll(self, kind: type | None, actions: Actions) -> SceneCase | str: + def make(ref: Ref | None) -> Action: + return Scroll(target=ref, dy=200) + + if kind is None: + return SceneCase(steps=(lambda seen: make(None),), check=_title_is("scrolled")) + return self._with_target(kind, actions, SCROLL_BOX, make, _title_is("scrolled")) + + def _hover(self, kind: type | None, actions: Actions) -> SceneCase | str: + return self._with_target(kind, actions, HOVER, lambda ref: Hover(target=ref), _title_is("hovered")) + + def _drag(self, kind: type | None, actions: Actions) -> SceneCase | str: + if kind is None: + return "Drag needs element refs" + + def step(seen: Sequence[Observation]) -> Action: + return Drag(source=_ref(seen[-1], DRAG_SOURCE, kind), destination=_ref(seen[-1], DROP_TARGET, kind)) + + return SceneCase(steps=(step,), check=_title_is("dragged")) + + def _wait(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase(steps=(lambda seen: Wait(seconds=0.2),)) + + def _switch_context(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase( + steps=( + lambda seen: OpenContext(url=self.url("second.html")), + lambda seen: SwitchContext(context=_other_tab(seen[-1], seen[-1].context.id)), + ), + check=lambda before, after, result: after.context.id != before.context.id, + cleanup=(_close_extra_tab,), + ) + + def _close_context(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase( + steps=( + lambda seen: OpenContext(url=self.url("second.html")), + lambda seen: CloseContext(context=_other_tab(seen[-1], seen[0].context.id)), + ), + check=lambda before, after, result: len(_tabs(after)) < len(_tabs(before)), + ) + + def _handle_dialog(self, kind: type | None, actions: Actions) -> SceneCase | str: + click_kind = _kind_for(actions, Click) + if isinstance(click_kind, str): + return click_kind + return SceneCase( + steps=( + lambda seen: Click(target=_ref(seen[-1], CONFIRM, click_kind)), + lambda seen: HandleDialog(accept=True), + ), + check=_title_is("confirm:true"), + observe_between=False, + ) + + def _run_script(self, kind: type | None, actions: Actions) -> SceneCase | str: + if kind is None: + code = "document.title = 'script:ran'" + return SceneCase(steps=(lambda seen: RunScript(code=code),), check=_title_is("script:ran")) + code = "(element) => { document.title = 'script:' + element.id; return element.id; }" + + def step(seen: Sequence[Observation]) -> Action: + return RunScript(code=code, target=_ref(seen[-1], SAVE, kind)) + + return SceneCase(steps=(step,), check=_title_is("script:save")) + + def _navigate(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase( + steps=(lambda seen: Navigate(url=self.url("second.html")),), + check=lambda before, after, result: _url(after).endswith("/second.html"), + ) + + def _back(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase( + steps=(lambda seen: Navigate(url=self.url("second.html")), lambda seen: Back()), + check=lambda before, after, result: _url(after).endswith("/index.html"), + ) + + def _open_context(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase( + steps=(lambda seen: OpenContext(url=self.url("second.html")),), + check=lambda before, after, result: len(_tabs(after)) > len(_tabs(before)), + cleanup=(_close_extra_tab,), + ) + + def _select_option(self, kind: type | None, actions: Actions) -> SceneCase | str: + if kind is None: + return "SelectOption needs an element ref" + + def step(seen: Sequence[Observation]) -> Action: + return SelectOption(target=_ref(seen[-1], SIZE, kind), values=("m",)) + + return SceneCase(steps=(step,), check=_title_is("size:m")) + + def _upload_file(self, kind: type | None, actions: Actions) -> SceneCase | str: + check = _title_is(f"upload:{UPLOAD_NAME}") + if kind is not None: + + def step(seen: Sequence[Observation]) -> Action: + return UploadFile(target=_ref(seen[-1], UPLOAD, kind), paths=(self.upload_path,)) + + return SceneCase(steps=(step,), check=check) + click_kind = _kind_for(actions, Click) + if isinstance(click_kind, str): + return click_kind + return SceneCase( + steps=( + lambda seen: Click(target=_ref(seen[-1], UPLOAD, click_kind)), + lambda seen: UploadFile(target=None, paths=(self.upload_path,)), + ), + check=check, + observe_between=False, + ) + + def _fill_form(self, kind: type | None, actions: Actions) -> SceneCase | str: + if kind is None: + return "FillForm needs element refs" + + def step(seen: Sequence[Observation]) -> Action: + return FillForm( + fields=( + FormField(target=_ref(seen[-1], NAME, kind), kind=FieldKind.TEXTBOX, value="Ada"), + FormField(target=_ref(seen[-1], GIFT, kind), kind=FieldKind.CHECKBOX, value="true"), + ) + ) + + return SceneCase(steps=(step,), check=_title_is("form:3:true")) + + def _drop(self, kind: type | None, actions: Actions) -> SceneCase | str: + if kind is None: + return "Drop needs an element ref" + + def step(seen: Sequence[Observation]) -> Action: + return Drop(target=_ref(seen[-1], FILE_DROP, kind), paths=(self.upload_path,)) + + return SceneCase(steps=(step,), check=_title_is("dropped:1")) + + def _get_pdf(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase(steps=(lambda seen: GetPdf(),), check=lambda before, after, result: bool(result.value)) + + def _get_console(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase( + steps=(lambda seen: GetConsole(),), + check=lambda before, after, result: "scene console message" in str(result.value), + ) + + def _get_network(self, kind: type | None, actions: Actions) -> SceneCase | str: + return SceneCase( + steps=(lambda seen: GetNetwork(),), check=lambda before, after, result: "data.json" in str(result.value) + ) + + def stale(self, actions: Actions) -> SceneCase | str: + if DriverRef not in actions.get(Click, frozenset()): + return "the scene makes a stale ref only as a DriverRef, and the driver does not click DriverRefs" + return SceneCase( + steps=( + lambda seen: Navigate(url=self.url("second.html")), + lambda seen: Click(target=_element(seen[0], LAST).ref), + ) + ) + + def _click_case(self, actions: Actions, target: _Target, kinds: Sequence[type], **options) -> SceneCase | str: + kind = _kind_for(actions, Click, kinds) + if isinstance(kind, str): + return kind + return SceneCase(steps=(lambda seen: Click(target=_ref(seen[-1], target, kind)),), **options) + + def intercepted(self, actions: Actions) -> SceneCase | str: + kind = _kind_for(actions, Click, _ELEMENT_KINDS) + if isinstance(kind, str): + return kind + if RunScript not in actions: + return "the driver does not handle RunScript, which this case needs to cover the button" + return SceneCase( + steps=(lambda seen: RunScript(code=_COVER_SCRIPT), lambda seen: Click(target=_ref(seen[0], COVERED, kind))), + observe_between=False, + ) + + def timeout(self, actions: Actions) -> SceneCase | str: + if Wait not in actions: + return "the driver does not handle Wait, which this case needs" + return SceneCase(steps=(lambda seen: Wait(until_text=_ABSENT_TEXT, seconds=_TIMEOUT_S),)) + + def sensitive(self) -> dict[str, Callable[[Actions, str], SceneCase | str]]: + return {"form-field": self._sensitive_form_field} + + def _sensitive_form_field(self, actions: Actions, secret: str) -> SceneCase | str: + kind = _kind_for(actions, FillForm) + if isinstance(kind, str): + return kind + + def step(seen: Sequence[Observation]) -> Action: + field = FormField(target=_ref(seen[-1], NAME, kind), kind=FieldKind.TEXTBOX, value=secret, sensitive=True) + return FillForm(fields=(field,)) + + return SceneCase(steps=(step,)) + + def context_changes(self) -> dict[str, Callable[[Actions], SceneCase | str]]: + def second(seen: Sequence[Observation]) -> Action: + return OpenContext(url=self.url("second.html")) + + return { + "navigate": lambda actions: SceneCase(steps=(lambda seen: Navigate(url=self.url("second.html")),)), + "same-url-reload": lambda actions: SceneCase(steps=(lambda seen: Navigate(url=self.url("index.html")),)), + "link-click": lambda actions: self._click_case(actions, SECOND_LINK, _ANY_KINDS), + "open-tab": lambda actions: SceneCase(steps=(second,), cleanup=(_close_extra_tab,)), + "popup-click": lambda actions: self._click_case(actions, POPUP, _ANY_KINDS, cleanup=(_close_extra_tab,)), + "switch-tab": lambda actions: SceneCase( + steps=(second, lambda seen: SwitchContext(context=_other_tab(seen[-1], seen[-1].context.id))), + cleanup=(_close_extra_tab,), + ), + "close-tab": lambda actions: SceneCase( + steps=(second, lambda seen: CloseContext(context=_other_tab(seen[-1], seen[0].context.id))) + ), + "dialog": lambda actions: self._click_case( + actions, + CONFIRM, + _ANY_KINDS, + check=_shows_confirm, + observe_between=False, + cleanup=(lambda seen: HandleDialog(accept=False),), + ), + } + + +def clipboard_read_case(actions: Actions) -> SceneCase | str: + """A clipboard read through an alias no script text rule sees; the page must reject the promise.""" + if RunScript not in actions: + return "the driver does not handle RunScript, which this case needs" + return SceneCase(steps=(lambda seen: RunScript(code=CLIPBOARD_ALIAS_SCRIPT),)) + + +def paste_command_case(actions: Actions) -> SceneCase | str: + """``document.execCommand('paste')`` answers ``false`` in the page.""" + if RunScript not in actions: + return "the driver does not handle RunScript, which this case needs" + return SceneCase( + steps=(lambda seen: RunScript(code=PASTE_COMMAND_SCRIPT),), + check=lambda before, after, result: result.value is False, + ) + + +def slow_typing_case(actions: Actions) -> SceneCase | str: + """``TypeText(slowly=True)`` into the empty name field leaves the value plain typing leaves.""" + kind = _kind_for(actions, TypeText, _ELEMENT_KINDS) + if isinstance(kind, str): + return kind + + def step(seen: Sequence[Observation]) -> Action: + return TypeText(target=_ref(seen[-1], NAME, kind), text="Ada", slowly=True) + + return SceneCase(steps=(step,), check=_title_is("form:3:false")) + + +def browser_scene() -> Scene: + """A new, not yet started browser scene for the conformance suite (pages on ``127.0.0.1``, no network).""" + scene = _BrowserScene() + return Scene( + surface="browser", + start=scene.start, + stop=scene.stop, + enter=scene.enter, + happy=scene.happy, + stale=scene.stale, + intercepted=scene.intercepted, + timeout=scene.timeout, + context_changes=scene.context_changes(), + sensitive=scene.sensitive(), + ) + + +__all__ = [ + "CLIPBOARD_ALIAS_SCRIPT", + "PASTE_COMMAND_SCRIPT", + "browser_scene", + "clipboard_read_case", + "paste_command_case", + "slow_typing_case", +] diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/data.json b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/data.json new file mode 100644 index 0000000000..66fba4aefa --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/data.json @@ -0,0 +1 @@ +{"scene": "data"} diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/index.html b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/index.html new file mode 100644 index 0000000000..f08086f55f --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/index.html @@ -0,0 +1,88 @@ +<!doctype html> +<html lang="en"> +<head> + <meta charset="utf-8"> + <link rel="icon" href="data:,"> + <title>Scene start + + + + +

Conformance scene

+ +
+ + + + +
+

+ + + + +

+

+ + + File drop zone + +

+

Scroll box content

+

+ + + + +

+
+ + + diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/second.html b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/second.html new file mode 100644 index 0000000000..88b2996550 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/pages/second.html @@ -0,0 +1,12 @@ + + + + + + Scene second + + +

Second

+ Back to start + + diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/server.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/server.py new file mode 100644 index 0000000000..f448aa2db0 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/scene/server.py @@ -0,0 +1,47 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""A tiny stdlib HTTP server for the scene pages, on ``127.0.0.1`` and a free port. No network beyond loopback.""" + +from __future__ import annotations + +import functools +import threading +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + + +class _QuietHandler(SimpleHTTPRequestHandler): + def log_message(self, format: str, *args: Any) -> None: + pass + + +class SceneServer: + """Serves one folder over HTTP in a daemon thread; ``start()`` returns the base URL (ending with ``/``).""" + + def __init__(self, root: Path) -> None: + self._root = root + self._server: ThreadingHTTPServer | None = None + self._thread: threading.Thread | None = None + + def start(self) -> str: + if self._server is None: + handler = functools.partial(_QuietHandler, directory=str(self._root)) + self._server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + self._thread = threading.Thread( + target=self._server.serve_forever, kwargs={"poll_interval": 0.05}, name="scene-server", daemon=True + ) + self._thread.start() + host, port = self._server.server_address[:2] + return f"http://{host}:{port}/" + + def stop(self) -> None: + if self._server is None: + return + self._server.shutdown() + self._server.server_close() + if self._thread is not None: + self._thread.join(timeout=5) + self._server = None + self._thread = None diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/__init__.py new file mode 100644 index 0000000000..f08df04bd3 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/__init__.py @@ -0,0 +1,4 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Semantic state of a browser task: what the page says the task has reached.""" diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/evidence.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/evidence.py new file mode 100644 index 0000000000..7b56a22f82 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/evidence.py @@ -0,0 +1,787 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Evidence records, evaluate compaction and the recent-action log.""" + +from __future__ import annotations + +import hashlib +import json +import re +from collections import deque +from typing import Any, Dict, Iterable, Optional +from urllib.parse import parse_qs, urlsplit + +from openjiuwen.harness.tools.browser_move.contract import Observe, RunScript +from openjiuwen.harness.tools.browser_move.runtime.session import ToolKind +from openjiuwen.harness.tools.browser_move.runtime.utils.parsing import extract_json_object +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.keys import ( + _BROWSER_FIELD_ALIASES, + _BROWSER_PHASE_STATE_KEY, + _BROWSER_SELECTED_SORT_LABELS, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.working_context.store import BrowserWorkingContextStore + +_BROWSER_CONTEXTUAL_EVALUATE_FIELD_ALIASES = frozenset( + { + "name", + "profile_name", + "active_tab", + "active_tabs", + "selected_tab", + "selected_tabs", + } +) +_BROWSER_UNKNOWN_VALUE_RE = re.compile( + r"^(?:unknown|n/?a|not available|not found|未找到|未知|暂无|未显示|不可用)$", + re.IGNORECASE, +) +_BROWSER_ZERO_COMMENT_RE = re.compile( + r"^(?:0|no comments?|no replies|还没有评论|暂无评论|没有评论|暂无回复|0\s*条评论)$", + re.IGNORECASE, +) + + +class _EvidenceMixin: + @classmethod + def _record_tool_evidence( + cls, + state: Dict[str, Any], + result: Dict[str, Any], + *, + kind: ToolKind, + tool_name: str, + tool_args: Dict[str, Any], + ) -> Dict[str, Any]: + previous_evidence = { + cls._evidence_signature(item) for item in state.get("structured_evidence") or [] if isinstance(item, dict) + } + previous_coverage = set(state.get("field_coverage") or []) + cls._record_structured_evidence(state, result, kind=kind, tool_name=tool_name, tool_args=tool_args) + evidence_added = any( + cls._evidence_signature(item) not in previous_evidence + for item in state.get("structured_evidence") or [] + if isinstance(item, dict) + ) + new_evidence_fields = sorted(set(state.get("field_coverage") or []) - previous_coverage) + return { + "new_evidence_fields": new_evidence_fields, + "evidence_added": evidence_added, + "recovered": bool(evidence_added or new_evidence_fields), + } + + @classmethod + def _record_price_interval( + cls, + details: Dict[str, Any], + phase: str, + kind: ToolKind, + args: Dict[str, Any], + ) -> None: + price_interval = cls._price_interval_signature(kind, args) + if phase != "filtering" or not price_interval: + return + visited_intervals = details.setdefault("visited_price_intervals", []) + if price_interval not in visited_intervals: + visited_intervals.append(price_interval) + + @classmethod + def _record_structured_evidence( + cls, + state: Dict[str, Any], + result: Dict[str, Any], + *, + kind: ToolKind, + tool_name: str, + tool_args: Dict[str, Any], + ) -> None: + evidence = cls._structured_evidence(result) + if not evidence and isinstance(kind.action, RunScript): + evidence = cls._evaluate_evidence( + result, + tool_args, + required_fields=state.get("required_fields") or [], + ) + if not evidence and "probe_interactives" in str(tool_name or "").lower(): + evidence = cls._interactive_probe_evidence(result) + stored_evidence = state.setdefault("structured_evidence", []) + known_signatures = {cls._evidence_signature(item) for item in stored_evidence if isinstance(item, dict)} + observation = cls._page_observation_evidence(state, result, kind, tool_name, tool_args) + for item in (evidence, observation): + if item and cls._evidence_signature(item) not in known_signatures: + stored_evidence.append(item) + del stored_evidence[:-20] + if not evidence: + return + if evidence.get("kind") == "card_probe": + state["observed_result_count"] = max( + int(state.get("observed_result_count") or 0), + int(evidence.get("observed_count") or 0), + ) + cls._record_evidence_slots( + state, + evidence, + result=result, + tool_name=tool_name, + tool_args=tool_args, + ) + BrowserWorkingContextStore.refresh_field_coverage(state) + + @classmethod + def _page_observation_evidence( + cls, + state: Dict[str, Any], + result: Dict[str, Any], + kind: ToolKind, + tool_name: str, + tool_args: Dict[str, Any], + ) -> Dict[str, Any]: + """Retain source text without inventing field coverage or another memory store.""" + if result.get("ok", True) is False: + return {} + if not isinstance(kind.action, (Observe, RunScript)): + return {} + if not cls._is_read_only_recovery(kind, tool_args): + return {} + value = cls._evaluate_result_value(result) + if value in (None, "", [], {}) or isinstance(value, (bool, int, float)): + return {} + if isinstance(value, dict) and (value.get("ok") is False or value.get("error")): + return {} + raw_text = value if isinstance(value, str) else cls._evidence_signature(value) + normalized = re.sub(r"\s*\[(?:ref|target_id|generation_id)=[^\]]+\]", "", raw_text) + normalized = " ".join(normalized.split()) + if len(normalized) < 16: + return {} + _, source = cls._evidence_variant_and_url(state, result, tool_args) + if not source: + return {} + page = result.get("page_state") or {} + generation = str(result.get("generation_id") or page.get("generation_id") or "") + target = str(tool_args.get("target") or tool_args.get("selector") or "")[:240] + return { + "kind": "page_observation", + "source": source, + "generation_id": generation, + "raw_text": normalized[:4_000], + "content_hash": hashlib.sha256(normalized.encode("utf-8")).hexdigest()[:20], + "provenance": { + "tool": tool_name, + **cls._evaluate_execution_provenance(tool_args, target), + }, + } + + @classmethod + def _record_evidence_slots( + cls, + state: Dict[str, Any], + evidence: Dict[str, Any], + *, + result: Dict[str, Any], + tool_name: str, + tool_args: Dict[str, Any], + ) -> None: + if not state.get("required_evidence_slots"): + return + variant, source_url = cls._evidence_variant_and_url(state, result, tool_args) + values, generation, provenance, field_status = cls._evidence_values(evidence) + if not values and not field_status: + return + slots = state.setdefault("evidence_slots", []) + covered = {cls._evidence_slot_key(slot): index for index, slot in enumerate(slots) if isinstance(slot, dict)} + for required in state.get("required_evidence_slots") or []: + if not isinstance(required, dict): + continue + key = cls._evidence_slot_key(required) + if key[1] != "default" and key[1] != variant: + continue + slot = cls._build_evidence_slot( + key, + values=values, + provenance=provenance, + field_status=field_status, + generation=generation, + source_url=source_url, + tool_name=tool_name, + ) + if slot is None: + continue + existing_index = covered.get(key) + if existing_index is not None: + existing = slots[existing_index] + existing_status = str(existing.get("status") or "present").strip().lower() + if existing_status == "present" or slot.get("status") != "present": + continue + slots[existing_index] = slot + else: + slots.append(slot) + covered[key] = len(slots) - 1 + del state["evidence_slots"][:-20] + + @staticmethod + def _build_evidence_slot( + key: tuple[str, str, str], + *, + values: Dict[str, Any], + provenance: Dict[str, Any], + field_status: Dict[str, Any], + generation: str, + source_url: str, + tool_name: str, + ) -> Optional[Dict[str, Any]]: + value = values.get(key[2]) + status = str(field_status.get(key[2]) or ("present" if value not in (None, "") else "")).lower() + if status not in {"present", "missing", "unknown"}: + return None + if status == "present" and value in (None, ""): + return None + item = provenance.get(key[2]) if isinstance(provenance, dict) else None + item = item if isinstance(item, dict) else {} + slot = { + "entity": key[0], + "variant": key[1], + "field": key[2], + "status": status, + "source": str(item.get("source") or source_url or tool_name or "")[:500], + "generation": str(item.get("generation_id") or generation or ""), + } + if status == "present": + slot["value"] = str(value)[:500] + for provenance_key in ("selector", "raw_text"): + provenance_value = str(item.get(provenance_key) or "").strip() + if provenance_value: + slot[provenance_key] = provenance_value[:600] + return slot + + @staticmethod + def _evidence_values( + evidence: Dict[str, Any], + ) -> tuple[Dict[str, Any], str, Dict[str, Any], Dict[str, str]]: + values = evidence.get("values") + if isinstance(values, dict): + provenance = evidence.get("provenance") + statuses = evidence.get("field_status") + return ( + values, + str(evidence.get("generation_id") or ""), + provenance if isinstance(provenance, dict) else {}, + dict(statuses) if isinstance(statuses, dict) else {}, + ) + cards = evidence.get("cards") + if not isinstance(cards, list): + return {}, str(evidence.get("generation_id") or ""), {}, {} + first = next((card for card in cards if isinstance(card, dict)), {}) + if not first: + return {}, "", {}, {} + provenance = first.get("provenance") + statuses = first.get("field_status") + return ( + first, + str(first.get("generation_id") or evidence.get("generation_id") or ""), + provenance if isinstance(provenance, dict) else {}, + dict(statuses) if isinstance(statuses, dict) else {}, + ) + + @staticmethod + def _evidence_variant_and_url( + state: Dict[str, Any], + result: Dict[str, Any], + tool_args: Dict[str, Any], + ) -> tuple[str, str]: + page_state = result.get("page_state") if isinstance(result.get("page_state"), dict) else {} + last_page = state.get("last_page") if isinstance(state.get("last_page"), dict) else {} + source_url = str( + result.get("url") or page_state.get("url") or tool_args.get("url") or last_page.get("url") or "" + ).strip() + query: Dict[str, list[str]] = {} + try: + query = parse_qs(urlsplit(source_url).query) + except ValueError: + pass + order = " ".join(query.get("order", []) + query.get("sort", [])).lower() + serialized_args = json.dumps(tool_args, ensure_ascii=False, default=str).lower() + if any(token in order for token in ("pubdate", "latest", "newest", "recent")) or any( + token in serialized_args for token in ("最新", "latest", "newest", "pubdate") + ): + return "latest", source_url + if any(token in order for token in ("totalrank", "relevance", "default", "comprehensive")) or any( + token in serialized_args for token in ("综合", "comprehensive", "relevance", "totalrank") + ): + return "comprehensive", source_url + required_variants = { + str(slot.get("variant") or "") + for slot in state.get("required_evidence_slots") or [] + if isinstance(slot, dict) + } + if required_variants == {"comprehensive", "latest"} and source_url and not order: + return "comprehensive", source_url + return "default", source_url + + @classmethod + def _evidence_signature(cls, evidence: Dict[str, Any]) -> str: + if isinstance(evidence, dict) and evidence.get("kind") == "page_observation": + return json.dumps([evidence.get("source"), evidence.get("content_hash")], ensure_ascii=False) + ignored_keys = { + "generation", + "generation_id", + "sel", + "selector", + "selector_hint", + "ref", + "target_id", + "target", + "provenance", + "execution_provenance", + } + + def semantic_value(value: Any) -> Any: + if isinstance(value, dict): + return {str(key): semantic_value(item) for key, item in value.items() if str(key) not in ignored_keys} + if isinstance(value, list): + return [semantic_value(item) for item in value] + return value + + return json.dumps( + semantic_value(evidence), + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + default=str, + ) + + @classmethod + def _evaluate_evidence( + cls, + result: Dict[str, Any], + tool_args: Dict[str, Any], + *, + required_fields: Iterable[str] = (), + ) -> Dict[str, Any]: + value = cls._evaluate_result_value(result) + if value is None: + return {} + trusted_fields, allowed_fields = cls._evaluate_field_scope(tool_args, required_fields) + compact_values, fields, field_status, raw_values = cls._compact_evaluate_values( + value, + allowed_fields=allowed_fields, + trusted_fields=trusted_fields, + ) + if not compact_values and not field_status: + return {} + raw_preview = json.dumps(value, ensure_ascii=False, default=str) if not isinstance(value, str) else value + generation_id = str(result.get("generation_id") or tool_args.get("generation_id") or "") + target = str(tool_args.get("target") or tool_args.get("element") or "")[:240] + return { + "kind": "targeted_evaluate", + "generation_id": generation_id, + "fields": fields, + "values": compact_values, + "field_status": field_status, + "preview": " ".join(raw_preview.split())[:800], + "provenance": cls._evaluate_field_provenance( + compact_values, + field_status=field_status, + raw_values=raw_values, + target=target, + generation_id=generation_id, + ), + "execution_provenance": cls._evaluate_execution_provenance(tool_args, target), + } + + @staticmethod + def _evaluate_result_value(result: Dict[str, Any]) -> Any: + value = next( + (result.get(key) for key in ("result", "value", "data") if result.get(key) is not None), + None, + ) + if not isinstance(value, str): + return value + try: + return json.loads(value) + except (TypeError, ValueError): + return extract_json_object(value) or value + + @classmethod + def _evaluate_field_scope( + cls, + tool_args: Dict[str, Any], + required_fields: Iterable[str], + ) -> tuple[set[str], set[str]]: + trusted_fields = { + cls._canonical_field_name(field_name) for field_name in tool_args.get("_runtime_evidence_fields") or [] + } + trusted_fields.discard("") + allowed_fields = {cls._canonical_field_name(field_name) for field_name in required_fields} + allowed_fields.discard("") + allowed_fields.update(trusted_fields) + return trusted_fields, allowed_fields + + @staticmethod + def _evaluate_execution_provenance(tool_args: Dict[str, Any], target: str) -> Dict[str, str]: + expression = str( + tool_args.get("function") + or tool_args.get("expression") + or tool_args.get("script") + or tool_args.get("code") + or "" + ) + return { + "target": target, + "expression_sha256": hashlib.sha256(expression.encode("utf-8")).hexdigest()[:16] if expression else "", + } + + @classmethod + def _compact_evaluate_values( + cls, + value: Any, + *, + allowed_fields: set[str], + trusted_fields: set[str], + ) -> tuple[Dict[str, str], list[str], Dict[str, str], Dict[str, str]]: + if isinstance(value, dict): + return cls._compact_evaluate_mapping( + value, + allowed_fields=allowed_fields, + trusted_fields=trusted_fields, + ) + if isinstance(value, list) and value and all(isinstance(item, dict) for item in value[:5]): + return cls._compact_evaluate_list( + value, + allowed_fields=allowed_fields, + trusted_fields=trusted_fields, + ) + return {}, [], {}, {} + + @classmethod + def _compact_evaluate_mapping( + cls, + value: Dict[str, Any], + *, + allowed_fields: set[str], + trusted_fields: set[str], + ) -> tuple[Dict[str, str], list[str], Dict[str, str], Dict[str, str]]: + compact_values: Dict[str, str] = {} + field_status: Dict[str, str] = {} + raw_values: Dict[str, str] = {} + for key, item in list(value.items())[:20]: + normalized = cls._normalize_evaluate_mapping_item( + key, + item, + allowed_fields=allowed_fields, + trusted_fields=trusted_fields, + ) + if normalized is None: + continue + field_name, field_value, status, raw_value = normalized + field_status[field_name] = status + raw_values[field_name] = raw_value + if status == "present": + compact_values[field_name] = field_value + return compact_values, sorted(compact_values), field_status, raw_values + + @classmethod + def _normalize_evaluate_mapping_item( + cls, + key: Any, + item: Any, + *, + allowed_fields: set[str], + trusted_fields: set[str], + ) -> tuple[str, str, str, str] | None: + selected_sort = cls._selected_sort_value(key, item) + normalized: tuple[str, str, str, str] | None = None + if selected_sort: + if not allowed_fields or "sort_state" in allowed_fields: + normalized = ("sort_state", selected_sort, "present", f"{key}={item}"[:600]) + else: + snake_name = cls._snake_case_field_name(key) + canonical = cls._canonical_evaluate_field_name(key) + field_allowed = canonical in _BROWSER_FIELD_ALIASES and (not allowed_fields or canonical in allowed_fields) + contextual_alias_allowed = ( + snake_name not in _BROWSER_CONTEXTUAL_EVALUATE_FIELD_ALIASES or canonical in trusted_fields + ) + if ( + field_allowed + and contextual_alias_allowed + and not (canonical == "sort_state" and isinstance(item, bool)) + ): + normalized_value, status = cls._normalize_evaluate_field_value(canonical, item) + normalized = (canonical, normalized_value, status, str(item)[:600]) + return normalized + + @staticmethod + def _merge_first_values(target: Dict[str, str], source: Dict[str, str]) -> None: + for field_name, field_value in source.items(): + target.setdefault(field_name, field_value) + + @classmethod + def _compact_evaluate_list( + cls, + value: list[Dict[str, Any]], + *, + allowed_fields: set[str], + trusted_fields: set[str], + ) -> tuple[Dict[str, str], list[str], Dict[str, str], Dict[str, str]]: + compact_values: Dict[str, str] = {} + field_status: Dict[str, str] = {} + raw_values: Dict[str, str] = {} + fields: set[str] = set() + for item in value[:5]: + item_values, item_fields, item_status, item_raw = cls._compact_evaluate_mapping( + item, + allowed_fields=allowed_fields, + trusted_fields=trusted_fields, + ) + fields.update(item_fields) + cls._merge_first_values(compact_values, item_values) + cls._merge_first_values(field_status, item_status) + cls._merge_first_values(raw_values, item_raw) + if not fields and not field_status: + return {}, [], {}, {} + compact_values["items"] = json.dumps(value[:3], ensure_ascii=False, default=str)[:800] + return compact_values, sorted(fields), field_status, raw_values + + @classmethod + def _selected_sort_value(cls, key: Any, value: Any) -> str: + snake_name = cls._snake_case_field_name(key) + if snake_name in {"active_tab", "active_tabs", "selected_tab", "selected_tabs"}: + if isinstance(value, bool): + return "" + text = " ".join(str(value or "").split()) + return text[:120] if text else "" + if not snake_name.endswith(("_selected", "_tab_selected")) or value is not True: + return "" + base_name = re.sub(r"(?:_tab)?_selected$", "", snake_name) + for token, label in _BROWSER_SELECTED_SORT_LABELS.items(): + if token in base_name: + return label + return "" + + @staticmethod + def _normalize_evaluate_field_value(field_name: str, value: Any) -> tuple[str, str]: + if value in (None, "", [], {}): + return "", "missing" + text = " ".join(str(value).split())[:300] + if field_name == "comments" and _BROWSER_ZERO_COMMENT_RE.fullmatch(text): + return "0", "present" + if _BROWSER_UNKNOWN_VALUE_RE.fullmatch(text): + return "", "unknown" + return text, "present" + + @staticmethod + def _evaluate_field_provenance( + values: Dict[str, str], + *, + field_status: Dict[str, str], + raw_values: Dict[str, str], + target: str, + generation_id: str, + ) -> Dict[str, Dict[str, str]]: + return { + field_name: { + "source": "browser_evaluate", + "selector": target, + "raw_text": str(raw_values.get(field_name) or values.get(field_name) or "")[:600], + "generation_id": generation_id, + } + for field_name in set(values) | set(field_status) + } + + @classmethod + def _record_recent_action( + cls, + session: Any, + *, + kind: ToolKind, + tool_name: str, + tool_args: Any, + tool_result: Any, + action_class: str, + elapsed_ms: int, + progress_delta: Dict[str, Any], + ) -> bool: + state = session.get_state(_BROWSER_PHASE_STATE_KEY) + if not isinstance(state, dict): + return False + state["action_sequence"] = int(state.get("action_sequence") or 0) + 1 + recent_actions = deque(state.get("recent_actions") or [], maxlen=6) + recent_actions.append( + cls._build_action_record( + state, + kind=kind, + tool_name=tool_name, + tool_args=tool_args, + tool_result=tool_result, + action_class=action_class, + elapsed_ms=elapsed_ms, + progress_delta=progress_delta, + ) + ) + state["recent_actions"] = list(recent_actions) + cls._update_last_page(state, tool_result) + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + return bool(progress_delta.get("recovered")) + + @classmethod + def _build_action_record( + cls, + state: Dict[str, Any], + *, + kind: ToolKind, + tool_name: str, + tool_args: Any, + tool_result: Any, + action_class: str, + elapsed_ms: int, + progress_delta: Dict[str, Any], + ) -> Dict[str, Any]: + succeeded = bool(progress_delta.get("success")) + error = str(tool_result.get("error") or "").strip() if isinstance(tool_result, dict) else "" + semantic_delta = "evidence_added" if progress_delta.get("evidence_added") else "pending" + outcome_status = "success" if succeeded else "failed" + if progress_delta.get("ambiguous"): + outcome_status = "ambiguous" + semantic_delta = "awaiting_observation" + elif not succeeded: + semantic_delta = "error" + return { + "seq": state["action_sequence"], + "phase": progress_delta.get("phase") or state.get("current_phase"), + "action_class": action_class or cls._classify_action_class(kind, tool_name, tool_args, state), + "target_summary": cls._target_summary(tool_name, tool_args), + "outcome": "success" if succeeded else f"error: {error[:180] or 'tool_failed'}", + "outcome_status": outcome_status, + "semantic_delta": semantic_delta, + "new_evidence_fields": list(progress_delta.get("new_evidence_fields") or []), + "elapsed_ms": max(0, int(elapsed_ms)), + } + + @staticmethod + def _update_last_page(state: Dict[str, Any], tool_result: Any) -> None: + result = tool_result if isinstance(tool_result, dict) else {} + page_state = result.get("page_state") if isinstance(result.get("page_state"), dict) else {} + url = str(result.get("url") or page_state.get("url") or "") + title = str(result.get("title") or page_state.get("title") or "") + if url or title: + last_page = state.setdefault("last_page", {}) + if url: + last_page["url"] = url + if title: + last_page["title"] = title + + @classmethod + def _target_summary(cls, tool_name: str, tool_args: Any) -> str: + args = cls._coerce_tool_args(tool_args) + name = str(tool_name or "").strip().rsplit(".", 1)[-1] + summary: Dict[str, Any] = {"tool": name} + for key in ("url", "target_id", "target", "ref", "element", "name", "label", "text", "value"): + value = args.get(key) + if value not in (None, "", [], {}): + summary[key] = str(value)[:160] + steps = args.get("steps") + if isinstance(steps, list): + summary["ops"] = [str(step.get("op") or "") for step in steps[:12] if isinstance(step, dict)] + expression = str(args.get("function") or args.get("expression") or args.get("script") or args.get("code") or "") + if expression: + summary["expression_sha256"] = hashlib.sha256(expression.encode("utf-8")).hexdigest()[:16] + summary["expression_chars"] = len(expression) + return json.dumps(summary, ensure_ascii=False, separators=(",", ":"))[:600] + + @classmethod + def _structured_evidence(cls, result: Dict[str, Any]) -> Dict[str, Any]: + extracted = result.get("extracted") + if isinstance(extracted, dict) and extracted: + return cls._structured_extraction_evidence(result, extracted) + condition_evidence = cls._condition_evidence(result) + if condition_evidence: + return condition_evidence + return cls._card_probe_evidence(result) + + @classmethod + def _structured_extraction_evidence( + cls, + result: Dict[str, Any], + extracted: Dict[str, Any], + ) -> Dict[str, Any]: + provenance = result.get("field_provenance") + compact_values: Dict[str, str] = {} + compact_provenance: Dict[str, Any] = {} + for key, value in list(extracted.items())[:20]: + if value in (None, "", [], {}): + continue + canonical = cls._canonical_field_name(key) or str(key) + compact_values[canonical] = str(value)[:300] + if isinstance(provenance, dict) and isinstance(provenance.get(key), dict): + compact_provenance[canonical] = dict(provenance[key]) + condition_values, condition_provenance = cls._successful_condition_values(result) + for key, value in condition_values.items(): + compact_values.setdefault(key, value) + if key in condition_provenance: + compact_provenance.setdefault(key, condition_provenance[key]) + return { + "kind": "structured_extraction", + "generation_id": str(result.get("generation_id") or ""), + "fields": sorted(compact_values), + "values": compact_values, + "provenance": compact_provenance, + } + + @classmethod + def _condition_evidence(cls, result: Dict[str, Any]) -> Dict[str, Any]: + values, provenance = cls._successful_condition_values(result) + if not values: + return {} + return { + "kind": "condition_observation", + "generation_id": str(result.get("generation_id") or ""), + "fields": sorted(values), + "values": values, + "provenance": provenance, + } + + @staticmethod + def _successful_condition_values( + result: Dict[str, Any], + ) -> tuple[Dict[str, str], Dict[str, Dict[str, str]]]: + field_by_op = { + "wait_for_sort_state": "sort_state", + "wait_for_result_count": "result_count", + "wait_for_first_card_title": "title", + "wait_for_url": "url", + "wait_for_dom_text_change": "dom_text", + "wait_for_tab": "tab_state", + } + values: Dict[str, str] = {} + provenance: Dict[str, Dict[str, str]] = {} + generation_id = str(result.get("generation_id") or "") + conditions = result.get("conditions") + for item in conditions if isinstance(conditions, list) else []: + if not isinstance(item, dict) or item.get("ok") is not True: + continue + field_name = field_by_op.get(str(item.get("op") or "").strip().lower()) + if not field_name: + continue + observed = item.get("observed") + if isinstance(observed, dict): + value = next( + ( + observed[key] + for key in ("value", "text", "url", "count", "tabs", "stable") + if observed.get(key) not in (None, "", [], {}) + ), + None, + ) + else: + value = observed + if value in (None, "", [], {}): + continue + rendered = ( + json.dumps(value, ensure_ascii=False, default=str) if isinstance(value, (dict, list)) else str(value) + ) + values[field_name] = rendered[:500] + provenance[field_name] = { + "source": "browser_batch_interact", + "raw_text": rendered[:600], + "generation_id": generation_id, + } + return values, provenance diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/phases.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/phases.py new file mode 100644 index 0000000000..1c31d16ab4 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/phases.py @@ -0,0 +1,1319 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""Task phases, strategy fingerprints and completion requirements.""" + +from __future__ import annotations + +import hashlib +import json +import re +from typing import Any, Dict, Iterable, Optional +from urllib.parse import parse_qsl, unquote, urlencode, urlsplit + +from openjiuwen.harness.tools.browser_move.contract import ( + Click, + Drag, + Effect, + Hover, + Observe, + RunScript, + Scroll, + SwitchContext, +) +from openjiuwen.harness.tools.browser_move.runtime.session import ToolKind +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.keys import ( + _BATCH_MUTATING_TARGET_OPS, + _BROWSER_FIELD_ALIASES, + _BROWSER_PHASE_STATE_KEY, + _BROWSER_TERMINAL_STATUSES, + _contains_any_token, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.semantic.state import price_interval_signature +from openjiuwen.harness.tools.browser_move.surfaces.browser.site_profiles import infer_profile_evidence_entity +from openjiuwen.harness.tools.browser_move.surfaces.browser.working_context.store import BrowserWorkingContextStore + +_BROWSER_REPLAN_DENIAL_LIMIT = 3 +_BROWSER_READ_ONLY_RECOVERY_LIMIT = 1 + +_BROWSER_OUTPUT_REQUEST_CUE_RE = re.compile( + r"(?:返回|告诉我|提取|获取|输出|给出|列出|汇报|报告|比较|对比|" + r"return|tell\s+me|extract|retrieve|output|provide|list|report|compare)" + r"[^。；;.!?\n]{0,240}", + re.IGNORECASE, +) + +_BATCH_SAFE_READ_SELECTOR_OPS = frozenset({"extract_text", "extract_value"}) +_BATCH_EXPLICIT_SELECTOR_OPS = frozenset( + { + "wait_for_selector", + "wait_for_first_card_title", + "wait_for_sort_state", + "wait_for_result_count", + "wait_for_dom_text_change", + "wait_for_stable", + } +) +_FORM_COMMAND_NAMES = frozenset({"TypeText", "PressKey", "SelectOption", "FillForm", "UploadFile"}) +_BROWSER_STRICT_EVIDENCE_FIELDS = frozenset({"rating", "product_rating", "shop_rating", "sort_state"}) +_BROWSER_EVALUATE_FIELD_ALIASES = { + "article_title": "title", + "page_title": "title", + "product_title": "title", + "result_title": "title", + "author_name": "author", + "author_raw": "author", + "creator_name": "author", + "profile_name": "author", + "writer_name": "author", + "like_count": "likes", + "likes_count": "likes", + "like_raw": "likes", + "upvote_count": "likes", + "favorite_count": "favorites", + "favorites_count": "favorites", + "favourite_count": "favorites", + "bookmark_count": "favorites", + "fav_raw": "favorites", + "comment_count": "comments", + "comments_count": "comments", + "reply_count": "comments", + "comment_raw": "comments", + "com_raw": "comments", + "active_tab": "sort_state", + "active_tabs": "sort_state", + "selected_tab": "sort_state", + "selected_tabs": "sort_state", + "sort_selected": "sort_state", +} +_BROWSER_PHASE_DEFINITIONS = { + "navigation": { + "budget": 12, + "completion_condition": "Target URL or expected page identity is observed.", + }, + "form": { + "budget": 24, + "completion_condition": "All required fields are populated and submission is evidenced.", + }, + "filtering": { + "budget": 20, + "completion_condition": "Requested filter/sort state and changed results are observed.", + }, + "extraction": { + "budget": 20, + "completion_condition": "One structured result contains every requested field.", + }, +} + + +def _has_non_empty_mapping_value( + value: Dict[str, Any], + keys: Iterable[str], +) -> bool: + for key in keys: + if value.get(key) not in (None, ""): + return True + return False + + +def _copy_non_none_values( + value: Dict[str, Any], + keys: Iterable[str], +) -> Dict[str, Any]: + copied: Dict[str, Any] = {} + for key in keys: + item = value.get(key) + if item is not None: + copied[key] = item + return copied + + +class _PhaseMixin: + @staticmethod + def _build_phase_state(task: str) -> Dict[str, Any]: + normalized = str(task or "").lower() + complex_tokens = ( + "form", + "filter", + "sort", + "compare", + "book", + "checkout", + "register", + "login", + "purchase", + "choose", + "select", + "fill", + "submit", + "switch", + "cart", + "basket", + "add to cart", + "\u8868\u5355", + "\u7b5b\u9009", + "\u6392\u5e8f", + "\u6bd4\u8f83", + "\u5bf9\u6bd4", + "\u9009\u62e9", + "\u586b\u5199", + "\u63d0\u4ea4", + "\u5207\u6362", + "\u8bbe\u7f6e", + "\u65e5\u671f", + "\u9884\u8ba2", + "\u767b\u5f55", + "\u6ce8\u518c", + "\u7ed3\u8d26", + "\u8d2d\u4e70", + "\u8d2d\u7269\u8f66", + "\u52a0\u8d2d", + "\u52a0\u5165\u8d2d\u7269\u8f66", + ) + task_type = "complex" if any(token in normalized for token in complex_tokens) else "simple" + phase_names = tuple(_BROWSER_PHASE_DEFINITIONS) if task_type == "complex" else ("navigation", "extraction") + phases = { + name: { + "status": "pending", + "attempts": 0, + "budget": _BROWSER_PHASE_DEFINITIONS[name]["budget"], + "completion_condition": _BROWSER_PHASE_DEFINITIONS[name]["completion_condition"], + "blocked_signature": "", + "visited_price_intervals": [], + } + for name in phase_names + } + required_slots = _PhaseMixin._infer_required_evidence_slots(task) + required_fields = list( + dict.fromkeys( + str(slot.get("field") or "") for slot in required_slots if str(slot.get("field") or "").strip() + ) + ) + return { + "task_id": hashlib.sha256(str(task).encode("utf-8")).hexdigest()[:16], + "task": task, + "goal": task, + "requirements_source": "inferred", + "task_type": task_type, + "status": "in_progress", + "phases": phases, + "current_phase": phase_names[0], + "known_urls": re.findall(r"https?://[^\s<>\"]+", task)[:4], + "constraints": [], + "required_fields": required_fields, + "required_evidence_slots": required_slots, + "requested_result_count": _PhaseMixin._infer_requested_result_count(task), + "observed_result_count": 0, + "replan_count": 0, + "replan_denial_count": 0, + "resume_count": 0, + "direct_navigation_guidance_count": 0, + "read_only_recovery_counts": {}, + "model_protocol_retry_count": 0, + "replan_required": False, + "replan_trial_pending": False, + "blocked_strategy": "", + "failed_strategies": [], + "trial_strategy": "", + "last_strategy_fingerprint": "", + "next_action_class": "navigation", + "semantic_revision": 0, + "structured_evidence": [], + "evidence_slots": [], + "field_coverage": [], + "blockers": [], + "recent_actions": [], + "action_sequence": 0, + "last_page": {}, + } + + @classmethod + def _infer_required_fields(cls, task: str) -> list[str]: + normalized = str(task or "").lower() + request_scopes = _BROWSER_OUTPUT_REQUEST_CUE_RE.findall(normalized) + output_scope = " ".join(request_scopes).strip() or normalized + inferred = [ + field + for field, aliases in _BROWSER_FIELD_ALIASES.items() + if any(cls._contains_field_alias(output_scope, alias) for alias in (field, *aliases)) + ] + if "product_rating" in inferred or "shop_rating" in inferred: + inferred = [field_name for field_name in inferred if field_name != "rating"] + if "product_rating" in inferred and not any( + cls._contains_field_alias(normalized, alias) for alias in ("title", "name", "标题", "名称", "电影", "结果") + ): + inferred = [field_name for field_name in inferred if field_name != "title"] + if "shop_rating" in inferred: + without_shop_rating = re.sub( + r"shop\s+rating|seller\s+rating|store\s+rating|店铺评分|卖家评分", + "", + normalized, + flags=re.IGNORECASE, + ) + if not any( + cls._contains_field_alias(without_shop_rating, alias) for alias in _BROWSER_FIELD_ALIASES["shop"] + ): + inferred = [field_name for field_name in inferred if field_name != "shop"] + return inferred + + @classmethod + def _infer_required_evidence_slots(cls, task: str) -> list[Dict[str, str]]: + """Create one provenance-bearing evidence slot per requested field.""" + normalized = str(task or "").lower() + required_fields = cls._infer_required_fields(task) + entity = cls._infer_evidence_entity(normalized) + comprehensive_requested = any( + token in normalized for token in ("comprehensive", "relevance", "综合", "默认排序") + ) + latest_requested = any(token in normalized for token in ("latest", "newest", "最新", "最近发布")) + split_title_variants = bool("title" in required_fields and comprehensive_requested and latest_requested) + slots = [ + {"entity": entity, "variant": "default", "field": field_name} + for field_name in required_fields + if not (field_name == "title" and split_title_variants) + ] + if split_title_variants: + slots.extend( + [ + {"entity": entity, "variant": "comprehensive", "field": "title"}, + {"entity": entity, "variant": "latest", "field": "title"}, + ] + ) + return slots + + @staticmethod + def _infer_evidence_entity(normalized_task: str) -> str: + profile_entity = infer_profile_evidence_entity(normalized_task) + if profile_entity: + return profile_entity + entities = ( + (("商品",), "product"), + (("回答",), "article_or_answer"), + (("article", "文章"), "article"), + (("电影",), "movie"), + (("weather", "天气", "气温"), "weather"), + (("hotel", "酒店"), "hotel"), + (("video", "视频"), "video"), + ) + for aliases, entity in entities: + if any(alias in normalized_task for alias in aliases): + return entity + return "task_result" + + @staticmethod + def _contains_field_alias(text: str, alias: str) -> bool: + normalized_alias = str(alias or "").strip().lower() + if not normalized_alias: + return False + if any("\u4e00" <= character <= "\u9fff" for character in normalized_alias): + return normalized_alias in text + return re.search(rf"(? int: + normalized = str(task or "").lower() + patterns = ( + r"(?:top|first)\s*(\d{1,2})(?:\s*(?:results?|items?|articles?|links?))?", + r"(?:前|最前)\s*(\d{1,2})\s*(?:条|个|篇|项|部|名|则)?", + r"(\d{1,2})\s*(?:条|个|篇|项|部)\s*(?:结果|记录|文章|商品|视频|链接)", + ) + for pattern in patterns: + match = re.search(pattern, normalized, re.IGNORECASE) + if match is not None: + return min(20, max(1, int(match.group(1)))) + number_words = "|".join(cls._NUMBER_WORDS) + verb_match = re.search( + rf"\b(?:find|discover|list|show|get|recommend|identify|locate)\s+(\d{{1,2}}|{number_words})\b", + normalized, + ) + if verb_match is not None: + token = verb_match.group(1) + count = int(token) if token.isdigit() else cls._NUMBER_WORDS[token] + return min(20, max(1, count)) + return 0 + + @staticmethod + def _snake_case_field_name(value: Any) -> str: + text = str(value or "").strip() + text = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", text) + text = re.sub(r"[^a-zA-Z0-9\u4e00-\u9fff]+", "_", text) + return text.strip("_").lower() + + @classmethod + def _canonical_field_name(cls, value: Any) -> str: + snake_name = cls._snake_case_field_name(value) + normalized = " ".join(snake_name.replace("_", " ").split()) + if not normalized: + return "" + for field_name, aliases in _BROWSER_FIELD_ALIASES.items(): + candidates = (field_name.replace("_", " "), *aliases) + if any(normalized == candidate.lower() for candidate in candidates): + return field_name + return str(value or "").strip()[:80] + + @classmethod + def _canonical_evaluate_field_name(cls, value: Any) -> str: + snake_name = cls._snake_case_field_name(value) + if not snake_name: + return "" + explicit = _BROWSER_EVALUATE_FIELD_ALIASES.get(snake_name) + if explicit: + return explicit + canonical = cls._canonical_field_name(snake_name) + if canonical != snake_name: + return canonical + for suffix in ("_raw", "_text", "_value"): + if not snake_name.endswith(suffix): + continue + base_name = snake_name[: -len(suffix)] + canonical_base = cls._canonical_field_name(base_name) + if canonical_base in _BROWSER_FIELD_ALIASES: + return canonical_base + return snake_name[:80] + + @classmethod + def _field_mentioned(cls, field_name: str, text: str) -> bool: + aliases = _BROWSER_FIELD_ALIASES.get(str(field_name), (str(field_name),)) + return any(cls._contains_field_alias(text, alias) for alias in aliases) + + def _sync_semantic_progress(self, session: Any) -> None: + if session is None: + return + progress = self._runtime.semantic_progress + if BrowserWorkingContextStore.sync_semantic_progress(session, progress): + self._runtime.acknowledge_semantic_replan() + + @classmethod + def _classify_tool_phase( + cls, + kind: ToolKind, + tool_args: Any, + state: Dict[str, Any], + ) -> str: + args = cls._coerce_tool_args(tool_args) + serialized = json.dumps(args, ensure_ascii=False).lower() + if kind.effect is Effect.NAVIGATE: + return "navigation" + filter_terms = ( + "filter", + "sort", + "price", + "rating", + "category", + "\u7b5b\u9009", + "\u6392\u5e8f", + "\u4ef7\u683c", + "\u8bc4\u5206", + "\u5206\u7c7b", + ) + filter_terms_present = _contains_any_token(serialized, filter_terms) + script_tool = isinstance(kind.action, RunScript) + filter_intent = bool( + filter_terms_present and (not script_tool or cls._operation_intent(kind, args) == "script_mutation") + ) + steps = args.get("steps") + if kind.helper and isinstance(steps, list): + operations = {str(step.get("op") or "").lower() for step in steps if isinstance(step, dict)} + extraction_operations = {"extract_text", "extract_value"} + mutating_operations = _BATCH_MUTATING_TARGET_OPS | {"press", "navigate", "navigate_back"} + if operations & extraction_operations and not operations & mutating_operations: + return "extraction" + if filter_intent: + return "filtering" + if operations & { + "fill", + "type", + "autocomplete", + "select_option", + "select_visible_text", + "set_checked", + }: + return "form" + if operations & {"extract_text", "extract_value"}: + return "extraction" + if filter_intent: + return "filtering" + if cls._command_name(kind) in _FORM_COMMAND_NAMES: + return "form" + if script_tool or (kind.read_only and not kind.exempt_from_budget): + return "extraction" + current = str(state.get("current_phase") or "navigation") + if kind.known and kind.effect is Effect.ACT and current == "extraction": + return "form" + return current if current in _BROWSER_PHASE_DEFINITIONS else "navigation" + + @classmethod + def _phase_action_signature( + cls, + tool_name: str, + tool_args: Any, + ) -> str: + args = cls._coerce_tool_args(tool_args) + semantic_keys = ( + "url", + "href", + "name", + "text", + "query", + "value", + "values", + "text_value", + "label_value", + "option_value", + "option_label", + "option_text", + "choose_text", + "checked", + "key", + "field", + "action", + "element", + ) + actions: list[Dict[str, Any]] = [] + steps = args.get("steps") + if isinstance(steps, list): + for step in steps: + if isinstance(step, dict): + compact = _copy_non_none_values(step, semantic_keys) + compact["op"] = str(step.get("op") or "").strip().lower() + actions.append(compact) + else: + compact = _copy_non_none_values(args, semantic_keys) + normalized_name = str(tool_name or "").strip().lower() + operation = normalized_name.rsplit("browser_", 1)[-1] + compact["op"] = operation + actions.append(compact) + return json.dumps( + actions, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + )[:600] + + @classmethod + def _strategy_fingerprint( + cls, + state: Dict[str, Any], + kind: ToolKind, + tool_args: Any, + action_class: str, + current_page_state: Optional[Dict[str, Any]] = None, + ) -> str: + """Identify a material strategy without selector or generation noise.""" + + args = cls._coerce_tool_args(tool_args) + fields = cls._requested_strategy_fields(state, args) + intent = cls._operation_intent(kind, args) + descriptor = { + "action_class": action_class, + "page": cls._normalize_navigation_url(cls._current_page_url(state, current_page_state)), + "target": cls._strategy_target(args, current_page_state), + "fields": fields, + "intent": intent, + } + serialized = json.dumps( + descriptor, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + ) + field_label = ",".join(fields[:4]) or "none" + return (f"{action_class}:{intent}:{field_label}:{hashlib.sha256(serialized.encode('utf-8')).hexdigest()[:16]}")[ + :240 + ] + + @staticmethod + def _current_page_url( + state: Dict[str, Any], + current_page_state: Optional[Dict[str, Any]], + ) -> str: + page_state = current_page_state if isinstance(current_page_state, dict) else {} + last_page = state.get("last_page") if isinstance(state.get("last_page"), dict) else {} + return str(page_state.get("url") or last_page.get("url") or "") + + @classmethod + def _strategy_target( + cls, + args: Dict[str, Any], + current_page_state: Optional[Dict[str, Any]] = None, + ) -> str: + url = str(args.get("url") or args.get("href") or "").strip() + if url: + return cls._normalize_navigation_url(url) + target_id = str(args.get("target_id") or "").strip() + if target_id: + return cls._target_semantic_identity(target_id, current_page_state) + for key in ("name", "label", "placeholder", "text", "element"): + value = " ".join(str(args.get(key) or "").split()).lower() + if value: + return f"semantic:{value[:120]}" + steps = args.get("steps") + if isinstance(steps, list): + labels = [] + for step in steps[:12]: + if not isinstance(step, dict): + continue + step_target_id = str(step.get("target_id") or "").strip() + if step_target_id: + labels.append(cls._target_semantic_identity(step_target_id, current_page_state)) + continue + label = next( + ( + " ".join(str(step.get(key) or "").split()).lower() + for key in ("name", "label", "placeholder", "text", "element") + if str(step.get(key) or "").strip() + ), + "", + ) + labels.append(label[:80] or "page_target") + if labels: + return "batch:" + "|".join(labels) + return ( + "page_target" if _has_non_empty_mapping_value(args, ("target_id", "target", "ref", "selector")) else "page" + ) + + @staticmethod + def _target_semantic_identity( + target_id: str, + current_page_state: Optional[Dict[str, Any]], + ) -> str: + page_state = current_page_state if isinstance(current_page_state, dict) else {} + + def find_target(value: Any) -> Optional[Dict[str, Any]]: + if isinstance(value, dict): + if str(value.get("target_id") or "") == target_id: + return value + for nested in value.values(): + match = find_target(nested) + if match is not None: + return match + elif isinstance(value, list): + for nested in value: + match = find_target(nested) + if match is not None: + return match + return None + + target = find_target(page_state) + if target is None: + return "registered_target" + identity = { + key: " ".join(str(target.get(key) or "").split()).lower()[:160] + for key in ("href", "field", "role", "kind", "region", "text") + if str(target.get(key) or "").strip() + } + return json.dumps(identity, ensure_ascii=False, sort_keys=True, separators=(",", ":"))[:400] + + @classmethod + def _requested_strategy_fields(cls, state: Dict[str, Any], args: Dict[str, Any]) -> list[str]: + fields: set[str] = set() + for value in (args.get("field"), args.get("fields")): + values = value if isinstance(value, list) else [value] + for item in values: + canonical = cls._canonical_field_name(item) + if canonical: + fields.add(canonical) + steps = args.get("steps") + if isinstance(steps, list): + for step in steps: + if not isinstance(step, dict): + continue + canonical = cls._canonical_field_name(step.get("field")) + if canonical: + fields.add(canonical) + serialized = json.dumps(args, ensure_ascii=False, default=str).lower() + for field_name in state.get("required_fields") or []: + if cls._field_mentioned(str(field_name), serialized): + fields.add(str(field_name)) + return sorted(fields) + + @classmethod + def _operation_intent(cls, kind: ToolKind, args: Dict[str, Any]) -> str: + steps = args.get("steps") + if isinstance(steps, list): + operations = sorted( + { + str(step.get("op") or "").strip().lower() + for step in steps + if isinstance(step, dict) and str(step.get("op") or "").strip() + } + ) + return "+".join(operations)[:120] or "batch" + if isinstance(kind.action, RunScript): + expression = str( + args.get("function") or args.get("expression") or args.get("script") or args.get("code") or "" + ).lower() + if re.search(r"\.click\s*\(|dispatchEvent|\.value\s*=|setAttribute\s*\(", expression): + return "script_mutation" + if re.search(r"textContent|innerText|return|querySelector|document\.title", expression): + return "script_extraction" + return "script_inspection" + return cls._command_name(kind).lower() or "other" + + @staticmethod + def _price_interval_signature(kind: ToolKind, tool_args: Any) -> str: + """Delegate cross-site filter normalization to semantic-state logic; a script carries no filter.""" + if isinstance(kind.action, RunScript): + return "" + return price_interval_signature("", tool_args) + + @classmethod + def _classify_action_class(cls, kind: ToolKind, tool_name: str, tool_args: Any, state: Dict[str, Any]) -> str: + name = str(tool_name or "").strip().lower() + args = cls._coerce_tool_args(tool_args) + serialized = json.dumps(args, ensure_ascii=False).lower() + if kind.effect is Effect.NAVIGATE: + return "navigation" + if isinstance(kind.action, Scroll): + return "viewport_exploration" + if isinstance(kind.action, RunScript): + return "script_exploration" + if "probe_cards" in name: + return "structured_extraction" + if isinstance(kind.action, Observe) or (kind.helper and kind.read_only and not kind.exempt_from_budget): + return "target_discovery" + phase = cls._classify_tool_phase(kind, tool_args, state) + if phase == "extraction": + return "structured_extraction" + if phase in {"form", "filtering"}: + return phase + if isinstance(kind.action, (Click, Hover, Drag)) or cls._command_name(kind) == "Drop": + return "interaction" + if any(token in serialized for token in ("filter", "sort", "price", "rating", "筛选", "排序", "价格")): + return "filtering" + return "other" + + @classmethod + def _consume_phase_budget( + cls, + session: Any, + kind: ToolKind, + tool_name: str, + tool_args: Any, + *, + current_page_state: Optional[Dict[str, Any]] = None, + replan_group_admitted: bool = False, + ) -> str: + if session is None: + return "other" + state = session.get_state(_BROWSER_PHASE_STATE_KEY) + if not isinstance(state, dict): + return "other" + cls._reject_terminal_state(state) + if kind.exempt_from_budget: + return cls._classify_action_class(kind, tool_name, tool_args, state) + phases = state.setdefault("phases", {}) + phase = cls._classify_tool_phase(kind, tool_args, state) + details = cls._phase_details(phases, phase) + signature = cls._phase_action_signature(tool_name, tool_args) + action_class = cls._classify_action_class(kind, tool_name, tool_args, state) + strategy_fingerprint = cls._strategy_fingerprint( + state, + kind, + tool_args, + action_class, + current_page_state, + ) + + attempts = int(details.get("attempts") or 0) + budget = max(1, int(details.get("budget") or 1)) + if attempts >= budget: + details["status"] = "replan_required" + details["blocked_signature"] = signature + state["current_phase"] = phase + state["status"] = "partial" + state["blockers"] = [f"{phase}_phase_budget_exhausted"] + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + raise ValueError( + f"Browser {phase} phase budget exhausted ({attempts}/{budget}). " + "Finish with available evidence or return partial/blocked." + ) + + cls._reject_revisited_price_interval(details, phase, kind, tool_args) + + if cls._direct_navigation_is_required(state, phases, phase, current_page_state): + state["direct_navigation_guidance_count"] = int(state.get("direct_navigation_guidance_count") or 0) + 1 + recommended_url = str((state.get("known_urls") or [""])[0]) + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + raise ValueError( + "The task already contains a known URL. Navigate to it directly before selector exploration: " + f"{recommended_url}" + ) + + try: + if not replan_group_admitted: + cls._consume_replan_gate( + state, + kind, + tool_args, + action_class, + strategy_fingerprint, + ) + except ValueError: + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + raise + + details["attempts"] = attempts + (0 if cls._repeated_script(details, phase, kind) else 1) + details["status"] = "in_progress" + details["last_signature"] = signature + details["last_semantic_signature"] = signature + state["current_phase"] = phase + state["last_action_class"] = action_class + state["last_strategy_fingerprint"] = strategy_fingerprint + state["next_action_class"] = "" + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + return action_class + + @staticmethod + def _phase_attempts(session: Any) -> int: + state = session.get_state(_BROWSER_PHASE_STATE_KEY) if session is not None else None + phases = state.get("phases") if isinstance(state, dict) else None + return sum(int(item.get("attempts") or 0) for item in (phases or {}).values() if isinstance(item, dict)) + + @staticmethod + def _repeated_script(details: Dict[str, Any], phase: str, kind: ToolKind) -> bool: + """A page script read in this phase before; each distinct script costs one attempt.""" + if phase != "extraction" or not isinstance(kind.action, RunScript): + return False + digest = hashlib.sha256(kind.action.code.encode("utf-8")).hexdigest()[:16] + scripts = details.setdefault("scripts", []) + if digest in scripts: + return True + scripts.append(digest) + return False + + @classmethod + def _direct_navigation_is_required( + cls, + state: Dict[str, Any], + phases: Dict[str, Any], + phase: str, + current_page_state: Optional[Dict[str, Any]], + ) -> bool: + if ( + not state.get("known_urls") + or phase == "navigation" + or int(state.get("direct_navigation_guidance_count") or 0) >= 1 + ): + return False + total_attempts = sum(int(item.get("attempts") or 0) for item in phases.values() if isinstance(item, dict)) + return total_attempts == 0 and not cls._known_url_navigation_satisfied( + state, + current_page_state, + ) + + @classmethod + def _known_url_navigation_satisfied( + cls, + state: Dict[str, Any], + current_page_state: Optional[Dict[str, Any]], + ) -> bool: + page_state = current_page_state if isinstance(current_page_state, dict) else {} + last_page = state.get("last_page") if isinstance(state.get("last_page"), dict) else {} + current_url = str(page_state.get("url") or last_page.get("url") or "").strip() + if not current_url: + return False + current_normalized = cls._normalize_navigation_url(current_url) + current_document = cls._normalize_navigation_url(current_url, include_query=False) + try: + current_parts = urlsplit(current_url) + except ValueError: + current_parts = None + for known_url in state.get("known_urls") or []: + known_normalized = cls._normalize_navigation_url(known_url) + if known_normalized and known_normalized == current_normalized: + return True + known_document = cls._normalize_navigation_url(known_url, include_query=False) + if known_document and known_document == current_document: + return True + try: + known_parts = urlsplit(str(known_url)) + except ValueError: + known_parts = None + if current_parts is None or known_parts is None: + continue + if current_parts.netloc.lower() != known_parts.netloc.lower(): + continue + known_path = (known_parts.path or "/").rstrip("/") or "/" + current_path = (current_parts.path or "/").rstrip("/") or "/" + if known_path == "/" and (current_path != "/" or bool(current_parts.query)): + return True + if known_path != "/" and current_path.startswith(f"{known_path}/"): + return True + return False + + @staticmethod + def _normalize_navigation_url(value: Any, *, include_query: bool = True) -> str: + raw = str(value or "").strip().rstrip(".,;，。；)") + if not raw: + return "" + try: + parsed = urlsplit(raw) + except ValueError: + return raw.lower() + if not parsed.netloc: + return raw.lower().rstrip("/") + path = unquote(parsed.path or "/").rstrip("/") or "/" + query = "" + if include_query: + query_items = [ + (key, item) + for key, item in parse_qsl(parsed.query, keep_blank_values=True) + if not key.lower().startswith("utm_") + ] + query = urlencode(sorted(query_items)) + normalized = f"{parsed.netloc.lower()}{path}" + return f"{normalized}?{query}" if query else normalized + + @staticmethod + def _reject_terminal_state(state: Dict[str, Any]) -> None: + status = str(state.get("status") or "in_progress").strip().lower() + if status in _BROWSER_TERMINAL_STATUSES: + raise ValueError( + f"Browser task is already {status}; do not issue another browser action. " + "Finish with the runtime evidence and blockers." + ) + + @staticmethod + def _phase_details(phases: Dict[str, Any], phase: str) -> Dict[str, Any]: + return phases.setdefault( + phase, + { + "status": "pending", + "attempts": 0, + "budget": _BROWSER_PHASE_DEFINITIONS[phase]["budget"], + "completion_condition": _BROWSER_PHASE_DEFINITIONS[phase]["completion_condition"], + "blocked_signature": "", + "visited_price_intervals": [], + }, + ) + + @classmethod + def _consume_replan_gate( + cls, + state: Dict[str, Any], + kind: ToolKind, + tool_args: Any, + action_class: str, + strategy_fingerprint: str = "", + ) -> None: + cls._reject_terminal_state(state) + if not state.get("replan_required"): + return + if kind.exempt_from_budget: + return + if state.get("replan_trial_pending"): + cls._consume_replan_denial(state, "replan_trial_pending") + raise ValueError( + "A replan trial already ran without verified semantic progress. " + "Wait for the runtime observation, or finish partial/blocked instead of trying " + "another selector, generation, tool, or phase." + ) + replan_count = int(state.get("replan_count") or 0) + budget_s = float(state.get("deadline_budget_s") or 0.0) + remaining_s = float(state.get("deadline_remaining_s") or 0.0) + replan_limit = 4 if budget_s and remaining_s / budget_s > 0.5 else 2 + if replan_count >= replan_limit: + state["status"] = "partial" if cls._has_task_evidence(state) else "blocked" + state["blockers"] = ["semantic_replan_budget_exhausted"] + state["terminal_reason"] = "semantic_replan_budget_exhausted" + state["next_action_class"] = "finish" + raise ValueError( + f"Semantic progress remained blocked after {replan_limit} replan trials. " + "Return blocked or partial with the available structured evidence." + ) + blocked_strategy = str( + state.get("blocked_strategy") + or state.get("last_strategy_fingerprint") + or state.get("last_action_class") + or "" + ) + failed_strategies = {str(item) for item in state.get("failed_strategies") or [] if str(item).strip()} + if blocked_strategy: + failed_strategies.add(blocked_strategy) + current_strategy = strategy_fingerprint or action_class + if current_strategy in failed_strategies or action_class in failed_strategies: + if cls._allow_read_only_recovery(state, kind, tool_args, current_strategy): + state["replan_count"] = replan_count + 1 + state["replan_trial_pending"] = True + state["trial_strategy"] = current_strategy + state["status"] = "replan_trial" + return + cls._consume_replan_denial(state, "repeated_strategy") + raise ValueError( + "Semantic loop detected. Changing selector, generation, tool name, or phase does not " + f"change the {action_class} target/field/intent strategy. Re-plan materially or finish " + "partial/blocked." + ) + state["replan_count"] = replan_count + 1 + state["replan_trial_pending"] = True + state["trial_strategy"] = current_strategy + state["status"] = "replan_trial" + + @classmethod + def _allow_read_only_recovery( + cls, + state: Dict[str, Any], + kind: ToolKind, + tool_args: Any, + strategy_fingerprint: str, + ) -> bool: + if not cls._is_read_only_recovery(kind, tool_args): + return False + counts = state.setdefault("read_only_recovery_counts", {}) + count = int(counts.get(strategy_fingerprint) or 0) + if count >= _BROWSER_READ_ONLY_RECOVERY_LIMIT: + return False + counts[strategy_fingerprint] = count + 1 + return True + + @classmethod + def _is_read_only_recovery(cls, kind: ToolKind, tool_args: Any) -> bool: + args = cls._coerce_tool_args(tool_args) + if kind.read_only: + return True + if isinstance(kind.action, SwitchContext): + return True + if isinstance(kind.action, RunScript): + return cls._operation_intent(kind, args) != "script_mutation" + steps = args.get("steps") + if not kind.helper or not isinstance(steps, list) or not steps: + return False + read_only_ops = ( + _BATCH_SAFE_READ_SELECTOR_OPS + | _BATCH_EXPLICIT_SELECTOR_OPS + | { + "wait_for_text", + "wait_for_load_state", + "wait_for_url", + "wait_for_tab", + } + ) + return all( + isinstance(step, dict) and str(step.get("op") or "").strip().lower() in read_only_ops for step in steps + ) + + @staticmethod + def _consume_replan_denial(state: Dict[str, Any], reason: str) -> None: + denial_count = int(state.get("replan_denial_count") or 0) + 1 + state["replan_denial_count"] = denial_count + state["last_denial_reason"] = str(reason or "")[:120] + if denial_count < _BROWSER_REPLAN_DENIAL_LIMIT: + return + blockers = list(state.get("blockers") or []) + if "semantic_replan_denial_budget_exhausted" not in blockers: + blockers.append("semantic_replan_denial_budget_exhausted") + state["blockers"] = blockers[:10] + state["status"] = "blocked" + state["next_action_class"] = "finish" + state["terminal_reason"] = "semantic_replan_denial_budget_exhausted" + + @classmethod + def _reject_revisited_price_interval( + cls, + details: Dict[str, Any], + phase: str, + kind: ToolKind, + tool_args: Any, + ) -> None: + price_interval = cls._price_interval_signature(kind, tool_args) + visited_intervals = details.setdefault("visited_price_intervals", []) + if phase == "filtering" and price_interval and price_interval in visited_intervals: + raise ValueError( + f"Price interval {price_interval} was already visited in this task. " + "Use the existing evidence or choose a different interval." + ) + + @classmethod + @classmethod + def _record_phase_result( + cls, + session: Any, + kind: ToolKind, + tool_name: str, + tool_args: Any, + tool_result: Any, + *, + progressed: bool = False, + charged: bool = True, + ) -> Dict[str, Any]: + state = session.get_state(_BROWSER_PHASE_STATE_KEY) + if not isinstance(state, dict): + return {} + result = tool_result if isinstance(tool_result, dict) else {"result": tool_result} + succeeded = result.get("ok", True) is not False + if str(state.get("status") or "").strip().lower() in _BROWSER_TERMINAL_STATUSES: + return { + "phase": state.get("current_phase") or "unknown", + "success": succeeded, + "new_evidence_fields": [], + "evidence_added": False, + "recovered": False, + } + phase = cls._classify_tool_phase(kind, tool_args, state) + phases = state.get("phases") + if not isinstance(phases, dict): + return {} + details = phases.get(phase) + if not isinstance(details, dict): + return {} + args = cls._coerce_tool_args(tool_args) + if not succeeded: + evidence_delta = cls._record_tool_evidence( + state, + result, + kind=kind, + tool_name=tool_name, + tool_args=args, + ) + completion_evidence = cls._phase_completion_evidence(phase, kind, result, state) + cls._record_failed_phase_result(state, details, tool_result) + missing_fields = cls._missing_completion_requirements(state) if phase == "extraction" else [] + if completion_evidence and not missing_fields: + cls._complete_phase(state, phases, phase, details, completion_evidence) + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + return {"phase": phase, "success": False, **evidence_delta} + + details["successes"] = int(details.get("successes") or 0) + 1 + details["last_error"] = "" + cls._record_price_interval(details, phase, kind, args) + evidence_delta = cls._record_tool_evidence(state, result, kind=kind, tool_name=tool_name, tool_args=args) + if charged and (evidence_delta["recovered"] if phase == "extraction" else progressed): + details["attempts"] = max(0, int(details.get("attempts") or 0) - 1) + if evidence_delta.get("recovered"): + BrowserWorkingContextStore.mark_replan_recovered(state) + completion_evidence = cls._phase_completion_evidence(phase, kind, result, state) + missing_fields = cls._missing_completion_requirements(state) if phase == "extraction" else [] + if completion_evidence and not missing_fields: + cls._complete_phase(state, phases, phase, details, completion_evidence) + elif details.get("status") != "replan_required": + details["status"] = "in_progress" + details["missing_fields"] = missing_fields + session.update_state({_BROWSER_PHASE_STATE_KEY: state}) + return {"phase": phase, "success": True, **evidence_delta} + + @staticmethod + def _missing_required_fields(state: Dict[str, Any]) -> list[str]: + required = {str(item) for item in state.get("required_fields") or [] if str(item).strip()} + coverage = {str(item) for item in state.get("field_coverage") or [] if str(item).strip()} + return sorted(required - coverage) + + @staticmethod + def _has_task_evidence(state: Dict[str, Any]) -> bool: + """Use evidence records, not compatibility coverage, as task truth.""" + + return bool(state.get("evidence_slots") or state.get("structured_evidence")) + + @staticmethod + def _task_observations(state: Dict[str, Any]) -> list[Dict[str, Any]]: + return [ + item + for item in state.get("structured_evidence") or [] + if isinstance(item, dict) and item.get("kind") == "page_observation" + ] + + @classmethod + def _advisory_missing_fields(cls, state: Dict[str, Any]) -> set[str]: + """Do not mistake an incomplete field adapter for an unsuccessful page read.""" + if state.get("requirements_source") != "inferred" or not cls._task_observations(state): + return set() + slots = state.get("required_evidence_slots") or [] + if any(isinstance(slot, dict) and slot.get("variant", "default") != "default" for slot in slots): + return set() + return set(cls._missing_required_fields(state)) - _BROWSER_STRICT_EVIDENCE_FIELDS + + @classmethod + def _missing_completion_requirements(cls, state: Dict[str, Any]) -> list[str]: + missing = cls._missing_required_fields(state) + missing_field_names = set(missing) + missing_slots = cls._missing_evidence_slots(state) + missing.extend( + f"evidence_slot:{slot['entity']}:{slot['variant']}:{slot['field']}" + for slot in missing_slots + if not (slot["variant"] == "default" and slot["field"] in missing_field_names) + ) + requested_count = int(state.get("requested_result_count") or 0) + observed_count = int(state.get("observed_result_count") or 0) + if requested_count and observed_count < requested_count: + missing.append(f"result_count:{observed_count}/{requested_count}") + return missing + + @classmethod + def _missing_evidence_slots(cls, state: Dict[str, Any]) -> list[Dict[str, str]]: + required_slots = set() + for slot in state.get("required_evidence_slots") or []: + if isinstance(slot, dict): + required_slots.add(cls._evidence_slot_key(slot)) + + covered_slots = set() + for slot in state.get("evidence_slots") or []: + if not isinstance(slot, dict): + continue + status = str(slot.get("status") or "").strip().lower() + if slot.get("value") not in (None, "") or status in {"missing", "unknown"}: + covered_slots.add(cls._evidence_slot_key(slot)) + return [ + {"entity": entity, "variant": variant, "field": field_name} + for entity, variant, field_name in sorted(required_slots - covered_slots) + ] + + @classmethod + def _unavailable_evidence_slots(cls, state: Dict[str, Any]) -> list[Dict[str, str]]: + required_slots = set() + for slot in state.get("required_evidence_slots") or []: + if isinstance(slot, dict): + required_slots.add(cls._evidence_slot_key(slot)) + + unavailable = [] + for slot in state.get("evidence_slots") or []: + if not isinstance(slot, dict): + continue + key = cls._evidence_slot_key(slot) + status = str(slot.get("status") or "").strip().lower() + if key not in required_slots or status not in {"missing", "unknown"}: + continue + unavailable.append( + { + "entity": key[0], + "variant": key[1], + "field": key[2], + "status": status or "unknown", + } + ) + return unavailable + + @staticmethod + def _evidence_slot_key(slot: Dict[str, Any]) -> tuple[str, str, str]: + return ( + str(slot.get("entity") or "").strip().lower(), + str(slot.get("variant") or "").strip().lower(), + str(slot.get("field") or "").strip().lower(), + ) + + @staticmethod + def _record_failed_phase_result(state: Dict[str, Any], details: Dict[str, Any], tool_result: Any) -> None: + if str(state.get("status") or "").strip().lower() in _BROWSER_TERMINAL_STATUSES: + return + details["status"] = "pending" + if isinstance(tool_result, dict): + details["last_error"] = str(tool_result.get("error") or "")[:300] + if not state.get("replan_trial_pending"): + return + trial_strategy = str(state.get("trial_strategy") or "") + state["replan_trial_pending"] = False + state["replan_required"] = True + state["blocked_strategy"] = trial_strategy + BrowserWorkingContextStore.record_failed_strategy(state, trial_strategy) + state["status"] = "replan_required" + + @staticmethod + def _phase_completion_evidence( + phase: str, + kind: ToolKind, + result: Dict[str, Any], + state: Dict[str, Any], + ) -> str: + successful_condition_operations = { + str(condition.get("op") or "").strip().lower() + for condition in (result.get("conditions") if isinstance(result.get("conditions"), list) else []) + if isinstance(condition, dict) and condition.get("ok") is True + } + if phase == "navigation": + result_url = str(result.get("url") or "") + if result_url or kind.effect is Effect.NAVIGATE: + return result_url or "navigation tool succeeded" + if phase == "form" and successful_condition_operations & { + "wait_for_url", + "wait_for_selector", + "wait_for_text", + "wait_for_first_card_title", + "wait_for_result_count", + "wait_for_dom_text_change", + "wait_for_stable", + "wait_for_tab", + }: + return "form batch completed with an observable condition" + if phase == "filtering" and successful_condition_operations & { + "wait_for_sort_state", + "wait_for_result_count", + "wait_for_dom_text_change", + "wait_for_first_card_title", + }: + return "requested state and changed results were observed" + if phase == "extraction": + extracted = result.get("extracted") + cards = result.get("cards") + if isinstance(extracted, dict) and extracted: + return f"structured extraction returned {len(extracted)} field(s)" + if isinstance(cards, list) and cards: + if int(state.get("requested_result_count") or 0) < 1: + return "" + excluded_regions = {"hot_search", "sidebar", "account", "chat"} + excluded_kinds = { + "hot_search", + "paid_column", + "promotion", + "activity", + "account", + "shop", + "chat", + } + natural_cards = [] + for card in cards: + if not isinstance(card, dict) or card.get("is_ad") is True: + continue + if str(card.get("region") or "") in excluded_regions: + continue + if str(card.get("kind") or "") in excluded_kinds: + continue + natural_cards.append(card) + if natural_cards: + return f"card probe returned {len(natural_cards)} structured result(s)" + return "" + + @staticmethod + def _complete_phase( + state: Dict[str, Any], + phases: Dict[str, Any], + phase: str, + details: Dict[str, Any], + completion_evidence: str, + ) -> None: + if str(state.get("status") or "").strip().lower() in _BROWSER_TERMINAL_STATUSES: + return + details["status"] = "completed" + details["completion_evidence"] = completion_evidence[:300] + phase_order = list(phases) + try: + remaining_phases = phase_order[phase_order.index(phase) + 1 :] + except ValueError: + remaining_phases = [] + next_phase = next( + ( + candidate + for candidate in remaining_phases + if isinstance(phases.get(candidate), dict) and phases[candidate].get("status") != "completed" + ), + phase, + ) + state["current_phase"] = next_phase + if all(isinstance(item, dict) and item.get("status") == "completed" for item in phases.values()): + state["status"] = "completed" + state["next_action_class"] = "finish" + else: + state["next_action_class"] = next_phase diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/semantic_state.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/state.py similarity index 97% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/semantic_state.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/state.py index 7802fa8a9b..d65e622585 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/semantic_state.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/semantic/state.py @@ -13,7 +13,6 @@ from typing import Any, Dict, Iterable, Mapping from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit - _FILTER_KEY_TOKENS = ( "filter", "sort", @@ -57,11 +56,7 @@ def _coerce_tool_args(tool_args: Any) -> Dict[str, Any]: def _numbers_from_filter_value(value: Any) -> list[str]: if isinstance(value, (list, tuple)): - return [ - number - for item in value - for number in _numbers_from_filter_value(item) - ][:2] + return [number for item in value for number in _numbers_from_filter_value(item)][:2] if value in (None, ""): return [] return re.findall(r"\d+(?:\.\d+)?", str(value).replace(",", ""))[:2] @@ -112,8 +107,7 @@ def price_interval_signature(tool_name: str, tool_args: Any) -> str: }: continue descriptor = " ".join( - str(step.get(key) or "") - for key in ("field", "name", "label", "placeholder", "description") + str(step.get(key) or "") for key in ("field", "name", "label", "placeholder", "description") ).lower() if not re.search(r"price|amount|价格|价位", descriptor, re.IGNORECASE): continue @@ -315,6 +309,7 @@ def observe( raw_state: Mapping[str, Any], *, action_group_id: str = "", + changed: bool = False, ) -> Dict[str, Any]: """Record one model action group and return its progress classification.""" normalized_group_id = str(action_group_id or "").strip() @@ -343,6 +338,9 @@ def observe( if not self._history: progress = "initial" + elif repeated_state and changed: + progress = "progress" + self._consecutive_no_progress = 0 elif repeated_state: progress = "no_progress" self._consecutive_no_progress += 1 diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/site_profiles.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/site_profiles.py similarity index 90% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/site_profiles.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/site_profiles.py index 4248a9f2bc..a1b250f7f0 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/site_profiles.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/site_profiles.py @@ -5,7 +5,6 @@ import json import logging -import os import re import time from copy import deepcopy @@ -13,7 +12,6 @@ from typing import Any, Dict, List, Mapping from urllib.parse import parse_qsl, urlparse - logger = logging.getLogger(__name__) @@ -341,9 +339,13 @@ def _natural_result_exclusion( reason = "missing_primary_link" if not href: return reason - if not reason and requirements.get("required_link_patterns") and not _matches_any( - href, - requirements.get("required_link_patterns"), + if ( + not reason + and requirements.get("required_link_patterns") + and not _matches_any( + href, + requirements.get("required_link_patterns"), + ) ): reason = "primary_link_not_natural" if not reason and _matches_any(href, requirements.get("excluded_link_patterns")): @@ -392,10 +394,7 @@ def profile_detail_link_key(profile: Mapping[str, Any] | None, value: Any) -> st host = (parsed.hostname or "").lower() if not _domain_matches(host, detail.get("domains") or profile.get("domains")): return "" - query = { - str(key).lower(): str(item) - for key, item in parse_qsl(parsed.query, keep_blank_values=True) - } + query = {str(key).lower(): str(item) for key, item in parse_qsl(parsed.query, keep_blank_values=True)} identifier = next( ( query.get(str(parameter or "").lower(), "").strip() @@ -415,10 +414,7 @@ def _card_semantic_inputs(card: Mapping[str, Any]) -> tuple[str, str, str]: href = str(card.get("primary_link") or card.get("href") or "")[:500] title = " ".join(str(card.get("title") or "").split())[:240] preview = " ".join(str(card.get("text_preview") or "").split())[:500] - badges = " ".join( - " ".join(str(item or "").split())[:100] - for item in (card.get("semantic_badges") or []) - ) + badges = " ".join(" ".join(str(item or "").split())[:100] for item in (card.get("semantic_badges") or [])) selector = " ".join( ( str(card.get("selector_hint") or "")[:500], @@ -643,11 +639,7 @@ def site_profiles_for_url(url: str) -> List[Dict[str, Any]]: host = domain_from_url(url) if not host: return builtin_site_profiles() - return [ - deepcopy(profile) - for profile in BUILTIN_SITE_PROFILES - if _domain_matches(host, profile.get("domains")) - ] + return [deepcopy(profile) for profile in BUILTIN_SITE_PROFILES if _domain_matches(host, profile.get("domains"))] def normalize_route_signature(url: str) -> str: @@ -655,7 +647,6 @@ def normalize_route_signature(url: str) -> str: parsed = urlparse(str(url or "")) path = parsed.path or "/" - # Avoid caching selectors against item-specific IDs too narrowly. path = re.sub(r"\d+", "*", path) path = re.sub(r"/+", "/", path) @@ -671,10 +662,6 @@ def domain_from_url(url: str) -> str: def _default_cache_path() -> Path: - raw = os.environ.get("OPENJIUWEN_BROWSER_SELECTOR_CACHE", "").strip() - if raw: - return Path(raw).expanduser() - return Path.home() / ".openjiuwen" / "browser_selector_cache.json" @@ -816,11 +803,7 @@ def _is_cacheable_card(card: Dict[str, Any]) -> bool: preview = str(card.get("text_preview") or "").strip() buttons = card.get("buttons") or [] - return ( - score >= 30 - and len(preview) >= 80 - and (bool(card.get("primary_link")) or bool(buttons)) - ) + return score >= 30 and len(preview) >= 80 and (bool(card.get("primary_link")) or bool(buttons)) def _generalize_selector(selector: str) -> List[str]: @@ -843,15 +826,10 @@ def _generalize_selector(selector: str) -> List[str]: if len(parts) >= 3: candidates.append(" > ".join(parts[-3:])) - # Keep full non-nth selector as a fallback, but not first. if no_nth and no_nth not in candidates: candidates.append(no_nth) - return [ - item - for item in _unique(candidates, limit=4) - if not _selector_is_too_broad(item) - ] + return [item for item in _unique(candidates, limit=4) if not _selector_is_too_broad(item)] class BrowserSelectorCache: @@ -929,11 +907,7 @@ def record_card_probe_result(self, result: Dict[str, Any]) -> None: if not isinstance(cards, list) or not cards: return - cacheable_cards = [ - card - for card in cards - if isinstance(card, dict) and _is_cacheable_card(card) - ] + cacheable_cards = [card for card in cards if isinstance(card, dict) and _is_cacheable_card(card)] selector_source = str(result.get("selector_source") or "").strip().lower() min_cacheable_cards = 3 if selector_source == "generic" else 2 @@ -966,37 +940,21 @@ def record_card_probe_result(self, result: Dict[str, Any]) -> None: if not isinstance(card, dict): continue - selectors["card_container_selectors"].extend( - _generalize_selector(str(card.get("selector_hint") or "")) - ) + selectors["card_container_selectors"].extend(_generalize_selector(str(card.get("selector_hint") or ""))) title_selector_hint = str(card.get("title_selector_hint") or "") if not _selector_looks_like_author_profile(title_selector_hint): - selectors["title_selectors"].extend( - _generalize_selector(title_selector_hint) - ) - selectors["price_selectors"].extend( - _generalize_selector(str(card.get("price_selector_hint") or "")) - ) - selectors["rating_selectors"].extend( - _generalize_selector(str(card.get("rating_selector_hint") or "")) - ) + selectors["title_selectors"].extend(_generalize_selector(title_selector_hint)) + selectors["price_selectors"].extend(_generalize_selector(str(card.get("price_selector_hint") or ""))) + selectors["rating_selectors"].extend(_generalize_selector(str(card.get("rating_selector_hint") or ""))) selectors["availability_selectors"].extend( _generalize_selector(str(card.get("availability_selector_hint") or "")) ) - selectors["author_selectors"].extend( - _generalize_selector(str(card.get("author_selector_hint") or "")) - ) - selectors["source_selectors"].extend( - _generalize_selector(str(card.get("source_selector_hint") or "")) - ) - selectors["summary_selectors"].extend( - _generalize_selector(str(card.get("summary_selector_hint") or "")) - ) + selectors["author_selectors"].extend(_generalize_selector(str(card.get("author_selector_hint") or ""))) + selectors["source_selectors"].extend(_generalize_selector(str(card.get("source_selector_hint") or ""))) + selectors["summary_selectors"].extend(_generalize_selector(str(card.get("summary_selector_hint") or ""))) primary_link_selector_hint = str(card.get("primary_link_selector_hint") or "") if not _selector_looks_like_author_profile(primary_link_selector_hint): - selectors["primary_link_selectors"].extend( - _generalize_selector(primary_link_selector_hint) - ) + selectors["primary_link_selectors"].extend(_generalize_selector(primary_link_selector_hint)) buttons = card.get("buttons") or [] if isinstance(buttons, list): @@ -1005,21 +963,11 @@ def record_card_probe_result(self, result: Dict[str, Any]) -> None: button_selector_hint = str(button.get("selector_hint") or "") generalized_button_selectors = _generalize_selector(button_selector_hint) if _selector_looks_like_article_link(button_selector_hint): - selectors["primary_link_selectors"].extend( - generalized_button_selectors - ) - selectors["title_selectors"].extend( - generalized_button_selectors - ) - selectors["button_selectors"].extend( - generalized_button_selectors - ) - - selectors = { - key: _unique(value, limit=20) - for key, value in selectors.items() - if _unique(value, limit=20) - } + selectors["primary_link_selectors"].extend(generalized_button_selectors) + selectors["title_selectors"].extend(generalized_button_selectors) + selectors["button_selectors"].extend(generalized_button_selectors) + + selectors = {key: _unique(value, limit=20) for key, value in selectors.items() if _unique(value, limit=20)} if not selectors: return @@ -1089,18 +1037,11 @@ def record_card_probe_result(self, result: Dict[str, Any]) -> None: len(cards), ) - # Keep file small. data["records"] = records[-200:] self._save(data) def record_card_probe_cache_rejection(self, result: Dict[str, Any]) -> None: - """Record that cached card-probe selectors were tried but rejected. - - A card probe can still succeed through a site profile or generic discovery - after cached selectors failed validation. Track that separately so stale - cache records are deprioritized on later exports instead of being retried - indefinitely. - """ + """Record that cached card-probe selectors were tried and rejected.""" if not isinstance(result, dict): return @@ -1126,9 +1067,7 @@ def record_card_probe_cache_rejection(self, result: Dict[str, Any]) -> None: now = time.time() rejection_reason = str( - result.get("cache_rejection_reason") - or result.get("selector_source") - or "cache_validation_failed" + result.get("cache_rejection_reason") or result.get("selector_source") or "cache_validation_failed" ).strip() updated = False @@ -1150,12 +1089,12 @@ def record_card_probe_cache_rejection(self, result: Dict[str, Any]) -> None: self._save(data) - _SELECTOR_CACHE: BrowserSelectorCache | None = None -def get_selector_cache() -> BrowserSelectorCache: +def get_selector_cache(path: Path | None = None) -> BrowserSelectorCache: + """The process-wide selector cache; ``path`` applies only to the call that creates it.""" global _SELECTOR_CACHE if _SELECTOR_CACHE is None: - _SELECTOR_CACHE = BrowserSelectorCache() + _SELECTOR_CACHE = BrowserSelectorCache(path) return _SELECTOR_CACHE diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_state_context_processor.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/state_processor.py similarity index 76% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/browser_state_context_processor.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/state_processor.py index debef42890..6cfb72a9c9 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_state_context_processor.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/state_processor.py @@ -18,8 +18,9 @@ PROMPT_ATTACHMENT_PRESERVE_TAIL_METADATA_KEY, ) -from .browser_logging import browser_agent_log_warning -from .browser_working_context import BrowserWorkingContextStore +from ...contract import Effect +from .runner.logging import browser_agent_log_warning +from .working_context.store import BrowserWorkingContextStore _BROWSER_STATE_MESSAGE_NAME = "current_browser_state" _BROWSER_STATE_METADATA_KEY = "browser_state_context" @@ -30,68 +31,6 @@ _PAGE_CHANGE_CHANGED = "changed" _PAGE_CHANGE_UNCHANGED = "unchanged" _PAGE_CHANGE_UNKNOWN = "unknown" -_BROWSER_STATE_REFRESH_TOOL_NAMES = frozenset( - { - # Runtime interaction helpers. - "browser_batch_interact", - "browser_click", - "browser_close", - "browser_custom_action", - "browser_drag", - "browser_drop", - "browser_evaluate", - "browser_file_upload", - "browser_fill_form", - "browser_handle_dialog", - "browser_hover", - "browser_navigate", - "browser_navigate_back", - "browser_press_key", - "browser_run_code", - "browser_run_code_unsafe", - "browser_select_option", - "browser_tabs", - "browser_type", - "browser_wait_for", - # Coordinate-based interactions can click, drag, hover, or scroll. - "browser_mouse_click_xy", - "browser_mouse_down", - "browser_mouse_drag_xy", - "browser_mouse_move_xy", - "browser_mouse_up", - "browser_mouse_wheel", - # Devtools actions that can alter rendering or resume page activity. - "browser_annotate", - "browser_hide_highlight", - "browser_highlight", - "browser_resize", - "browser_resume", - "browser_video_hide_actions", - "browser_video_show_actions", - # Network and storage mutations can affect the current application. - "browser_network_state_set", - "browser_route", - "browser_unroute", - "browser_cookie_clear", - "browser_cookie_delete", - "browser_cookie_set", - "browser_localstorage_clear", - "browser_localstorage_delete", - "browser_localstorage_set", - "browser_sessionstorage_clear", - "browser_sessionstorage_delete", - "browser_sessionstorage_set", - "browser_set_storage_state", - } -) -_BROWSER_STATE_OBSERVATION_TOOL_NAMES = frozenset( - { - "browser_find", - "browser_probe_cards", - "browser_probe_interactives", - "browser_snapshot", - } -) class BrowserStateContextProcessorConfig(BaseModel): @@ -128,9 +67,6 @@ async def trigger_get_context_window( **kwargs: Any, ) -> bool: del context, context_window, kwargs - # The hook must run for every model call because this processor's state - # message is ephemeral. The expensive browser capture inside the hook is - # gated on the initial call and completed browser-mutating tool calls. return True async def on_get_context_window( @@ -193,16 +129,8 @@ def _completed_refresh_action_group( action_group_id, refresh_ids, observation_only = cls._completed_state_action_group(messages) if observation_only: return "", set() - mutation_ids: set[str] = set() - for message in messages: - if not isinstance(message, AssistantMessage): - continue - for tool_call in message.tool_calls or []: - call_id = str(tool_call.id or "") - if not call_id or call_id not in refresh_ids: - continue - if cls._is_refresh_tool_name(tool_call.name): - mutation_ids.add(call_id) + _, tool_messages, _ = cls._completed_tool_message_index(messages) + mutation_ids = {call_id for call_id in refresh_ids if cls._effect(tool_messages.get(call_id)) != Effect.READ} return action_group_id, mutation_ids @classmethod @@ -270,10 +198,14 @@ def _classify_completed_action_group( if not call_id or call_id not in executed_call_ids: continue tool_message = tool_messages.get(call_id) - if cls._is_refresh_tool_name(tool_call.name) and cls._tool_message_changed_state(tool_message): + effect = cls._effect(tool_message) + if effect is None: + continue + if effect == Effect.READ: + if cls._tool_message_succeeded(tool_message): + observation_ids.add(call_id) + elif cls._tool_message_changed_state(tool_message): mutation_ids.add(call_id) - if cls._is_observation_tool_name(tool_call.name) and cls._tool_message_succeeded(tool_message): - observation_ids.add(call_id) return call_ids, mutation_ids, observation_ids @classmethod @@ -288,82 +220,32 @@ def _completed_tool_message_index( } completed_call_ids = set(tool_messages) executed_call_ids = { - call_id - for call_id, message in tool_messages.items() - if cls._tool_message_was_executed(message) + call_id for call_id, message in tool_messages.items() if cls._tool_message_was_executed(message) } return completed_call_ids, tool_messages, executed_call_ids @staticmethod - def _tool_message_was_executed(message: ToolMessage) -> bool: - metadata = message.metadata if isinstance(message.metadata, dict) else {} - if metadata.get("denied") is True or metadata.get("executed") is False: - return False - content = message.content - if isinstance(content, str): - try: - payload = json.loads(content) - except (TypeError, ValueError): - payload = None - if isinstance(payload, dict) and (payload.get("denied") is True or payload.get("executed") is False): - return False - return True - - @staticmethod - def _tool_message_changed_state(message: ToolMessage | None) -> bool: - if message is None: - return False - metadata = message.metadata if isinstance(message.metadata, dict) else {} - if metadata.get("state_changed") is False: - return False - content = message.content - if isinstance(content, str): - try: - payload = json.loads(content) - except (TypeError, ValueError): - payload = None - if isinstance(payload, dict) and payload.get("state_changed") is False: - return False - return True + def _metadata(message: ToolMessage | None) -> dict[str, Any]: + metadata = message.metadata if message is not None else None + return metadata if isinstance(metadata, dict) else {} - @staticmethod - def _tool_message_succeeded(message: ToolMessage | None) -> bool: - if message is None: - return False - metadata = message.metadata if isinstance(message.metadata, dict) else {} - if metadata.get("success") is False: - return False - content = message.content - if isinstance(content, str): - try: - payload = json.loads(content) - except (TypeError, ValueError): - payload = None - if isinstance(payload, dict) and ( - payload.get("ok") is False or payload.get("success") is False - ): - return False - return True + @classmethod + def _effect(cls, message: ToolMessage | None) -> str | None: + effect = cls._metadata(message).get("effect") + return effect if isinstance(effect, str) else None @classmethod - def _completed_refresh_tool_call_ids(cls, messages: list[BaseMessage]) -> set[str]: - """Compatibility projection for callers that only need completed IDs.""" - _, refresh_tool_call_ids = cls._completed_refresh_action_group(messages) - return refresh_tool_call_ids + def _tool_message_was_executed(cls, message: ToolMessage) -> bool: + metadata = cls._metadata(message) + return metadata.get("denied") is not True and metadata.get("executed") is not False - @staticmethod - def _is_refresh_tool_name(tool_name: str) -> bool: - return any( - tool_name == expected or tool_name.endswith(f".{expected}") or tool_name.endswith(f"_{expected}") - for expected in _BROWSER_STATE_REFRESH_TOOL_NAMES - ) + @classmethod + def _tool_message_changed_state(cls, message: ToolMessage | None) -> bool: + return message is not None and cls._metadata(message).get("state_changed") is not False - @staticmethod - def _is_observation_tool_name(tool_name: str) -> bool: - return any( - tool_name == expected or tool_name.endswith(f".{expected}") or tool_name.endswith(f"_{expected}") - for expected in _BROWSER_STATE_OBSERVATION_TOOL_NAMES - ) + @classmethod + def _tool_message_succeeded(cls, message: ToolMessage | None) -> bool: + return message is not None and cls._metadata(message).get("success") is not False async def _capture_state(self, *, action_group_id: str) -> Dict[str, Any]: try: @@ -492,17 +374,19 @@ def _format_state_text(self, state: Dict[str, Any]) -> str: if not isinstance(page_state, dict): page_state = {} - state_header = self._fit_state_header({ - "ok": bool(state.get("ok")), - "error": state.get("error"), - "url": state.get("url") or "", - "title": state.get("title") or "", - "tabs": state.get("tabs") or [], - "page_position": state.get("page_position") or {}, - "semantic_state": state.get("semantic_state") or {}, - "dom_error": state.get("dom_error"), - "page_state": page_state, - }) + state_header = self._fit_state_header( + { + "ok": bool(state.get("ok")), + "error": state.get("error"), + "url": state.get("url") or "", + "title": state.get("title") or "", + "tabs": state.get("tabs") or [], + "page_position": state.get("page_position") or {}, + "semantic_state": state.get("semantic_state") or {}, + "dom_error": state.get("dom_error"), + "page_state": page_state, + } + ) return ( "\n" "This observation was captured initially or after the latest detected browser mutation and " @@ -570,8 +454,7 @@ def _fit_state_header(self, value: Dict[str, Any]) -> Dict[str, Any]: values = compact_semantic.get(key) if isinstance(values, dict): compact_semantic[key] = { - str(item_key)[:80]: str(item_value)[:160] - for item_key, item_value in list(values.items())[:8] + str(item_key)[:80]: str(item_value)[:160] for item_key, item_value in list(values.items())[:8] } elif isinstance(values, list): compact_semantic[key] = [str(item)[:160] for item in values[:8]] diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/status_logging.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/status_logging.py similarity index 89% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/status_logging.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/status_logging.py index 3ac0150d26..7c649f42e6 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/status_logging.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/status_logging.py @@ -6,19 +6,19 @@ import hashlib import json import logging -import os import re import time from collections import Counter from typing import Any, Callable, Mapping, Optional from urllib.parse import urlsplit, urlunsplit -from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_logging import ( + +from openjiuwen.harness.tools.browser_move.runtime.config import CoreSettings +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import CommandRecord, command_record +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.logging import ( browser_agent_log_info, ) _STATUS_KEY = "_browser_subagent_status_logging" -_TRUE_VALUES = {"1", "true", "yes", "on"} -_FALSE_VALUES = {"0", "false", "no", "off"} _TEXT_VALUE_KEYS = { "value", "text", @@ -61,18 +61,18 @@ } _TOOL_HISTORY_LIMIT = 20 _URL_RE = re.compile(r"https?://[^\s)>'\"]+", re.IGNORECASE) +_BATCH_TOOL = "browser_batch_interact" +_SCRIPT_COMMAND = "RunScript" +_NAVIGATE_COMMAND = "Navigate" +_OBSERVE_COMMAND = "Observe" MetadataProvider = Callable[[], Mapping[str, Any]] +CommandOf = Callable[[str, Any], str] -def is_browser_subagent_status_log_enabled() -> bool: +def is_browser_subagent_status_log_enabled(settings: CoreSettings | None = None) -> bool: """Return whether browser subagent status logging is enabled.""" - raw = (os.getenv("BROWSER_SUBAGENT_STATUS_LOG") or "").strip().lower() - if raw in _FALSE_VALUES: - return False - if raw in _TRUE_VALUES: - return True - return True + return (settings or CoreSettings.from_process_env()).status_log def _safe_str(value: Any, limit: int = 240) -> str: @@ -126,10 +126,12 @@ def _sanitize_url(value: Any) -> str: def _extract_text_payload(value: Any) -> Any: - """Unwrap common ToolOutput/MCP text result shapes.""" + """Unwrap common ToolOutput result shapes; a command tool's output yields its rendered content.""" if hasattr(value, "success") and hasattr(value, "data"): data = getattr(value, "data", None) error = getattr(value, "error", None) + if isinstance(data, Mapping) and "command" in data: + return data.get("content") if data is not None: return data if error: @@ -161,10 +163,12 @@ def __init__( logger: Optional[logging.Logger] = None, metadata_provider: Optional[MetadataProvider] = None, marker: str = "[BROWSER_SUBAGENT]", + command_of: Optional[CommandOf] = None, ) -> None: self._logger = logger or logging.getLogger(__name__) self._metadata_provider = metadata_provider self._marker = marker + self._command_of = command_of self._runs: dict[str, dict[str, Any]] = {} def before_invoke(self, ctx: Any) -> None: @@ -293,15 +297,16 @@ def before_tool_call(self, ctx: Any) -> None: state.setdefault("tool_starts", {})[tool_key] = time.monotonic() state.setdefault("tool_counts", Counter())[tool_name] += 1 tool_args = _mapping_get(inputs, "tool_args", None) + command = self._command_of(tool_name, tool_args) if self._command_of is not None else "" previous_batch = state.get("last_failed_batch") - if previous_batch and tool_name and tool_name != "browser_batch_interact": + if previous_batch and tool_name and tool_name != _BATCH_TOOL: state["fallback_count"] = int(state.get("fallback_count", 0)) + 1 self._emit( "fallback_detected", ctx, { - "from_tool": "browser_batch_interact", + "from_tool": _BATCH_TOOL, "to_tool": _safe_str(tool_name, 180), "previous_batch": previous_batch, }, @@ -315,7 +320,8 @@ def before_tool_call(self, ctx: Any) -> None: "iteration": state.get("model_calls", 0), "tool_index": state.get("tool_calls", 0), "tool_name": _safe_str(tool_name, 180), - "args_summary": self.summarize_args(str(tool_name), tool_args), + "command": command, + "args_summary": self.summarize_args(str(tool_name), tool_args, command), }, ) @@ -333,7 +339,7 @@ def after_tool_call(self, ctx: Any) -> None: tool_result = _mapping_get(inputs, "tool_result", None) result_summary = self.summarize_result(str(tool_name), tool_result) self._record_tool_history(state, tool_name, elapsed_ms, result_summary) - if tool_name == "browser_batch_interact": + if tool_name == _BATCH_TOOL: self._accumulate_batch_result(state, result_summary) self._emit( "tool_end", @@ -369,17 +375,17 @@ def on_tool_exception(self, ctx: Any) -> None: }, ) - def summarize_args(self, tool_name: str, tool_args: Any) -> dict[str, Any]: + def summarize_args(self, tool_name: str, tool_args: Any, command: str = "") -> dict[str, Any]: + """Shape the arguments by the command they decode to; the batch helper by its own name.""" parsed = _parse_jsonish(tool_args) if isinstance(parsed, list): return {"kind": "list", "length": len(parsed)} if not isinstance(parsed, Mapping): return {"kind": type(parsed).__name__, "repr_length": len(str(parsed))} - lowered_name = (tool_name or "").lower() - if lowered_name == "browser_batch_interact": + if tool_name == _BATCH_TOOL: return self._summarize_batch_args(parsed) - if "run_code" in lowered_name or "evaluate" in lowered_name: + if command == _SCRIPT_COMMAND: code = parsed.get("code") or parsed.get("script") or parsed.get("expression") or parsed.get("function") return { "kind": "code_execution", @@ -387,7 +393,7 @@ def summarize_args(self, tool_name: str, tool_args: Any) -> dict[str, Any]: "code_length": len(str(code or "")), "code_redacted": True, } - if "navigate" in lowered_name: + if command == _NAVIGATE_COMMAND: return { "kind": "navigation", "url": _sanitize_url(parsed.get("url") or parsed.get("href") or parsed.get("target")), @@ -403,7 +409,14 @@ def summarize_args(self, tool_name: str, tool_args: Any) -> dict[str, Any]: if key_lower in _TEXT_VALUE_KEYS or any(token in key_lower for token in ("password", "token", "secret")): redacted[key_str] = _redacted_text(value) elif key_lower in { - "selector", "role", "label", "placeholder", "checked", "timeout", "timeout_ms", "max_items" + "selector", + "role", + "label", + "placeholder", + "checked", + "timeout", + "timeout_ms", + "max_items", }: safe_values[key_str] = value if isinstance(value, (bool, int, float)) else _safe_str(value, 120) if safe_values: @@ -413,17 +426,20 @@ def summarize_args(self, tool_name: str, tool_args: Any) -> dict[str, Any]: return summary def summarize_result(self, tool_name: str, tool_result: Any) -> dict[str, Any]: + """Shape the result by its command record; a helper's payload by its shape, the batch by its name.""" payload = _extract_text_payload(tool_result) parsed = _parse_jsonish(payload) if isinstance(parsed, str): parsed_json = _parse_jsonish(parsed) if not isinstance(parsed_json, str): parsed = parsed_json - lowered_name = (tool_name or "").lower() if isinstance(parsed, Mapping): - if lowered_name == "browser_batch_interact": + if tool_name == _BATCH_TOOL: return self._summarize_batch_result(parsed) - return self._summarize_mapping_result(parsed, tool_name=lowered_name) + return self._summarize_mapping_result(parsed) + record = command_record(tool_result) + if record is not None: + return self._summarize_command_result(record, parsed) if isinstance(parsed, list): return {"kind": "list", "length": len(parsed)} text = "" if parsed is None else str(parsed) @@ -437,7 +453,7 @@ def summarize_result(self, tool_name: str, tool_result: Any) -> dict[str, Any]: def _summarize_batch_args(args: Mapping[str, Any]) -> dict[str, Any]: steps = args.get("steps") if not isinstance(steps, list): - return {"kind": "browser_batch_interact", "step_count": 0, "keys": sorted(str(key) for key in args.keys())} + return {"kind": _BATCH_TOOL, "step_count": 0, "keys": sorted(str(key) for key in args.keys())} op_counts = Counter(str(step.get("op") or "") for step in steps if isinstance(step, Mapping)) step_summaries = [] for idx, step in enumerate(steps[:10]): @@ -454,7 +470,7 @@ def _summarize_batch_args(args: Mapping[str, Any]) -> dict[str, Any]: } ) return { - "kind": "browser_batch_interact", + "kind": _BATCH_TOOL, "step_count": len(steps), "op_counts": dict(sorted(op_counts.items())), "steps_preview": step_summaries, @@ -501,7 +517,28 @@ def _summarize_batch_result(result: Mapping[str, Any]) -> dict[str, Any]: return summary @staticmethod - def _summarize_mapping_result(result: Mapping[str, Any], *, tool_name: str = "") -> dict[str, Any]: + def _summarize_command_result(record: CommandRecord, content: Any) -> dict[str, Any]: + command = record.command.__name__ + summary: dict[str, Any] = { + "kind": "command", + "command": command, + "effect": record.effect.value, + "ok": record.ok, + "dispatched": record.dispatched, + } + if record.error: + summary["error"] = record.error + if isinstance(content, str): + summary["result_length"] = len(content) + if command == _OBSERVE_COMMAND: + summary["line_count"] = content.count("\n") + 1 if content else 0 + if command == _SCRIPT_COMMAND and record.value is not None: + summary["result_kind"] = type(record.value).__name__ + summary["result_length"] = len(str(record.value)) + return summary + + @staticmethod + def _summarize_mapping_result(result: Mapping[str, Any]) -> dict[str, Any]: keys = sorted(str(key) for key in result.keys()) summary: dict[str, Any] = {"kind": "dict", "keys": keys} if "ok" in result: @@ -522,21 +559,6 @@ def _summarize_mapping_result(result: Mapping[str, Any], *, tool_name: str = "") "url": _sanitize_url(page.get("url")), "title": _safe_str(page.get("title"), 160), } - if "screenshot" in tool_name or "take_screenshot" in tool_name: - for key in ("path", "filename", "file", "name"): - if result.get(key): - summary["file"] = _safe_str(result.get(key), 180) - break - if "snapshot" in tool_name: - snapshot_text = result.get("snapshot") or result.get("text") or result.get("result") - if isinstance(snapshot_text, str): - summary["result_length"] = len(snapshot_text) - summary["line_count"] = snapshot_text.count("\n") + 1 if snapshot_text else 0 - if "run_code" in tool_name or "evaluate" in tool_name: - payload = result.get("result") or result.get("data") or result.get("text") - if payload is not None: - summary["result_kind"] = type(payload).__name__ - summary["result_length"] = len(str(payload)) return summary @staticmethod @@ -570,12 +592,10 @@ def _summarize_query(query: Any) -> dict[str, Any]: domain = "" if domain and domain not in domains: domains.append(domain) - + task_hash = "" if text: - task_hash = hashlib.sha256( - text.encode("utf-8", "ignore") - ).hexdigest()[:16] + task_hash = hashlib.sha256(text.encode("utf-8", "ignore")).hexdigest()[:16] return { "length": len(text), @@ -602,7 +622,7 @@ def _extract_line_value(text: str, label: str) -> str: prefix = f"{label}:" for line in text.splitlines()[:12]: if line.startswith(prefix): - return _safe_str(line[len(prefix):].strip(), 120) + return _safe_str(line[len(prefix) :].strip(), 120) return "" def _state(self, ctx: Any) -> dict[str, Any]: diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/tools.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/tools.py new file mode 100644 index 0000000000..acae1bf4b3 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/tools.py @@ -0,0 +1,908 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The browser-dialect tools: names, descriptions and schemas as the model knows them, and their argument decoders.""" + +from __future__ import annotations + +import datetime +from collections.abc import Mapping +from typing import Any, Iterable, Optional + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + Click, + CloseContext, + ContextRef, + Drag, + DriverRef, + HandleDialog, + Hover, + Locator, + Observe, + PressKey, + Ref, + RefResolver, + RunScript, + Scroll, + SwitchContext, + ToolSpec, + TypeText, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FieldKind, + FillForm, + FormField, + GetConsole, + GetNetwork, + GetPdf, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) + +_TRUE = frozenset({"1", "true", "yes"}) +_FALSE = frozenset({"0", "false", "no"}) +_CHECKED = frozenset({"1", "true", "yes", "on", "checked"}) +_FIELD_KINDS = { + "textbox": FieldKind.TEXTBOX, + "spinbutton": FieldKind.TEXTBOX, + "searchbox": FieldKind.TEXTBOX, + "slider": FieldKind.SLIDER, + "combobox": FieldKind.COMBOBOX, + "listbox": FieldKind.COMBOBOX, + "select": FieldKind.COMBOBOX, + "checkbox": FieldKind.CHECKBOX, + "switch": FieldKind.CHECKBOX, + "radio": FieldKind.RADIO, +} +_TAB_ACTIONS = ("list", "new", "select", "close") + +_ELEMENT_TARGET_PROPERTIES: dict[str, Any] = { + "generation_id": { + "type": "string", + "description": "Current PageState generation_id for the element target.", + }, + "target_id": { + "type": "string", + "description": "Generation-scoped target_id from PageState / probes.", + }, + "ref": { + "type": "string", + "description": "Element ref from the current page observation.", + }, + "selector": { + "type": "string", + "description": "CSS selector when a validated selector is already known.", + }, +} + +_NAVIGATE_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "Absolute URL to open, e.g. https://example.com", + }, + "wait_until": { + "type": "string", + "description": "Navigation wait condition. Default load.", + }, + "timeout_ms": { + "type": "integer", + "description": "Optional navigation timeout in milliseconds.", + }, + }, + "required": ["url"], +} + +_NO_PARAMS: dict[str, Any] = {"type": "object", "properties": {}, "required": []} + +_CLICK_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + **_ELEMENT_TARGET_PROPERTIES, + "button": { + "type": "string", + "description": "Mouse button: left, right, or middle. Default left.", + }, + "click_count": { + "type": "integer", + "description": "Number of clicks. Default 1.", + }, + }, + "required": ["generation_id"], +} + +_TYPE_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + **_ELEMENT_TARGET_PROPERTIES, + "text": { + "type": "string", + "description": "Text to type into the element.", + }, + "clear": { + "type": "boolean", + "description": "Clear existing content before typing. Default true.", + }, + "press_enter": { + "type": "boolean", + "description": "Press Enter after typing. Default false.", + }, + "sensitive": { + "type": "boolean", + "description": "Mark input as sensitive (e.g. password). Default false.", + }, + }, + "required": ["generation_id", "text"], +} + +_PRESS_KEY_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "key": { + "type": "string", + "description": "Key or combination, e.g. Enter, Control+a, Escape.", + }, + }, + "required": ["key"], +} + +_SCREENSHOT_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "full_page": { + "type": "boolean", + "description": "Capture the full scrollable page. Default false.", + }, + }, + "required": [], +} + +_TABS_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "action": { + "type": "string", + "description": "Tab action: list, select, or close. Default list.", + "enum": ["list", "select", "close", "new"], + }, + "index": { + "type": "integer", + "description": "Zero-based tab index for select/close.", + }, + }, + "required": [], +} + +_SELECT_OPTION_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + **_ELEMENT_TARGET_PROPERTIES, + "value": { + "type": "string", + "description": "Option value attribute to select.", + }, + "label": { + "type": "string", + "description": "Visible option label to select.", + }, + }, + "required": ["generation_id"], +} + +_EVALUATE_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "function": { + "type": "string", + "description": "JavaScript source to evaluate (expression or function body).", + }, + "args": { + "description": "Optional JSON-serializable argument passed to the script.", + }, + }, + "required": ["function"], +} + +_DRAG_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "generation_id": _ELEMENT_TARGET_PROPERTIES["generation_id"], + "source_target_id": { + "type": "string", + "description": "Source element target_id from PageState.", + }, + "source_ref": { + "type": "string", + "description": "Source element ref.", + }, + "source_selector": { + "type": "string", + "description": "Source CSS selector.", + }, + "target_target_id": { + "type": "string", + "description": "Target element target_id from PageState.", + }, + "target_ref": { + "type": "string", + "description": "Target element ref.", + }, + "target_selector": { + "type": "string", + "description": "Target CSS selector.", + }, + "steps": { + "type": "integer", + "description": "Intermediate drag steps. Default 10.", + }, + }, + "required": ["generation_id"], +} + +_FILE_UPLOAD_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + **_ELEMENT_TARGET_PROPERTIES, + "paths": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Absolute or workspace-relative file paths to upload. " + "Relative/bare names also resolve under BROWSER_UPLOAD_ROOT when set " + "(see list_upload_files)." + ), + }, + }, + "required": ["generation_id", "paths"], +} + +_HOVER_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + **_ELEMENT_TARGET_PROPERTIES, + }, + "required": ["generation_id"], +} + +_HANDLE_DIALOG_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "accept": { + "type": "boolean", + "description": "True to accept/OK, false to dismiss/Cancel.", + }, + "promptText": { + "type": "string", + "description": "Optional text to submit for a prompt() dialog.", + }, + "prompt_text": { + "type": "string", + "description": "Alias for promptText.", + }, + }, + "required": ["accept"], +} + +_DROP_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + **_ELEMENT_TARGET_PROPERTIES, + "paths": { + "type": "array", + "items": {"type": "string"}, + "description": ( + "Absolute or workspace-relative file paths to drop. " + "Resolved like browser_file_upload (BROWSER_UPLOAD_ROOT)." + ), + }, + "data": { + "type": "array", + "description": "MIME-typed drag data items (mimeType + data).", + "items": { + "type": "object", + "properties": { + "mimeType": {"type": "string"}, + "mime_type": {"type": "string"}, + "data": {"type": "string"}, + }, + }, + }, + }, + "required": ["generation_id"], +} + +_FILL_FORM_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "generation_id": _ELEMENT_TARGET_PROPERTIES["generation_id"], + "fields": { + "type": "array", + "description": "Ordered field fill steps.", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string", + "description": "Field type: textbox, combobox/select, checkbox/radio.", + }, + "target_id": {"type": "string"}, + "ref": {"type": "string"}, + "selector": {"type": "string"}, + "value": {}, + "label": {"type": "string"}, + "sensitive": {"type": "boolean"}, + }, + "required": [], + }, + }, + }, + "required": ["generation_id", "fields"], +} + +_SNAPSHOT_PARAMS: dict[str, Any] = { + "type": "object", + "properties": { + "include_screenshot": { + "type": "boolean", + "description": "Include a screenshot in the observation. Default false.", + }, + }, + "required": [], +} + + +def _text(args: Mapping[str, Any], name: str) -> str: + return str(args.get(name) or "").strip() + + +def _flag(args: Mapping[str, Any], name: str, default: bool) -> bool: + value = args.get(name, default) + if isinstance(value, str): + word = value.strip().lower() + return True if word in _TRUE else False if word in _FALSE else default + return bool(value) + + +def _integer(args: Mapping[str, Any], name: str, default: int | None) -> int | None: + value = args.get(name) + if value in (None, ""): + return default + try: + return int(value) + except (TypeError, ValueError): + raise ValueError(f"'{name}' must be an integer") from None + + +def _generation(args: Mapping[str, Any]) -> str: + generation = _text(args, "generation_id") + if not generation: + raise ValueError("'generation_id' is required") + return generation + + +def _target(args: Mapping[str, Any], resolver: RefResolver, generation: str, prefix: str = "") -> Ref: + target_id, ref, selector = (_text(args, prefix + name) for name in ("target_id", "ref", "selector")) + if target_id: + return resolver.resolve(target_id, generation) + if ref: + return DriverRef(ref, generation) + if selector: + return Locator(css=selector, text=None, generation=generation) + raise ValueError(f"exactly one of {prefix}target_id, {prefix}ref, or {prefix}selector is required") + + +def _element(args: Mapping[str, Any], resolver: RefResolver) -> Ref: + return _target(args, resolver, _generation(args)) + + +def _paths(args: Mapping[str, Any]) -> tuple[str, ...] | None: + value = args.get("paths") + if value is None: + return None + if not isinstance(value, list): + raise ValueError("'paths' must be a list of strings") + return tuple(str(path) for path in value) + + +def _current_context(resolver: RefResolver) -> ContextRef: + context = resolver.current_context + if context is None: + raise ValueError("no open tab") + return context + + +def build_navigate(args: Mapping[str, Any], resolver: RefResolver) -> Action: + url = _text(args, "url") + if not url: + raise ValueError("'url' is required") + return Navigate(url) + + +def build_navigate_back(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Back() + + +def build_click(args: Mapping[str, Any], resolver: RefResolver) -> Action: + count = _integer(args, "click_count", 1) + return Click(_element(args, resolver), button=_text(args, "button") or "left", count=max(1, count)) + + +def build_type(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return TypeText( + _element(args, resolver), + str(args.get("text") or ""), + submit=_flag(args, "press_enter", False), + sensitive=_flag(args, "sensitive", False), + clear=_flag(args, "clear", True), + ) + + +def build_press_key(args: Mapping[str, Any], resolver: RefResolver) -> Action: + key = str(args.get("key") or "") + if not key: + raise ValueError("'key' is required") + return PressKey(key) + + +def build_take_screenshot(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Observe(include_screenshot=True, full_page=_flag(args, "full_page", False)) + + +def build_tabs(args: Mapping[str, Any], resolver: RefResolver) -> Action: + action = _text(args, "action").lower() or "list" + if action not in _TAB_ACTIONS: + raise ValueError(f"action must be one of: {', '.join(_TAB_ACTIONS)}") + if action == "list": + return Observe() + if action == "new": + return OpenContext() + index = _integer(args, "index", None) + if index is None: + if action == "select": + raise ValueError("action=select requires integer index") + return CloseContext(_current_context(resolver)) + contexts = resolver.contexts + if not 0 <= index < len(contexts): + raise ValueError(f"tab index {index} is out of range: {len(contexts)} tab(s) open") + return SwitchContext(contexts[index]) if action == "select" else CloseContext(contexts[index]) + + +def build_close(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return CloseContext(_current_context(resolver)) + + +def build_select_option(args: Mapping[str, Any], resolver: RefResolver) -> Action: + value = _text(args, "value") or _text(args, "label") + if not value: + raise ValueError("exactly one of value or label is required") + return SelectOption(_element(args, resolver), (value,)) + + +def build_evaluate(args: Mapping[str, Any], resolver: RefResolver) -> Action: + code = str(args.get("function") or args.get("source") or "") + if not code: + raise ValueError("'function' is required") + return RunScript(code, args.get("args")) + + +def build_drag(args: Mapping[str, Any], resolver: RefResolver) -> Action: + generation = _generation(args) + _integer(args, "steps", None) + return Drag( + _target(args, resolver, generation, prefix="source_"), _target(args, resolver, generation, prefix="target_") + ) + + +def build_file_upload(args: Mapping[str, Any], resolver: RefResolver) -> Action: + paths = _paths(args) + if paths is None: + raise ValueError("'paths' must be a list of strings") + return UploadFile(_element(args, resolver), paths) + + +def build_hover(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Hover(_element(args, resolver)) + + +def build_handle_dialog(args: Mapping[str, Any], resolver: RefResolver) -> Action: + if "accept" not in args: + raise ValueError("'accept' is required") + prompt = args.get("promptText", args.get("prompt_text")) + return HandleDialog(_flag(args, "accept", False), None if prompt is None else str(prompt)) + + +def build_drop(args: Mapping[str, Any], resolver: RefResolver) -> Action: + target = _element(args, resolver) + raw_data = args.get("data") + data: tuple[tuple[str, str], ...] = () + if raw_data is not None: + if not isinstance(raw_data, list): + raise ValueError("'data' must be a list of objects") + if not all(isinstance(item, Mapping) for item in raw_data): + raise ValueError("'data' items must be objects") + data = tuple( + (str(item.get("mimeType") or item.get("mime_type") or ""), str(item.get("data") or "")) for item in raw_data + ) + return Drop(target, paths=_paths(args) or (), data=data) + + +def build_fill_form(args: Mapping[str, Any], resolver: RefResolver) -> Action: + generation = _generation(args) + raw_fields = args.get("fields") + if not isinstance(raw_fields, list) or not raw_fields: + raise ValueError("'fields' must be a non-empty list") + fields = [] + for index, raw in enumerate(raw_fields): + if not isinstance(raw, Mapping): + raise ValueError(f"fields[{index}] must be an object") + name = _text(raw, "type").lower() or "textbox" + kind = _FIELD_KINDS.get(name) + if kind is None: + raise ValueError(f"fields[{index}] unsupported type={name!r}") + value = raw.get("value") + if kind in (FieldKind.CHECKBOX, FieldKind.RADIO): + checked = value if isinstance(value, bool) else str(value).strip().lower() in _CHECKED + text = "true" if checked else "false" + elif kind is FieldKind.COMBOBOX: + text = "" if value in (None, "") else str(value) + text = text or _text(raw, "label") + else: + text = "" if value is None else str(value) + target = _target(raw, resolver, generation) + fields.append(FormField(target, kind, text, sensitive=_flag(raw, "sensitive", False))) + return FillForm(tuple(fields)) + + +def build_snapshot(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Observe(include_screenshot=_flag(args, "include_screenshot", False)) + + +BROWSER_TOOL_SPECS: tuple[ToolSpec, ...] = ( + ToolSpec("browser_navigate", (Navigate,), "Navigate to a URL.", schema=_NAVIGATE_PARAMS, build=build_navigate), + ToolSpec( + "browser_navigate_back", + (Back,), + "Navigate back in browser history.", + schema=_NO_PARAMS, + build=build_navigate_back, + ), + ToolSpec("browser_click", (Click,), "Click an element.", schema=_CLICK_PARAMS, build=build_click), + ToolSpec("browser_type", (TypeText,), "Type text into an element.", schema=_TYPE_PARAMS, build=build_type), + ToolSpec( + "browser_press_key", + (PressKey,), + "Press a keyboard key or key combination.", + schema=_PRESS_KEY_PARAMS, + build=build_press_key, + ), + ToolSpec( + "browser_take_screenshot", + (Observe,), + "Take a screenshot of the current page.", + schema=_SCREENSHOT_PARAMS, + build=build_take_screenshot, + ), + ToolSpec( + "browser_tabs", + (Observe, OpenContext, SwitchContext, CloseContext), + "List, select, or close browser tabs.", + schema=_TABS_PARAMS, + build=build_tabs, + ), + ToolSpec("browser_close", (CloseContext,), "Close the active browser tab.", schema=_NO_PARAMS, build=build_close), + ToolSpec( + "browser_select_option", + (SelectOption,), + "Select an option from a native select element.", + schema=_SELECT_OPTION_PARAMS, + build=build_select_option, + ), + ToolSpec( + "browser_evaluate", + (RunScript,), + "Evaluate a small JavaScript expression in the page. Do not dump the full document.", + schema=_EVALUATE_PARAMS, + build=build_evaluate, + ), + ToolSpec( + "browser_drag", + (Drag,), + "Drag from a source element to a target element.", + schema=_DRAG_PARAMS, + build=build_drag, + ), + ToolSpec( + "browser_file_upload", + (UploadFile,), + "Upload one or more files to a file input element.", + schema=_FILE_UPLOAD_PARAMS, + build=build_file_upload, + ), + ToolSpec( + "browser_hover", + (Hover,), + "Hover the mouse over an element (tooltips / :hover menus).", + schema=_HOVER_PARAMS, + build=build_hover, + ), + ToolSpec( + "browser_handle_dialog", + (HandleDialog,), + "Accept or dismiss a JavaScript alert/confirm/prompt. " + "If no dialog is open yet, arms handling for the next dialog — " + "call this before the click/type that opens it.", + schema=_HANDLE_DIALOG_PARAMS, + build=build_handle_dialog, + ), + ToolSpec( + "browser_drop", + (Drop,), + "Drop local files and/or MIME-typed data onto an element as if dragged from " + "outside the page. Distinct from browser_drag (element→element). " + "At least one of paths or data is required. Supported MIME types: " + "text/plain, text/uri-list, text/html.", + schema=_DROP_PARAMS, + build=build_drop, + ), + ToolSpec( + "browser_fill_form", + (FillForm,), + "Fill multiple form fields in one call (type / select / checkbox).", + schema=_FILL_FORM_PARAMS, + build=build_fill_form, + ), + ToolSpec( + "browser_snapshot", + (Observe,), + "Capture an accessibility/page observation of the current page.", + schema=_SNAPSHOT_PARAMS, + build=build_snapshot, + ), + ToolSpec("browser_mouse_click_xy", (Click,)), + ToolSpec("browser_mouse_move_xy", (Hover,)), + ToolSpec("browser_mouse_drag_xy", (Drag,)), + ToolSpec("browser_mouse_wheel", (Scroll,)), + ToolSpec("browser_pdf_save", (GetPdf,)), + ToolSpec("browser_console_messages", (GetConsole,)), + ToolSpec("browser_network_requests", (GetNetwork,)), +) + +_BROWSER_IMAGE_CAPABILITY_SECTION_NAME = "browser_image_input_capability" +_BROWSER_DATE_PICKER_SECTION_NAME = "browser_date_picker_guidance" +_BROWSER_DATE_PICKER_GUIDANCE = { + "en": ( + "For calendar/date-picker widgets, do not type the date into a text field. Click the " + 'specific day cell instead (look for a data-iso="YYYY-MM-DD" attribute or an aria-label ' + "naming the full date), then confirm the selection before proceeding." + ), + "cn": ( + "对于日历/日期选择器组件，不要向文本框输入日期，而应点击对应的日期单元格" + '（查找 data-iso="YYYY-MM-DD" 属性或包含完整日期的 aria-label），' + "确认选中后再继续。" + ), +} +_BROWSER_TASK_RULES_SECTION_NAME = "browser_task_rules" +_BROWSER_TASK_RULES = { + "en": ( + "Task rules:\n" + "- Today's date is {today}. A date without a year means its next occurrence on or after today.\n" + "- Apply every filter, category and option the task names (a property type, a class year, a price range) " + "before reading results.\n" + "- Check on the page what is today's, the latest or the newest; never answer that from memory.\n" + "- When the task names a source, open that source and read the answer there.\n" + "- Perform every UI step the task names, even when the answer is reachable another way.\n" + "- When the question has two readings, answer both.\n" + "- When the site misreads an entity, query it again disambiguated (full name, kind or unit).\n" + "- When the page shows the answer, state it plainly, without caveats that undermine it.\n" + "- Do not retry a hand-built search URL more than twice; open a candidate result instead." + ), + "cn": ( + "任务规则：\n" + "- 今天的日期是 {today}。未写年份的日期指今天或之后最近的那一天。\n" + "- 读取结果前，应用任务点名的每一个筛选、类别和选项（如住宿类型、年级、价格区间）。\n" + "- “今天”“最新”“最近”的内容要在页面上核实，不要凭记忆回答。\n" + "- 任务点名了来源时，打开该来源并在那里读取答案。\n" + "- 任务点名的每个界面操作都要执行，即使答案可以通过别的途径得到。\n" + "- 问题有两种理解时，两种都回答。\n" + "- 网站误解了某个实体时，用消歧后的写法重新查询（全名、类别或单位）。\n" + "- 页面上已有答案时直接给出，不要附加削弱答案的保留说明。\n" + "- 手工拼接的搜索 URL 最多重试两次，之后改为打开一个候选结果。" + ), +} + + +def browser_task_rules() -> dict[str, str]: + """The task rules per language, with today's date.""" + today = datetime.date.today().isoformat() + return {language: text.format(today=today) for language, text in _BROWSER_TASK_RULES.items()} + + +_BROWSER_IMAGE_CAPABILITY_GUIDANCE = { + True: { + "en": ( + "The current model can inspect image input. Use browser_take_screenshot only when " + "pixel-level visual evidence is required and DOM probes, accessibility snapshots, " + "or targeted evaluation cannot answer the task." + ), + "cn": ( + "当前模型可以理解图片输入。仅当任务需要像素级视觉证据，且 DOM 探测、无障碍快照或" + "定向脚本无法解决时，才使用 browser_take_screenshot。" + ), + }, + False: { + "en": ( + "Image input is unavailable or unverified for this run. Do not request screenshots, " + "including browser_take_screenshot or browser_batch_interact with op=screenshot. " + "Use DOM probes, accessibility snapshots, and targeted evaluation instead." + ), + "cn": ( + "本次运行的图片输入能力不可用或尚未确认。不要请求截图，包括 " + "browser_take_screenshot 或 browser_batch_interact 的 op=screenshot；" + "请改用 DOM 探测、无障碍快照和定向脚本。" + ), + }, +} + + +_BROWSER_WORKER_HELPER_TOOL_NAMES = ( + "browser_probe_interactives", + "browser_probe_cards", + "browser_batch_interact", + "browser_custom_action", + "browser_list_custom_actions", +) + + +def build_browser_worker_system_prompt( + screenshot_subdir: str = "screenshots", + artifacts_subdir: str = "artifacts", + *, + tool_names: Optional[Iterable[str]] = None, + driver_name: str, +) -> str: + """Render the worker system prompt; helper guidance only for helpers in ``tool_names`` (``None``: all).""" + available = set(_BROWSER_WORKER_HELPER_TOOL_NAMES if tool_names is None else tool_names) + interactives = "browser_probe_interactives" in available + cards = "browser_probe_cards" in available + probes = interactives or cards + batch = "browser_batch_interact" in available + batch_vs_custom_action = batch and interactives and cards and "browser_custom_action" in available + helper_guidance = "".join( + text + for enabled, text in ( + ( + probes, + "Before broad page snapshots, full-body scans, or generic DOM scraping, " + "choose the smallest compact probe that matches the task. ", + ), + ( + interactives, + "Use browser_probe_interactives for buttons, links, inputs, forms, navigation controls, " + "login controls, pagination controls, menus, and other visible interactive elements. " + "When using browser_probe_interactives only for page-level controls, prefer max_items around 20-30 " + "unless the task explicitly requires a larger inventory. ", + ), + ( + cards, + "On product pages, marketplace pages, search-result pages, catalog pages, article-list pages, " + "or any page with repeated visible cards/listings, call browser_probe_cards before broad extraction. " + "Use browser_probe_cards to identify compact repeated structures such as product cards, result cards, " + "book cards, article cards, listing rows, title/author/source/summary, " + "title/price/rating/review/availability fields, primary links, " + "visible controls, generation-scoped target IDs, and recurring structure signatures. ", + ), + ( + cards and interactives, + "For product/listing/item-data tasks, prefer browser_probe_cards first; call " + "browser_probe_interactives " + "only if you also need page-level navigation, filters, forms, or controls outside the cards. ", + ), + ( + cards, + "If browser_probe_cards returns the fields needed for the task, including article/result-row " + "title, link, author/source, or summary fields, use that compact result as the evidence " + "for extraction and final reporting instead of taking screenshots, snapshots, or running extra DOM " + "scans. " + "After a successful card probe, use browser_evaluate, or a browser_run_code tool only when it is " + "explicitly present, for a clearly missing required field or a specific target-based action; " + "do not repeat broad evaluation " + "just to re-read the same cards. ", + ), + ( + probes, + "Treat page_id/generation_id/url/title/interactives/cards/" + "field_coverage/blockers as one PageState contract. ", + ), + ( + probes and batch, + "Pass the current generation_id with a " + "probe target_id directly to browser_batch_interact; never translate it into guessed CSS. " + "Explicit selectors are reserved for validated read-only extraction and condition waits. ", + ), + ( + cards, + "If a card has primary_link or href, navigate to it directly. ", + ), + ( + batch_vs_custom_action, + "browser_batch_interact is a standalone runtime helper like browser_probe_interactives and " + "browser_probe_cards, not a browser_custom_action wrapper. Use it directly. ", + ), + ( + batch, + "For multi-field forms with several known controls, search flows with autocomplete, " + "dropdown/date-picker " + "flows, filter panels, or any short sequence where two or more next click/type/wait/extract steps are " + "already known, call browser_batch_interact before falling back to repeated " + "browser_click/browser_type and separate wait turns. " + "Extract multiple fields from the same page in one batch with named extract_text/extract_value " + "steps, or use one card probe result that already contains every requested field. " + "Do not force browser_batch_interact for a single uncertain click, one simple text field, or a page " + "state " + "that still needs inspection. Prefer generation-scoped target_id in batch steps. " + "Use autocomplete steps for type-then-choose widgets, and condition-based " + "wait_for_selector/wait_for_text " + "steps instead of fixed sleeps. For dynamic pages use wait_for_url, " + "wait_for_first_card_title, wait_for_sort_state, wait_for_result_count, " + "wait_for_dom_text_change, wait_for_stable, or wait_for_tab with a short poll interval and total " + "timeout. " + "When an action opens a new tab, use click plus wait_for_tab in the same batch; wait_for_tab " + "activates the matched tab by default. Do not wait for a new URL on the old tab. ", + ), + ( + True, + "Interpret 'fully loaded' or an 'appropriate pause' as condition stability, not a fixed sleep. ", + ), + ( + batch, + "Do not split click+type+wait, click+wait+click, " + "date open+choose, or search submit+result wait into separate ReAct turns unless " + "browser_batch_interact " + "failed or the next target is genuinely unknown. ", + ), + ) + if enabled + ) + return ( + "You are a browser worker agent.\n" + f"Execute browser tasks step-by-step with {driver_name} tools and approved runtime helper tools only.\n" + "Before interacting, ensure page or selector readiness.\n" + "Keep actions targeted and avoid unnecessary page snapshots.\n" + f"{helper_guidance}" + "Keep using browser_fill_form for ordinary visible text fields when that official tool is enough. " + "Use browser_snapshot only when the compact probes are insufficient, when accessibility structure is needed, " + f"or when you need exact element references required by a {driver_name} action. " + "Use browser_evaluate for small exact computations. Use a browser_run_code tool only when it is present " + "in the current tool list and compact probes, evaluate, snapshot, and deterministic actions are " + "insufficient. Never use run-code to dump the entire document body.\n" + "If actions repeatedly fail, stop and report the exact failing action.\n" + "If you use browser_tabs, action MUST be one of: list, new, close, select.\n" + "Use specialized operations only when their explicit primitive tool is present in the current tool list. " + "Do not launch nested browser tasks from the browser worker. If the available tools cannot complete an " + "operation, report the exact blocker instead of inventing a tool name.\n" + "IMPORTANT: Do NOT use browser_take_screenshot unless strictly necessary. " + f"If a screenshot is needed, always save it under '{screenshot_subdir}/'. " + "Use browser_take_screenshot when it is present; do not create screenshots through run-code.\n" + f"If you produce any output files (reports, notes, summaries, markdown, text files, etc.), " + f"write them to the '{artifacts_subdir}/' directory relative to the working directory. " + "Never write output files to the project root or any other location.\n" + f"{browser_task_rules()['en']}\n" + "Final output MUST be a single JSON object with keys:\n" + "ok (boolean), final (string), page (object with url and title), " + "screenshot (string|null), error (string|null).\n" + "Also include status (completed|partial|blocked|failed) whenever possible. " + "If the task is not fully complete, include progress as an object with " + "completed_steps, remaining_steps, next_step, completion_evidence, and missing_requirements.\n" + "Set ok=true only when the exact user-visible goal is fully satisfied and you can cite concrete evidence " + "from the page, compact probe results, browser snapshots, screenshots, or generated artifacts. " + "If the task is incomplete or blocked, set ok=false and fill the progress fields so a continuation can " + "resume with minimal repetition.\n" + "Return JSON only, even on failures. " + "Do not output markdown, code fences, or plain text outside the JSON object." + ) diff --git a/openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/__init__.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/__init__.py new file mode 100644 index 0000000000..34a1953a54 --- /dev/null +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/__init__.py @@ -0,0 +1,4 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +"""The browser working context: its store and the context processor that renders it.""" diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context_processor.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/processor.py similarity index 95% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context_processor.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/processor.py index 6177479462..f44cfa59c5 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context_processor.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/processor.py @@ -11,13 +11,12 @@ from openjiuwen.core.context_engine.processor.base import ContextEvent, ContextProcessor from openjiuwen.core.foundation.llm import BaseMessage, ToolMessage, UserMessage -from .browser_working_context import ( +from .store import ( BROWSER_TOOL_MEMORY_METADATA_KEY, BrowserWorkingContextConfig, BrowserWorkingContextStore, ) - _BROWSER_WORKING_CONTEXT_MESSAGE_NAME = "browser_working_context" _BROWSER_WORKING_CONTEXT_METADATA_KEY = "browser_working_context" _BROWSER_WORKING_CONTEXT_MESSAGE_ID = "openjiuwen:browser-working-context" @@ -84,10 +83,7 @@ def _is_browser_state_projection(message: BaseMessage) -> bool: if message.name in {"current_browser_state", "browser_state_progress"}: return True metadata = message.metadata if isinstance(message.metadata, dict) else {} - return bool( - metadata.get("browser_state_context") - or metadata.get("browser_state_progress_context") - ) + return bool(metadata.get("browser_state_context") or metadata.get("browser_state_progress_context")) @staticmethod def _is_working_context_message(message: BaseMessage) -> bool: diff --git a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context.py b/openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/store.py similarity index 94% rename from openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context.py rename to openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/store.py index 850a675087..0291aa27ef 100644 --- a/openjiuwen/harness/tools/browser_move/playwright_runtime/browser_working_context.py +++ b/openjiuwen/harness/tools/browser_move/surfaces/browser/working_context/store.py @@ -11,8 +11,10 @@ from pydantic import BaseModel, ConfigDict, Field, ValidationError from openjiuwen.core.foundation.llm import BaseMessage, ToolMessage, UserMessage - -from .browser_logging import browser_agent_log_info, browser_agent_log_warning +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.logging import ( + browser_agent_log_info, + browser_agent_log_warning, +) BROWSER_WORKING_CONTEXT_STATE_KEY = "__browser_subagent_working_context__" BROWSER_TASK_STATE_KEY = "__browser_phase_budget_state__" @@ -246,9 +248,6 @@ def stage_model_step( sanitized_memory = self.sanitize_memory(memory) if memory is not None else None if sanitized_memory is None and state.current.task_list: - # A missing/malformed update must not erase the last valid state. - # The explicit model_update_error remains in the step record so - # the next model call can repair the omission. sanitized_memory = self.sanitize_memory(state.current) sanitized_error = _bounded_text( model_update_error, @@ -394,7 +393,7 @@ def tool_memory_from_message( memory.tool_call_id = str(tool_call_id or "") if not memory.error: - inferred_error = self._infer_tool_error(message.content) + inferred_error = self._infer_tool_error(message) if inferred_error: memory.error = inferred_error if memory.has_prompt_content() and not isinstance(payload, dict): @@ -428,12 +427,12 @@ def render_and_consume_one_step(self, session: Any) -> str: }, "task": self._project_task_state(task_state), "runtime_directive": self._runtime_directive(task_state), - "recent_actions": list(task_state.get("recent_actions") or [])[-self.config.max_recent_steps:], + "recent_actions": list(task_state.get("recent_actions") or [])[-self.config.max_recent_steps :], } if not self.config.runtime_projection_only: model_notes = { - "key_facts": state.current.key_facts[-self.config.max_list_items:], - "important_information": state.current.important_information[-self.config.max_list_items:], + "key_facts": state.current.key_facts[-self.config.max_list_items :], + "important_information": state.current.important_information[-self.config.max_list_items :], } if any(model_notes.values()): rendered_state["model_notes"] = model_notes @@ -449,7 +448,7 @@ def render_and_consume_one_step(self, session: Any) -> str: def _compact_retained_tool_evidence(self, steps: Iterable[BrowserStepRecord]) -> list[Dict[str, Any]]: evidence: list[Dict[str, Any]] = [] - for step in list(steps)[-self.config.max_recent_steps:]: + for step in list(steps)[-self.config.max_recent_steps :]: for memory in step.tool_memories: if not (memory.durable_content or memory.error): continue @@ -464,7 +463,7 @@ def _compact_retained_tool_evidence(self, steps: Iterable[BrowserStepRecord]) -> "source": memory.content_source or ("error" if memory.error else "retained"), } ) - return evidence[-self.config.max_recent_steps:] + return evidence[-self.config.max_recent_steps :] def _serialize_prompt_payload(self, value: Dict[str, Any]) -> str: """Serialize a bounded projection without ever truncating JSON text.""" @@ -480,8 +479,7 @@ def _serialize_prompt_payload(self, value: Dict[str, Any]) -> str: notes = payload.get("model_notes") if isinstance(notes, dict): payload["model_notes"] = { - key: [_bounded_text(item, 240) for item in list(items or [])[-4:]] - for key, items in notes.items() + key: [_bounded_text(item, 240) for item in list(items or [])[-4:]] for key, items in notes.items() } payload["recent_actions"] = list(payload.get("recent_actions") or [])[-3:] body = json.dumps(payload, ensure_ascii=False, separators=(",", ":")) @@ -520,9 +518,7 @@ def _serialize_prompt_payload(self, value: Dict[str, Any]) -> str: fallback["task"]["requirements"]["evidence"] = [] fallback["task"]["requirements"]["missing"] = list(requirements.get("missing") or [])[:6] fallback["task"]["requirements"]["unavailable"] = list(requirements.get("unavailable") or [])[:4] - fallback["task"]["blockers"] = [ - _bounded_text(item, 160) for item in list(task.get("blockers") or [])[:4] - ] + fallback["task"]["blockers"] = [_bounded_text(item, 160) for item in list(task.get("blockers") or [])[:4]] fallback["task"].pop("last_page", None) fallback["recent_actions"] = [] body = json.dumps(fallback, ensure_ascii=False, separators=(",", ":")) @@ -556,11 +552,7 @@ def load_task_state(session: Any) -> Dict[str, Any]: @classmethod def sync_semantic_progress(cls, session: Any, progress: Any) -> bool: - """Merge one browser observation into authoritative task state. - - Returns ``True`` only when a permitted replan trial produced observable - semantic progress and the runtime gate may be cleared. - """ + """Merge one observation into the task state; ``True`` when a replan trial made progress.""" if session is None or not isinstance(progress, dict) or not progress: return False @@ -638,8 +630,7 @@ def _merge_semantic_evidence(state: Dict[str, Any], progress: Dict[str, Any]) -> continue filter_text = f"{item.get('key', '')} {item.get('value', '')}".lower() if any( - token in filter_text - for token in ("sort", "order", "sales", "销量", "排序", "价格", "最新", "综合") + token in filter_text for token in ("sort", "order", "sales", "销量", "排序", "价格", "最新", "综合") ): sort_candidates.append(item) if sort_candidates: @@ -819,10 +810,11 @@ def _sort_from_changed_results( target = BrowserWorkingContextStore._successful_click_target(state, progress) if target is None: return None - target_text = " ".join( - str(target.get(key) or "") - for key in ("element", "name", "label", "text", "value") - ).strip().lower() + target_text = ( + " ".join(str(target.get(key) or "") for key in ("element", "name", "label", "text", "value")) + .strip() + .lower() + ) for token, label in ( ("销量", "销量"), ("sales", "销量"), @@ -839,12 +831,7 @@ def _sort_from_changed_results( ("rating", "评分"), ): if token in target_text: - selector = str( - target.get("target_id") - or target.get("target") - or target.get("ref") - or "" - ) + selector = str(target.get("target_id") or target.get("target") or target.get("ref") or "") return label, selector[:600] return None @@ -887,11 +874,7 @@ def _reconcile_observed_action(state: Dict[str, Any], progress: Dict[str, Any]) semantic_state = progress.get("semantic_state") semantic_state = semantic_state if isinstance(semantic_state, dict) else {} - changed_fields = { - str(field) - for field in progress.get("changed_fields") or [] - if str(field).strip() - } + changed_fields = {str(field) for field in progress.get("changed_fields") or [] if str(field).strip()} phase = str(action.get("phase") or "") phase_progress = BrowserWorkingContextStore._phase_has_semantic_progress( phase, @@ -906,7 +889,7 @@ def _reconcile_observed_action(state: Dict[str, Any], progress: Dict[str, Any]) details["completion_evidence"] = "post-action semantic observation" phase_names = list(phases) try: - remaining = phase_names[phase_names.index(phase) + 1:] + remaining = phase_names[phase_names.index(phase) + 1 :] except ValueError: remaining = [] next_phase = phase @@ -1038,18 +1021,11 @@ def _project_task_state(state: Dict[str, Any]) -> Dict[str, Any]: for field_name in (state.get("field_coverage") or [])[:16] if str(field_name).strip() ] - covered_keys = { - BrowserWorkingContextStore._evidence_slot_key(slot) - for slot in evidence_slots - } + covered_keys = {BrowserWorkingContextStore._evidence_slot_key(slot) for slot in evidence_slots} missing_slots = [ - slot - for slot in required_slots - if BrowserWorkingContextStore._evidence_slot_key(slot) not in covered_keys - ] - unavailable_slots = [ - slot for slot in evidence_slots if slot.get("status") in {"missing", "unknown"} + slot for slot in required_slots if BrowserWorkingContextStore._evidence_slot_key(slot) not in covered_keys ] + unavailable_slots = [slot for slot in evidence_slots if slot.get("status") in {"missing", "unknown"}] return { "task_id": state.get("task_id"), "goal": _bounded_text(state.get("goal") or state.get("task"), 1_000), @@ -1067,8 +1043,11 @@ def _project_task_state(state: Dict[str, Any]) -> Dict[str, Any]: "evidence": evidence_slots, }, "observations": [ - {"source": item.get("source"), "generation_id": item.get("generation_id"), - "raw_text": _bounded_text(item.get("raw_text"), 1_500)} + { + "source": item.get("source"), + "generation_id": item.get("generation_id"), + "raw_text": _bounded_text(item.get("raw_text"), 1_500), + } for item in state.get("structured_evidence") or [] if isinstance(item, dict) and item.get("kind") == "page_observation" ][-2:], @@ -1205,33 +1184,22 @@ def _commit_pending_as_incomplete( def _limit_history(self, state: BrowserWorkingContextState) -> None: if len(state.recent_steps) > self.config.max_recent_steps: - state.recent_steps = state.recent_steps[-self.config.max_recent_steps:] + state.recent_steps = state.recent_steps[-self.config.max_recent_steps :] - def _infer_tool_error(self, content: Any) -> Optional[str]: - text = self.message_content_to_text(content) + def _infer_tool_error(self, message: ToolMessage) -> Optional[str]: + text = self.message_content_to_text(message.content) if not text: return None - lowered = text.lower().strip() - if lowered.startswith(_ERROR_PREFIXES): + if text.lower().strip().startswith(_ERROR_PREFIXES): return _bounded_text(text, self.config.max_item_chars) - + if message.metadata.get("success") is not False: + return None try: parsed = json.loads(text) except (TypeError, ValueError): parsed = None - if isinstance(parsed, dict): - error = parsed.get("error") - failure_markers = ( - parsed.get("success") is False, - parsed.get("ok") is False, - parsed.get("isError") is True, - ) - if error and any(failure_markers): - return _bounded_text(error, self.config.max_item_chars) - - if "success=false" in lowered and "error=" in lowered: - return _bounded_text(text, self.config.max_item_chars) - return None + error = parsed.get("error") if isinstance(parsed, dict) else None + return _bounded_text(error or text, self.config.max_item_chars) @staticmethod def message_content_to_text(content: Any) -> str: diff --git a/openjiuwen/harness/tools/browser_move/utils/env.py b/openjiuwen/harness/tools/browser_move/utils/env.py deleted file mode 100644 index 2fdf74c96c..0000000000 --- a/openjiuwen/harness/tools/browser_move/utils/env.py +++ /dev/null @@ -1,248 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Environment and settings helpers for the runtime.""" - -from __future__ import annotations - -import json -import os -import shlex -from pathlib import Path - -try: - from ..playwright_runtime import REPO_ROOT -except ImportError: # pragma: no cover - from playwright_runtime import REPO_ROOT - -SUPPORTED_MODEL_PROVIDERS = frozenset({"openai", "openrouter", "siliconflow", "dashscope"}) -TRUTHY_ENV_VALUES = frozenset({"1", "true", "yes", "on"}) -FALSY_ENV_VALUES = frozenset({"0", "false", "no", "off"}) -DEFAULT_MODEL_NAME = "anthropic/claude-sonnet-4.5" -DEFAULT_BROWSER_TIMEOUT_S = 600 -DEFAULT_GUARDRAIL_MAX_STEPS = 100 -DEFAULT_GUARDRAIL_MAX_FAILURES = 4 -DEFAULT_GUARDRAIL_RETRY_ONCE = True -DEFAULT_PLAYWRIGHT_MCP_COMMAND = "npx" -DEFAULT_PLAYWRIGHT_MCP_ARGS = "-y @playwright/mcp@0.0.78" -DEFAULT_BROWSER_UPLOAD_ROOT = "" -MISSING_API_KEY_MESSAGE = ( - "Missing API key. Set API_KEY (or OPENROUTER_API_KEY / SILICONFLOW_API_KEY / " - "OPENAI_API_KEY / DASHSCOPE_API_KEY)." -) - - -def resolve_repo_dotenv_path() -> Path: - return Path(REPO_ROOT).expanduser().resolve() / ".env" - - -def load_repo_dotenv(*, override: bool = False) -> bool: - env_path = resolve_repo_dotenv_path() - if not env_path.exists(): - return False - try: - from dotenv import load_dotenv - except ImportError: - return False - return bool(load_dotenv(dotenv_path=env_path, override=override)) - - -def parse_command_args(value: str) -> list[str]: - value = (value or "").strip() - if not value: - return [] - if value.startswith("["): - try: - parsed = json.loads(value) - if isinstance(parsed, list): - return [str(x) for x in parsed] - except json.JSONDecodeError: - pass - return shlex.split(value) - - -def first_non_empty_env(*keys: str) -> str: - for key in keys: - value = (os.getenv(key) or "").strip() - if value: - return value - return "" - - -def normalize_provider(provider: str) -> str: - raw = (provider or "").strip() - lowered = raw.lower() - if lowered in SUPPORTED_MODEL_PROVIDERS: - return lowered - if lowered in {"alibaba", "aliyun"}: - return "dashscope" - if lowered in {"silicon-flow", "silicon_flow"}: - return "siliconflow" - return raw - - -def is_truthy_env(value: str) -> bool: - lowered = (value or "").strip().lower() - return lowered in TRUTHY_ENV_VALUES - - -def is_falsy_env(value: str) -> bool: - lowered = (value or "").strip().lower() - return lowered in FALSY_ENV_VALUES - - -def resolve_int_env(*keys: str, default: int, minimum: int | None = None) -> int: - for key in keys: - raw = (os.getenv(key) or "").strip() - if not raw: - continue - try: - value = int(raw) - except ValueError: - continue - if minimum is None or value >= minimum: - return value - return default - - -def resolve_bool_env(*keys: str, default: bool) -> bool: - for key in keys: - raw = (os.getenv(key) or "").strip() - if not raw: - continue - if is_truthy_env(raw): - return True - if is_falsy_env(raw): - return False - return default - - -def infer_provider_from_api_base(api_base: str) -> str: - base = (api_base or "").strip().lower() - if not base: - return "" - if "openrouter.ai" in base: - return "openrouter" - if "siliconflow.cn" in base or "siliconflow" in base: - return "siliconflow" - if "dashscope.aliyuncs.com" in base or "dashscope" in base: - return "dashscope" - return "openai" - - -def resolve_model_name() -> str: - return first_non_empty_env("MODEL_NAME") or DEFAULT_MODEL_NAME - - -def resolve_browser_timeout_s() -> int: - return resolve_int_env( - "BROWSER_TIMEOUT_S", - "PLAYWRIGHT_TOOL_TIMEOUT_S", - default=DEFAULT_BROWSER_TIMEOUT_S, - minimum=1, - ) - - -def resolve_upload_root() -> Path | None: - """Return the configured upload root directory, or None if not set. - - Reads BROWSER_UPLOAD_ROOT. Returns None when unset or blank so callers - can decide whether to enforce path sandboxing. - """ - raw = first_non_empty_env("BROWSER_UPLOAD_ROOT") - if not raw: - return None - return Path(raw).expanduser().resolve() - - -def resolve_model_settings() -> tuple[str, str, str]: - provider_mode = normalize_provider(first_non_empty_env("MODEL_PROVIDER", "MODEL_CLIENT_PROVIDER")) - if provider_mode and provider_mode not in SUPPORTED_MODEL_PROVIDERS: - raise ValueError( - f"Unsupported MODEL_PROVIDER '{provider_mode}'. " - "Supported: openai, openrouter, siliconflow, dashscope." - ) - - explicit_api_base = first_non_empty_env("API_BASE", "MODEL_API_BASE") - - if provider_mode: - provider = provider_mode - else: - base_hint = explicit_api_base or first_non_empty_env( - "OPENROUTER_BASE_URL", - "OPENROUTER_API_BASE", - "SILICONFLOW_BASE_URL", - "SILICONFLOW_API_BASE", - "DASHSCOPE_BASE_URL", - "DASHSCOPE_API_BASE", - "OPENAI_BASE_URL", - "OPENAI_API_BASE", - ) - provider = infer_provider_from_api_base(base_hint) - if not provider: - has_openrouter_key = bool(first_non_empty_env("OPENROUTER_API_KEY")) - has_siliconflow_key = bool(first_non_empty_env("SILICONFLOW_API_KEY")) - has_dashscope_key = bool(first_non_empty_env("DASHSCOPE_API_KEY")) - if has_openrouter_key: - provider = "openrouter" - elif has_siliconflow_key: - provider = "siliconflow" - elif has_dashscope_key: - provider = "dashscope" - else: - provider = "openai" - - if provider == "openrouter": - api_key = first_non_empty_env( - "API_KEY", - "MODEL_API_KEY", - "OPENROUTER_API_KEY", - "OPENAI_API_KEY", - ) - api_base = first_non_empty_env( - "API_BASE", - "MODEL_API_BASE", - "OPENROUTER_BASE_URL", - "OPENROUTER_API_BASE", - ) or "https://openrouter.ai/api/v1" - elif provider == "siliconflow": - api_key = first_non_empty_env( - "API_KEY", - "MODEL_API_KEY", - "SILICONFLOW_API_KEY", - "OPENAI_API_KEY", - "OPENROUTER_API_KEY", - ) - api_base = first_non_empty_env( - "API_BASE", - "MODEL_API_BASE", - "SILICONFLOW_BASE_URL", - "SILICONFLOW_API_BASE", - ) or "https://api.siliconflow.cn/v1" - elif provider == "dashscope": - api_key = first_non_empty_env( - "API_KEY", - "MODEL_API_KEY", - "DASHSCOPE_API_KEY", - "OPENAI_API_KEY", - "OPENROUTER_API_KEY", - ) - api_base = first_non_empty_env( - "API_BASE", - "MODEL_API_BASE", - "DASHSCOPE_BASE_URL", - "DASHSCOPE_API_BASE", - ) or "https://dashscope.aliyuncs.com/compatible-mode/v1" - else: - api_key = first_non_empty_env( - "API_KEY", - "MODEL_API_KEY", - "OPENAI_API_KEY", - "OPENROUTER_API_KEY", - ) - api_base = first_non_empty_env( - "API_BASE", - "MODEL_API_BASE", - "OPENAI_BASE_URL", - "OPENAI_API_BASE", - ) or "https://api.openai.com/v1" - return provider, api_key, api_base diff --git a/openjiuwen/harness/tools/browser_move/utils/parsing.py b/openjiuwen/harness/tools/browser_move/utils/parsing.py deleted file mode 100644 index 17740886d6..0000000000 --- a/openjiuwen/harness/tools/browser_move/utils/parsing.py +++ /dev/null @@ -1,109 +0,0 @@ -# coding: utf-8 -# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. -"""Shared parsing helpers for runtime/controller responses.""" - -from __future__ import annotations - -import json -from typing import Any - -# Top-level schema keys that OpenAI-compatible APIs (including Dashscope) reject. -_UNSUPPORTED_SCHEMA_KEYS = {"$schema", "$id", "$defs", "definitions", "$comment", "$anchor", "$vocabulary"} - - -def sanitize_json_schema(schema: Any) -> Any: - """Recursively strip schema keywords unsupported by OpenAI-compatible APIs. - - Converts ``anyOf: [{type: X}, {type: "null"}]`` nullable patterns to the - plain type so Dashscope doesn't reject the schema with error 181001. - """ - if not isinstance(schema, dict): - return schema - - # Collapse anyOf/oneOf nullable shorthands: [{type: X}, {type: "null"}] → type: X - for kw in ("anyOf", "oneOf"): - variants = schema.get(kw) - if isinstance(variants, list) and len(variants) == 2: - non_null = [v for v in variants if v != {"type": "null"} and v.get("type") != "null"] - null_count = len(variants) - len(non_null) - if null_count == 1 and len(non_null) == 1: - merged = {k: v for k, v in schema.items() if k != kw} - merged.update(non_null[0]) - schema = merged - - cleaned = {k: v for k, v in schema.items() if k not in _UNSUPPORTED_SCHEMA_KEYS} - - # Normalize null type to "object" — MCP servers sometimes emit {"type": null} - # for tools with no parameters, but OpenAI-compatible APIs require type to be a string. - if cleaned.get("type") is None and "type" in cleaned: - cleaned["type"] = "object" - - if "properties" in cleaned and isinstance(cleaned["properties"], dict): - cleaned["properties"] = {k: sanitize_json_schema(v) for k, v in cleaned["properties"].items()} - - for kw in ("items", "additionalProperties", "not"): - if kw in cleaned: - cleaned[kw] = sanitize_json_schema(cleaned[kw]) - - for kw in ("anyOf", "oneOf", "allOf"): - if kw in cleaned and isinstance(cleaned[kw], list): - cleaned[kw] = [sanitize_json_schema(v) for v in cleaned[kw]] - - return cleaned - - -def extract_json_object(text: Any) -> dict[str, Any]: - """Best-effort JSON extraction from model or tool text.""" - if isinstance(text, dict): - return text - if text is None: - return {} - - raw = str(text).strip() - if not raw: - return {} - - marker_result = "### Result" - marker_ran = "### Ran Playwright code" - if marker_result in raw and marker_ran in raw: - start = raw.find(marker_result) + len(marker_result) - end = raw.find(marker_ran, start) - if end > start: - raw = raw[start:end].strip() - - for _ in range(2): - try: - parsed = json.loads(raw) - except json.JSONDecodeError: - break - if isinstance(parsed, dict): - return parsed - if isinstance(parsed, str): - raw = parsed.strip() - continue - break - - if "```json" in raw: - start = raw.find("```json") + len("```json") - end = raw.find("```", start) - if end > start: - block = raw[start:end].strip() - try: - parsed = json.loads(block) - except json.JSONDecodeError: - parsed = None - if isinstance(parsed, dict): - return parsed - - first = raw.find("{") - last = raw.rfind("}") - if first >= 0 and last > first: - snippet = raw[first:last + 1] - try: - parsed = json.loads(snippet) - except json.JSONDecodeError: - parsed = None - if isinstance(parsed, dict): - return parsed - - return {} diff --git a/openjiuwen/harness/tools/subagent/task_tool.py b/openjiuwen/harness/tools/subagent/task_tool.py index 965ca401a3..72b72d0beb 100644 --- a/openjiuwen/harness/tools/subagent/task_tool.py +++ b/openjiuwen/harness/tools/subagent/task_tool.py @@ -40,14 +40,6 @@ ) from openjiuwen.harness.tools.base_tool import ToolOutput -try: - from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_logging import ( - browser_agent_log_info, - ) -except Exception: # pragma: no cover - browser runtime is optional here - browser_agent_log_info = None - - # Keep one delegation above the browser runtime's 540-second complex-task # slice so startup, state reconciliation, and final result assembly can finish. DEFAULT_SUBAGENT_TASK_TIMEOUT_S = 720.0 @@ -979,7 +971,9 @@ async def invoke(self, inputs: Input, **kwargs) -> ToolOutput: "[TaskTool] Invoking subagent with isolated session: %s, " "subagent_type=%s, query_summary=%s" ) - if normalized_type == "browser_agent" and browser_agent_log_info is not None: + if normalized_type == "browser_agent": + from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.logging import browser_agent_log_info + browser_agent_log_info(invoke_log, sub_session_id, normalized_type, query_summary) else: logger.info(invoke_log, sub_session_id, normalized_type, query_summary) diff --git a/pyproject.toml b/pyproject.toml index 6978478e2b..007c98027b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,6 +220,8 @@ openjiuwen = [ "rsi/**/*.md", "rsi/**/*.template", "rsi/**/*.yaml", + "harness/tools/browser_move/surfaces/browser/scene/pages/*", + "harness/tools/browser_move/surfaces/browser/probes/*.js", ] [tool.coverage.run] @@ -234,6 +236,7 @@ norecursedirs = [".*", "dist", "CVS", "_darcs", "{arch}", "*.egg", "venv", "node markers = [ "level0: smoke / happy-path coverage; PR gate must stay green", "level1: feature branches, error paths, and lifecycle edges", + "live: runs the conformance suite against a real driver and a live scene", ] # Formatting & Linting diff --git a/tests/system_tests/harness/tools/browser_move/test_browser_tools.py b/tests/system_tests/harness/tools/browser_move/test_browser_tools.py index 64f8289969..606016aa67 100644 --- a/tests/system_tests/harness/tools/browser_move/test_browser_tools.py +++ b/tests/system_tests/harness/tools/browser_move/test_browser_tools.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding: utf-8 # Copyright (c) Huawei Technologies Co., Ltd. 2025. All rights reserved. -"""System test for browser_move browser_tools registration flow.""" +"""System test: an agent uses browser_move served as an MCP server.""" from __future__ import annotations @@ -19,12 +19,8 @@ import pytest from openjiuwen.harness.tools.browser_move import REPO_ROOT -from openjiuwen.harness.tools.browser_move.playwright_runtime.config import ( - MISSING_API_KEY_MESSAGE, - build_runtime_settings, - load_repo_dotenv, -) - +from openjiuwen.harness.tools.browser_move.runtime.config import MISSING_API_KEY_MESSAGE, load_repo_dotenv +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings import build_runtime_settings SERVER_MODULE = "openjiuwen.harness.tools.browser_move.playwright_runtime_mcp_server" @@ -120,9 +116,8 @@ async def _run_browser_tools_check(query: str, session_id: str) -> Dict[str, obj from openjiuwen.core.runner import Runner from openjiuwen.core.single_agent.agents.react_agent import ReActAgent, ReActAgentConfig from openjiuwen.core.single_agent.schema.agent_card import AgentCard - from openjiuwen.harness.tools.browser_move.playwright_runtime.browser_tools import ( + from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.server_config import ( build_browser_runtime_mcp_config, - register_browser_runtime_mcp_server, ) settings = build_runtime_settings() @@ -168,7 +163,9 @@ async def _run_browser_tools_check(query: str, session_id: str) -> Dict[str, obj ) ) - registered = await register_browser_runtime_mcp_server(agent, tag="agent.main") + added = await Runner.resource_mgr.add_mcp_server(browser_tools_cfg, tag="agent.main") + registered = added is None or added.is_ok() + agent.ability_manager.add(browser_tools_cfg) result = await Runner.run_agent(agent, {"query": query}, session=session_id) result_type = result.get("result_type") if isinstance(result, dict) else None diff --git a/tests/system_tests/harness/tools/browser_move/test_playwright_runtime.py b/tests/system_tests/harness/tools/browser_move/test_runtime.py similarity index 86% rename from tests/system_tests/harness/tools/browser_move/test_playwright_runtime.py rename to tests/system_tests/harness/tools/browser_move/test_runtime.py index 594314f0f2..43c2e53557 100644 --- a/tests/system_tests/harness/tools/browser_move/test_playwright_runtime.py +++ b/tests/system_tests/harness/tools/browser_move/test_runtime.py @@ -11,12 +11,9 @@ import pytest -from openjiuwen.harness.tools.browser_move.playwright_runtime.config import ( - MISSING_API_KEY_MESSAGE, - build_runtime_settings, - load_repo_dotenv, -) -from openjiuwen.harness.tools.browser_move.playwright_runtime.runtime import BrowserAgentRuntime +from openjiuwen.harness.tools.browser_move.runtime.config import MISSING_API_KEY_MESSAGE, load_repo_dotenv +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade import BrowserAgentRuntime +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings import build_runtime_settings def _system_tests_enabled() -> bool: @@ -66,7 +63,7 @@ async def _run_live_check(query: str, session_id: str, cancel_after_s: float = 0 await runtime.shutdown() -def test_playwright_runtime_end_to_end() -> None: +def test_runtime_end_to_end() -> None: load_repo_dotenv(override=True) result = asyncio.run( _run_live_check( diff --git a/tests/unit_tests/agent/react_agent/test_react_agent_tool_call_logging.py b/tests/unit_tests/agent/react_agent/test_react_agent_tool_call_logging.py new file mode 100644 index 0000000000..1a2ee28fe5 --- /dev/null +++ b/tests/unit_tests/agent/react_agent/test_react_agent_tool_call_logging.py @@ -0,0 +1,62 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from openjiuwen.core.common.logging import tool_call +from openjiuwen.core.foundation.llm.schema.tool_call import ToolCall +from openjiuwen.core.single_agent.agents import react_agent +from openjiuwen.core.single_agent.agents.react_agent import ReActAgent +from openjiuwen.core.single_agent.schema.agent_card import AgentCard + + +class _Recorder: + def __init__(self): + self.lines = [] + + def info(self, message, *args): + self.lines.append(message % args) + + def __call__(self, message, *args): + self.info(message, *args) + + +@pytest.fixture +def core_log(monkeypatch): + monkeypatch.setattr(tool_call, "_TOOL_CALL_LOGGERS", {}) + recorder = _Recorder() + monkeypatch.setattr(react_agent, "logger", recorder) + return recorder + + +def _execute(*tool_names: str) -> None: + agent = ReActAgent(card=AgentCard(name="tool_log_agent", description="tool-call log routing")) + agent.ability_manager.execute = AsyncMock(return_value=[]) + calls = [ + ToolCall(id=f"c{index}", type="function", name=name, arguments="{}") for index, name in enumerate(tool_names) + ] + asyncio.run(agent._execute_tool_call(MagicMock(), calls, None, MagicMock())) + + +_BROWSER_LINE = "Executing tool: browser_click with args: %s" % ( + ReActAgent._summarize_tool_args_for_log("browser_click", "{}"), +) + + +def test_tool_calls_go_to_core_logger_without_registered_logger(core_log): + _execute("browser_click", "read_file") + + assert core_log.lines == [_BROWSER_LINE, "Executing tool: read_file with args: {}"] + + +def test_registered_logger_takes_matching_tool_calls(core_log): + routed = _Recorder() + tool_call.register_tool_call_logger(lambda name: name.startswith("browser_"), routed) + + _execute("browser_click", "read_file") + + assert routed.lines == [_BROWSER_LINE] + assert core_log.lines == ["Executing tool: read_file with args: {}"] diff --git a/tests/unit_tests/core/common/log/test_tool_call.py b/tests/unit_tests/core/common/log/test_tool_call.py new file mode 100644 index 0000000000..eb18427899 --- /dev/null +++ b/tests/unit_tests/core/common/log/test_tool_call.py @@ -0,0 +1,39 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +import pytest + +from openjiuwen.core.common.logging import tool_call + + +@pytest.fixture(autouse=True) +def _empty_registry(monkeypatch): + monkeypatch.setattr(tool_call, "_TOOL_CALL_LOGGERS", {}) + + +def _is_web_tool(name: str) -> bool: + return name.startswith("web_") + + +def test_no_tool_call_logger_without_registration(): + assert tool_call.tool_call_logger("web_search") is None + + +def test_registered_logger_serves_only_matching_tool_names(): + lines = [] + tool_call.register_tool_call_logger(_is_web_tool, lambda message, *args: lines.append(message % args)) + + tool_call.tool_call_logger("web_search")("Executing tool: %s", "web_search") + + assert lines == ["Executing tool: web_search"] + assert tool_call.tool_call_logger("read_file") is None + + +def test_registering_a_matcher_again_replaces_its_logger(): + first, second = [], [] + tool_call.register_tool_call_logger(_is_web_tool, lambda message, *args: first.append(message)) + tool_call.register_tool_call_logger(_is_web_tool, lambda message, *args: second.append(message)) + + tool_call.tool_call_logger("web_search")("line") + + assert (first, second) == ([], ["line"]) diff --git a/tests/unit_tests/core/context_engine/test_tool_result_budget_processor.py b/tests/unit_tests/core/context_engine/test_tool_result_budget_processor.py index 74680a7cd0..3afbf15294 100644 --- a/tests/unit_tests/core/context_engine/test_tool_result_budget_processor.py +++ b/tests/unit_tests/core/context_engine/test_tool_result_budget_processor.py @@ -7,12 +7,15 @@ import json import os +import re +from pathlib import Path from typing import List from unittest.mock import AsyncMock, MagicMock import pytest from openjiuwen.core.context_engine import ContextEngine, ContextEngineConfig +from openjiuwen.core.context_engine.context.context_utils import ContextUtils from openjiuwen.core.context_engine.processor.offloader.tool_result_budget_processor import ( ToolResultBudgetProcessor, ToolResultBudgetProcessorConfig, @@ -693,6 +696,48 @@ async def test_reload_from_filesystem_with_real_sys_operation(self, tmp_path): assert "ORIGINAL_TOOL_CONTENT_" in reloaded_messages[0].content assert "_END_MARKER" in reloaded_messages[0].content + @pytest.mark.asyncio + async def test_session_id_with_colons_offloads_to_a_path_safe_dir_and_reloads(self, tmp_path): + from openjiuwen.core.sys_operation import OperationMode, SysOperation, SysOperationCard + from openjiuwen.core.sys_operation.config import LocalWorkConfig + + workspace_root = tmp_path / "workspace" + card = SysOperationCard(id="test_real_sys_op", mode=OperationMode.LOCAL, work_config=LocalWorkConfig()) + engine = ContextEngine( + ContextEngineConfig(default_window_message_num=100), + workspace=MockWorkspace(root_path=str(workspace_root)), + sys_operation=SysOperation(card=card), + ) + session_id = "wv-task:direct:req-1:0123abcd" + context = await engine.create_context( + context_id="test_ctx", + session=create_mock_session(session_id), + history_messages=[], + processors=[ + ( + "ToolResultBudgetProcessor", + ToolResultBudgetProcessorConfig(tokens_threshold=50, large_message_threshold=50, trim_size=20), + ) + ], + ) + original_content = "ORIGINAL_" + "x" * 500 + await context.add_messages( + [ + UserMessage(content="Get detailed output"), + AssistantMessage(content="", tool_calls=create_tool_call_list(["tc-colon"])), + ToolMessage(content=original_content, tool_call_id="tc-colon", name="grep"), + AssistantMessage(content="Done."), + ] + ) + + tool_msg = context.get_messages()[2] + assert getattr(tool_msg, "offload_type", None) == "filesystem" + session_dirs = [path.name for path in (workspace_root / "context").iterdir()] + assert len(session_dirs) == 1 and ":" not in session_dirs[0] + assert session_dirs[0].startswith("wv-task-direct-req-1-0123abcd-") + reloaded = await context._offload_message_buffer.reload(tool_msg.offload_handle, "filesystem") + assert [message.content for message in reloaded] == [original_content] + @pytest.mark.asyncio async def test_reload_from_in_memory(self, tmp_path): """测试从内存 reload 已 offload 的消息 (workspace=None, sys_operation=None 场景)""" @@ -864,3 +909,70 @@ async def test_reload_filesystem_without_sys_operation_returns_empty(self, tmp_p ) assert reloaded_messages == [] + + +class TestOffloadSessionDirName: + @pytest.mark.parametrize("session_id", ["browser-session", "wv-Amazon--1__b0f215c.direct.r1.abc", "a_b.c-d"]) + def test_path_safe_ids_are_kept(self, session_id): + assert ContextUtils.offload_session_dir_name(session_id) == session_id + + @pytest.mark.parametrize("session_id", ["a:b", "a:b:c", "../x", "..", ".", "", "C:/x", "a b", "日本"]) + def test_other_ids_become_one_path_safe_component(self, session_id): + name = ContextUtils.offload_session_dir_name(session_id) + assert re.fullmatch(r"[A-Za-z0-9_.-]+", name) and name not in {".", ".."} and not name.startswith(".") + + def test_ids_that_read_alike_stay_distinct(self): + names = {ContextUtils.offload_session_dir_name(value) for value in ("a:b", "a/b", "a b", "a-b")} + assert len(names) == 4 + + +class TestOffloadMessageBufferLegacyFolder: + @staticmethod + def _buffer(workspace: Path, session_id: str): + from openjiuwen.core.context_engine.context.message_buffer import OffloadMessageBuffer + from openjiuwen.core.sys_operation import OperationMode, SysOperation, SysOperationCard + from openjiuwen.core.sys_operation.config import LocalWorkConfig + + buffer = OffloadMessageBuffer() + card = SysOperationCard(id="test_legacy_offload", mode=OperationMode.LOCAL, work_config=LocalWorkConfig()) + buffer.set_sys_operation(SysOperation(card=card)) + buffer.set_workspace_info(str(workspace), session_id) + return buffer + + @staticmethod + def _write(workspace: Path, session_dir: str, file_name: str, content: str) -> None: + folder = workspace / "context" / f"{session_dir}_context" / "offload" + folder.mkdir(parents=True, exist_ok=True) + payload = {"messages": [{"role": "tool", "content": content, "tool_call_id": "tc-1", "name": "grep"}]} + (folder / file_name).write_text(json.dumps(payload), encoding="utf-8") + + @pytest.mark.asyncio + async def test_a_session_stored_under_its_literal_id_is_still_found(self, tmp_path): + session_id = "wv task 1" + self._write(tmp_path, session_id, "ToolResultBudgetProcessor_h1.json", "old folder") + + reloaded = await self._buffer(tmp_path, session_id).reload("h1", "filesystem") + + assert [message.content for message in reloaded] == ["old folder"] + + @pytest.mark.asyncio + async def test_the_path_safe_folder_wins_over_the_literal_one(self, tmp_path): + session_id = "wv task 1" + self._write(tmp_path, session_id, "ToolResultBudgetProcessor_h1.json", "old folder") + self._write(tmp_path, ContextUtils.offload_session_dir_name(session_id), "h1.json", "new folder") + + reloaded = await self._buffer(tmp_path, session_id).reload("h1", "filesystem") + + assert [message.content for message in reloaded] == ["new folder"] + + @pytest.mark.parametrize("session_id", ["a/b", "a\\b", "../x", ".", ".."]) + def test_an_id_with_a_path_separator_never_names_a_folder(self, tmp_path, session_id): + paths = self._buffer(tmp_path, session_id)._filesystem_reload_paths("h1") + + safe = ContextUtils.offload_session_dir_name(session_id) + assert paths == [str(tmp_path / "context" / f"{safe}_context" / "offload" / "h1.json")] + + def test_a_path_safe_id_is_looked_up_once(self, tmp_path): + paths = self._buffer(tmp_path, "browser-session")._filesystem_reload_paths("h1") + + assert paths == [str(tmp_path / "context" / "browser-session_context" / "offload" / "h1.json")] diff --git a/tests/unit_tests/core/foundation/llm/test_anthropic_client_pooling.py b/tests/unit_tests/core/foundation/llm/test_anthropic_client_pooling.py index 02f382f4ad..c43e2a3c2e 100644 --- a/tests/unit_tests/core/foundation/llm/test_anthropic_client_pooling.py +++ b/tests/unit_tests/core/foundation/llm/test_anthropic_client_pooling.py @@ -2,6 +2,7 @@ # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. """Tests for AnthropicModelClient shared/cached AsyncAnthropic client lifecycle.""" +import asyncio from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -101,6 +102,52 @@ async def test_aclose_closes_cached_clients_and_clears_cache(self): assert AnthropicModelClient._client_cache == {} +class TestSharedClientAcrossEventLoops: + """Regression tests for a batch runner driving tasks through separate + ``asyncio.run()`` calls, each with its own (short-lived) event loop. + """ + + def test_a_client_cached_from_a_closed_loop_is_rebuilt_not_reused(self): + model = _make_model(use_shared=True) + client = model._client + + built = [] + + def fake_build(timeout=None): + mc = _mock_async_client() + built.append(mc) + return mc + + with patch.object(client, "_build_async_anthropic_client", side_effect=fake_build): + asyncio.run(model.invoke(messages=[UserMessage(content="hi")])) + # The loop from the first asyncio.run() call is now closed. Reusing + # the client built on it would raise "RuntimeError: Event loop is + # closed" on the first request of the second run. + asyncio.run(model.invoke(messages=[UserMessage(content="hi again")])) + + # A fresh client was built for the second (new) loop, not reused from + # the first (closed) one. + assert len(built) == 2 + assert built[0].messages.create.call_count == 1 + assert built[1].messages.create.call_count == 1 + + def test_stale_entries_from_closed_loops_are_evicted_on_the_next_cache_miss(self): + model = _make_model(use_shared=True) + client = model._client + + with patch.object( + client, "_build_async_anthropic_client", side_effect=lambda timeout=None: _mock_async_client() + ): + asyncio.run(model.invoke(messages=[UserMessage(content="hi")])) + assert len(AnthropicModelClient._client_cache) == 1 + + asyncio.run(model.invoke(messages=[UserMessage(content="hi again")])) + + # The first loop's entry was dropped (its loop is closed); only the + # second, live-loop entry remains -- the cache does not grow forever. + assert len(AnthropicModelClient._client_cache) == 1 + + class TestAcloseConnections: def _cfg(self, api_base: str) -> ModelClientConfig: return ModelClientConfig( @@ -137,6 +184,28 @@ async def test_aclose_connections_unknown_config_is_noop(self): assert client.close.call_count == 0 assert AnthropicModelClient.connection_key(cfg) in AnthropicModelClient._client_cache + @pytest.mark.asyncio + async def test_aclose_connections_closes_every_loop_entry_for_a_dropped_connection(self): + # A connection identity can have several cache entries, one per event + # loop it has ever been used from (see ``_client_cache_key``). + # aclose_connections must close all of them, not just one. + cfg_keep = self._cfg("https://api.anthropic.com") + cfg_drop = self._cfg("https://old.example.com") + keep_client = AsyncMock() + drop_client_loop_a, drop_client_loop_b = AsyncMock(), AsyncMock() + base_drop_key = AnthropicModelClient.connection_key(cfg_drop) + AnthropicModelClient._client_cache[AnthropicModelClient.connection_key(cfg_keep)] = keep_client + AnthropicModelClient._client_cache[(*base_drop_key, "loop-a")] = drop_client_loop_a + AnthropicModelClient._client_cache[(*base_drop_key, "loop-b")] = drop_client_loop_b + + await AnthropicModelClient.aclose_connections([cfg_drop]) + + assert drop_client_loop_a.close.call_count == 1 + assert drop_client_loop_b.close.call_count == 1 + assert keep_client.close.call_count == 0 + assert AnthropicModelClient.connection_key(cfg_keep) in AnthropicModelClient._client_cache + assert not any(key[: len(base_drop_key)] == base_drop_key for key in AnthropicModelClient._client_cache) + def test_connection_key_includes_auth_mode(self): api_key_cfg = self._cfg("https://api.anthropic.com") custom_headers_cfg = ModelClientConfig( diff --git a/tests/unit_tests/core/foundation/llm/test_openai_client_pooling.py b/tests/unit_tests/core/foundation/llm/test_openai_client_pooling.py index cd2dc774c0..f6192be7c0 100644 --- a/tests/unit_tests/core/foundation/llm/test_openai_client_pooling.py +++ b/tests/unit_tests/core/foundation/llm/test_openai_client_pooling.py @@ -2,6 +2,7 @@ # Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. """Tests for OpenAIModelClient shared/cached AsyncOpenAI client lifecycle.""" +import asyncio from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -108,6 +109,50 @@ async def test_aclose_closes_cached_clients_and_clears_cache(self): assert OpenAIModelClient._client_cache == {} +class TestSharedClientAcrossEventLoops: + """Regression tests for a batch runner driving tasks through separate + ``asyncio.run()`` calls, each with its own (short-lived) event loop. + """ + + def test_a_client_cached_from_a_closed_loop_is_rebuilt_not_reused(self): + model = _make_model(use_shared=True) + client = model._client + + built = [] + + def fake_build(timeout=None): + mc = _mock_async_client() + built.append(mc) + return mc + + with patch.object(client, "_build_async_openai_client", side_effect=fake_build): + asyncio.run(model.invoke(messages=[UserMessage(content="hi")])) + # The loop from the first asyncio.run() call is now closed. Reusing + # the client built on it would raise "RuntimeError: Event loop is + # closed" on the first request of the second run. + asyncio.run(model.invoke(messages=[UserMessage(content="hi again")])) + + # A fresh client was built for the second (new) loop, not reused from + # the first (closed) one. + assert len(built) == 2 + assert built[0].chat.completions.create.call_count == 1 + assert built[1].chat.completions.create.call_count == 1 + + def test_stale_entries_from_closed_loops_are_evicted_on_the_next_cache_miss(self): + model = _make_model(use_shared=True) + client = model._client + + with patch.object(client, "_build_async_openai_client", side_effect=lambda timeout=None: _mock_async_client()): + asyncio.run(model.invoke(messages=[UserMessage(content="hi")])) + assert len(OpenAIModelClient._client_cache) == 1 + + asyncio.run(model.invoke(messages=[UserMessage(content="hi again")])) + + # The first loop's entry was dropped (its loop is closed); only the + # second, live-loop entry remains -- the cache does not grow forever. + assert len(OpenAIModelClient._client_cache) == 1 + + class TestAcloseConnections: def _cfg(self, api_base: str) -> ModelClientConfig: return ModelClientConfig( @@ -133,6 +178,28 @@ async def test_aclose_connections_closes_only_given(self): assert OpenAIModelClient.connection_key(cfg_keep) in OpenAIModelClient._client_cache assert OpenAIModelClient.connection_key(cfg_drop) not in OpenAIModelClient._client_cache + @pytest.mark.asyncio + async def test_aclose_connections_closes_every_loop_entry_for_a_dropped_connection(self): + # A connection identity can have several cache entries, one per event + # loop it has ever been used from (see ``_client_cache_key``). + # aclose_connections must close all of them, not just one. + cfg_keep = self._cfg("https://api.openai.com/v1") + cfg_drop = self._cfg("https://old.example.com/v1") + keep_client = AsyncMock() + drop_client_loop_a, drop_client_loop_b = AsyncMock(), AsyncMock() + base_drop_key = OpenAIModelClient.connection_key(cfg_drop) + OpenAIModelClient._client_cache[OpenAIModelClient.connection_key(cfg_keep)] = keep_client + OpenAIModelClient._client_cache[(*base_drop_key, "loop-a")] = drop_client_loop_a + OpenAIModelClient._client_cache[(*base_drop_key, "loop-b")] = drop_client_loop_b + + await OpenAIModelClient.aclose_connections([cfg_drop]) + + assert drop_client_loop_a.close.call_count == 1 + assert drop_client_loop_b.close.call_count == 1 + assert keep_client.close.call_count == 0 + assert OpenAIModelClient.connection_key(cfg_keep) in OpenAIModelClient._client_cache + assert not any(key[: len(base_drop_key)] == base_drop_key for key in OpenAIModelClient._client_cache) + class TestFallbackClient: @pytest.mark.asyncio diff --git a/tests/unit_tests/harness/test_deep_agent_terminal_stream.py b/tests/unit_tests/harness/test_deep_agent_terminal_stream.py index 036f0c821b..ec6639e755 100644 --- a/tests/unit_tests/harness/test_deep_agent_terminal_stream.py +++ b/tests/unit_tests/harness/test_deep_agent_terminal_stream.py @@ -22,10 +22,10 @@ from openjiuwen.core.single_agent.schema.agent_card import AgentCard from openjiuwen.harness.deep_agent import DeepAgent from openjiuwen.harness.schema.config import DeepAgentConfig -from openjiuwen.harness.tools.browser_move.playwright_runtime.runtime import ( - BrowserAgentRuntime, - BrowserRuntimeRail, -) +from openjiuwen.harness.tools.browser_move.contract import Effect, RunScript +from openjiuwen.harness.tools.browser_move.runtime.session import ToolKind +from openjiuwen.harness.tools.browser_move.surfaces.browser.rails.runtime_rail import BrowserRuntimeRail +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade import BrowserAgentRuntime from openjiuwen.harness.tools.subagent.task_tool import TaskTool, _run_subagent_with_observable_stream from tests.unit_tests.harness.test_deep_agent import FakeReactAgent, _create_dummy_model @@ -230,7 +230,7 @@ async def test_real_browser_rail_stream_reaches_tasktool(monkeypatch, outcome): runtime.service.guardrails.timeout_s = 600 runtime.service.build_failure_summary.return_value = "browser task incomplete" rail = BrowserRuntimeRail(runtime) - monkeypatch.setattr(rail, "_ensure_browser_mcp_ability", AsyncMock()) + monkeypatch.setattr(rail, "_add_agent_parts", AsyncMock()) browser.add_rail(rail) seen_sessions = [] @@ -262,6 +262,7 @@ async def call_model(ctx, context, tools): "result": {"sel": "#weather", "text": "Singapore weather: high 33 C, low 26 C"}, "generation_id": "g1", }, + kind=ToolKind(action=RunScript("() => 1"), effect=RunScript.effect, progress=True, known=True), tool_name="mcp_playwright-official_browser_evaluate", tool_args={"function": "() => document.querySelector('#weather').textContent"}, ) @@ -279,9 +280,10 @@ async def call_model(ctx, context, tools): parent = SimpleNamespace(create_subagent=lambda *_args, **_kwargs: browser) tool = TaskTool(card=ToolCard(id="terminal-task", name="task_tool"), parent_agent=parent) result = await tool.invoke( - {"subagent_type": "browser_agent", "task_description": ( - "Report Singapore low_temperature" if outcome == "raw_weather" else "Read the title" - )}, + { + "subagent_type": "browser_agent", + "task_description": ("Report Singapore low_temperature" if outcome == "raw_weather" else "Read the title"), + }, session=Session(session_id="parent"), ) assert result.success @@ -338,7 +340,7 @@ def create_browser(*_args, **_kwargs): runtime.service.guardrails.timeout_s = 600 runtime.semantic_progress = {} rail = BrowserRuntimeRail(runtime) - monkeypatch.setattr(rail, "_ensure_browser_mcp_ability", AsyncMock()) + monkeypatch.setattr(rail, "_add_agent_parts", AsyncMock()) browser.add_rail(rail) async def call_model(ctx, _context, _tools): @@ -353,8 +355,11 @@ async def call_model(ctx, _context, _tools): assert state["resume_count"] == 1 extracted = {"product_rating": "4.8"} if observed_states else {"title": "Keyboard"} BrowserRuntimeRail._record_tool_evidence( - state, {"extracted": extracted, "generation_id": "g1"}, - tool_name="browser_batch_interact", tool_args={}, + state, + {"extracted": extracted, "generation_id": "g1"}, + kind=ToolKind(effect=Effect.ACT, progress=True, known=True, helper=True), + tool_name="browser_batch_interact", + tool_args={}, ) ctx.session.update_state({"__browser_phase_budget_state__": state}) observed_states.append(dict(state)) @@ -363,19 +368,30 @@ async def call_model(ctx, _context, _tools): monkeypatch.setattr(browser.react_agent, "_call_model", call_model) return browser - tool = TaskTool(card=ToolCard(id="resume-test", name="task_tool"), parent_agent=SimpleNamespace( - create_subagent=create_browser, - )) + tool = TaskTool( + card=ToolCard(id="resume-test", name="task_tool"), + parent_agent=SimpleNamespace( + create_subagent=create_browser, + ), + ) session = Session(session_id="resume-parent") - first = await tool.invoke({ - "subagent_type": "browser_agent", "task_description": "Return product title and product_rating", - }, session=session) + first = await tool.invoke( + { + "subagent_type": "browser_agent", + "task_description": "Return product title and product_rating", + }, + session=session, + ) assert first.success assert first.data["browser_result"]["status"] == "partial" - second = await tool.invoke({ - "subagent_type": "browser_agent", "task_description": "Only read the missing product rating", - "resume_task_id": first.data["resume_task_id"], - }, session=session) + second = await tool.invoke( + { + "subagent_type": "browser_agent", + "task_description": "Only read the missing product rating", + "resume_task_id": first.data["resume_task_id"], + }, + session=session, + ) assert second.success assert len(observed_states) == 2 assert second.data["browser_result"]["status"] == "completed" diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/__init__.py b/tests/unit_tests/harness/tools/browser_move/architecture/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/_scan.py b/tests/unit_tests/harness/tools/browser_move/architecture/_scan.py new file mode 100644 index 0000000000..94f768ba34 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/_scan.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Static-analysis helpers for the architecture tests: files are parsed with ``ast``, never imported.""" + +from __future__ import annotations + +import ast +import re +import warnings +from functools import lru_cache +from pathlib import Path +from typing import Iterable, Mapping + +REPO_ROOT = Path(__file__).resolve().parents[6] +PKG_DOTTED = "openjiuwen.harness.tools.browser_move" +PKG_DIR = REPO_ROOT.joinpath(*PKG_DOTTED.split(".")) +CORE_DIR = REPO_ROOT / "openjiuwen" / "core" + +CORE_FOLDERS = ("runtime",) +CORE_MODULES = frozenset({"drivers/middleware.py"}) +CONTRACT_FOLDER = "contract" +SURFACES_FOLDER = "surfaces" +CONFIG_MODULE_RE = re.compile(r"^(?:runtime/config\.py|surfaces/[^/]+/(?:runner/settings|config)\.py)$") +ASSEMBLY_MODULES = frozenset({"runtime/assembly.py"}) +SETTINGS_FACADE_MODULES = frozenset({"surfaces/browser/runner/settings.py"}) + + +def rel(path: Path) -> str: + return path.relative_to(PKG_DIR).as_posix() + + +@lru_cache(maxsize=None) +def pkg_files() -> tuple[str, ...]: + return tuple(sorted(rel(p) for p in PKG_DIR.rglob("*.py"))) + + +def in_folder(relpath: str, folder: str) -> bool: + return relpath.startswith(folder + "/") + + +def is_core(relpath: str) -> bool: + return relpath in CORE_MODULES or any(in_folder(relpath, folder) for folder in CORE_FOLDERS) + + +def is_contract(relpath: str) -> bool: + return in_folder(relpath, CONTRACT_FOLDER) + + +def is_surface(relpath: str) -> bool: + return in_folder(relpath, SURFACES_FOLDER) + + +def is_config_module(relpath: str) -> bool: + return bool(CONFIG_MODULE_RE.match(relpath)) + + +def files_where(predicate) -> list[str]: + return [f for f in pkg_files() if predicate(f)] + + +@lru_cache(maxsize=None) +def read_text(relpath: str) -> str: + return (PKG_DIR / relpath).read_text(encoding="utf-8") + + +def parse_source(text: str, filename: str) -> ast.Module: + with warnings.catch_warnings(): + warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning)) + return ast.parse(text, filename=filename) + + +@lru_cache(maxsize=None) +def parse(relpath: str) -> ast.Module: + return parse_source(read_text(relpath), relpath) + + +def module_parts(relpath: str) -> tuple[str, ...]: + parts = relpath[: -len(".py")].split("/") + if parts[-1] == "__init__": + parts = parts[:-1] + return tuple(parts) + + +def resolve_import(relpath: str, module: str | None, level: int) -> tuple[str, ...] | None: + """Package-relative module path of an import, or ``None`` for a module outside the package.""" + if level: + base = module_parts(relpath) + if not relpath.endswith("__init__.py"): + base = base[:-1] + if level - 1 > len(base): + return None + base = base[: len(base) - (level - 1)] + return base + (tuple(module.split(".")) if module else ()) + if not module: + return None + if module == PKG_DOTTED or module.startswith(PKG_DOTTED + "."): + return tuple(module[len(PKG_DOTTED) + 1 :].split(".")) if module != PKG_DOTTED else () + return None + + +def is_package_module(parts: tuple[str, ...]) -> bool: + """True when ``parts`` names a module or package that exists in the package.""" + if not parts: + return True + path = PKG_DIR.joinpath(*parts) + return path.with_suffix(".py").is_file() or (path / "__init__.py").is_file() + + +def iter_imports(tree: ast.AST) -> Iterable[tuple[ast.stmt, bool]]: + """Yield ``(import node, lazy)``; lazy means inside a function body.""" + + def walk(node: ast.AST, lazy: bool): + for child in ast.iter_child_nodes(node): + if isinstance(child, (ast.Import, ast.ImportFrom)): + yield child, lazy + yield from walk(child, lazy or isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda))) + + yield from walk(tree, False) + + +def compare_with_allowlist(actual: Mapping, known: Mapping, label: str) -> list[str]: + """Ratchet: new keys, growth and entries down to zero fail; a count that dropped above zero passes.""" + problems: list[str] = [] + for key in sorted(set(actual) | set(known), key=repr): + have = actual.get(key, 0) + allowed = known.get(key) + if allowed is None: + problems.append(f"new {label}: {key!r} (count {have}); fix it, do not allowlist it") + elif have > allowed: + problems.append(f"{label} grew: {key!r} has {have}, allowlist says {allowed}") + elif have == 0: + problems.append(f"stale entry — remove it: {key!r} no longer has any {label}") + return problems diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/__init__.py b/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/surface.py b/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/surface.py new file mode 100644 index 0000000000..d1fa31d540 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/surface.py @@ -0,0 +1,182 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""A screen surface written from outside the package: its command, renderer, policy, driver and provisioner.""" + +from __future__ import annotations + +from collections import deque +from collections.abc import Iterable, Mapping +from dataclasses import dataclass +from typing import Any, ClassVar + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + Capability, + Click, + ContentBlock, + ContextRef, + DriverError, + DriverInfo, + Effect, + HandlerDriver, + Health, + Observation, + Observe, + PointRef, + PolicyRule, + RefResolver, + Registry, + Screenshot, + SurfaceBundle, + SurfaceContext, + SurfaceProfile, + ToolSpec, + handles, +) +from openjiuwen.harness.tools.browser_move.provisioning import ProvisionedDevice +from openjiuwen.harness.tools.browser_move.runtime.assembly import Registries + +BOARD = 100 +SCREEN = ContextRef("screen-1", "screen", "Board") +DEVICE = ProvisionedDevice(kind="fake-screen", endpoint="screen://1", extras={}) +MESSAGES = { + "driver_restarted": "the screen restarted; look again", + "driver_lost": "the screen is gone", + "retry": "Then poke again.", +} + + +@dataclass(frozen=True) +class Poke(Action): + """Poke the screen at a point.""" + + effect: ClassVar[Effect] = Effect.ACT + + target: PointRef + + +@dataclass(frozen=True) +class ScreenContext(SurfaceContext): + app: str + + @property + def identity(self) -> str: + return self.app + + +@dataclass(frozen=True) +class ScreenSettings: + provisioning: str = "screen" + + +class ScreenRenderer: + def observation(self, observation: Observation) -> tuple[ContentBlock, ...]: + return (f"screen {observation.context.id}", observation.screenshot) + + def result(self, action: Action, result: ActionResult) -> tuple[ContentBlock, ...]: + if isinstance(action, Observe): + return (result.value,) + return (f"{type(action).__name__} done",) + + def error(self, action: Action, error: DriverError, observation: Observation | None) -> tuple[ContentBlock, ...]: + text = f"### Error\n{type(error).__name__}: {error}" + if observation is not None: + text += f"\nfresh look at {observation.context.id} in {observation.generation}" + return (text,) + + +def _on_the_board(action: Action, context: SurfaceContext | None) -> str | None: + target = action.target + return None if 0 <= target.x < BOARD and 0 <= target.y < BOARD else f"({target.x}, {target.y}) is off the board" + + +def _look(args: Mapping[str, Any], refs: RefResolver) -> Action: + return Observe(include_screenshot=True) + + +PROFILE = SurfaceProfile( + name="fake", + commands=(Poke,), + renderer=ScreenRenderer(), + tool_specs=( + ToolSpec("look", (Observe,), "Look at the screen.", schema={"type": "object", "properties": {}}, build=_look), + ToolSpec("click", (Click,), "Click a point of the screen."), + ToolSpec("poke", (Poke,), "Poke a point of the screen."), + ), + capability_catalog={"core": frozenset({(Observe, None), (Click, PointRef), (Poke, PointRef)})}, + capability_tools={"core": ("look", "click", "poke")}, + capabilities=(Capability("core", "Look at, click and poke the screen.", ("look", "click", "poke")),), + policy_rules=(PolicyRule(Poke, _on_the_board),), + timeouts=dict.fromkeys(Effect, 5.0), + messages=MESSAGES, + settings=ScreenSettings(), +) + + +class ScreenDriver(HandlerDriver): + """Screenshot-only observations; each command answers with the next scripted outcome.""" + + def __init__(self, outcomes: Iterable[ActionResult | DriverError]) -> None: + self.outcomes = deque(outcomes) + self.calls: list[str | Action] = [] + self._shots = 0 + + def describe(self) -> DriverInfo: + return DriverInfo(name="screen_driver", surfaces=frozenset({"fake"})) + + async def start(self, config: Any = None) -> None: + self.calls.append("start") + + async def stop(self) -> None: + self.calls.append("stop") + + async def health(self) -> Health: + return Health(ok=True) + + async def observe(self, command: Observe) -> Observation: + self.calls.append(command) + self._shots += 1 + screenshot = Screenshot(b"\x89PNG-%d" % self._shots, BOARD, BOARD) + return Observation(context=SCREEN, screenshot=screenshot, surface=ScreenContext("board")) + + @handles(Click, refs={PointRef}) + async def _click(self, action: Click) -> ActionResult: + return self._next(action) + + @handles(Poke, refs={PointRef}) + async def _poke(self, action: Poke) -> ActionResult: + return self._next(action) + + def _next(self, action: Action) -> ActionResult: + self.calls.append(action) + outcome = self.outcomes.popleft() + if isinstance(outcome, DriverError): + raise outcome + return outcome + + +class ScreenProvisioner: + def __init__(self) -> None: + self.log: list[tuple[str, Any]] = [] + + def provision(self, surface_settings: Any) -> ProvisionedDevice: + self.log.append(("provision", surface_settings)) + return DEVICE + + def release(self, device: ProvisionedDevice) -> None: + self.log.append(("release", device)) + + def is_alive(self, device: ProvisionedDevice) -> bool: + return True + + +def screen_registries(driver: ScreenDriver, provisioner: ScreenProvisioner) -> Registries: + """Test-scoped registries: the screen surface, its driver and its provisioner by name.""" + surfaces: Registry[SurfaceBundle] = Registry("surface") + surfaces.register("fake", lambda environ, instance=None, settings=None: SurfaceBundle(PROFILE.settings, PROFILE)) + drivers: Registry[Any] = Registry("driver") + drivers.register("screen_driver", lambda device, settings: driver) + provisioners: Registry[Any] = Registry("provisioner") + provisioners.register("screen", lambda: provisioner) + return Registries(drivers=drivers, surfaces=surfaces, provisioners=provisioners) diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/test_fake_surface.py b/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/test_fake_surface.py new file mode 100644 index 0000000000..4d6ee57bde --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/test_fake_surface.py @@ -0,0 +1,188 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""A new surface, driver and provisioner reach a running agent through registries alone, with no core change.""" + +from __future__ import annotations + +import asyncio +import base64 +import json +from dataclasses import dataclass, field +from typing import Any + +import pytest + +from openjiuwen.core.foundation.llm import AssistantMessage, Model, ModelClientConfig, ModelRequestConfig, ToolCall +from openjiuwen.core.single_agent.rail.base import AgentCallbackContext, AgentRail +from openjiuwen.harness.tools.browser_move.contract import ( + ActionResult, + Click, + ConnectionLost, + Observe, + PointRef, + PolicyDenied, + StaleRef, +) +from openjiuwen.harness.tools.browser_move.runtime.assembly import build_agent +from openjiuwen.harness.tools.browser_move.runtime.config import CoreSettings +from openjiuwen.harness.tools.browser_move.runtime.page_state import PageState +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import command_record + +from .surface import DEVICE, MESSAGES, Poke, ScreenDriver, ScreenProvisioner, screen_registries + +SCRIPT = ( + ("look", {}), + ("poke", {"x": 500, "y": 5}), + ("click", {"x": 3, "y": 4}), + ("poke", {"x": 10, "y": 10}), + ("look", {}), + ("click", {"x": 3, "y": 4}), +) + + +class ScriptedModel(Model): + """Calls the scripted tools one per turn, then answers; keeps every request it was sent.""" + + def __init__(self) -> None: + super().__init__( + model_client_config=ModelClientConfig( + client_provider="OpenAI", api_key="mock-api-key", api_base="http://mock.invalid", verify_ssl=False + ), + model_config=ModelRequestConfig(model="scripted"), + ) + self.requests: list[dict[str, Any]] = [] + + async def invoke(self, *args: Any, **kwargs: Any) -> AssistantMessage: + self.requests.append(kwargs) + turn = len(self.requests) + if turn > len(SCRIPT): + return AssistantMessage(content="done") + name, arguments = SCRIPT[turn - 1] + call = ToolCall(id=f"call-{turn}", type="function", name=name, arguments=json.dumps(arguments)) + return AssistantMessage(content="", tool_calls=[call]) + + +class ToolResults(AgentRail): + def __init__(self) -> None: + super().__init__() + self.by_call: dict[str, Any] = {} + self.identities: list[str] = [] + self.page_state: PageState | None = None + + async def after_tool_call(self, ctx: AgentCallbackContext) -> None: + self.by_call[ctx.inputs.tool_call.id] = ctx.inputs.tool_result + self.identities.append(self.page_state.identity) + + +@dataclass +class Run: + model: ScriptedModel = field(default_factory=ScriptedModel) + driver: ScreenDriver = field( + default_factory=lambda: ScreenDriver( + [ActionResult(ok=True, context_changed=True), ConnectionLost("the cable came out", dispatched=True)] + ) + ) + provisioner: ScreenProvisioner = field(default_factory=ScreenProvisioner) + results: ToolResults = field(default_factory=ToolResults) + output: Any = None + + def tool_message(self, turn: int) -> str: + """What the model read for the tool it called on ``turn`` (1-based).""" + messages = [message.content for message in self.model.requests[turn]["messages"] if message.role == "tool"] + return messages[-1] + + +@pytest.fixture(scope="module") +def run(tmp_path_factory: pytest.TempPathFactory) -> Run: + run = Run() + workspace = tmp_path_factory.mktemp("fake-surface-workspace") + + async def scenario() -> None: + agent, session = await build_agent( + CoreSettings(driver="screen_driver", surface="fake"), + {}, + model=run.model, + capabilities=("core",), + image_support=True, + language="en", + registries=screen_registries(run.driver, run.provisioner), + enable_task_loop=False, + enable_read_image_multimodal=False, + max_iterations=len(SCRIPT) + 2, + workspace=str(workspace), + ) + agent.add_rail(run.results) + run.results.page_state = session.page_state + try: + run.output = await agent.invoke({"query": "Poke the board", "conversation_id": "fake-surface"}) + finally: + await session.close() + + asyncio.run(scenario()) + return run + + +def _schemas(request: dict[str, Any]) -> dict[str, Any]: + return {tool.name: tool.parameters for tool in request["tools"]} + + +def _images(request: dict[str, Any]) -> list[str]: + return [ + part["image_url"]["url"] + for message in request["messages"] + if message.role == "user" and isinstance(message.content, list) + for part in message.content + if part.get("type") == "image_url" + ] + + +def test_the_scripted_task_runs_to_its_answer(run: Run) -> None: + assert run.output["output"] == "done" + assert len(run.model.requests) == len(SCRIPT) + 1 + + +def test_the_model_is_offered_the_surface_tools_with_point_arguments(run: Run) -> None: + schemas = _schemas(run.model.requests[0]) + + assert set(schemas) == {"look", "click", "poke"} + assert schemas["poke"]["properties"] == {"x": {"type": "integer"}, "y": {"type": "integer"}} + assert schemas["click"]["properties"] == {"x": {"type": "integer"}, "y": {"type": "integer"}} + + +def test_the_model_receives_the_screenshot_as_an_image(run: Run) -> None: + first_shot = "data:image/png;base64," + base64.b64encode(b"\x89PNG-1").decode("ascii") + + assert _images(run.model.requests[0]) == [] + assert _images(run.model.requests[1]) == [first_shot] + + +def test_a_poke_off_the_board_is_denied_before_the_driver(run: Run) -> None: + assert isinstance(run.results.by_call["call-2"], PolicyDenied) + assert run.tool_message(2) == "### Error\n(500, 5) is off the board" + assert not [call for call in run.driver.calls if isinstance(call, Poke)] + + +def test_a_stale_point_observes_once_and_reports_the_fresh_observation(run: Run) -> None: + click = run.driver.calls.index(Click(PointRef(3, 4, "g0"))) + next_look = run.driver.calls.index(Observe(include_screenshot=True), click) + + assert run.driver.calls[click + 1 : next_look] == [Observe()] + assert command_record(run.results.by_call["call-4"]).error == StaleRef.__name__ + assert run.tool_message(4) == ( + "### Error\nStaleRef: Stale PointRef belongs to g0; current generation is g1.\nfresh look at screen-1 in g1" + ) + + +def test_the_core_page_state_tracks_the_screen_context_by_its_own_identity(run: Run) -> None: + assert type(run.results.page_state) is PageState + assert run.results.identities[0] == "board" + + +def test_a_lost_connection_restarts_the_driver_once(run: Run) -> None: + last_click = run.driver.calls.index(Click(PointRef(3, 4, "g1"))) + + assert run.driver.calls[last_click + 1 :] == ["stop", "start", "stop"] + assert run.driver.calls.count("start") == 2 + assert run.tool_message(6) == f"### Error\nConnectionLost: {MESSAGES['driver_restarted']}" + assert [entry[0] for entry in run.provisioner.log] == ["provision", "release"] + assert run.provisioner.log[-1] == ("release", DEVICE) diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/test_fake_surface_runtime.py b/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/test_fake_surface_runtime.py new file mode 100644 index 0000000000..46c3e7dee6 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/fake_surface/test_fake_surface_runtime.py @@ -0,0 +1,134 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The screen surface runs a task through the core runtime on both topologies, with no browser input anywhere.""" + +from __future__ import annotations + +import asyncio +import json +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, patch + +import pytest + +from openjiuwen.core.foundation.llm import AssistantMessage, Model, ModelClientConfig, ModelRequestConfig, ToolCall +from openjiuwen.harness.tools.browser_move.contract import ActionResult, Click, PointRef, SurfaceBundle +from openjiuwen.harness.tools.browser_move.runtime import assembly as assembly_module +from openjiuwen.harness.tools.browser_move.runtime import topology as topology_module +from openjiuwen.harness.tools.browser_move.runtime.config import CoreSettings, RunGuardrails +from openjiuwen.harness.tools.browser_move.runtime.page_state import PageState +from openjiuwen.harness.tools.browser_move.runtime.session import AgentRuntime +from openjiuwen.harness.tools.browser_move.runtime.topology import DelegatedWorker, Direct + +from .surface import DEVICE, PROFILE, Poke, ScreenDriver, ScreenProvisioner, screen_registries + +SCRIPT = (("look", {}), ("poke", {"x": 10, "y": 10}), ("look", {}), ("click", {"x": 3, "y": 4})) +ANSWERS = {"direct": "poked", "delegated_worker": json.dumps({"ok": True, "final": "poked"})} +TOPOLOGIES = {"direct": Direct, "delegated_worker": DelegatedWorker} + + +class ScriptedModel(Model): + """One scripted tool call per turn, then ``answer``; keeps every request it was sent.""" + + def __init__(self, answer: str) -> None: + super().__init__( + model_client_config=ModelClientConfig( + client_provider="openai", api_key="mock-api-key", api_base="http://mock.invalid", verify_ssl=False + ), + model_config=ModelRequestConfig(model="scripted"), + ) + self.answer = answer + self.requests: list[dict[str, Any]] = [] + + async def invoke(self, *args: Any, **kwargs: Any) -> AssistantMessage: + self.requests.append(kwargs) + turn = len(self.requests) + if turn > len(SCRIPT): + return AssistantMessage(content=self.answer) + name, arguments = SCRIPT[turn - 1] + call = ToolCall(id=f"call-{turn}", type="function", name=name, arguments=json.dumps(arguments)) + return AssistantMessage(content="", tool_calls=[call]) + + +class Run: + def __init__(self, topology: str, tmp_path: Path) -> None: + self.driver = ScreenDriver([ActionResult(ok=True, context_changed=True), ActionResult(ok=True)]) + self.provisioner = ScreenProvisioner() + self.model = ScriptedModel(ANSWERS[topology]) + self.output_dir = tmp_path / "out" + self.runtime = AgentRuntime( + CoreSettings(driver="screen_driver", surface="fake", topology=topology), + SurfaceBundle(PROFILE.settings, PROFILE), + provider="openai", + api_key="mock-api-key", + api_base="http://mock.invalid", + model_name="scripted", + guardrails=RunGuardrails(max_steps=len(SCRIPT) + 2, max_failures=1, timeout_s=30, retry_once=False), + workspace=str(tmp_path / "ws"), + language="en", + output_dir=self.output_dir, + identity="screen:test", + ) + self.answer: dict[str, Any] = {} + self.topology: type | None = None + + async def scenario(self) -> None: + registries = screen_registries(self.driver, self.provisioner) + build_worker = topology_module.build_worker + + async def scripted_worker(*args: Any, **kwargs: Any) -> Any: + worker = await build_worker(*args, **kwargs) + worker.set_llm(self.model) + return worker + + with ( + patch.object(assembly_module, "DRIVERS", registries.drivers), + patch.object(assembly_module, "SURFACES", registries.surfaces), + patch.object(assembly_module, "PROVISIONERS", registries.provisioners), + patch.object(topology_module, "_model", return_value=self.model), + patch.object(topology_module, "probe_image_support", AsyncMock(return_value=True)), + patch.object(topology_module, "build_worker", scripted_worker), + ): + await self.runtime.ensure_started() + try: + self.answer = await self.runtime.run_task("Poke the board", session_id="screen") + self.topology = type(self.runtime.topology) + finally: + await self.runtime.shutdown() + + +@pytest.fixture(params=sorted(TOPOLOGIES)) +def run(request: pytest.FixtureRequest, tmp_path: Path) -> Run: + run = Run(request.param, tmp_path) + asyncio.run(run.scenario()) + return run + + +def test_the_task_runs_to_its_answer_on_the_core_page_state(run: Run) -> None: + assert (run.answer["ok"], run.answer["final"], run.answer["session_id"]) == (True, "poked", "screen") + assert run.answer["error"] is None + assert type(run.runtime.page_state) is PageState + assert run.topology is TOPOLOGIES[run.runtime.core.topology] + + +def test_the_answer_carries_the_screen_identity_and_no_metadata(run: Run) -> None: + assert run.answer["context"] == {"identity": "board", "metadata": {}} + + +def test_the_model_called_the_surface_tools_and_the_driver_saw_the_commands(run: Run) -> None: + assert len(run.model.requests) == len(SCRIPT) + 1 + assert {tool.name for tool in run.model.requests[0]["tools"]} == {"look", "click", "poke"} + assert Poke(PointRef(10, 10, "g0")) in run.driver.calls + assert Click(PointRef(3, 4, "g1")) in run.driver.calls + + +def test_the_device_is_provisioned_once_and_released_at_shutdown(run: Run) -> None: + assert run.driver.calls.count("start") == 1 and run.driver.calls[-1] == "stop" + assert [entry[0] for entry in run.provisioner.log] == ["provision", "release"] + assert run.provisioner.log[-1] == ("release", DEVICE) + + +def test_the_output_folder_and_the_identity_are_the_given_ones(run: Run) -> None: + assert (run.output_dir / "screenshots").is_dir() + assert run.runtime.service.identity == "screen:test" diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/test_browser_words.py b/tests/unit_tests/harness/tools/browser_move/architecture/test_browser_words.py new file mode 100644 index 0000000000..a92acf27c1 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/test_browser_words.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""No whole-word browser, page, tab or url in the contract or the core, outside import statements.""" + +from __future__ import annotations + +import re +from collections import Counter + +from . import _scan + +BROWSER_WORD_RE = re.compile(r"\b(?:browser|page|tab|url)\b", re.IGNORECASE) + +KNOWN_BROWSER_WORDS: dict[str, int] = { + "runtime/rails/repeated_call_guard.py": 1, + "runtime/worker.py": 37, +} + + +def _text_without_imports(relpath: str) -> str: + lines = _scan.read_text(relpath).splitlines() + for node, _lazy in _scan.iter_imports(_scan.parse(relpath)): + for lineno in range(node.lineno, node.end_lineno + 1): + lines[lineno - 1] = "" + return "\n".join(lines) + + +def _collect_browser_words() -> Counter: + found: Counter = Counter() + for relpath in _scan.files_where(lambda f: _scan.is_contract(f) or _scan.is_core(f)): + hits = len(BROWSER_WORD_RE.findall(_text_without_imports(relpath))) + if hits: + found[relpath] = hits + return found + + +class TestBrowserWords: + def test_no_browser_words_in_contract_or_core(self): + problems = _scan.compare_with_allowlist(_collect_browser_words(), KNOWN_BROWSER_WORDS, "browser word") + assert not problems, "\n".join(problems) + + def test_the_core_page_state_names_no_browser_field(self): + fields = re.compile(r"\b(?:url|title|tabs|price|rating|shop|href|captcha|login)\b", re.IGNORECASE) + hits = fields.findall(_text_without_imports("runtime/page_state.py")) + assert not hits, f"runtime/page_state.py names browser fields: {sorted(set(hits))}" diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/test_driver_names.py b/tests/unit_tests/harness/tools/browser_move/architecture/test_driver_names.py new file mode 100644 index 0000000000..0c445ac3db --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/test_driver_names.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""No driver names and no Playwright MCP tool names in the core; ``surfaces/browser/`` may name tools.""" + +from __future__ import annotations + +import re +from collections import Counter + +from . import _scan + +DRIVER_NAME_RE = re.compile(r"playwright_mcp|browser_use(?![a-z])|browser-use|\bmcp\b", re.IGNORECASE) + +MCP_TOOL_NAMES = frozenset( + { + "browser_annotate", + "browser_click", + "browser_close", + "browser_console_messages", + "browser_cookie_clear", + "browser_cookie_delete", + "browser_cookie_get", + "browser_cookie_list", + "browser_cookie_set", + "browser_drag", + "browser_drop", + "browser_evaluate", + "browser_file_upload", + "browser_fill_form", + "browser_find", + "browser_generate_locator", + "browser_get_config", + "browser_handle_dialog", + "browser_hide_highlight", + "browser_highlight", + "browser_hover", + "browser_localstorage_clear", + "browser_localstorage_delete", + "browser_localstorage_get", + "browser_localstorage_list", + "browser_localstorage_set", + "browser_mouse_click_xy", + "browser_mouse_down", + "browser_mouse_drag_xy", + "browser_mouse_move_xy", + "browser_mouse_up", + "browser_mouse_wheel", + "browser_navigate", + "browser_navigate_back", + "browser_network_request", + "browser_network_requests", + "browser_network_state_set", + "browser_pdf_save", + "browser_press_key", + "browser_resize", + "browser_resume", + "browser_route", + "browser_route_list", + "browser_run_code", + "browser_run_code_unsafe", + "browser_select_option", + "browser_sessionstorage_clear", + "browser_sessionstorage_delete", + "browser_sessionstorage_get", + "browser_sessionstorage_list", + "browser_sessionstorage_set", + "browser_set_storage_state", + "browser_snapshot", + "browser_start_tracing", + "browser_start_video", + "browser_stop_tracing", + "browser_stop_video", + "browser_storage_state", + "browser_tabs", + "browser_take_screenshot", + "browser_type", + "browser_unroute", + "browser_verify_element_visible", + "browser_verify_list_visible", + "browser_verify_text_visible", + "browser_verify_value", + "browser_video_chapter", + "browser_video_hide_actions", + "browser_video_show_actions", + "browser_wait_for", + } +) +MCP_TOOL_NAME_RE = re.compile( + r"(? bool: + return (_scan.is_core(relpath) or _scan.is_surface(relpath)) and not _scan.is_config_module(relpath) + + +def _in_mcp_name_scope(relpath: str) -> bool: + return _scan.is_core(relpath) or (_scan.is_surface(relpath) and not _scan.in_folder(relpath, "surfaces/browser")) + + +def _count(pattern: re.Pattern, predicate) -> Counter: + found: Counter = Counter() + for relpath in _scan.files_where(predicate): + hits = len(pattern.findall(_scan.read_text(relpath))) + if hits: + found[relpath] = hits + return found + + +def _collect_driver_names() -> Counter: + return _count(DRIVER_NAME_RE, _in_driver_name_scope) + + +def _collect_mcp_tool_names() -> Counter: + return _count(MCP_TOOL_NAME_RE, _in_mcp_name_scope) + + +class TestDriverNames: + def test_mcp_tool_name_list_is_complete(self): + assert len(MCP_TOOL_NAMES) == 70 + + def test_no_driver_names_in_core_or_surfaces(self): + problems = _scan.compare_with_allowlist(_collect_driver_names(), KNOWN_DRIVER_NAMES, "driver name") + assert not problems, "\n".join(problems) + + def test_no_mcp_tool_names_in_core(self): + problems = _scan.compare_with_allowlist(_collect_mcp_tool_names(), KNOWN_MCP_TOOL_NAMES, "MCP tool name") + assert not problems, "\n".join(problems) diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/test_env_reads.py b/tests/unit_tests/harness/tools/browser_move/architecture/test_env_reads.py new file mode 100644 index 0000000000..0335aff768 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/test_env_reads.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Environment variables are read only in the config modules.""" + +from __future__ import annotations + +import ast +import re +from collections import Counter +from functools import lru_cache + +from . import _scan + +ENV_MODULE = ("runtime", "utils", "env") +UPPER_SNAKE_RE = re.compile(r"^[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)+$") + +KNOWN_ENV_READS: dict[str, dict[str, int]] = { + "surfaces/browser/runner/facade.py": { + "env_map.get(PLAYWRIGHT_MCP_CDP_ENDPOINT)": 1, + }, +} + + +def _module_file(parts: tuple[str, ...]) -> str | None: + for candidate in ("/".join(parts) + ".py", "/".join(parts + ("__init__",)) + ".py"): + if candidate in _scan.pkg_files(): + return candidate + return None + + +def _reads_process_env(function: ast.AST) -> bool: + return any( + isinstance(node, ast.Attribute) + and isinstance(node.value, ast.Name) + and node.value.id == "os" + and node.attr in ("getenv", "environ") + for node in ast.walk(function) + ) + + +@lru_cache(maxsize=None) +def _env_functions() -> frozenset[str]: + """``runtime/utils/env.py`` functions reading the process environment; pure helpers over a mapping do not count.""" + tree = _scan.parse(_module_file(ENV_MODULE)) + return frozenset( + n.name for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef)) and _reads_process_env(n) + ) + + +@lru_cache(maxsize=None) +def _bindings(relpath: str) -> dict[str, tuple[str, ...]]: + """Local names bound by imports: name -> package path or ``("", attr)``.""" + bound: dict[str, tuple[str, ...]] = {} + for node, _lazy in _scan.iter_imports(_scan.parse(relpath)): + if isinstance(node, ast.Import): + for alias in node.names: + if alias.name == "os": + bound[alias.asname or "os"] = ("",) + continue + target = _scan.resolve_import(relpath, alias.name, 0) + if target is not None and alias.asname: + bound[alias.asname] = target + elif node.module == "os" and not node.level: + for alias in node.names: + bound[alias.asname or alias.name] = ("", alias.name) + else: + base = _scan.resolve_import(relpath, node.module, node.level) + if base is None: + continue + for alias in node.names: + if alias.name != "*": + bound[alias.asname or alias.name] = base + (alias.name,) + if relpath == _module_file(ENV_MODULE): + for name in _env_functions(): + bound.setdefault(name, ENV_MODULE + (name,)) + return bound + + +def _origin(target: tuple[str, ...], depth: int = 0) -> tuple[str, ...]: + """Follow re-exports until the defining module (or give up after a few hops).""" + if depth > 8 or len(target) < 2 or target[0] == "" or target[:-1] == ENV_MODULE: + return target + owner = _module_file(target[:-1]) + if owner is None: + return target + rebound = _bindings(owner).get(target[-1]) + return _origin(rebound, depth + 1) if rebound and rebound != target else target + + +def _resolve(relpath: str, expr: ast.expr) -> tuple[str, ...] | None: + attrs: list[str] = [] + while isinstance(expr, ast.Attribute): + attrs.insert(0, expr.attr) + expr = expr.value + if not isinstance(expr, ast.Name): + return None + root = _bindings(relpath).get(expr.id) + if root is None: + return None + return _origin(root + tuple(attrs)) + + +def _literal_args(call: ast.Call) -> list[str]: + values: list[str] = [] + for arg in list(call.args) + [kw.value for kw in call.keywords]: + items = arg.elts if isinstance(arg, (ast.Tuple, ast.List, ast.Set)) else [arg] + values += [i.value for i in items if isinstance(i, ast.Constant) and isinstance(i.value, str)] + return values + + +def _key(call: ast.Call | None) -> str: + if call is None or not call.args: + return "?" + first = call.args[0] + return first.value if isinstance(first, ast.Constant) and isinstance(first.value, str) else "?" + + +def _parents(tree: ast.AST) -> dict[ast.AST, ast.AST]: + return {child: node for node in ast.walk(tree) for child in ast.iter_child_nodes(node)} + + +@lru_cache(maxsize=None) +def _direct_and_helper_reads(relpath: str) -> tuple[Counter, set[int], set[str]]: + """Direct and helper reads: descriptors, ids of the calls they cover, literal keys.""" + tree = _scan.parse(relpath) + parents = _parents(tree) + found: Counter = Counter() + covered: set[int] = set() + keys: set[str] = set() + for node in ast.walk(tree): + if not isinstance(node, (ast.Name, ast.Attribute)) or isinstance(parents.get(node), ast.Attribute): + continue + target = _resolve(relpath, node) + if target is None: + continue + parent = parents.get(node) + if target[:1] == ("",) and len(target) >= 2 and target[1] in ("getenv", "environ"): + call = parent if isinstance(parent, ast.Call) and parent.func is node else None + if target[1] == "getenv" or len(target) > 2: + label = "os.getenv" if target[1] == "getenv" else f"os.environ.{target[2]}" + found[f"{label}({_key(call)})"] += 1 + elif isinstance(parent, ast.Subscript) and isinstance(parent.slice, ast.Constant): + found[f"os.environ[{parent.slice.value}]"] += 1 + keys.add(str(parent.slice.value)) + else: + found["os.environ"] += 1 + if call is not None: + covered.add(id(call)) + keys.update(_literal_args(call)[:1]) + elif target[:-1] == ENV_MODULE and target[-1] in _env_functions(): + if isinstance(parent, ast.Call) and parent.func is node: + names = [v for v in _literal_args(parent) if UPPER_SNAKE_RE.match(v)] + found[f"utils.env.{target[-1]}({','.join(names)})"] += 1 + covered.add(id(parent)) + keys.update(names) + return found, covered, keys + + +@lru_cache(maxsize=None) +def _known_env_names() -> frozenset[str]: + names: set[str] = set() + for relpath in _scan.pkg_files(): + names |= _direct_and_helper_reads(relpath)[2] + if _scan.is_config_module(relpath): + tree = _scan.parse(relpath) + names |= {n.value for n in ast.walk(tree) if isinstance(n, ast.Constant) and isinstance(n.value, str)} + return frozenset(n for n in names if UPPER_SNAKE_RE.match(n)) + + +def _collect_env_reads() -> dict[str, Counter]: + known = _known_env_names() + result: dict[str, Counter] = {} + for relpath in _scan.files_where(lambda f: not _scan.is_config_module(f)): + direct, covered, _keys = _direct_and_helper_reads(relpath) + found = Counter(direct) + for node in ast.walk(_scan.parse(relpath)): + if isinstance(node, ast.Call) and id(node) not in covered: + names = [v for v in _literal_args(node) if v in known] + if names: + found[f"{ast.unparse(node.func)}({','.join(names)})"] += 1 + if found: + result[relpath] = found + return result + + +class TestEnvReads: + def test_env_is_read_only_in_config_modules(self): + actual = _collect_env_reads() + flat_actual = {(f, d): n for f, reads in actual.items() for d, n in reads.items()} + flat_known = {(f, d): n for f, reads in KNOWN_ENV_READS.items() for d, n in reads.items()} + problems = _scan.compare_with_allowlist(flat_actual, flat_known, "env read outside the config modules") + assert not problems, "\n".join(problems) diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/test_import_boundaries.py b/tests/unit_tests/harness/tools/browser_move/architecture/test_import_boundaries.py new file mode 100644 index 0000000000..37ef410855 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/test_import_boundaries.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Import boundaries between the contract, drivers, provisioning, surfaces and the core.""" + +from __future__ import annotations + +import ast +import sys +from collections import Counter + +from . import _scan + +REGISTRY_IMPORTERS: dict[str, frozenset[str]] = { + "drivers": frozenset({"runtime/assembly.py"}), + "provisioning": frozenset({"runtime/assembly.py"}), + "surfaces": frozenset({"runtime/assembly.py", "runtime/config.py"}), +} + +KNOWN_CORE_IMPORTS_OF_CONCRETE_MODULES: dict[tuple[str, str], int] = {} + +KNOWN_CORE_DETOURS_INTO_CONCRETE_MODULES: dict[tuple[str, str], int] = {} + +KNOWN_CORE_TO_HARNESS_IMPORTS: dict[tuple[str, str], int] = { + ("context_engine/context/session_memory_manager.py", "openjiuwen.harness.tools.filesystem"): 2, + ("context_engine/processor/forked/compressor/session_memory_agent.py", "openjiuwen.harness.tools.filesystem"): 1, + ("memory/lite/coding_memory_tool_ops.py", "openjiuwen.harness.workspace.workspace"): 1, + ("memory/lite/coding_memory_tools.py", "openjiuwen.harness.workspace.workspace"): 1, + ("memory/lite/internal.py", "openjiuwen.harness.workspace.workspace"): 1, + ("memory/lite/manager.py", "openjiuwen.harness.workspace.workspace"): 1, + ("memory/lite/memory_tool_context_base.py", "openjiuwen.harness.workspace.workspace"): 1, + ("memory/lite/memory_tool_ops.py", "openjiuwen.harness.workspace.workspace"): 1, + ("memory/lite/memory_tools.py", "openjiuwen.harness.workspace.workspace"): 1, +} + +KNOWN_DRIVER_IMPORTS_OUTSIDE_THEIR_LAYER: dict[tuple[str, str], int] = {} +DRIVER_LAYER_IMPORTERS = frozenset({"runtime/assembly.py"}) +LAZY_DRIVER_LAYER_IMPORTERS = _scan.SETTINGS_FACADE_MODULES +ROOT_MODULES = frozenset({"__init__.py", "playwright_runtime_mcp_server.py"}) +SURFACE_MODULES_A_DRIVER_MAY_IMPORT = frozenset({"commands", "context"}) + +_STDLIB = frozenset(sys.stdlib_module_names) | {"__future__"} +_CONTRACT_TESTING_EXTRA = frozenset({"pytest"}) + + +def _concrete_kind(parts: tuple[str, ...]) -> str | None: + """Classify a package module path as a concrete driver, surface or provisioner.""" + if len(parts) < 2: + return None + head, name = parts[0], parts[1] + if head == "drivers" and name != "middleware": + return "driver" + if head == "surfaces": + return "surface" + if head == "provisioning": + return "provisioning" + return None + + +def _import_targets(relpath: str, node: ast.stmt) -> list[tuple[str, ...]]: + """Package-relative module paths an import statement reaches (outside ones dropped).""" + if isinstance(node, ast.Import): + targets = [_scan.resolve_import(relpath, alias.name, 0) for alias in node.names] + return [t for t in targets if t is not None] + base = _scan.resolve_import(relpath, node.module, node.level) + if base is None: + return [] + targets = [] + for alias in node.names: + sub = base + (alias.name,) + targets.append(sub if alias.name != "*" and _scan.is_package_module(sub) else base) + return targets + + +def _collect_core_concrete_imports() -> Counter: + found: Counter = Counter() + for relpath in _scan.files_where(_scan.is_core): + lazy_allowed = relpath in _scan.ASSEMBLY_MODULES | _scan.SETTINGS_FACADE_MODULES + for node, lazy in _scan.iter_imports(_scan.parse(relpath)): + for target in _import_targets(relpath, node): + if _concrete_kind(target) and not (lazy_allowed and lazy): + found[(relpath, ".".join(target))] += 1 + return found + + +def _collect_core_imports_of_registries() -> list[str]: + bad = [] + for relpath in _scan.files_where(_scan.is_core): + for node, _lazy in _scan.iter_imports(_scan.parse(relpath)): + for target in _import_targets(relpath, node): + allowed = REGISTRY_IMPORTERS.get(target[0]) if len(target) == 1 else None + if allowed is not None and relpath not in allowed: + bad.append(f"{relpath}:{node.lineno} imports the {target[0]} package") + return bad + + +def _collect_surface_imports_of_drivers_or_provisioners() -> list[str]: + bad = [] + for relpath in _scan.files_where(_scan.is_surface): + for node, lazy in _scan.iter_imports(_scan.parse(relpath)): + for target in _import_targets(relpath, node): + if lazy and relpath in _scan.SETTINGS_FACADE_MODULES: + continue + if _concrete_kind(target) in ("driver", "provisioning"): + bad.append(f"{relpath}:{node.lineno} imports {'.'.join(target)}") + return bad + + +def _collect_contract_imports() -> list[str]: + bad = [] + for relpath in _scan.files_where(_scan.is_contract): + for node, _lazy in _scan.iter_imports(_scan.parse(relpath)): + if isinstance(node, ast.ImportFrom) and node.level: + target = _scan.resolve_import(relpath, node.module, node.level) + if target is not None and target[:1] == (_scan.CONTRACT_FOLDER,): + continue + bad.append(f"{relpath}:{node.lineno} relative import outside contract/") + continue + names = [a.name for a in node.names] if isinstance(node, ast.Import) else [node.module or ""] + for name in names: + target = _scan.resolve_import(relpath, name, 0) + if target is not None and target[:1] == (_scan.CONTRACT_FOLDER,): + continue + allowed = _STDLIB | ( + _CONTRACT_TESTING_EXTRA if relpath.startswith("contract/testing/") else frozenset() + ) + if name.split(".")[0] not in allowed or target is not None: + bad.append(f"{relpath}:{node.lineno} imports {name}") + return bad + + +def _collect_core_to_harness_imports() -> Counter: + found: Counter = Counter() + for path in sorted(_scan.CORE_DIR.rglob("*.py")): + text = path.read_text(encoding="utf-8") + if "harness" not in text: + continue + relpath = path.relative_to(_scan.CORE_DIR).as_posix() + package = ("openjiuwen", "core") + tuple(path.relative_to(_scan.CORE_DIR).parent.parts) + for node, _lazy in _scan.iter_imports(_scan.parse_source(text, str(path))): + if isinstance(node, ast.Import): + modules = [alias.name for alias in node.names] + elif node.level: + base = package[: len(package) - (node.level - 1)] + modules = [".".join(base + ((node.module,) if node.module else ()))] + else: + modules = [node.module or ""] + for module in modules: + if module == "openjiuwen.harness" or module.startswith("openjiuwen.harness."): + found[(relpath, module)] += 1 + return found + + +def _outside_modules(node: ast.stmt) -> list[str]: + if isinstance(node, ast.Import): + return [alias.name for alias in node.names] + return [node.module or ""] if not node.level else [] + + +def _imports_harness_outside_core(module: str) -> bool: + return module.startswith("openjiuwen.") and not module.startswith(("openjiuwen.core.", _scan.PKG_DOTTED)) + + +def _driver_may_import(driver: str, target: tuple[str, ...]) -> bool: + if target[:1] == (_scan.CONTRACT_FOLDER,) or target[:2] == ("drivers", driver): + return True + if target == ("drivers", "middleware"): + return True + return ( + target[:1] == (_scan.SURFACES_FOLDER,) and len(target) >= 3 and target[2] in SURFACE_MODULES_A_DRIVER_MAY_IMPORT + ) + + +def _collect_driver_imports_outside_their_layer() -> Counter: + found: Counter = Counter() + for relpath in _scan.files_where(lambda f: f.startswith("drivers/") and f.count("/") >= 2): + driver = relpath.split("/")[1] + for node, _lazy in _scan.iter_imports(_scan.parse(relpath)): + for target in _import_targets(relpath, node): + if not _driver_may_import(driver, target): + found[(relpath, ".".join(target))] += 1 + for module in _outside_modules(node): + if _imports_harness_outside_core(module): + found[(relpath, module)] += 1 + return found + + +def _collect_provisioning_imports_outside_the_contract() -> list[str]: + bad = [] + for relpath in _scan.files_where(lambda f: _scan.in_folder(f, "provisioning")): + for node, _lazy in _scan.iter_imports(_scan.parse(relpath)): + for target in _import_targets(relpath, node): + if target[:1] not in ((_scan.CONTRACT_FOLDER,), ("provisioning",)): + bad.append(f"{relpath}:{node.lineno} imports {'.'.join(target)}") + for module in _outside_modules(node): + if _imports_harness_outside_core(module): + bad.append(f"{relpath}:{node.lineno} imports {module}") + return bad + + +def _absolute_modules(node: ast.stmt, package: tuple[str, ...]) -> list[str]: + if isinstance(node, ast.Import): + return [alias.name for alias in node.names] + if not node.level: + return [node.module or ""] + base = package[: len(package) - (node.level - 1)] + return [".".join(base + ((node.module,) if node.module else ()))] + + +def _concrete_modules_imported_by(module: str) -> list[str]: + """Concrete driver, surface or provisioner modules that a module outside the package imports itself.""" + path = _scan.REPO_ROOT.joinpath(*module.split(".")) + source = path.with_suffix(".py") if path.with_suffix(".py").is_file() else path / "__init__.py" + if not source.is_file(): + return [] + package = tuple(module.split(".")) if source.name == "__init__.py" else tuple(module.split(".")[:-1]) + reached = [] + for node, _lazy in _scan.iter_imports(_scan.parse_source(source.read_text(encoding="utf-8"), str(source))): + for name in _absolute_modules(node, package): + target = _scan.resolve_import("", name, 0) + if target is not None and _concrete_kind(target): + reached.append(".".join(target)) + return reached + + +def _collect_core_detours_into_concrete_modules() -> Counter: + found: Counter = Counter() + for relpath in _scan.files_where(_scan.is_core): + for node, _lazy in _scan.iter_imports(_scan.parse(relpath)): + for module in _outside_modules(node): + if module.startswith("openjiuwen.") and _scan.resolve_import(relpath, module, 0) is None: + if _concrete_modules_imported_by(module): + found[(relpath, module)] += 1 + return found + + +def _collect_imports_of_the_driver_layer() -> list[str]: + bad = [] + for relpath in _scan.files_where(lambda f: _scan.in_folder(f, "runtime") or _scan.is_surface(f)): + if relpath in DRIVER_LAYER_IMPORTERS: + continue + for node, lazy in _scan.iter_imports(_scan.parse(relpath)): + if lazy and relpath in LAZY_DRIVER_LAYER_IMPORTERS: + continue + for target in _import_targets(relpath, node): + if target[:1] == ("drivers",): + bad.append(f"{relpath}:{node.lineno} imports {'.'.join(target)}") + return bad + + +class TestImportBoundaries: + def test_contract_imports_only_stdlib_and_itself(self): + bad = _collect_contract_imports() + assert not bad, "contract/ must import only the standard library and contract/:\n" + "\n".join(bad) + + def test_core_never_imports_concrete_driver_surface_or_provisioner(self): + problems = _scan.compare_with_allowlist( + _collect_core_concrete_imports(), + KNOWN_CORE_IMPORTS_OF_CONCRETE_MODULES, + "core import of a concrete driver/surface/provisioner", + ) + assert not problems, "\n".join(problems) + + def test_core_never_reaches_a_concrete_module_through_a_module_outside_the_package(self): + problems = _scan.compare_with_allowlist( + _collect_core_detours_into_concrete_modules(), + KNOWN_CORE_DETOURS_INTO_CONCRETE_MODULES, + "core import of an outside module that imports a concrete driver/surface/provisioner", + ) + assert not problems, "\n".join(problems) + + def test_core_reaches_the_registries_from_the_composition_root_only(self): + bad = _collect_core_imports_of_registries() + message = "the registries are imported by runtime/assembly.py (surfaces also by config.py) only:\n" + assert not bad, message + "\n".join(bad) + + def test_surfaces_never_import_a_driver_or_provisioner(self): + bad = _collect_surface_imports_of_drivers_or_provisioners() + assert not bad, ( + "surfaces/ may import contract/ and runtime/ only (the settings facade a driver config, lazily):\n" + + "\n".join(bad) + ) + + def test_openjiuwen_core_never_imports_harness(self): + problems = _scan.compare_with_allowlist( + _collect_core_to_harness_imports(), + KNOWN_CORE_TO_HARNESS_IMPORTS, + "openjiuwen.core import of openjiuwen.harness", + ) + assert not problems, "\n".join(problems) + + def test_drivers_import_only_the_contract_their_package_and_surface_commands_and_contexts(self): + problems = _scan.compare_with_allowlist( + _collect_driver_imports_outside_their_layer(), + KNOWN_DRIVER_IMPORTS_OUTSIDE_THEIR_LAYER, + "driver import outside contract/, its own package and surface commands/contexts", + ) + assert not problems, "\n".join(problems) + + def test_provisioning_imports_only_the_contract(self): + bad = _collect_provisioning_imports_outside_the_contract() + assert not bad, "provisioning/ may import contract/ only:\n" + "\n".join(bad) + + def test_runtime_and_surfaces_reach_drivers_from_the_composition_root_only(self): + bad = _collect_imports_of_the_driver_layer() + message = "only runtime/assembly.py imports drivers/ (the settings facade lazily, for the driver config):\n" + assert not bad, message + "\n".join(bad) + + def test_every_module_at_the_package_root_is_known(self): + stray = sorted(f for f in _scan.pkg_files() if "/" not in f and f not in ROOT_MODULES) + assert not stray, f"modules at the package root belong to no layer; move them: {stray}" diff --git a/tests/unit_tests/harness/tools/browser_move/architecture/test_module_size.py b/tests/unit_tests/harness/tools/browser_move/architecture/test_module_size.py new file mode 100644 index 0000000000..91445464c7 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/architecture/test_module_size.py @@ -0,0 +1,31 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""No module in the core or in ``surfaces/`` longer than 1,500 lines; allowlisted ones may only shrink.""" + +from __future__ import annotations + +from collections import Counter + +from . import _scan + +MAX_LINES = 1500 + +KNOWN_OVERSIZED_MODULES: dict[str, int] = {} + + +def _collect_oversized_modules() -> Counter: + found: Counter = Counter() + for relpath in _scan.files_where(lambda f: _scan.is_core(f) or _scan.is_surface(f)): + lines = len(_scan.read_text(relpath).splitlines()) + if lines > MAX_LINES: + found[relpath] = lines + return found + + +class TestModuleSize: + def test_no_oversized_modules_in_core_or_surfaces(self): + problems = _scan.compare_with_allowlist( + _collect_oversized_modules(), KNOWN_OVERSIZED_MODULES, f"module over {MAX_LINES} lines" + ) + assert not problems, "\n".join(problems) diff --git a/tests/unit_tests/harness/tools/browser_move/command_tools.py b/tests/unit_tests/harness/tools/browser_move/command_tools.py new file mode 100644 index 0000000000..a05bc3c0f8 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/command_tools.py @@ -0,0 +1,91 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Browser tools as ``CommandTool`` over ``FakeDriver``, with the Playwright MCP names and arguments.""" + +from __future__ import annotations + +from collections.abc import Iterable +from typing import Any + +from openjiuwen.core.foundation.llm import ToolMessage +from openjiuwen.core.foundation.tool import Tool +from openjiuwen.core.foundation.tool.schema import ToolOutput +from openjiuwen.harness.tools.browser_move.contract import ( + ActionResult, + Click, + ContextRef, + DriverInfo, + Hover, + Observation, + Observe, + PressKey, + RunScript, + SurfaceProfile, + ToolSpec, + TypeText, +) +from openjiuwen.harness.tools.browser_move.contract.testing import FakeDriver +from openjiuwen.harness.tools.browser_move.contract.testing.fake import DEFAULT_ACTIONS +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.tools import DECODERS +from openjiuwen.harness.tools.browser_move.provisioning import ProvisionedDevice +from openjiuwen.harness.tools.browser_move.runtime.assembly import DriverAssembly +from openjiuwen.harness.tools.browser_move.runtime.driver_session import DriverSession +from openjiuwen.harness.tools.browser_move.runtime.tool_factory import ToolFactory +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import Navigate +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext +from openjiuwen.harness.tools.browser_move.surfaces.browser.page_state import BrowserPageState +from openjiuwen.harness.tools.browser_move.surfaces.browser.renderer import BrowserRenderer + +COMMANDS = { + "browser_click": (Click,), + "browser_evaluate": (RunScript,), + "browser_hover": (Hover,), + "browser_navigate": (Navigate,), + "browser_press_key": (PressKey,), + "browser_snapshot": (Observe,), + "browser_type": (TypeText,), +} +HOME = ContextRef("0", "tab", "Home") +SURFACE = BrowserContext( + url="https://www.allrecipes.com/", + title="Home", + tabs=(HOME,), + tab_urls=("https://www.allrecipes.com/",), + current_tab=0, +) +_DEVICE = ProvisionedDevice(kind="fake-device", endpoint="fake://1", extras={}) +_MESSAGES = {"driver_restarted": "restarted; observe again", "driver_lost": "the device is gone"} + + +def fake_driver(results: Iterable[ActionResult] = (), observations: Iterable[Observation] = ()) -> FakeDriver: + info = DriverInfo(name="fake", surfaces=frozenset({"browser"}), actions={**DEFAULT_ACTIONS, Navigate: frozenset()}) + return FakeDriver(observations, results, info=info) + + +def command_tools(driver: FakeDriver) -> dict[str, Tool]: + specs = tuple( + ToolSpec(name, commands, name, schema={"type": "object"}, build=DECODERS[name]) + for name, commands in COMMANDS.items() + ) + profile = SurfaceProfile( + name="browser", + commands=(), + renderer=BrowserRenderer(), + tool_specs=specs, + capability_tools={"core": tuple(COMMANDS)}, + ) + page_state = BrowserPageState(page_id="page-test") + session = DriverSession(DriverAssembly(driver, _DEVICE, None), page_state, _MESSAGES) + built = ToolFactory(profile, driver.info).build( + ("core",), image_support=False, session=session, page_state=page_state + ) + return {tool.card.name: tool for tool in built.tools} + + +def result(value: Any = None, *, detail: str | None = None) -> ActionResult: + return ActionResult(ok=True, value=value, detail=detail, surface=SURFACE) + + +def tool_message(tool: Tool, output: ToolOutput, call_id: str) -> ToolMessage: + """The message the ability manager builds from ``output``.""" + return ToolMessage(content=tool.render_for_llm(output), tool_call_id=call_id, name=tool.card.name) diff --git a/tests/unit_tests/harness/tools/browser_move/conftest.py b/tests/unit_tests/harness/tools/browser_move/conftest.py new file mode 100644 index 0000000000..52055e9174 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/conftest.py @@ -0,0 +1,32 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. + +import os +import tempfile +from pathlib import Path +from unittest.mock import AsyncMock + +import pytest + +os.environ.setdefault( + "OPENJIUWEN_BROWSER_AGENT_LOG_FILE", str(Path(tempfile.mkdtemp(prefix="browser-agent-log-")) / "browser_agent.log") +) + +from openjiuwen.harness.tools.browser_move.runtime import topology # noqa: E402 + +_TEST_ROOT = Path(__file__).parent +_SCAFFOLDING = ("AGENT.md", "SOUL.md") + + +@pytest.fixture(autouse=True) +def _no_image_probe(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(topology, "probe_image_support", AsyncMock(return_value=None)) + + +def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None: + stray = sorted(str(path.relative_to(_TEST_ROOT)) for name in _SCAFFOLDING for path in _TEST_ROOT.rglob(name)) + if stray: + session.exitstatus = pytest.ExitCode.TESTS_FAILED + reporter = session.config.pluginmanager.get_plugin("terminalreporter") + if reporter is not None: + reporter.write_line(f"agent workspace files written into the test tree: {stray}", red=True) diff --git a/tests/unit_tests/harness/tools/browser_move/contract/__init__.py b/tests/unit_tests/harness/tools/browser_move/contract/__init__.py new file mode 100644 index 0000000000..57d631c3f0 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/__init__.py @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/tests/unit_tests/harness/tools/browser_move/contract/_lazy_target.py b/tests/unit_tests/harness/tools/browser_move/contract/_lazy_target.py new file mode 100644 index 0000000000..913db256db --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/_lazy_target.py @@ -0,0 +1,16 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Imported only by the lazy-import registry test, through an import string.""" + +CREATED: list[tuple] = [] + + +def make_thing(*args, **kwargs): + CREATED.append((args, kwargs)) + return ("thing", args, kwargs) + + +class Nested: + @staticmethod + def factory(): + return "nested" diff --git a/tests/unit_tests/harness/tools/browser_move/contract/test_conformance_fake.py b/tests/unit_tests/harness/tools/browser_move/contract/test_conformance_fake.py new file mode 100644 index 0000000000..bf77b39148 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/test_conformance_fake.py @@ -0,0 +1,544 @@ +#!/usr/bin/env python +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Self-test of the conformance suite on ``FakeDriver``; each deliberately wrong driver makes its case fail.""" + +from __future__ import annotations + +import asyncio +import dataclasses +import logging + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionIntercepted, + ActionResult, + ActionTimeout, + Click, + CloseContext, + ContextRef, + Drag, + DriverError, + DriverInfo, + DriverRef, + Element, + ElementNotFound, + HandleDialog, + Hover, + IndexRef, + Observation, + Observe, + PointRef, + PressKey, + Ref, + RunScript, + Scene, + SceneCase, + Scroll, + StaleRef, + SurfaceContext, + SwitchContext, + TypeText, + Unsupported, + Wait, + ensure_supported, +) +from openjiuwen.harness.tools.browser_move.contract.testing import FakeDriver +from openjiuwen.harness.tools.browser_move.contract.testing.conformance import ( + SECRET_TEXT, + DriverConformance, + TraceSink, + check_context_change, + check_declared_actions, + check_fingerprint, + check_happy_path, + check_intercepted, + check_sensitive_not_logged, + check_sensitive_not_traced, + check_stale_ref, + check_surface, + check_timeout, + check_unhandled, +) + +pytest_plugins = ["openjiuwen.harness.tools.browser_move.contract.testing.conformance"] + +HOME = ("Save", "Name", "Covered", "Next", "Alert", "Popup", "Box", "Target") +NEXT = ("Home",) +ROW = 10.0 + + +@dataclasses.dataclass(frozen=True) +class FakeScreen(SurfaceContext): + dialog_open: bool = False + + +class FakeWorld: + """What a ``WorldDriver`` acts on. Element ``i`` of a document is ``e`` at ``y = 10 * i``.""" + + def __init__(self) -> None: + self.documents = {"c1": "home"} + self.loads = {"c1": 1} + self.current = "c1" + self.title = "" + self.late_waits = 0 + self.dialog_open = False + self._opened = 1 + + def generation(self) -> str: + return f"{self.current}.{self.loads[self.current]}" + + def names(self) -> tuple[str, ...]: + return HOME if self.documents[self.current] == "home" else NEXT + + def observe(self) -> Observation: + generation = self.generation() + elements = tuple( + Element(ref=DriverRef(f"e{i}", generation), role="button", name=name, bounds=(0.0, ROW * i, 100.0, ROW)) + for i, name in enumerate(self.names(), start=1) + ) + return Observation( + context=ContextRef(self.current, "screen", self.title), + elements=elements, + surface=FakeScreen(dialog_open=self.dialog_open), + ) + + def resolve(self, ref: Ref) -> str: + if isinstance(ref, DriverRef): + if ref.generation != self.generation(): + raise StaleRef(f"{ref.ref} belongs to {ref.generation}", dispatched=False) + index = int(ref.ref[1:]) + elif isinstance(ref, PointRef): + index = int(ref.y // ROW) + else: + raise ElementNotFound(f"cannot resolve {ref!r}", dispatched=False) + if not 1 <= index <= len(self.names()): + raise ElementNotFound(f"no element {index}", dispatched=False) + return self.names()[index - 1] + + def _load(self, document: str) -> ActionResult: + self.documents[self.current] = document + self.loads[self.current] += 1 + return ActionResult(ok=True, context_changed=True) + + def click(self, name: str) -> ActionResult: + if name == "Covered": + raise ActionIntercepted("another element would receive the click", by="overlay", dispatched=False) + if name == "Next": + return self._load("next") + if name == "Home": + return self._load("home") + if name == "Alert": + self.dialog_open = True + return ActionResult(ok=True, context_changed=True) + if name == "Popup": + self._opened += 1 + source = ContextRef(self.current, "screen") + self.current = f"c{self._opened}" + self.documents[self.current] = "next" + self.loads[self.current] = 1 + opened = (ContextRef(self.current, "screen"),) + return ActionResult(ok=True, context_changed=True, source_context=source, opened_contexts=opened) + self.title = "clicked" if name == "Save" else f"clicked:{name}" + return ActionResult(ok=True) + + def wait(self, action: Wait) -> ActionResult: + """A text that is not on the screen appears late: the first wait for it times out, a second one finds it.""" + if action.until_text is not None and action.until_text not in self.names(): + self.late_waits += 1 + if self.late_waits == 1: + raise ActionTimeout(f"{action.until_text!r} did not appear in {action.seconds}s", dispatched=True) + return ActionResult(ok=True) + + def apply(self, action: Action) -> ActionResult: + if isinstance(action, Wait): + return self.wait(action) + if isinstance(action, Click): + return self.click(self.resolve(action.target)) + if isinstance(action, Drag): + self.resolve(action.source) + self.resolve(action.destination) + self.title = "dragged" + elif isinstance(action, (TypeText, PressKey, Scroll, Hover)) and action.target is not None: + self.resolve(action.target) + if isinstance(action, TypeText): + self.title = f"typed:{len(action.text)}" + elif isinstance(action, PressKey): + self.title = f"key:{action.key}" + elif isinstance(action, Scroll): + self.title = "scrolled" + elif isinstance(action, Hover): + self.title = "hovered" + elif isinstance(action, HandleDialog): + self.dialog_open = False + self.title = f"dialog:{action.accept}" + elif isinstance(action, RunScript): + return ActionResult(ok=True, value="ran") + elif isinstance(action, SwitchContext): + self.current = action.context.id + return ActionResult(ok=True, context_changed=True) + elif isinstance(action, CloseContext): + del self.documents[action.context.id] + self.current = next(iter(self.documents)) + return ActionResult(ok=True, context_changed=True) + return ActionResult(ok=True) + + +FAKE_ACTIONS: dict[type[Action], frozenset[type]] = { + Click: frozenset({DriverRef, PointRef, IndexRef}), + TypeText: frozenset({DriverRef}), + PressKey: frozenset({DriverRef}), + Scroll: frozenset({DriverRef}), + Hover: frozenset({DriverRef}), + Drag: frozenset({DriverRef}), + Wait: frozenset(), + SwitchContext: frozenset(), + CloseContext: frozenset(), + HandleDialog: frozenset(), + RunScript: frozenset(), +} +FAKE_INFO = DriverInfo(name="fake-world", surfaces=frozenset({"fake"}), actions=FAKE_ACTIONS) + + +class WorldDriver(FakeDriver): + """A correct driver: a ``FakeDriver`` whose scripted answers come from a ``FakeWorld``, with a masking trace.""" + + def __init__(self, sink: TraceSink, info: DriverInfo = FAKE_INFO) -> None: + super().__init__(info=info) + self.world = FakeWorld() + self.sink = sink + + async def observe(self, command: Observe) -> Observation: + self.script(observations=(self.world.observe(),)) + return await super().observe(command) + + async def execute(self, action: Action) -> ActionResult: + ensure_supported(self.info.name, self.info.actions, action) + self.trace(action) + try: + self.script(results=(self.world.apply(action),)) + except DriverError as error: + self.fail_next(error) + return await super().execute(action) + + def trace(self, action: Action) -> None: + if isinstance(action, TypeText) and action.sensitive: + action = dataclasses.replace(action, text="***") + elif isinstance(action, RunScript) and action.sensitive: + action = dataclasses.replace(action, code="***") + self.sink(action) + + +def _element(observation: Observation, name: str) -> Element: + for element in observation.elements: + if element.name == name: + return element + raise LookupError(f"no element {name!r}") + + +def _ref(observation: Observation, name: str, kind: type = DriverRef) -> Ref: + element = _element(observation, name) + if kind is PointRef: + x, y, width, height = element.bounds + return PointRef(x=int(x + width / 2), y=int(y + height / 2), generation=observation.generation) + return element.ref + + +def _title_is(expected: str): + return lambda before, after, result: after.context.title == expected + + +def _context_moved(before: Observation, after: Observation, result: ActionResult) -> bool: + return after.context.id != before.context.id + + +def _shows_dialog(before: Observation, after: Observation, result: ActionResult) -> bool: + return isinstance(after.surface, FakeScreen) and after.surface.dialog_open + + +def _click(name: str): + return lambda seen: Click(target=_ref(seen[-1], name)) + + +def _happy(command: type[Action], kind: type | None, actions) -> SceneCase | str: + if kind is IndexRef: + return "an IndexRef is internal to the driver" + cases = { + Click: lambda: SceneCase( + steps=(lambda seen: Click(target=_ref(seen[-1], "Save", kind)),), check=_title_is("clicked") + ), + TypeText: lambda: SceneCase( + steps=(lambda seen: TypeText(target=_ref(seen[-1], "Name"), text="Ada"),), check=_title_is("typed:3") + ), + PressKey: lambda: SceneCase( + steps=(lambda seen: PressKey(key="Enter", target=_ref(seen[-1], "Name")),), check=_title_is("key:Enter") + ), + Scroll: lambda: SceneCase( + steps=(lambda seen: Scroll(target=_ref(seen[-1], "Box"), dy=100),), check=_title_is("scrolled") + ), + Hover: lambda: SceneCase(steps=(lambda seen: Hover(target=_ref(seen[-1], "Box")),), check=_title_is("hovered")), + Drag: lambda: SceneCase( + steps=(lambda seen: Drag(source=_ref(seen[-1], "Box"), destination=_ref(seen[-1], "Target")),), + check=_title_is("dragged"), + ), + Wait: lambda: SceneCase(steps=(lambda seen: Wait(seconds=0),)), + SwitchContext: lambda: SceneCase( + steps=(_click("Popup"), lambda seen: SwitchContext(context=seen[0].context)), check=_context_moved + ), + CloseContext: lambda: SceneCase( + steps=(_click("Popup"), lambda seen: CloseContext(context=seen[-1].context)), check=_context_moved + ), + HandleDialog: lambda: SceneCase( + steps=(_click("Alert"), lambda seen: HandleDialog(accept=True)), + check=_title_is("dialog:True"), + observe_between=False, + ), + RunScript: lambda: SceneCase( + steps=(lambda seen: RunScript(code="noop"),), check=lambda before, after, result: result.value == "ran" + ), + } + build = cases.get(command) + return build() if build is not None else f"no happy path for {command.__name__}" + + +def _scene(**overrides) -> Scene: + fields = dict( + surface="fake", + start=lambda: None, + stop=lambda: None, + happy=_happy, + stale=lambda actions: SceneCase( + steps=(_click("Next"), lambda seen: Click(target=_element(seen[0], "Save").ref)) + ), + intercepted=lambda actions: SceneCase(steps=(_click("Covered"),)), + timeout=lambda actions: SceneCase(steps=(lambda seen: Wait(until_text="Late", seconds=0.1),)), + context_changes={ + "new-document": lambda actions: SceneCase(steps=(_click("Next"),)), + "dialog": lambda actions: SceneCase( + steps=(_click("Alert"),), + check=_shows_dialog, + observe_between=False, + cleanup=(lambda seen: HandleDialog(accept=False),), + ), + "opened": lambda actions: SceneCase(steps=(_click("Popup"),)), + "switched": lambda actions: SceneCase( + steps=(_click("Popup"), lambda seen: SwitchContext(context=seen[0].context)) + ), + "closed": lambda actions: SceneCase( + steps=(_click("Popup"), lambda seen: CloseContext(context=seen[-1].context)) + ), + }, + sensitive={ + "script": lambda actions, secret: SceneCase(steps=(lambda seen: RunScript(code=secret, sensitive=True),)), + }, + ) + fields.update(overrides) + return Scene(**fields) + + +FAKE_SCENE = _scene() + + +class TestConformanceOnFakeDriver(DriverConformance): + scene = FAKE_SCENE + driver_actions = FAKE_ACTIONS + + @pytest.fixture + def conformance_driver_factory(self): + return WorldDriver + + +class RetriesDispatchedAction(WorldDriver): + async def execute(self, action: Action) -> ActionResult: + try: + return await super().execute(action) + except ActionTimeout: + return await super().execute(action) + + +class StaleAfterDispatch(WorldDriver): + async def execute(self, action: Action) -> ActionResult: + try: + return await super().execute(action) + except StaleRef as error: + raise StaleRef(str(error), dispatched=True) from error + + +class NoContextSignal(WorldDriver): + async def execute(self, action: Action) -> ActionResult: + return dataclasses.replace(await super().execute(action), context_changed=False) + + +class DialogNotObserved(WorldDriver): + async def observe(self, command: Observe) -> Observation: + return dataclasses.replace(await super().observe(command), surface=None) + + +class InterceptedAfterDispatch(WorldDriver): + async def execute(self, action: Action) -> ActionResult: + try: + return await super().execute(action) + except ActionIntercepted as error: + raise ActionIntercepted(str(error), by=error.by, dispatched=True) from error + + +class AnonymousIntercept(WorldDriver): + async def execute(self, action: Action) -> ActionResult: + try: + return await super().execute(action) + except ActionIntercepted as error: + raise ActionIntercepted(str(error), by="", dispatched=error.dispatched) from error + + +class UnsupportedAsGenericError(WorldDriver): + async def execute(self, action: Action) -> ActionResult: + try: + return await super().execute(action) + except Unsupported as error: + raise DriverError(str(error)) from error + + +class UnstableFingerprint(WorldDriver): + async def observe(self, command: Observe) -> Observation: + observation = await super().observe(command) + count = len(self.calls) + elements = tuple(dataclasses.replace(e, name=f"{e.name} {count}") for e in observation.elements) + return dataclasses.replace(observation, elements=elements) + + +class LeakyTrace(WorldDriver): + def trace(self, action: Action) -> None: + self.sink(action) + + +_LEAKY_LOGGER = logging.getLogger("conformance.selftest.leaky") +_LEAKY_LOGGER.propagate = False + + +class LeakyLog(WorldDriver): + async def execute(self, action: Action) -> ActionResult: + if isinstance(action, TypeText): + _LEAKY_LOGGER.debug("typing %s", action.text) + elif isinstance(action, RunScript): + _LEAKY_LOGGER.debug("running %s", action.code) + return await super().execute(action) + + +class ClickWithoutEffect(WorldDriver): + async def execute(self, action: Action) -> ActionResult: + if isinstance(action, Click) and isinstance(action.target, DriverRef): + ensure_supported(self.info.name, self.info.actions, action) + return ActionResult(ok=True) + return await super().execute(action) + + +def _driver_with(info: DriverInfo): + return lambda sink: WorldDriver(sink, info=info) + + +def _run(check, factory, *args) -> None: + asyncio.run(check(factory, FAKE_SCENE, *args)) + + +@pytest.mark.parametrize( + "check, factory, args, message", + [ + pytest.param(check_timeout, RetriesDispatchedAction, (), "expected ActionTimeout", id="retries-dispatched"), + pytest.param(check_stale_ref, StaleAfterDispatch, (), "before dispatch", id="stale-ref-dispatched"), + pytest.param( + check_context_change, NoContextSignal, ("new-document",), "did not report context_changed", id="no-signal" + ), + pytest.param( + check_context_change, DialogNotObserved, ("dialog",), "fails the scene's check", id="dialog-not-observed" + ), + pytest.param(check_intercepted, AnonymousIntercept, (), "must describe the element", id="intercept-no-by"), + pytest.param( + check_intercepted, InterceptedAfterDispatch, (), "never reached the target", id="intercept-dispatched" + ), + pytest.param(check_unhandled, UnsupportedAsGenericError, (), "expected Unsupported", id="unhandled-generic"), + pytest.param(check_fingerprint, UnstableFingerprint, (), "fingerprint changed", id="unstable-fingerprint"), + pytest.param(check_sensitive_not_traced, LeakyTrace, (), "reached the trace", id="leaky-trace"), + pytest.param(check_sensitive_not_traced, LeakyTrace, ("script",), "reached the trace", id="leaky-trace-script"), + pytest.param(check_sensitive_not_logged, LeakyLog, (), "reached the logs", id="leaky-log"), + pytest.param(check_sensitive_not_logged, LeakyLog, ("script",), "reached the logs", id="leaky-log-script"), + pytest.param(check_happy_path, ClickWithoutEffect, (Click, DriverRef), "did not take effect", id="no-effect"), + pytest.param( + check_surface, + _driver_with(dataclasses.replace(FAKE_INFO, surfaces=frozenset({"other"}))), + (), + "not the scene's 'fake'", + id="wrong-surface", + ), + pytest.param( + check_declared_actions, + _driver_with(dataclasses.replace(FAKE_INFO, actions={Click: frozenset({DriverRef})})), + (FAKE_ACTIONS,), + "differs from driver_actions", + id="drifted-actions", + ), + ], +) +def test_case_fails_for_a_wrong_driver(check, factory, args, message): + with pytest.raises(AssertionError, match=message): + _run(check, factory, *args) + + +def test_retrying_driver_really_repeats_the_dispatched_action(): + drivers: list[RetriesDispatchedAction] = [] + + def factory(sink: TraceSink) -> RetriesDispatchedAction: + drivers.append(RetriesDispatchedAction(sink)) + return drivers[-1] + + with pytest.raises(AssertionError, match="expected ActionTimeout"): + _run(check_timeout, factory) + assert drivers[0].world.late_waits == 2 + + +def test_a_timeout_case_that_is_not_a_bounded_wait_for_text_fails(): + scene = _scene(timeout=lambda actions: SceneCase(steps=(lambda seen: Wait(until_text="Late"),))) + with pytest.raises(AssertionError, match=r"must end in Wait\(until_text=..., seconds=...\)"): + asyncio.run(check_timeout(WorldDriver, scene)) + + +def test_a_sensitive_case_that_does_not_send_the_secret_fails(): + scene = _scene(sensitive={"script": lambda actions, secret: SceneCase(steps=(lambda seen: RunScript(code="x"),))}) + with pytest.raises(AssertionError, match="does not send the secret"): + asyncio.run(check_sensitive_not_logged(WorldDriver, scene, "script")) + + +def test_the_correct_driver_keeps_the_secret_out_of_a_non_empty_trace(): + sinks: list[TraceSink] = [] + + def factory(sink: TraceSink) -> WorldDriver: + sinks.append(sink) + return WorldDriver(sink) + + _run(check_sensitive_not_traced, factory) + assert sinks[0].records + assert SECRET_TEXT not in sinks[0].text + + +def test_a_case_the_scene_cannot_provide_is_skipped_with_its_reason(): + with pytest.raises(pytest.skip.Exception, match="IndexRef is internal to the driver"): + _run(check_happy_path, WorldDriver, Click, IndexRef) + + +def test_a_scene_without_a_case_skips_with_the_default_reason(): + bare = Scene(surface="fake", start=lambda: None, stop=lambda: None) + with pytest.raises(pytest.skip.Exception, match="does not provide this case"): + asyncio.run(check_stale_ref(WorldDriver, bare)) + + +def test_a_case_needing_an_unhandled_setup_command_is_skipped_naming_it(): + without_click = dataclasses.replace(FAKE_INFO, actions={HandleDialog: frozenset()}) + with pytest.raises(pytest.skip.Exception, match="does not handle Click"): + _run(check_happy_path, _driver_with(without_click), HandleDialog, None) + + +def test_a_scene_built_with_the_wrong_ref_kind_fails(): + scene = _scene(happy=lambda command, kind, actions: _happy(command, DriverRef, actions)) + with pytest.raises(AssertionError, match="for ref kind PointRef"): + asyncio.run(check_happy_path(WorldDriver, scene, Click, PointRef)) diff --git a/tests/unit_tests/harness/tools/browser_move/contract/test_driver.py b/tests/unit_tests/harness/tools/browser_move/contract/test_driver.py new file mode 100644 index 0000000000..5bdf14492c --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/test_driver.py @@ -0,0 +1,219 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""``HandlerDriver`` dispatches by command type, derives ``info.actions`` from ``@handles`` and refuses the rest.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import Any, ClassVar + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + Click, + ContextRef, + Driver, + DriverInfo, + DriverRef, + Effect, + HandlerDriver, + Health, + Hover, + IndexRef, + Locator, + Observation, + Observe, + PointRef, + PressKey, + TypeText, + Unsupported, + handles, + iter_refs, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import FieldKind, FillForm, FormField, Navigate + +E1 = DriverRef("e1", "g1") +P1 = PointRef(10, 20, "g1") + + +@dataclass(frozen=True) +class Poke(Action): + """A command defined outside ``contract/`` (like a surface-owned one).""" + + effect: ClassVar[Effect] = Effect.ACT + + target: PointRef + + +class _Base(HandlerDriver): + def __init__(self) -> None: + self.handled: list[tuple[str, Action]] = [] + + def describe(self) -> DriverInfo: + return DriverInfo( + name="toy", surfaces=frozenset({"toy"}), versions={"toy": "1"}, observes=frozenset({"elements"}) + ) + + async def start(self, config: Any = None) -> None: + return None + + async def stop(self) -> None: + return None + + async def health(self) -> Health: + return Health(ok=True) + + async def observe(self, command: Observe) -> Observation: + return Observation(context=ContextRef("c1", "window")) + + def _done(self, name: str, action: Action) -> ActionResult: + self.handled.append((name, action)) + return ActionResult(ok=True, value=name) + + +class ToyDriver(_Base): + @handles(Click, refs={DriverRef, PointRef}) + async def click(self, action: Click) -> ActionResult: + return self._done("click", action) + + @handles(Poke, refs={PointRef}) + async def poke(self, action: Poke) -> ActionResult: + return self._done("poke", action) + + @handles(Navigate) + async def navigate(self, action: Navigate) -> ActionResult: + return self._done("navigate", action) + + @handles(PressKey) + @handles(Hover, refs={DriverRef}) + async def key_or_hover(self, action: Action) -> ActionResult: + return self._done("key_or_hover", action) + + @handles(FillForm, refs={DriverRef}) + async def fill(self, action: FillForm) -> ActionResult: + return self._done("fill", action) + + +def _run(coro): + return asyncio.run(coro) + + +def test_handler_driver_is_a_driver() -> None: + assert isinstance(ToyDriver(), Driver) + + +def test_info_actions_are_derived_from_the_marks() -> None: + info = ToyDriver().info + + assert dict(info.actions) == { + Click: frozenset({DriverRef, PointRef}), + Poke: frozenset({PointRef}), + Navigate: frozenset(), + PressKey: frozenset(), + Hover: frozenset({DriverRef}), + FillForm: frozenset({DriverRef}), + } + assert (info.name, info.surfaces, dict(info.versions)) == ("toy", frozenset({"toy"}), {"toy": "1"}) + assert info.observes == frozenset({"elements"}) + + +def test_info_actions_set_in_describe_are_replaced_by_the_marks() -> None: + class Liar(ToyDriver): + def describe(self) -> DriverInfo: + return DriverInfo(name="liar", surfaces=frozenset({"toy"}), actions={TypeText: frozenset({DriverRef})}) + + assert TypeText not in Liar().info.actions + assert Click in Liar().info.actions + + +@pytest.mark.parametrize( + ("action", "handler"), + [ + (Click(E1), "click"), + (Click(P1), "click"), + (Poke(P1), "poke"), + (Navigate("https://example.test/"), "navigate"), + (PressKey("Enter"), "key_or_hover"), + (Hover(E1), "key_or_hover"), + (FillForm((FormField(E1, FieldKind.TEXTBOX, "x"),)), "fill"), + ], + ids=lambda v: v if isinstance(v, str) else type(v).__name__, +) +def test_execute_dispatches_by_command_type(action: Action, handler: str) -> None: + driver = ToyDriver() + + result = _run(driver.execute(action)) + + assert result == ActionResult(ok=True, value=handler) + assert driver.handled == [(handler, action)] + + +def test_unhandled_command_is_unsupported_and_not_dispatched() -> None: + driver = ToyDriver() + + with pytest.raises(Unsupported, match="ToyDriver does not handle TypeText") as raised: + _run(driver.execute(TypeText(E1, "secret"))) + + assert raised.value.dispatched is False + assert driver.handled == [] + assert TypeText not in driver.info.actions + + +@pytest.mark.parametrize( + "action", + [ + Click(IndexRef(1, "g1")), + Click(Locator("#go", None, "g1")), + Poke(E1), + Hover(P1), + FillForm((FormField(E1, FieldKind.TEXTBOX, "x"), FormField(P1, FieldKind.CHECKBOX, "true"))), + ], + ids=["click-index", "click-locator", "poke-driver-ref", "hover-point", "nested-form-field"], +) +def test_ref_kind_the_handler_does_not_accept_is_unsupported(action: Action) -> None: + driver = ToyDriver() + + with pytest.raises(Unsupported, match="does not accept") as raised: + _run(driver.execute(action)) + + assert raised.value.dispatched is False + assert driver.handled == [] + + +def test_subclass_keeps_inherited_handlers_and_may_override_them() -> None: + class Child(ToyDriver): + @handles(Click, refs={DriverRef}) + async def click_by_ref(self, action: Click) -> ActionResult: + return self._done("child-click", action) + + driver = Child() + + assert driver.info.actions[Click] == frozenset({DriverRef}) + assert Poke in driver.info.actions + assert _run(driver.execute(Click(E1))).value == "child-click" + assert ToyDriver().info.actions[Click] == frozenset({DriverRef, PointRef}), "the parent is unchanged" + + +def test_handles_rejects_kinds_that_are_not_element_refs() -> None: + with pytest.raises(TypeError, match="ContextRef"): + handles(Click, refs={ContextRef}) + + +def test_iter_refs_finds_nested_element_refs_only() -> None: + form = FillForm((FormField(E1, FieldKind.TEXTBOX, "x"), FormField(P1, FieldKind.RADIO, "y"))) + + assert list(iter_refs(form)) == [E1, P1] + assert list(iter_refs(TypeText(None, "x"))) == [] + assert list(iter_refs(Navigate("https://example.test/"))) == [] + + +def test_tool_hints_default_is_not_shared_between_instances() -> None: + first = DriverInfo(name="a", surfaces=frozenset({"x"})) + second = DriverInfo(name="b", surfaces=frozenset({"x"})) + + assert first.tool_hints == {} and first.tool_hints is not second.tool_hints + assert first.versions is not second.versions + assert first.actions is not second.actions diff --git a/tests/unit_tests/harness/tools/browser_move/contract/test_errors.py b/tests/unit_tests/harness/tools/browser_move/contract/test_errors.py new file mode 100644 index 0000000000..c46e623ff1 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/test_errors.py @@ -0,0 +1,64 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Driver errors carry ``dispatched``; an unknown outcome counts as dispatched.""" + +from __future__ import annotations + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + ActionIntercepted, + ActionTimeout, + AmbiguousElement, + ConnectionLost, + ContextRef, + DriverError, + ElementNotFound, + NavigationError, + Observation, + PolicyDenied, + StaleRef, + Unsupported, +) + +DEFAULT_DISPATCHED = { + DriverError: True, + StaleRef: False, + ElementNotFound: True, + AmbiguousElement: True, + ActionTimeout: True, + NavigationError: True, + ConnectionLost: True, + Unsupported: False, + PolicyDenied: False, +} + + +@pytest.mark.parametrize("error_type", list(DEFAULT_DISPATCHED), ids=lambda t: t.__name__) +def test_dispatched_default(error_type: type[DriverError]) -> None: + error = error_type("boom") + + assert isinstance(error, DriverError) + assert error.dispatched is DEFAULT_DISPATCHED[error_type] + assert str(error) == "boom" + + +@pytest.mark.parametrize("error_type", list(DEFAULT_DISPATCHED), ids=lambda t: t.__name__) +@pytest.mark.parametrize("dispatched", [True, False]) +def test_dispatched_can_be_set_explicitly(error_type: type[DriverError], dispatched: bool) -> None: + assert error_type("boom", dispatched=dispatched).dispatched is dispatched + + +def test_action_intercepted_names_the_blocker() -> None: + error = ActionIntercepted("click not delivered", by="
") + + assert error.by == "
" + assert error.dispatched is False + assert ActionIntercepted(by="x", dispatched=True).dispatched is True + + +def test_stale_ref_carries_an_optional_fresh_observation() -> None: + fresh = Observation(context=ContextRef("c1", "screen")) + + assert StaleRef("old").observation is None + assert StaleRef("old", observation=fresh).observation is fresh diff --git a/tests/unit_tests/harness/tools/browser_move/contract/test_fake_driver.py b/tests/unit_tests/harness/tools/browser_move/contract/test_fake_driver.py new file mode 100644 index 0000000000..25abdf2ca9 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/test_fake_driver.py @@ -0,0 +1,191 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""``FakeDriver`` records calls, replays scripted answers and raises contract errors on request.""" + +from __future__ import annotations + +import asyncio + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + ActionResult, + Click, + ConnectionLost, + ContextRef, + Driver, + DriverError, + DriverInfo, + DriverRef, + Observation, + Observe, + PointRef, + TypeText, + Unsupported, +) +from openjiuwen.harness.tools.browser_move.contract.testing import DEFAULT_ACTIONS, Call, FakeDriver +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import Navigate + +E1 = DriverRef("e1", "g1") +OBS_1 = Observation(context=ContextRef("c1", "window")) +OBS_2 = Observation(context=ContextRef("c2", "window")) +RES_1 = ActionResult(ok=True, value=1) +RES_2 = ActionResult(ok=True, value=2) + + +def test_is_a_driver_with_a_neutral_default_info() -> None: + driver = FakeDriver() + + assert isinstance(driver, Driver) + assert (driver.info.name, driver.info.surfaces) == ("fake", frozenset({"fake"})) + assert driver.info.actions == DEFAULT_ACTIONS + assert Observe not in driver.info.actions + + +def test_records_every_call_in_order() -> None: + driver = FakeDriver(observations=[OBS_1], results=[RES_1]) + + async def scenario() -> None: + await driver.start({"headless": True}) + await driver.observe(Observe()) + await driver.execute(Click(E1)) + await driver.stop() + + asyncio.run(scenario()) + + assert driver.calls == [ + Call("start", {"headless": True}), + Call("observe", Observe()), + Call("execute", Click(E1)), + Call("stop"), + ] + assert driver.started is False + + +def test_returns_scripted_observations_and_results_in_order() -> None: + driver = FakeDriver(observations=[OBS_1, OBS_2], results=[RES_1, RES_2]) + + async def scenario() -> list[object]: + return [ + await driver.observe(Observe()), + await driver.execute(Click(E1)), + await driver.execute(Click(E1)), + await driver.observe(Observe()), + ] + + assert asyncio.run(scenario()) == [OBS_1, RES_1, RES_2, OBS_2] + + +def test_script_appends_answers_after_the_constructor_ones() -> None: + driver = FakeDriver(observations=[OBS_1], results=[RES_1]) + driver.script(observations=[OBS_2], results=[RES_2]) + driver.script(results=[RES_1]) + + async def scenario() -> list[object]: + return [ + await driver.observe(Observe()), + await driver.observe(Observe()), + await driver.execute(Click(E1)), + await driver.execute(Click(E1)), + await driver.execute(Click(E1)), + ] + + assert asyncio.run(scenario()) == [OBS_1, OBS_2, RES_1, RES_2, RES_1] + + +def test_running_out_of_script_fails_loudly() -> None: + driver = FakeDriver() + + with pytest.raises(AssertionError, match="no scripted answer left for execute"): + asyncio.run(driver.execute(Click(E1))) + assert driver.kinds() == ["execute"] + + +@pytest.mark.parametrize("dispatched", [True, False]) +def test_scripted_error_is_a_driver_error_with_the_chosen_dispatched_value(dispatched: bool) -> None: + driver = FakeDriver(results=[RES_1]) + driver.fail_next(dispatched=dispatched) + + with pytest.raises(DriverError) as raised: + asyncio.run(driver.execute(Click(E1))) + + assert type(raised.value) is DriverError + assert raised.value.dispatched is dispatched + assert driver.kinds() == ["execute"], "the failed call is still recorded" + assert asyncio.run(driver.execute(Click(E1))) == RES_1, "the error does not consume a scripted result" + + +def test_any_scripted_exception_can_be_raised() -> None: + driver = FakeDriver() + driver.fail_next(TimeoutError("slow scene")) + + with pytest.raises(TimeoutError, match="slow scene"): + asyncio.run(driver.observe(Observe())) + + +def test_connection_is_lost_on_the_nth_call_until_restart() -> None: + driver = FakeDriver(observations=[OBS_1, OBS_2], results=[RES_1]) + driver.lose_connection_on(2) + + async def scenario() -> list[object]: + outcomes: list[object] = [] + await driver.start() + for call in (driver.observe(Observe()), driver.execute(Click(E1)), driver.observe(Observe())): + try: + outcomes.append(await call) + except ConnectionLost as error: + outcomes.append(("lost", error.dispatched)) + outcomes.append((await driver.health()).ok) + await driver.start() + outcomes.append((await driver.health()).ok) + outcomes.append(await driver.execute(Click(E1))) + outcomes.append(await driver.observe(Observe())) + return outcomes + + assert asyncio.run(scenario()) == [OBS_1, ("lost", True), ("lost", True), False, True, RES_1, OBS_2] + assert driver.kinds() == ["start", "observe", "execute", "observe", "start", "execute", "observe"] + + +def test_connection_loss_can_be_marked_not_dispatched() -> None: + driver = FakeDriver() + driver.lose_connection_on(1, dispatched=False) + + with pytest.raises(ConnectionLost) as raised: + asyncio.run(driver.execute(Click(E1))) + + assert raised.value.dispatched is False + + +def test_health_reflects_start_and_stop() -> None: + driver = FakeDriver() + + async def scenario() -> list[bool]: + before = (await driver.health()).ok + await driver.start() + running = (await driver.health()).ok + await driver.stop() + return [before, running, (await driver.health()).ok] + + assert asyncio.run(scenario()) == [False, True, False] + + +def test_command_missing_from_info_actions_is_unsupported() -> None: + driver = FakeDriver(results=[RES_1]) + + with pytest.raises(Unsupported) as raised: + asyncio.run(driver.execute(Navigate("https://example.test/"))) + + assert raised.value.dispatched is False + assert asyncio.run(driver.execute(Click(E1))) == RES_1, "an unsupported call consumes no scripted result" + + +def test_info_can_restrict_commands_and_ref_kinds() -> None: + info = DriverInfo(name="points", surfaces=frozenset({"other"}), actions={Click: frozenset({PointRef})}) + driver = FakeDriver(results=[RES_1], info=info) + + assert driver.info is info + with pytest.raises(Unsupported, match="does not accept DriverRef"): + asyncio.run(driver.execute(Click(E1))) + with pytest.raises(Unsupported, match="does not handle TypeText"): + asyncio.run(driver.execute(TypeText(None, "x"))) + assert asyncio.run(driver.execute(Click(PointRef(1, 2, "g1")))) == RES_1 diff --git a/tests/unit_tests/harness/tools/browser_move/contract/test_registry.py b/tests/unit_tests/harness/tools/browser_move/contract/test_registry.py new file mode 100644 index 0000000000..00145fb491 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/test_registry.py @@ -0,0 +1,155 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""``Registry`` resolves lazy import strings on first ``create`` and refuses duplicate or unknown names.""" + +from __future__ import annotations + +import sys + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import registry as registry_module +from openjiuwen.harness.tools.browser_move.contract.registry import ( + DRIVERS, + PROVISIONERS, + SURFACES, + Registry, + default_topology, + register_driver, + register_provisioner, + register_surface, +) + +LAZY_MODULE = "tests.unit_tests.harness.tools.browser_move.contract._lazy_target" + + +def test_lazy_import_string_is_resolved_only_on_create(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delitem(sys.modules, LAZY_MODULE, raising=False) + registry: Registry[object] = Registry("thing") + + registry.register("lazy", f"{LAZY_MODULE}:make_thing") + + assert LAZY_MODULE not in sys.modules + assert registry.names() == ("lazy",) + assert registry.create("lazy", 1, key="v") == ("thing", (1,), {"key": "v"}) + assert LAZY_MODULE in sys.modules + + +def test_lazy_import_string_may_name_a_nested_attribute(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delitem(sys.modules, LAZY_MODULE, raising=False) + registry: Registry[object] = Registry("thing") + registry.register("nested", f"{LAZY_MODULE}:Nested.factory") + + assert registry.create("nested") == "nested" + + +def test_callable_factory_is_called_with_the_given_arguments() -> None: + registry: Registry[object] = Registry("thing") + registry.register("direct", lambda device, settings: (device, settings)) + + assert registry.create("direct", "dev", settings="s") == ("dev", "s") + + +def test_names_keep_registration_order() -> None: + registry: Registry[object] = Registry("thing") + for name in ("b", "a", "c"): + registry.register(name, dict) + + assert registry.names() == ("b", "a", "c") + + +def test_duplicate_name_is_refused() -> None: + registry: Registry[object] = Registry("driver") + registry.register("x", dict) + + with pytest.raises(ValueError, match="driver 'x' is already registered"): + registry.register("x", list) + assert registry.create("x") == {} + + +def test_unknown_name_lists_the_registered_ones() -> None: + registry: Registry[object] = Registry("surface") + registry.register("known", dict) + + with pytest.raises(KeyError, match="unknown surface 'missing'; registered: known"): + registry.create("missing") + + +@pytest.mark.parametrize("factory", ["no_colon", ":attr", "pkg.module:"]) +def test_malformed_import_string_is_refused_at_register(factory: str) -> None: + with pytest.raises(ValueError, match="pkg.module:attr"): + Registry("driver").register("bad", factory) + + +def test_non_callable_factory_is_refused() -> None: + with pytest.raises(TypeError, match="callable"): + Registry("driver").register("bad", 42) # type: ignore[arg-type] + + +def test_lazy_import_string_of_a_missing_module_says_not_built_yet() -> None: + registry: Registry[object] = Registry("driver") + registry.register("later", f"{LAZY_MODULE}_not_written_yet:create") + + with pytest.raises(ModuleNotFoundError, match="driver 'later' is registered but not built yet"): + registry.create("later") + + +def test_missing_dependency_inside_a_lazy_module_is_raised_as_it_is(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: + (tmp_path / "lazy_with_missing_dependency.py").write_text("import not_installed_dependency_xyz\n") + monkeypatch.syspath_prepend(str(tmp_path)) + registry: Registry[object] = Registry("driver") + registry.register("broken", "lazy_with_missing_dependency:create") + + with pytest.raises(ModuleNotFoundError, match="No module named 'not_installed_dependency_xyz'"): + registry.create("broken") + + +@pytest.mark.parametrize( + ("registry", "kind"), [(DRIVERS, "driver"), (SURFACES, "surface"), (PROVISIONERS, "provisioner")] +) +def test_default_registries_name_their_kind(registry: Registry[object], kind: str) -> None: + with pytest.raises(KeyError, match=f"unknown {kind} '__missing__'"): + registry.create("__missing__") + + +@pytest.mark.parametrize( + ("register", "attribute"), + [(register_driver, "DRIVERS"), (register_surface, "SURFACES"), (register_provisioner, "PROVISIONERS")], +) +def test_register_functions_use_the_default_registries( + monkeypatch: pytest.MonkeyPatch, register, attribute: str +) -> None: + fresh: Registry[object] = Registry(attribute.lower()) + monkeypatch.setattr(registry_module, attribute, fresh) + + register("mine", f"{LAZY_MODULE}:make_thing") + + assert fresh.names() == ("mine",) + + +@pytest.fixture +def fresh_drivers(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(registry_module, "DRIVERS", Registry("driver")) + monkeypatch.setattr(registry_module, "DRIVER_TOPOLOGIES", {}) + + +def test_register_driver_records_the_default_topology_and_direct_when_none_is_given(fresh_drivers: None) -> None: + register_driver("plain", f"{LAZY_MODULE}:make_thing") + register_driver("delegating", f"{LAZY_MODULE}:make_thing", default_topology="delegated_worker") + + assert (default_topology("plain"), default_topology("delegating")) == ("direct", "delegated_worker") + + +def test_default_topology_of_an_unknown_driver_lists_the_registered_ones(fresh_drivers: None) -> None: + register_driver("known", f"{LAZY_MODULE}:make_thing") + + with pytest.raises(KeyError, match="unknown driver 'missing'; registered: known"): + default_topology("missing") + + +def test_a_duplicate_driver_is_refused_before_its_default_topology_is_recorded(fresh_drivers: None) -> None: + register_driver("x", dict) + + with pytest.raises(ValueError, match="driver 'x' is already registered"): + register_driver("x", list, default_topology="delegated_worker") + assert default_topology("x") == "direct" diff --git a/tests/unit_tests/harness/tools/browser_move/contract/test_tool_types.py b/tests/unit_tests/harness/tools/browser_move/contract/test_tool_types.py new file mode 100644 index 0000000000..c134dbaff7 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/test_tool_types.py @@ -0,0 +1,179 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Native tools, tool specs with decoders, the ref resolver, the renderer protocol and contextual policy rules.""" + +from __future__ import annotations + +from collections.abc import Mapping +from typing import Any + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + Capability, + Click, + ContentBlock, + ContextRef, + DriverError, + DriverInfo, + DriverRef, + HandlerDriver, + Health, + NativeTool, + Observation, + Observe, + PolicyRule, + Ref, + RefResolver, + Renderer, + Screenshot, + SurfaceContext, + SurfaceProfile, + ToolSpec, + handles, +) + +CTX = ContextRef("c1", "window", "Home") + + +class _Resolver: + contexts = (CTX,) + current_context = CTX + + def resolve(self, model_id: str, generation: str) -> Ref: + if model_id != "t7": + raise ValueError(f"unknown id {model_id}") + return DriverRef("e7", generation) + + def generation_of(self, ref: str) -> str: + return "g2" if ref == "e1" else "g3" + + +def _click_by_id(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Click(resolver.resolve(args["id"], args["generation"])) + + +def _click_by_ref(args: Mapping[str, Any], resolver: RefResolver) -> Action: + return Click(DriverRef(args["ref"], resolver.generation_of(args["ref"]))) + + +class _Renderer: + def observation(self, observation: Observation) -> tuple[ContentBlock, ...]: + return (observation.native_text or "",) + + def result(self, action: Action, result: ActionResult) -> tuple[ContentBlock, ...]: + return (result.detail or "",) + + def error(self, action: Action, error: DriverError, observation: Observation | None) -> tuple[ContentBlock, ...]: + return (f"### Error\n{error}", Screenshot(b"png", 1, 1)) + + +class _NativeDriver(HandlerDriver): + def describe(self) -> DriverInfo: + return DriverInfo( + name="native", + surfaces=frozenset({"toy"}), + native_tools={"toy_click": NativeTool("Click it.", {"type": "object"}, _click_by_ref)}, + tool_name_prefix="native_", + ) + + async def start(self, config: Any = None) -> None: + pass + + async def stop(self) -> None: + pass + + async def health(self) -> Health: + return Health(ok=True) + + async def observe(self, command: Observe) -> Observation: + return Observation(context=CTX) + + @handles(Click, refs={DriverRef}) + async def _click(self, action: Click) -> ActionResult: + return ActionResult(ok=True) + + +def test_driver_info_has_no_native_tools_and_no_prefix_by_default() -> None: + first = DriverInfo(name="a", surfaces=frozenset({"x"})) + second = DriverInfo(name="b", surfaces=frozenset({"x"})) + assert first.native_tools == {} and first.tool_name_prefix == "" + assert first.native_tools is not second.native_tools + + +def test_handler_driver_info_keeps_native_tools_and_prefix() -> None: + info = _NativeDriver().info + assert info.tool_name_prefix == "native_" + assert info.native_tools["toy_click"].description == "Click it." + assert info.actions == {Click: frozenset({DriverRef})} + hash(info) + + +def test_native_tool_decodes_arguments_with_the_resolver() -> None: + tool = _NativeDriver().info.native_tools["toy_click"] + assert tool.build({"ref": "e1"}, _Resolver()) == Click(DriverRef("e1", "g2")) + assert tool.build({"ref": "e9"}, _Resolver()) == Click(DriverRef("e9", "g3")) + + +def test_native_tool_has_no_decoder_by_default() -> None: + assert NativeTool("Find it.", {"type": "object"}).build is None + + +def test_tool_spec_has_no_schema_and_no_decoder_by_default() -> None: + spec = ToolSpec("toy_click", (Click,)) + assert spec.schema is None and spec.build is None + + +def test_tool_spec_with_schema_stays_hashable_and_decodes() -> None: + spec = ToolSpec("toy_click", (Click,), schema={"type": "object"}, build=_click_by_id) + hash(spec) + assert spec.build({"id": "t7", "generation": "g4"}, _Resolver()) == Click(DriverRef("e7", "g4")) + + +def test_resolver_exposes_contexts_and_the_current_one() -> None: + resolver: RefResolver = _Resolver() + assert resolver.contexts == (CTX,) and resolver.current_context == CTX + + +def test_surface_profile_takes_a_renderer_with_three_methods() -> None: + renderer: Renderer = _Renderer() + profile = SurfaceProfile(name="toy", commands=(), renderer=renderer) + blocks = profile.renderer.error(Click(DriverRef("e1", "g1")), DriverError("boom"), None) + assert blocks[0] == "### Error\nboom" + assert isinstance(blocks[1], Screenshot) + assert profile.renderer.observation(Observation(context=CTX, native_text="tree")) == ("tree",) + assert profile.renderer.result(Observe(), ActionResult(ok=True, detail="done")) == ("done",) + + +def test_policy_rule_check_gets_the_action_and_the_current_context() -> None: + seen: list[tuple[Action, SurfaceContext | None]] = [] + + def deny_without_context(action: Action, context: SurfaceContext | None) -> str | None: + seen.append((action, context)) + return None if context is not None else "no context yet" + + rule = PolicyRule(Click, deny_without_context) + click = Click(DriverRef("e1", "g1")) + assert rule.check(click, None) == "no context yet" + assert rule.check(click, SurfaceContext()) is None + assert seen == [(click, None), (click, SurfaceContext())] + + +def test_surface_profile_has_no_capability_tools_by_default() -> None: + first = SurfaceProfile(name="toy", commands=(), renderer=_Renderer()) + second = SurfaceProfile(name="toy", commands=(), renderer=_Renderer()) + assert first.capability_tools == {} and first.capability_tools is not second.capability_tools + + +def test_surface_profile_has_no_catalog_and_no_hooks_by_default() -> None: + profile = SurfaceProfile(name="toy", commands=(), renderer=_Renderer()) + assert profile.capabilities == () + assert (profile.custom_actions, profile.page_metadata, profile.policy_probe) == (None, None, None) + assert (profile.activate, profile.semantic_tracker, profile.worker_prompt) == (None, None, None) + + +def test_capability_is_a_name_its_tools_and_whether_policy_alone_grants_it() -> None: + look = Capability("core", "Look around.", ("look",)) + assert (look.name, look.description, look.tool_names) == ("core", "Look around.", ("look",)) + assert look.policy_only is False + assert Capability("unsafe", "Anything.", (), policy_only=True).policy_only diff --git a/tests/unit_tests/harness/tools/browser_move/contract/test_types.py b/tests/unit_tests/harness/tools/browser_move/contract/test_types.py new file mode 100644 index 0000000000..71529a4b50 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/contract/test_types.py @@ -0,0 +1,269 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Contract data types are frozen, have the documented defaults, and the fingerprint is stable.""" + +from __future__ import annotations + +import dataclasses + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + Click, + CloseContext, + ContextRef, + Drag, + DriverRef, + Effect, + Element, + HandleDialog, + Hover, + IndexRef, + Locator, + Observation, + Observe, + PointRef, + PressKey, + RunScript, + Screenshot, + Scroll, + SurfaceContext, + SwitchContext, + TypeText, + Viewport, + Wait, + iter_refs, +) +from openjiuwen.harness.tools.browser_move.contract import actions as contract_actions +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FieldKind, + FillForm, + FormField, + GetConsole, + GetNetwork, + GetPdf, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext + +CTX = ContextRef("c1", "window", "Home") +E1 = DriverRef("e1", "g1") + + +def _observation(**overrides) -> Observation: + fields = { + "context": CTX, + "elements": (Element(E1, "button", "Go"), Element(DriverRef("e2", "g1"), "link", "More")), + } + fields.update(overrides) + return Observation(**fields) + + +FROZEN_SAMPLES = [ + E1, + PointRef(1, 2, "g1"), + IndexRef(3, "g1"), + Locator("#go", None, "g1"), + CTX, + Viewport(800, 600), + Screenshot(b"png", 800, 600), + Element(E1, "button", "Go"), + _observation(), + ActionResult(ok=True), + Click(E1), + Navigate("https://example.test/"), + BrowserContext(url="https://example.test/"), +] + + +@pytest.mark.parametrize("sample", FROZEN_SAMPLES, ids=lambda s: type(s).__name__) +def test_contract_values_are_frozen(sample) -> None: + name = dataclasses.fields(sample)[0].name + + with pytest.raises(dataclasses.FrozenInstanceError): + setattr(sample, name, None) + + +EFFECTS = { + Observe: Effect.READ, + Click: Effect.ACT, + TypeText: Effect.ACT, + PressKey: Effect.ACT, + Scroll: Effect.ACT, + Hover: Effect.ACT, + Drag: Effect.ACT, + Wait: Effect.READ, + SwitchContext: Effect.NAVIGATE, + CloseContext: Effect.NAVIGATE, + HandleDialog: Effect.ACT, + RunScript: Effect.SCRIPT, + Navigate: Effect.NAVIGATE, + Back: Effect.NAVIGATE, + OpenContext: Effect.NAVIGATE, + SelectOption: Effect.ACT, + UploadFile: Effect.ACT, + FillForm: Effect.ACT, + Drop: Effect.ACT, + GetPdf: Effect.READ, + GetConsole: Effect.READ, + GetNetwork: Effect.READ, +} + + +@pytest.mark.parametrize("command", list(EFFECTS), ids=lambda c: c.__name__) +def test_every_command_has_its_class_level_effect(command: type[Action]) -> None: + assert issubclass(command, Action) + assert command.effect is EFFECTS[command] + assert "effect" not in {f.name for f in dataclasses.fields(command)}, "effect is not an instance field" + + +def test_shared_commands_are_exactly_the_approved_list() -> None: + shared = { + obj + for obj in vars(contract_actions).values() + if isinstance(obj, type) and issubclass(obj, Action) and obj is not Action + } + + assert {c.__name__ for c in shared} == { + "Observe", + "Click", + "TypeText", + "PressKey", + "Scroll", + "Hover", + "Drag", + "Wait", + "SwitchContext", + "CloseContext", + "HandleDialog", + "RunScript", + }, "GetText, Forward and Reload were removed (decision C14)" + + +def test_an_observe_is_scoped_by_a_depth_or_a_root_the_stale_check_sees() -> None: + assert not Observe().scoped and not Observe(include_screenshot=True).scoped + assert Observe(depth=1).scoped and Observe(root=E1).scoped + assert list(iter_refs(Observe(root=E1))) == [E1] + + +def test_command_defaults() -> None: + assert Observe() == Observe(include_screenshot=False, full_page=False, depth=None, root=None) + assert Click(E1) == Click(E1, button="left", modifiers=(), count=1) + typed = TypeText(None, "hi") + assert (typed.submit, typed.sensitive, typed.clear, typed.slowly) == (False, False, True, False) + assert PressKey("Enter").target is None + assert Scroll() == Scroll(target=None, dx=0, dy=0) + assert HandleDialog(True).prompt_text is None + script = RunScript("() => 1") + assert (script.args, script.sensitive, script.target) == (None, False, None) + assert OpenContext().url is None + assert UploadFile(None).paths == () + assert GetPdf().filename is None + assert GetConsole() == GetConsole(level="info", all=False) + assert GetNetwork() == GetNetwork(static=False, filter=None) + + +def test_wait_needs_seconds_or_text() -> None: + with pytest.raises(ValueError, match="seconds, until_text or both"): + Wait() + + +def test_wait_accepts_seconds_text_or_text_with_a_deadline() -> None: + assert Wait(seconds=0.5).until_text is None + assert Wait(until_text="done").seconds is None + bounded = Wait(until_text="done", seconds=2.0) + assert (bounded.until_text, bounded.seconds) == ("done", 2.0) + + +def test_drop_needs_paths_or_data() -> None: + with pytest.raises(ValueError, match="paths or data"): + Drop(E1) + assert Drop(E1, paths=("/tmp/a.txt",)).data == () + assert Drop(E1, data=(("text/plain", "hello"),)).paths == () + + +def test_fill_form_fields_use_todays_field_kinds() -> None: + form = FillForm((FormField(E1, FieldKind.CHECKBOX, "true"),)) + + assert {kind.value for kind in FieldKind} == {"textbox", "checkbox", "radio", "combobox", "slider"} + assert form.fields[0].kind == "checkbox" + + +def test_form_field_is_not_sensitive_unless_marked() -> None: + assert FormField(E1, FieldKind.TEXTBOX, "Ann").sensitive is False + assert FormField(E1, FieldKind.TEXTBOX, "pw", sensitive=True).sensitive is True + + +def test_observation_and_result_defaults() -> None: + observation = Observation(context=CTX) + assert (observation.generation, observation.elements, observation.screenshot) == ("", (), None) + assert (observation.viewport, observation.native_text, observation.surface) == (None, None, None) + + result = ActionResult(ok=True) + assert (result.value, result.detail, result.context_changed, result.generation) == (None, None, False, "") + assert (result.observation, result.source_context, result.opened_contexts, result.surface) == (None, None, (), None) + + +def test_element_defaults_and_extra_not_shared() -> None: + first, second = Element(E1, "button", "Go"), Element(E1, "button", "Go") + + assert (first.value, first.bounds, first.enabled, first.focused) == (None, None, True, False) + assert first.extra == {} and first.extra is not second.extra + assert hash(first) == hash(second) + + +def test_screenshot_scale_defaults_to_one() -> None: + assert Screenshot(b"png", 10, 20).scale == 1.0 + + +def test_surface_context_is_an_empty_base() -> None: + assert dataclasses.fields(SurfaceContext) == () + context = BrowserContext(url="https://example.test/", tabs=(CTX,)) + assert isinstance(context, SurfaceContext) + assert (context.title, context.is_pdf_viewer, context.pixels_above, context.pixels_below) == ("", False, 0, 0) + assert context.last_status is None + + +def test_fingerprint_is_stable_for_the_same_scene() -> None: + first = _observation() + second = _observation( + generation="g9", + screenshot=Screenshot(b"other", 1, 1), + native_text="tree", + surface=BrowserContext(url="https://example.test/"), + ) + + assert first.fingerprint == second.fingerprint + assert first.fingerprint == _observation().fingerprint + assert len(first.fingerprint) == 16 + + +def test_fingerprint_ignores_ref_generation_and_values() -> None: + elements = ( + Element(DriverRef("e1", "g7"), "button", "Go", value="x"), + Element(DriverRef("e2", "g7"), "link", "More"), + ) + + assert _observation(elements=elements).fingerprint == _observation().fingerprint + + +@pytest.mark.parametrize( + "overrides", + [ + {"context": ContextRef("c2", "window", "Home")}, + {"elements": (Element(E1, "button", "Stop"), Element(DriverRef("e2", "g1"), "link", "More"))}, + {"elements": (Element(E1, "link", "Go"), Element(DriverRef("e2", "g1"), "link", "More"))}, + {"elements": (Element(DriverRef("e3", "g1"), "button", "Go"), Element(DriverRef("e2", "g1"), "link", "More"))}, + {"elements": (Element(E1, "button", "Go"),)}, + ], + ids=["context-id", "name", "role", "ref", "fewer-elements"], +) +def test_fingerprint_changes_with_context_refs_roles_and_names(overrides) -> None: + assert _observation(**overrides).fingerprint != _observation().fingerprint diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/__init__.py b/tests/unit_tests/harness/tools/browser_move/drivers/__init__.py new file mode 100644 index 0000000000..57d631c3f0 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/__init__.py @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/browser_scene_extras.py b/tests/unit_tests/harness/tools/browser_move/drivers/browser_scene_extras.py new file mode 100644 index 0000000000..2267c7e82c --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/browser_scene_extras.py @@ -0,0 +1,47 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Live cases of the browser scene beyond the shared suite, mixed into both drivers' conformance classes.""" + +from __future__ import annotations + +import asyncio +from collections.abc import Callable + +from openjiuwen.harness.tools.browser_move.contract import DriverError, Observe, Scene +from openjiuwen.harness.tools.browser_move.contract.testing.conformance import ( + DriverFactory, + _expect_error, + _prepare, + _session, + _skip_if_reason, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.scene import ( + clipboard_read_case, + paste_command_case, + slow_typing_case, +) + + +async def _check(factory: DriverFactory, scene: Scene, case_of: Callable) -> None: + async with _session(factory, scene) as driver: + case = _skip_if_reason(scene, case_of(driver.info.actions)) + prepared = await _prepare(driver, case) + result = await driver.execute(prepared.action) + after = await driver.observe(Observe()) + assert case.check is not None and case.check(prepared.before, after, result), f"{prepared.action!r}: {result!r}" + + +class BrowserSceneExtras: + def test_a_clipboard_read_through_an_alias_is_rejected_in_the_page( + self, conformance_scene, conformance_driver_factory + ): + error = asyncio.run( + _expect_error(conformance_driver_factory, conformance_scene, clipboard_read_case, DriverError) + ) + assert "clipboard belongs to the host machine" in str(error) + + def test_the_paste_command_is_refused_in_the_page(self, conformance_scene, conformance_driver_factory): + asyncio.run(_check(conformance_driver_factory, conformance_scene, paste_command_case)) + + def test_slow_typing_leaves_the_value_plain_typing_leaves(self, conformance_scene, conformance_driver_factory): + asyncio.run(_check(conformance_driver_factory, conformance_scene, slow_typing_case)) diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/__init__.py b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/__init__.py new file mode 100644 index 0000000000..285ed342d6 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/__init__.py @@ -0,0 +1,2 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_conformance_playwright_mcp_live.py b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_conformance_playwright_mcp_live.py new file mode 100644 index 0000000000..dc1600a303 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_conformance_playwright_mcp_live.py @@ -0,0 +1,68 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The driver conformance suite, live on ``PlaywrightMcpDriver`` with ``@playwright/mcp`` and a headless Chrome.""" + +from __future__ import annotations + +import os +import shutil +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import pytest + +from openjiuwen.harness.tools.browser_move.contract.testing.conformance import DriverConformance +from openjiuwen.harness.tools.browser_move.drivers.middleware import Tracing +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.driver import ( + PlaywrightMcpDriver, + create_playwright_mcp_driver, +) +from openjiuwen.harness.tools.browser_move.provisioning import ProvisionedDevice +from openjiuwen.harness.tools.browser_move.provisioning.managed_browser import _candidate_chrome_binaries +from openjiuwen.harness.tools.browser_move.surfaces.browser.config import BrowserSettings +from openjiuwen.harness.tools.browser_move.surfaces.browser.scene import browser_scene +from tests.unit_tests.harness.tools.browser_move.drivers.browser_scene_extras import BrowserSceneExtras +from tests.unit_tests.harness.tools.browser_move.drivers.test_conformance_browser_use_live import ( # noqa: F401 + chrome_device, +) + +pytest_plugins = ["openjiuwen.harness.tools.browser_move.contract.testing.conformance"] + +SERVER_ARGS = "-y @playwright/mcp@0.0.78" + + +LIVE_MCP_ENV = "BROWSER_MOVE_LIVE_MCP" + + +def _skip_reason() -> str | None: + if os.getenv(LIVE_MCP_ENV) != "1": + return f"set {LIVE_MCP_ENV}=1 to run against a real @playwright/mcp server and Chrome" + settings = BrowserSettings.from_process_env() + if shutil.which(settings.mcp_command) is None: + return f"{settings.mcp_command!r} is not on PATH (it starts @playwright/mcp)" + if not _candidate_chrome_binaries(settings.browser_binary_candidates): + return "no Chrome binary found" + return None + + +@pytest.fixture(scope="class") +def mcp_settings() -> Iterator[BrowserSettings]: + """Settings that pin the server version and give it a fresh working directory for its output files.""" + work_dir = Path(tempfile.mkdtemp(prefix="mcp-conformance-")) + try: + environ = {"PLAYWRIGHT_MCP_ARGS": SERVER_ARGS, "PLAYWRIGHT_RUNTIME_MCP_CWD": str(work_dir)} + yield BrowserSettings.from_env(environ) + finally: + shutil.rmtree(work_dir, ignore_errors=True) + + +@pytest.mark.live +@pytest.mark.skipif(_skip_reason() is not None, reason=_skip_reason() or "") +class TestPlaywrightMcpConformance(DriverConformance, BrowserSceneExtras): + scene = browser_scene() + driver_actions = PlaywrightMcpDriver.handled_actions() + + @pytest.fixture + def conformance_driver_factory(self, chrome_device: ProvisionedDevice, mcp_settings: BrowserSettings): # noqa: F811 + return lambda sink: Tracing(create_playwright_mcp_driver(chrome_device, mcp_settings), sink) diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_driver.py b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_driver.py new file mode 100644 index 0000000000..20b88b16a4 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_driver.py @@ -0,0 +1,1009 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""``PlaywrightMcpDriver`` on a fake MCP session: tool calls sent, observation, lifecycle, context tracking, factory.""" + +from __future__ import annotations + +import asyncio +import contextlib +import dataclasses +import json +import os +import sys +import tempfile +import threading +from collections.abc import Callable, Iterator +from pathlib import Path +from typing import Any +from unittest.mock import PropertyMock, patch + +import pytest + +from openjiuwen.core.foundation.tool import McpServerConfig +from openjiuwen.harness.tools.browser_move.contract import ( + ActionResult, + ActionTimeout, + Click, + CloseContext, + ConnectionLost, + ContextRef, + Drag, + DriverError, + DriverRef, + HandleDialog, + Health, + Hover, + IndexRef, + Locator, + Observation, + Observe, + PointRef, + PressKey, + RunScript, + Scroll, + StaleRef, + SwitchContext, + TypeText, + Unsupported, + Wait, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp import driver as driver_module +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.clients import BrowserMoveStdioClient +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.clients.stdio_client import sanitize_json_schema +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.driver import ( + PlaywrightMcpConfig, + PlaywrightMcpDriver, + create_playwright_mcp_driver, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.result import result_text +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.snapshot import without_boxes +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.tools import DECODERS +from openjiuwen.harness.tools.browser_move.provisioning import ProvisionedDevice +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FieldKind, + FillForm, + FormField, + GetConsole, + GetNetwork, + GetPdf, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.config import BrowserSettings +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings import ( + BrowserInstanceConfig, + build_playwright_mcp_config, +) +from tests.unit_tests.harness.tools.browser_move.snapshots.builder import FakeMcpSession, load_mcp_tools + +FIXTURES_DIR = Path(__file__).resolve().parents[2] / "fixtures" / "mcp" +TIMESTAMP = "2026-10-10T00-00-00-000Z" +SNAPSHOT_FILE = Path(".playwright-mcp") / f"page-{TIMESTAMP}.yml" +OK = { + "content": [ + { + "type": "text", + "text": ( + "### Ran Playwright code\n```js\nawait page.x();\n```\n### Page\n- Page URL: http://x/\n- Page Title: X" + ), + } + ], + "isError": False, +} + +E12 = DriverRef("e12", "g3") +E5 = DriverRef("e5", "g3") +CSS = Locator("#save", None, "g3") +POINT = PointRef(10, 20, "g3") +POINT_2 = PointRef(30, 40, "g3") +TAB_1 = ContextRef(id="1", kind="tab", title="Second") + +HANG = "hang" +MISSING = str((Path(tempfile.gettempdir()) / "openjiuwen-missing-upload" / "a.csv").resolve()) + +Call = tuple[str, dict[str, Any]] + + +def _tab(index: int, title: str) -> ContextRef: + return ContextRef(id=str(index), kind="tab", title=title) + + +def _raw(case: str) -> dict[str, Any]: + fixture = json.loads((FIXTURES_DIR / f"{case}.json").read_text(encoding="utf-8")) + return json.loads(json.dumps(fixture["raw_result"]).replace("", TIMESTAMP)) + + +def _inline_yaml(case: str) -> str: + text = _raw(case)["content"][0]["text"] + return text.split("```yaml\n", 1)[1].rsplit("```", 1)[0] + + +class _Session(FakeMcpSession): + """Replies per tool name with a raw result, a list of them in order, or an exception to raise.""" + + broken = False + + async def initialize(self) -> Any: + return None + + async def list_tools(self) -> Any: + if self.broken: + raise RuntimeError("server gone") + return await super().list_tools() + + async def call_tool(self, name: str, arguments: dict[str, Any] | None = None, **kwargs: Any) -> Any: + from mcp import types + + del kwargs + self.calls.append((name, dict(arguments or {}))) + reply = self.replies[name] + if isinstance(reply, list): + reply = reply.pop(0) + if isinstance(reply, Exception): + raise reply + if reply == HANG: + await asyncio.sleep(3600) + return types.CallToolResult.model_validate(reply) + + +@contextlib.asynccontextmanager +async def _no_process(*args: Any, **kwargs: Any): + del args, kwargs + yield None, None + + +@contextlib.contextmanager +def _mcp(replies: dict[str, Any]) -> Iterator[list[Call]]: + import mcp + import mcp.client.stdio + + calls: list[Call] = [] + session = type("Session", (_Session,), {"tools": load_mcp_tools(), "calls": calls, "replies": replies}) + with patch.object(mcp, "ClientSession", session), patch.object(mcp.client.stdio, "stdio_client", _no_process): + yield calls + + +def _config(cwd: Path, timeout_s: float = 5, upload_root: Path | None = None) -> PlaywrightMcpConfig: + server = McpServerConfig( + server_id="playwright_official_stdio", + server_name="playwright-official", + server_path="stdio://playwright", + client_type="stdio", + params={ + "command": sys.executable, + "args": ["-y", "@playwright/mcp@0.0.78", "--caps=vision,pdf"], + "cwd": str(cwd), + "timeout_s": timeout_s, + }, + ) + return PlaywrightMcpConfig(server, upload_root=upload_root) + + +def _run( + cwd: Path, replies: dict[str, Any], body: Callable[[PlaywrightMcpDriver], Any], upload_root: Path | None = None +) -> tuple[Any, list[Call]]: + """Start a driver on the fake session, run ``body`` on it, stop it; returns its outcome and the tool calls sent.""" + + async def scenario() -> Any: + driver = PlaywrightMcpDriver(_config(cwd, upload_root=upload_root)) + await driver.start() + try: + return await body(driver) + finally: + await driver.stop() + + with _mcp(replies) as calls: + outcome = asyncio.run(scenario()) + return outcome, [call for call in calls if call != "tools/list"] + + +def _execute( + cwd: Path, replies: dict[str, Any], *actions: Any, upload_root: Path | None = None +) -> tuple[list[Any], list[Call]]: + """Outcomes of the actions in order: the ``ActionResult`` or the ``DriverError`` each raised.""" + + async def body(driver: PlaywrightMcpDriver) -> list[Any]: + outcomes: list[Any] = [] + for action in actions: + try: + outcomes.append(await driver.execute(action)) + except DriverError as exc: + outcomes.append(exc) + return outcomes + + return _run(cwd, replies, body, upload_root) + + +EXPECTED_ACTIONS = { + Click: {DriverRef, Locator, PointRef}, + TypeText: {DriverRef, Locator}, + PressKey: set(), + Hover: {DriverRef, Locator, PointRef}, + Drag: {DriverRef, Locator, PointRef}, + Scroll: set(), + SelectOption: {DriverRef, Locator}, + FillForm: {DriverRef, Locator}, + Drop: {DriverRef, Locator}, + UploadFile: set(), + HandleDialog: set(), + Wait: set(), + RunScript: {DriverRef, Locator}, + Navigate: set(), + Back: set(), + OpenContext: set(), + SwitchContext: set(), + CloseContext: set(), + GetPdf: set(), + GetConsole: set(), + GetNetwork: set(), +} + + +def test_start_installs_the_clipboard_guard_as_the_server_init_script(tmp_path: Path) -> None: + import mcp.client.stdio + + seen: list[Any] = [] + + @contextlib.asynccontextmanager + async def record(params: Any, *args: Any, **kwargs: Any): + del args, kwargs + seen.append(params) + yield None, None + + async def scenario() -> None: + driver = PlaywrightMcpDriver(_config(tmp_path)) + await driver.start() + await driver.stop() + + with _mcp({}), patch.object(mcp.client.stdio, "stdio_client", record): + asyncio.run(scenario()) + + [params] = seen + assert params.args[:4] == ["-y", "@playwright/mcp@0.0.78", "--caps=vision,pdf", "--init-script"] + guard = Path(params.args[4]) + assert guard.name == "clipboard_guard.js" and "clipboard belongs to the host machine" in guard.read_text("utf-8") + + +def test_info_actions_are_pinned() -> None: + actions = PlaywrightMcpDriver().info.actions + assert {command: set(refs) for command, refs in actions.items()} == EXPECTED_ACTIONS + + +def test_index_ref_is_never_a_model_facing_ref_kind() -> None: + for refs in PlaywrightMcpDriver().info.actions.values(): + assert IndexRef not in refs + + +def test_info_describes_the_browser_surface_and_the_server_version(tmp_path: Path) -> None: + info = PlaywrightMcpDriver(_config(tmp_path)).info + assert info.name == "playwright_mcp" + assert info.surfaces == frozenset({"browser"}) + assert info.observes == frozenset({"elements", "screenshot", "native_text", "surface"}) + assert dict(info.versions) == {"@playwright/mcp": "0.0.78"} + assert dict(info.tool_hints) == {} + + +def test_info_without_a_config_has_no_version() -> None: + assert dict(PlaywrightMcpDriver().info.versions) == {} + + +def test_tool_hints_are_the_server_descriptions_after_start(tmp_path: Path) -> None: + described = {tool["name"]: tool["description"] for tool in load_mcp_tools()} + + hints, _calls = _run(tmp_path, {}, lambda driver: _value(dict(driver.info.tool_hints))) + + assert set(hints) == set(EXPECTED_ACTIONS) + assert hints[Click] == described["browser_click"] + assert hints[Scroll] == described["browser_mouse_wheel"] + assert hints[OpenContext] == hints[CloseContext] == described["browser_tabs"] + + +def test_native_tools_are_the_server_entries_with_a_decoder_after_start(tmp_path: Path) -> None: + listed = {tool["name"]: tool for tool in load_mcp_tools()} + + native, _calls = _run(tmp_path, {}, lambda driver: _value(dict(driver.info.native_tools))) + + assert set(native) == set(DECODERS) | {"browser_find"} + for name, tool in native.items(): + assert tool.description == listed[name]["description"] + assert tool.schema == sanitize_json_schema(listed[name]["inputSchema"]) + assert "$schema" not in tool.schema + assert tool.build is DECODERS.get(name) + + +def test_find_is_listed_as_a_server_tool_no_command_backs(tmp_path: Path) -> None: + native, _calls = _run(tmp_path, {}, lambda driver: _value(dict(driver.info.native_tools))) + + assert "browser_find" not in DECODERS + assert native["browser_find"].build is None + + +def test_native_tools_are_empty_before_start(tmp_path: Path) -> None: + assert dict(PlaywrightMcpDriver(_config(tmp_path)).info.native_tools) == {} + + +def test_tool_name_prefix_is_the_model_prefix_of_the_server(tmp_path: Path) -> None: + assert PlaywrightMcpDriver(_config(tmp_path)).info.tool_name_prefix == "mcp_playwright-official_" + assert PlaywrightMcpDriver().info.tool_name_prefix == "" + + +def test_the_display_name_is_playwright_mcp() -> None: + assert PlaywrightMcpDriver().info.display_name == "Playwright MCP" + + +async def _value(value: Any) -> Any: + return value + + +def test_start_connects_and_health_pings_the_session(tmp_path: Path) -> None: + health, calls = _run(tmp_path, {}, lambda driver: driver.health()) + + assert health == Health(ok=True) + assert calls == [] + + +def test_health_before_start_and_after_stop(tmp_path: Path) -> None: + driver = PlaywrightMcpDriver(_config(tmp_path)) + assert asyncio.run(driver.health()) == Health(ok=False, detail="not started") + + async def body(started: PlaywrightMcpDriver) -> Health: + await started.stop() + return await started.health() + + health, _calls = _run(tmp_path, {}, body) + assert health == Health(ok=False, detail="not started") + + +def test_health_reports_a_server_that_stopped_answering(tmp_path: Path) -> None: + async def body(driver: PlaywrightMcpDriver) -> Health: + _Session.broken = True + try: + return await driver.health() + finally: + _Session.broken = False + + health, _calls = _run(tmp_path, {}, body) + assert health == Health(ok=False, detail="the server does not answer") + + +def test_start_without_a_config_raises_before_anything_runs() -> None: + with pytest.raises(DriverError, match="needs a PlaywrightMcpConfig") as caught: + asyncio.run(PlaywrightMcpDriver().start()) + assert caught.value.dispatched is False + + +def test_execute_before_start_is_connection_lost_before_dispatch(tmp_path: Path) -> None: + with pytest.raises(ConnectionLost, match="not started") as caught: + asyncio.run(PlaywrightMcpDriver(_config(tmp_path)).execute(PressKey("Enter"))) + assert caught.value.dispatched is False + + +def test_transport_failure_is_connection_lost_dispatched(tmp_path: Path) -> None: + (outcome,), calls = _execute(tmp_path, {"browser_press_key": RuntimeError("broken pipe")}, PressKey("Enter")) + + assert isinstance(outcome, ConnectionLost) + assert outcome.dispatched is True + assert "broken pipe" in str(outcome) + assert calls == [("browser_press_key", {"key": "Enter"})] + + +def test_a_client_error_that_is_not_a_transport_failure_is_a_driver_error_before_dispatch(tmp_path: Path) -> None: + (outcome,), calls = _execute(tmp_path, {"browser_press_key": ValueError("bad argument")}, PressKey("Enter")) + + assert type(outcome) is DriverError + assert outcome.dispatched is False + assert "ValueError: bad argument" in str(outcome) + assert calls == [("browser_press_key", {"key": "Enter"})] + + +def test_a_call_that_times_out_is_sent_once_and_counts_as_dispatched(tmp_path: Path) -> None: + async def scenario() -> Any: + driver = PlaywrightMcpDriver(_config(tmp_path, timeout_s=0.2)) + await driver.start() + try: + return await driver.execute(PressKey("Enter")) + except DriverError as exc: + return exc + finally: + await driver.stop() + + with _mcp({"browser_press_key": [HANG, OK]}) as calls: + outcome = asyncio.run(scenario()) + + assert isinstance(outcome, ActionTimeout) + assert outcome.dispatched is True + assert [call for call in calls if call != "tools/list"] == [("browser_press_key", {"key": "Enter"})] + + +def test_a_closed_transport_is_not_resent(tmp_path: Path) -> None: + replies = {"browser_press_key": [RuntimeError("ClosedResourceError: connection closed"), OK]} + + (outcome,), calls = _execute(tmp_path, replies, PressKey("Enter")) + + assert isinstance(outcome, ConnectionLost) + assert outcome.dispatched is True + assert calls == [("browser_press_key", {"key": "Enter"})] + + +def test_a_call_without_a_connection_is_connection_lost_before_dispatch(tmp_path: Path) -> None: + with patch.object(BrowserMoveStdioClient, "connected", new_callable=PropertyMock, return_value=False): + (outcome,), calls = _execute(tmp_path, {"browser_press_key": OK}, PressKey("Enter")) + + assert isinstance(outcome, ConnectionLost) + assert outcome.dispatched is False + assert calls == [] + + +def test_observe_reads_the_snapshot_header_tabs_elements_and_full_text(tmp_path: Path) -> None: + raw = _raw("snapshot_two_tabs") + + observation, calls = _run(tmp_path, {"browser_snapshot": raw}, lambda driver: driver.observe(Observe())) + + assert isinstance(observation, Observation) + assert observation.context == _tab(0, "Fixture popup page") + assert observation.surface is not None + assert observation.surface.tabs == (_tab(0, "Fixture popup page"), _tab(1, "Fixture second page")) + assert observation.surface.url.endswith("/popup.html") + assert observation.native_text == result_text(raw) + assert observation.native_text.startswith("### Open tabs\n") + assert [element.ref.ref for element in observation.elements] == ["f9e1", "f9e2", "f9e3", "f9e4", "f9e5", "f9e6"] + assert observation.screenshot is None + assert observation.generation == "" + assert calls == [("browser_snapshot", {"boxes": True})] + + +def test_observe_gives_bounds_and_keeps_the_boxes_out_of_the_text(tmp_path: Path) -> None: + raw = _raw("snapshot_with_boxes") + + observation, _calls = _run(tmp_path, {"browser_snapshot": raw}, lambda driver: driver.observe(Observe())) + + assert observation.native_text == without_boxes(result_text(raw)) + assert "[box=" not in observation.native_text + assert {element.ref.ref: element.bounds for element in observation.elements}["f7e2"] == (8.0, 21.0, 1264.0, 37.0) + + +@pytest.mark.parametrize("full_page", [False, True]) +def test_observe_with_a_screenshot_takes_it_at_css_scale(tmp_path: Path, full_page: bool) -> None: + replies = {"browser_snapshot": _raw("snapshot_form_page"), "browser_take_screenshot": _raw("screenshot_element")} + + observation, calls = _run( + tmp_path, replies, lambda driver: driver.observe(Observe(include_screenshot=True, full_page=full_page)) + ) + + assert calls == [ + ("browser_snapshot", {"boxes": True}), + ("browser_take_screenshot", {"type": "png", "fullPage": full_page}), + ] + assert observation.screenshot is not None + assert (observation.screenshot.width, observation.screenshot.height, observation.screenshot.scale) == ( + 1264, + 38, + 1.0, + ) + + +def test_observe_while_a_dialog_is_open_reads_the_tab_list_and_takes_no_screenshot(tmp_path: Path) -> None: + refused = _raw("snapshot_during_dialog") + replies = {"browser_snapshot": refused, "browser_tabs": _raw("tabs_list_during_dialog")} + + observation, calls = _run(tmp_path, replies, lambda driver: driver.observe(Observe(include_screenshot=True))) + + assert calls == [("browser_snapshot", {"boxes": True}), ("browser_tabs", {"action": "list"})] + assert (observation.elements, observation.screenshot) == ((), None) + assert observation.context == _tab(0, "") + assert observation.native_text == result_text(refused) + assert observation.surface is not None + assert observation.surface.url.endswith("/index.html") + assert observation.surface.dialog == "confirm: Delete order?" + + +def test_a_depth_limited_observe_sends_the_depth_alone_and_reads_the_servers_text(tmp_path: Path) -> None: + raw = _raw("snapshot_depth_1") + + observation, calls = _run(tmp_path, {"browser_snapshot": raw}, lambda driver: driver.observe(Observe(depth=1))) + + assert calls == [("browser_snapshot", {"depth": 1})] + assert observation.native_text == result_text(raw) + refs = [element.ref.ref for element in observation.elements] + assert refs == ["e1", "e2", "e3", "e7", "e13", "e14", "e15", "e16", "e17", "e18"] + assert all(element.bounds is None for element in observation.elements) + assert observation.context == _tab(0, "Fixture form page") + + +def test_an_observe_rooted_at_a_ref_sends_the_target_and_reads_only_the_subtree(tmp_path: Path) -> None: + raw = _raw("snapshot_target") + + observation, calls = _run( + tmp_path, {"browser_snapshot": raw}, lambda driver: driver.observe(Observe(root=DriverRef("e3", "g3"))) + ) + + assert calls == [("browser_snapshot", {"target": "e3"})] + assert observation.native_text == result_text(raw) + assert [element.ref.ref for element in observation.elements] == ["e3", "e4", "e5", "e6"] + + +def test_an_observe_rooted_at_a_css_locator_with_a_depth_sends_both(tmp_path: Path) -> None: + raw = _raw("snapshot_target") + + _observation, calls = _run( + tmp_path, {"browser_snapshot": raw}, lambda driver: driver.observe(Observe(depth=2, root=CSS)) + ) + + assert calls == [("browser_snapshot", {"depth": 2, "target": "#save"})] + + +def test_an_observe_rooted_at_a_point_is_unsupported_before_any_call(tmp_path: Path) -> None: + async def body(driver: PlaywrightMcpDriver) -> DriverError: + try: + await driver.observe(Observe(root=POINT)) + except DriverError as exc: + return exc + raise AssertionError("observe accepted a PointRef root") + + error, calls = _run(tmp_path, {}, body) + + assert isinstance(error, Unsupported) and error.dispatched is False + assert calls == [] + + +SENT = [ + (Click(E12), "browser_click", {"element": "click e12", "target": "e12"}), + ( + Click(CSS, button="right", modifiers=("Shift",), count=2), + "browser_click", + {"element": "click #save", "target": "#save", "doubleClick": True, "button": "right", "modifiers": ["Shift"]}, + ), + (Click(POINT), "browser_mouse_click_xy", {"x": 10, "y": 20}), + ( + Click(POINT, button="middle", count=3), + "browser_mouse_click_xy", + {"x": 10, "y": 20, "button": "middle", "clickCount": 3}, + ), + (TypeText(E12, "hello"), "browser_type", {"element": "type into e12", "target": "e12", "text": "hello"}), + ( + TypeText(E12, "hello", submit=True, clear=False, slowly=True), + "browser_type", + {"element": "type into e12", "target": "e12", "text": "hello", "submit": True, "slowly": True}, + ), + (PressKey("Enter"), "browser_press_key", {"key": "Enter"}), + (Hover(E12), "browser_hover", {"element": "hover e12", "target": "e12"}), + (Hover(POINT), "browser_mouse_move_xy", {"x": 10, "y": 20}), + ( + Drag(E12, E5), + "browser_drag", + {"startElement": "drag e12", "startTarget": "e12", "endElement": "drop on e5", "endTarget": "e5"}, + ), + (Drag(POINT, POINT_2), "browser_mouse_drag_xy", {"startX": 10, "startY": 20, "endX": 30, "endY": 40}), + (Scroll(dx=5, dy=-300), "browser_mouse_wheel", {"deltaX": 5, "deltaY": -300}), + ( + SelectOption(E12, ("Large", "Medium")), + "browser_select_option", + {"element": "select in e12", "target": "e12", "values": ["Large", "Medium"]}, + ), + ( + FillForm((FormField(E12, FieldKind.TEXTBOX, "Ada"), FormField(CSS, FieldKind.CHECKBOX, "true"))), + "browser_fill_form", + { + "fields": [ + {"name": "textbox e12", "type": "textbox", "target": "e12", "value": "Ada"}, + {"name": "checkbox #save", "type": "checkbox", "target": "#save", "value": "true"}, + ] + }, + ), + ( + Drop(E12, paths=(MISSING,), data=(("text/plain", "x"),)), + "browser_drop", + {"element": "drop on e12", "target": "e12", "paths": [MISSING], "data": {"text/plain": "x"}}, + ), + (UploadFile(None, (MISSING,)), "browser_file_upload", {"paths": [MISSING]}), + (UploadFile(None), "browser_file_upload", {}), + (HandleDialog(accept=True), "browser_handle_dialog", {"accept": True}), + (HandleDialog(accept=False, prompt_text="x"), "browser_handle_dialog", {"accept": False, "promptText": "x"}), + (Wait(seconds=1.5), "browser_wait_for", {"time": 1.5}), + (Wait(until_text="Done", seconds=3), "browser_wait_for", {"text": "Done"}), + (RunScript("() => 1"), "browser_evaluate", {"function": "() => 1"}), + ( + RunScript("(e) => e.id", target=E12), + "browser_evaluate", + {"element": "evaluate on e12", "target": "e12", "function": "(e) => e.id"}, + ), + (RunScript("(n) => n + 1", args=41), "browser_evaluate", {"function": "() => ((n) => n + 1)(41)"}), + ( + RunScript("(el, n) => n", args={"a": 1}, target=CSS), + "browser_evaluate", + { + "element": "evaluate on #save", + "target": "#save", + "function": '(element) => ((el, n) => n)(element, {"a": 1})', + }, + ), + (Navigate("http://x/"), "browser_navigate", {"url": "http://x/"}), + (Back(), "browser_navigate_back", {}), + (OpenContext(), "browser_tabs", {"action": "new"}), + (OpenContext("http://x/"), "browser_tabs", {"action": "new", "url": "http://x/"}), + (SwitchContext(TAB_1), "browser_tabs", {"action": "select", "index": 1}), + (CloseContext(TAB_1), "browser_tabs", {"action": "close", "index": 1}), + (GetPdf(), "browser_pdf_save", {}), + (GetPdf("out.pdf"), "browser_pdf_save", {"filename": "out.pdf"}), + (GetConsole(), "browser_console_messages", {"level": "info"}), + (GetConsole(level="error", all=True), "browser_console_messages", {"level": "error", "all": True}), + (GetNetwork(), "browser_network_requests", {"static": False}), + (GetNetwork(static=True, filter="/api/"), "browser_network_requests", {"static": True, "filter": "/api/"}), +] + + +@pytest.mark.parametrize(("action", "tool", "arguments"), SENT, ids=[f"{t}:{i}" for i, (_a, t, _x) in enumerate(SENT)]) +def test_handler_sends_one_tool_call(tmp_path: Path, action: Any, tool: str, arguments: dict[str, Any]) -> None: + reply = _raw("tabs_new") if tool == "browser_tabs" else OK + + (result,), calls = _execute(tmp_path, {tool: reply}, action) + + assert isinstance(result, ActionResult) + assert result.ok is True + assert calls == [(tool, arguments)] + + +def _outside_and_server(tmp_path: Path) -> tuple[Path, Path]: + outside = tmp_path / "home" / "cv.pdf" + outside.parent.mkdir() + outside.write_bytes(b"%PDF-cv") + server = tmp_path / "server" + server.mkdir() + return outside, server + + +def test_a_file_outside_the_server_directory_is_uploaded_from_a_copy_under_it(tmp_path: Path) -> None: + outside, server = _outside_and_server(tmp_path) + + async def upload(driver: PlaywrightMcpDriver) -> bytes: + await driver.execute(UploadFile(None, (str(outside),))) + return next((server / "uploads").glob("*/cv.pdf")).read_bytes() + + copied, [(tool, arguments)] = _run(server, {"browser_file_upload": OK}, upload) + + [staged] = map(Path, arguments["paths"]) + assert tool == "browser_file_upload" + assert staged.parent.parent == server / "uploads" and staged.name == "cv.pdf" + assert copied == b"%PDF-cv" + + +def test_stop_removes_the_upload_copies_and_the_uploads_folder_they_left_empty(tmp_path: Path) -> None: + outside, server = _outside_and_server(tmp_path) + + _execute(server, {"browser_file_upload": OK}, UploadFile(None, (str(outside),)), UploadFile(None, (str(outside),))) + + assert not (server / "uploads").exists() + + +def test_stop_leaves_everything_in_the_uploads_folder_it_did_not_stage(tmp_path: Path) -> None: + outside, server = _outside_and_server(tmp_path) + mine = server / "uploads" / "mine" / "keep.txt" + mine.parent.mkdir(parents=True) + mine.write_text("keep", encoding="utf-8") + + _execute(server, {"browser_file_upload": OK}, UploadFile(None, (str(outside),))) + + assert [path.relative_to(server).as_posix() for path in sorted(server.rglob("*"))] == [ + "uploads", + "uploads/mine", + "uploads/mine/keep.txt", + ] + + +def test_a_drop_stages_its_files_too(tmp_path: Path) -> None: + outside, server = _outside_and_server(tmp_path) + + _outcomes, [(_tool, arguments)] = _execute(server, {"browser_drop": OK}, Drop(E12, paths=(str(outside),))) + + assert Path(arguments["paths"][0]).is_relative_to(server / "uploads") + + +def test_files_under_the_server_directory_and_missing_files_go_resolved_without_a_copy(tmp_path: Path) -> None: + _outside, server = _outside_and_server(tmp_path) + (server / "in.csv").write_text("a", encoding="utf-8") + paths = ("in.csv", str(server / "in.csv"), str(tmp_path / "gone.csv"), str(tmp_path / "home")) + + _outcomes, [(_tool, arguments)] = _execute( + server, {"browser_file_upload": OK}, UploadFile(None, paths), upload_root=server + ) + + assert arguments["paths"] == [str(Path(path).resolve()) for path in (server / "in.csv", *paths[1:])] + assert not (server / "uploads").exists() + + +def test_a_relative_name_is_the_file_under_the_upload_root_not_under_the_server_directory(tmp_path: Path) -> None: + allowed, server = tmp_path / "allowed", tmp_path / "server" + for folder in (allowed, server): + folder.mkdir() + (folder / "report.txt").write_text(folder.name, encoding="utf-8") + + async def upload(driver: PlaywrightMcpDriver) -> bytes: + await driver.execute(UploadFile(None, ("report.txt",))) + return next((server / "uploads").glob("*/report.txt")).read_bytes() + + copied, [(_tool, arguments)] = _run(server, {"browser_file_upload": OK}, upload, upload_root=allowed) + + assert Path(arguments["paths"][0]).is_relative_to(server / "uploads") + assert copied == b"allowed" + + +@pytest.mark.parametrize( + "action", [UploadFile(None, ("report.txt",)), Drop(E12, paths=("report.txt",))], ids=["upload", "drop"] +) +def test_a_relative_name_without_an_upload_root_is_refused_before_any_call(tmp_path: Path, action: Any) -> None: + (tmp_path / "report.txt").write_text("x", encoding="utf-8") + + [outcome], calls = _execute(tmp_path, {}, action) + + assert isinstance(outcome, ActionResult) and outcome.ok is False + assert "no upload folder is configured" in outcome.detail + assert calls == [] + + +UNSUPPORTED = [ + Click(E12, count=3), + Click(POINT, modifiers=("Shift",)), + Click(Locator(None, "Save", "g3")), + Click(Locator("#save", "Save", "g3")), + Click(IndexRef(1, "g3")), + TypeText(None, "x"), + TypeText(E12, "x", clear=False), + Drag(E12, POINT), + Drag(POINT, CSS), + PressKey("a", target=E12), + Scroll(E12, dy=10), + UploadFile(E12, ("C:/a.csv",)), + RunScript("() => 1", target=POINT), + FillForm((FormField(POINT, FieldKind.TEXTBOX, "x"),)), +] + + +@pytest.mark.parametrize("action", UNSUPPORTED, ids=[f"{type(a).__name__}:{i}" for i, a in enumerate(UNSUPPORTED)]) +def test_unsupported_commands_send_nothing(tmp_path: Path, action: Any) -> None: + (outcome,), calls = _execute(tmp_path, {}, action) + + assert isinstance(outcome, Unsupported) + assert outcome.dispatched is False + assert calls == [] + + +SECRET = "s3cret-value" +ECHOED = { + "content": [ + { + "type": "text", + "text": f"### Ran Playwright code\n```js\nawait page.x('{SECRET}');\n```\n### Page\n- Page URL: http://x/", + } + ], + "isError": False, +} + + +@pytest.mark.parametrize( + ("action", "tool"), + [ + (TypeText(E12, SECRET, sensitive=True), "browser_type"), + (FillForm((FormField(E12, FieldKind.TEXTBOX, SECRET, sensitive=True),)), "browser_fill_form"), + (RunScript(f"() => '{SECRET}'", sensitive=True), "browser_evaluate"), + ], +) +def test_a_sensitive_command_keeps_the_echoed_code_out_of_the_detail(tmp_path: Path, action: Any, tool: str) -> None: + (result,), _calls = _execute(tmp_path, {tool: ECHOED}, action) + + assert SECRET not in (result.detail or "") + + +def test_a_plain_command_keeps_the_echoed_code(tmp_path: Path) -> None: + (result,), _calls = _execute(tmp_path, {"browser_type": ECHOED}, TypeText(E12, SECRET)) + + assert SECRET in result.detail + + +def test_open_context_value_is_the_new_tab(tmp_path: Path) -> None: + replies = {"browser_navigate": _raw("navigate_second_page"), "browser_tabs": _raw("tabs_new")} + + (_navigated, opened), _calls = _execute(tmp_path, replies, Navigate("http://x/second.html"), OpenContext()) + + assert opened.value == _tab(1, "") + assert opened.opened_contexts == (_tab(1, ""),) + assert opened.context_changed is True + + +def test_close_context_names_the_closed_tab(tmp_path: Path) -> None: + (closed,), _calls = _execute(tmp_path, {"browser_tabs": _raw("tabs_close")}, CloseContext(TAB_1)) + + assert closed.source_context == TAB_1 + assert closed.context_changed is True + + +def test_a_click_without_an_earlier_context_reports_no_change(tmp_path: Path) -> None: + (clicked,), _calls = _execute(tmp_path, {"browser_click": _raw("click_success")}, Click(E12)) + + assert clicked.context_changed is False + + +def test_the_context_of_every_result_is_carried_to_the_next_decode(tmp_path: Path) -> None: + replies = {"browser_navigate": _raw("navigate_second_page"), "browser_click": _raw("click_success")} + + (_navigated, clicked), _calls = _execute(tmp_path, replies, Navigate("http://x/second.html"), Click(E12)) + + assert clicked.context_changed is True + + +def test_the_context_of_an_observation_is_carried_to_the_next_decode(tmp_path: Path) -> None: + replies = {"browser_snapshot": _raw("snapshot_two_tabs"), "browser_click": _raw("click_navigates")} + + async def body(driver: PlaywrightMcpDriver) -> ActionResult: + await driver.observe(Observe()) + return await driver.execute(Click(E12)) + + clicked, _calls = _run(tmp_path, replies, body) + + assert clicked.context_changed is True + assert clicked.opened_contexts == () + + +def test_a_linked_snapshot_is_a_free_observation(tmp_path: Path) -> None: + (tmp_path / SNAPSHOT_FILE).parent.mkdir() + (tmp_path / SNAPSHOT_FILE).write_text(_inline_yaml("snapshot_form_page"), encoding="utf-8") + + (clicked,), calls = _execute(tmp_path, {"browser_click": _raw("click_success")}, Click(E12)) + + assert clicked.observation is not None + assert [element.ref.ref for element in clicked.observation.elements][-3:] == ["f1e1", "f1e2", "f1e3"] + assert [tool for tool, _args in calls] == ["browser_click"] + + +def test_decode_runs_off_the_event_loop(tmp_path: Path) -> None: + threads: list[threading.Thread] = [] + real_decode = driver_module.decode + + def recording_decode(*args: Any, **kwargs: Any) -> ActionResult: + threads.append(threading.current_thread()) + return real_decode(*args, **kwargs) + + with patch.object(driver_module, "decode", recording_decode): + _execute(tmp_path, {"browser_click": _raw("click_success")}, Click(E12)) + + assert threads and all(thread is not threading.main_thread() for thread in threads) + + +def test_a_stale_ref_is_raised_before_dispatch(tmp_path: Path) -> None: + (outcome,), _calls = _execute(tmp_path, {"browser_click": _raw("click_stale_ref")}, Click(E12)) + + assert isinstance(outcome, StaleRef) + assert outcome.dispatched is False + + +def test_an_action_refused_by_a_dialog_is_not_ok(tmp_path: Path) -> None: + (outcome,), _calls = _execute(tmp_path, {"browser_click": _raw("click_blocked_by_dialog")}, Click(E12)) + + assert isinstance(outcome, ActionResult) + assert outcome.ok is False + + +def test_evaluate_returns_the_value(tmp_path: Path) -> None: + (outcome,), _calls = _execute(tmp_path, {"browser_evaluate": _raw("evaluate_result")}, RunScript("() => 1")) + + assert outcome.value == {"name": "Ada", "size": "l"} + + +REMOTE = ProvisionedDevice(kind="chrome", endpoint="http://127.0.0.1:9222", extras={}) +MANAGED = ProvisionedDevice(kind="chrome", endpoint="http://127.0.0.1:9333", extras={"port": 9333}) +EXTENSION = ProvisionedDevice(kind="chrome", endpoint="", extras={"extension": True, "extension_token": "tok"}) +DEVICES = { + "remote": ({"BROWSER_PROVISIONING": "remote", "PLAYWRIGHT_MCP_CDP_ENDPOINT": REMOTE.endpoint}, REMOTE), + "managed": ({"BROWSER_PROVISIONING": "managed"}, MANAGED), + "extension": ({"BROWSER_PROVISIONING": "extension", "PLAYWRIGHT_MCP_EXTENSION_TOKEN": "tok"}, EXTENSION), +} + + +@pytest.fixture +def environ(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Callable[[dict[str, str]], None]: + for key in list(os.environ): + if key.startswith(("BROWSER_", "PLAYWRIGHT_")): + monkeypatch.delenv(key) + monkeypatch.setenv("PLAYWRIGHT_MCP_COMMAND", "npx") + monkeypatch.setenv("PLAYWRIGHT_RUNTIME_WORKDIR", str(tmp_path)) + + def apply(values: dict[str, str]) -> None: + for key, value in values.items(): + monkeypatch.setenv(key, value) + + return apply + + +@pytest.mark.parametrize("provisioning", sorted(DEVICES)) +def test_factory_builds_the_server_config_the_runtime_builds_today(environ, provisioning: str) -> None: + values, device = DEVICES[provisioning] + environ(values) + expected = build_playwright_mcp_config(BrowserInstanceConfig(cdp_url=device.endpoint)) + + driver = create_playwright_mcp_driver(device, BrowserSettings.from_env(os.environ)) + + assert isinstance(driver, PlaywrightMcpDriver) + assert driver.config is not None + assert driver.config.server == expected + assert driver.config.output_dir == Path(expected.params["cwd"]) + caps = [arg for arg in driver.config.server.params["args"] if arg.startswith("--caps=")] + assert len(caps) == 1 and "vision" in caps[0] + + +@pytest.mark.parametrize("provisioning", ["remote", "managed"]) +def test_factory_points_the_server_at_the_device_endpoint(environ, provisioning: str) -> None: + values, device = DEVICES[provisioning] + environ(values) + + server = create_playwright_mcp_driver(device, BrowserSettings.from_env(os.environ)).config.server + + assert server.params["env"]["PLAYWRIGHT_MCP_CDP_ENDPOINT"] == device.endpoint + assert server.params["env"]["PLAYWRIGHT_MCP_BROWSER"] == "chrome" + assert "--extension" not in server.params["args"] + + +def test_factory_keeps_a_given_server_command_line_under_its_own_name_and_endpoint(environ) -> None: + values, device = DEVICES["managed"] + environ(values) + given = McpServerConfig( + server_id="runner-id", + server_name="playwright-official-agent-a", + server_path="stdio://playwright", + client_type="stdio", + params={"command": "node", "args": ["mcp.js", "--isolated"], "cwd": "C:/runs", "env": {"X": "1"}}, + ) + settings = dataclasses.replace(BrowserSettings.from_env(os.environ), mcp_server=given) + + driver = create_playwright_mcp_driver(device, settings) + + server = driver.config.server + assert (server.server_id, server.server_name) == ("playwright_official_stdio", "playwright-official") + assert driver.info.tool_name_prefix == "mcp_playwright-official_" + assert (server.params["command"], server.params["args"], server.params["cwd"]) == ( + "node", + ["mcp.js", "--isolated"], + "C:/runs", + ) + assert server.params["env"]["X"] == "1" + assert server.params["env"]["PLAYWRIGHT_MCP_CDP_ENDPOINT"] == device.endpoint + assert server.params["env"]["PLAYWRIGHT_MCP_BROWSER"] == "chrome" + assert driver.config.output_dir == Path("C:/runs") + + +def test_a_given_server_env_keeps_its_browser_and_loses_its_device(environ) -> None: + values, device = DEVICES["managed"] + environ(values) + given = McpServerConfig( + server_id="runner-id", + server_name="playwright-official-agent-a", + server_path="stdio://playwright", + client_type="stdio", + params={ + "command": "node", + "args": [], + "cwd": "C:/runs", + "env": {"PLAYWRIGHT_MCP_BROWSER": "msedge", "PLAYWRIGHT_MCP_DEVICE": "iPhone 15"}, + }, + ) + settings = dataclasses.replace(BrowserSettings.from_env(os.environ), mcp_server=given) + + env = create_playwright_mcp_driver(device, settings).config.server.params["env"] + + assert env["PLAYWRIGHT_MCP_BROWSER"] == "msedge" + assert "PLAYWRIGHT_MCP_DEVICE" not in env + assert env["PLAYWRIGHT_MCP_CDP_ENDPOINT"] == device.endpoint + + +def test_factory_builds_the_extension_command_line(environ) -> None: + values, device = DEVICES["extension"] + environ(values) + + server = create_playwright_mcp_driver(device, BrowserSettings.from_env(os.environ)).config.server + + assert "--extension" in server.params["args"] + assert server.params["env"]["PLAYWRIGHT_MCP_EXTENSION"] == "true" + assert server.params["env"]["PLAYWRIGHT_MCP_EXTENSION_TOKEN"] == "tok" + assert "PLAYWRIGHT_MCP_CDP_ENDPOINT" not in server.params["env"] diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_result.py b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_result.py new file mode 100644 index 0000000000..be8e49278c --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_result.py @@ -0,0 +1,384 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The Playwright MCP result decoder on recorded @playwright/mcp answers.""" + +from __future__ import annotations + +import dataclasses +import json +from pathlib import Path +from typing import Any + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + ActionIntercepted, + ActionResult, + ActionTimeout, + ConnectionLost, + ContextRef, + DriverError, + Screenshot, + StaleRef, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.result import decode +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext + +FIXTURES_DIR = Path(__file__).resolve().parents[2] / "fixtures" / "mcp" +BASE = "http://127.0.0.1:" +TIMESTAMP = "2026-10-10T00-00-00-000Z" +SNAPSHOT_FILE = Path(".playwright-mcp") / f"page-{TIMESTAMP}.yml" + + +def _fixture(case: str) -> dict[str, Any]: + return json.loads((FIXTURES_DIR / f"{case}.json").read_text(encoding="utf-8")) + + +def _raw(case: str) -> dict[str, Any]: + return json.loads(json.dumps(_fixture(case)["raw_result"]).replace("", TIMESTAMP)) + + +def _decode(case: str, output_dir: Path, before: BrowserContext | None = None) -> ActionResult: + return decode(_fixture(case)["request"]["tool"], _raw(case), output_dir=output_dir, before=before) + + +def _tab(index: int, title: str) -> ContextRef: + return ContextRef(id=str(index), kind="tab", title=title) + + +def _inline_yaml(case: str) -> str: + text = _fixture(case)["raw_result"]["content"][0]["text"] + return text.split("```yaml\n", 1)[1].rsplit("```", 1)[0] + + +def _one_tab(url: str, title: str) -> BrowserContext: + return BrowserContext(url=url, title=title, tabs=(_tab(0, title),), tab_urls=(url,), current_tab=0) + + +FORM = _one_tab(f"{BASE}/index.html", "Fixture form page") +POPUP = _one_tab(f"{BASE}/popup.html", "Fixture popup page") +SECOND = _one_tab(f"{BASE}/second.html", "Fixture second page") +POPUP_AND_SECOND = BrowserContext( + url=f"{BASE}/popup.html", + title="Fixture popup page", + tabs=(_tab(0, "Fixture popup page"), _tab(1, "Fixture second page")), + tab_urls=(f"{BASE}/popup.html", f"{BASE}/second.html"), + current_tab=0, +) +SECOND_AND_BLANK = BrowserContext( + url=f"{BASE}/second.html", + title="Fixture second page", + tabs=(_tab(0, "Fixture second page"), _tab(1, "")), + tab_urls=(f"{BASE}/second.html", "about:blank"), + current_tab=0, +) + + +DECODED = { + "snapshot_form_page": (FORM, True, False), + "snapshot_depth_1": (FORM, True, False), + "snapshot_target": (FORM, True, False), + "snapshot_with_boxes": (FORM, True, False), + "snapshot_after_fill_form": (FORM, True, False), + "snapshot_two_tabs": (POPUP_AND_SECOND, True, False), + "click_success": (FORM, True, False), + "events_console_message": (FORM, True, False), + "events_download": (FORM, True, False), + "dialog_open_by_click": (FORM, True, True), + "dialog_handle_accept": (FORM, True, False), + "click_blocked_by_dialog": (FORM, False, False), + "snapshot_during_dialog": (FORM, False, False), + "tabs_list_during_dialog": (FORM, True, True), + "navigate_same_url_reload": (FORM, True, True), + "navigate_second_page": (FORM, True, True), + "navigate_back": (POPUP_AND_SECOND, True, True), + "click_navigates": (POPUP_AND_SECOND, True, True), + "click_opens_tab": (POPUP, True, True), + "tabs_list": (SECOND, True, True), + "tabs_new": (SECOND, True, True), + "tabs_select": (SECOND_AND_BLANK, True, True), + "tabs_close": (SECOND_AND_BLANK, True, True), + "fill_form": (FORM, True, False), + "select_option": (FORM, True, False), + "evaluate_result": (FORM, True, False), + "screenshot_element": (FORM, True, False), + "file_chooser_open_by_click": ( + _one_tab(f"{BASE}/upload.html", "Fixture upload page"), + True, + False, + ), + "file_upload": ( + _one_tab(f"{BASE}/upload.html", ""), + True, + False, + ), +} + +RAISED = { + "click_intercepted_by_overlay": (ActionIntercepted, False), + "click_timeout_not_enabled": (ActionTimeout, False), + "click_timeout_after_action": (ActionTimeout, True), + "wait_for_timeout": (ActionTimeout, True), + "click_stale_ref": (StaleRef, False), + "evaluate_new_tab_failed": (ConnectionLost, True), + "evaluate_browser_closed": (ConnectionLost, True), + "evaluate_error": (DriverError, True), + "drop_outside_roots": (DriverError, False), +} + + +def test_every_fixture_is_covered() -> None: + assert set(DECODED) | set(RAISED) == {path.stem for path in FIXTURES_DIR.glob("*.json")} + + +@pytest.mark.parametrize("case", sorted(DECODED)) +def test_decoded_outcome(case: str, tmp_path: Path) -> None: + before, ok, changed = DECODED[case] + + result = _decode(case, tmp_path, before) + + assert (result.ok, result.context_changed) == (ok, changed) + assert result.generation == "" + + +@pytest.mark.parametrize("case", sorted(RAISED)) +def test_error_type_and_dispatched(case: str, tmp_path: Path) -> None: + error_type, dispatched = RAISED[case] + + with pytest.raises(DriverError) as caught: + _decode(case, tmp_path, FORM) + + assert type(caught.value) is error_type + assert caught.value.dispatched is dispatched + assert "\x1b" not in str(caught.value) + + +def test_intercepted_click_names_the_blocker(tmp_path: Path) -> None: + with pytest.raises(ActionIntercepted) as caught: + _decode("click_intercepted_by_overlay", tmp_path) + + assert caught.value.by == "div#consent-backdrop.consent-dark-filter" + + +def test_unclassified_error_keeps_the_text(tmp_path: Path) -> None: + with pytest.raises(DriverError) as caught: + _decode("evaluate_error", tmp_path) + + assert str(caught.value).startswith("Error: fixture failure\n at eval") + + +def test_detail_drops_events_and_the_snapshot_link(tmp_path: Path) -> None: + result = _decode("events_download", tmp_path) + + assert result.detail == ( + "### Ran Playwright code\n```js\n" + "await page.getByRole('link', { name: 'Download report' }).click();\n```\n" + f"### Page\n- Page URL: {BASE}/index.html\n- Page Title: Fixture form page" + ) + + +def test_a_sensitive_call_leaves_the_code_out_of_the_detail(tmp_path: Path) -> None: + raw = _raw("click_success") + + sensitive = decode("browser_click", raw, output_dir=tmp_path, sensitive=True) + plain = decode("browser_click", raw, output_dir=tmp_path) + + assert plain.detail is not None and "### Ran Playwright code" in plain.detail + assert sensitive.detail == f"### Page\n- Page URL: {BASE}/index.html\n- Page Title: Fixture form page" + + +@pytest.mark.parametrize("case", ["events_console_message", "click_success", "navigate_second_page"]) +def test_detail_has_no_events_or_file_links(case: str, tmp_path: Path) -> None: + detail = _decode(case, tmp_path).detail + + assert detail is not None + assert "### Events" not in detail + assert "### Snapshot" not in detail + assert ".playwright-mcp" not in detail + + +def test_same_url_reload_changes_the_context(tmp_path: Path) -> None: + result = _decode("navigate_same_url_reload", tmp_path, FORM) + + assert result.context_changed is True + assert result.surface == FORM + + +def test_navigation_changes_the_context_without_before(tmp_path: Path) -> None: + assert _decode("navigate_second_page", tmp_path).context_changed is True + + +def test_click_without_url_or_tab_change_keeps_the_context(tmp_path: Path) -> None: + result = _decode("click_success", tmp_path, FORM) + + assert result.context_changed is False + assert result.opened_contexts == () + + +def test_click_that_changes_the_url_changes_the_context(tmp_path: Path) -> None: + result = _decode("click_navigates", tmp_path, POPUP_AND_SECOND) + + assert result.context_changed is True + assert result.surface == BrowserContext( + url=f"{BASE}/second.html", + title="Fixture second page", + tabs=(_tab(0, "Fixture second page"), _tab(1, "Fixture second page")), + tab_urls=(f"{BASE}/second.html", f"{BASE}/second.html"), + current_tab=0, + ) + + +def test_click_that_opens_a_tab_reports_it(tmp_path: Path) -> None: + result = _decode("click_opens_tab", tmp_path, POPUP) + + assert result.context_changed is True + assert result.opened_contexts == (_tab(1, "Fixture second page"),) + assert result.surface == POPUP_AND_SECOND + + +def test_new_tab_from_the_tab_list(tmp_path: Path) -> None: + result = _decode("tabs_new", tmp_path, SECOND) + + assert result.opened_contexts == (_tab(1, ""),) + assert result.surface == BrowserContext( + url="about:blank", title="", tabs=SECOND_AND_BLANK.tabs, tab_urls=SECOND_AND_BLANK.tab_urls, current_tab=1 + ) + assert (result.value, result.detail) == (None, None) + + +def test_closing_a_tab_reports_the_remaining_tabs(tmp_path: Path) -> None: + result = _decode("tabs_close", tmp_path, SECOND_AND_BLANK) + + assert result.opened_contexts == () + assert result.surface == SECOND + + +def test_dialog_opened_by_click_changes_the_context(tmp_path: Path) -> None: + result = _decode("dialog_open_by_click", tmp_path) + + assert result.context_changed is True + assert result.detail is not None + assert '["confirm" dialog with message "Delete order?"]' in result.detail + assert result.surface == dataclasses.replace(_one_tab(f"{BASE}/index.html", ""), dialog="confirm: Delete order?") + + +def test_file_chooser_is_reported_but_keeps_the_context(tmp_path: Path) -> None: + result = _decode("file_chooser_open_by_click", tmp_path) + + assert result.context_changed is False + assert result.detail is not None + assert "- [File chooser]: can be handled by browser_file_upload" in result.detail + assert result.surface is not None and result.surface.dialog is None + + +def test_action_refused_by_an_open_dialog_is_not_ok(tmp_path: Path) -> None: + result = _decode("click_blocked_by_dialog", tmp_path) + + assert result.ok is False + assert result.detail == ( + '### Error\nError: Tool "browser_click" does not handle the modal state.\n' + '### Modal state\n- ["confirm" dialog with message "Delete order?"]: can be handled by browser_handle_dialog' + ) + assert result.surface is None + + +def test_action_refused_by_an_open_dialog_shows_it_on_the_page_before(tmp_path: Path) -> None: + result = _decode("click_blocked_by_dialog", tmp_path, FORM) + + assert result.surface == dataclasses.replace(FORM, dialog="confirm: Delete order?") + + +def test_tab_list_during_a_dialog_shows_the_page_and_the_dialog(tmp_path: Path) -> None: + result = _decode("tabs_list_during_dialog", tmp_path, FORM) + + assert result.surface == dataclasses.replace(_one_tab(f"{BASE}/index.html", ""), dialog="confirm: Delete order?") + + +def test_dialog_message_keeps_quotes_and_line_breaks(tmp_path: Path) -> None: + text = ( + "### Page\n- Page URL: http://x/\n### Modal state\n" + '- ["prompt" dialog with message "Say "hi"\nand go"]: can be handled by browser_handle_dialog\n' + "- [File chooser]: can be handled by browser_file_upload" + ) + raw = {"content": [{"type": "text", "text": text}], "isError": False} + + result = decode("browser_click", raw, output_dir=tmp_path) + + assert result.surface is not None + assert result.surface.dialog == 'prompt: Say "hi"\nand go' + + +def test_inline_snapshot_becomes_the_observation(tmp_path: Path) -> None: + result = _decode("snapshot_form_page", tmp_path) + + observation = result.observation + assert observation is not None + assert observation.native_text == _inline_yaml("snapshot_form_page") + assert len(observation.elements) == 21 + assert observation.context == _tab(0, "Fixture form page") + assert observation.surface == FORM == result.surface + assert observation.generation == "" + + +def test_linked_snapshot_file_becomes_the_observation(tmp_path: Path) -> None: + yaml = _inline_yaml("snapshot_form_page") + (tmp_path / SNAPSHOT_FILE).parent.mkdir() + (tmp_path / SNAPSHOT_FILE).write_text(yaml, encoding="utf-8") + + result = _decode("click_success", tmp_path, FORM) + + assert result.observation is not None + assert result.observation.native_text == yaml + assert [element.ref.ref for element in result.observation.elements][-3:] == ["f1e1", "f1e2", "f1e3"] + assert result.observation.surface == FORM + + +def test_missing_snapshot_file_gives_no_observation(tmp_path: Path) -> None: + assert _decode("click_success", tmp_path).observation is None + + +def test_empty_snapshot_file_gives_no_observation(tmp_path: Path) -> None: + (tmp_path / SNAPSHOT_FILE).parent.mkdir() + (tmp_path / SNAPSHOT_FILE).write_text("", encoding="utf-8") + + assert _decode("dialog_open_by_click", tmp_path).observation is None + + +def test_snapshot_link_outside_the_output_dir_is_not_read(tmp_path: Path) -> None: + output_dir = tmp_path / "server" + output_dir.mkdir() + (tmp_path / "outside.yml").write_text(_inline_yaml("snapshot_form_page"), encoding="utf-8") + raw = _raw("click_success") + raw["content"][0]["text"] = raw["content"][0]["text"].replace( + f".playwright-mcp\\page-{TIMESTAMP}.yml", "..\\outside.yml" + ) + + assert decode("browser_click", raw, output_dir=output_dir).observation is None + + +def test_evaluate_result_is_the_value(tmp_path: Path) -> None: + result = _decode("evaluate_result", tmp_path) + + assert result.value == {"name": "Ada", "size": "l"} + assert result.detail is not None + assert result.detail.startswith("### Ran Playwright code\n") + assert "### Result" not in result.detail + assert result.surface is None + + +def test_screenshot_image_is_the_value(tmp_path: Path) -> None: + result = _decode("screenshot_element", tmp_path) + + assert isinstance(result.value, Screenshot) + assert result.value.png.startswith(b"\x89PNG\r\n\x1a\n") + assert (result.value.width, result.value.height, result.value.scale) == (1264, 38, 1.0) + assert result.detail is not None + assert "### Result" not in result.detail + + +def test_result_without_page_has_no_surface(tmp_path: Path) -> None: + result = _decode("fill_form", tmp_path, FORM) + + assert (result.surface, result.observation, result.value) == (None, None, None) + assert result.detail is not None + assert "setChecked(true)" in result.detail diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_snapshot.py b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_snapshot.py new file mode 100644 index 0000000000..7b0012b9d4 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_snapshot.py @@ -0,0 +1,155 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The Playwright MCP snapshot parser on recorded @playwright/mcp answers.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from openjiuwen.harness.tools.browser_move.contract import ContextRef, Element +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.snapshot import ( + parse_elements, + parse_snapshot, + parse_tabs, + split_sections, + without_boxes, +) + +FIXTURES_DIR = Path(__file__).resolve().parents[2] / "fixtures" / "mcp" + + +def _text(case: str) -> str: + fixture = json.loads((FIXTURES_DIR / f"{case}.json").read_text(encoding="utf-8")) + return "\n".join(block["text"] for block in fixture["raw_result"]["content"] if block["type"] == "text") + + +def _by_ref(elements: tuple[Element, ...]) -> dict[str, Element]: + return {element.ref.ref: element for element in elements} + + +def test_form_page_has_every_ref_including_the_iframe() -> None: + elements, _ = parse_snapshot(_text("snapshot_form_page")) + + assert [element.ref.ref for element in elements] == [f"e{n}" for n in range(1, 19)] + ["f1e1", "f1e2", "f1e3"] + assert all(element.ref.generation == "" for element in elements) + + +def test_form_page_roles_and_names() -> None: + elements = _by_ref(parse_snapshot(_text("snapshot_form_page"))[0]) + + assert [(e.role, e.name) for e in elements.values()][:6] == [ + ("generic", ""), + ("heading", "Fixture form page"), + ("navigation", ""), + ("link", "Second page"), + ("link", "Blocked page"), + ("link", "Download report"), + ] + assert (elements["e18"].role, elements["f1e3"].role, elements["f1e3"].name) == ("iframe", "button", "Frame button") + + +def test_form_page_attributes_properties_and_text_go_to_extra() -> None: + elements = _by_ref(parse_snapshot(_text("snapshot_form_page"))[0]) + + assert elements["e2"].extra == {"level": "1"} + assert elements["e4"].extra == {"cursor": "pointer", "url": "second.html"} + assert elements["e8"].extra == {"placeholder": "Your name"} + assert elements["e13"].extra == {"text": "Not saved"} + assert elements["f1e2"].extra == {"text": "Inside the frame"} + + +def test_form_page_values_focus_and_bounds() -> None: + elements = _by_ref(parse_snapshot(_text("snapshot_form_page"))[0]) + + assert elements["e8"].value is None + assert elements["e9"].value == "Small" + assert elements["e1"].focused is True + assert not any(element.focused for ref, element in elements.items() if ref != "e1") + assert all(element.enabled and element.bounds is None for element in elements.values()) + + +def test_form_page_header() -> None: + _, header = parse_snapshot(_text("snapshot_form_page")) + + tab = ContextRef(id="0", kind="tab", title="Fixture form page") + assert header is not None + assert (header.url, header.title, header.tabs, header.focused) == ( + "http://127.0.0.1:/index.html", + "Fixture form page", + (tab,), + tab, + ) + + +def test_boxes_become_bounds() -> None: + elements = _by_ref(parse_snapshot(_text("snapshot_with_boxes"))[0]) + + assert len(elements) == 21 + assert elements["f7e2"].bounds == (8.0, 21.0, 1264.0, 37.0) + assert elements["f8e3"].bounds == (8.0, 50.0, 95.0, 21.0) + assert "box" not in elements["f7e2"].extra + + +def test_without_boxes_removes_only_the_boxes() -> None: + text = _text("snapshot_with_boxes") + + plain = without_boxes(text) + + assert "[box=" not in plain + assert plain.count("[ref=") == text.count("[ref=") + assert ' - heading "Fixture form page" [level=1] [ref=f7e2]\n' in plain + + +def test_filled_form_values_and_checked_state() -> None: + elements = _by_ref(parse_snapshot(_text("snapshot_after_fill_form"))[0]) + + assert elements["f7e8"].value == "Ada" + assert elements["f7e9"].value == "Medium" + assert elements["f7e11"].extra == {"checked": True} + assert elements["f7e11"].focused is True + + +def test_quoted_yaml_lines_are_unquoted() -> None: + elements = _by_ref(parse_snapshot(_text("snapshot_two_tabs"))[0]) + + assert (elements["f9e5"].role, elements["f9e5"].name) == ("button", "Note: read me") + assert elements["f9e6"].extra == {"text": "42"} + + +def test_open_tabs_section_lists_every_tab() -> None: + _, header = parse_snapshot(_text("snapshot_two_tabs")) + + assert header is not None + assert header.tabs == ( + ContextRef(id="0", kind="tab", title="Fixture popup page"), + ContextRef(id="1", kind="tab", title="Fixture second page"), + ) + assert header.focused == header.tabs[0] + assert header.url == "http://127.0.0.1:/popup.html" + + +def test_tab_list_result_gives_tabs_and_the_focused_page() -> None: + header = parse_tabs(split_sections(_text("tabs_new"))["Result"]) + + assert header is not None + assert header.tabs == ( + ContextRef(id="0", kind="tab", title="Fixture second page"), + ContextRef(id="1", kind="tab", title=""), + ) + assert (header.focused, header.url, header.title) == (header.tabs[1], "about:blank", "") + + +def test_tab_list_without_tabs_is_none() -> None: + assert parse_tabs("No open tabs. Navigate to a URL to create one.") is None + + +def test_disabled_flag_and_nodes_without_ref() -> None: + elements = parse_elements('- button "Disabled button" [disabled] [ref=f5e4]\n- text: no ref\n- option "Small"') + + assert len(elements) == 1 + assert (elements[0].ref.ref, elements[0].enabled, elements[0].extra) == ("f5e4", False, {"disabled": True}) + + +def test_sections_keep_their_order() -> None: + assert list(split_sections(_text("click_opens_tab"))) == ["Ran Playwright code", "Open tabs", "Page", "Snapshot"] diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_tools.py b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_tools.py new file mode 100644 index 0000000000..0338f00a96 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/playwright_mcp/test_tools.py @@ -0,0 +1,222 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Decoders of the @playwright/mcp tool arguments: the inverse of the driver's handlers.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + Click, + CloseContext, + ContextRef, + Drag, + DriverRef, + Hover, + Locator, + Observe, + PointRef, + Ref, + RunScript, + Scroll, + SwitchContext, + TypeText, +) +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.tools import DECODERS +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Drop, + FieldKind, + FillForm, + FormField, + GetConsole, + GetNetwork, + OpenContext, + UploadFile, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.tools import BROWSER_TOOL_SPECS +from tests.unit_tests.harness.tools.browser_move.drivers.playwright_mcp.test_driver import OK, SENT, _execute, _raw +from tests.unit_tests.harness.tools.browser_move.snapshots.builder import load_mcp_tools + +TAB_0 = ContextRef("0", "tab", "First") +TAB_1 = ContextRef("1", "tab", "Second") + + +@dataclass +class _Resolver: + contexts: tuple[ContextRef, ...] = (TAB_0, TAB_1) + current_context: ContextRef | None = TAB_0 + generation: str = "g3" + screenshot_generation: str = "" + + def resolve(self, model_id: str, generation: str) -> Ref: + raise ValueError(f"unknown target_id {model_id}") + + def generation_of(self, ref: str) -> str: + return self.generation + + +RESOLVER = _Resolver() +ROUND_TRIPS = [(tool, arguments) for _action, tool, arguments in SENT if tool in DECODERS] + + +def _decode(tool: str, arguments: dict[str, Any], resolver: _Resolver = RESOLVER) -> Any: + return DECODERS[tool](arguments, resolver) + + +@pytest.mark.parametrize(("tool", "arguments"), ROUND_TRIPS, ids=[f"{t}:{i}" for i, (t, _a) in enumerate(ROUND_TRIPS)]) +def test_decoding_then_sending_gives_back_the_arguments(tmp_path: Path, tool: str, arguments: dict[str, Any]) -> None: + reply = _raw("tabs_new") if tool == "browser_tabs" else OK + + _outcomes, calls = _execute(tmp_path, {tool: reply}, _decode(tool, arguments)) + + assert calls == [(tool, arguments)] + + +def test_every_tool_spec_the_server_offers_has_a_decoder_and_nothing_else_does() -> None: + offered = {tool["name"] for tool in load_mcp_tools()} + specs = {spec.name for spec in BROWSER_TOOL_SPECS} + assert set(DECODERS) == specs & offered + + +@pytest.mark.parametrize( + ("target", "ref"), + [ + ("e12", DriverRef("e12", "")), + ("f1e3", DriverRef("f1e3", "")), + ("#save", Locator("#save", None, "")), + ("e12 > span", Locator("e12 > span", None, "")), + ], +) +def test_target_is_a_driver_ref_for_an_aria_ref_and_a_css_locator_otherwise(target: str, ref: Ref) -> None: + assert _decode("browser_hover", {"target": target}) == Hover(ref) + + +def test_a_target_carries_no_generation_even_when_the_resolver_saw_it_in_an_older_one() -> None: + older = _Resolver(generation="g1") + assert _decode("browser_hover", {"target": "e1"}, older) == Hover(DriverRef("e1", "")) + assert _decode("browser_hover", {"target": "#save"}, older) == Hover(Locator("#save", None, "")) + + +def test_double_click_is_a_click_counted_twice() -> None: + assert _decode("browser_click", {"target": "e1", "doubleClick": True}).count == 2 + assert _decode("browser_click", {"target": "e1", "doubleClick": False}).count == 1 + + +def test_typing_slowly_keeps_the_content() -> None: + assert _decode("browser_type", {"target": "e1", "text": "a"}) == TypeText(DriverRef("e1", ""), "a") + slow = _decode("browser_type", {"target": "e1", "text": "a", "slowly": True}) + assert (slow.slowly, slow.clear) == (True, False) + + +@pytest.mark.parametrize( + ("tool", "arguments", "action"), + [ + ("browser_click", {"target": "e1", "element": "Save button"}, Click(DriverRef("e1", ""))), + ("browser_snapshot", {"boxes": True, "filename": "a.md"}, Observe()), + ( + "browser_take_screenshot", + {"target": "e1", "element": "logo", "type": "jpeg", "scale": "device", "filename": "a.png"}, + Observe(include_screenshot=True), + ), + ("browser_evaluate", {"function": "() => 1", "filename": "out.json"}, RunScript("() => 1")), + ("browser_mouse_click_xy", {"x": 1, "y": 2, "delay": 50}, Click(PointRef(1, 2, ""))), + ("browser_console_messages", {"level": "error", "filename": "c.txt"}, GetConsole(level="error")), + ("browser_network_requests", {"static": False, "filename": "n.txt"}, GetNetwork()), + ], +) +def test_cosmetic_and_dropped_arguments_do_not_change_the_command(tool: str, arguments: dict, action: Any) -> None: + assert _decode(tool, arguments) == action + + +def test_snapshot_depth_and_target_scope_the_observation() -> None: + assert _decode("browser_snapshot", {}) == Observe() + assert _decode("browser_snapshot", {"depth": 1}) == Observe(depth=1) + assert _decode("browser_snapshot", {"target": "e3"}) == Observe(root=DriverRef("e3", "")) + assert _decode("browser_snapshot", {"target": "nav", "depth": 2.0}) == Observe( + depth=2, root=Locator("nav", None, "") + ) + with pytest.raises(ValueError, match="'depth' must be a number"): + _decode("browser_snapshot", {"depth": "shallow"}) + + +def test_points_take_rounded_screenshot_pixels_and_no_generation_before_a_screenshot() -> None: + assert _decode("browser_mouse_move_xy", {"x": 10.6, "y": 20.2}) == Hover(PointRef(11, 20, "")) + dragged = _decode("browser_mouse_drag_xy", {"startX": 1, "startY": 2, "endX": 3, "endY": 4}) + assert dragged == Drag(PointRef(1, 2, ""), PointRef(3, 4, "")) + + +def test_points_take_the_generation_of_the_last_screenshot() -> None: + shot = _Resolver(screenshot_generation="g2") + assert _decode("browser_mouse_click_xy", {"x": 1, "y": 2}, shot) == Click(PointRef(1, 2, "g2")) + dragged = _decode("browser_mouse_drag_xy", {"startX": 1, "startY": 2, "endX": 3, "endY": 4}, shot) + assert dragged == Drag(PointRef(1, 2, "g2"), PointRef(3, 4, "g2")) + + +def test_wheel_scrolls_the_whole_view() -> None: + assert _decode("browser_mouse_wheel", {"deltaX": 0, "deltaY": 300}) == Scroll(None, dx=0, dy=300) + + +def test_fill_form_and_drop_take_the_server_shapes() -> None: + form = _decode("browser_fill_form", {"fields": [{"target": "#a", "name": "A", "type": "radio", "value": "true"}]}) + assert form == FillForm((FormField(Locator("#a", None, ""), FieldKind.RADIO, "true"),)) + drop = _decode("browser_drop", {"target": "e1", "data": {"text/plain": "x", "text/uri-list": "https://a.test"}}) + assert drop == Drop(DriverRef("e1", ""), data=(("text/plain", "x"), ("text/uri-list", "https://a.test"))) + + +def test_file_upload_answers_the_open_chooser() -> None: + assert _decode("browser_file_upload", {"paths": ["C:/a.csv"]}) == UploadFile(None, ("C:/a.csv",)) + assert _decode("browser_file_upload", {}) == UploadFile(None) + + +@pytest.mark.parametrize( + ("arguments", "action"), + [ + ({"action": "list"}, Observe()), + ({"action": "new"}, OpenContext()), + ({"action": "new", "url": "https://a.test/"}, OpenContext("https://a.test/")), + ({"action": "select", "index": 1}, SwitchContext(TAB_1)), + ({"action": "close", "index": 1}, CloseContext(TAB_1)), + ({"action": "close"}, CloseContext(TAB_0)), + ], +) +def test_tabs_action_picks_the_command(arguments: dict, action: Any) -> None: + assert _decode("browser_tabs", arguments) == action + + +def test_close_takes_the_current_tab() -> None: + assert _decode("browser_close", {}) == CloseContext(TAB_0) + + +@pytest.mark.parametrize( + ("tool", "arguments", "message"), + [ + ("browser_click", {}, "'target' is required"), + ("browser_type", {"target": "e1"}, "'text' is required"), + ("browser_navigate", {}, "'url' is required"), + ("browser_select_option", {"target": "e1", "values": []}, "at least one option"), + ("browser_select_option", {"target": "e1", "values": "Large"}, "must be a list"), + ("browser_fill_form", {"fields": []}, "non-empty list"), + ("browser_fill_form", {"fields": [{"target": "e1", "type": "date", "value": "x"}]}, "unsupported type"), + ("browser_drop", {"target": "e1"}, "Drop needs paths or data"), + ("browser_drop", {"target": "e1", "data": ["x"]}, "MIME types"), + ("browser_mouse_click_xy", {"y": 1}, "'x' is required"), + ("browser_mouse_click_xy", {"x": "left", "y": 1}, "'x' must be a number"), + ("browser_handle_dialog", {}, "'accept' is required"), + ("browser_tabs", {}, "'action' is required"), + ("browser_tabs", {"action": "move"}, "action must be one of"), + ("browser_tabs", {"action": "select"}, "requires index"), + ("browser_tabs", {"action": "select", "index": 2}, "out of range: 2 tab(s) open"), + ], +) +def test_invalid_arguments_are_a_value_error(tool: str, arguments: dict, message: str) -> None: + with pytest.raises(ValueError, match=message.replace("(", r"\(").replace(")", r"\)")): + _decode(tool, arguments) + + +def test_closing_with_no_open_tab_is_a_value_error() -> None: + with pytest.raises(ValueError, match="no open tab"): + _decode("browser_tabs", {"action": "close"}, _Resolver(contexts=(), current_context=None)) diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/test_browser_use_driver.py b/tests/unit_tests/harness/tools/browser_move/drivers/test_browser_use_driver.py new file mode 100644 index 0000000000..c26cf2e143 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/test_browser_use_driver.py @@ -0,0 +1,1006 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""``BrowserUseDriver`` on a mocked sidecar transport: wire requests, observation and error mapping, factory.""" + +from __future__ import annotations + +import asyncio +import base64 +import importlib +import json +import shutil +import struct +import subprocess +import sys +import tempfile +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + ActionIntercepted, + ActionResult, + ActionTimeout, + AmbiguousElement, + Click, + CloseContext, + ConnectionLost, + ContextRef, + Drag, + DriverError, + DriverRef, + ElementNotFound, + HandleDialog, + Hover, + IndexRef, + Locator, + NavigationError, + Observe, + PointRef, + PressKey, + RunScript, + Scroll, + StaleRef, + SwitchContext, + TypeText, + Unsupported, + Wait, +) +from openjiuwen.harness.tools.browser_move.drivers.browser_use import driver as driver_module +from openjiuwen.harness.tools.browser_move.drivers.browser_use.driver import ( + BrowserUseConfig, + BrowserUseDriver, + create_browser_use_driver, +) +from openjiuwen.harness.tools.browser_move.drivers.browser_use.sidecar import wire +from openjiuwen.harness.tools.browser_move.drivers.browser_use.transport import SidecarTransport +from openjiuwen.harness.tools.browser_move.provisioning import ProvisionedDevice +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FieldKind, + FillForm, + FormField, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext + +E1 = DriverRef("bnid:42", "g3") +E2 = DriverRef("bnid:43", "g3") +WIRE_E1 = {"kind": "driver", "handle": "bnid:42", "driver_generation": 0} +WIRE_E2 = {"kind": "driver", "handle": "bnid:43", "driver_generation": 0} +CSS = Locator("#save", None, "g3") +CSS_2 = Locator("form .size", None, "g3") +WIRE_CSS = {"kind": "selector", "css": "#save", "nth": 0} +WIRE_CSS_2 = {"kind": "selector", "css": "form .size", "nth": 0} +TAB_2 = ContextRef(id="T2", kind="tab", title="Second") +WIRE_TAB_2 = {"target_id": "T2", "url": "", "title": "Second", "active": False} +ACT_OK = {"ok": True, "detail": "done", "document_changed": False, "driver_generation": 0} +NAV_SAME_URL = {"url": "http://x/", "title": "X", "changed_document": False, "driver_generation": 0} + + +def _driver(*replies: Any) -> tuple[BrowserUseDriver, AsyncMock]: + transport = MagicMock() + transport.request = AsyncMock(side_effect=list(replies)) + driver = BrowserUseDriver() + driver._transport = transport + return driver, transport.request + + +def _calls(request: AsyncMock) -> list[tuple[str, dict[str, Any]]]: + return [call.args for call in request.call_args_list] + + +def _png(width: int, height: int) -> bytes: + return b"\x89PNG\r\n\x1a\n" + b"\x00\x00\x00\rIHDR" + struct.pack(">II", width, height) + b"\x08\x06\x00\x00\x00" + + +def _sidecar_error(code: str, message: str = "boom", data: dict[str, Any] | None = None) -> Exception: + try: + SidecarTransport._unwrap_reply({"id": 1, "error": {"code": code, "message": message, "data": data or {}}}) + except Exception as exc: # noqa: BLE001 + return exc + raise AssertionError("transport did not raise") + + +EXPECTED_ACTIONS = { + Click: {DriverRef, Locator}, + TypeText: {DriverRef, Locator}, + PressKey: set(), + Scroll: {DriverRef}, + Hover: {DriverRef, Locator}, + Drag: {DriverRef, Locator}, + Wait: set(), + SwitchContext: set(), + CloseContext: set(), + HandleDialog: set(), + RunScript: {DriverRef}, + Navigate: set(), + Back: set(), + OpenContext: set(), + SelectOption: {DriverRef, Locator}, + UploadFile: {DriverRef, Locator}, + FillForm: {DriverRef, Locator}, + Drop: {DriverRef, Locator}, +} + + +def test_info_actions_are_pinned(): + actions = BrowserUseDriver().info.actions + assert {command: set(refs) for command, refs in actions.items()} == EXPECTED_ACTIONS + + +def test_index_ref_is_never_a_model_facing_ref_kind(): + for refs in BrowserUseDriver().info.actions.values(): + assert IndexRef not in refs + + +def test_info_describes_the_browser_surface(): + info = BrowserUseDriver().info + assert info.name == "browser_use" + assert info.surfaces == frozenset({"browser"}) + assert info.observes == frozenset({"elements", "screenshot", "native_text", "surface"}) + assert info.display_name == "browser-use" + + +@pytest.mark.parametrize( + "action", + [Click(IndexRef(1, "g3")), Click(PointRef(1, 2, "g3")), Scroll(CSS, dy=10), RunScript("() => 1", target=CSS)], + ids=["index-ref", "point-ref", "scroll-locator", "run-script-locator"], +) +def test_other_ref_kinds_are_unsupported_before_dispatch(action): + driver, request = _driver() + with pytest.raises(Unsupported) as info: + asyncio.run(driver.execute(action)) + assert info.value.dispatched is False + request.assert_not_called() + + +@pytest.mark.parametrize( + "locator", [Locator(None, "Save", "g3"), Locator("#save", "Save", "g3")], ids=["text", "css-and-text"] +) +def test_a_locator_by_text_is_unsupported_before_dispatch(locator): + driver, request = _driver() + with pytest.raises(Unsupported, match="by css only") as info: + asyncio.run(driver.execute(Click(locator))) + assert info.value.dispatched is False + request.assert_not_called() + + +WIRE_CASES = [ + ( + Click(E1, button="right", modifiers=("Shift",), count=2), + "click", + {"ref": WIRE_E1, "button": "right", "click_count": 2, "modifiers": ["Shift"]}, + ), + ( + TypeText(E1, "hello", submit=True, sensitive=True, clear=False), + "type_text", + {"ref": WIRE_E1, "text": "hello", "clear": False, "press_enter": True, "sensitive": True, "slowly": False}, + ), + ( + TypeText(E1, "Ada", slowly=True), + "type_text", + {"ref": WIRE_E1, "text": "Ada", "clear": True, "press_enter": False, "sensitive": False, "slowly": True}, + ), + (PressKey("Control+A"), "press_key", {"keys": "Control+A"}), + (Scroll(dy=300), "scroll", {"direction": "down", "amount": 300, "ref": None}), + (Scroll(dy=-200), "scroll", {"direction": "up", "amount": 200, "ref": None}), + (Scroll(E1, dx=50), "scroll", {"direction": "right", "amount": 50, "ref": WIRE_E1}), + (Scroll(dx=-50), "scroll", {"direction": "left", "amount": 50, "ref": None}), + (Hover(E1), "hover", {"ref": WIRE_E1}), + (Drag(E1, E2), "drag", {"source": WIRE_E1, "target": WIRE_E2, "steps": 10, "delay_ms": 0}), + (SelectOption(E1, ("Blue",)), "select_option", {"ref": WIRE_E1, "value": "Blue", "label": None}), + (HandleDialog(accept=False, prompt_text="x"), "handle_dialog", {"accept": False, "prompt_text": "x"}), +] + + +@pytest.mark.parametrize(("action", "method", "params"), WIRE_CASES, ids=lambda case: getattr(case, "__name__", None)) +def test_act_handler_sends_one_wire_request(action, method, params): + driver, request = _driver(dict(ACT_OK)) + result = asyncio.run(driver.execute(action)) + assert _calls(request) == [(method, params)] + assert result.ok is True + assert result.detail == "done" + assert result.generation == "" + + +@pytest.mark.parametrize( + ("action", "method", "params"), + [ + ( + Navigate("http://x/"), + "navigate", + {"url": "http://x/", "wait_until": "load", "timeout_ms": None, "new_tab": False}, + ), + (Back(), "go_back", {}), + ], +) +def test_navigation_handler_sends_one_wire_request(action, method, params): + driver, request = _driver(dict(NAV_SAME_URL)) + result = asyncio.run(driver.execute(action)) + assert _calls(request) == [(method, params)] + assert result.ok is True + assert result.surface == BrowserContext(url="http://x/", title="X") + + +@pytest.mark.parametrize("action", [Navigate("http://x/"), Back()]) +def test_a_dialog_the_loaded_page_opened_is_in_the_navigation_result(action): + driver, _request = _driver({**NAV_SAME_URL, "dialog": {"type": "alert", "message": "Welcome"}}) + result = asyncio.run(driver.execute(action)) + assert result.surface == BrowserContext(url="http://x/", title="X", dialog="alert: Welcome") + + +@pytest.mark.parametrize( + ("action", "method", "params"), + [ + (Click(CSS), "click", {"ref": WIRE_CSS, "button": "left", "click_count": 1, "modifiers": []}), + ( + TypeText(CSS, "hi"), + "type_text", + {"ref": WIRE_CSS, "text": "hi", "clear": True, "press_enter": False, "sensitive": False, "slowly": False}, + ), + (Hover(CSS), "hover", {"ref": WIRE_CSS}), + (Drag(CSS, CSS_2), "drag", {"source": WIRE_CSS, "target": WIRE_CSS_2, "steps": 10, "delay_ms": 0}), + (SelectOption(CSS_2, ("m",)), "select_option", {"ref": WIRE_CSS_2, "value": "m", "label": None}), + ], + ids=["click", "type-text", "hover", "drag", "select-option"], +) +def test_a_css_locator_is_sent_as_the_wire_selector_ref(action, method, params): + driver, request = _driver(dict(ACT_OK)) + assert asyncio.run(driver.execute(action)).ok is True + assert _calls(request) == [(method, params)] + + +def test_upload_drop_and_form_fields_accept_a_css_locator(tmp_path): + upload = tmp_path / "a.txt" + upload.write_text("x", encoding="utf-8") + driver, request = _driver(dict(ACT_OK), dict(ACT_OK), dict(ACT_OK)) + asyncio.run(driver.execute(UploadFile(CSS, (str(upload),)))) + asyncio.run(driver.execute(Drop(CSS, data=(("text/plain", "x"),)))) + asyncio.run(driver.execute(FillForm((FormField(CSS, FieldKind.TEXTBOX, "Ada"),)))) + assert [params["ref"] for _method, params in _calls(request)] == [WIRE_CSS, WIRE_CSS, WIRE_CSS] + + +def test_a_form_with_a_text_locator_fills_no_field(): + driver, request = _driver(dict(ACT_OK)) + fields = (FormField(CSS, FieldKind.TEXTBOX, "Ada"), FormField(Locator(None, "Gift", "g3"), FieldKind.CHECKBOX, "1")) + with pytest.raises(Unsupported) as info: + asyncio.run(driver.execute(FillForm(fields))) + assert info.value.dispatched is False + request.assert_not_called() + + +def test_wire_refs_carry_the_last_driver_generation(): + driver, request = _driver({**ACT_OK, "driver_generation": 7}, dict(ACT_OK)) + asyncio.run(driver.execute(Hover(E1))) + asyncio.run(driver.execute(Hover(E1))) + assert request.call_args_list[1].args[1]["ref"]["driver_generation"] == 7 + + +def test_run_script_without_target_evaluates_page_js(): + driver, request = _driver({"answer": 42}) + result = asyncio.run(driver.execute(RunScript("(a) => a", args={"x": 1}))) + assert _calls(request) == [ + ("evaluate", {"source": "(a) => a", "args": {"x": 1}, "await_promise": True, "return_by_value": True}) + ] + assert result.ok is True + assert result.value == {"answer": 42} + assert result.context_changed is False + + +def test_run_script_with_target_stamps_the_element_then_scopes_the_script(): + selector = '[data-openjiuwen-target-id="bnid:42"]' + driver, request = _driver(selector, "Buy") + result = asyncio.run(driver.execute(RunScript("(el, a) => el.textContent", args=[1], target=E1))) + (stamp_method, stamp_params), (eval_method, eval_params) = _calls(request) + assert (stamp_method, stamp_params) == ( + "stamp", + {"ref": WIRE_E1, "attribute": "data-openjiuwen-target-id", "value": "bnid:42"}, + ) + assert eval_method == "evaluate" + assert eval_params["args"] == {"selector": selector, "args": [1]} + assert "document.querySelector(input.selector)" in eval_params["source"] + assert "const value = ((el, a) => el.textContent" in eval_params["source"] + assert result.value == "Buy" + + +def test_open_context_reports_the_new_tab(): + tab_1 = {"target_id": "T1", "url": "http://x/", "title": "X", "active": True} + tab_2 = {"target_id": "T2", "url": "http://y/", "title": "Y", "active": False} + nav = {"url": "http://y/", "title": "Y", "changed_document": True, "driver_generation": 2} + driver, request = _driver([tab_1], nav, [tab_1, tab_2]) + result = asyncio.run(driver.execute(OpenContext("http://y/"))) + assert _calls(request) == [ + ("list_tabs", {}), + ("navigate", {"url": "http://y/", "wait_until": "load", "timeout_ms": None, "new_tab": True}), + ("list_tabs", {}), + ] + opened = ContextRef(id="T2", kind="tab", title="Y") + assert result.opened_contexts == (opened,) + assert result.value == opened + assert result.context_changed is True + assert result.surface == BrowserContext( + url="http://y/", + title="Y", + tabs=(ContextRef(id="T1", kind="tab", title="X"), opened), + tab_urls=("http://x/", "http://y/"), + current_tab=0, + ) + + +def test_open_context_without_url_opens_a_blank_tab(): + driver, request = _driver([], dict(NAV_SAME_URL), []) + asyncio.run(driver.execute(OpenContext())) + assert request.call_args_list[1].args[1]["url"] == "about:blank" + + +def test_upload_file_sends_resolved_paths(): + driver, request = _driver(dict(ACT_OK)) + with tempfile.NamedTemporaryFile(suffix=".txt", delete=False) as handle: + handle.write(b"x") + path = handle.name + try: + asyncio.run(driver.execute(UploadFile(E1, (path,)))) + finally: + Path(path).unlink(missing_ok=True) + assert _calls(request) == [("upload_files", {"ref": WIRE_E1, "paths": [str(Path(path).resolve())]})] + + +def test_drop_data_only_sends_mime_items(): + driver, request = _driver(dict(ACT_OK)) + asyncio.run(driver.execute(Drop(E1, data=(("text/plain", "hi"), ("text/uri-list", "http://x/"))))) + assert _calls(request) == [ + ( + "drop", + { + "ref": WIRE_E1, + "paths": [], + "data": [{"mimeType": "text/plain", "data": "hi"}, {"mimeType": "text/uri-list", "data": "http://x/"}], + }, + ) + ] + + +def test_fill_form_fills_fields_in_order(): + driver, request = _driver(dict(ACT_OK), dict(ACT_OK), dict(ACT_OK), dict(ACT_OK)) + form = FillForm( + ( + FormField(E1, FieldKind.TEXTBOX, "Ann"), + FormField(E1, FieldKind.COMBOBOX, "Blue"), + FormField(E2, FieldKind.CHECKBOX, "true"), + FormField(E2, FieldKind.RADIO, "false"), + ) + ) + result = asyncio.run(driver.execute(form)) + assert _calls(request) == [ + ("type_text", {"ref": WIRE_E1, "text": "Ann", "clear": True, "press_enter": False, "sensitive": False}), + ("select_option", {"ref": WIRE_E1, "value": "Blue", "label": None}), + ("set_checked", {"ref": WIRE_E2, "checked": True}), + ("set_checked", {"ref": WIRE_E2, "checked": False}), + ] + assert result.ok is True + assert result.context_changed is False + + +def test_fill_form_passes_each_field_sensitivity_to_the_wire(): + driver, request = _driver(dict(ACT_OK), dict(ACT_OK)) + form = FillForm((FormField(E1, FieldKind.TEXTBOX, "Ann"), FormField(E2, FieldKind.TEXTBOX, "pw", sensitive=True))) + asyncio.run(driver.execute(form)) + assert [params["sensitive"] for _method, params in _calls(request)] == [False, True] + + +def test_fill_form_stops_at_the_first_failed_field(): + failed = {"ok": False, "detail": "option not found", "document_changed": False, "driver_generation": 0} + driver, request = _driver(dict(ACT_OK), failed) + form = FillForm( + ( + FormField(E1, FieldKind.TEXTBOX, "Ann"), + FormField(E1, FieldKind.COMBOBOX, "Nope"), + FormField(E2, FieldKind.CHECKBOX, "true"), + ) + ) + result = asyncio.run(driver.execute(form)) + assert request.call_count == 2 + assert result.ok is False + assert result.detail == "fields[1] failed: option not found" + + +@pytest.mark.parametrize(("failing_field", "dispatched"), [(0, False), (1, True)]) +def test_fill_form_error_after_a_filled_field_counts_as_dispatched(failing_field, dispatched): + replies = [dict(ACT_OK)] * failing_field + [_sidecar_error("ElementNotFound")] + driver, _request = _driver(*replies) + form = FillForm((FormField(E1, FieldKind.TEXTBOX, "a"), FormField(E2, FieldKind.TEXTBOX, "b"))) + with pytest.raises(ElementNotFound) as info: + asyncio.run(driver.execute(form)) + assert info.value.dispatched is dispatched + + +def test_wait_seconds_sends_nothing(): + driver, request = _driver() + assert asyncio.run(driver.execute(Wait(seconds=0))).ok is True + request.assert_not_called() + + +def test_wait_until_text_polls_page_text(monkeypatch): + monkeypatch.setattr(driver_module, "_WAIT_POLL_S", 0) + driver, request = _driver(False, False, True) + assert asyncio.run(driver.execute(Wait(until_text="Done"))).ok is True + assert request.call_count == 3 + method, params = request.call_args.args + assert method == "evaluate" + assert params["args"] == "Done" + assert "innerText" in params["source"] + + +def test_wait_until_text_times_out(monkeypatch): + monkeypatch.setattr(driver_module, "_WAIT_TIMEOUT_S", 0) + driver, _request = _driver(False) + with pytest.raises(ActionTimeout) as info: + asyncio.run(driver.execute(Wait(until_text="Never"))) + assert info.value.dispatched is True + + +def test_wait_until_text_uses_seconds_as_its_deadline(monkeypatch): + monkeypatch.setattr(driver_module, "_WAIT_TIMEOUT_S", 3600.0) + driver, request = _driver(False) + with pytest.raises(ActionTimeout, match=r"within 0\.0s"): + asyncio.run(driver.execute(Wait(until_text="Never", seconds=0.0))) + assert request.call_count == 1 + + +@pytest.mark.parametrize( + "action", + [ + SelectOption(E1, ("a", "b")), + SelectOption(E1, ()), + TypeText(None, "x"), + Scroll(dx=10, dy=10), + Scroll(), + UploadFile(None, ("a.txt",)), + PressKey("Enter", target=E1), + ], +) +def test_unsupported_forms_raise_before_dispatch(action): + driver, request = _driver() + with pytest.raises(Unsupported) as info: + asyncio.run(driver.execute(action)) + assert info.value.dispatched is False + request.assert_not_called() + + +@pytest.mark.parametrize("action", [Navigate("http://x/"), Back()]) +def test_same_url_reload_sets_context_changed(action): + driver, _request = _driver(dict(NAV_SAME_URL)) + assert asyncio.run(driver.execute(action)).context_changed is True + + +@pytest.mark.parametrize(("document_changed", "expected"), [(True, True), (False, False)]) +def test_act_result_follows_the_sidecar_document_change(document_changed, expected): + driver, _request = _driver({**ACT_OK, "document_changed": document_changed}) + assert asyncio.run(driver.execute(Click(E1))).context_changed is expected + + +def test_a_dialog_opened_by_the_command_changes_the_context_and_is_named(): + dialog = {"type": "confirm", "message": "Proceed?"} + driver, _request = _driver( + {**ACT_OK, "detail": "the input waits until the dialog is handled", "dialog_opened": dialog} + ) + result = asyncio.run(driver.execute(Click(E1))) + assert result.ok is True + assert result.context_changed is True + assert result.detail == "the input waits until the dialog is handled; confirm dialog opened: 'Proceed?'" + + +def test_no_dialog_leaves_the_detail_alone(): + driver, _request = _driver({**ACT_OK, "dialog_opened": None}) + result = asyncio.run(driver.execute(Click(E1))) + assert (result.context_changed, result.detail, result.surface) == (False, "done", None) + + +def test_a_dialog_open_after_the_command_is_in_the_result_surface(): + dialog = {"type": "confirm", "message": "Proceed?"} + driver, _request = _driver({**ACT_OK, "url": "http://x/", "dialog_opened": dialog, "dialog": dialog}) + result = asyncio.run(driver.execute(Click(E1))) + assert result.surface == BrowserContext(url="http://x/", dialog="confirm: Proceed?") + + +def test_a_command_refused_by_an_open_dialog_shows_it_but_changes_nothing(): + refused = { + **ACT_OK, + "ok": False, + "detail": "a confirm dialog is open ('Proceed?'); handle it first", + "url": "http://x/", + "dialog": {"type": "confirm", "message": "Proceed?"}, + } + driver, _request = _driver(refused) + result = asyncio.run(driver.execute(Click(E1))) + assert (result.ok, result.context_changed) == (False, False) + assert result.surface == BrowserContext(url="http://x/", dialog="confirm: Proceed?") + + +def test_pages_opened_by_the_command_are_opened_contexts(): + popup = {"target_id": "T9", "url": "", "title": "", "active": False} + driver, _request = _driver({**ACT_OK, "new_targets": [popup]}) + result = asyncio.run(driver.execute(Click(E1))) + assert result.opened_contexts == (ContextRef(id="T9", kind="tab", title=""),) + assert result.context_changed is True + + +TABS_ON_2 = [ + {"target_id": "T1", "url": "http://x/", "title": "X", "active": False}, + {"target_id": "T2", "url": "http://y/", "title": "Second", "active": True}, +] + + +def test_switch_context_always_changes_context(): + driver, _request = _driver(dict(ACT_OK), TABS_ON_2) + assert asyncio.run(driver.execute(SwitchContext(TAB_2))).context_changed is True + + +def test_switch_context_reports_the_tabs_after_the_switch(): + driver, request = _driver({**ACT_OK, "url": "http://y/"}, TABS_ON_2) + result = asyncio.run(driver.execute(SwitchContext(TAB_2))) + assert _calls(request) == [("switch_tab", {"tab": WIRE_TAB_2}), ("list_tabs", {})] + assert result.surface == BrowserContext( + url="http://y/", + title="Second", + tabs=(ContextRef(id="T1", kind="tab", title="X"), TAB_2), + tab_urls=("http://x/", "http://y/"), + current_tab=1, + ) + + +def test_close_context_changes_context_and_names_the_closed_one(): + driver, request = _driver(dict(ACT_OK), TABS_ON_2[:1]) + result = asyncio.run(driver.execute(CloseContext(TAB_2))) + assert _calls(request) == [("close_tab", {"tab": WIRE_TAB_2}), ("list_tabs", {})] + assert result.context_changed is True + assert result.source_context == TAB_2 + assert result.surface.tab_urls == ("http://x/",) and result.surface.current_tab is None + + +def test_a_tab_list_that_fails_after_the_switch_counts_as_dispatched(): + driver, _request = _driver(dict(ACT_OK), _sidecar_error("DriverConnectionLost")) + with pytest.raises(DriverError) as raised: + asyncio.run(driver.execute(SwitchContext(TAB_2))) + assert raised.value.dispatched is True + + +TITLE_CALL = ( + "evaluate", + {"source": "() => document.title", "args": None, "await_promise": True, "return_by_value": True}, +) + + +def _wire_observation(**overrides: Any) -> dict[str, Any]: + data = { + "url": "http://x/", + "title": "X", + "tabs": [ + {"target_id": "T1", "url": "http://x/", "title": "X", "active": True}, + {"target_id": "T2", "url": "http://y/", "title": "Y", "active": False}, + ], + "elements": [ + { + "index": 3, + "backend_node_id": 42, + "driver_ref": {"handle": "bnid:42", "driver_generation": 9}, + "frame_id": None, + "tag": "button", + "role": "button", + "name": "Buy", + "value": None, + "attributes": {"id": "buy"}, + "box": {"x": 1.0, "y": 2.0, "width": 30.0, "height": 10.0}, + "visible": True, + "scrollable": False, + } + ], + "ax_text": "[3]", + "screenshot_b64": None, + "pixels_above": 10, + "pixels_below": 900, + "errors": [], + "is_pdf_viewer": False, + "captured_at": 0.0, + "driver_generation": 9, + } + data.update(overrides) + return data + + +def test_observe_maps_the_sidecar_observation(): + png = _png(800, 600) + driver, request = _driver(_wire_observation(screenshot_b64=base64.b64encode(png).decode("ascii")), "X") + observation = asyncio.run(driver.observe(Observe(include_screenshot=True))) + + assert _calls(request) == [ + ("observe", {"include_dom": True, "include_screenshot": True, "cached": False}), + TITLE_CALL, + ] + assert observation.generation == "" + assert observation.context == ContextRef(id="T1", kind="tab", title="X") + (element,) = observation.elements + assert element.ref == DriverRef(ref="bnid:42", generation="") + assert (element.role, element.name, element.value) == ("button", "Buy", None) + assert element.bounds == (1.0, 2.0, 30.0, 10.0) + assert element.extra["index"] == 3 + assert element.extra["tag"] == "button" + assert element.extra["attributes"] == {"id": "buy"} + assert observation.native_text == "[3]" + assert observation.screenshot is not None + assert observation.screenshot.png == png + assert (observation.screenshot.width, observation.screenshot.height) == (800, 600) + assert observation.surface == BrowserContext( + url="http://x/", + title="X", + tabs=(ContextRef(id="T1", kind="tab", title="X"), ContextRef(id="T2", kind="tab", title="Y")), + tab_urls=("http://x/", "http://y/"), + current_tab=0, + is_pdf_viewer=False, + pixels_above=10, + pixels_below=900, + last_status=None, + ) + + +def test_observe_full_page_takes_a_separate_screenshot(): + png = _png(800, 4000) + driver, request = _driver(_wire_observation(), "X", base64.b64encode(png).decode("ascii")) + observation = asyncio.run(driver.observe(Observe(include_screenshot=True, full_page=True))) + assert _calls(request) == [ + ("observe", {"include_dom": True, "include_screenshot": False, "cached": False}), + TITLE_CALL, + ("screenshot", {"full_page": True, "clip": None}), + ] + assert observation.screenshot.height == 4000 + + +def test_observe_without_screenshot_has_none(): + driver, _request = _driver(_wire_observation(), "X") + assert asyncio.run(driver.observe(Observe())).screenshot is None + + +def test_a_scoped_observe_is_unsupported_before_anything_is_sent(): + driver, request = _driver() + + with pytest.raises(Unsupported) as raised: + asyncio.run(driver.observe(Observe(depth=1))) + + assert raised.value.dispatched is False + assert _calls(request) == [] + + +def test_observe_without_an_active_tab_falls_back_to_the_page(): + tabs = [{"target_id": "T1", "url": "http://other/", "title": "O", "active": False}] + driver, _request = _driver(_wire_observation(tabs=tabs), "X") + observation = asyncio.run(driver.observe(Observe())) + assert observation.context == ContextRef(id="", kind="tab", title="X") + assert (observation.surface.tab_urls, observation.surface.current_tab) == (("http://other/",), None) + + +def test_observe_twice_on_an_unchanged_page_gives_the_same_fingerprint(): + driver, _request = _driver(_wire_observation(driver_generation=1), "X", _wire_observation(driver_generation=2), "X") + first = asyncio.run(driver.observe(Observe())) + second = asyncio.run(driver.observe(Observe())) + assert first.fingerprint == second.fingerprint + + +def test_observe_reports_the_document_title_not_the_sidecar_title(): + """browser-use's summary title lags behind ``document.title`` (seen live: the URL instead of the title).""" + driver, request = _driver(_wire_observation(title="x/index.html"), "clicked") + observation = asyncio.run(driver.observe(Observe())) + assert _calls(request)[1] == TITLE_CALL + assert observation.surface.title == "clicked" + + +def test_observe_keeps_the_sidecar_title_when_the_page_runs_no_script(): + driver, _request = _driver(_wire_observation(title="Viewer"), _sidecar_error("EvaluateError")) + assert asyncio.run(driver.observe(Observe())).surface.title == "Viewer" + + +def test_observe_while_a_dialog_is_open_runs_no_script_and_takes_no_screenshot(): + blocked = { + "url": "http://x/", + "title": "X", + "tabs": [{"target_id": "T1", "url": "http://x/", "title": "X", "active": True}], + "elements": [], + "dialog": {"type": "alert", "message": "Saved"}, + "driver_generation": 9, + } + driver, request = _driver(blocked) + observation = asyncio.run(driver.observe(Observe(include_screenshot=True, full_page=True))) + assert _calls(request) == [("observe", {"include_dom": True, "include_screenshot": False, "cached": False})] + assert (observation.elements, observation.screenshot) == ((), None) + assert observation.context == ContextRef(id="T1", kind="tab", title="X") + assert observation.surface == BrowserContext( + url="http://x/", + title="X", + tabs=(ContextRef(id="T1", kind="tab", title="X"),), + tab_urls=("http://x/",), + current_tab=0, + dialog="alert: Saved", + ) + + +ERROR_TABLE = { + "StaleIndexError": (StaleRef, False), + "ElementNotFound": (ElementNotFound, False), + "StaleNodeError": (StaleRef, False), + "AmbiguousElement": (AmbiguousElement, True), + "ActionTimeout": (ActionTimeout, True), + "NavigationError": (NavigationError, True), + "DriverNotConnected": (ConnectionLost, False), + "DriverConnectionError": (ConnectionLost, True), + "DriverConnectionLost": (ConnectionLost, True), + "DriverUnsupported": (Unsupported, False), + "ActionIntercepted": (ActionIntercepted, False), + "DriverError": (DriverError, True), + "ObserveError": (DriverError, True), + "EvaluateError": (DriverError, True), +} + + +def test_error_table_covers_every_wire_error_code(): + assert set(ERROR_TABLE) == set(wire.ERROR_CODES) + + +@pytest.mark.parametrize(("code", "expected"), sorted(ERROR_TABLE.items())) +def test_sidecar_error_maps_to_typed_error(code, expected): + error_type, dispatched = expected + driver, _request = _driver(_sidecar_error(code, "boom")) + with pytest.raises(DriverError) as info: + asyncio.run(driver.execute(Hover(E1))) + assert type(info.value) is error_type + assert info.value.dispatched is dispatched + assert "boom" in str(info.value) + + +@pytest.mark.parametrize("raised", [_sidecar_error("SomethingNew"), RuntimeError("raw failure")]) +def test_unclassified_error_is_dispatched_and_logged(monkeypatch, raised): + logger = MagicMock() + monkeypatch.setattr(driver_module, "_logger", logger) + driver, _request = _driver(raised) + with pytest.raises(DriverError) as info: + asyncio.run(driver.execute(Hover(E1))) + assert type(info.value) is DriverError + assert info.value.dispatched is True + logger.warning.assert_called_once() + assert str(raised) in [str(arg) for arg in logger.warning.call_args.args] + + +@pytest.mark.parametrize( + "message", + [ + "[DownloadsWatchdog.on_BrowserStateRequestEvent] Reconnection failed — CDP still not connected", + "[ScreenshotWatchdog.on_ScreenshotEvent] Reconnection wait timed out after 30.0s", + "Cannot switch tabs - browser not connected", + "No valid agent focus available - target may have detached and recovery failed.", + ], +) +def test_a_browser_that_went_away_is_a_lost_connection(monkeypatch, message): + logger = MagicMock() + monkeypatch.setattr(driver_module, "_logger", logger) + driver, _request = _driver(_sidecar_error("DriverError", message)) + with pytest.raises(ConnectionLost) as info: + asyncio.run(driver.execute(Hover(E1))) + assert info.value.dispatched is True + assert message in str(info.value) + logger.warning.assert_not_called() + + +def test_intercepted_input_names_the_element_that_would_receive_it(): + driver, _request = _driver(_sidecar_error("ActionIntercepted", "click would reach it", {"by": "span#overlay"})) + with pytest.raises(ActionIntercepted) as info: + asyncio.run(driver.execute(Click(E1))) + assert info.value.by == "span#overlay" + assert info.value.dispatched is False + + +def test_classified_error_is_not_logged(monkeypatch): + logger = MagicMock() + monkeypatch.setattr(driver_module, "_logger", logger) + driver, _request = _driver(_sidecar_error("ElementNotFound")) + with pytest.raises(ElementNotFound): + asyncio.run(driver.execute(Hover(E1))) + logger.warning.assert_not_called() + + +def test_unclassified_error_log_does_not_contain_the_typed_text(monkeypatch): + logger = MagicMock() + monkeypatch.setattr(driver_module, "_logger", logger) + driver, _request = _driver(_sidecar_error("DriverError", "sidecar crashed")) + with pytest.raises(DriverError): + asyncio.run(driver.execute(TypeText(E1, "s3cret-password", sensitive=True))) + assert "s3cret-password" not in repr(logger.warning.call_args) + + +def test_command_before_start_is_connection_lost_and_not_dispatched(): + driver = BrowserUseDriver() + with pytest.raises(ConnectionLost) as info: + asyncio.run(driver.execute(Hover(E1))) + assert info.value.dispatched is False + + +def _patched_transport(monkeypatch, *, connect_reply: Any) -> MagicMock: + transport = MagicMock() + transport.start = AsyncMock(return_value={"kind": "hello"}) + transport.request = AsyncMock(side_effect=[connect_reply]) + transport.close = AsyncMock() + monkeypatch.setattr(driver_module, "SidecarTransport", MagicMock(return_value=transport)) + return transport + + +CONNECT_REPLY = { + "backend": "browser_use", + "browser_version": "Chrome/140", + "protocol_version": "1.3", + "backend_version": "0.13.10", + "initial_url": "about:blank", +} + + +def test_factory_connects_to_the_device_endpoint(monkeypatch): + transport = _patched_transport(monkeypatch, connect_reply=dict(CONNECT_REPLY)) + device = ProvisionedDevice(kind="chrome", endpoint="http://127.0.0.1:9333", extras={}, cleanup=None) + driver = create_browser_use_driver(device, None) + asyncio.run(driver.start()) + transport.start.assert_awaited_once_with(timeout_s=30.0) + assert transport.request.call_args.args == ( + "connect", + {"cdp_url": "http://127.0.0.1:9333", "timeout_s": 30.0, "init_script": driver_module._clipboard_guard()}, + ) + assert driver.info.versions == {"browser": "Chrome/140", "protocol": "1.3", "browser_use": "0.13.10"} + + +def test_start_with_an_explicit_config_overrides_the_factory_config(monkeypatch): + transport = _patched_transport(monkeypatch, connect_reply=dict(CONNECT_REPLY)) + driver = BrowserUseDriver(BrowserUseConfig(cdp_url="http://a")) + asyncio.run(driver.start(BrowserUseConfig(cdp_url="http://b", timeout_s=5.0))) + assert transport.request.call_args.args == ( + "connect", + {"cdp_url": "http://b", "timeout_s": 5.0, "init_script": driver_module._clipboard_guard()}, + ) + + +def test_start_hands_the_clipboard_guard_to_the_sidecar(monkeypatch): + transport = _patched_transport(monkeypatch, connect_reply=dict(CONNECT_REPLY)) + asyncio.run(BrowserUseDriver(BrowserUseConfig(cdp_url="http://a")).start()) + script = transport.request.call_args.args[1]["init_script"] + assert "the clipboard belongs to the host machine" in script and "addScriptToEvaluateOnNewDocument" not in script + + +def test_start_without_config_is_refused(): + with pytest.raises(DriverError) as info: + asyncio.run(BrowserUseDriver().start()) + assert info.value.dispatched is False + + +def test_a_target_that_detached_at_connect_is_a_lost_connection(monkeypatch): + message = "No valid agent focus available - target may have detached and recovery failed." + transport = _patched_transport(monkeypatch, connect_reply=_sidecar_error("DriverError", message)) + driver = BrowserUseDriver(BrowserUseConfig(cdp_url="http://a")) + with pytest.raises(ConnectionLost) as info: + asyncio.run(driver.start()) + assert info.value.dispatched is True + assert str(info.value) == "connect: " + message + transport.close.assert_awaited_once() + + +def test_failed_connect_closes_the_sidecar_and_raises_connection_lost(monkeypatch): + transport = _patched_transport(monkeypatch, connect_reply=_sidecar_error("DriverConnectionError")) + driver = BrowserUseDriver(BrowserUseConfig(cdp_url="http://a")) + with pytest.raises(ConnectionLost): + asyncio.run(driver.start()) + transport.close.assert_awaited_once() + assert asyncio.run(driver.health()).ok is False + + +def test_stop_closes_the_transport(): + driver, _request = _driver() + transport = driver._transport + transport.close = AsyncMock() + asyncio.run(driver.stop()) + transport.close.assert_awaited_once() + assert asyncio.run(driver.health()).detail == "not started" + + +def test_health_maps_the_sidecar_reply(): + driver, request = _driver({"connected": True, "url": "http://x/", "tab_count": 1, "latency_ms": 1.0, "error": None}) + health = asyncio.run(driver.health()) + assert (health.ok, health.detail) == (True, None) + assert _calls(request) == [("health", {})] + + +def test_health_never_raises(): + driver, _request = _driver(_sidecar_error("DriverConnectionLost", "gone")) + health = asyncio.run(driver.health()) + assert (health.ok, health.detail) == (False, "gone") + + +def test_result_types_are_contract_results(): + driver, _request = _driver(dict(ACT_OK)) + assert isinstance(asyncio.run(driver.execute(Hover(E1))), ActionResult) + + +def test_importing_browser_use_driver_does_not_load_browser_use_package() -> None: + snapshot = set(sys.modules) + importlib.reload(importlib.import_module("openjiuwen.harness.tools.browser_move.drivers.browser_use.driver")) + added = set(sys.modules) - snapshot + browser_use_modules = [name for name in added if name == "browser_use" or name.startswith("browser_use.")] + assert not browser_use_modules, f"unexpected browser_use imports: {browser_use_modules}" + + +@pytest.mark.parametrize( + ("code", "value"), [("element.id", "save"), ("(el, a) => el.id + a", "save!"), ("() => document.title", "T")] +) +def test_a_scoped_script_takes_an_expression_or_a_function(code: str, value: str) -> None: + node = shutil.which("node") + if node is None: + pytest.skip("node is not installed") + driver, request = _driver('[data-openjiuwen-target-id="bnid:42"]', None) + asyncio.run(driver.execute(RunScript(code, args="!", target=E1))) + params = _calls(request)[1][1] + script = ( + "globalThis.document = { title: 'T', querySelector: () => ({ id: 'save' }) };\n" + f"console.log(JSON.stringify(({params['source']})({json.dumps(params['args'])})));" + ) + + completed = subprocess.run([node, "-e", script], check=True, text=True, capture_output=True) + + assert json.loads(completed.stdout) == value + + +def _timeout(method: str = "evaluate") -> Exception: + return _sidecar_error("ActionTimeout", f"{method} timed out after 30.0s") + + +def test_a_second_request_timeout_in_a_row_is_a_lost_connection() -> None: + driver, _ = _driver(_timeout("navigate"), _timeout("observe")) + + with pytest.raises(ActionTimeout): + asyncio.run(driver.execute(Navigate("https://www.google.com/maps"))) + with pytest.raises(ConnectionLost) as info: + asyncio.run(driver.execute(RunScript("document.title"))) + + assert info.value.dispatched is True + assert "stopped answering" in str(info.value) + + +def test_an_answer_between_two_timeouts_resets_the_count() -> None: + driver, _ = _driver(_timeout(), {"answer": 1}, _timeout(), {"answer": 2}) + + for _ in range(2): + with pytest.raises(ActionTimeout): + asyncio.run(driver.execute(RunScript("document.title"))) + assert asyncio.run(driver.execute(RunScript("document.title"))).ok is True + + +@pytest.mark.parametrize( + "css", + ['button:has-text("Done")', "text=One way", "div[role=combobox] >> span", "li:text-is('Rome')", "input:visible"], +) +def test_a_playwright_selector_is_unsupported_before_dispatch(css: str) -> None: + driver, request = _driver() + + with pytest.raises(Unsupported) as info: + asyncio.run(driver.execute(Click(Locator(css, None, "g3")))) + + assert info.value.dispatched is False + assert "plain CSS selectors only" in str(info.value) and "target_id" in str(info.value) + request.assert_not_awaited() + + +@pytest.mark.parametrize("css", ["#save", "a:nth-of-type(2)", '[id="app-container-Copy Link"]', ".text-label"]) +def test_plain_css_still_goes_to_the_sidecar(css: str) -> None: + driver, request = _driver(ACT_OK) + + asyncio.run(driver.execute(Click(Locator(css, None, "g3")))) + + assert _calls(request)[0][1]["ref"]["css"] == css diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/test_browser_use_sidecar.py b/tests/unit_tests/harness/tools/browser_move/drivers/test_browser_use_sidecar.py new file mode 100644 index 0000000000..b2ca7d4cd4 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/test_browser_use_sidecar.py @@ -0,0 +1,746 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The sidecar's ``SessionAdapter`` on a fake CDP client; flat sidecar modules (``import cdp``) load by path.""" + +from __future__ import annotations + +import asyncio +import importlib.util +import inspect +import json +import shutil +import subprocess +import sys +from collections.abc import Callable +from pathlib import Path +from types import ModuleType, SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock + +import pytest + +_SIDECAR_DIR = ( + Path(__file__).resolve().parents[6] + / "openjiuwen" + / "harness" + / "tools" + / "browser_move" + / "drivers" + / "browser_use" + / "sidecar" +) + + +def _load_sidecar_module(name: str) -> Any: + if name in sys.modules and getattr(sys.modules[name], "__file__", "").startswith(str(_SIDECAR_DIR)): + return sys.modules[name] + if str(_SIDECAR_DIR) not in sys.path: + sys.path.insert(0, str(_SIDECAR_DIR)) + spec = importlib.util.spec_from_file_location(name, _SIDECAR_DIR / f"{name}.py") + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +session_adapter = _load_sidecar_module("session_adapter") +errors = session_adapter.exceptions + +SESSION = "S1" +TARGET = "T1" +POPUP_SESSION = "S2" +BUTTON = {"kind": "driver", "handle": "bnid:7", "driver_generation": 0} +QUICK_S = 2.0 + + +class _Registry: + """cdp-use's ``EventRegistry``: one handler per event method.""" + + def __init__(self) -> None: + self._handlers: dict[str, Callable] = {} + + def register(self, method: str, callback: Callable) -> None: + self._handlers[method] = callback + + def unregister(self, method: str) -> None: + self._handlers.pop(method, None) + + +class _Domain: + def __init__(self, client: FakeCdp, domain: str) -> None: + self._client = client + self._domain = domain + + def __getattr__(self, method: str) -> Callable: + name = f"{self._domain}.{method}" + + async def send(params: dict[str, Any] | None = None, session_id: str | None = None) -> Any: + params = dict(params or {}) + self._client.calls.append((name, params, session_id)) + reply = self._client.replies.get(name) + if reply is None: + return {} + result = reply(params, session_id) + return await result if inspect.isawaitable(result) else result + + return send + + +class FakeCdp: + """Answers ``send..`` from ``replies`` and records every call.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, dict[str, Any], str | None]] = [] + self.replies: dict[str, Callable[[dict[str, Any], str | None], Any]] = {} + self._event_registry = _Registry() + self.send = _Send(self) + + def emit(self, method: str, params: dict[str, Any], session_id: str | None = SESSION) -> None: + handler = self._event_registry._handlers.get(method) + if handler is not None: + handler(params, session_id) + + def sent(self, name: str) -> list[dict[str, Any]]: + return [params for called, params, _sid in self.calls if called == name] + + +class _Send: + def __init__(self, client: FakeCdp) -> None: + self._client = client + + def __getattr__(self, domain: str) -> _Domain: + return _Domain(self._client, domain) + + +class Page: + """A page with one button at (10, 20, 100, 40); a dialog or popup it opens blocks the release, as in Chrome.""" + + def __init__( + self, + client: FakeCdp, + *, + dialog: dict[str, str] | None = None, + popup: dict[str, str] | None = None, + navigation: dict[str, Any] | None = None, + commit_after: float | None = None, + covered_by: str | None = None, + ) -> None: + self.client = client + self.covered_by = covered_by + self.navigation = navigation + self.commit_after = commit_after + self.dialog = dialog + self.dialog_closed: asyncio.Future | None = None + self.popup = popup + self.opener_resumed: asyncio.Future | None = None + client.replies["DOM.resolveNode"] = lambda params, sid: {"object": {"objectId": "obj-7"}} + client.replies["Runtime.callFunctionOn"] = self._call_function + client.replies["Input.dispatchMouseEvent"] = self._mouse + client.replies["Page.handleJavaScriptDialog"] = self._handle_dialog + client.replies["Runtime.runIfWaitingForDebugger"] = self._run_if_waiting + + def _call_function(self, params: dict[str, Any], sid: str | None) -> dict[str, Any]: + if "elementFromPoint" in params["functionDeclaration"]: + return {"result": {"value": self.covered_by}} + if "getBoundingClientRect" in params["functionDeclaration"]: + return {"result": {"value": {"x": 10, "y": 20, "width": 100, "height": 40}}} + return {"result": {"value": None}} + + async def _mouse(self, params: dict[str, Any], sid: str | None) -> dict[str, Any]: + if params["type"] == "mouseReleased" and self.dialog is not None: + self.dialog_closed = asyncio.get_running_loop().create_future() + self.client.emit("Page.javascriptDialogOpening", self.dialog) + await self.dialog_closed + if params["type"] == "mouseReleased" and self.navigation is not None: + self.client.emit("Page.frameRequestedNavigation", self.navigation) + if self.commit_after is not None: + frame = {"id": self.navigation["frameId"], "loaderId": "L2", "url": self.navigation["url"]} + asyncio.get_running_loop().call_later( + self.commit_after, self.client.emit, "Page.frameNavigated", {"frame": frame} + ) + if params["type"] == "mouseReleased" and self.popup is not None: + self.opener_resumed = asyncio.get_running_loop().create_future() + self.client.emit("Target.targetCreated", {"targetInfo": self.popup}, session_id=None) + await self.opener_resumed + return {} + + def _run_if_waiting(self, params: dict[str, Any], sid: str | None) -> dict[str, Any]: + if sid == SESSION and self.opener_resumed is not None and not self.opener_resumed.done(): + self.opener_resumed.set_result(True) + return {} + + def _handle_dialog(self, params: dict[str, Any], sid: str | None) -> dict[str, Any]: + if self.dialog_closed is not None and not self.dialog_closed.done(): + self.dialog_closed.set_result(params["accept"]) + return {} + + +def _adapter(client: FakeCdp) -> Any: + adapter = session_adapter.SessionAdapter() + adapter._session = SimpleNamespace( + get_current_page_url=AsyncMock(return_value="http://x/index.html"), + get_tabs=AsyncMock(return_value=[]), + stop=AsyncMock(), + ) + adapter._cdp_session = SimpleNamespace(cdp_client=client, session_id=SESSION, target_id=TARGET) + return adapter + + +def _quick(awaitable: Any) -> Any: + """Fails the test when the call blocks, as the click on a dialog used to (30 s live).""" + return asyncio.wait_for(awaitable, QUICK_S) + + +def test_a_click_that_opens_a_dialog_returns_with_the_dialog(): + async def scenario() -> None: + client = FakeCdp() + page = Page(client, dialog={"type": "confirm", "message": "Proceed?"}) + adapter = _adapter(client) + reply = await _quick(adapter.click(BUTTON)) + assert reply["ok"] is True + assert reply["dialog_opened"] == {"type": "confirm", "message": "Proceed?"} + assert not page.dialog_closed.done(), "the click must not wait for the dialog" + await adapter.close() + + asyncio.run(scenario()) + + +def test_handle_dialog_resolves_the_open_dialog_and_the_waiting_input(): + async def scenario() -> None: + client = FakeCdp() + page = Page(client, dialog={"type": "confirm", "message": "Proceed?"}) + adapter = _adapter(client) + await _quick(adapter.click(BUTTON)) + reply = await _quick(adapter.handle_dialog(accept=True)) + assert reply["ok"] is True + assert reply["detail"] == "accepted confirm" + assert client.sent("Page.handleJavaScriptDialog") == [{"accept": True}] + assert page.dialog_closed.result() is True + await asyncio.sleep(0) + assert not adapter._waiting_inputs, "the click's input finished once the dialog closed" + + asyncio.run(scenario()) + + +def test_the_dialog_is_handled_on_the_session_that_reported_it(): + async def scenario() -> None: + client = FakeCdp() + Page(client) + adapter = _adapter(client) + await adapter._ensure_listeners() + client.emit("Page.javascriptDialogOpening", {"type": "alert", "message": "Hi"}, session_id=POPUP_SESSION) + await _quick(adapter.handle_dialog(accept=True)) + assert [sid for name, _params, sid in client.calls if name == "Page.handleJavaScriptDialog"] == [POPUP_SESSION] + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + ("call", "error"), + [ + (lambda adapter: adapter.evaluate("() => document.title"), "EvaluateError"), + (lambda adapter: adapter.screenshot(), "ObserveError"), + (lambda adapter: adapter.stamp(BUTTON, attribute="data-x", value="1"), "EvaluateError"), + ], + ids=["evaluate", "screenshot", "stamp"], +) +def test_page_reads_fail_at_once_while_a_dialog_is_open(call, error): + async def scenario() -> None: + client = FakeCdp() + Page(client, dialog={"type": "alert", "message": "Saved"}) + adapter = _adapter(client) + await _quick(adapter.click(BUTTON)) + sent_before = len(client.calls) + with pytest.raises(getattr(errors, error), match="alert dialog is open"): + await _quick(call(adapter)) + assert len(client.calls) == sent_before, "nothing reaches the blocked page" + await adapter.close() + + asyncio.run(scenario()) + + +def test_observe_while_a_dialog_is_open_shows_the_page_and_the_dialog_and_sends_nothing(): + async def scenario() -> None: + client = FakeCdp() + Page(client, dialog={"type": "alert", "message": "Saved"}) + adapter = _adapter(client) + tab = SimpleNamespace(target_id=TARGET, url="http://x/index.html", title="Scene") + adapter._session.get_tabs = AsyncMock(return_value=[tab]) + await _quick(adapter.click(BUTTON)) + sent_before = len(client.calls) + observed = await _quick(adapter.observe(include_screenshot=True)) + assert observed["dialog"] == {"type": "alert", "message": "Saved"} + assert (observed["url"], observed["title"], observed["elements"]) == ("http://x/index.html", "Scene", []) + assert [tab["target_id"] for tab in observed["tabs"]] == [TARGET] + assert len(client.calls) == sent_before, "nothing reaches the blocked page" + await adapter.close() + + asyncio.run(scenario()) + + +def test_act_replies_carry_the_url_and_the_dialog_open_when_they_return(): + async def scenario() -> None: + client = FakeCdp() + Page(client, dialog={"type": "confirm", "message": "Proceed?"}) + adapter = _adapter(client) + opened = await _quick(adapter.click(BUTTON)) + refused = await _quick(adapter.click(BUTTON)) + handled = await _quick(adapter.handle_dialog(accept=True)) + assert opened["url"] == "http://x/index.html" + assert opened["dialog"] == refused["dialog"] == {"type": "confirm", "message": "Proceed?"} + assert handled["dialog"] is None + + asyncio.run(scenario()) + + +def test_an_action_while_a_dialog_is_open_sends_nothing(): + async def scenario() -> None: + client = FakeCdp() + Page(client, dialog={"type": "confirm", "message": "Proceed?"}) + adapter = _adapter(client) + await _quick(adapter.click(BUTTON)) + presses = len(client.sent("Input.dispatchMouseEvent")) + reply = await _quick(adapter.click(BUTTON)) + assert reply["ok"] is False + assert "confirm dialog is open" in reply["detail"] + assert reply["dialog_opened"] is None, "the dialog opened during the earlier action" + assert len(client.sent("Input.dispatchMouseEvent")) == presses + await adapter.close() + + asyncio.run(scenario()) + + +def test_an_armed_dialog_is_handled_and_still_reported(): + async def scenario() -> None: + client = FakeCdp() + page = Page(client, dialog={"type": "confirm", "message": "Proceed?"}) + adapter = _adapter(client) + armed = await _quick(adapter.handle_dialog(accept=False)) + assert armed["detail"].startswith("armed") + reply = await _quick(adapter.click(BUTTON)) + assert reply["ok"] is True + assert reply["detail"] == "clicked" + assert reply["dialog_opened"] == {"type": "confirm", "message": "Proceed?"} + assert reply["dialog"] is None, "the armed dialog is already handled" + assert page.dialog_closed.result() is False + + asyncio.run(scenario()) + + +class _DoneEvent: + def __await__(self) -> Any: + return iter(()) + + async def event_result(self, **_kwargs: Any) -> None: + return None + + +@pytest.fixture +def browser_use_events(monkeypatch: pytest.MonkeyPatch) -> None: + events = ModuleType("browser_use.browser.events") + for name in ("NavigateToUrlEvent", "GoBackEvent", "GoForwardEvent", "RefreshEvent"): + setattr(events, name, lambda **_fields: _DoneEvent()) + monkeypatch.setitem(sys.modules, "browser_use", ModuleType("browser_use")) + monkeypatch.setitem(sys.modules, "browser_use.browser", ModuleType("browser_use.browser")) + monkeypatch.setitem(sys.modules, "browser_use.browser.events", events) + + +@pytest.mark.parametrize("dialog", [None, {"type": "alert", "message": "Welcome"}]) +@pytest.mark.parametrize("navigation", ["navigate", "go_back", "go_forward", "reload"]) +def test_navigation_replies_carry_the_dialog_the_loaded_page_opened(browser_use_events, navigation, dialog): + async def scenario() -> None: + client = FakeCdp() + Page(client) + adapter = _adapter(client) + await _quick(adapter.click(BUTTON)) + + def dispatch(event: _DoneEvent) -> _DoneEvent: + if dialog is not None: + client.emit("Page.javascriptDialogOpening", dialog) + return event + + adapter._session.event_bus = SimpleNamespace(dispatch=dispatch) + adapter._session.get_current_page_title = AsyncMock(return_value="Scene") + call = ( + adapter.navigate(url="http://x/next.html") if navigation == "navigate" else getattr(adapter, navigation)() + ) + reply = await _quick(call) + assert reply["dialog"] == dialog + await adapter.close() + + asyncio.run(scenario()) + + +def test_every_action_reclaims_the_dialog_event_from_browser_use(): + async def scenario() -> None: + client = FakeCdp() + Page(client, dialog={"type": "alert", "message": "Hi"}) + adapter = _adapter(client) + await adapter._ensure_listeners() + client._event_registry.register("Page.javascriptDialogOpening", lambda event, sid: None) + reply = await _quick(adapter.click(BUTTON)) + assert reply["dialog_opened"] == {"type": "alert", "message": "Hi"} + await adapter.close() + + asyncio.run(scenario()) + + +def test_page_events_are_enabled_once_per_session(): + async def scenario() -> None: + client = FakeCdp() + Page(client) + adapter = _adapter(client) + await _quick(adapter.click(BUTTON)) + await _quick(adapter.click(BUTTON)) + assert len(client.sent("Page.enable")) == 1 + + asyncio.run(scenario()) + + +POPUP = {"targetId": "T2", "type": "page", "title": "", "url": "", "openerId": TARGET} + + +def test_a_click_that_opens_a_popup_resumes_the_page_and_reports_the_new_target(): + async def scenario() -> None: + client = FakeCdp() + Page(client, popup=POPUP) + adapter = _adapter(client) + reply = await _quick(adapter.click(BUTTON)) + assert reply["ok"] is True + assert reply["new_targets"] == [{"target_id": "T2", "url": "", "title": "", "active": False}] + assert [sid for name, _params, sid in client.calls if name == "Runtime.runIfWaitingForDebugger"] == [SESSION] + + asyncio.run(scenario()) + + +def test_targets_that_are_not_pages_are_not_new_contexts(): + async def scenario() -> None: + client = FakeCdp() + Page(client) + adapter = _adapter(client) + await adapter._ensure_listeners() + + async def body(act: Any) -> tuple[bool, str]: + client.emit("Target.targetCreated", {"targetInfo": {**POPUP, "type": "service_worker"}}, session_id=None) + return True, "done" + + reply = await _quick(adapter._act(body)) + assert reply["new_targets"] == [] + + asyncio.run(scenario()) + + +def test_a_page_opened_between_actions_is_not_reported_by_the_next_one(): + async def scenario() -> None: + client = FakeCdp() + Page(client) + adapter = _adapter(client) + await adapter._ensure_listeners() + client.emit("Target.targetCreated", {"targetInfo": POPUP}, session_id=None) + reply = await _quick(adapter.click(BUTTON)) + assert reply["new_targets"] == [] + assert client.sent("Runtime.runIfWaitingForDebugger") == [] + + asyncio.run(scenario()) + + +def test_new_target_listener_keeps_the_handler_browser_use_registered(): + async def scenario() -> None: + client = FakeCdp() + Page(client) + seen: list[str] = [] + client._event_registry.register("Target.targetCreated", lambda event, sid: seen.append("browser-use")) + adapter = _adapter(client) + await adapter._ensure_listeners() + await adapter._ensure_listeners() + client.emit("Target.targetCreated", {"targetInfo": POPUP}, session_id=None) + assert seen == ["browser-use"] + + asyncio.run(scenario()) + + +LINK = {"frameId": TARGET, "reason": "anchorClick", "url": "http://x/second.html", "disposition": "currentTab"} +RELOAD = {**LINK, "reason": "formSubmissionGet", "url": "http://x/index.html"} + + +@pytest.mark.parametrize("navigation", [LINK, RELOAD], ids=["link", "same-url-reload"]) +def test_a_navigation_the_action_started_is_waited_for_and_changes_the_document(navigation): + async def scenario() -> None: + client = FakeCdp() + Page(client, navigation=navigation, commit_after=0.05) + adapter = _adapter(client) + reply = await _quick(adapter.click(BUTTON)) + assert reply["document_changed"] is True + assert reply["driver_generation"] == 1 + + asyncio.run(scenario()) + + +def test_a_navigation_that_does_not_commit_in_time_is_not_a_document_change(monkeypatch): + monkeypatch.setattr(session_adapter, "_NAVIGATION_COMMIT_WAIT_S", 0.05) + + async def scenario() -> None: + client = FakeCdp() + Page(client, navigation=LINK, commit_after=None) + adapter = _adapter(client) + reply = await _quick(adapter.click(BUTTON)) + assert reply["document_changed"] is False + + asyncio.run(scenario()) + + +@pytest.mark.parametrize( + "navigation", + [{**LINK, "disposition": "newForegroundTab"}, {**LINK, "frameId": "T2"}], + ids=["new-tab", "other-frame"], +) +def test_navigations_outside_the_main_frame_are_not_waited_for(monkeypatch, navigation): + monkeypatch.setattr(session_adapter, "_NAVIGATION_COMMIT_WAIT_S", 60.0) + + async def scenario() -> None: + client = FakeCdp() + Page(client, navigation=navigation, commit_after=None) + adapter = _adapter(client) + reply = await _quick(adapter.click(BUTTON)) + assert reply["document_changed"] is False + + asyncio.run(scenario()) + + +def test_a_child_frame_document_is_not_a_document_change(): + async def scenario() -> None: + client = FakeCdp() + Page(client) + adapter = _adapter(client) + await adapter._ensure_listeners() + + async def body(act: Any) -> tuple[bool, str]: + frame = {"id": TARGET, "parentId": "P", "loaderId": "L2", "url": "http://x/frame.html"} + client.emit("Page.frameNavigated", {"frame": frame}) + return True, "done" + + reply = await _quick(adapter._act(body)) + assert reply["document_changed"] is False + + asyncio.run(scenario()) + + +cdp = session_adapter.cdp +NOT_IN_DOCUMENT = RuntimeError({"code": -32000, "message": "Node with given id does not belong to the document"}) +NO_SUCH_NODE = RuntimeError({"code": -32000, "message": "No node with given id found"}) + + +def _resolve_fails_from(client: FakeCdp, call: int, error: Exception) -> None: + calls = [] + + def resolve(params: dict[str, Any], sid: str | None) -> dict[str, Any]: + calls.append(params) + if len(calls) >= call: + raise error + return {"object": {"objectId": "obj-7"}} + + client.replies["DOM.resolveNode"] = resolve + + +@pytest.mark.parametrize("error", [NOT_IN_DOCUMENT, NO_SUCH_NODE], ids=["not-in-document", "no-such-node"]) +def test_a_node_that_is_not_in_the_document_is_stale(error): + client = FakeCdp() + _resolve_fails_from(client, 1, error) + with pytest.raises(errors.StaleNodeError, match="backend node 7 is not in the current document"): + asyncio.run(cdp.resolve_object_id_by_backend_node(client, SESSION, 7)) + + +def test_other_resolve_failures_are_not_stale(): + client = FakeCdp() + _resolve_fails_from(client, 1, RuntimeError({"code": -32000, "message": "Target closed"})) + with pytest.raises(RuntimeError, match="Target closed"): + asyncio.run(cdp.resolve_object_id_by_backend_node(client, SESSION, 7)) + + +@pytest.mark.parametrize( + "act", + [lambda adapter: adapter.click(BUTTON), lambda adapter: adapter.hover(BUTTON)], + ids=["click", "hover"], +) +def test_an_action_on_a_node_of_an_old_document_is_stale_and_sends_nothing(act): + async def scenario() -> None: + client = FakeCdp() + Page(client) + _resolve_fails_from(client, 1, NOT_IN_DOCUMENT) + adapter = _adapter(client) + with pytest.raises(errors.StaleNodeError): + await _quick(act(adapter)) + assert client.sent("Input.dispatchMouseEvent") == [] + + asyncio.run(scenario()) + + +def test_a_node_lost_after_the_input_is_not_stale(): + async def scenario() -> None: + client = FakeCdp() + Page(client) + _resolve_fails_from(client, 2, NOT_IN_DOCUMENT) + adapter = _adapter(client) + with pytest.raises(errors.DriverError, match="after input reached the page") as info: + await _quick(adapter.type_text(BUTTON, "Ada")) + assert type(info.value) is errors.DriverError + assert client.sent("Input.dispatchMouseEvent"), "the click reached the page" + + asyncio.run(scenario()) + + +def test_a_cached_node_of_an_old_document_is_stale(): + async def scenario() -> None: + client = FakeCdp() + _resolve_fails_from(client, 1, NOT_IN_DOCUMENT) + adapter = _adapter(client) + adapter._index_cache = {1: SimpleNamespace(backend_node_id=7)} + with pytest.raises(errors.StaleNodeError): + await adapter._node_for_event(BUTTON) + + asyncio.run(scenario()) + + +OVERLAY = 'span#overlay "Overlay banner"' +OTHER = {"kind": "driver", "handle": "bnid:8", "driver_generation": 0} + + +@pytest.mark.parametrize( + ("method", "act"), + [ + ("click", lambda adapter: adapter.click(BUTTON)), + ("hover", lambda adapter: adapter.hover(BUTTON)), + ("drag", lambda adapter: adapter.drag(BUTTON, OTHER)), + ], + ids=["click", "hover", "drag-source"], +) +def test_input_another_element_would_receive_is_intercepted_before_it_is_sent(method, act): + async def scenario() -> None: + client = FakeCdp() + Page(client, covered_by=OVERLAY) + adapter = _adapter(client) + with pytest.raises(errors.ActionIntercepted, match=f"{method} at \\(60, 40\\) would reach") as info: + await _quick(act(adapter)) + assert info.value.by == OVERLAY + assert client.sent("Input.dispatchMouseEvent") == [] + + asyncio.run(scenario()) + + +def test_the_hit_test_uses_the_point_the_mouse_will_use(): + async def scenario() -> None: + client = FakeCdp() + Page(client) + adapter = _adapter(client) + reply = await _quick(adapter.click(BUTTON)) + assert reply["ok"] is True + hit_tests = [p for p in client.sent("Runtime.callFunctionOn") if "elementFromPoint" in p["functionDeclaration"]] + assert [p["arguments"] for p in hit_tests] == [[{"value": 60.0}, {"value": 40.0}]] + presses = [p for p in client.sent("Input.dispatchMouseEvent") if p["type"] == "mousePressed"] + assert [(p["x"], p["y"]) for p in presses] == [(60.0, 40.0)] + + asyncio.run(scenario()) + + +def test_the_wire_error_carries_the_receiver(): + main = _load_sidecar_module("main") + exc = errors.ActionIntercepted("click would reach it", by=OVERLAY) + assert main._error_code_for(exc) == "ActionIntercepted" + assert main._error_data_for(exc) == {"by": OVERLAY} + + +cdp = _load_sidecar_module("cdp") + + +def _sent_expression(source: str, args: Any = None) -> str: + client = SimpleNamespace(send=SimpleNamespace(Runtime=SimpleNamespace(evaluate=AsyncMock(return_value={})))) + asyncio.run(cdp.evaluate_expression(client, SESSION, source, args)) + return client.send.Runtime.evaluate.await_args.kwargs["params"]["expression"] + + +@pytest.mark.parametrize( + ("source", "args", "value"), + [ + ("document.title", None, "Wolfram"), + ("() => document.title", None, "Wolfram"), + ("(input) => input.a + 1", {"a": 1}, 2), + ("document.title.length // a comment", None, 7), + ("async () => document.title", None, "Wolfram"), + ], +) +def test_evaluate_takes_an_expression_or_a_function(source: str, args: Any, value: Any) -> None: + node = shutil.which("node") + if node is None: + pytest.skip("node is not installed") + script = f"globalThis.document = {{ title: 'Wolfram' }};\nPromise.resolve({_sent_expression(source, args)})" + script += ".then((value) => console.log(JSON.stringify(value)));" + completed = subprocess.run([node, "-e", script], check=True, text=True, capture_output=True) + assert json.loads(completed.stdout) == value + + +def test_a_script_error_names_the_javascript_error_not_just_uncaught() -> None: + reply = { + "exceptionDetails": { + "text": "Uncaught", + "exception": {"description": "TypeError: document.title is not a function\n at :1:16"}, + } + } + + with pytest.raises(errors.EvaluateError) as info: + cdp._unwrap_js_result(reply) + + assert info.value.js_message == "TypeError: document.title is not a function" + assert str(info.value) == "TypeError: document.title is not a function" + + +def _guard_calls(client: FakeCdp) -> tuple[list[str | None], list[str]]: + installed = [sid for name, _params, sid in client.calls if name == "Page.addScriptToEvaluateOnNewDocument"] + evaluated = [params["expression"] for name, params, _sid in client.calls if name == "Runtime.evaluate"] + return installed, evaluated + + +def test_the_init_script_is_installed_once_per_session_for_new_documents_and_the_current_one(): + client = FakeCdp() + adapter = session_adapter.SessionAdapter() + adapter._init_script = "guard();" + ids = (SESSION, POPUP_SESSION) + sessions = [SimpleNamespace(cdp_client=client, session_id=sid, target_id=TARGET) for sid in ids] + adapter._session = SimpleNamespace(get_or_create_cdp_session=AsyncMock(side_effect=sessions)) + + async def scenario() -> None: + await adapter._ensure_cdp_session() + await adapter._ensure_cdp_session() + adapter._cdp_session = None + await adapter._ensure_cdp_session() + + asyncio.run(scenario()) + + installed, evaluated = _guard_calls(client) + assert installed == [SESSION, POPUP_SESSION] + assert client.sent("Page.addScriptToEvaluateOnNewDocument") == [{"source": "guard();"}] * 2 + assert len(evaluated) == 2 and all("guard();" in expression for expression in evaluated) + + +def test_without_an_init_script_nothing_is_installed(): + client = FakeCdp() + adapter = session_adapter.SessionAdapter() + session = SimpleNamespace(cdp_client=client, session_id=SESSION, target_id=TARGET) + adapter._session = SimpleNamespace(get_or_create_cdp_session=AsyncMock(return_value=session)) + + asyncio.run(adapter._ensure_cdp_session()) + + assert _guard_calls(client) == ([], []) + + +@pytest.mark.parametrize(("slowly", "inserted"), [(False, ["Ada"]), (True, ["A", "d", "a"])]) +def test_slow_typing_inserts_one_character_at_a_time(monkeypatch, slowly: bool, inserted: list[str]): + monkeypatch.setattr(session_adapter, "_SLOW_TYPING_DELAY_S", 0) + client = FakeCdp() + Page(client) + adapter = _adapter(client) + + reply = asyncio.run(_quick(adapter.type_text(BUTTON, "Ada", slowly=slowly))) + + assert reply["ok"] is True + assert [params["text"] for params in client.sent("Input.insertText")] == inserted diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/test_conformance_browser_use_live.py b/tests/unit_tests/harness/tools/browser_move/drivers/test_conformance_browser_use_live.py new file mode 100644 index 0000000000..5912b9f0b3 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/test_conformance_browser_use_live.py @@ -0,0 +1,123 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The driver conformance suite, live on ``BrowserUseDriver`` with the sidecar and a local headless Chrome.""" + +from __future__ import annotations + +import json +import os +import shutil +import socket +import subprocess +import tempfile +import time +from collections.abc import Iterator +from pathlib import Path +from urllib.error import URLError +from urllib.request import urlopen + +import pytest + +from openjiuwen.harness.tools.browser_move.contract.testing.conformance import DriverConformance +from openjiuwen.harness.tools.browser_move.drivers.browser_use.driver import ( + BrowserUseDriver, + create_browser_use_driver, +) +from openjiuwen.harness.tools.browser_move.drivers.middleware import Tracing +from openjiuwen.harness.tools.browser_move.provisioning import ProvisionedDevice +from openjiuwen.harness.tools.browser_move.provisioning.managed_browser import _candidate_chrome_binaries +from openjiuwen.harness.tools.browser_move.surfaces.browser.config import BrowserSettings +from openjiuwen.harness.tools.browser_move.surfaces.browser.scene import browser_scene +from tests.unit_tests.harness.tools.browser_move.drivers.browser_scene_extras import BrowserSceneExtras + +pytest_plugins = ["openjiuwen.harness.tools.browser_move.contract.testing.conformance"] + +_SIDECAR_PYTHON_ENV = "BROWSER_USE_SIDECAR_PYTHON" +_CHROME_READY_TIMEOUT_S = 30.0 + + +def _chrome_binaries() -> list[str]: + return _candidate_chrome_binaries(BrowserSettings.from_process_env().browser_binary_candidates) + + +def _skip_reason() -> str | None: + sidecar_python = os.getenv(_SIDECAR_PYTHON_ENV, "") + if not sidecar_python: + return f"{_SIDECAR_PYTHON_ENV} is not set (the browser-use sidecar interpreter)" + if not Path(sidecar_python).is_file(): + return f"{_SIDECAR_PYTHON_ENV}={sidecar_python!r} is not an existing interpreter" + if not _chrome_binaries(): + return "no Chrome binary found" + return None + + +def _free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe: + probe.bind(("127.0.0.1", 0)) + return int(probe.getsockname()[1]) + + +def _wait_for_cdp(endpoint: str, process: subprocess.Popen) -> None: + deadline = time.monotonic() + _CHROME_READY_TIMEOUT_S + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError(f"Chrome exited with code {process.returncode} before CDP was ready") + try: + with urlopen(f"{endpoint}/json/version", timeout=1.0) as response: # nosec + json.load(response) + return + except (URLError, OSError, ValueError): + time.sleep(0.2) + raise RuntimeError(f"Chrome CDP endpoint {endpoint} not ready after {_CHROME_READY_TIMEOUT_S}s") + + +def _kill_tree(process: subprocess.Popen) -> None: + """Kill the Chrome we started and its child processes, nothing else.""" + if os.name == "nt": + subprocess.run( # noqa: S603, S607 + ["taskkill", "/F", "/T", "/PID", str(process.pid)], capture_output=True, check=False + ) + else: + process.kill() + process.wait(timeout=10) + + +@pytest.fixture(scope="class") +def chrome_device() -> Iterator[ProvisionedDevice]: + """A headless Chrome with a fresh profile on a free CDP port, as a provisioner would hand it to the driver.""" + port = _free_port() + profile = Path(tempfile.mkdtemp(prefix="bu-conformance-chrome-")) + process = subprocess.Popen( # noqa: S603 + [ + _chrome_binaries()[0], + "--headless=new", + "--remote-debugging-address=127.0.0.1", + f"--remote-debugging-port={port}", + f"--user-data-dir={profile}", + "--no-first-run", + "--no-default-browser-check", + "--window-size=1280,900", + "about:blank", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + endpoint = f"http://127.0.0.1:{port}" + try: + _wait_for_cdp(endpoint, process) + yield ProvisionedDevice(kind="chrome", endpoint=endpoint, extras={"user_data_dir": str(profile)}, cleanup=None) + finally: + _kill_tree(process) + shutil.rmtree(profile, ignore_errors=True) + + +@pytest.mark.live +@pytest.mark.skipif(_skip_reason() is not None, reason=_skip_reason() or "") +class TestBrowserUseConformance(DriverConformance, BrowserSceneExtras): + scene = browser_scene() + driver_actions = BrowserUseDriver.handled_actions() + + @pytest.fixture + def conformance_driver_factory(self, chrome_device: ProvisionedDevice): + settings = BrowserSettings.from_process_env() + return lambda sink: Tracing(create_browser_use_driver(chrome_device, settings), sink) diff --git a/tests/unit_tests/harness/tools/browser_move/drivers/test_middleware.py b/tests/unit_tests/harness/tools/browser_move/drivers/test_middleware.py new file mode 100644 index 0000000000..36fb7ef05b --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/drivers/test_middleware.py @@ -0,0 +1,343 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Timeouts, RetryOnStale and Tracing on ``FakeDriver``, and the conformance suite through the full stack.""" + +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from typing import ClassVar + +import pytest + +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + ActionTimeout, + Click, + ContextRef, + DriverError, + DriverInfo, + DriverRef, + Effect, + Health, + Observation, + Observe, + PressKey, + RunScript, + StaleRef, + TypeText, + Wait, +) +from openjiuwen.harness.tools.browser_move.contract.testing import FakeDriver +from openjiuwen.harness.tools.browser_move.contract.testing.conformance import DriverConformance, TraceSink +from openjiuwen.harness.tools.browser_move.drivers.middleware import MASK, RetryOnStale, Timeouts, Tracing +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import FieldKind, FillForm, FormField + +from ..contract.test_conformance_fake import FAKE_ACTIONS, FAKE_SCENE, WorldDriver + +pytest_plugins = ["openjiuwen.harness.tools.browser_move.contract.testing.conformance"] + +REF = DriverRef("e1", "g1") +OK = ActionResult(ok=True) +SECRET = "hunter2-middleware" + + +def _observation(context: str = "c1") -> Observation: + return Observation(context=ContextRef(context, "screen")) + + +class SlowDriver(FakeDriver): + def __init__(self, delay: float, **kwargs) -> None: + super().__init__(**kwargs) + self.delay = delay + self.cancelled = 0 + + async def _slow(self) -> None: + try: + await asyncio.sleep(self.delay) + except asyncio.CancelledError: + self.cancelled += 1 + raise + + async def observe(self, command: Observe) -> Observation: + await self._slow() + return await super().observe(command) + + async def execute(self, action: Action) -> ActionResult: + await self._slow() + return await super().execute(action) + + +class TestTimeouts: + def test_execute_over_its_effect_bound_is_a_dispatched_timeout(self): + inner = SlowDriver(1.0, results=[OK]) + driver = Timeouts(inner, {Effect.ACT: 0.01}) + + with pytest.raises(ActionTimeout) as raised: + asyncio.run(driver.execute(Click(target=REF))) + + assert raised.value.dispatched is True + assert inner.cancelled == 1 + + def test_observe_is_bounded_as_read(self): + driver = Timeouts(SlowDriver(1.0, observations=[_observation()]), {Effect.READ: 0.01, Effect.ACT: 5.0}) + + with pytest.raises(ActionTimeout) as raised: + asyncio.run(driver.observe(Observe())) + + assert raised.value.dispatched is True + + def test_each_effect_gets_its_own_bound(self): + driver = Timeouts(SlowDriver(0.05, results=[OK]), {Effect.ACT: 5.0, Effect.READ: 0.01}) + + assert asyncio.run(driver.execute(Click(target=REF))) is OK + with pytest.raises(ActionTimeout): + asyncio.run(driver.execute(Wait(seconds=0))) + + def test_an_unlisted_effect_is_not_bounded(self): + driver = Timeouts(SlowDriver(0.05, results=[OK]), {Effect.READ: 0.01}) + + assert asyncio.run(driver.execute(RunScript(code="noop"))) is OK + + def test_inner_errors_pass_through_unchanged(self): + inner = FakeDriver() + inner.fail_next(StaleRef("old", dispatched=False)) + + with pytest.raises(StaleRef): + asyncio.run(Timeouts(inner, {Effect.ACT: 5.0}).execute(Click(target=REF))) + + +class TestRetryOnStale: + def test_stale_ref_is_raised_again_with_one_fresh_observation(self): + fresh = _observation("c2") + inner = FakeDriver(observations=[fresh]) + inner.fail_next(StaleRef("e1 is old", dispatched=False)) + + with pytest.raises(StaleRef) as raised: + asyncio.run(RetryOnStale(inner).execute(Click(target=REF))) + + assert raised.value.observation is fresh + assert raised.value.dispatched is False + assert inner.kinds() == ["execute", "observe"] + + def test_a_dispatched_stale_ref_is_neither_observed_nor_retried(self): + inner = FakeDriver(observations=[_observation()], results=[OK]) + inner.fail_next(StaleRef("sent already", dispatched=True)) + + with pytest.raises(StaleRef) as raised: + asyncio.run(RetryOnStale(inner).execute(Click(target=REF))) + + assert raised.value.observation is None + assert inner.kinds() == ["execute"] + + @pytest.mark.parametrize( + "error", + [ActionTimeout("slow"), DriverError("unknown outcome")], + ids=["timeout", "unknown-outcome"], + ) + def test_a_dispatched_error_is_never_retried(self, error): + inner = FakeDriver(observations=[_observation()], results=[OK]) + inner.fail_next(error) + + with pytest.raises(type(error)): + asyncio.run(RetryOnStale(inner).execute(Click(target=REF))) + + assert inner.kinds() == ["execute"] + + def test_the_command_is_sent_once_even_when_the_driver_stays_stale(self): + inner = FakeDriver(observations=[_observation(), _observation()], results=[OK]) + inner.fail_next(StaleRef("old", dispatched=False)) + inner.fail_next(StaleRef("still old", dispatched=False)) + + with pytest.raises(StaleRef): + asyncio.run(RetryOnStale(inner).execute(Click(target=REF))) + + assert inner.kinds() == ["execute", "observe"] + + def test_a_success_is_returned_without_observing(self): + inner = FakeDriver(results=[OK]) + + assert asyncio.run(RetryOnStale(inner).execute(Click(target=REF))) is OK + assert inner.kinds() == ["execute"] + + +@dataclass(frozen=True) +class Unlock(Action): + effect: ClassVar[Effect] = Effect.ACT + __sensitive_fields__: ClassVar[tuple[str, ...]] = ("pin",) + + pin: str + sensitive: bool = True + + +def _traced(action: Action) -> TraceSink: + sink = TraceSink() + info = DriverInfo(name="fake", surfaces=frozenset({"fake"}), actions={type(action): frozenset({DriverRef})}) + asyncio.run(Tracing(FakeDriver(results=[OK], info=info), sink).execute(action)) + return sink + + +class TestTracing: + @pytest.mark.parametrize( + "action", + [ + TypeText(target=REF, text=SECRET, sensitive=True), + RunScript(code=SECRET, sensitive=True), + FillForm(fields=(FormField(target=REF, kind=FieldKind.TEXTBOX, value=SECRET, sensitive=True),)), + Unlock(pin=SECRET), + ], + ids=["type-text", "run-script", "form-field", "surface-command"], + ) + def test_a_sensitive_field_is_masked(self, action): + sink = _traced(action) + + assert SECRET not in sink.text + assert MASK in sink.text + + def test_a_field_is_traced_when_not_sensitive(self): + sink = _traced(TypeText(target=REF, text="Ada")) + + assert sink.records == [ + { + "call": "execute", + "command": "TypeText", + "args": { + "target": {"ref": "e1", "generation": "g1"}, + "text": "Ada", + "submit": False, + "sensitive": False, + "clear": True, + "slowly": False, + }, + "ok": True, + "detail": None, + "context_changed": False, + } + ] + + def test_only_the_sensitive_form_field_is_masked(self): + form = FillForm( + fields=( + FormField(target=REF, kind=FieldKind.TEXTBOX, value="Ada"), + FormField(target=REF, kind=FieldKind.TEXTBOX, value=SECRET, sensitive=True), + ) + ) + sink = _traced(form) + + values = [field["value"] for field in sink.records[0]["args"]["fields"]] + assert values == ["Ada", MASK] + + def test_an_error_is_traced_and_raised(self): + inner = FakeDriver() + inner.fail_next(StaleRef("e1 is old", dispatched=False)) + sink = TraceSink() + + with pytest.raises(StaleRef): + asyncio.run(Tracing(inner, sink).execute(PressKey(key="Enter"))) + + assert sink.records == [ + { + "call": "execute", + "command": "PressKey", + "args": {"key": "Enter", "target": None}, + "error": "StaleRef", + "message": "e1 is old", + "dispatched": False, + } + ] + + def test_sensitive_script_args_are_masked(self): + sink = _traced(RunScript(code="(pin) => pin", args={"pin": SECRET}, sensitive=True)) + + assert sink.records[0]["args"]["args"] == MASK + assert SECRET not in sink.text + + @pytest.mark.parametrize( + "action", + [TypeText(target=REF, text=SECRET, sensitive=True), RunScript(code=SECRET, sensitive=True)], + ids=["type-text", "run-script"], + ) + def test_the_error_message_of_a_sensitive_command_is_dropped(self, action): + inner = FakeDriver() + inner.fail_next(DriverError(f"Error: {SECRET} is not defined", dispatched=True)) + sink = TraceSink() + + with pytest.raises(DriverError): + asyncio.run(Tracing(inner, sink).execute(action)) + + assert "message" not in sink.records[0] + assert sink.records[0]["error"] == "DriverError" + assert SECRET not in sink.text + + def test_the_error_message_is_kept_when_the_command_is_not_sensitive(self): + inner = FakeDriver() + inner.fail_next(DriverError("Error: x is not defined", dispatched=True)) + sink = TraceSink() + + with pytest.raises(DriverError): + asyncio.run(Tracing(inner, sink).execute(RunScript(code="x"))) + + assert sink.records[0]["message"] == "Error: x is not defined" + + def test_observe_is_traced_without_the_element_contents(self): + observation = Observation(context=ContextRef("c1", "screen"), native_text=SECRET) + sink = TraceSink() + + assert asyncio.run(Tracing(FakeDriver(observations=[observation]), sink).observe(Observe())) is observation + + assert SECRET not in sink.text + assert sink.records[0]["context"] == "c1" + assert sink.records[0]["elements"] == 0 + + +def _stack(inner: FakeDriver, sink: TraceSink) -> Tracing: + return Tracing(RetryOnStale(Timeouts(inner, {Effect.READ: 5.0, Effect.ACT: 5.0})), sink) + + +class TestStack: + def test_info_lifecycle_and_health_reach_the_inner_driver(self): + inner = FakeDriver() + driver = _stack(inner, TraceSink()) + + async def run() -> Health: + await driver.start("config") + health = await driver.health() + await driver.stop() + return health + + assert driver.info is inner.info + assert asyncio.run(run()) == Health(ok=True) + assert inner.calls[0].payload == "config" + assert inner.kinds() == ["start", "stop"] + + def test_a_stale_ref_is_traced_once_with_its_fresh_observation(self): + fresh = _observation("c2") + inner = FakeDriver(observations=[fresh]) + inner.fail_next(StaleRef("old", dispatched=False)) + sink = TraceSink() + + with pytest.raises(StaleRef) as raised: + asyncio.run(_stack(inner, sink).execute(Click(target=REF))) + + assert raised.value.observation is fresh + assert [record.get("error") for record in sink.records] == ["StaleRef"] + + def test_a_timeout_is_traced_as_dispatched(self): + sink = TraceSink() + driver = Tracing(RetryOnStale(Timeouts(SlowDriver(1.0, results=[OK]), {Effect.ACT: 0.01})), sink) + + with pytest.raises(ActionTimeout): + asyncio.run(driver.execute(Click(target=REF))) + + assert sink.records[0]["error"] == "ActionTimeout" + assert sink.records[0]["dispatched"] is True + + +class TestConformanceThroughTheStack(DriverConformance): + scene = FAKE_SCENE + driver_actions = FAKE_ACTIONS + + @pytest.fixture + def conformance_driver_factory(self): + return lambda sink: _stack(WorldDriver(TraceSink()), sink) diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/__init__.py b/tests/unit_tests/harness/tools/browser_move/fixtures/__init__.py new file mode 100644 index 0000000000..285ed342d6 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/__init__.py @@ -0,0 +1,2 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/README.md b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/README.md new file mode 100644 index 0000000000..b522394a92 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/README.md @@ -0,0 +1,120 @@ +# Recorded `@playwright/mcp@0.0.78` answers + +Real answers of the Playwright MCP server, recorded once from a local headless +Chrome against the static pages in `pages/`. They are the input for the +snapshot parser and envelope decoder (`drivers/playwright_mcp/`) and for the +rail tests. `test_mcp_fixtures_wellformed.py` only checks that the +files are complete; `drivers/playwright_mcp/test_snapshot.py` and +`test_result.py` parse them. + +## Re-recording + +Needs Node.js (`npx`) and Google Chrome installed. No internet access is needed +apart from `npx` fetching the pinned server package the first time. From the +repository root: + +```bash +uv run python tests/unit_tests/harness/tools/browser_move/fixtures/mcp/record_mcp_fixtures.py +``` + +The recorder: + +- serves `pages/` on a free port of `127.0.0.1`, plus `/hang`, which answers + only after 30 s; +- launches the server the way the runtime does: `build_playwright_mcp_config()` + and `BrowserMoveStdioClient`, with the WebVoyager baseline arguments + (`-y @playwright/mcp@0.0.78 --isolated --config `) plus + `--headless`; `--isolated` keeps the profile in memory, so no real profile + is ever used. The server's working directory is a temporary directory that + is deleted afterwards; +- calls each tool through `MCPTool.invoke`, which is what `ToolMgr` registers + for every MCP tool today, and records: + - `request`: tool name and arguments; + - `raw_result`: the `CallToolResult` the MCP session returned + (`model_dump(mode="json", by_alias=True, exclude_none=True)`), before any + browser_move or core processing; + - `model_text`: `MCPTool.render_for_llm` of the invoke output, the tool + message content the model receives before any rail rewrites it. A + successful call renders as the plain text; a call with `isError: true` + renders as the JSON envelope `{"result": ""}`. + +Run-specific values are replaced: the page server's port by ``, the +temporary directory by ``, a second temporary directory outside the +server's roots by ``, and the timestamps in the server's output-file +names (`page-.yml`, `console-.log`) by ``. With these placeholders, +running the recorder again produces the same files. + +## Cases + +All cases run in one server session, in this order. The task's item number is +in brackets. + +| File | What it records | +|---|---| +| `snapshot_form_page` [1] | `browser_snapshot` of `index.html`: text input, select, checkbox, links, a download link, an iframe (refs `f1e*`), alert/confirm triggers | +| `snapshot_depth_1` | `browser_snapshot` `depth: 1` of `index.html`: the root and its children only; the navigation, the form container and the iframe appear without their subtrees | +| `snapshot_target` | `browser_snapshot` `target: ` of `index.html`: the navigation subtree alone, under the usual `### Page` header | +| `click_success` [2] | `browser_click` on "Save"; the page text changes | +| `events_console_message` [6] | `browser_click` that runs `console.log`; result has `### Events` with `New console entries: …` | +| `events_download` [6] | `browser_click` on a `download` link; `### Events` reports the download start and the saved file | +| `dialog_open_by_click` [7] | `browser_click` that opens `confirm()`; result has `### Modal state` and no page title | +| `dialog_handle_accept` [7] | `browser_handle_dialog` `accept: true` | +| `navigate_same_url_reload` [8] | `browser_navigate` to the URL the tab is already on | +| `navigate_second_page` [4] | `browser_navigate` to `second.html` | +| `tabs_list` [5] | `browser_tabs` `list` with one tab | +| `tabs_new` [5] | `browser_tabs` `new` (opens `about:blank`) | +| `tabs_select` [5] | `browser_tabs` `select` index 0 | +| `tabs_close` [5] | `browser_tabs` `close` index 1 | +| `click_intercepted_by_overlay` [3] | `browser_click` on a button under a fixed backdrop: `isError`, Playwright's call log with `intercepts pointer events`, ending in the 5000 ms action timeout | +| `click_timeout_not_enabled` [9] | `browser_click` on a disabled button: `isError`, call log with `element is not enabled`, the same 5000 ms action timeout | +| `click_stale_ref` | `browser_click` with a ref of `blocked.html` after navigating away: `Ref … not found in the current page snapshot` | +| `snapshot_with_boxes` | `browser_snapshot` `boxes: true` of `index.html`: every ref line has `[box=x,y,width,height]` | +| `fill_form` | `browser_fill_form` with a textbox, a checkbox and a combobox: only `### Ran Playwright code`, no page or snapshot | +| `snapshot_after_fill_form` | `browser_snapshot` after it: the textbox value is a child `text:` line, the combobox value its `[selected]` option, the checkbox `[checked]` | +| `select_option` | `browser_select_option` `values: ["Large"]` | +| `evaluate_result` | `browser_evaluate` returning an object: `### Result` with pretty-printed JSON before `### Ran Playwright code` | +| `evaluate_error` | `browser_evaluate` whose function throws: `isError`, the JS error and its stack | +| `screenshot_element` | `browser_take_screenshot` of the heading: a `### Result` file link plus an `image/png` content block | +| `wait_for_timeout` | `browser_wait_for` text that never appears: `TimeoutError: Timeout 5000ms exceeded.`, call log with only the visibility wait | +| `click_blocked_by_dialog` | `browser_click` while a `confirm()` is open: `Tool "browser_click" does not handle the modal state.` plus `### Modal state` | +| `click_opens_tab` | `browser_click` on a `target="_blank"` link of `popup.html`: `### Open tabs` lists both tabs, the focused tab is unchanged | +| `snapshot_two_tabs` | `browser_snapshot` with two tabs: `### Open tabs` before `### Page`; YAML quoting (`'button "Note: read me" …'`, `paragraph …: "42"`) | +| `click_navigates` | `browser_click` on a same-tab link: only the URL in `### Page` shows the navigation | +| `navigate_back` | `browser_navigate_back` | +| `file_chooser_open_by_click` | `browser_click` on a file input of `upload.html`: `### Modal state` with `[File chooser]` | +| `file_upload` | `browser_file_upload` of a file in the server's working directory | +| `click_timeout_after_action` | `browser_click` on a link to `/hang` (the page server answers after 30 s): call log with `performing click action`, `click action done`, then the 5000 ms timeout while waiting for the navigation | +| `snapshot_during_dialog` | `browser_snapshot` `boxes: true` while a `confirm()` is open: refused like any other page tool (`does not handle the modal state` plus `### Modal state`), no `### Page` | +| `tabs_list_during_dialog` | `browser_tabs` `list` while the `confirm()` is open: not a page tool, so it answers, with an empty title and `### Modal state` | +| `drop_outside_roots` | `browser_drop` of a file outside the server's working and output directories: `File access denied: … is outside allowed roots`, refused before the page is touched | +| `evaluate_new_tab_failed` | first `browser_evaluate` after the browser was closed during a call (set up with `browser_run_code_unsafe`): `Failed to open a new tab` | +| `evaluate_browser_closed` | the next `browser_evaluate`: `Target page, context or browser has been closed` | + +## Things to know about the raw format + +- Every result is a single `text` content block; no `structuredContent`. +- Element arguments are named `target` in 0.0.78 (not `ref`). +- Action results (`browser_click`, `browser_navigate`) do not inline the + snapshot: their `### Snapshot` section is a link to a file the server writes + into its working directory (`.playwright-mcp\page-.yml` on Windows). + Only `browser_snapshot` returns the YAML inline. +- Refs are not always bare `e`: refs inside an iframe are `fe`, and + even the main frame of `blocked.html` (opened after several navigations and + the tab operations) gets `f5e*` refs. A parser must accept `(f\d+)?e\d+`. +- The intercepted click and the plain timeout share the error header + `TimeoutError: browserBackend.callTool: Timeout 5000ms exceeded.`; only the + call log tells them apart. The call log keeps the ANSI dim escapes + (`\u001b[2m … \u001b[22m`) and folds repeats as `N ×`. +- The linked snapshot file holds only the YAML. While a dialog is open the + server writes it empty. +- While a dialog or a file chooser is open, the server refuses every page + tool, `browser_snapshot` and `browser_take_screenshot` included; + `browser_tabs` still answers (the title it cannot read is empty). +- Nothing marks a navigation by click or a new tab: only the URL in + `### Page` and the `### Open tabs` list (shown only with more than one tab) + differ from the result before. +- Closing the browser with a tool call does not break the session: the server + starts a new one on the next call. Only a close during a running call leaves + it broken, which is how the two `evaluate_*` closed cases are set up. +- The temporary directory also appears with doubled backslashes inside the JS + code lines; both spellings become ``. diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/__init__.py b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/__init__.py new file mode 100644 index 0000000000..285ed342d6 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/__init__.py @@ -0,0 +1,2 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_blocked_by_dialog.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_blocked_by_dialog.json new file mode 100644 index 0000000000..9b1f66c035 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_blocked_by_dialog.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "click_blocked_by_dialog", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Save button", + "target": "f7e12" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nError: Tool \"browser_click\" does not handle the modal state.\n### Modal state\n- [\"confirm\" dialog with message \"Delete order?\"]: can be handled by browser_handle_dialog" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nError: Tool \\\"browser_click\\\" does not handle the modal state.\\n### Modal state\\n- [\\\"confirm\\\" dialog with message \\\"Delete order?\\\"]: can be handled by browser_handle_dialog\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_intercepted_by_overlay.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_intercepted_by_overlay.json new file mode 100644 index 0000000000..58196aad3b --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_intercepted_by_overlay.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "click_intercepted_by_overlay", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Covered button", + "target": "f5e3" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nTimeoutError: browserBackend.callTool: Timeout 5000ms exceeded.\nCall log:\n\u001b[2m - waiting for locator('aria-ref=f5e3')\u001b[22m\n\u001b[2m - locator resolved to \u001b[22m\n\u001b[2m - attempting click action\u001b[22m\n\u001b[2m 2 × waiting for element to be visible, enabled and stable\u001b[22m\n\u001b[2m - element is visible, enabled and stable\u001b[22m\n\u001b[2m - scrolling into view if needed\u001b[22m\n\u001b[2m - done scrolling\u001b[22m\n\u001b[2m -
intercepts pointer events\u001b[22m\n\u001b[2m - retrying click action\u001b[22m\n\u001b[2m - waiting 20ms\u001b[22m\n\u001b[2m 2 × waiting for element to be visible, enabled and stable\u001b[22m\n\u001b[2m - element is visible, enabled and stable\u001b[22m\n\u001b[2m - scrolling into view if needed\u001b[22m\n\u001b[2m - done scrolling\u001b[22m\n\u001b[2m -
intercepts pointer events\u001b[22m\n\u001b[2m - retrying click action\u001b[22m\n\u001b[2m - waiting 100ms\u001b[22m\n\u001b[2m 10 × waiting for element to be visible, enabled and stable\u001b[22m\n\u001b[2m - element is visible, enabled and stable\u001b[22m\n\u001b[2m - scrolling into view if needed\u001b[22m\n\u001b[2m - done scrolling\u001b[22m\n\u001b[2m -
intercepts pointer events\u001b[22m\n\u001b[2m - retrying click action\u001b[22m\n\u001b[2m - waiting 500ms\u001b[22m\n" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nTimeoutError: browserBackend.callTool: Timeout 5000ms exceeded.\\nCall log:\\n\\u001b[2m - waiting for locator('aria-ref=f5e3')\\u001b[22m\\n\\u001b[2m - locator resolved to \\u001b[22m\\n\\u001b[2m - attempting click action\\u001b[22m\\n\\u001b[2m 2 × waiting for element to be visible, enabled and stable\\u001b[22m\\n\\u001b[2m - element is visible, enabled and stable\\u001b[22m\\n\\u001b[2m - scrolling into view if needed\\u001b[22m\\n\\u001b[2m - done scrolling\\u001b[22m\\n\\u001b[2m -
intercepts pointer events\\u001b[22m\\n\\u001b[2m - retrying click action\\u001b[22m\\n\\u001b[2m - waiting 20ms\\u001b[22m\\n\\u001b[2m 2 × waiting for element to be visible, enabled and stable\\u001b[22m\\n\\u001b[2m - element is visible, enabled and stable\\u001b[22m\\n\\u001b[2m - scrolling into view if needed\\u001b[22m\\n\\u001b[2m - done scrolling\\u001b[22m\\n\\u001b[2m -
intercepts pointer events\\u001b[22m\\n\\u001b[2m - retrying click action\\u001b[22m\\n\\u001b[2m - waiting 100ms\\u001b[22m\\n\\u001b[2m 10 × waiting for element to be visible, enabled and stable\\u001b[22m\\n\\u001b[2m - element is visible, enabled and stable\\u001b[22m\\n\\u001b[2m - scrolling into view if needed\\u001b[22m\\n\\u001b[2m - done scrolling\\u001b[22m\\n\\u001b[2m -
intercepts pointer events\\u001b[22m\\n\\u001b[2m - retrying click action\\u001b[22m\\n\\u001b[2m - waiting 500ms\\u001b[22m\\n\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_navigates.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_navigates.json new file mode 100644 index 0000000000..d04fdb1455 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_navigates.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "click_navigates", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Second page link", + "target": "f9e4" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByRole('link', { name: 'Go to second page' }).click();\n```\n### Open tabs\n- 0: (current) [Fixture second page](http://127.0.0.1:/second.html)\n- 1: [Fixture second page](http://127.0.0.1:/second.html)\n### Page\n- Page URL: http://127.0.0.1:/second.html\n- Page Title: Fixture second page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByRole('link', { name: 'Go to second page' }).click();\n```\n### Open tabs\n- 0: (current) [Fixture second page](http://127.0.0.1:/second.html)\n- 1: [Fixture second page](http://127.0.0.1:/second.html)\n### Page\n- Page URL: http://127.0.0.1:/second.html\n- Page Title: Fixture second page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_opens_tab.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_opens_tab.json new file mode 100644 index 0000000000..c02ee8744a --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_opens_tab.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "click_opens_tab", + "request": { + "tool": "browser_click", + "arguments": { + "element": "New tab link", + "target": "f9e3" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByRole('link', { name: 'Open second page in a new tab' }).click();\n```\n### Open tabs\n- 0: (current) [Fixture popup page](http://127.0.0.1:/popup.html)\n- 1: [Fixture second page](http://127.0.0.1:/second.html)\n### Page\n- Page URL: http://127.0.0.1:/popup.html\n- Page Title: Fixture popup page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByRole('link', { name: 'Open second page in a new tab' }).click();\n```\n### Open tabs\n- 0: (current) [Fixture popup page](http://127.0.0.1:/popup.html)\n- 1: [Fixture second page](http://127.0.0.1:/second.html)\n### Page\n- Page URL: http://127.0.0.1:/popup.html\n- Page Title: Fixture popup page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_stale_ref.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_stale_ref.json new file mode 100644 index 0000000000..e385020af1 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_stale_ref.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "click_stale_ref", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Covered button", + "target": "f5e3" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nError: Ref f5e3 not found in the current page snapshot. Try capturing new snapshot." + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nError: Ref f5e3 not found in the current page snapshot. Try capturing new snapshot.\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_success.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_success.json new file mode 100644 index 0000000000..5dec2d5dc2 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_success.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "click_success", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Save button", + "target": "e12" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByRole('button', { name: 'Save' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByRole('button', { name: 'Save' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_timeout_after_action.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_timeout_after_action.json new file mode 100644 index 0000000000..9e23ac5486 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_timeout_after_action.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "click_timeout_after_action", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Slow link", + "target": "f13e3" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nTimeoutError: browserBackend.callTool: Timeout 5000ms exceeded.\nCall log:\n\u001b[2m - waiting for locator('aria-ref=f13e3')\u001b[22m\n\u001b[2m - locator resolved to Slow link\u001b[22m\n\u001b[2m - attempting click action\u001b[22m\n\u001b[2m - waiting for element to be visible, enabled and stable\u001b[22m\n\u001b[2m - element is visible, enabled and stable\u001b[22m\n\u001b[2m - scrolling into view if needed\u001b[22m\n\u001b[2m - done scrolling\u001b[22m\n\u001b[2m - performing click action\u001b[22m\n\u001b[2m - click action done\u001b[22m\n\u001b[2m - waiting for scheduled navigations to finish\u001b[22m\n" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nTimeoutError: browserBackend.callTool: Timeout 5000ms exceeded.\\nCall log:\\n\\u001b[2m - waiting for locator('aria-ref=f13e3')\\u001b[22m\\n\\u001b[2m - locator resolved to Slow link\\u001b[22m\\n\\u001b[2m - attempting click action\\u001b[22m\\n\\u001b[2m - waiting for element to be visible, enabled and stable\\u001b[22m\\n\\u001b[2m - element is visible, enabled and stable\\u001b[22m\\n\\u001b[2m - scrolling into view if needed\\u001b[22m\\n\\u001b[2m - done scrolling\\u001b[22m\\n\\u001b[2m - performing click action\\u001b[22m\\n\\u001b[2m - click action done\\u001b[22m\\n\\u001b[2m - waiting for scheduled navigations to finish\\u001b[22m\\n\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_timeout_not_enabled.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_timeout_not_enabled.json new file mode 100644 index 0000000000..465a0ea37c --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/click_timeout_not_enabled.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "click_timeout_not_enabled", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Disabled button", + "target": "f5e4" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nTimeoutError: browserBackend.callTool: Timeout 5000ms exceeded.\nCall log:\n\u001b[2m - waiting for locator('aria-ref=f5e4')\u001b[22m\n\u001b[2m - locator resolved to \u001b[22m\n\u001b[2m - attempting click action\u001b[22m\n\u001b[2m 2 × waiting for element to be visible, enabled and stable\u001b[22m\n\u001b[2m - element is not enabled\u001b[22m\n\u001b[2m - retrying click action\u001b[22m\n\u001b[2m - waiting 20ms\u001b[22m\n\u001b[2m 2 × waiting for element to be visible, enabled and stable\u001b[22m\n\u001b[2m - element is not enabled\u001b[22m\n\u001b[2m - retrying click action\u001b[22m\n\u001b[2m - waiting 100ms\u001b[22m\n\u001b[2m 10 × waiting for element to be visible, enabled and stable\u001b[22m\n\u001b[2m - element is not enabled\u001b[22m\n\u001b[2m - retrying click action\u001b[22m\n\u001b[2m - waiting 500ms\u001b[22m\n" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nTimeoutError: browserBackend.callTool: Timeout 5000ms exceeded.\\nCall log:\\n\\u001b[2m - waiting for locator('aria-ref=f5e4')\\u001b[22m\\n\\u001b[2m - locator resolved to \\u001b[22m\\n\\u001b[2m - attempting click action\\u001b[22m\\n\\u001b[2m 2 × waiting for element to be visible, enabled and stable\\u001b[22m\\n\\u001b[2m - element is not enabled\\u001b[22m\\n\\u001b[2m - retrying click action\\u001b[22m\\n\\u001b[2m - waiting 20ms\\u001b[22m\\n\\u001b[2m 2 × waiting for element to be visible, enabled and stable\\u001b[22m\\n\\u001b[2m - element is not enabled\\u001b[22m\\n\\u001b[2m - retrying click action\\u001b[22m\\n\\u001b[2m - waiting 100ms\\u001b[22m\\n\\u001b[2m 10 × waiting for element to be visible, enabled and stable\\u001b[22m\\n\\u001b[2m - element is not enabled\\u001b[22m\\n\\u001b[2m - retrying click action\\u001b[22m\\n\\u001b[2m - waiting 500ms\\u001b[22m\\n\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/dialog_handle_accept.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/dialog_handle_accept.json new file mode 100644 index 0000000000..6f169dcc83 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/dialog_handle_accept.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "dialog_handle_accept", + "request": { + "tool": "browser_handle_dialog", + "arguments": { + "accept": true + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page" + } + ], + "isError": false + }, + "model_text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/dialog_open_by_click.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/dialog_open_by_click.json new file mode 100644 index 0000000000..307613310d --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/dialog_open_by_click.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "dialog_open_by_click", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Delete order button", + "target": "e15" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByRole('button', { name: 'Delete order' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n### Modal state\n- [\"confirm\" dialog with message \"Delete order?\"]: can be handled by browser_handle_dialog\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByRole('button', { name: 'Delete order' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n### Modal state\n- [\"confirm\" dialog with message \"Delete order?\"]: can be handled by browser_handle_dialog\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/drop_outside_roots.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/drop_outside_roots.json new file mode 100644 index 0000000000..03df000f6b --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/drop_outside_roots.json @@ -0,0 +1,24 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "drop_outside_roots", + "request": { + "tool": "browser_drop", + "arguments": { + "element": "Page heading", + "target": "f16e2", + "paths": [ + "\\outside.csv" + ] + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nError: File access denied: \\outside.csv is outside allowed roots. Allowed roots: \\.playwright-mcp, " + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nError: File access denied: \\\\outside.csv is outside allowed roots. Allowed roots: \\\\.playwright-mcp, \"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_browser_closed.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_browser_closed.json new file mode 100644 index 0000000000..83f690be61 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_browser_closed.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "evaluate_browser_closed", + "request": { + "tool": "browser_evaluate", + "arguments": { + "function": "() => document.title" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nError: browserBackend.callTool: Target page, context or browser has been closed" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nError: browserBackend.callTool: Target page, context or browser has been closed\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_error.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_error.json new file mode 100644 index 0000000000..035bc8c143 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_error.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "evaluate_error", + "request": { + "tool": "browser_evaluate", + "arguments": { + "function": "() => { throw new Error('fixture failure'); }" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nError: fixture failure\n at eval (eval at (eval at evaluate (:303:30)), :1:16)\n at eval (eval at evaluate (:303:30), :4:50)\n at UtilityScript.evaluate (:305:16)\n at UtilityScript. (:1:44)" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nError: fixture failure\\n at eval (eval at (eval at evaluate (:303:30)), :1:16)\\n at eval (eval at evaluate (:303:30), :4:50)\\n at UtilityScript.evaluate (:305:16)\\n at UtilityScript. (:1:44)\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_new_tab_failed.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_new_tab_failed.json new file mode 100644 index 0000000000..d005dcb4e9 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_new_tab_failed.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "evaluate_new_tab_failed", + "request": { + "tool": "browser_evaluate", + "arguments": { + "function": "() => document.title" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nError: browserBackend.callTool: Protocol error (Target.createTarget): Failed to open a new tab" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nError: browserBackend.callTool: Protocol error (Target.createTarget): Failed to open a new tab\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_result.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_result.json new file mode 100644 index 0000000000..b08316325f --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/evaluate_result.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "evaluate_result", + "request": { + "tool": "browser_evaluate", + "arguments": { + "function": "() => ({ name: document.getElementById('name').value, size: document.getElementById('size').value })" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Result\n{\n \"name\": \"Ada\",\n \"size\": \"l\"\n}\n### Ran Playwright code\n```js\nawait page.evaluate('() => ({ name: document.getElementById(\\'name\\').value, size: document.getElementById(\\'size\\').value })');\n```" + } + ], + "isError": false + }, + "model_text": "### Result\n{\n \"name\": \"Ada\",\n \"size\": \"l\"\n}\n### Ran Playwright code\n```js\nawait page.evaluate('() => ({ name: document.getElementById(\\'name\\').value, size: document.getElementById(\\'size\\').value })');\n```" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/events_console_message.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/events_console_message.json new file mode 100644 index 0000000000..161d55300a --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/events_console_message.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "events_console_message", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Log to console button", + "target": "e17" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByRole('button', { name: 'Log to console' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)\n### Events\n- New console entries: .playwright-mcp\\console-.log#L1" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByRole('button', { name: 'Log to console' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)\n### Events\n- New console entries: .playwright-mcp\\console-.log#L1" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/events_download.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/events_download.json new file mode 100644 index 0000000000..d55aa0c096 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/events_download.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "events_download", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Download report link", + "target": "e6" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByRole('link', { name: 'Download report' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)\n### Events\n- Downloading file report.csv ...\n- Downloaded file report.csv to \".playwright-mcp\\report.csv\"" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByRole('link', { name: 'Download report' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)\n### Events\n- Downloading file report.csv ...\n- Downloaded file report.csv to \".playwright-mcp\\report.csv\"" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/file_chooser_open_by_click.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/file_chooser_open_by_click.json new file mode 100644 index 0000000000..515b8e7659 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/file_chooser_open_by_click.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "file_chooser_open_by_click", + "request": { + "tool": "browser_click", + "arguments": { + "element": "Attachment input", + "target": "f12e3" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByRole('button', { name: 'Attachment' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/upload.html\n### Modal state\n- [File chooser]: can be handled by browser_file_upload\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByRole('button', { name: 'Attachment' }).click();\n```\n### Page\n- Page URL: http://127.0.0.1:/upload.html\n### Modal state\n- [File chooser]: can be handled by browser_file_upload\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/file_upload.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/file_upload.json new file mode 100644 index 0000000000..b933470011 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/file_upload.json @@ -0,0 +1,22 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "file_upload", + "request": { + "tool": "browser_file_upload", + "arguments": { + "paths": [ + "\\attachment.csv" + ] + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait fileChooser.setFiles([\"\\\\attachment.csv\"])\n```\n### Page\n- Page URL: http://127.0.0.1:/upload.html\n- Page Title: Fixture upload page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait fileChooser.setFiles([\"\\\\attachment.csv\"])\n```\n### Page\n- Page URL: http://127.0.0.1:/upload.html\n- Page Title: Fixture upload page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/fill_form.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/fill_form.json new file mode 100644 index 0000000000..a67b136436 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/fill_form.json @@ -0,0 +1,39 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "fill_form", + "request": { + "tool": "browser_fill_form", + "arguments": { + "fields": [ + { + "target": "f7e8", + "name": "Full name", + "type": "textbox", + "value": "Ada" + }, + { + "target": "f7e11", + "name": "Gift wrap", + "type": "checkbox", + "value": "true" + }, + { + "target": "f7e9", + "name": "Size", + "type": "combobox", + "value": "Medium" + } + ] + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByRole('textbox', { name: 'Full name' }).fill('Ada');\nawait page.getByRole('checkbox', { name: 'Gift wrap' }).setChecked(true);\nawait page.getByLabel('Size').selectOption('Medium');\n```" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByRole('textbox', { name: 'Full name' }).fill('Ada');\nawait page.getByRole('checkbox', { name: 'Gift wrap' }).setChecked(true);\nawait page.getByLabel('Size').selectOption('Medium');\n```" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_back.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_back.json new file mode 100644 index 0000000000..5f347220af --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_back.json @@ -0,0 +1,18 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "navigate_back", + "request": { + "tool": "browser_navigate_back", + "arguments": {} + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.goBack();\n```\n### Open tabs\n- 0: (current) [Fixture popup page](http://127.0.0.1:/popup.html)\n- 1: [Fixture second page](http://127.0.0.1:/second.html)\n### Page\n- Page URL: http://127.0.0.1:/popup.html\n- Page Title: Fixture popup page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.goBack();\n```\n### Open tabs\n- 0: (current) [Fixture popup page](http://127.0.0.1:/popup.html)\n- 1: [Fixture second page](http://127.0.0.1:/second.html)\n### Page\n- Page URL: http://127.0.0.1:/popup.html\n- Page Title: Fixture popup page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_same_url_reload.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_same_url_reload.json new file mode 100644 index 0000000000..de576bdee1 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_same_url_reload.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "navigate_same_url_reload", + "request": { + "tool": "browser_navigate", + "arguments": { + "url": "http://127.0.0.1:/index.html" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.goto('http://127.0.0.1:/index.html');\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.goto('http://127.0.0.1:/index.html');\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_second_page.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_second_page.json new file mode 100644 index 0000000000..fde47c5e42 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/navigate_second_page.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "navigate_second_page", + "request": { + "tool": "browser_navigate", + "arguments": { + "url": "http://127.0.0.1:/second.html" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.goto('http://127.0.0.1:/second.html');\n```\n### Page\n- Page URL: http://127.0.0.1:/second.html\n- Page Title: Fixture second page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.goto('http://127.0.0.1:/second.html');\n```\n### Page\n- Page URL: http://127.0.0.1:/second.html\n- Page Title: Fixture second page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/blocked.html b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/blocked.html new file mode 100644 index 0000000000..df3675f156 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/blocked.html @@ -0,0 +1,17 @@ + + + + + + Fixture blocked page + + + +

Fixture blocked page

+ + + + + diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/frame.html b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/frame.html new file mode 100644 index 0000000000..4a5e00d58c --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/frame.html @@ -0,0 +1,12 @@ + + + + + + Embedded frame + + +

Inside the frame

+ + + diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/index.html b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/index.html new file mode 100644 index 0000000000..4b1ab287a7 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/index.html @@ -0,0 +1,37 @@ + + + + + + Fixture form page + + +

Fixture form page

+ +
+ + + + + + +
+

Not saved

+ + +

No answer

+ + + + diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/popup.html b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/popup.html new file mode 100644 index 0000000000..6edd9e62b2 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/popup.html @@ -0,0 +1,15 @@ + + + + + + Fixture popup page + + +

Fixture popup page

+ Open second page in a new tab + Go to second page + +

42

+ + diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/report.csv b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/report.csv new file mode 100644 index 0000000000..a57d296632 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/report.csv @@ -0,0 +1,2 @@ +id,name +1,fixture report diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/second.html b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/second.html new file mode 100644 index 0000000000..148b0f45f4 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/second.html @@ -0,0 +1,12 @@ + + + + + + Fixture second page + + +

Fixture second page

+ Back to form + + diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/slow.html b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/slow.html new file mode 100644 index 0000000000..5c7238ab62 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/slow.html @@ -0,0 +1,12 @@ + + + + + + Fixture slow page + + +

Fixture slow page

+ Slow link + + diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/upload.html b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/upload.html new file mode 100644 index 0000000000..a61c4a6c2f --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/pages/upload.html @@ -0,0 +1,13 @@ + + + + + + Fixture upload page + + +

Fixture upload page

+ + + + diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/record_mcp_fixtures.py b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/record_mcp_fixtures.py new file mode 100644 index 0000000000..3ed953aae4 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/record_mcp_fixtures.py @@ -0,0 +1,382 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""Record real @playwright/mcp answers as JSON fixtures; a script run from the repository root, not a test.""" + +from __future__ import annotations + +import asyncio +import functools +import json +import os +import re +import shutil +import sys +import tempfile +import threading +import time +from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any + +from openjiuwen.core.foundation.tool.mcp.base import MCPTool +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.clients import BrowserMoveStdioClient +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.settings import build_playwright_mcp_config + +SERVER = "@playwright/mcp@0.0.78" +HERE = Path(__file__).resolve().parent +PAGES_DIR = HERE / "pages" + +_CLEARED_ENV = ( + "BROWSER_DRIVER", + "PLAYWRIGHT_MCP_COMMAND", + "PLAYWRIGHT_MCP_EXTENSION", + "PLAYWRIGHT_MCP_EXTENSION_TOKEN", + "PLAYWRIGHT_MCP_CDP_ENDPOINT", + "PLAYWRIGHT_CDP_URL", + "PLAYWRIGHT_MCP_CDP_HEADERS", + "PLAYWRIGHT_CDP_HEADERS", + "PLAYWRIGHT_MCP_CDP_TIMEOUT", + "PLAYWRIGHT_CDP_TIMEOUT_MS", + "PLAYWRIGHT_MCP_BROWSER", + "PLAYWRIGHT_MCP_DEVICE", + "PLAYWRIGHT_MCP_ENV_JSON", +) + +_BASELINE_MCP_CONFIG = { + "browser": { + "contextOptions": {"locale": "en-US", "timezoneId": "America/New_York"}, + "launchOptions": { + "args": [ + "--disable-background-timer-throttling", + "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", + ] + }, + } +} + + +_HANG_S = 30 + + +class _QuietHandler(SimpleHTTPRequestHandler): + def log_message(self, format: str, *args: Any) -> None: # noqa: A002 + return + + def do_GET(self) -> None: # noqa: N802 + if self.path == "/hang": + time.sleep(_HANG_S) + self.send_response(204) + self.end_headers() + return + super().do_GET() + + +def _start_page_server() -> ThreadingHTTPServer: + handler = functools.partial(_QuietHandler, directory=str(PAGES_DIR)) + server = ThreadingHTTPServer(("127.0.0.1", 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + return server + + +def _configure_env(work_dir: Path) -> None: + """Launch args of the WebVoyager baseline, headless, with work_dir as the server cwd.""" + for key in _CLEARED_ENV: + os.environ.pop(key, None) + config_path = work_dir / "playwright-mcp.config.json" + config_path.write_text(json.dumps(_BASELINE_MCP_CONFIG), encoding="utf-8") + os.environ["PLAYWRIGHT_MCP_ARGS"] = json.dumps( + ["-y", SERVER, "--isolated", "--config", str(config_path), "--headless"] + ) + os.environ["PLAYWRIGHT_RUNTIME_MCP_CWD"] = str(work_dir) + + +_OUTPUT_FILE_TIMESTAMP = re.compile(r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z") + + +class _Placeholders: + def __init__(self, port: int, tmp_dir: Path, outside_dir: Path) -> None: + variants = { + (spelling, token) + for directory, token in ((tmp_dir, ""), (outside_dir, "")) + for path in (str(directory), str(directory.resolve())) + for spelling in (path, path.replace("\\", "/"), path.replace("\\", "\\\\")) + } + self._paths = sorted(variants, key=lambda variant: len(variant[0]), reverse=True) + self._port = re.compile(rf"(127\.0\.0\.1|localhost):{port}\b") + + def apply(self, value: Any) -> Any: + if isinstance(value, str): + for path, token in self._paths: + value = value.replace(path, token) + value = _OUTPUT_FILE_TIMESTAMP.sub("", value) + return self._port.sub(r"\1:", value) + if isinstance(value, list): + return [self.apply(item) for item in value] + if isinstance(value, dict): + return {key: self.apply(item) for key, item in value.items()} + return value + + +class _Recorder: + def __init__(self, client: BrowserMoveStdioClient, tools: dict[str, MCPTool], placeholders: _Placeholders): + self._client = client + self._tools = tools + self._placeholders = placeholders + self._raw: list[Any] = [] + self.written: list[str] = [] + original_call_tool = client._session.call_tool + + async def capturing_call_tool(*args: Any, **kwargs: Any) -> Any: + result = await original_call_tool(*args, **kwargs) + self._raw.append(result) + return result + + client._session.call_tool = capturing_call_tool + + async def call(self, tool: str, arguments: dict[str, Any]) -> tuple[Any, str]: + self._raw.clear() + mcp_tool = self._tools[tool] + output = await mcp_tool.invoke(arguments) + if len(self._raw) != 1: + raise RuntimeError(f"expected exactly one MCP call for {tool}, captured {len(self._raw)}") + return self._raw[0], mcp_tool.render_for_llm(output) + + async def record(self, case: str, tool: str, arguments: dict[str, Any]) -> str: + raw, model_text = await self.call(tool, arguments) + fixture = { + "server": SERVER, + "case": case, + "request": {"tool": tool, "arguments": arguments}, + "raw_result": raw.model_dump(mode="json", by_alias=True, exclude_none=True), + "model_text": model_text, + } + path = HERE / f"{case}.json" + text = json.dumps(self._placeholders.apply(fixture), ensure_ascii=False, indent=2) + "\n" + path.write_text(text, encoding="utf-8", newline="\n") + self.written.append(path.name) + print(f"recorded {path.name}") + return model_text + + +def _ref(snapshot_text: str, role: str, name: str | None) -> str: + named = "" if name is None else f' "{re.escape(name)}"' + match = re.search(rf"- {re.escape(role)}{named}[^\n]*\[ref=((?:f\d+)?e\d+)\]", snapshot_text) + if match is None: + raise RuntimeError(f"no {role}{named} in snapshot:\n{snapshot_text}") + return match.group(1) + + +_CLOSE_BROWSER_MID_CALL = ( + "async (page) => { const pending = page.locator('#never').click({ timeout: 3000 }); " + "await page.context().browser().close(); await pending; }" +) + + +async def _run_cases(rec: _Recorder, base_url: str, work_dir: Path, outside_dir: Path) -> None: + index_url = f"{base_url}/index.html" + await rec.call("browser_navigate", {"url": index_url}) + + snapshot = await rec.record( + "snapshot_form_page", + "browser_snapshot", + {}, + ) + await rec.record("snapshot_depth_1", "browser_snapshot", {"depth": 1}) + await rec.record("snapshot_target", "browser_snapshot", {"target": _ref(snapshot, "navigation", None)}) + await rec.record( + "click_success", + "browser_click", + {"element": "Save button", "target": _ref(snapshot, "button", "Save")}, + ) + await rec.record( + "events_console_message", + "browser_click", + {"element": "Log to console button", "target": _ref(snapshot, "button", "Log to console")}, + ) + await rec.record( + "events_download", + "browser_click", + {"element": "Download report link", "target": _ref(snapshot, "link", "Download report")}, + ) + await rec.record( + "dialog_open_by_click", + "browser_click", + {"element": "Delete order button", "target": _ref(snapshot, "button", "Delete order")}, + ) + await rec.record( + "dialog_handle_accept", + "browser_handle_dialog", + {"accept": True}, + ) + await rec.record( + "navigate_same_url_reload", + "browser_navigate", + {"url": index_url}, + ) + await rec.record( + "navigate_second_page", + "browser_navigate", + {"url": f"{base_url}/second.html"}, + ) + + await rec.record("tabs_list", "browser_tabs", {"action": "list"}) + await rec.record("tabs_new", "browser_tabs", {"action": "new"}) + await rec.record("tabs_select", "browser_tabs", {"action": "select", "index": 0}) + await rec.record("tabs_close", "browser_tabs", {"action": "close", "index": 1}) + + await rec.call("browser_navigate", {"url": f"{base_url}/blocked.html"}) + _, blocked_snapshot = await rec.call("browser_snapshot", {}) + await rec.record( + "click_intercepted_by_overlay", + "browser_click", + {"element": "Covered button", "target": _ref(blocked_snapshot, "button", "Covered button")}, + ) + await rec.record( + "click_timeout_not_enabled", + "browser_click", + {"element": "Disabled button", "target": _ref(blocked_snapshot, "button", "Disabled button")}, + ) + + await rec.call("browser_navigate", {"url": f"{base_url}/second.html"}) + await rec.record( + "click_stale_ref", + "browser_click", + {"element": "Covered button", "target": _ref(blocked_snapshot, "button", "Covered button")}, + ) + + await rec.call("browser_navigate", {"url": index_url}) + _, form = await rec.call("browser_snapshot", {}) + await rec.record("snapshot_with_boxes", "browser_snapshot", {"boxes": True}) + await rec.record( + "fill_form", + "browser_fill_form", + { + "fields": [ + {"target": _ref(form, "textbox", "Full name"), "name": "Full name", "type": "textbox", "value": "Ada"}, + { + "target": _ref(form, "checkbox", "Gift wrap"), + "name": "Gift wrap", + "type": "checkbox", + "value": "true", + }, + {"target": _ref(form, "combobox", "Size"), "name": "Size", "type": "combobox", "value": "Medium"}, + ] + }, + ) + await rec.record("snapshot_after_fill_form", "browser_snapshot", {}) + await rec.record( + "select_option", + "browser_select_option", + {"element": "Size select", "target": _ref(form, "combobox", "Size"), "values": ["Large"]}, + ) + await rec.record( + "evaluate_result", + "browser_evaluate", + { + "function": "() => ({ name: document.getElementById('name').value, " + "size: document.getElementById('size').value })" + }, + ) + await rec.record( + "evaluate_error", + "browser_evaluate", + {"function": "() => { throw new Error('fixture failure'); }"}, + ) + await rec.record( + "screenshot_element", + "browser_take_screenshot", + {"element": "Page heading", "target": _ref(form, "heading", "Fixture form page"), "type": "png"}, + ) + await rec.record("wait_for_timeout", "browser_wait_for", {"text": "Never shown"}) + await rec.call("browser_click", {"element": "Delete order button", "target": _ref(form, "button", "Delete order")}) + await rec.record( + "click_blocked_by_dialog", + "browser_click", + {"element": "Save button", "target": _ref(form, "button", "Save")}, + ) + await rec.call("browser_handle_dialog", {"accept": False}) + + await rec.call("browser_navigate", {"url": f"{base_url}/popup.html"}) + _, popup = await rec.call("browser_snapshot", {}) + await rec.record( + "click_opens_tab", + "browser_click", + {"element": "New tab link", "target": _ref(popup, "link", "Open second page in a new tab")}, + ) + await rec.record("snapshot_two_tabs", "browser_snapshot", {}) + await rec.record( + "click_navigates", + "browser_click", + {"element": "Second page link", "target": _ref(popup, "link", "Go to second page")}, + ) + await rec.record("navigate_back", "browser_navigate_back", {}) + await rec.call("browser_tabs", {"action": "close", "index": 1}) + + attachment = work_dir / "attachment.csv" + attachment.write_text("name,size\nAda,Large\n", encoding="utf-8") + await rec.call("browser_navigate", {"url": f"{base_url}/upload.html"}) + _, upload = await rec.call("browser_snapshot", {}) + await rec.record( + "file_chooser_open_by_click", + "browser_click", + {"element": "Attachment input", "target": _ref(upload, "button", "Attachment")}, + ) + await rec.record("file_upload", "browser_file_upload", {"paths": [str(attachment)]}) + + await rec.call("browser_navigate", {"url": f"{base_url}/slow.html"}) + _, slow = await rec.call("browser_snapshot", {}) + await rec.record( + "click_timeout_after_action", + "browser_click", + {"element": "Slow link", "target": _ref(slow, "link", "Slow link")}, + ) + + await rec.call("browser_navigate", {"url": index_url}) + _, form = await rec.call("browser_snapshot", {}) + await rec.call("browser_click", {"element": "Delete order button", "target": _ref(form, "button", "Delete order")}) + await rec.record("snapshot_during_dialog", "browser_snapshot", {"boxes": True}) + await rec.record("tabs_list_during_dialog", "browser_tabs", {"action": "list"}) + await rec.call("browser_handle_dialog", {"accept": False}) + + outside = outside_dir / "outside.csv" + outside.write_text("name\nAda\n", encoding="utf-8") + await rec.call("browser_navigate", {"url": f"{base_url}/upload.html"}) + _, upload = await rec.call("browser_snapshot", {}) + await rec.record( + "drop_outside_roots", + "browser_drop", + {"element": "Page heading", "target": _ref(upload, "heading", "Fixture upload page"), "paths": [str(outside)]}, + ) + + await rec.call("browser_run_code_unsafe", {"code": _CLOSE_BROWSER_MID_CALL}) + await rec.record("evaluate_new_tab_failed", "browser_evaluate", {"function": "() => document.title"}) + await rec.record("evaluate_browser_closed", "browser_evaluate", {"function": "() => document.title"}) + + +async def main() -> int: + tmp_dir = Path(tempfile.mkdtemp(prefix="mcp-fixtures-")) + outside_dir = Path(tempfile.mkdtemp(prefix="mcp-outside-")) + page_server = _start_page_server() + port = page_server.server_address[1] + _configure_env(tmp_dir) + client = BrowserMoveStdioClient(build_playwright_mcp_config()) + try: + if not await client.connect(): + print("could not start the MCP server", file=sys.stderr) + return 1 + tools = {card.name: MCPTool(client, card) for card in await client.list_tools()} + recorder = _Recorder(client, tools, _Placeholders(port, tmp_dir, outside_dir)) + await _run_cases(recorder, f"http://127.0.0.1:{port}", tmp_dir, outside_dir) + print(f"wrote {len(recorder.written)} fixtures to {HERE}") + return 0 + finally: + await client.disconnect() + page_server.shutdown() + page_server.server_close() + shutil.rmtree(tmp_dir, ignore_errors=True) + shutil.rmtree(outside_dir, ignore_errors=True) + + +if __name__ == "__main__": + sys.exit(asyncio.run(main())) diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/screenshot_element.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/screenshot_element.json new file mode 100644 index 0000000000..ce0cd8b37c --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/screenshot_element.json @@ -0,0 +1,27 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "screenshot_element", + "request": { + "tool": "browser_take_screenshot", + "arguments": { + "element": "Page heading", + "target": "f7e2", + "type": "png" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Result\n- [Screenshot of Page heading](.playwright-mcp\\element-.png)\n### Ran Playwright code\n```js\n// Screenshot Page heading and save it as .playwright-mcp\\element-.png\nawait page.getByRole('heading', { name: 'Fixture form page' }).screenshot({\n path: '.playwright-mcp\\\\element-.png',\n scale: 'css',\n type: 'png'\n});\n```" + }, + { + "type": "image", + "data": "iVBORw0KGgoAAAANSUhEUgAABPAAAAAmCAIAAADFifJ1AAAPZklEQVR4nOzdb0gbef4H8MliIYEWEuhCAhYaseCEFn6RFppwPjCyP2jEgya00AwVNG7hqitckwqr2T7oqgs1uUKrt9A6FlaSQos5OHGEKx0f7JEctJiDFUeobApbyMAuZOAKGaiQy+TvN5o0UXPbat+vR07m/zd54Hu+3/l+mtLpNAUAAAAAAABw0HxGAQAAAAAAABxACLQAAAAAAABwICHQAgAAAAAAwIGEQAsAAAAAAAAHEgItAAAAAAAAHEgItAAAAAAAAHAgIdACAAAAAADAgfQZtcio9qCXKx5CWvG2azIfadpv8BIFCnmTC/S2G3SF5tKZnFNRaYsCAAAAAACARvmM6gmm0+lU4vnY2fI1p/0/pwn/Sf78YmHson77Ad6G3LZATM78Jcfudrkfy9SnToresmpOdXvnY9KFheS7iCfTZpIQHrEab0QpAAAAAAAAaJD8kGO13jY+6nrfhke1xrOO8XAiGXZpyc/FhEAsCW8SVCPIi0zXVJw6gMSHTuu3+eBqNJu1TXrD8fwq6T7LUQAAAAAAANAYxDu0TXXtoL0Y5L82lpb1BppYSzcbqP17yw31hhqTjH9vwvQkv/0zdfEPtY4CAAAAAACAxtjLpFBmh6s08vioi+U9ZiWzqc1/fs5eUVP7JXHXGfagvowbj7/e9onRfcutzzwsaKI93LiFAgAAAAAAgMaor1uWUt6V9X1vGb+Z7Zs9a7URa7Sd/tWUn2oMWfjO1j1/qOaW0vbMJt7NUgAAAAAAANBQdffQbqzHSgv24A92qvGk6KjVNBqjAAAAAAAAAGqpN9BGn7A75miKB85sr+Zjys/kVGFVbi3XW2UXOc5epq3fldKsMNJS2IThKhyQ4SqfyBTYJK5xR1Gi3OnkzbDX1qIUGzpiGFgkOoS3pNgjbxddqLeja2fu11duZzNgUnboDhGf5W+ht3wqqPwpDJrcKY7oDOeYwLIgb9Vs3uwtb0nR+0x79gI1HQFB2YvbUXkpu6UksDe6WjT5s7RfC8cLU1Bnb9+kO5K7x2xJIaqmnWfJNLUUnXKaCq2lMbQPPBKqHUre4ALXukwnClsf0ZlsXvbfVTaX49zUQOmLqIRZLN8jVyop36yaFps3vIk5twEAAAAADrP6Au1GYHBK3PGp0fNTKhFyVXpr1uiJpdbnHNryD5kLRvsP6fS71OptZSYp+suF9WRqXRnGHPUaWwaeilRVRs+LRLDCC7rKiSI3jVX36wmmYuN02Uey9LcB4ylnYCWb77ZE9rIvPyvxb9wQrWvvD/BHveupdDq1Pt4aCw1bdRZftFGDoH8LD5zInkK0TgsppWCS4Lduhrx2k+aEM/SGKru1nbcsC4EOnXU4FMtej/xP79AjUekwTyVme8o3XAtYPzcN3OXzITaToh86W7IBWHjYZVBuX8gH9VxJoeGaJYWUsyz9qaypV2/brCPhYoSVxRjbbzL+PytsC+dbYuiyQUN3B4+P8a+S6VRi4ao2c0nCSmDArLPeFbadSbn4Yy3dIyy/IZm/jiTfpVM/jZup95CF+5mb6vbOx22hZKZVk2FHciXgPGVwPhYpAAAAAAA4pGoFWlkSln1dFm+VccBq/RXGUXFNk5ruW4jcIWNI3DcaUrJPkxwXZEcosf7AQWtzgc3iT2RiyBJZOIi+U6yDG7RnT2UxGyueyNLxvrmW1P9nJi8i+YShJ/XhX1ORrwpxW04mKWUuKuZU94zSu2scezRGZ65LTY/dGVSu7+WEtTdUI9K2etbTVW6hODxbOYWTzSYsy+0Zd1t2Kq1Wd/Cv2SYUw8wZhiNPs/2WY74OW+Taz6lfZovvMCekVG5LWwcZ20NO80x7OJFJzMl/DJYO8dLLdLSb7hhZIZvXb5d2ke772JrRT623XyCbWgjNxx1z66l36fSvkWIdY+nZgLWs4q7MXaMZ5WmFmj7XrlcaVu+45SueO3pjqOzUb9juM95oLhKrB2cmLdomSn16jP26MBOZ1rWUzP4sChlemLKahnml5S7OsBeVr1V7MTB+XrmWsMvqe0kBAAAAAMChVD3QrnmV8bIanck+we+1f5K+yQfJnsPFQe+yLC0O+k8uLFzRUx+C+FJyPxq3HFdbRgpzDqt1ukzoGnbmM2ubmzld2PpcYfqrxcHxf1H7FL3FFGKx3mYp3b7aYs1HbinkHOaqD5MVYicCwT6jutk99qd8z61Bq6m0pXHwH6vTPUp41H4x6G4rrYht0EsvZh1t2bx+xU2E4Ehsk9otejKy0EermyjquGU8F/5z93HfM1PsbRaDgUe525ZDrkHu7c7D8PyL0kL0zlCp8NFFWzFAmy2FFJ9ppVGiNtJmwDmSf97iulocL6C3duZaOD4xyqKXFgAAAADgUKoeaE/7le7F/yRW59x03XMh76B1/cVPdJBKrMtIT7azt83Uh6J3OXN5tXmQVTqQ9e4n45ZM6JovBMlz5lLMO6otdONKM6Gag3Lf623If7/4YECnI0djn6SLZ5TnA8Hq8ct+yZELbLbJBVfmCOf9030VnwtY7J3FE6jV5NfXw9i1FY+d66feFdp9iUjEnXamtBANL1e6DVkQfqPeKx5ZIRJ9U+UqUPLf+OKQgeg9X2HIMt1+prSNVlco+vsstIRECwAAAABwGNUacnxUb+6bXU/M7n1S41YP+zUxblYSDZcd+0jI+/aFtRim6Zur6XRitkdLveD5WvvJ0Uic2odnS+HSgplurbYdz1cNzrTtXCHgae3BZDod9XzIltyOpk+XFvgfV/N/6d3TDxxKJd5MV/Q3M+6TmWwvhO8F99WYJeUBuDI+ipmzAQAAAAAOo/omhTru9n9jpPbKPDpNzhwVG/VVGnf6O6HpCjcS3yASz3w3MZMuMWvxm+R+ZoaKb9YbqmKb1bKe2XiS+ohptJ8TS4JQvA36y4XEO2UCrMgwFew1qU745B5nld+T0dqppmqymApdw8LqWumU3lOlb65lpDTXVELCdMcAAAAAAIdQvWV76D+69p5oj9oD92ylRTk0NPnhusyaauWlnmAqXUli/MONk85Tf0T9sTvpDc3E0rHyt3vlePiaSfW5deipeeHVgstY9VuwjEyXfivPIsUfCvHQweyfrDi3tn7sRcVvLr10pY6QDAAAAAAAB029gZZqM+0nzukvuohES8W/87JvqI+HoZlI60SIaiBja73tZ27d+6ODD0qWyC7sZkPp7d4NttvY4nyodJna52Ycx997mGb30k/T9tzO4oT7O0FWCvlMOHM9rlqbP8p7StNcGYyl04jRGF6WBQAAAAD4hNQdaI+6xm/uOWhJoX6f9kuyLC0/cD202xG8ZbGzodSnzKVYJLPBFarxLDYy0stkpdbXAlGJ1WZ7XxGij1kiQYyVtnW05/96yw1YBrh80nS5L2lrHkh9enDpl2TkpvIIIDZq0qhUmjO+eJvNM7ea/PW55zx5BNp0rrTAzy0g0QIAAAAAfDrqDrT7IEzZxk9zCw/YmR0lfKjdUB8lk4wgvC78tbbvLtWzjDJZUZ480z+0vVKRxHlHuH29iKlnPFeLA19j8dfEKlkuHfmim/kw9Yz2T4yXCv9YHBfytyE+DrBEY9Y1anpLDF+32lac68Tg76Tw3N9n1m7fXe24QhRC/tcQ8/32SCvcHZjZfUUiAAAAAAD4+BGBdqt8zZbckIl0hIddtifMglKnZ2cJHzdXlhvJ4aOU8CMvbcni4lD3t4W8SrcTVWJi/Iqk1IG532UdFai6JStP7UT7/uouxeXXM13nBtiXotICmWtYCXRZgrZR+/5exFTb72Vr7SgE/kWpdeVYJH8DWtfSnKv6WSTpfzqZ1tYut6fiqzGiMdeWiolf+1VgsPA+bUpKELuEg08lakvi7s2Q31n8NZlCJa6fdj4UHKNjdB0trr7i958tLfLX6a5bnJBtKFkSwv0m95ZnsJUCAAAAAIDDJx9oZZH3TYbK1mz4fY8EuUbIkcXHQaIaDSX8EIoVQpcSJ66ZTNd4qsNqzHWsnWTcncTWUqj7i4lYKaSZ3SNE4F0c0B3RGHqTg18VPmx1+6+WUifXr1OpNKbhdedVsqiQwD8TySgur8XIDlzxMcuJFaK6+sJs5IGtdPRNduCcQaNSqTLX4Iq4nwXttYfKyuLTIEdeCtEaCq09GJ3NvR3KXe+e+bdyGfIGy1zP7qS3z0bLz7IlxWLkjMccW+0bkePcMpkQ+fByPmrKmxy3Rqx5FuZ+K+yzzJH7hJ+Gpd1lWqMm5mWmeKU5ZWGibyJ/ref9kb+Uhk0bLUSrUnLosk51ghEuj7mIA0WHDSpLQMid/Z8+57yUvZ6QWNczFaPn2XMiskr8t92mY8pExxqdyd/McjdpCgAAAAAADqPPqEVG+dff0DXxctsqKdO7pTmiUp0JVCkjEw+cyYS9UFnoWPO1H1MxixTXq8SJ3DxA4l2rRmUKbHLMEcPQttdTX2a2z6zKLxn/vJoIu825ANSkNV+djsTJjKe1z8Ujdxz5KXKbtHSnZ+FVfLb8tUzuukHTW8iVi4zmjK+sA/f1TLdBY5qqcE/0l88Tr5b8V836QsegWm923XueiC+4mqkaNgOmTCteLn8xONsaql4i5La5l35Jrs55bPp1r1nJyxp6KKLPvh36y5K7jdw5HjDrnI/LWpcfznwjDLf93ByjaSlvWJG165R7zNz+qSG+fE3350qDx6faW4bL1siPnTpzYFflYen+2ZkOwdOhyzxY8L2k1K2ZG1lPbquO+4fpyIP8V5bZwH0vkrlTzx/cM7yn8EXrbd8sJX/M7xWP8nL+epjsE4UyGoPBZBsILMfL2kVrmxayrdpW+CXkfhtCKnLbUvtBBAAAAAAAHEyqdDpNAdQp8/jjj8WefNr/at3T6NG8mVytcYVrbqbvWxLm7AirAAAAAACfst9jUiiA+qkvjZPvxFYjPhryr1EAAAAAAPApQ6CFj0wT7Vn5efpCze3i8V0NjwYAAAAAgEMHgRY+LuJjp+FYy9CyeYxPpN6ly6WSifWFvlw5Yr3RQAEAAAAAwKcMgRb2Tt51pZ+a4sHJsFLDp83JdOp3FK1Va/W045pDmSi6c3yojpHJAAAAAABwiCHQQt1kkVuOEstC8Gms0ZnW6LiRLfOz4evuZ2OvJeL4siQK/PcDpo6AeH4sEnbrKQAAAAAA+KRhlmOoE8eoukOVVrj+ng72UA0kizFuPsg+Ca5uimKxDlKTVt/abrvkcLvctjY1BQAAAAAAnzwEWgAAAAAAADiQMOQYAAAAAAAADqT/AgAA//8KZmtKAAAABklEQVQDADxTI5+8fEUqAAAAAElFTkSuQmCC", + "mimeType": "image/png" + } + ], + "isError": false + }, + "model_text": "### Result\n- [Screenshot of Page heading](.playwright-mcp\\element-.png)\n### Ran Playwright code\n```js\n// Screenshot Page heading and save it as .playwright-mcp\\element-.png\nawait page.getByRole('heading', { name: 'Fixture form page' }).screenshot({\n path: '.playwright-mcp\\\\element-.png',\n scale: 'css',\n type: 'png'\n});\n```\n[binary content: image/png]" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/select_option.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/select_option.json new file mode 100644 index 0000000000..92c9c728db --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/select_option.json @@ -0,0 +1,24 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "select_option", + "request": { + "tool": "browser_select_option", + "arguments": { + "element": "Size select", + "target": "f7e9", + "values": [ + "Large" + ] + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Ran Playwright code\n```js\nawait page.getByLabel('Size').selectOption(['Large']);\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" + } + ], + "isError": false + }, + "model_text": "### Ran Playwright code\n```js\nawait page.getByLabel('Size').selectOption(['Large']);\n```\n### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n- [Snapshot](.playwright-mcp\\page-.yml)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_after_fill_form.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_after_fill_form.json new file mode 100644 index 0000000000..9e5343ce1e --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_after_fill_form.json @@ -0,0 +1,18 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "snapshot_after_fill_form", + "request": { + "tool": "browser_snapshot", + "arguments": {} + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- generic [ref=f7e1]:\n - heading \"Fixture form page\" [level=1] [ref=f7e2]\n - navigation [ref=f7e3]:\n - link \"Second page\" [ref=f7e4] [cursor=pointer]:\n - /url: second.html\n - link \"Blocked page\" [ref=f7e5] [cursor=pointer]:\n - /url: blocked.html\n - link \"Download report\" [ref=f7e6] [cursor=pointer]:\n - /url: report.csv\n - generic [ref=f7e7]:\n - text: Full name\n - textbox \"Full name\" [ref=f7e8]:\n - /placeholder: Your name\n - text: Ada\n - text: Size\n - combobox \"Size\" [ref=f7e9]:\n - option \"Small\"\n - option \"Medium\" [selected]\n - option \"Large\"\n - generic [ref=f7e10]:\n - checkbox \"Gift wrap\" [checked] [active] [ref=f7e11]\n - text: Gift wrap\n - button \"Save\" [ref=f7e12]\n - paragraph [ref=f7e13]: Not saved\n - button \"Show alert\" [ref=f7e14]\n - button \"Delete order\" [ref=f7e15]\n - paragraph [ref=f7e16]: No answer\n - button \"Log to console\" [ref=f7e17]\n - iframe [ref=f7e18]:\n - generic [ref=f8e1]:\n - paragraph [ref=f8e2]: Inside the frame\n - button \"Frame button\" [ref=f8e3]\n```" + } + ], + "isError": false + }, + "model_text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- generic [ref=f7e1]:\n - heading \"Fixture form page\" [level=1] [ref=f7e2]\n - navigation [ref=f7e3]:\n - link \"Second page\" [ref=f7e4] [cursor=pointer]:\n - /url: second.html\n - link \"Blocked page\" [ref=f7e5] [cursor=pointer]:\n - /url: blocked.html\n - link \"Download report\" [ref=f7e6] [cursor=pointer]:\n - /url: report.csv\n - generic [ref=f7e7]:\n - text: Full name\n - textbox \"Full name\" [ref=f7e8]:\n - /placeholder: Your name\n - text: Ada\n - text: Size\n - combobox \"Size\" [ref=f7e9]:\n - option \"Small\"\n - option \"Medium\" [selected]\n - option \"Large\"\n - generic [ref=f7e10]:\n - checkbox \"Gift wrap\" [checked] [active] [ref=f7e11]\n - text: Gift wrap\n - button \"Save\" [ref=f7e12]\n - paragraph [ref=f7e13]: Not saved\n - button \"Show alert\" [ref=f7e14]\n - button \"Delete order\" [ref=f7e15]\n - paragraph [ref=f7e16]: No answer\n - button \"Log to console\" [ref=f7e17]\n - iframe [ref=f7e18]:\n - generic [ref=f8e1]:\n - paragraph [ref=f8e2]: Inside the frame\n - button \"Frame button\" [ref=f8e3]\n```" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_depth_1.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_depth_1.json new file mode 100644 index 0000000000..46f503be9a --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_depth_1.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "snapshot_depth_1", + "request": { + "tool": "browser_snapshot", + "arguments": { + "depth": 1 + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- generic [active] [ref=e1]:\n - heading \"Fixture form page\" [level=1] [ref=e2]\n - navigation [ref=e3]\n - generic [ref=e7]\n - paragraph [ref=e13]: Not saved\n - button \"Show alert\" [ref=e14]\n - button \"Delete order\" [ref=e15]\n - paragraph [ref=e16]: No answer\n - button \"Log to console\" [ref=e17]\n - iframe [ref=e18]:\n \n```" + } + ], + "isError": false + }, + "model_text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- generic [active] [ref=e1]:\n - heading \"Fixture form page\" [level=1] [ref=e2]\n - navigation [ref=e3]\n - generic [ref=e7]\n - paragraph [ref=e13]: Not saved\n - button \"Show alert\" [ref=e14]\n - button \"Delete order\" [ref=e15]\n - paragraph [ref=e16]: No answer\n - button \"Log to console\" [ref=e17]\n - iframe [ref=e18]:\n \n```" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_during_dialog.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_during_dialog.json new file mode 100644 index 0000000000..3c1e2ee5ae --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_during_dialog.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "snapshot_during_dialog", + "request": { + "tool": "browser_snapshot", + "arguments": { + "boxes": true + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nError: Tool \"browser_snapshot\" does not handle the modal state.\n### Modal state\n- [\"confirm\" dialog with message \"Delete order?\"]: can be handled by browser_handle_dialog" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nError: Tool \\\"browser_snapshot\\\" does not handle the modal state.\\n### Modal state\\n- [\\\"confirm\\\" dialog with message \\\"Delete order?\\\"]: can be handled by browser_handle_dialog\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_form_page.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_form_page.json new file mode 100644 index 0000000000..5283ab9a46 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_form_page.json @@ -0,0 +1,18 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "snapshot_form_page", + "request": { + "tool": "browser_snapshot", + "arguments": {} + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- generic [active] [ref=e1]:\n - heading \"Fixture form page\" [level=1] [ref=e2]\n - navigation [ref=e3]:\n - link \"Second page\" [ref=e4] [cursor=pointer]:\n - /url: second.html\n - link \"Blocked page\" [ref=e5] [cursor=pointer]:\n - /url: blocked.html\n - link \"Download report\" [ref=e6] [cursor=pointer]:\n - /url: report.csv\n - generic [ref=e7]:\n - text: Full name\n - textbox \"Full name\" [ref=e8]:\n - /placeholder: Your name\n - text: Size\n - combobox \"Size\" [ref=e9]:\n - option \"Small\" [selected]\n - option \"Medium\"\n - option \"Large\"\n - generic [ref=e10]:\n - checkbox \"Gift wrap\" [ref=e11]\n - text: Gift wrap\n - button \"Save\" [ref=e12]\n - paragraph [ref=e13]: Not saved\n - button \"Show alert\" [ref=e14]\n - button \"Delete order\" [ref=e15]\n - paragraph [ref=e16]: No answer\n - button \"Log to console\" [ref=e17]\n - iframe [ref=e18]:\n - generic [ref=f1e1]:\n - paragraph [ref=f1e2]: Inside the frame\n - button \"Frame button\" [ref=f1e3]\n```" + } + ], + "isError": false + }, + "model_text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- generic [active] [ref=e1]:\n - heading \"Fixture form page\" [level=1] [ref=e2]\n - navigation [ref=e3]:\n - link \"Second page\" [ref=e4] [cursor=pointer]:\n - /url: second.html\n - link \"Blocked page\" [ref=e5] [cursor=pointer]:\n - /url: blocked.html\n - link \"Download report\" [ref=e6] [cursor=pointer]:\n - /url: report.csv\n - generic [ref=e7]:\n - text: Full name\n - textbox \"Full name\" [ref=e8]:\n - /placeholder: Your name\n - text: Size\n - combobox \"Size\" [ref=e9]:\n - option \"Small\" [selected]\n - option \"Medium\"\n - option \"Large\"\n - generic [ref=e10]:\n - checkbox \"Gift wrap\" [ref=e11]\n - text: Gift wrap\n - button \"Save\" [ref=e12]\n - paragraph [ref=e13]: Not saved\n - button \"Show alert\" [ref=e14]\n - button \"Delete order\" [ref=e15]\n - paragraph [ref=e16]: No answer\n - button \"Log to console\" [ref=e17]\n - iframe [ref=e18]:\n - generic [ref=f1e1]:\n - paragraph [ref=f1e2]: Inside the frame\n - button \"Frame button\" [ref=f1e3]\n```" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_target.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_target.json new file mode 100644 index 0000000000..bcb4488305 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_target.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "snapshot_target", + "request": { + "tool": "browser_snapshot", + "arguments": { + "target": "e3" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- navigation [ref=e3]:\n - link \"Second page\" [ref=e4] [cursor=pointer]:\n - /url: second.html\n - link \"Blocked page\" [ref=e5] [cursor=pointer]:\n - /url: blocked.html\n - link \"Download report\" [ref=e6] [cursor=pointer]:\n - /url: report.csv\n```" + } + ], + "isError": false + }, + "model_text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- navigation [ref=e3]:\n - link \"Second page\" [ref=e4] [cursor=pointer]:\n - /url: second.html\n - link \"Blocked page\" [ref=e5] [cursor=pointer]:\n - /url: blocked.html\n - link \"Download report\" [ref=e6] [cursor=pointer]:\n - /url: report.csv\n```" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_two_tabs.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_two_tabs.json new file mode 100644 index 0000000000..dedfdac79b --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_two_tabs.json @@ -0,0 +1,18 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "snapshot_two_tabs", + "request": { + "tool": "browser_snapshot", + "arguments": {} + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Open tabs\n- 0: (current) [Fixture popup page](http://127.0.0.1:/popup.html)\n- 1: [Fixture second page](http://127.0.0.1:/second.html)\n### Page\n- Page URL: http://127.0.0.1:/popup.html\n- Page Title: Fixture popup page\n### Snapshot\n```yaml\n- generic [ref=f9e1]:\n - heading \"Fixture popup page\" [level=1] [ref=f9e2]\n - link \"Open second page in a new tab\" [active] [ref=f9e3] [cursor=pointer]:\n - /url: second.html\n - link \"Go to second page\" [ref=f9e4] [cursor=pointer]:\n - /url: second.html\n - 'button \"Note: read me\" [ref=f9e5]'\n - paragraph [ref=f9e6]: \"42\"\n```" + } + ], + "isError": false + }, + "model_text": "### Open tabs\n- 0: (current) [Fixture popup page](http://127.0.0.1:/popup.html)\n- 1: [Fixture second page](http://127.0.0.1:/second.html)\n### Page\n- Page URL: http://127.0.0.1:/popup.html\n- Page Title: Fixture popup page\n### Snapshot\n```yaml\n- generic [ref=f9e1]:\n - heading \"Fixture popup page\" [level=1] [ref=f9e2]\n - link \"Open second page in a new tab\" [active] [ref=f9e3] [cursor=pointer]:\n - /url: second.html\n - link \"Go to second page\" [ref=f9e4] [cursor=pointer]:\n - /url: second.html\n - 'button \"Note: read me\" [ref=f9e5]'\n - paragraph [ref=f9e6]: \"42\"\n```" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_with_boxes.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_with_boxes.json new file mode 100644 index 0000000000..b2972fdbe8 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/snapshot_with_boxes.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "snapshot_with_boxes", + "request": { + "tool": "browser_snapshot", + "arguments": { + "boxes": true + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- generic [active] [ref=f7e1] [box=8,21,1264,309]:\n - heading \"Fixture form page\" [level=1] [ref=f7e2] [box=8,21,1264,37]\n - navigation [ref=f7e3] [box=8,80,1264,18]:\n - link \"Second page\" [ref=f7e4] [cursor=pointer] [box=8,80,81,17]:\n - /url: second.html\n - link \"Blocked page\" [ref=f7e5] [cursor=pointer] [box=93,80,88,17]:\n - /url: blocked.html\n - link \"Download report\" [ref=f7e6] [cursor=pointer] [box=185,80,109,17]:\n - /url: report.csv\n - generic [ref=f7e7] [box=8,98,1264,22]:\n - text: Full name\n - textbox \"Full name\" [ref=f7e8] [box=76,99,177,21]:\n - /placeholder: Your name\n - text: Size\n - combobox \"Size\" [ref=f7e9] [box=289,100,70,19]:\n - option \"Small\" [selected] [box=0,0,0,0]\n - option \"Medium\" [box=0,0,0,0]\n - option \"Large\" [box=0,0,0,0]\n - generic [ref=f7e10] [box=363,100,86,17]:\n - checkbox \"Gift wrap\" [ref=f7e11] [box=367,101,13,13]\n - text: Gift wrap\n - button \"Save\" [ref=f7e12] [box=453,99,46,21]\n - paragraph [ref=f7e13] [box=8,136,1264,18]: Not saved\n - button \"Show alert\" [ref=f7e14] [box=8,170,79,21]\n - button \"Delete order\" [ref=f7e15] [box=91,170,89,21]\n - paragraph [ref=f7e16] [box=8,207,1264,18]: No answer\n - button \"Log to console\" [ref=f7e17] [box=8,310,103,21]\n - iframe [ref=f7e18] [box=115,241,324,84]:\n - generic [ref=f8e1] [box=8,16,304,55]:\n - paragraph [ref=f8e2] [box=8,16,304,18]: Inside the frame\n - button \"Frame button\" [ref=f8e3] [box=8,50,95,21]\n```" + } + ], + "isError": false + }, + "model_text": "### Page\n- Page URL: http://127.0.0.1:/index.html\n- Page Title: Fixture form page\n### Snapshot\n```yaml\n- generic [active] [ref=f7e1] [box=8,21,1264,309]:\n - heading \"Fixture form page\" [level=1] [ref=f7e2] [box=8,21,1264,37]\n - navigation [ref=f7e3] [box=8,80,1264,18]:\n - link \"Second page\" [ref=f7e4] [cursor=pointer] [box=8,80,81,17]:\n - /url: second.html\n - link \"Blocked page\" [ref=f7e5] [cursor=pointer] [box=93,80,88,17]:\n - /url: blocked.html\n - link \"Download report\" [ref=f7e6] [cursor=pointer] [box=185,80,109,17]:\n - /url: report.csv\n - generic [ref=f7e7] [box=8,98,1264,22]:\n - text: Full name\n - textbox \"Full name\" [ref=f7e8] [box=76,99,177,21]:\n - /placeholder: Your name\n - text: Size\n - combobox \"Size\" [ref=f7e9] [box=289,100,70,19]:\n - option \"Small\" [selected] [box=0,0,0,0]\n - option \"Medium\" [box=0,0,0,0]\n - option \"Large\" [box=0,0,0,0]\n - generic [ref=f7e10] [box=363,100,86,17]:\n - checkbox \"Gift wrap\" [ref=f7e11] [box=367,101,13,13]\n - text: Gift wrap\n - button \"Save\" [ref=f7e12] [box=453,99,46,21]\n - paragraph [ref=f7e13] [box=8,136,1264,18]: Not saved\n - button \"Show alert\" [ref=f7e14] [box=8,170,79,21]\n - button \"Delete order\" [ref=f7e15] [box=91,170,89,21]\n - paragraph [ref=f7e16] [box=8,207,1264,18]: No answer\n - button \"Log to console\" [ref=f7e17] [box=8,310,103,21]\n - iframe [ref=f7e18] [box=115,241,324,84]:\n - generic [ref=f8e1] [box=8,16,304,55]:\n - paragraph [ref=f8e2] [box=8,16,304,18]: Inside the frame\n - button \"Frame button\" [ref=f8e3] [box=8,50,95,21]\n```" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_close.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_close.json new file mode 100644 index 0000000000..eb385c64a8 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_close.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "tabs_close", + "request": { + "tool": "browser_tabs", + "arguments": { + "action": "close", + "index": 1 + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Result\n- 0: (current) [Fixture second page](http://127.0.0.1:/second.html)" + } + ], + "isError": false + }, + "model_text": "### Result\n- 0: (current) [Fixture second page](http://127.0.0.1:/second.html)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_list.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_list.json new file mode 100644 index 0000000000..9b5aa3e9e9 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_list.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "tabs_list", + "request": { + "tool": "browser_tabs", + "arguments": { + "action": "list" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Result\n- 0: (current) [Fixture second page](http://127.0.0.1:/second.html)" + } + ], + "isError": false + }, + "model_text": "### Result\n- 0: (current) [Fixture second page](http://127.0.0.1:/second.html)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_list_during_dialog.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_list_during_dialog.json new file mode 100644 index 0000000000..9caaee2033 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_list_during_dialog.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "tabs_list_during_dialog", + "request": { + "tool": "browser_tabs", + "arguments": { + "action": "list" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Result\n- 0: (current) [](http://127.0.0.1:/index.html)\n### Modal state\n- [\"confirm\" dialog with message \"Delete order?\"]: can be handled by browser_handle_dialog" + } + ], + "isError": false + }, + "model_text": "### Result\n- 0: (current) [](http://127.0.0.1:/index.html)\n### Modal state\n- [\"confirm\" dialog with message \"Delete order?\"]: can be handled by browser_handle_dialog" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_new.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_new.json new file mode 100644 index 0000000000..a975944c70 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_new.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "tabs_new", + "request": { + "tool": "browser_tabs", + "arguments": { + "action": "new" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Result\n- 0: [Fixture second page](http://127.0.0.1:/second.html)\n- 1: (current) [](about:blank)" + } + ], + "isError": false + }, + "model_text": "### Result\n- 0: [Fixture second page](http://127.0.0.1:/second.html)\n- 1: (current) [](about:blank)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_select.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_select.json new file mode 100644 index 0000000000..93e86dfc98 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/tabs_select.json @@ -0,0 +1,21 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "tabs_select", + "request": { + "tool": "browser_tabs", + "arguments": { + "action": "select", + "index": 0 + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Result\n- 0: (current) [Fixture second page](http://127.0.0.1:/second.html)\n- 1: [](about:blank)" + } + ], + "isError": false + }, + "model_text": "### Result\n- 0: (current) [Fixture second page](http://127.0.0.1:/second.html)\n- 1: [](about:blank)" +} diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/test_mcp_fixtures_wellformed.py b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/test_mcp_fixtures_wellformed.py new file mode 100644 index 0000000000..099c05e9bb --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/test_mcp_fixtures_wellformed.py @@ -0,0 +1,108 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""The recorded @playwright/mcp fixtures are complete and carry the markers the MCP driver parses.""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any + +import pytest + +FIXTURES_DIR = Path(__file__).resolve().parent +SERVER = "@playwright/mcp@0.0.78" +KEYS = {"server", "case", "request", "raw_result", "model_text"} +CASES = { + "snapshot_form_page", + "snapshot_depth_1", + "snapshot_target", + "click_success", + "click_intercepted_by_overlay", + "navigate_second_page", + "tabs_list", + "tabs_new", + "tabs_select", + "tabs_close", + "events_console_message", + "events_download", + "dialog_open_by_click", + "dialog_handle_accept", + "navigate_same_url_reload", + "click_timeout_not_enabled", + "click_stale_ref", + "snapshot_with_boxes", + "fill_form", + "snapshot_after_fill_form", + "select_option", + "evaluate_result", + "evaluate_error", + "screenshot_element", + "wait_for_timeout", + "click_blocked_by_dialog", + "click_opens_tab", + "snapshot_two_tabs", + "click_navigates", + "navigate_back", + "file_chooser_open_by_click", + "file_upload", + "click_timeout_after_action", + "snapshot_during_dialog", + "tabs_list_during_dialog", + "drop_outside_roots", + "evaluate_new_tab_failed", + "evaluate_browser_closed", +} + + +def _load(case: str) -> dict[str, Any]: + return json.loads((FIXTURES_DIR / f"{case}.json").read_text(encoding="utf-8")) + + +def _raw_text(fixture: dict[str, Any]) -> str: + return "\n".join(block.get("text", "") for block in fixture["raw_result"]["content"]) + + +def test_every_case_is_recorded() -> None: + assert {path.stem for path in FIXTURES_DIR.glob("*.json")} == CASES + + +@pytest.mark.parametrize("case", sorted(CASES)) +def test_fixture_is_wellformed(case: str) -> None: + fixture = _load(case) + + assert set(fixture) == KEYS + assert fixture["server"] == SERVER + assert fixture["case"] == case + assert set(fixture["request"]) == {"tool", "arguments"} + assert fixture["request"]["tool"].startswith("browser_") + assert isinstance(fixture["request"]["arguments"], dict) + assert fixture["raw_result"]["content"], "the server returned no content blocks" + assert isinstance(fixture["raw_result"]["isError"], bool) + assert isinstance(fixture["model_text"], str) and fixture["model_text"] + + +@pytest.mark.parametrize("case", sorted(CASES)) +def test_run_specific_values_are_placeholders(case: str) -> None: + text = (FIXTURES_DIR / f"{case}.json").read_text(encoding="utf-8") + + assert not re.search(r"(127\.0\.0\.1|localhost):\d", text), "local port not replaced by " + assert "mcp-fixtures-" not in text, "temp path not replaced by " + assert not re.search(r"\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z", text), "output-file time not replaced by " + + +def test_intercepted_click_carries_the_retry_log() -> None: + fixture = _load("click_intercepted_by_overlay") + + assert fixture["raw_result"]["isError"] is True + assert "intercepts pointer events" in _raw_text(fixture) + assert "intercepts pointer events" in fixture["model_text"] + + +@pytest.mark.parametrize("case", ["events_console_message", "events_download"]) +def test_events_section_is_present(case: str) -> None: + fixture = _load(case) + + assert "### Events" in _raw_text(fixture) + assert "### Events" in fixture["model_text"] diff --git a/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/wait_for_timeout.json b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/wait_for_timeout.json new file mode 100644 index 0000000000..7d288469fd --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/fixtures/mcp/wait_for_timeout.json @@ -0,0 +1,20 @@ +{ + "server": "@playwright/mcp@0.0.78", + "case": "wait_for_timeout", + "request": { + "tool": "browser_wait_for", + "arguments": { + "text": "Never shown" + } + }, + "raw_result": { + "content": [ + { + "type": "text", + "text": "### Error\nTimeoutError: Timeout 5000ms exceeded.\nCall log:\n\u001b[2m - waiting for getByText('Never shown').first() to be visible\u001b[22m\n" + } + ], + "isError": true + }, + "model_text": "{\"result\": \"### Error\\nTimeoutError: Timeout 5000ms exceeded.\\nCall log:\\n\\u001b[2m - waiting for getByText('Never shown').first() to be visible\\u001b[22m\\n\"}" +} diff --git a/tests/unit_tests/harness/tools/browser_move/runtime_fixtures.py b/tests/unit_tests/harness/tools/browser_move/runtime_fixtures.py new file mode 100644 index 0000000000..1e77fe2129 --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/runtime_fixtures.py @@ -0,0 +1,197 @@ +# coding: utf-8 +# Copyright (c) Huawei Technologies Co., Ltd. 2026. All rights reserved. +"""A ``BrowserAgentRuntime`` over ``FakeDriver`` with the Playwright MCP tool dialect, for the rail and facade tests.""" + +from __future__ import annotations + +import asyncio +from concurrent.futures import ThreadPoolExecutor +from pathlib import Path +from typing import Any, Iterable +from unittest.mock import AsyncMock, patch + +from openjiuwen.core.foundation.tool import McpServerConfig +from openjiuwen.harness.tools.browser_move.contract import ( + Action, + ActionResult, + ContextRef, + DriverInfo, + DriverRef, + Effect, + Element, + NativeTool, + Observation, + Screenshot, +) +from openjiuwen.harness.tools.browser_move.contract.testing import FakeDriver +from openjiuwen.harness.tools.browser_move.contract.testing.fake import DEFAULT_ACTIONS +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.clients.stdio_client import sanitize_json_schema +from openjiuwen.harness.tools.browser_move.drivers.playwright_mcp.tools import DECODERS +from openjiuwen.harness.tools.browser_move.provisioning import ProvisionedDevice +from openjiuwen.harness.tools.browser_move.runtime import assembly as assembly_module +from openjiuwen.harness.tools.browser_move.runtime.assembly import DriverAssembly +from openjiuwen.harness.tools.browser_move.runtime.config import CoreSettings, RunGuardrails +from openjiuwen.harness.tools.browser_move.runtime.session import ToolKind +from openjiuwen.harness.tools.browser_move.surfaces.browser.commands import ( + Back, + Drop, + FillForm, + GetConsole, + GetNetwork, + GetPdf, + Navigate, + OpenContext, + SelectOption, + UploadFile, +) +from openjiuwen.harness.tools.browser_move.surfaces.browser.context import BrowserContext +from openjiuwen.harness.tools.browser_move.surfaces.browser.profile import create_browser_surface +from openjiuwen.harness.tools.browser_move.surfaces.browser.runner.facade import BrowserAgentRuntime +from tests.unit_tests.harness.tools.browser_move.snapshots import builder + +MCP_PREFIX = "mcp_playwright-official_" +DEVICE = ProvisionedDevice(kind="chrome", endpoint="http://127.0.0.1:9222", extras={}) +HOME = ContextRef("0", "tab", "Home") +HOME_URL = "https://example.com/" +_REFS = next(iter(DEFAULT_ACTIONS.values())) +_SURFACE_COMMANDS = ( + Navigate, + Back, + OpenContext, + SelectOption, + UploadFile, + FillForm, + Drop, + GetPdf, + GetConsole, + GetNetwork, +) + + +def mcp_driver_info() -> DriverInfo: + """The Playwright MCP driver's view of itself over the 0.0.78 tool list, served by a fake.""" + native = { + tool["name"]: NativeTool( + tool["description"], sanitize_json_schema(tool["inputSchema"]), DECODERS.get(tool["name"]) + ) + for tool in builder.load_mcp_tools() + if tool["name"] in DECODERS or tool["name"] == "browser_find" + } + actions = {**DEFAULT_ACTIONS, **{command: _REFS for command in _SURFACE_COMMANDS}} + return DriverInfo( + name="fake", + surfaces=frozenset({"browser"}), + actions=actions, + native_tools=native, + tool_name_prefix=MCP_PREFIX, + ) + + +def mcp_fake_driver(observations: Iterable[Observation] = (), results: Iterable[ActionResult] = ()) -> FakeDriver: + return FakeDriver(observations, results, info=mcp_driver_info()) + + +def surface(url: str = HOME_URL, title: str = "Home") -> BrowserContext: + return BrowserContext(url=url, title=title, tabs=(HOME,), tab_urls=(url,), current_tab=0) + + +def element(ref: str, role: str = "button", name: str = "Save") -> Element: + return Element(ref=DriverRef(ref, ""), role=role, name=name) + + +def observation( + elements: Iterable[Element] = (), + *, + text: str | None = None, + url: str = HOME_URL, + title: str = "Home", + screenshot: Screenshot | None = None, +) -> Observation: + elements = tuple(elements) + native_text = text if text is not None else "\n".join(f'- {e.role} "{e.name}" [ref={e.ref.ref}]' for e in elements) + return Observation( + context=HOME, elements=elements, native_text=native_text, surface=surface(url, title), screenshot=screenshot + ) + + +def result( + *, ok: bool = True, url: str = HOME_URL, title: str = "Home", context_changed: bool = False, value: Any = None +) -> ActionResult: + return ActionResult(ok=ok, value=value, context_changed=context_changed, surface=surface(url, title)) + + +def run(coro: Any) -> Any: + """Run ``coro`` to completion, on a worker thread when the caller already runs a loop (async tests).""" + try: + asyncio.get_running_loop() + except RuntimeError: + return asyncio.run(coro) + with ThreadPoolExecutor(max_workers=1) as pool: + return pool.submit(asyncio.run, coro).result() + + +class FakeProvisioner: + """Hands out ``DEVICE`` and records releases; the device is always alive.""" + + def __init__(self) -> None: + self.released: list[ProvisionedDevice] = [] + + def provision(self, surface_settings: Any) -> ProvisionedDevice: + return DEVICE + + def release(self, device: ProvisionedDevice) -> None: + self.released.append(device) + + def is_alive(self, device: ProvisionedDevice) -> bool: + return True + + +def fake_assembly(driver: FakeDriver) -> DriverAssembly: + bundle = create_browser_surface({}) + return DriverAssembly(driver, DEVICE, FakeProvisioner(), surface=bundle) + + +def make_runtime( + driver: FakeDriver | None = None, + *, + allowed_tool_names: Iterable[str] | None = None, + image_support: bool | None = False, + language: str = "en", + workspace: Any = None, + core_settings: CoreSettings | None = None, +) -> BrowserAgentRuntime: + """A started runtime whose driver session is ``driver`` (a FakeDriver with the MCP dialect by default).""" + runtime = BrowserAgentRuntime( + provider="openai", + api_key="test-key", + api_base="https://example.invalid/v1", + model_name="test-model", + mcp_cfg=McpServerConfig( + server_id="playwright_official_stdio", + server_name="playwright-official", + server_path="stdio://playwright", + client_type="stdio", + params={"cwd": str(Path.cwd())}, + ), + guardrails=RunGuardrails(max_steps=3, max_failures=1, timeout_s=30, retry_once=False), + allowed_tool_names=allowed_tool_names, + language=language, + workspace=workspace, + core_settings=core_settings, + ) + driver = driver if driver is not None else mcp_fake_driver() + with patch.object(assembly_module, "assemble_driver", AsyncMock(return_value=fake_assembly(driver))): + run(runtime.ensure_runtime_ready()) + if image_support is not None: + runtime.agent_parts(image_support=image_support, language=language) + return runtime + + +def command_kind(action: Action) -> ToolKind: + return ToolKind(action=action, effect=action.effect, progress=True, known=True) + + +PROBE = ToolKind(effect=Effect.READ, progress=True, known=True, helper=True) +BATCH = ToolKind(effect=Effect.ACT, progress=True, known=True, helper=True) +RECALL = ToolKind(effect=Effect.READ, exempt_from_budget=True, progress=True, known=True, helper=True) +UNKNOWN = ToolKind() diff --git a/tests/unit_tests/harness/tools/browser_move/snapshots/__init__.py b/tests/unit_tests/harness/tools/browser_move/snapshots/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit_tests/harness/tools/browser_move/snapshots/builder.py b/tests/unit_tests/harness/tools/browser_move/snapshots/builder.py new file mode 100644 index 0000000000..864e56d9df --- /dev/null +++ b/tests/unit_tests/harness/tools/browser_move/snapshots/builder.py @@ -0,0 +1,511 @@ +# coding: utf-8 +"""Tool/prompt snapshot of the agent on either driver; the LLM client, the driver transport and Chrome are stubbed.""" + +from __future__ import annotations + +import asyncio +import contextlib +import json +import os +import re +import sys +from pathlib import Path +from typing import Any, Iterator +from unittest.mock import patch + +HERE = Path(__file__).resolve().parent +GOLDEN_DIR = HERE / "golden" / "playwright_mcp" +BROWSER_USE_GOLDEN_DIR = HERE / "golden" / "browser_use" +MCP_TOOLS_FIXTURE = HERE / "fixtures" / "playwright-mcp-0.0.78-tools.all-caps.json" +REGENERATE_COMMAND = "uv run python -m tests.unit_tests.harness.tools.browser_move.snapshots.regenerate" + +DRIVER = "playwright_mcp" +BROWSER_USE = "browser_use" +RUNNER_VIEW = "runner" +RUNNER_DIRECT_VIEW = "runner_direct" +RUNNER_WORKER_VIEW = "runner_worker" +RUNNER_VIEWS: dict[str, str | None] = { + RUNNER_VIEW: None, + RUNNER_DIRECT_VIEW: "direct", + RUNNER_WORKER_VIEW: "delegated_worker", +} +ROLES = {"direct": "main", "delegated_worker": "worker"} + +MODEL_NAME = "anthropic/claude-sonnet-5.5" +API_BASE = "https://openrouter.ai/api/v1" +STUB_API_KEY = "stub-key-not-used" +LANGUAGE = "en" +TASK_QUERY = ( + "Go to https://example.com and complete this task:\n\n" + "Report the main heading of the page.\n\n" + "When finished, provide your final answer as text." +) +STUB_ANSWER = "The main heading of the page is Example Domain." +MAX_STUB_CALLS = 8 + +RUNNER_HELPERS = ( + "browser_probe_interactives", + "browser_probe_cards", + "browser_batch_interact", + "browser_custom_action", + "browser_list_custom_actions", +) + +CLEARED_ENV_PREFIXES = ( + "BROWSER_", + "PLAYWRIGHT_", + "MODEL_", + "OPENAI_", + "OPENROUTER_", + "LLM_", + "OPENJIUWEN_", + "ANTHROPIC_", + "DASHSCOPE_", + "SILICONFLOW_", + "JUDGE_", + "WV_", + "API_", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "AGENT_CORE_DIR", +) + +_BLANK_SNAPSHOT = "### Page\n- Page URL: about:blank\n- Page Title: \n### Snapshot\n```yaml\n\n```" +_BLANK_TABS = "### Open tabs\n- 0: (current) [] (about:blank)" +_BLANK_WIRE_OBSERVATION = { + "url": "about:blank", + "title": "", + "tabs": [{"target_id": "T1", "url": "about:blank", "title": "", "active": True}], + "elements": [], + "ax_text": "", + "screenshot_b64": None, + "pixels_above": 0, + "pixels_below": 0, + "errors": [], + "is_pdf_viewer": False, + "captured_at": 0.0, + "driver_generation": 0, +} +_CONNECT_REPLY = { + "backend": "browser_use", + "browser_version": "Chrome/140", + "protocol_version": "1.3", + "backend_version": "0.13.10", + "initial_url": "about:blank", +} + + +def view_env(workdir: Path) -> dict[str, str]: + """Variables set for every view (the runner view reads them through build_runtime_settings).""" + return { + "PYTHONHASHSEED": "0", + "BROWSER_RUNTIME_WORKDIR": str(workdir), + "BROWSER_TIMEOUT_S": "600", + "PLAYWRIGHT_MCP_COMMAND": sys.executable, + "MODEL_PROVIDER": "openrouter", + "MODEL_NAME": MODEL_NAME, + "OPENROUTER_API_KEY": STUB_API_KEY, + "OPENAI_API_KEY": STUB_API_KEY, + "OPENAI_BASE_URL": API_BASE, + } + + +@contextlib.contextmanager +def isolated_env(workdir: Path, extra: dict[str, str] | None = None) -> Iterator[None]: + """Clear machine settings, set the view variables and cwd; restore everything after.""" + saved_env = dict(os.environ) + saved_cwd = os.getcwd() + try: + for key in list(os.environ): + if any(key.upper().startswith(p) for p in CLEARED_ENV_PREFIXES): + del os.environ[key] + os.environ.update(view_env(workdir)) + os.environ.update(extra or {}) + os.chdir(workdir) + yield + finally: + os.chdir(saved_cwd) + os.environ.clear() + os.environ.update(saved_env) + + +def load_mcp_tools() -> list[dict[str, Any]]: + return json.loads(MCP_TOOLS_FIXTURE.read_text(encoding="utf-8")) + + +class FakeMcpSession: + """Stands in for ``mcp.ClientSession`` over the Playwright MCP stdio pipe.""" + + tools: list[dict[str, Any]] = [] + calls: list[str] = [] + replies: dict[str, str] = {} + + def __init__(self, *args: Any, **kwargs: Any) -> None: + del args, kwargs + + async def __aenter__(self) -> "FakeMcpSession": + return self + + async def __aexit__(self, *exc: Any) -> None: + return None + + async def initialize(self) -> Any: + self.calls.append("initialize") + return None + + async def list_tools(self) -> Any: + from mcp import types + + self.calls.append("tools/list") + return types.ListToolsResult(tools=[types.Tool.model_validate(t) for t in self.tools]) + + async def call_tool(self, name: str, arguments: dict[str, Any] | None = None, **kwargs: Any) -> Any: + from mcp import types + + del arguments, kwargs + self.calls.append(name) + text = self.replies.get(name, "### Result\nok") + return types.CallToolResult(content=[types.TextContent(type="text", text=text)]) + + +class FakeSidecarTransport: + """Stands in for the browser-use sidecar process: a blank page that answers every request.""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + del args, kwargs + + async def start(self, timeout_s: float | None = None) -> dict[str, Any]: + del timeout_s + return {"kind": "hello"} + + async def request(self, method: str, params: Any = None, timeout_s: float | None = None) -> Any: + del params, timeout_s + if method == "connect": + return dict(_CONNECT_REPLY) + if method == "observe": + return dict(_BLANK_WIRE_OBSERVATION) + if method == "health": + return {"connected": True} + if method == "evaluate": + return None + return {} + + async def close(self) -> None: + return None + + +@contextlib.asynccontextmanager +async def _fake_stdio_client(*args: Any, **kwargs: Any): + del args, kwargs + yield None, None + + +def _plain(value: Any) -> Any: + if hasattr(value, "model_dump"): + return _plain(value.model_dump(mode="json")) + if isinstance(value, dict): + return {str(k): _plain(v) for k, v in value.items()} + if isinstance(value, (list, tuple)): + return [_plain(v) for v in value] + if isinstance(value, (str, int, float, bool)) or value is None: + return value + return repr(value) + + +def _tool_dict(tool: Any) -> dict[str, Any]: + if isinstance(tool, dict): + fn = tool.get("function", tool) + name, description, parameters = fn.get("name"), fn.get("description"), fn.get("parameters") + else: + name, description, parameters = tool.name, tool.description, tool.parameters + if isinstance(parameters, type) and hasattr(parameters, "model_json_schema"): + parameters = parameters.model_json_schema() + return {"name": name, "description": description, "parameters": _plain(parameters)} + + +def _make_capturing_client_factory(captured: list[dict[str, Any]], answer: str): + from openjiuwen.core.foundation.llm import AssistantMessage + from openjiuwen.core.foundation.llm.model_clients.base_model_client import BaseModelClient + from openjiuwen.core.foundation.llm.schema.message_chunk import AssistantMessageChunk + + def _is_probe(messages: Any) -> bool: + return "image_url" in json.dumps(_plain(messages), ensure_ascii=False) + + class CapturingClient(BaseModelClient): + __client_name__ = "snapshot-capture" + + def _record(self, messages: Any, tools: Any) -> AssistantMessage: + if _is_probe(messages): + raise RuntimeError("snapshot stub: image probe refused (no network)") + if len(captured) >= MAX_STUB_CALLS: + raise RuntimeError("snapshot stub: call budget exhausted") + captured.append( + { + "client_provider": str(getattr(self.model_client_config, "client_provider", "")), + "messages": _plain(messages), + "tools": [_tool_dict(t) for t in (tools or [])], + } + ) + return AssistantMessage(content=answer) + + async def invoke(self, messages, *, tools=None, **kwargs): + return self._record(messages, tools) + + async def stream(self, messages, *, tools=None, **kwargs): + msg = self._record(messages, tools) + yield AssistantMessageChunk(content=msg.content) + + async def generate_image(self, *args, **kwargs): + raise RuntimeError("snapshot stub: generate_image not supported") + + async def generate_speech(self, *args, **kwargs): + raise RuntimeError("snapshot stub: generate_speech not supported") + + async def generate_video(self, *args, **kwargs): + raise RuntimeError("snapshot stub: generate_video not supported") + + def fake_create_model_client(client_config, model_config): + return CapturingClient(model_config=model_config, model_client_config=client_config) + + return fake_create_model_client + + +def _refuse_chrome_start(self, *args: Any, **kwargs: Any) -> str: + raise AssertionError("snapshot stub: Chrome must not be started") + + +@contextlib.contextmanager +def io_stubs( + captured: list[dict[str, Any]], + mcp_calls: list[str], + answer: str = STUB_ANSWER, + mcp_replies: dict[str, str] | None = None, + driver: str = DRIVER, +) -> Iterator[None]: + """Stub the LLM client, the driver transport and Chrome start; restore them after.""" + import mcp + import mcp.client.stdio + + from openjiuwen.core.foundation.llm import model as model_mod + from openjiuwen.harness.tools.browser_move.drivers.browser_use import driver as browser_use_driver + from openjiuwen.harness.tools.browser_move.provisioning.managed_browser import ManagedBrowserProvisioner + + session_cls = type( + "FakeMcpSession", + (FakeMcpSession,), + { + "tools": load_mcp_tools(), + "calls": mcp_calls, + "replies": {"browser_snapshot": _BLANK_SNAPSHOT, "browser_tabs": _BLANK_TABS, **(mcp_replies or {})}, + }, + ) + with contextlib.ExitStack() as stack: + stack.enter_context( + patch.object(model_mod, "create_model_client", _make_capturing_client_factory(captured, answer)) + ) + stack.enter_context(patch.object(mcp, "ClientSession", session_cls)) + stack.enter_context(patch.object(mcp.client.stdio, "stdio_client", _fake_stdio_client)) + if driver == BROWSER_USE: + stack.enter_context(patch.object(browser_use_driver, "SidecarTransport", FakeSidecarTransport)) + stack.enter_context(patch.object(ManagedBrowserProvisioner, "start", _refuse_chrome_start)) + yield + + +@contextlib.contextmanager +def fresh_checkpointer() -> Iterator[None]: + """A fresh checkpointer, so a second build does not restore the first build's finished task.""" + from openjiuwen.core.session.checkpointer import checkpointer as checkpointer_mod + from openjiuwen.core.session.checkpointer.inmemory import InMemoryCheckpointer + + factory = checkpointer_mod.CheckpointerFactory + with contextlib.ExitStack() as stack: + stack.enter_context(patch.object(checkpointer_mod, "default_inmemory_checkpointer", InMemoryCheckpointer())) + stack.enter_context(patch.object(factory, "_default_checkpointer", None)) + stack.enter_context(patch.object(factory, "_type_checkpointers", {})) + yield + + +async def reset_process_state() -> None: + """Drop process-global state a previous view may have left behind.""" + from openjiuwen.core.runner import Runner + from openjiuwen.harness.image_modality_probe import reset_image_support_cache + + reset_image_support_cache() + for server_id in ("playwright_official_stdio",): + with contextlib.suppress(Exception): + await Runner.resource_mgr.remove_mcp_server(server_id=server_id, ignore_exception=True) + with contextlib.suppress(Exception): + await Runner.stop() + + +def _role(message: dict[str, Any]) -> str: + return str(message.get("role") or "") + + +def _content_text(message: dict[str, Any]) -> str: + content = message.get("content") + if isinstance(content, str): + return content + return json.dumps(content, ensure_ascii=False, sort_keys=True) + + +def agent_entry(role: str, call: dict[str, Any]) -> dict[str, Any]: + system = [_content_text(m) for m in call["messages"] if _role(m) == "system"] + return { + "role": role, + "tools": sorted(call["tools"], key=lambda t: t["name"]), + "system_prompt": "\n\n".join(system), + } + + +def stabilize(doc: Any, workdir: Path) -> Any: + """Replace run-dependent values with the placeholders of the browser-use snapshot.""" + home = Path.home() + literal = [ + (str(workdir), ""), + (workdir.as_posix(), ""), + (str(home), ""), + (home.as_posix(), ""), + ] + patterns = [ + (re.compile(r"\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}(:\d{2}(\.\d+)?)?([+-]\d{2}:?\d{2}|Z)?"), ""), + (re.compile(r"\b\d{4}-\d{2}-\d{2}\b"), ""), + (re.compile(r"\b\d{1,2}:\d{2}:\d{2}\b"), "