diff --git a/apps/desktop/src/main/app-shell-guard.test.ts b/apps/desktop/src/main/app-shell-guard.test.ts new file mode 100644 index 00000000..279027ae --- /dev/null +++ b/apps/desktop/src/main/app-shell-guard.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from 'vitest' +import type { WebContents } from 'electron' + +const { openExternal } = vi.hoisted(() => ({ openExternal: vi.fn() })) + +vi.mock('electron', () => ({ + shell: { openExternal }, +})) + +import { attachAppShellNavigationGuard } from './app-shell-guard' + +function fakeContents(): { + contents: WebContents + fire: (event: string, url: string) => boolean +} { + const listeners = new Map void>>() + const contents = { + setWindowOpenHandler: vi.fn(), + on: (event: string, listener: (event: { preventDefault(): void }, url: string) => void) => { + const list = listeners.get(event) ?? [] + list.push(listener) + listeners.set(event, list) + return contents + }, + } as unknown as WebContents + return { + contents, + fire: (event, url) => { + let prevented = false + const navEvent = { preventDefault: () => { prevented = true } } + for (const listener of listeners.get(event) ?? []) listener(navEvent, url) + return prevented + }, + } +} + +describe('attachAppShellNavigationGuard', () => { + it('sends YouTube to the OS browser instead of replacing the ADE', () => { + const { contents, fire } = fakeContents() + attachAppShellNavigationGuard(contents) + expect(fire('will-navigate', 'https://youtube.com/')).toBe(true) + expect(openExternal).toHaveBeenCalledWith('https://youtube.com/') + }) +}) diff --git a/apps/desktop/src/main/app-shell-guard.ts b/apps/desktop/src/main/app-shell-guard.ts new file mode 100644 index 00000000..3206f84f --- /dev/null +++ b/apps/desktop/src/main/app-shell-guard.ts @@ -0,0 +1,27 @@ +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { shell, type WebContents } from 'electron' +import { isAppUrl, isExternalOpenable } from './external-links' + +/** + * Pins the ADE renderer so transcript links cannot replace the app. Must be + * attached only to a BrowserWindow's own webContents — a WebContentsView + * guest (the in-app browser) has to navigate http(s) itself. + */ +export function attachAppShellNavigationGuard(contents: WebContents): void { + contents.setWindowOpenHandler(({ url }) => { + if (isExternalOpenable(url)) void shell.openExternal(url) + return { action: 'deny' } + }) + const devServerUrl = process.env['ELECTRON_RENDERER_URL'] + const appFileUrl = devServerUrl + ? undefined + : pathToFileURL(join(import.meta.dirname, '../renderer/index.html')).href + const guardNavigation = (event: { preventDefault(): void }, url: string): void => { + if (isAppUrl(url, devServerUrl, appFileUrl)) return + event.preventDefault() + if (isExternalOpenable(url)) void shell.openExternal(url) + } + contents.on('will-navigate', guardNavigation) + contents.on('will-redirect', guardNavigation) +} diff --git a/apps/desktop/src/main/browser-electron.ts b/apps/desktop/src/main/browser-electron.ts new file mode 100644 index 00000000..b7b0f5c8 --- /dev/null +++ b/apps/desktop/src/main/browser-electron.ts @@ -0,0 +1,88 @@ +import { WebContentsView, type BrowserWindow, type WebContents } from 'electron' +import type { BrowserGuest } from './browser-service' +import { isBrowserNavigable } from './browser-url' + +export const BROWSER_PARTITION = 'persist:ari-browser' + +/** True when this webContents is the in-app browser, not the ADE shell. */ +export function isBrowserGuest(contents: WebContents): boolean { + const session = contents.session as unknown as { partition?: string } + return session.partition === BROWSER_PARTITION +} + +/** + * Chromium guest for one in-app tab. Isolated partition, sandboxed, no Node. + * Popups navigate the same view instead of spawning windows. + */ +export function createElectronBrowserGuest( + win: BrowserWindow, + onUpdated: () => void, +): BrowserGuest { + const view = new WebContentsView({ + webPreferences: { + sandbox: true, + contextIsolation: true, + nodeIntegration: false, + partition: BROWSER_PARTITION, + }, + }) + win.contentView.addChildView(view) + view.setBounds({ x: 0, y: 0, width: 800, height: 600 }) + view.setVisible(false) + const wc = view.webContents + wc.setUserAgent(wc.getUserAgent().replace(/\sElectron\/\S+/g, '')) + + const emit = (): void => onUpdated() + wc.on('did-navigate', emit) + wc.on('did-navigate-in-page', emit) + wc.on('page-title-updated', emit) + wc.on('did-start-loading', emit) + wc.on('did-stop-loading', emit) + wc.on('did-fail-load', (_event, _code, _desc, _url, isMainFrame) => { + if (isMainFrame) emit() + }) + + const blockDisallowed = (event: { preventDefault(): void }, url: string): void => { + if (!isBrowserNavigable(url)) event.preventDefault() + } + // will-navigate misses 3xx. A guest redirect to file: or an OS protocol + // has to be cancelled on will-redirect, same as the shell guard. + wc.on('will-navigate', blockDisallowed) + wc.on('will-redirect', blockDisallowed) + wc.setWindowOpenHandler(({ url }) => { + if (isBrowserNavigable(url)) void wc.loadURL(url) + return { action: 'deny' } + }) + + return { + loadURL: (url) => wc.loadURL(url), + goBack: () => { + if (wc.navigationHistory.canGoBack()) wc.navigationHistory.goBack() + }, + goForward: () => { + if (wc.navigationHistory.canGoForward()) wc.navigationHistory.goForward() + }, + reload: () => wc.reload(), + canGoBack: () => wc.navigationHistory.canGoBack(), + canGoForward: () => wc.navigationHistory.canGoForward(), + getURL: () => wc.getURL(), + getTitle: () => wc.getTitle(), + isLoading: () => wc.isLoading(), + executeJavaScript: (code) => wc.executeJavaScript(code, true), + capturePage: async (rect) => { + try { + const image = await wc.capturePage(rect) + const png = image.toPNG() + return png.length > 0 ? png : null + } catch { + return null + } + }, + setBounds: (bounds) => view.setBounds(bounds), + setVisible: (visible) => view.setVisible(visible), + destroy: () => { + if (!win.isDestroyed()) win.contentView.removeChildView(view) + if (!wc.isDestroyed()) wc.close() + }, + } +} diff --git a/apps/desktop/src/main/browser-element.test.ts b/apps/desktop/src/main/browser-element.test.ts new file mode 100644 index 00000000..89ac6418 --- /dev/null +++ b/apps/desktop/src/main/browser-element.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { + elementChipLabel, + formatElementContext, + formatElementContexts, + isPickedElement, + type PickedElement, +} from './browser-element' + +const button: PickedElement = { + url: 'http://localhost:5173/', + selector: 'button.primary', + tag: 'button', + text: 'Save changes', + role: 'button', + ariaLabel: null, + html: '', + x: 10, + y: 20, + width: 80, + height: 32, +} + +describe('formatElementContext', () => { + it('names the page, selector, and text so the agent can find the node', () => { + const block = formatElementContext(button) + expect(block).toContain('http://localhost:5173/') + expect(block).toContain('button.primary') + expect(block).toContain('Save changes') + expect(formatElementContexts([button])).toContain('') + }) + + it('labels chips from visible text', () => { + expect(elementChipLabel(button)).toBe('Save changes') + expect(isPickedElement(button)).toBe(true) + expect(isPickedElement({ tag: 'div' })).toBe(false) + }) +}) diff --git a/apps/desktop/src/main/browser-element.ts b/apps/desktop/src/main/browser-element.ts new file mode 100644 index 00000000..329dae5d --- /dev/null +++ b/apps/desktop/src/main/browser-element.ts @@ -0,0 +1,60 @@ +/** One DOM node the user (or agent snapshot) named on the in-app page. */ +export interface PickedElement { + url: string + selector: string + tag: string + text: string + role: string | null + ariaLabel: string | null + html: string + x: number + y: number + width: number + height: number +} + +const TEXT_CAP = 400 +const HTML_CAP = 1_500 + +export function isPickedElement(value: unknown): value is PickedElement { + if (typeof value !== 'object' || value === null) return false + const row = value as Record + return ( + typeof row['url'] === 'string' && + typeof row['selector'] === 'string' && + typeof row['tag'] === 'string' && + typeof row['text'] === 'string' && + typeof row['html'] === 'string' && + typeof row['x'] === 'number' && + typeof row['y'] === 'number' && + typeof row['width'] === 'number' && + typeof row['height'] === 'number' + ) +} + +/** Prompt block the agent reads when the user mentions a picked element. */ +export function formatElementContext(element: PickedElement): string { + const lines = [ + `Selected element on ${element.url}`, + `- tag: ${element.tag}`, + `- selector: ${element.selector}`, + ] + if (element.role) lines.push(`- role: ${element.role}`) + if (element.ariaLabel) lines.push(`- aria-label: ${element.ariaLabel}`) + if (element.text) lines.push(`- text: ${element.text.slice(0, TEXT_CAP)}`) + if (element.html) lines.push(`- html: ${element.html.slice(0, HTML_CAP)}`) + return lines.join('\n') +} + +export function formatElementContexts(elements: readonly PickedElement[]): string { + if (elements.length === 0) return '' + const body = elements.map((el, i) => `Element ${String(i + 1)}\n${formatElementContext(el)}`).join('\n\n') + return `\n${body}\n` +} + +export function elementChipLabel(element: PickedElement): string { + const text = element.text.trim().replace(/\s+/g, ' ') + if (text.length > 0) return text.length > 32 ? `${text.slice(0, 31)}…` : text + if (element.ariaLabel) return element.ariaLabel + return element.selector +} diff --git a/apps/desktop/src/main/browser-mcp.test.ts b/apps/desktop/src/main/browser-mcp.test.ts new file mode 100644 index 00000000..bf425fcf --- /dev/null +++ b/apps/desktop/src/main/browser-mcp.test.ts @@ -0,0 +1,79 @@ +import { mkdtemp, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { BrowserService, type BrowserGuest } from './browser-service' +import { dispatchBrowserMcpTool, startBrowserMcpServer, writeBrowserMcpProxy } from './browser-mcp' + +function guest(): BrowserGuest { + let url = 'about:blank' + return { + loadURL: vi.fn(async (next: string) => { + url = next + }), + goBack: vi.fn(), + goForward: vi.fn(), + reload: vi.fn(), + canGoBack: () => false, + canGoForward: () => false, + getURL: () => url, + getTitle: () => 'Example', + isLoading: () => false, + executeJavaScript: vi.fn(async (code: string) => { + if (code.includes('el.click()') || code.includes('el.focus()')) return true + return { url, title: 'Example', text: 'hi', elements: [] } + }), + capturePage: vi.fn(async () => null), + setBounds: vi.fn(), + setVisible: vi.fn(), + destroy: vi.fn(), + } +} + +describe('dispatchBrowserMcpTool', () => { + it('navigates and snapshots the shared guest', async () => { + const service = new BrowserService(() => guest(), vi.fn()) + const nav = await dispatchBrowserMcpTool(service, 'browser_navigate', { url: 'example.com' }) + expect(nav.isError).toBeUndefined() + expect(nav.content[0]?.text).toContain('https://example.com') + const snap = await dispatchBrowserMcpTool(service, 'browser_snapshot', {}) + expect(snap.content[0]?.text).toContain('Example') + const bad = await dispatchBrowserMcpTool(service, 'browser_navigate', { url: 'file:///x' }) + expect(bad.isError).toBe(true) + }) +}) + +describe('writeBrowserMcpProxy', () => { + it('writes a stdio proxy the ACP agent can spawn', async () => { + const dir = await mkdtemp(join(tmpdir(), 'ari-mcp-')) + const path = await writeBrowserMcpProxy(dir) + const source = await readFile(path, 'utf8') + expect(source).toContain('ARI_BROWSER_MCP_URL') + expect(source).toContain('fetch(url') + }) +}) + +describe('startBrowserMcpServer', () => { + it('serves tools/list over authorized HTTP', async () => { + const service = new BrowserService(() => guest(), vi.fn()) + const handle = await startBrowserMcpServer(service) + try { + const denied = await fetch(handle.url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' }), + }) + expect(denied.status).toBe(401) + const ok = await fetch(handle.url, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${handle.token}` }, + body: JSON.stringify({ jsonrpc: '2.0', id: 2, method: 'tools/list' }), + }) + expect(ok.status).toBe(200) + const body = (await ok.json()) as { result?: { tools?: { name: string }[] } } + expect(body.result?.tools?.map((t) => t.name)).toContain('browser_navigate') + } finally { + handle.close() + } + }) +}) diff --git a/apps/desktop/src/main/browser-mcp.ts b/apps/desktop/src/main/browser-mcp.ts new file mode 100644 index 00000000..0e58fdea --- /dev/null +++ b/apps/desktop/src/main/browser-mcp.ts @@ -0,0 +1,291 @@ +import { writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' +import { randomBytes } from 'node:crypto' +import { createLogger } from '@ari/shared/logger' +import type { BrowserService } from './browser-service' +import { resolveBrowserUrl } from './browser-url' + +const log = createLogger('desktop:browser-mcp') + +const DEFAULT_TAB = 'inspector' + +export interface BrowserMcpHandle { + url: string + token: string + close: () => void +} + +interface JsonRpc { + jsonrpc?: string + id?: number | string + method?: string + params?: unknown +} + +function json(res: ServerResponse, status: number, body: unknown): void { + const payload = JSON.stringify(body) + res.writeHead(status, { + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(payload), + }) + res.end(payload) +} + +function asRecord(value: unknown): Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {} +} + +function textResult( + text: string, + isError = false, +): { content: { type: 'text'; text: string }[]; isError?: boolean } { + return { content: [{ type: 'text', text }], ...(isError ? { isError: true } : {}) } +} + +export const BROWSER_MCP_TOOLS = [ + { + name: 'browser_status', + description: + "URL, title, and loading state of Ari's visible in-app browser. Prefer this over any other browser.", + inputSchema: { type: 'object', properties: {}, additionalProperties: false }, + }, + { + name: 'browser_navigate', + description: + "Open a URL in Ari's visible in-app browser (http/https). The user sees this page. Use this instead of any other browser, web fetch, or computer-use tool.", + inputSchema: { + type: 'object', + properties: { url: { type: 'string', description: 'http(s) URL or host[:port]' } }, + required: ['url'], + additionalProperties: false, + }, + }, + { + name: 'browser_snapshot', + description: + 'Page text plus interactive elements (tag, selector, label) from the in-app browser.', + inputSchema: { type: 'object', properties: {}, additionalProperties: false }, + }, + { + name: 'browser_click', + description: 'Click the first element matching a CSS selector in the in-app browser.', + inputSchema: { + type: 'object', + properties: { selector: { type: 'string' } }, + required: ['selector'], + additionalProperties: false, + }, + }, + { + name: 'browser_type', + description: 'Type into the first element matching a CSS selector in the in-app browser.', + inputSchema: { + type: 'object', + properties: { selector: { type: 'string' }, text: { type: 'string' } }, + required: ['selector', 'text'], + additionalProperties: false, + }, + }, +] as const + +export async function dispatchBrowserMcpTool( + browsers: BrowserService, + name: string, + args: Record, + tabId = DEFAULT_TAB, +): Promise<{ content: { type: 'text'; text: string }[]; isError?: boolean }> { + try { + if (!browsers.has(tabId)) await browsers.open(tabId) + if (name === 'browser_status') { + return textResult(JSON.stringify(browsers.snapshot(tabId))) + } + if (name === 'browser_navigate') { + const url = typeof args['url'] === 'string' ? args['url'] : '' + const resolved = resolveBrowserUrl(url) + if (!resolved.ok) return textResult(resolved.error, true) + const result = await browsers.navigate(tabId, resolved.url) + browsers.reveal(tabId) + return result.ok ? textResult(JSON.stringify(result.tab)) : textResult(result.error, true) + } + if (name === 'browser_snapshot') { + const snap = await browsers.pageSnapshot(tabId) + return textResult(JSON.stringify(snap)) + } + if (name === 'browser_click') { + const selector = typeof args['selector'] === 'string' ? args['selector'] : '' + if (!selector) return textResult('selector is required', true) + const clicked = await browsers.runScript( + tabId, + `(() => { const el = document.querySelector(${JSON.stringify(selector)}); if (!el) return false; el.click(); return true })()`, + ) + return clicked === true + ? textResult('clicked') + : textResult(`no element for ${selector}`, true) + } + if (name === 'browser_type') { + const selector = typeof args['selector'] === 'string' ? args['selector'] : '' + const text = typeof args['text'] === 'string' ? args['text'] : '' + if (!selector) return textResult('selector is required', true) + const typed = await browsers.runScript( + tabId, + `(() => { const el = document.querySelector(${JSON.stringify(selector)}); if (!el) return false; el.focus(); if ('value' in el) el.value = ${JSON.stringify(text)}; el.dispatchEvent(new Event('input', { bubbles: true })); return true })()`, + ) + return typed === true ? textResult('typed') : textResult(`no element for ${selector}`, true) + } + return textResult(`unknown tool ${name}`, true) + } catch (error: unknown) { + return textResult(error instanceof Error ? error.message : String(error), true) + } +} + +function authorized(req: IncomingMessage, token: string): boolean { + const header = req.headers['authorization'] + if (typeof header === 'string' && header === `Bearer ${token}`) return true + const url = new URL(req.url ?? '/', 'http://127.0.0.1') + return url.searchParams.get('token') === token +} + +function readBody(req: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + req.on('data', (chunk: Buffer) => chunks.push(chunk)) + req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))) + req.on('error', reject) + }) +} + +async function handleRpc( + browsers: BrowserService, + message: JsonRpc, +): Promise<{ + jsonrpc: '2.0' + id?: number | string + result?: unknown + error?: { code: number; message: string } +}> { + const id = message.id + const method = message.method ?? '' + if (method === 'initialize') { + return { + jsonrpc: '2.0', + id, + result: { + protocolVersion: '2024-11-05', + capabilities: { tools: {} }, + serverInfo: { name: 'ari-browser', version: '0.1.0' }, + instructions: + "You are connected to Ari's in-app browser. The user can see the page. Use browser_navigate, browser_snapshot, browser_click, and browser_type for every web task. Never say a browser is unavailable and do not use a separate browser or computer-use tool.", + }, + } + } + if (method === 'notifications/initialized' || method === 'notifications/cancelled') { + return { jsonrpc: '2.0', id } + } + if (method === 'ping' || method === 'tools/list') { + if (method === 'ping') return { jsonrpc: '2.0', id, result: {} } + return { jsonrpc: '2.0', id, result: { tools: BROWSER_MCP_TOOLS } } + } + if (method === 'tools/call') { + const params = asRecord(message.params) + const name = typeof params['name'] === 'string' ? params['name'] : '' + const args = asRecord(params['arguments']) + const result = await dispatchBrowserMcpTool(browsers, name, args) + return { jsonrpc: '2.0', id, result } + } + return { jsonrpc: '2.0', id, error: { code: -32601, message: `Method not found: ${method}` } } +} + +/** Loopback Streamable HTTP MCP for the in-app browser. */ +export function startBrowserMcpServer(browsers: BrowserService): Promise { + const token = randomBytes(24).toString('base64url') + const server = createServer((req, res) => { + void (async () => { + if (req.method === 'OPTIONS') { + res.writeHead(204) + res.end() + return + } + if (!authorized(req, token)) { + json(res, 401, { error: 'unauthorized' }) + return + } + if (req.method !== 'POST') { + json(res, 405, { error: 'POST required' }) + return + } + let parsed: JsonRpc + try { + parsed = JSON.parse(await readBody(req)) as JsonRpc + } catch { + json(res, 400, { error: 'invalid json' }) + return + } + if (parsed.id === undefined) { + void handleRpc(browsers, parsed) + res.writeHead(202) + res.end() + return + } + json(res, 200, await handleRpc(browsers, parsed)) + })().catch((error: unknown) => { + log.warn('browser mcp request failed', { error: String(error) }) + if (!res.headersSent) json(res, 500, { error: 'internal' }) + }) + }) + return new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + if (address === null || typeof address === 'string') { + reject(new Error('browser mcp failed to bind')) + return + } + resolve({ + url: `http://127.0.0.1:${String(address.port)}/mcp`, + token, + close: () => server.close(), + }) + }) + }) +} + +/** + * ACP requires stdio MCP. This proxy is spawned with ELECTRON_RUN_AS_NODE + * and forwards JSON-RPC lines to the loopback HTTP server. + */ +const MCP_PROXY_SOURCE = [ + "import { createInterface } from 'node:readline'", + "import { stdin, stdout } from 'node:process'", + "const url = process.env.ARI_BROWSER_MCP_URL", + "const token = process.env.ARI_BROWSER_MCP_TOKEN", + "async function rpc(msg) {", + " const res = await fetch(url, {", + " method: 'POST',", + " headers: { 'content-type': 'application/json', authorization: 'Bearer ' + token },", + " body: JSON.stringify(msg),", + " })", + " if (res.status === 202) return null", + " return await res.json()", + "}", + "const rl = createInterface({ input: stdin })", + "rl.on('line', (line) => {", + " if (!line.trim()) return", + " const msg = JSON.parse(line)", + " rpc(msg).then((out) => {", + " if (out && msg.id !== undefined) stdout.write(JSON.stringify(out) + '\\n')", + " }).catch((err) => {", + " if (msg.id !== undefined) {", + " stdout.write(JSON.stringify({ jsonrpc: '2.0', id: msg.id, error: { code: -32000, message: String(err) } }) + '\\n')", + " }", + " })", + "})", +].join('\n') + +export async function writeBrowserMcpProxy(dir: string): Promise { + const path = join(dir, 'browser-mcp-proxy.mjs') + await writeFile(path, MCP_PROXY_SOURCE, 'utf8') + return path +} diff --git a/apps/desktop/src/main/browser-pick-script.ts b/apps/desktop/src/main/browser-pick-script.ts new file mode 100644 index 00000000..349bcdb5 --- /dev/null +++ b/apps/desktop/src/main/browser-pick-script.ts @@ -0,0 +1,123 @@ +/** + * Injected into the guest page. Built from line arrays so the main bundle + * never carries a multi-line template literal (esbuild CJS shims split those). + */ +function script(lines: string[]): string { + return lines.join('\n') +} + +export const PICKER_SOURCE = script([ + 'new Promise((resolve) => {', + ' window.__ariCancelPick && window.__ariCancelPick()', + " const box = document.createElement('div')", + " box.setAttribute('data-ari-pick', '1')", + " box.style.cssText = 'position:fixed;z-index:2147483647;pointer-events:none;border:2px solid #7c6cf0;background:rgba(124,108,240,.12);border-radius:4px;'", + ' document.documentElement.appendChild(box)', + ' function cssPath(el) {', + ' if (el.id) {', + " const id = '#' + CSS.escape(el.id)", + ' try { if (document.querySelectorAll(id).length === 1) return id } catch (e) {}', + ' }', + ' const parts = []', + ' let cur = el', + ' for (let i = 0; i < 5 && cur && cur.nodeType === 1 && cur !== document.documentElement; i++) {', + ' let part = cur.tagName.toLowerCase()', + ' if (cur.classList && cur.classList.length) {', + " part += '.' + Array.from(cur.classList).slice(0, 2).map(function(c) { return CSS.escape(c) }).join('.')", + ' }', + ' const parent = cur.parentElement', + ' if (parent) {', + ' const same = Array.from(parent.children).filter(function(c) { return c.tagName === cur.tagName })', + " if (same.length > 1) part += ':nth-of-type(' + (same.indexOf(cur) + 1) + ')'", + ' }', + ' parts.unshift(part)', + ' cur = parent', + ' }', + " return parts.join(' > ')", + ' }', + ' function targetOf(event) {', + ' const el = document.elementFromPoint(event.clientX, event.clientY)', + ' if (!el || el === box) return null', + ' return el', + ' }', + ' function cleanup() {', + ' window.__ariCancelPick = undefined', + " window.removeEventListener('mousemove', onMove, true)", + " window.removeEventListener('click', onClick, true)", + " window.removeEventListener('keydown', onKey, true)", + ' box.remove()', + ' }', + ' function onMove(event) {', + ' const el = targetOf(event)', + ' if (!el) return', + ' const r = el.getBoundingClientRect()', + " box.style.left = r.left + 'px'", + " box.style.top = r.top + 'px'", + " box.style.width = r.width + 'px'", + " box.style.height = r.height + 'px'", + ' }', + ' function onClick(event) {', + ' event.preventDefault()', + ' event.stopPropagation()', + ' const el = targetOf(event)', + ' cleanup()', + ' if (!el) { resolve(null); return }', + ' const r = el.getBoundingClientRect()', + ' resolve({', + ' url: location.href,', + ' selector: cssPath(el),', + ' tag: el.tagName.toLowerCase(),', + " text: (el.innerText || el.value || '').trim().slice(0, 400),", + " role: el.getAttribute('role'),", + " ariaLabel: el.getAttribute('aria-label'),", + " html: (el.outerHTML || '').slice(0, 1500),", + ' x: r.x, y: r.y, width: r.width, height: r.height', + ' })', + ' }', + ' function onKey(event) {', + " if (event.key === 'Escape') { event.preventDefault(); cleanup(); resolve(null) }", + ' }', + ' window.__ariCancelPick = function() { cleanup(); resolve(null) }', + " window.addEventListener('mousemove', onMove, true)", + " window.addEventListener('click', onClick, true)", + " window.addEventListener('keydown', onKey, true)", + '})', +]) + +export const CANCEL_PICK_SOURCE = + 'void (window.__ariCancelPick && window.__ariCancelPick())' + +export const SNAPSHOT_SOURCE = script([ + '(function() {', + ' function cssPath(el) {', + ' if (el.id) {', + " const id = '#' + CSS.escape(el.id)", + ' try { if (document.querySelectorAll(id).length === 1) return id } catch (e) {}', + ' }', + ' const parts = []', + ' let cur = el', + ' for (let i = 0; i < 4 && cur && cur.nodeType === 1 && cur !== document.documentElement; i++) {', + ' let part = cur.tagName.toLowerCase()', + ' if (cur.classList && cur.classList.length) {', + " part += '.' + Array.from(cur.classList).slice(0, 2).map(function(c) { return CSS.escape(c) }).join('.')", + ' }', + ' parts.unshift(part)', + ' cur = cur.parentElement', + ' }', + " return parts.join(' > ')", + ' }', + " const nodes = Array.from(document.querySelectorAll('a,button,input,textarea,select,[role=button],[role=link]')).slice(0, 80)", + ' return {', + ' url: location.href,', + ' title: document.title,', + " text: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 4000),", + ' elements: nodes.map(function(el) {', + ' return {', + ' tag: el.tagName.toLowerCase(),', + ' selector: cssPath(el),', + " text: ((el.innerText || el.getAttribute('aria-label') || el.getAttribute('placeholder') || '') + '').trim().slice(0, 80)", + ' }', + ' })', + ' }', + '})()', +]) diff --git a/apps/desktop/src/main/browser-service.test.ts b/apps/desktop/src/main/browser-service.test.ts new file mode 100644 index 00000000..ab9f7a2c --- /dev/null +++ b/apps/desktop/src/main/browser-service.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, vi } from 'vitest' +import { BrowserService, type BrowserGuest } from './browser-service' + +function fakeGuest(): { + guest: BrowserGuest + loadURL: ReturnType + goBack: ReturnType + setBounds: ReturnType + setVisible: ReturnType + destroy: ReturnType +} { + let url = 'about:blank' + const loadURL = vi.fn(async (next: string) => { + url = next + }) + const goBack = vi.fn() + const setBounds = vi.fn() + const setVisible = vi.fn() + const destroy = vi.fn() + const guest: BrowserGuest = { + loadURL, + goBack, + goForward: vi.fn(), + reload: vi.fn(), + canGoBack: () => false, + canGoForward: () => false, + getURL: () => url, + getTitle: () => 'Example', + isLoading: () => false, + executeJavaScript: vi.fn(async () => null), + capturePage: vi.fn(async () => null), + setBounds, + setVisible, + destroy, + } + return { guest, loadURL, goBack, setBounds, setVisible, destroy } +} + +describe('BrowserService', () => { + it('opens a tab, navigates http(s), and refuses file URLs', async () => { + const { guest, loadURL } = fakeGuest() + const service = new BrowserService(() => guest, vi.fn()) + + const opened = await service.open('tab1') + expect(opened.url).toBe('about:blank') + + const ok = await service.navigate('tab1', 'example.com') + expect(ok.ok).toBe(true) + if (ok.ok) expect(ok.tab.url).toBe('https://example.com/') + expect(loadURL).toHaveBeenCalledWith('https://example.com/') + + const refused = await service.navigate('tab1', 'file:///etc/passwd') + expect(refused.ok).toBe(false) + expect(loadURL).toHaveBeenCalledTimes(1) + }) + + it('moves, hides, and destroys the guest', async () => { + const { guest, goBack, setBounds, setVisible, destroy } = fakeGuest() + const service = new BrowserService(() => guest, vi.fn()) + await service.open('tab1') + expect(service.go('tab1', 'back').id).toBe('tab1') + expect(goBack).toHaveBeenCalledOnce() + expect(service.layout('tab1', { x: 10, y: 20, width: 400, height: 300 }, true)).toBe(true) + expect(setVisible).toHaveBeenCalledWith(true) + expect(setBounds).toHaveBeenCalledWith({ x: 10, y: 20, width: 400, height: 300 }) + expect(service.close('tab1')).toBe(true) + expect(destroy).toHaveBeenCalledOnce() + expect(service.layout('tab1', { x: 0, y: 0, width: 1, height: 1 }, false)).toBe(false) + }) + + it('returns a picked element from the guest script', async () => { + const picked = { + url: 'https://example.com/', + selector: 'button', + tag: 'button', + text: 'Go', + role: 'button', + ariaLabel: null, + html: '', + x: 1, + y: 2, + width: 3, + height: 4, + } + const { guest } = fakeGuest() + guest.executeJavaScript = vi.fn(async (code: string) => + code.includes('__ariCancelPick') && !code.includes('new Promise') ? undefined : picked, + ) + guest.capturePage = vi.fn(async () => Buffer.from('png')) + const service = new BrowserService(() => guest, vi.fn()) + await service.open('tab1') + const result = await service.pick('tab1') + expect(result.ok).toBe(true) + if (result.ok) { + expect(result.element.selector).toBe('button') + expect(result.pngBase64).toBe(Buffer.from('png').toString('base64')) + } + }) +}) diff --git a/apps/desktop/src/main/browser-service.ts b/apps/desktop/src/main/browser-service.ts new file mode 100644 index 00000000..2a89b1f2 --- /dev/null +++ b/apps/desktop/src/main/browser-service.ts @@ -0,0 +1,195 @@ +import { isPickedElement, type PickedElement } from './browser-element' +import { CANCEL_PICK_SOURCE, PICKER_SOURCE, SNAPSHOT_SOURCE } from './browser-pick-script' +import { resolveBrowserUrl } from './browser-url' + +export interface BrowserBounds { + x: number + y: number + width: number + height: number +} + +export interface BrowserTabState { + id: string + url: string + title: string + canGoBack: boolean + canGoForward: boolean + loading: boolean + error: string | null + /** True when an agent just drove this tab — the shell should show the pane. */ + reveal?: boolean +} + +/** One Chromium guest the service positions and navigates. Injected in tests. */ +export interface BrowserGuest { + loadURL(url: string): Promise + goBack(): void + goForward(): void + reload(): void + canGoBack(): boolean + canGoForward(): boolean + getURL(): string + getTitle(): string + isLoading(): boolean + executeJavaScript(code: string): Promise + capturePage(rect?: BrowserBounds): Promise + setBounds(bounds: BrowserBounds): void + setVisible(visible: boolean): void + destroy(): void +} + +export type BrowserGuestFactory = (id: string, onUpdated: () => void) => BrowserGuest + +interface Tab { + guest: BrowserGuest + error: string | null +} + +/** + * Owns in-app browser tabs. The factory builds the Electron guest; this + * class is the navigation, layout, and snapshot policy so tests do not + * need Chromium. + */ +export class BrowserService { + readonly #tabs = new Map() + readonly #createGuest: BrowserGuestFactory + readonly #onUpdated: (state: BrowserTabState) => void + + constructor(createGuest: BrowserGuestFactory, onUpdated: (state: BrowserTabState) => void) { + this.#createGuest = createGuest + this.#onUpdated = onUpdated + } + + async open(id: string, url?: string): Promise { + if (this.#tabs.get(id) === undefined) { + const tab: Tab = { guest: this.#createGuest(id, () => this.#emit(id)), error: null } + this.#tabs.set(id, tab) + } + if (url !== undefined && url !== '') { + const result = await this.navigate(id, url) + if (result.ok) return result.tab + } + return this.snapshot(id) + } + + async navigate( + id: string, + input: string, + ): Promise<{ ok: true; tab: BrowserTabState } | { ok: false; error: string }> { + const tab = this.#tab(id) + const resolved = resolveBrowserUrl(input) + if (!resolved.ok) return resolved + tab.error = null + try { + await tab.guest.loadURL(resolved.url) + } catch (error: unknown) { + tab.error = error instanceof Error ? error.message : String(error) + } + this.#emit(id) + return { ok: true, tab: this.snapshot(id) } + } + + go(id: string, action: 'back' | 'forward' | 'reload'): BrowserTabState { + const tab = this.#tab(id) + if (action === 'back') tab.guest.goBack() + else if (action === 'forward') tab.guest.goForward() + else tab.guest.reload() + tab.error = null + this.#emit(id) + return this.snapshot(id) + } + + layout(id: string, bounds: BrowserBounds, visible: boolean): boolean { + const tab = this.#tabs.get(id) + if (tab === undefined) return false + tab.guest.setVisible(visible) + if (visible) tab.guest.setBounds(bounds) + return true + } + + close(id: string): boolean { + const tab = this.#tabs.get(id) + if (tab === undefined) return false + tab.guest.destroy() + this.#tabs.delete(id) + return true + } + + snapshot(id: string): BrowserTabState { + const tab = this.#tab(id) + const url = tab.guest.getURL() + return { + id, + url: url.length > 0 ? url : 'about:blank', + title: tab.guest.getTitle(), + canGoBack: tab.guest.canGoBack(), + canGoForward: tab.guest.canGoForward(), + loading: tab.guest.isLoading(), + error: tab.error, + } + } + + async pick( + id: string, + ): Promise< + { ok: true; element: PickedElement; pngBase64: string | null } | { ok: false; error: string } + > { + const tab = this.#tab(id) + await tab.guest.executeJavaScript(CANCEL_PICK_SOURCE) + const raw = await tab.guest.executeJavaScript(PICKER_SOURCE) + if (raw === null || raw === undefined) return { ok: false, error: 'cancelled' } + if (!isPickedElement(raw)) return { ok: false, error: 'pick returned no element' } + const png = await tab.guest.capturePage({ + x: Math.round(raw.x), + y: Math.round(raw.y), + width: Math.max(1, Math.round(raw.width)), + height: Math.max(1, Math.round(raw.height)), + }) + return { ok: true, element: raw, pngBase64: png ? png.toString('base64') : null } + } + + async cancelPick(id: string): Promise { + const tab = this.#tabs.get(id) + if (tab === undefined) return false + await tab.guest.executeJavaScript(CANCEL_PICK_SOURCE) + return true + } + + async pageSnapshot(id: string): Promise { + const tab = this.#tab(id) + return tab.guest.executeJavaScript(SNAPSHOT_SOURCE) + } + + async runScript(id: string, code: string): Promise { + const tab = this.#tab(id) + return tab.guest.executeJavaScript(code) + } + + async screenshot(id: string): Promise { + const tab = this.#tab(id) + return tab.guest.capturePage() + } + + reveal(id: string): void { + this.#onUpdated({ ...this.snapshot(id), reveal: true }) + } + + has(id: string): boolean { + return this.#tabs.has(id) + } + + dispose(): void { + for (const id of [...this.#tabs.keys()]) this.close(id) + } + + #tab(id: string): Tab { + const tab = this.#tabs.get(id) + if (tab === undefined) throw new Error(`unknown browser tab ${id}`) + return tab + } + + #emit(id: string): void { + this.#onUpdated(this.snapshot(id)) + } +} diff --git a/apps/desktop/src/main/browser-url.test.ts b/apps/desktop/src/main/browser-url.test.ts new file mode 100644 index 00000000..cb4a2131 --- /dev/null +++ b/apps/desktop/src/main/browser-url.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest' +import { isBrowserNavigable, resolveBrowserUrl } from './browser-url' + +describe('isBrowserNavigable', () => { + it('allows http, https, and about:blank', () => { + expect(isBrowserNavigable('https://example.com/x')).toBe(true) + expect(isBrowserNavigable('http://localhost:5173/')).toBe(true) + expect(isBrowserNavigable('about:blank')).toBe(true) + }) + + it('refuses file, javascript, data, and mailto', () => { + expect(isBrowserNavigable('file:///etc/passwd')).toBe(false) + expect(isBrowserNavigable('javascript:alert(1)')).toBe(false) + expect(isBrowserNavigable('data:text/html,hi')).toBe(false) + expect(isBrowserNavigable('mailto:hi@example.com')).toBe(false) + }) +}) + +describe('resolveBrowserUrl', () => { + it('keeps an empty bar as a blank tab', () => { + expect(resolveBrowserUrl('')).toEqual({ ok: true, url: 'about:blank' }) + expect(resolveBrowserUrl(' about:blank ')).toEqual({ ok: true, url: 'about:blank' }) + }) + + it('accepts http(s) URLs as written', () => { + expect(resolveBrowserUrl('https://example.com/a')).toEqual({ + ok: true, + url: 'https://example.com/a', + }) + expect(resolveBrowserUrl('http://127.0.0.1:3000')).toEqual({ + ok: true, + url: 'http://127.0.0.1:3000/', + }) + }) + + it('prefixes https for public hosts and http for loopback', () => { + expect(resolveBrowserUrl('example.com/docs')).toEqual({ + ok: true, + url: 'https://example.com/docs', + }) + expect(resolveBrowserUrl('localhost:5173')).toEqual({ + ok: true, + url: 'http://localhost:5173/', + }) + }) + + it('rejects schemes the guest must never load', () => { + expect(resolveBrowserUrl('file:///tmp/x')).toMatchObject({ ok: false }) + expect(resolveBrowserUrl('javascript:alert(1)')).toMatchObject({ ok: false }) + expect(resolveBrowserUrl('not a url')).toMatchObject({ ok: false }) + }) +}) diff --git a/apps/desktop/src/main/browser-url.ts b/apps/desktop/src/main/browser-url.ts new file mode 100644 index 00000000..bf4aa8db --- /dev/null +++ b/apps/desktop/src/main/browser-url.ts @@ -0,0 +1,57 @@ +/** + * What the in-app browser may load. http(s) only, plus about:blank as the + * empty tab. file:/javascript:/data: never reach the guest — those belong + * outside this view, or nowhere. + */ + +const NAVIGABLE = new Set(['http:', 'https:']) + +/** True when a fully-resolved URL may become the guest's document. */ +export function isBrowserNavigable(rawUrl: string): boolean { + if (rawUrl === 'about:blank') return true + try { + const parsed = new URL(rawUrl) + return NAVIGABLE.has(parsed.protocol) + } catch { + return false + } +} + +function isLoopbackHost(value: string): boolean { + const host = value.split('/')[0]?.split(':')[0]?.toLowerCase() ?? '' + return host === 'localhost' || host === '127.0.0.1' || host === '[::1]' || host === '::1' +} + +/** + * Turns an address-bar string into a loadable URL. Bare hosts get https, + * except loopback which stays http so local dev servers work. + */ +export function resolveBrowserUrl( + input: string, +): { ok: true; url: string } | { ok: false; error: string } { + const trimmed = input.trim() + if (trimmed === '' || trimmed.toLowerCase() === 'about:blank') { + return { ok: true, url: 'about:blank' } + } + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) { + try { + const parsed = new URL(trimmed) + if (!NAVIGABLE.has(parsed.protocol)) { + return { ok: false, error: `${parsed.protocol} URLs cannot open in the in-app browser` } + } + return { ok: true, url: parsed.href } + } catch { + return { ok: false, error: 'Enter an http(s) URL' } + } + } + const candidate = `${isLoopbackHost(trimmed) ? 'http' : 'https'}://${trimmed}` + try { + const parsed = new URL(candidate) + if (!NAVIGABLE.has(parsed.protocol) || parsed.hostname.length === 0) { + return { ok: false, error: 'Enter an http(s) URL' } + } + return { ok: true, url: parsed.href } + } catch { + return { ok: false, error: 'Enter an http(s) URL' } + } +} diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index f39d3ce6..66234896 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -1,16 +1,15 @@ -import { app, BrowserWindow, shell } from 'electron' -import { join } from 'node:path' -import { pathToFileURL } from 'node:url' +import { app, BrowserWindow } from 'electron' import { isolateDevInstance } from './dev-instance' import { registerRpc, startAppUpdateChecks } from './rpc' import { createTray, type TrayHandle } from './tray' import { updateTrayStatus } from './tray-status' import { createMainWindow } from './window' -import { isAppUrl, isExternalOpenable } from './external-links' // The launch animation's signature sound is Web Audio; without this switch // Chromium blocks it until the first user gesture. app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required') +// Google properties (YouTube) often fail with net::ERR_FAILED under HTTP/3 in Electron. +app.commandLine.appendSwitch('disable-quic') // Unpackaged `pnpm dev` must take this identity before the lock: Electron keys // the mutex (and every userData store) off app name. Installed Ari stays on @@ -62,27 +61,4 @@ if (!gotLock) { app.on('window-all-closed', () => { if (process.platform !== 'darwin') app.quit() }) - - app.on('web-contents-created', (_event, contents) => { - // Popups (window.open / target=_blank) open in the OS browser. - contents.setWindowOpenHandler(({ url }) => { - if (isExternalOpenable(url)) void shell.openExternal(url) - return { action: 'deny' } - }) - // Same-window clicks (bare from transcript markdown) never - // navigate the ADE: the app entry stays, everything openable goes to the - // OS browser, everything else is dropped. Only the packaged renderer's - // own entry file counts as the app — never an arbitrary file:/data: URL. - const devServerUrl = process.env['ELECTRON_RENDERER_URL'] - const appFileUrl = devServerUrl - ? undefined - : pathToFileURL(join(import.meta.dirname, '../renderer/index.html')).href - const guardNavigation = (_navEvent: { preventDefault(): void }, url: string): void => { - if (isAppUrl(url, devServerUrl, appFileUrl)) return - _navEvent.preventDefault() - if (isExternalOpenable(url)) void shell.openExternal(url) - } - contents.on('will-navigate', guardNavigation) - contents.on('will-redirect', guardNavigation) - }) } diff --git a/apps/desktop/src/main/ipc-methods.ts b/apps/desktop/src/main/ipc-methods.ts index 4b49703c..26c85730 100644 --- a/apps/desktop/src/main/ipc-methods.ts +++ b/apps/desktop/src/main/ipc-methods.ts @@ -57,6 +57,13 @@ export const IPC_METHODS = [ 'terminal.write', 'terminal.resize', 'terminal.kill', + 'browser.open', + 'browser.navigate', + 'browser.go', + 'browser.close', + 'browser.layout', + 'browser.pick', + 'browser.cancelPick', 'project.list', 'project.add', 'project.open', diff --git a/apps/desktop/src/main/rpc.ts b/apps/desktop/src/main/rpc.ts index d40685ac..49cdc844 100644 --- a/apps/desktop/src/main/rpc.ts +++ b/apps/desktop/src/main/rpc.ts @@ -45,6 +45,9 @@ import { type PtyFactory, type PtyLike, } from './terminal-service' +import { BrowserService } from './browser-service' +import { createElectronBrowserGuest } from './browser-electron' +import { startBrowserMcpServer, writeBrowserMcpProxy } from './browser-mcp' import { ensureProjectWatched, getIndexedFiles, stopWatchingProject } from './watcher-bridge' import { createAppUpdater } from './updater' import type { UpdateController } from './update-controller' @@ -81,7 +84,7 @@ import { planFor } from '@ari/providers/package-manager' import { runInstall, type InstallHandle } from '@ari/providers/install' import { AcpDriver } from '@ari/providers/acp' import { resolveAcpLaunch, probeLaunch } from '@ari/providers/acp/launches' -import type { AcpLaunch } from '@ari/providers/acp/connection' +import type { AcpLaunch, AcpMcpServer } from '@ari/providers/acp/connection' import type { AcpTerminalLogin } from '@ari/providers/acp/protocol' import { ProviderAuthState, @@ -96,6 +99,7 @@ import type { SessionImportDeps } from './session-import' import type { Driver } from '@ari/providers/driver' import { AriCoreDriver } from '@ari/ari-core/driver' import { FileConversationStore } from '@ari/ari-core/conversation-store' +import type { McpServerConfig } from '@ari/ari-core/mcp-servers' import { todoFilenameFor } from '@ari/ari-core/todo' const log = createLogger('desktop:rpc') @@ -469,6 +473,9 @@ function publishAuthWall( } satisfies ProvidersUpdateFrame) } +let browserMcpServers: AcpMcpServer[] = [] +let browserCoreMcpServers: McpServerConfig[] = [] + /** * Registers each installed CLI driver as its detection resolves, preferring * the ACP transport (M16) with the legacy one-shot CLI driver as automatic @@ -497,8 +504,12 @@ function hydrateDrivers(registry: DriverRegistry): void { cliBinaryPath: detection.binaryPath, }) registry.register( - new AcpDriver(candidate.kind, launch, candidate.make(detection.binaryPath), (wall) => - publishAuthWall(candidate.kind, wall), + new AcpDriver( + candidate.kind, + launch, + candidate.make(detection.binaryPath), + (wall) => publishAuthWall(candidate.kind, wall), + () => browserMcpServers, ), ) log.info('driver registered', { @@ -586,6 +597,7 @@ export function registerRpc(contents: WebContents, options: RegisterRpcOptions = conversations: new FileConversationStore( join(app.getPath('userData'), 'ari-core', 'conversations'), ), + mcpServers: () => browserCoreMcpServers, }), ) driverRegistryRef = driverRegistry @@ -724,6 +736,58 @@ export function registerRpc(contents: WebContents, options: RegisterRpcOptions = }, ptyFactory, ) + const browsers = new BrowserService( + (_id, onUpdated) => { + const win = BrowserWindow.fromWebContents(contents) + if (win === null) throw new Error('browser host window is gone') + return createElectronBrowserGuest(win, onUpdated) + }, + (state) => rpcRegistry.publish('browser.updated', state), + ) + contents.once('destroyed', () => { + browsers.dispose() + browserMcpServers = [] + browserCoreMcpServers = [] + }) + void startBrowserMcpServer(browsers) + .then(async (handle) => { + const proxyPath = await writeBrowserMcpProxy(app.getPath('userData')) + const stdioEnv = [ + { name: 'ELECTRON_RUN_AS_NODE', value: '1' }, + { name: 'ARI_BROWSER_MCP_URL', value: handle.url }, + { name: 'ARI_BROWSER_MCP_TOKEN', value: handle.token }, + ] + browserMcpServers = [ + { + type: 'http', + name: 'ari-browser', + url: handle.url, + headers: [{ name: 'Authorization', value: `Bearer ${handle.token}` }], + }, + { + name: 'ari-browser', + command: process.execPath, + args: [proxyPath], + env: stdioEnv, + }, + ] + browserCoreMcpServers = [ + { + id: 'ari-browser', + name: 'ari-browser', + command: process.execPath, + args: [proxyPath], + env: { + ELECTRON_RUN_AS_NODE: '1', + ARI_BROWSER_MCP_URL: handle.url, + ARI_BROWSER_MCP_TOKEN: handle.token, + }, + disabled: false, + }, + ] + contents.once('destroyed', () => handle.close()) + }) + .catch((error: unknown) => log.warn('browser mcp failed to start', { error: String(error) })) const r = rpcRegistry r.register('ping', () => ({ pong: true, at: Date.now() })) @@ -1204,6 +1268,22 @@ export function registerRpc(contents: WebContents, options: RegisterRpcOptions = return { killed: true } }) + r.register('browser.open', async (params) => browsers.open(params.id, params.url)) + r.register('browser.navigate', async (params) => browsers.navigate(params.id, params.url)) + r.register('browser.go', (params) => browsers.go(params.id, params.action)) + r.register('browser.close', (params) => ({ closed: browsers.close(params.id) })) + r.register('browser.layout', (params) => ({ + applied: browsers.layout( + params.id, + { x: params.x, y: params.y, width: params.width, height: params.height }, + params.visible, + ), + })) + r.register('browser.pick', async (params) => browsers.pick(params.id)) + r.register('browser.cancelPick', async (params) => ({ + cancelled: await browsers.cancelPick(params.id), + })) + r.register('project.list', async () => getProjectStore().load()) r.register('project.add', async (params) => { diff --git a/apps/desktop/src/main/window.ts b/apps/desktop/src/main/window.ts index bac945f4..2bb9fb47 100644 --- a/apps/desktop/src/main/window.ts +++ b/apps/desktop/src/main/window.ts @@ -4,6 +4,7 @@ import { oklchToHex } from '@ari/ui/color' import { themeOf } from '@ari/ui/themes' import type { Theme } from '@ari/ui/themes' import { appDisplayName } from './dev-instance' +import { attachAppShellNavigationGuard } from './app-shell-guard' import { getSettingsStore } from './store' /** @@ -124,6 +125,8 @@ export function createMainWindow(): BrowserWindow { win.on('maximize', persistBounds) win.on('unmaximize', persistBounds) + attachAppShellNavigationGuard(win.webContents) + if (process.env['ELECTRON_RENDERER_URL']) { void win.loadURL(process.env['ELECTRON_RENDERER_URL']) } else { diff --git a/apps/desktop/src/renderer/src/App.tsx b/apps/desktop/src/renderer/src/App.tsx index 5ace5729..77a83fa4 100644 --- a/apps/desktop/src/renderer/src/App.tsx +++ b/apps/desktop/src/renderer/src/App.tsx @@ -6,7 +6,12 @@ import { ToastProvider, useToast } from '@ari/ui/toast' import { SessionImportDialog } from './features/providers' import { useUpdateToasts } from './features/providers/use-update-toasts' import { useAppUpdateToast } from './features/updates' -import type { RpcResults, SessionEventFrame, SessionSummary } from '@ari/contracts/rpc' +import type { + BrowserTabState, + RpcResults, + SessionEventFrame, + SessionSummary, +} from '@ari/contracts/rpc' import type { DriverKind, PermissionMode } from '@ari/contracts/common' import { createLogger } from '@ari/shared/logger' import { rpc } from './lib/rpc' @@ -22,6 +27,7 @@ import { sidebarOrder, } from './features/session/session-nav' import { descendantIds } from './features/session/session-tree' +import { BrowserPanel } from './features/browser/BrowserPanel' import { TerminalDock, TerminalPane } from './features/terminal' import { openTerminalTab, @@ -80,6 +86,7 @@ type InspectorId = Exclude /** Rail headings, and the accessible name of the rail itself. */ const INSPECTOR_TITLES: Record = { terminal: 'Terminal', + browser: 'Browser', changes: 'Changes', files: 'Files', usage: 'Usage', @@ -214,10 +221,20 @@ function Shell() { setInspector((prev) => (prev === 'terminal' ? null : 'terminal')) }, [leaveWorkspaceTool]) + useEffect(() => { + return rpc.subscribe('browser.updated', {}, (payload) => { + const next = payload as BrowserTabState + if (next.reveal !== true) return + leaveWorkspaceTool() + setFullPage(null) + setInspector('browser') + }) + }, [leaveWorkspaceTool]) + // Switching chats must not kill a running shell; every other rail still // yields to the session view the way it always has. const clearTransientInspector = useCallback(() => { - setInspector((prev) => (prev === 'terminal' ? prev : null)) + setInspector((prev) => (prev === 'terminal' || prev === 'browser' ? prev : null)) }, []) // Visiting a session lands on it and clears its settled badge — done/error @@ -391,6 +408,11 @@ function Shell() { setSettingsOpen(false) setFullPage(null) setInspector('terminal') + } else if (view === 'browser') { + setHubOpen(false) + setSettingsOpen(false) + setFullPage(null) + setInspector('browser') } else { setHubOpen(false) setSettingsOpen(false) @@ -1075,6 +1097,12 @@ function Shell() { /> + ) : inspector === 'browser' ? ( +
+ + setInspector(null)} /> + +
) : ( <>
diff --git a/apps/desktop/src/renderer/src/features/browser/BrowserPanel.test.tsx b/apps/desktop/src/renderer/src/features/browser/BrowserPanel.test.tsx new file mode 100644 index 00000000..98ad73ce --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/BrowserPanel.test.tsx @@ -0,0 +1,69 @@ +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { BrowserPanel } from './BrowserPanel' + +const { invokeFn, subscribeFn } = vi.hoisted(() => ({ + invokeFn: vi.fn(), + subscribeFn: vi.fn(() => () => undefined), +})) + +vi.mock('../../lib/rpc', () => ({ + rpc: { + invoke: invokeFn, + subscribe: subscribeFn, + }, +})) + +describe('BrowserPanel', () => { + beforeEach(() => { + invokeFn.mockReset() + subscribeFn.mockReset() + subscribeFn.mockReturnValue(() => undefined) + invokeFn.mockImplementation(async (method: string) => { + if (method === 'browser.open') { + return { + id: 'inspector', + url: 'about:blank', + title: '', + canGoBack: false, + canGoForward: false, + loading: false, + error: null, + } + } + if (method === 'browser.navigate') return { ok: true, tab: { id: 'inspector' } } + if (method === 'browser.go') return { id: 'inspector' } + if (method === 'browser.layout') return { applied: true } + if (method === 'browser.cancelPick') return { cancelled: true } + if (method === 'browser.pick') return { ok: false, error: 'cancelled' } + if (method === 'shell.openUrl') return { opened: true } + throw new Error(`unexpected method: ${method}`) + }) + }) + + it('opens a guest tab and navigates from the address bar', async () => { + const user = userEvent.setup() + render() + + expect(await screen.findByLabelText('Address')).toBeInTheDocument() + expect(screen.getByLabelText('Pick element for agent')).toBeDisabled() + expect(invokeFn).toHaveBeenCalledWith('browser.open', { id: 'inspector' }) + + await user.type(screen.getByLabelText('Address'), 'example.com') + await user.keyboard('{Enter}') + expect(invokeFn).toHaveBeenCalledWith('browser.navigate', { + id: 'inspector', + url: 'example.com', + }) + }) + + it('hides the guest when the panel unmounts', () => { + const { unmount } = render() + unmount() + expect(invokeFn).toHaveBeenCalledWith( + 'browser.layout', + expect.objectContaining({ id: 'inspector', visible: false }), + ) + }) +}) diff --git a/apps/desktop/src/renderer/src/features/browser/BrowserPanel.tsx b/apps/desktop/src/renderer/src/features/browser/BrowserPanel.tsx new file mode 100644 index 00000000..5eb44420 --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/BrowserPanel.tsx @@ -0,0 +1,222 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { + ArrowLeft, + ArrowRight, + ExternalLink, + Globe, + MousePointer2, + RotateCw, + X, +} from 'lucide-react' +import { IconButton } from '@ari/ui/icon-button' +import { Input } from '@ari/ui/input' +import type { BrowserTabState } from '@ari/contracts/rpc' +import { rpc } from '../../lib/rpc' +import { addBrowserPick, fileFromPngBase64 } from './browser-picks' + +const TAB_ID = 'inspector' + +const EMPTY: BrowserTabState = { + id: TAB_ID, + url: 'about:blank', + title: '', + canGoBack: false, + canGoForward: false, + loading: false, + error: null, +} + +/** + * Inspector-rail browser: address chrome in the renderer, Chromium guest in + * the main process. The host div is only a bounds target — the page itself + * is a WebContentsView overlaid by main. + */ +export function BrowserPanel({ onClose }: { onClose?: () => void }) { + const hostRef = useRef(null) + const [tab, setTab] = useState(EMPTY) + const [draft, setDraft] = useState('') + const [picking, setPicking] = useState(false) + + const syncLayout = useCallback((visible: boolean) => { + const host = hostRef.current + if (host === null) return + const rect = host.getBoundingClientRect() + void rpc + .invoke('browser.layout', { + id: TAB_ID, + visible, + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.round(rect.width), + height: Math.round(rect.height), + }) + .catch(() => undefined) + }, []) + + useEffect(() => { + void rpc + .invoke('browser.open', { id: TAB_ID }) + .then((next) => { + setTab(next) + if (next.url !== 'about:blank') setDraft(next.url) + }) + .catch(() => undefined) + const unsub = rpc.subscribe('browser.updated', { id: TAB_ID }, (payload) => { + const next = payload as BrowserTabState + if (next.id !== TAB_ID) return + setTab(next) + if (!next.loading) setDraft(next.url === 'about:blank' ? '' : next.url) + }) + return () => { + unsub() + void rpc.invoke('browser.cancelPick', { id: TAB_ID }) + void rpc.invoke('browser.layout', { + id: TAB_ID, + visible: false, + x: 0, + y: 0, + width: 0, + height: 0, + }) + } + }, []) + + const showGuest = tab.url !== 'about:blank' || tab.loading + + useEffect(() => { + const host = hostRef.current + if (host === null) return + const observer = new ResizeObserver(() => syncLayout(showGuest)) + observer.observe(host) + syncLayout(showGuest) + return () => observer.disconnect() + }, [syncLayout, showGuest]) + + const submit = (): void => { + void rpc + .invoke('browser.navigate', { id: TAB_ID, url: draft }) + .then((result) => { + if (!result.ok) setTab((prev) => ({ ...prev, error: result.error })) + }) + .catch(() => undefined) + } + + const go = (action: 'back' | 'forward' | 'reload'): void => { + void rpc.invoke('browser.go', { id: TAB_ID, action }).catch(() => undefined) + } + + const blank = !showGuest + + const openExternal = (): void => { + if (tab.url === 'about:blank') return + void rpc.invoke('shell.openUrl', { url: tab.url }).catch(() => undefined) + } + + const pickElement = (): void => { + if (blank || picking) return + setPicking(true) + void rpc + .invoke('browser.pick', { id: TAB_ID }) + .then((result) => { + if (result.ok) { + const image = + result.pngBase64 !== null + ? fileFromPngBase64(result.pngBase64, `${result.element.tag}.png`) + : null + addBrowserPick(result.element, image) + } + }) + .catch(() => undefined) + .finally(() => setPicking(false)) + } + + return ( +
+
+ + Browser +
+ {onClose !== undefined ? ( + } + aria-label="Close browser panel" + size="sm" + variant="ghost" + onClick={onClose} + /> + ) : null} +
+
+ } + aria-label="Back" + size="sm" + variant="ghost" + disabled={!tab.canGoBack} + onClick={() => go('back')} + /> + } + aria-label="Forward" + size="sm" + variant="ghost" + disabled={!tab.canGoForward} + onClick={() => go('forward')} + /> + } + aria-label="Reload" + size="sm" + variant="ghost" + onClick={() => go('reload')} + /> +
{ + event.preventDefault() + submit() + }} + > + setDraft(event.target.value)} + placeholder="Enter a URL" + aria-label="Address" + className="h-7 text-2xs" + invalid={tab.error !== null} + /> +
+ } + aria-label="Pick element for agent" + title="Pick an element to mention to the agent" + size="sm" + variant="ghost" + disabled={blank || picking} + onClick={pickElement} + /> + } + aria-label="Open in system browser" + size="sm" + variant="ghost" + disabled={blank} + onClick={openExternal} + /> +
+ {tab.error !== null ? ( +

{tab.error}

+ ) : null} +
+ {blank ? ( +
+

In-app browser

+

+ Type a URL, then pick an element to mention it to the agent. +

+
+ ) : null} +
+
+ ) +} diff --git a/apps/desktop/src/renderer/src/features/browser/ElementChips.tsx b/apps/desktop/src/renderer/src/features/browser/ElementChips.tsx new file mode 100644 index 00000000..9d8300f1 --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/ElementChips.tsx @@ -0,0 +1,34 @@ +import { MousePointer2, X } from 'lucide-react' +import { chipLabelFor, removeBrowserPick, type BrowserPick } from './browser-picks' + +export function ElementChips({ picks }: { picks: readonly BrowserPick[] }) { + if (picks.length === 0) return null + return ( +
+ + {picks.length} element{picks.length > 1 ? 's' : ''} with your next message: + + {picks.map((pick) => ( + + + {chipLabelFor(pick)} + + + ))} +
+ ) +} diff --git a/apps/desktop/src/renderer/src/features/browser/browser-picks.test.ts b/apps/desktop/src/renderer/src/features/browser/browser-picks.test.ts new file mode 100644 index 00000000..4783316f --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/browser-picks.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest' +import { addBrowserPick, promptForPicks, takeBrowserPicks } from './browser-picks' +import type { PickedElement } from '@ari/contracts/rpc' + +const el: PickedElement = { + url: 'https://example.com/', + selector: 'h1', + tag: 'h1', + text: 'Hello', + role: null, + ariaLabel: null, + html: '

Hello

', + x: 0, + y: 0, + width: 10, + height: 10, +} + +describe('browser-picks', () => { + it('formats taken picks as element_context for the agent', () => { + takeBrowserPicks() + addBrowserPick(el, null) + const taken = takeBrowserPicks() + expect(taken).toHaveLength(1) + expect(promptForPicks(taken)).toContain('') + expect(promptForPicks(taken)).toContain('h1') + expect(takeBrowserPicks()).toHaveLength(0) + }) +}) diff --git a/apps/desktop/src/renderer/src/features/browser/browser-picks.ts b/apps/desktop/src/renderer/src/features/browser/browser-picks.ts new file mode 100644 index 00000000..ae29e357 --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/browser-picks.ts @@ -0,0 +1,64 @@ +import type { PickedElement } from '@ari/contracts/rpc' +import { formatElementContexts, elementChipLabel } from './element-context' + +export interface BrowserPick { + id: string + element: PickedElement + image: File | null +} + +let picks: BrowserPick[] = [] +const listeners = new Set<() => void>() +let seq = 0 + +function emit(): void { + for (const listener of listeners) listener() +} + +export function subscribeBrowserPicks(listener: () => void): () => void { + listeners.add(listener) + return () => listeners.delete(listener) +} + +export function browserPicksState(): readonly BrowserPick[] { + return picks +} + +export function addBrowserPick(element: PickedElement, image: File | null): BrowserPick { + const pick: BrowserPick = { id: `el_${++seq}`, element, image } + picks = [...picks, pick] + emit() + return pick +} + +export function removeBrowserPick(id: string): void { + picks = picks.filter((pick) => pick.id !== id) + emit() +} + +export function takeBrowserPicks(): BrowserPick[] { + const taken = picks + picks = [] + emit() + return taken +} + +export function restoreBrowserPicks(next: readonly BrowserPick[]): void { + picks = [...next] + emit() +} + +export function promptForPicks(items: readonly BrowserPick[]): string { + return formatElementContexts(items.map((pick) => pick.element)) +} + +export function chipLabelFor(pick: BrowserPick): string { + return elementChipLabel(pick.element) +} + +export function fileFromPngBase64(base64: string, name: string): File { + const binary = atob(base64) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return new File([bytes], name, { type: 'image/png' }) +} diff --git a/apps/desktop/src/renderer/src/features/browser/element-context.ts b/apps/desktop/src/renderer/src/features/browser/element-context.ts new file mode 100644 index 00000000..8ae9ab6d --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/element-context.ts @@ -0,0 +1,32 @@ +import type { PickedElement } from '@ari/contracts/rpc' + +const TEXT_CAP = 400 +const HTML_CAP = 1_500 + +export function formatElementContext(element: PickedElement): string { + const lines = [ + `Selected element on ${element.url}`, + `- tag: ${element.tag}`, + `- selector: ${element.selector}`, + ] + if (element.role) lines.push(`- role: ${element.role}`) + if (element.ariaLabel) lines.push(`- aria-label: ${element.ariaLabel}`) + if (element.text) lines.push(`- text: ${element.text.slice(0, TEXT_CAP)}`) + if (element.html) lines.push(`- html: ${element.html.slice(0, HTML_CAP)}`) + return lines.join('\n') +} + +export function formatElementContexts(elements: readonly PickedElement[]): string { + if (elements.length === 0) return '' + const body = elements + .map((el, i) => `Element ${String(i + 1)}\n${formatElementContext(el)}`) + .join('\n\n') + return `\n${body}\n` +} + +export function elementChipLabel(element: PickedElement): string { + const text = element.text.trim().replace(/\s+/g, ' ') + if (text.length > 0) return text.length > 32 ? `${text.slice(0, 31)}…` : text + if (element.ariaLabel) return element.ariaLabel + return element.selector +} diff --git a/apps/desktop/src/renderer/src/features/palette/useCommands.test.ts b/apps/desktop/src/renderer/src/features/palette/useCommands.test.ts index a7e26192..06e9fe42 100644 --- a/apps/desktop/src/renderer/src/features/palette/useCommands.test.ts +++ b/apps/desktop/src/renderer/src/features/palette/useCommands.test.ts @@ -20,6 +20,7 @@ describe('buildAppCommands', () => { for (const view of [ 'sessions', 'terminal', + 'browser', 'changes', 'settings', 'files', diff --git a/apps/desktop/src/renderer/src/features/palette/useCommands.ts b/apps/desktop/src/renderer/src/features/palette/useCommands.ts index 01a5472f..fa5db950 100644 --- a/apps/desktop/src/renderer/src/features/palette/useCommands.ts +++ b/apps/desktop/src/renderer/src/features/palette/useCommands.ts @@ -4,6 +4,7 @@ import { Folder, Gauge, GitPullRequest, + Globe, Images, Maximize2, MessageSquare, @@ -35,6 +36,7 @@ export interface PaletteCommand { export type NavigableView = | 'sessions' | 'terminal' + | 'browser' | 'changes' | 'settings' | 'files' @@ -123,6 +125,12 @@ export function buildAppCommands(ctx: CommandsContext): PaletteCommand[] { hint: 'Ctrl+`', run: () => ctx.onNavigate('terminal'), }, + { + id: 'nav.browser', + label: 'Go to Browser', + icon: Globe, + run: () => ctx.onNavigate('browser'), + }, { id: 'nav.changes', label: 'Go to Changes', diff --git a/apps/desktop/src/renderer/src/features/session/SessionView.tsx b/apps/desktop/src/renderer/src/features/session/SessionView.tsx index 10edc83e..4efadd4a 100644 --- a/apps/desktop/src/renderer/src/features/session/SessionView.tsx +++ b/apps/desktop/src/renderer/src/features/session/SessionView.tsx @@ -1,4 +1,4 @@ -import { useCallback, useEffect, useMemo, useRef, useState } from 'react' +import { useCallback, useEffect, useMemo, useRef, useState, useSyncExternalStore } from 'react' import { ChildSessionActivity } from './ChildSessionActivity' import type { SessionActivity } from './session-activity' import { Check, ChevronDown, X } from 'lucide-react' @@ -24,6 +24,14 @@ import { useEngineSettings } from '../settings/useEngineSettings' import { PlanPanel } from './PlanPanel' import { SessionBranchChip } from './SessionBranchChip' import { TurnErrorBanner } from './TurnErrorBanner' +import { ElementChips } from '../browser/ElementChips' +import { + browserPicksState, + promptForPicks, + restoreBrowserPicks, + subscribeBrowserPicks, + takeBrowserPicks, +} from '../browser/browser-picks' interface PendingApproval { approvalId: string @@ -222,6 +230,7 @@ export function SessionView({ const [reviewNotes, setReviewNotes] = useState< { path: string; line: number | null; text: string }[] >([]) + const elementPicks = useSyncExternalStore(subscribeBrowserPicks, browserPicksState) const sessionTitleRef = useRef('Session') const fetchedTurnIdsRef = useRef(new Set()) const fetchTurnDiffRef = useRef<(turnId: string) => void>(() => {}) @@ -544,18 +553,26 @@ export function SessionView({ // defer an updater, and one that has not run yet would leave `outgoing` // without the notes this dispatch is about to send. const notes = reviewNotes - const outgoing = + const picks = takeBrowserPicks() + const elementBlock = promptForPicks(picks) + const outgoing = [ + elementBlock, notes.length > 0 ? `Review notes on your changes:\n${notes .map((n) => `- ${n.path}${n.line !== null ? `:${n.line}` : ''} — ${n.text}`) - .join('\n')}\n\n${text}` - : text + .join('\n')}` + : '', + text, + ] + .filter((part) => part.length > 0) + .join('\n\n') if (notes.length > 0) setReviewNotes([]) // A rejected send must cost a retry, not the message: the composer // already cleared its draft — text, images and the notes above — so put // all of it back and the retry carries the same context. const restoreDraft = (): void => { if (notes.length > 0) setReviewNotes((prev) => (prev.length === 0 ? notes : prev)) + if (picks.length > 0) restoreBrowserPicks(picks) setComposerSeed((prev) => ({ text, files, nonce: (prev?.nonce ?? 0) + 1 })) } if (running) { @@ -582,19 +599,23 @@ export function SessionView({ const handleSend = useCallback( (text: string, files: File[]) => { + const pickImages = browserPicksState() + .map((pick) => pick.image) + .filter((file): file is File => file !== null) + const allFiles = [...pickImages, ...files] // Staging is async; imageless sends skip it and dispatch synchronously. - if (files.length === 0) { + if (allFiles.length === 0) { dispatchSend(text, [], []) return } - void stageImages(files).then( - (attachments) => dispatchSend(text, attachments, files), + void stageImages(allFiles).then( + (attachments) => dispatchSend(text, attachments, allFiles), (err: unknown) => { // The composer already cleared: restore the draft so the failure // costs a retry, not the message, and never send text-only behind // images the user explicitly attached. The notes were never spent — // no dispatch happened — so they are still in state. - setComposerSeed((prev) => ({ text, files, nonce: (prev?.nonce ?? 0) + 1 })) + setComposerSeed((prev) => ({ text, files: allFiles, nonce: (prev?.nonce ?? 0) + 1 })) toast({ title: 'Couldn’t attach images', description: err instanceof Error ? err.message : String(err), @@ -883,6 +904,7 @@ export function SessionView({ onDismiss={() => setTurnError(null)} /> ) : null} + {reviewNotes.length > 0 ? (
McpServerConfig[]) /** Connection seam for tests; production connects over real stdio. */ mcpConnect?: (server: McpServerConfig) => Promise /** @@ -305,7 +305,7 @@ export class AriCoreDriver implements Driver { readonly #clients: AriCoreDriverClients readonly #contextCharLimit: number readonly #allowlist: AllowRule[] | undefined - readonly #mcpServers: McpServerConfig[] + readonly #mcpServers: McpServerConfig[] | (() => McpServerConfig[]) readonly #mcpConnectOverride?: (server: McpServerConfig) => Promise readonly #conversations: ConversationStore readonly #compaction: boolean @@ -345,7 +345,9 @@ export class AriCoreDriver implements Driver { const allowlist = this.#allowlist const conversations = this.#conversations const compaction = this.#compaction - const mcpServers = this.#mcpServers.filter((s) => !s.disabled) + const configured = + typeof this.#mcpServers === 'function' ? this.#mcpServers() : this.#mcpServers + const mcpServers = configured.filter((s) => !s.disabled) const mcpConnect = this.#mcpConnectOverride ?? ((server: McpServerConfig) => diff --git a/packages/contracts/src/contracts.test.ts b/packages/contracts/src/contracts.test.ts index 08e22b9d..9a73b621 100644 --- a/packages/contracts/src/contracts.test.ts +++ b/packages/contracts/src/contracts.test.ts @@ -241,6 +241,22 @@ describe('contracts', () => { }) }) + it('validates in-app browser RPC params', () => { + expect(rpcParams['browser.open'].parse({ id: 'inspector' })).toEqual({ id: 'inspector' }) + expect( + rpcParams['browser.navigate'].parse({ id: 'inspector', url: 'https://example.com' }), + ).toEqual({ + id: 'inspector', + url: 'https://example.com', + }) + expect(rpcParams['browser.go'].parse({ id: 'inspector', action: 'back' })).toEqual({ + id: 'inspector', + action: 'back', + }) + expect(() => rpcParams['browser.navigate'].parse({ id: 'inspector', url: '' })).toThrow() + expect(() => rpcParams['browser.go'].parse({ id: 'inspector', action: 'jump' })).toThrow() + }) + it('validates fs.writeTextFile scope params and rejects malformed payloads', () => { const params = { projectId: 'proj_1', path: 'src/main.ts', content: 'export {}\n' } expect(rpcParams['fs.writeTextFile'].parse(params)).toEqual(params) diff --git a/packages/contracts/src/rpc.ts b/packages/contracts/src/rpc.ts index 5760e96a..8fe537a7 100644 --- a/packages/contracts/src/rpc.ts +++ b/packages/contracts/src/rpc.ts @@ -187,12 +187,42 @@ export const usageSummarySchema = z.object({ }) export type UsageSummary = z.infer +/** Live state of one in-app browser tab. */ +export const browserTabStateSchema = z.object({ + id: z.string().min(1), + url: z.string(), + title: z.string(), + canGoBack: z.boolean(), + canGoForward: z.boolean(), + loading: z.boolean(), + error: z.string().nullable(), + reveal: z.boolean().optional(), +}) +export type BrowserTabState = z.infer + +/** One DOM node picked from the in-app browser for the next agent turn. */ +export const pickedElementSchema = z.object({ + url: z.string(), + selector: z.string(), + tag: z.string(), + text: z.string(), + role: z.string().nullable(), + ariaLabel: z.string().nullable(), + html: z.string(), + x: z.number(), + y: z.number(), + width: z.number(), + height: z.number(), +}) +export type PickedElement = z.infer + /** Stream names the renderer may subscribe to. */ export const streamNames = [ 'session.events', 'terminal.data', 'providers.updates', 'app.updates', + 'browser.updated', ] as const export type StreamName = (typeof streamNames)[number] @@ -488,6 +518,23 @@ export const rpcParams = { rows: z.number().int().positive(), }), 'terminal.kill': z.object({ id: z.string().min(1) }), + 'browser.open': z.object({ id: z.string().min(1), url: z.string().max(2048).optional() }), + 'browser.navigate': z.object({ id: z.string().min(1), url: z.string().min(1).max(2048) }), + 'browser.go': z.object({ + id: z.string().min(1), + action: z.enum(['back', 'forward', 'reload']), + }), + 'browser.close': z.object({ id: z.string().min(1) }), + 'browser.layout': z.object({ + id: z.string().min(1), + visible: z.boolean(), + x: z.number(), + y: z.number(), + width: z.number().nonnegative(), + height: z.number().nonnegative(), + }), + 'browser.pick': z.object({ id: z.string().min(1) }), + 'browser.cancelPick': z.object({ id: z.string().min(1) }), 'project.list': z.undefined(), 'project.add': z.object({ path: z.string().min(1), name: z.string().optional() }), 'project.open': z.object({ path: z.string().min(1), name: z.string().optional() }), @@ -781,6 +828,14 @@ export interface RpcResults { 'terminal.write': { written: boolean } 'terminal.resize': { resized: boolean } 'terminal.kill': { killed: boolean } + 'browser.open': BrowserTabState + 'browser.navigate': { ok: true; tab: BrowserTabState } | { ok: false; error: string } + 'browser.go': BrowserTabState + 'browser.close': { closed: boolean } + 'browser.layout': { applied: boolean } + 'browser.pick': + { ok: true; element: PickedElement; pngBase64: string | null } | { ok: false; error: string } + 'browser.cancelPick': { cancelled: boolean } 'project.list': Project[] 'project.add': Project 'project.open': Project diff --git a/packages/providers/src/acp/acp-driver.ts b/packages/providers/src/acp/acp-driver.ts index ab28b3ae..245c44ec 100644 --- a/packages/providers/src/acp/acp-driver.ts +++ b/packages/providers/src/acp/acp-driver.ts @@ -4,7 +4,7 @@ import { createLogger } from '@ari/shared/logger' import { formatUnknownError } from '@ari/shared/result' import type { AdapterSession, Driver, ProviderAdapter } from '../driver' import { AcpAuthRequiredError, AcpConnection, AcpConnectionError } from './connection' -import type { AcpChildProcess, AcpLaunch } from './connection' +import type { AcpChildProcess, AcpLaunch, AcpMcpServer } from './connection' import { encodeQuestionnaire, isAskUserQuestionMethod, @@ -90,6 +90,7 @@ export async function createAcpAdapter( session: AdapterSession, spawn?: (childLaunch: AcpLaunch, cwd: string) => AcpChildProcess, onAuthRequired?: AcpAuthRequiredHandler, + mcpServers: AcpMcpServer[] = [], ): Promise { const pendingPermissions = new Map< string, @@ -236,6 +237,7 @@ export async function createAcpAdapter( ...(session.runtimeEnv ? { runtimeEnv: session.runtimeEnv } : {}), cwd: session.workspacePath, ...(spawn !== undefined ? { spawn } : {}), + ...(mcpServers.length > 0 ? { mcpServers } : {}), }) } catch (error) { throw setupFailure(error) @@ -933,6 +935,7 @@ export class AcpDriver implements Driver { private readonly fallback: Driver | null, /** Notified whenever the agent refuses for want of a login. */ private readonly onAuthRequired: AcpAuthRequiredHandler | null = null, + private readonly mcpServers: () => AcpMcpServer[] = () => [], ) { this.kind = kind } @@ -945,6 +948,7 @@ export class AcpDriver implements Driver { session, undefined, this.onAuthRequired ?? undefined, + this.mcpServers(), ) log.info('turn started over ACP', { kind: this.kind, launch: this.launch.label }) publishAdvertisedEfforts(this.kind, adapter.advertisedEfforts) diff --git a/packages/providers/src/acp/connection.test.ts b/packages/providers/src/acp/connection.test.ts index 031ca5a7..fbb01223 100644 --- a/packages/providers/src/acp/connection.test.ts +++ b/packages/providers/src/acp/connection.test.ts @@ -1,6 +1,11 @@ import { PassThrough } from 'node:stream' import { describe, expect, it } from 'vitest' -import { AcpAuthRequiredError, AcpConnection, AcpConnectionError, acpPromptStallMs } from './connection' +import { + AcpAuthRequiredError, + AcpConnection, + AcpConnectionError, + acpPromptStallMs, +} from './connection' import type { AcpChildProcess, AcpLaunch } from './connection' const LAUNCH: AcpLaunch = { label: 'test-agent', command: 'fake', args: [] } @@ -65,11 +70,7 @@ function fakeChild(): FakeChild { return child } -type AgentHandler = ( - method: string | undefined, - params: unknown, - id: number | undefined, -) => unknown +type AgentHandler = (method: string | undefined, params: unknown, id: number | undefined) => unknown /** Sentinel: the scripted agent never replies to this request (wedge sim). */ export const NO_REPLY = Symbol('no-reply') @@ -117,7 +118,8 @@ function script(child: FakeChild, handler: AgentHandler): void { } const STANDARD_AGENT: AgentHandler = (method) => { - if (method === 'initialize') return { protocolVersion: 1, agentInfo: { name: 'TestAgent', version: '1.2.3' } } + if (method === 'initialize') + return { protocolVersion: 1, agentInfo: { name: 'TestAgent', version: '1.2.3' } } if (method === 'session/new') return { sessionId: 'sess_9' } if (method === 'session/prompt') return { stopReason: 'end_turn' } return undefined @@ -130,11 +132,14 @@ async function drain(ms = 15): Promise { describe('AcpConnection', () => { it('bounds a usage prompt even when the agent never answers', async () => { const child = fakeChild() - script(child, (method, params, id) => method === 'session/prompt' ? NO_REPLY : STANDARD_AGENT(method, params, id)) + script(child, (method, params, id) => + method === 'session/prompt' ? NO_REPLY : STANDARD_AGENT(method, params, id), + ) const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/', spawn: () => child }) try { - await expect(connection.prompt('probe', '/usage', { timeoutMs: 25, stallSilenceMs: 0 })) - .rejects.toThrow('timed out after 25ms') + await expect( + connection.prompt('probe', '/usage', { timeoutMs: 25, stallSilenceMs: 0 }), + ).rejects.toThrow('timed out after 25ms') } finally { await connection.shutdown() } @@ -142,10 +147,16 @@ describe('AcpConnection', () => { it('reads namespaced billing extensions through the response multiplexer', async () => { const child = fakeChild() - script(child, (method, params, id) => method === '_x.ai/billing' ? { config: { creditUsagePercent: 23 } } : STANDARD_AGENT(method, params, id)) + script(child, (method, params, id) => + method === '_x.ai/billing' + ? { config: { creditUsagePercent: 23 } } + : STANDARD_AGENT(method, params, id), + ) const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/', spawn: () => child }) try { - await expect(connection.requestExtension('_x.ai/billing', {})).resolves.toEqual({ config: { creditUsagePercent: 23 } }) + await expect(connection.requestExtension('_x.ai/billing', {})).resolves.toEqual({ + config: { creditUsagePercent: 23 }, + }) } finally { await connection.shutdown() } @@ -158,7 +169,9 @@ describe('AcpConnection', () => { label: 'environment fixture', command: process.execPath, env: { CODEX_PATH: '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/fixture/CLI with spaces/codex' }, - args: ['-e', ` + args: [ + '-e', + ` require('node:readline').createInterface({ input: process.stdin }).on('line', (line) => { const message = JSON.parse(line); if (message.method === 'initialize') { @@ -168,12 +181,15 @@ describe('AcpConnection', () => { } })); } }); - `], + `, + ], }, }) try { expect(connection.initialize.agentInfo?.name).toBe('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/fixture/CLI with spaces/codex') - expect(connection.initialize.agentInfo?.version).toBe(process.env['PATH'] ?? process.env['Path']) + expect(connection.initialize.agentInfo?.version).toBe( + process.env['PATH'] ?? process.env['Path'], + ) } finally { await connection.shutdown() } @@ -182,7 +198,11 @@ describe('AcpConnection', () => { it('completes the initialize handshake and exposes agent info', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) expect(connection.initialize.agentInfo?.name).toBe('TestAgent') const created = await connection.newSession('/w') @@ -193,16 +213,155 @@ describe('AcpConnection', () => { expect(child.killed).toBe(true) }) + it('hands HTTP MCP servers to session/new unless the agent opted out', async () => { + const httpServer = { + type: 'http' as const, + name: 'ari-browser', + url: 'http://127.0.0.1:9/mcp', + headers: [{ name: 'Authorization', value: 'Bearer t' }], + } + const withHttp = fakeChild() + script(withHttp, (method) => { + if (method === 'initialize') { + return { + protocolVersion: 1, + agentCapabilities: { mcpCapabilities: { http: true } }, + } + } + if (method === 'session/new') return { sessionId: 'sess_mcp' } + return undefined + }) + const httpConn = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => withHttp, + mcpServers: [httpServer], + }) + await httpConn.newSession('/w') + const created = withHttp.sent.find((m) => m['method'] === 'session/new') as + { params?: { mcpServers?: unknown[] } } | undefined + expect(created?.params?.mcpServers).toEqual([httpServer]) + httpConn.kill() + + const implied = fakeChild() + script(implied, STANDARD_AGENT) + const impliedConn = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => implied, + mcpServers: [httpServer], + }) + await impliedConn.newSession('/w') + const impliedNew = implied.sent.find((m) => m['method'] === 'session/new') as + { params?: { mcpServers?: unknown[] } } | undefined + // No `mcpCapabilities.http` means the agent did not advertise HTTP. + expect(impliedNew?.params?.mcpServers).toEqual([]) + impliedConn.kill() + + const optedOut = fakeChild() + script(optedOut, (method) => { + if (method === 'initialize') { + return { protocolVersion: 1, agentCapabilities: { mcpCapabilities: { http: false } } } + } + if (method === 'session/new') return { sessionId: 'sess_no_http' } + return undefined + }) + const opted = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => optedOut, + mcpServers: [httpServer], + }) + await opted.newSession('/w') + const skipped = optedOut.sent.find((m) => m['method'] === 'session/new') as + { params?: { mcpServers?: unknown[] } } | undefined + expect(skipped?.params?.mcpServers).toEqual([]) + opted.kill() + }) + + it('always forwards stdio MCP servers so the in-app browser reaches every agent', async () => { + const stdioServer = { + name: 'ari-browser', + command: '/bin/proxy', + args: ['/tmp/proxy.mjs'], + env: [{ name: 'ARI_BROWSER_MCP_URL', value: 'http://127.0.0.1:9/mcp' }], + } + const child = fakeChild() + script(child, STANDARD_AGENT) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + mcpServers: [stdioServer], + }) + await connection.newSession('/w') + const created = child.sent.find((m) => m['method'] === 'session/new') as + { params?: { mcpServers?: unknown[] } } | undefined + expect(created?.params?.mcpServers).toEqual([stdioServer]) + connection.kill() + }) + + it('sends HTTP MCP only when advertised, and stdio otherwise', async () => { + const httpServer = { + type: 'http' as const, + name: 'ari-browser', + url: 'http://127.0.0.1:9/mcp', + headers: [{ name: 'Authorization', value: 'Bearer t' }], + } + const stdioServer = { + name: 'ari-browser', + command: '/bin/proxy', + args: ['/tmp/proxy.mjs'], + } + const both = [httpServer, stdioServer] + const httpChild = fakeChild() + script(httpChild, (method) => { + if (method === 'initialize') { + return { protocolVersion: 1, agentCapabilities: { mcpCapabilities: { http: true } } } + } + if (method === 'session/new') return { sessionId: 'sess_http' } + return undefined + }) + const httpConn = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => httpChild, + mcpServers: both, + }) + await httpConn.newSession('/w') + const httpNew = httpChild.sent.find((m) => m['method'] === 'session/new') as + { params?: { mcpServers?: unknown[] } } | undefined + expect(httpNew?.params?.mcpServers).toEqual([httpServer]) + httpConn.kill() + + const stdioChild = fakeChild() + script(stdioChild, STANDARD_AGENT) + const stdioConn = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => stdioChild, + mcpServers: both, + }) + await stdioConn.newSession('/w') + const stdioNew = stdioChild.sent.find((m) => m['method'] === 'session/new') as + { params?: { mcpServers?: unknown[] } } | undefined + expect(stdioNew?.params?.mcpServers).toEqual([stdioServer]) + stdioConn.kill() + }) + it('sends prompts as text content blocks and resolves the stop reason', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const created = await connection.newSession('/w') const stopReason = await connection.prompt(created.sessionId as string, 'do things') expect(stopReason).toBe('end_turn') const promptRequest = child.sent.find((m) => m['method'] === 'session/prompt') as - | { params?: { prompt?: { type: string; text: string }[] } } - | undefined + { params?: { prompt?: { type: string; text: string }[] } } | undefined expect(promptRequest?.params?.prompt?.[0]).toEqual({ type: 'text', text: 'do things' }) connection.kill() }) @@ -210,14 +369,20 @@ describe('AcpConnection', () => { it('sends staged images as image blocks after the text', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const created = await connection.newSession('/w') const stopReason = await connection.prompt(created.sessionId as string, 'look', { images: [{ data: 'aGk=', mimeType: 'image/png' }], }) expect(stopReason).toBe('end_turn') const promptRequest = child.sent.find((m) => m['method'] === 'session/prompt') as - | { params?: { prompt?: { type: string; text?: string; data?: string; mimeType?: string }[] } } + | { + params?: { prompt?: { type: string; text?: string; data?: string; mimeType?: string }[] } + } | undefined expect(promptRequest?.params?.prompt).toEqual([ { type: 'text', text: 'look' }, @@ -230,17 +395,21 @@ describe('AcpConnection', () => { // Spec: session/load's response body is null; the agent re-attaches the id. const child = fakeChild() script(child, (method) => { - if (method === 'initialize') return { protocolVersion: 1, agentCapabilities: { loadSession: true } } + if (method === 'initialize') + return { protocolVersion: 1, agentCapabilities: { loadSession: true } } if (method === 'session/load') return null return undefined }) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const resumed = await connection.loadSession('sess_old', '/next') expect(resumed.sessionId).toBe('sess_old') const load = child.sent.find((m) => m['method'] === 'session/load') as - | { params?: Record } - | undefined + { params?: Record } | undefined expect(load?.params).toEqual({ sessionId: 'sess_old', cwd: '/next', mcpServers: [] }) connection.kill() }) @@ -248,7 +417,11 @@ describe('AcpConnection', () => { it('routes session/update notifications to the hook', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const seen: unknown[] = [] connection.onSessionUpdate = (notification) => seen.push(notification) child.stdout.write( @@ -269,7 +442,11 @@ describe('AcpConnection', () => { it('bridges server permission requests through the handler', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) connection.onRequestPermission = async () => ({ outcome: { outcome: 'selected', optionId: 'allow_once' }, }) @@ -278,20 +455,33 @@ describe('AcpConnection', () => { jsonrpc: '2.0', id: 42, method: 'session/request_permission', - params: { sessionId: 's', options: [{ optionId: 'allow_once', name: 'Allow', kind: 'allow_once' }] }, + params: { + sessionId: 's', + options: [{ optionId: 'allow_once', name: 'Allow', kind: 'allow_once' }], + }, })}\n`, ) await drain() const reply = child.sent.find((m) => m['id'] === 42 && m['method'] === undefined) - expect(reply).toMatchObject({ id: 42, result: { outcome: { outcome: 'selected', optionId: 'allow_once' } } }) + expect(reply).toMatchObject({ + id: 42, + result: { outcome: { outcome: 'selected', optionId: 'allow_once' } }, + }) connection.kill() }) it('bridges elicitation/create through onClientRequest as a JSON-RPC success', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) - connection.onClientRequest = async () => ({ action: 'accept', content: { strategy: 'balanced' } }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) + connection.onClientRequest = async () => ({ + action: 'accept', + content: { strategy: 'balanced' }, + }) child.stdout.write( `${JSON.stringify({ jsonrpc: '2.0', @@ -318,12 +508,15 @@ describe('AcpConnection', () => { if (method === 'session/resume') return null return undefined }) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const resumed = await connection.resumeSession('sess_old', '/next') expect(resumed.sessionId).toBe('sess_old') const resume = child.sent.find((m) => m['method'] === 'session/resume') as - | { params?: Record } - | undefined + { params?: Record } | undefined expect(resume?.params).toEqual({ sessionId: 'sess_old', cwd: '/next', mcpServers: [] }) connection.kill() }) @@ -331,7 +524,11 @@ describe('AcpConnection', () => { it('answers unadvertised client methods with method-not-found', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) child.stdout.write( `${JSON.stringify({ jsonrpc: '2.0', id: 7, method: 'fs/read_text_file', params: { path: '/x' } })}\n`, ) @@ -374,7 +571,11 @@ describe('AcpConnection', () => { ? NO_REPLY : { ok: true }, ) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const created = await connection.newSession('/w') await expect( connection.prompt(created.sessionId as string, 'hello?', { stallSilenceMs: 120 }), @@ -396,7 +597,11 @@ describe('AcpConnection', () => { } return { ok: true } }) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const created = await connection.newSession('/w') let resolvePermission: ((outcome: unknown) => void) | undefined connection.onRequestPermission = () => @@ -414,7 +619,10 @@ describe('AcpConnection', () => { jsonrpc: '2.0', id: 99, method: 'session/request_permission', - params: { sessionId: created.sessionId, options: [{ optionId: 'allow_once', name: 'Allow', kind: 'allow_once' }] }, + params: { + sessionId: created.sessionId, + options: [{ optionId: 'allow_once', name: 'Allow', kind: 'allow_once' }], + }, })}\n`, ) await new Promise((resolve) => setTimeout(resolve, 200)) @@ -449,7 +657,11 @@ describe('AcpConnection', () => { it('advertises the terminal-auth capability so agents offer their logins', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const initialize = child.sent[0] as { params?: { clientCapabilities?: { _meta?: Record } } } @@ -468,7 +680,9 @@ describe('AcpConnection', () => { id: 'claude-ai-login', name: 'Claude Subscription', type: 'terminal', - _meta: { 'terminal-auth': { command: 'node', args: ['acp.js', '--cli', 'auth', 'login'] } }, + _meta: { + 'terminal-auth': { command: 'node', args: ['acp.js', '--cli', 'auth', 'login'] }, + }, }, ], } @@ -481,7 +695,11 @@ describe('AcpConnection', () => { } return undefined }) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) expect(connection.terminalLogins.map((l) => l.methodId)).toEqual(['claude-ai-login']) const failure = await connection.newSession('/w').catch((error: unknown) => error) @@ -505,15 +723,25 @@ describe('AcpConnection', () => { } return undefined }) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) - await expect(connection.newSession('/w')).rejects.toThrow(/not authenticated yet — run its login flow/) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) + await expect(connection.newSession('/w')).rejects.toThrow( + /not authenticated yet — run its login flow/, + ) connection.kill() }) it('inbound traffic proves liveness and disarms the stall watchdog', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) const created = await connection.newSession('/w') // Stream updates every 40ms while the (delayed) answer is pending. const spam = setInterval(() => { @@ -545,7 +773,11 @@ describe('AcpConnection.shutdown', () => { child.stdin.on('end', () => { if (!child.killed) child.kill() }) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) await connection.shutdown() expect(connection.closed).toBe(true) expect(child.signals).toEqual([]) @@ -554,7 +786,11 @@ describe('AcpConnection.shutdown', () => { it('escalates to SIGTERM when the agent ignores the EOF', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) await connection.shutdown() expect(child.signals).toEqual(['SIGTERM']) expect(child.killed).toBe(true) @@ -563,7 +799,11 @@ describe('AcpConnection.shutdown', () => { it('is a no-op on an already-closed connection', async () => { const child = fakeChild() script(child, STANDARD_AGENT) - const connection = await AcpConnection.connect({ launch: LAUNCH, cwd: '/w', spawn: () => child }) + const connection = await AcpConnection.connect({ + launch: LAUNCH, + cwd: '/w', + spawn: () => child, + }) connection.kill() await connection.waitClosed() child.signals.length = 0 diff --git a/packages/providers/src/acp/connection.ts b/packages/providers/src/acp/connection.ts index 0f758e19..e6cdb632 100644 --- a/packages/providers/src/acp/connection.ts +++ b/packages/providers/src/acp/connection.ts @@ -3,11 +3,7 @@ import { createLogger } from '@ari/shared/logger' import { explainExitCode } from '../exit-codes' import { spawnCli } from '../spawn-cli' import { teardownChild } from '../teardown' -import { - AUTH_REQUIRED_ERROR, - describeAcpFailure, - terminalLoginsFrom, -} from './protocol' +import { AUTH_REQUIRED_ERROR, describeAcpFailure, terminalLoginsFrom } from './protocol' import { isInteractiveClientMethod } from './client-requests' import type { AcpInitializeResult, @@ -55,11 +51,28 @@ export interface AcpChildProcess { pid?: number | undefined } +/** MCP server handed to the agent on session/new so it can use client tools. */ +export type AcpMcpServer = + | { + name: string + command: string + args: string[] + env?: { name: string; value: string }[] + } + | { + type: 'http' + name: string + url: string + headers: { name: string; value: string }[] + } + export interface AcpConnectOptions { launch: AcpLaunch cwd: string clientName?: string clientVersion?: string + /** Client-hosted MCP servers (Ari's in-app browser). Empty when none. */ + mcpServers?: AcpMcpServer[] /** Handshake ceiling; provider runtimes can spend time starting on first use. */ initializeTimeoutMs?: number /** Client-side handler for `session/request_permission` server calls. */ @@ -147,6 +160,7 @@ export class AcpConnection { launch: AcpLaunch initialize: AcpInitializeResult + #mcpServers: AcpMcpServer[] /** Hook for `session/update` notifications; assigned by the driver. */ onSessionUpdate: ((notification: AcpSessionNotification) => void) | null = null @@ -174,6 +188,7 @@ export class AcpConnection { this.#child = child this.launch = launch this.initialize = {} + this.#mcpServers = [] this.onRequestPermission = onRequestPermission this.onClientRequest = null this.#closeWaiter = closeWaiter @@ -241,7 +256,12 @@ export class AcpConnection { child.stdout.once('close', () => resolveClose()) }) - const connection = new AcpConnection(child, launch, options.onRequestPermission ?? null, closeWaiter) + const connection = new AcpConnection( + child, + launch, + options.onRequestPermission ?? null, + closeWaiter, + ) child.stderr.on('data', (chunk: string) => { if (connection.#stderrTail.length > 6) connection.#stderrTail.shift() @@ -257,9 +277,12 @@ export class AcpConnection { // this listener they crash the host as unhandled 'error' events. child.on('error', (error: Error) => { log.debug('acp: process error', { label: launch.label, error: error.message }) - connection.#failAllPending(new AcpConnectionError(`${launch.label} process error: ${error.message}`)) + connection.#failAllPending( + new AcpConnectionError(`${launch.label} process error: ${error.message}`), + ) }) + connection.#mcpServers = options.mcpServers ?? [] connection.#wireStdout() connection.#watchExit() @@ -316,11 +339,9 @@ export class AcpConnection { // An auth wall on the handshake itself predates any authMethods, so // there is nothing to offer — but the kind must survive so callers still // route it to the sign-in path rather than a generic transport failure. - if (error instanceof AcpAuthRequiredError) throw new AcpAuthRequiredError(detail, error.logins) - throw new AcpConnectionError( - detail, - error instanceof AcpConnectionError ? error.code : null, - ) + if (error instanceof AcpAuthRequiredError) + throw new AcpAuthRequiredError(detail, error.logins) + throw new AcpConnectionError(detail, error instanceof AcpConnectionError ? error.code : null) } } @@ -418,7 +439,11 @@ export class AcpConnection { if (method === 'session/request_permission' && this.onRequestPermission !== null) { try { const result = await this.onRequestPermission(params as AcpRequestPermission) - this.#write({ jsonrpc: '2.0', id, result: result ?? { outcome: { outcome: 'cancelled' } } }) + this.#write({ + jsonrpc: '2.0', + id, + result: result ?? { outcome: { outcome: 'cancelled' } }, + }) } catch (error) { log.debug('acp: permission handler failed', { error: String(error) }) this.#write({ @@ -468,7 +493,12 @@ export class AcpConnection { return true } - #request(method: string, params: unknown, timeoutMs?: number, stallSilenceMs?: number): Promise { + #request( + method: string, + params: unknown, + timeoutMs?: number, + stallSilenceMs?: number, + ): Promise { if (this.#closed) { return Promise.reject(new AcpConnectionError(`${this.launch.label} connection is closed`)) } @@ -482,7 +512,9 @@ export class AcpConnection { if (timer !== null) clearTimeout(timer) if (stallTimer !== null) clearInterval(stallTimer) reject( - new AcpConnectionError(`${this.launch.label}: ${method} timed out after ${timeoutMs}ms`), + new AcpConnectionError( + `${this.launch.label}: ${method} timed out after ${timeoutMs}ms`, + ), ) }, timeoutMs) timer.unref?.() @@ -492,23 +524,26 @@ export class AcpConnection { // proves liveness; total silence past the ceiling fails the request — // unless the agent is parked on an unanswered server→client request, // where the silence is Ari's user taking their time, not a wedge. - const interval = setInterval(() => { - if (Date.now() - this.#lastInboundAt < stallSilenceMs) return - if (this.#pendingServerRequests > 0) return - if (timer !== null) clearTimeout(timer) - clearInterval(interval) - this.#pending.delete(id) - const quiet = - stallSilenceMs < 1000 - ? `${stallSilenceMs}ms` - : `${Math.round(stallSilenceMs / 1000)}s` - reject( - new AcpConnectionError( - `${this.launch.label} went silent for ${quiet} mid-${method} — ` + - `the agent may be wedged or waiting for login${this.#tailReport()}`, - ), - ) - }, Math.min(2000, Math.max(25, Math.floor(stallSilenceMs / 8)))) + const interval = setInterval( + () => { + if (Date.now() - this.#lastInboundAt < stallSilenceMs) return + if (this.#pendingServerRequests > 0) return + if (timer !== null) clearTimeout(timer) + clearInterval(interval) + this.#pending.delete(id) + const quiet = + stallSilenceMs < 1000 + ? `${stallSilenceMs}ms` + : `${Math.round(stallSilenceMs / 1000)}s` + reject( + new AcpConnectionError( + `${this.launch.label} went silent for ${quiet} mid-${method} — ` + + `the agent may be wedged or waiting for login${this.#tailReport()}`, + ), + ) + }, + Math.min(2000, Math.max(25, Math.floor(stallSilenceMs / 8))), + ) interval.unref?.() stallTimer = interval } @@ -526,9 +561,26 @@ export class AcpConnection { this.#write({ jsonrpc: '2.0', method, params }) } + /** + * Send HTTP MCP only when the agent advertised `mcpCapabilities.http`. + * Omitted means unsupported. Otherwise stdio, which every agent can spawn. + * Never send both — duplicate tool names confuse the model. + */ + #sessionMcpServers(): AcpMcpServer[] { + const allowHttp = this.initialize.agentCapabilities?.mcpCapabilities?.http === true + const http = this.#mcpServers.filter((server) => 'type' in server && server.type === 'http') + const stdio = this.#mcpServers.filter((server) => !('type' in server && server.type === 'http')) + if (allowHttp && http.length > 0) return http + return stdio + } + /** Creates a session bound to `cwd`; throws descriptive errors on auth walls. */ async newSession(cwd: string): Promise { - const result = await this.#request('session/new', { cwd, mcpServers: [] }, 30_000) + const result = await this.#request( + 'session/new', + { cwd, mcpServers: this.#sessionMcpServers() }, + 30_000, + ) const created = (result ?? {}) as AcpNewSessionResult if (typeof created.sessionId !== 'string') { throw new AcpConnectionError(`${this.launch.label} returned no sessionId`) @@ -547,7 +599,11 @@ export class AcpConnection { * only after this promise resolves. */ async loadSession(sessionId: string, cwd: string): Promise { - const result = await this.#request('session/load', { sessionId, cwd, mcpServers: [] }, 60_000) + const result = await this.#request( + 'session/load', + { sessionId, cwd, mcpServers: this.#sessionMcpServers() }, + 60_000, + ) return { ...((result ?? {}) as AcpNewSessionResult), sessionId } } @@ -556,7 +612,11 @@ export class AcpConnection { * {@link loadSession} when the agent advertised `sessionCapabilities.resume`. */ async resumeSession(sessionId: string, cwd: string): Promise { - const result = await this.#request('session/resume', { sessionId, cwd, mcpServers: [] }, 60_000) + const result = await this.#request( + 'session/resume', + { sessionId, cwd, mcpServers: this.#sessionMcpServers() }, + 60_000, + ) return { ...((result ?? {}) as AcpNewSessionResult), sessionId } } @@ -569,7 +629,11 @@ export class AcpConnection { async prompt( sessionId: string, text: string, - options: { images?: { data: string; mimeType: string }[]; stallSilenceMs?: number; timeoutMs?: number } = {}, + options: { + images?: { data: string; mimeType: string }[] + stallSilenceMs?: number + timeoutMs?: number + } = {}, ): Promise { const { images = [], stallSilenceMs = acpPromptStallMs() } = options const blocks: { type: string; text?: string; data?: string; mimeType?: string }[] = [] diff --git a/packages/providers/src/acp/protocol.ts b/packages/providers/src/acp/protocol.ts index f93d22b7..97e266c9 100644 --- a/packages/providers/src/acp/protocol.ts +++ b/packages/providers/src/acp/protocol.ts @@ -168,6 +168,10 @@ export interface AcpInitializeResult { agentInfo?: { name?: string; version?: string } agentCapabilities?: { loadSession?: boolean + mcpCapabilities?: { + http?: boolean + sse?: boolean + } sessionCapabilities?: { resume?: boolean close?: boolean