diff --git a/apps/desktop/src/main/app-shell-guard.test.ts b/apps/desktop/src/main/app-shell-guard.test.ts new file mode 100644 index 00000000..279027ae --- /dev/null +++ b/apps/desktop/src/main/app-shell-guard.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it, vi } from 'vitest' +import type { WebContents } from 'electron' + +const { openExternal } = vi.hoisted(() => ({ openExternal: vi.fn() })) + +vi.mock('electron', () => ({ + shell: { openExternal }, +})) + +import { attachAppShellNavigationGuard } from './app-shell-guard' + +function fakeContents(): { + contents: WebContents + fire: (event: string, url: string) => boolean +} { + const listeners = new Map void>>() + const contents = { + setWindowOpenHandler: vi.fn(), + on: (event: string, listener: (event: { preventDefault(): void }, url: string) => void) => { + const list = listeners.get(event) ?? [] + list.push(listener) + listeners.set(event, list) + return contents + }, + } as unknown as WebContents + return { + contents, + fire: (event, url) => { + let prevented = false + const navEvent = { preventDefault: () => { prevented = true } } + for (const listener of listeners.get(event) ?? []) listener(navEvent, url) + return prevented + }, + } +} + +describe('attachAppShellNavigationGuard', () => { + it('sends YouTube to the OS browser instead of replacing the ADE', () => { + const { contents, fire } = fakeContents() + attachAppShellNavigationGuard(contents) + expect(fire('will-navigate', 'https://youtube.com/')).toBe(true) + expect(openExternal).toHaveBeenCalledWith('https://youtube.com/') + }) +}) diff --git a/apps/desktop/src/main/app-shell-guard.ts b/apps/desktop/src/main/app-shell-guard.ts new file mode 100644 index 00000000..3206f84f --- /dev/null +++ b/apps/desktop/src/main/app-shell-guard.ts @@ -0,0 +1,27 @@ +import { join } from 'node:path' +import { pathToFileURL } from 'node:url' +import { shell, type WebContents } from 'electron' +import { isAppUrl, isExternalOpenable } from './external-links' + +/** + * Pins the ADE renderer so transcript links cannot replace the app. Must be + * attached only to a BrowserWindow's own webContents — a WebContentsView + * guest (the in-app browser) has to navigate http(s) itself. + */ +export function attachAppShellNavigationGuard(contents: WebContents): void { + contents.setWindowOpenHandler(({ url }) => { + if (isExternalOpenable(url)) void shell.openExternal(url) + return { action: 'deny' } + }) + const devServerUrl = process.env['ELECTRON_RENDERER_URL'] + const appFileUrl = devServerUrl + ? undefined + : pathToFileURL(join(import.meta.dirname, '../renderer/index.html')).href + const guardNavigation = (event: { preventDefault(): void }, url: string): void => { + if (isAppUrl(url, devServerUrl, appFileUrl)) return + event.preventDefault() + if (isExternalOpenable(url)) void shell.openExternal(url) + } + contents.on('will-navigate', guardNavigation) + contents.on('will-redirect', guardNavigation) +} diff --git a/apps/desktop/src/main/browser-electron.ts b/apps/desktop/src/main/browser-electron.ts new file mode 100644 index 00000000..390ce92e --- /dev/null +++ b/apps/desktop/src/main/browser-electron.ts @@ -0,0 +1,84 @@ +import { WebContentsView, type BrowserWindow, type WebContents } from 'electron' +import type { BrowserGuest } from './browser-service' +import { isBrowserNavigable } from './browser-url' + +export const BROWSER_PARTITION = 'persist:ari-browser' + +/** True when this webContents is the in-app browser, not the ADE shell. */ +export function isBrowserGuest(contents: WebContents): boolean { + const session = contents.session as unknown as { partition?: string } + return session.partition === BROWSER_PARTITION +} + +/** + * Chromium guest for one in-app tab. Isolated partition, sandboxed, no Node. + * Popups navigate the same view instead of spawning windows. + */ +export function createElectronBrowserGuest( + win: BrowserWindow, + onUpdated: () => void, +): BrowserGuest { + const view = new WebContentsView({ + webPreferences: { + sandbox: true, + contextIsolation: true, + nodeIntegration: false, + partition: BROWSER_PARTITION, + }, + }) + win.contentView.addChildView(view) + view.setBounds({ x: 0, y: 0, width: 800, height: 600 }) + view.setVisible(false) + const wc = view.webContents + wc.setUserAgent(wc.getUserAgent().replace(/\sElectron\/\S+/g, '')) + + const emit = (): void => onUpdated() + wc.on('did-navigate', emit) + wc.on('did-navigate-in-page', emit) + wc.on('page-title-updated', emit) + wc.on('did-start-loading', emit) + wc.on('did-stop-loading', emit) + wc.on('did-fail-load', (_event, _code, _desc, _url, isMainFrame) => { + if (isMainFrame) emit() + }) + + wc.on('will-navigate', (event, url) => { + if (!isBrowserNavigable(url)) event.preventDefault() + }) + wc.setWindowOpenHandler(({ url }) => { + if (isBrowserNavigable(url)) void wc.loadURL(url) + return { action: 'deny' } + }) + + return { + loadURL: (url) => wc.loadURL(url), + goBack: () => { + if (wc.navigationHistory.canGoBack()) wc.navigationHistory.goBack() + }, + goForward: () => { + if (wc.navigationHistory.canGoForward()) wc.navigationHistory.goForward() + }, + reload: () => wc.reload(), + canGoBack: () => wc.navigationHistory.canGoBack(), + canGoForward: () => wc.navigationHistory.canGoForward(), + getURL: () => wc.getURL(), + getTitle: () => wc.getTitle(), + isLoading: () => wc.isLoading(), + executeJavaScript: (code) => wc.executeJavaScript(code, true), + capturePage: async (rect) => { + try { + const image = await wc.capturePage(rect) + const png = image.toPNG() + return png.length > 0 ? png : null + } catch { + return null + } + }, + setBounds: (bounds) => view.setBounds(bounds), + setVisible: (visible) => view.setVisible(visible), + destroy: () => { + if (!win.isDestroyed()) win.contentView.removeChildView(view) + if (!wc.isDestroyed()) wc.close() + }, + } +} diff --git a/apps/desktop/src/main/browser-element.test.ts b/apps/desktop/src/main/browser-element.test.ts new file mode 100644 index 00000000..89ac6418 --- /dev/null +++ b/apps/desktop/src/main/browser-element.test.ts @@ -0,0 +1,38 @@ +import { describe, expect, it } from 'vitest' +import { + elementChipLabel, + formatElementContext, + formatElementContexts, + isPickedElement, + type PickedElement, +} from './browser-element' + +const button: PickedElement = { + url: 'http://localhost:5173/', + selector: 'button.primary', + tag: 'button', + text: 'Save changes', + role: 'button', + ariaLabel: null, + html: '', + x: 10, + y: 20, + width: 80, + height: 32, +} + +describe('formatElementContext', () => { + it('names the page, selector, and text so the agent can find the node', () => { + const block = formatElementContext(button) + expect(block).toContain('http://localhost:5173/') + expect(block).toContain('button.primary') + expect(block).toContain('Save changes') + expect(formatElementContexts([button])).toContain('') + }) + + it('labels chips from visible text', () => { + expect(elementChipLabel(button)).toBe('Save changes') + expect(isPickedElement(button)).toBe(true) + expect(isPickedElement({ tag: 'div' })).toBe(false) + }) +}) diff --git a/apps/desktop/src/main/browser-element.ts b/apps/desktop/src/main/browser-element.ts new file mode 100644 index 00000000..329dae5d --- /dev/null +++ b/apps/desktop/src/main/browser-element.ts @@ -0,0 +1,60 @@ +/** One DOM node the user (or agent snapshot) named on the in-app page. */ +export interface PickedElement { + url: string + selector: string + tag: string + text: string + role: string | null + ariaLabel: string | null + html: string + x: number + y: number + width: number + height: number +} + +const TEXT_CAP = 400 +const HTML_CAP = 1_500 + +export function isPickedElement(value: unknown): value is PickedElement { + if (typeof value !== 'object' || value === null) return false + const row = value as Record + return ( + typeof row['url'] === 'string' && + typeof row['selector'] === 'string' && + typeof row['tag'] === 'string' && + typeof row['text'] === 'string' && + typeof row['html'] === 'string' && + typeof row['x'] === 'number' && + typeof row['y'] === 'number' && + typeof row['width'] === 'number' && + typeof row['height'] === 'number' + ) +} + +/** Prompt block the agent reads when the user mentions a picked element. */ +export function formatElementContext(element: PickedElement): string { + const lines = [ + `Selected element on ${element.url}`, + `- tag: ${element.tag}`, + `- selector: ${element.selector}`, + ] + if (element.role) lines.push(`- role: ${element.role}`) + if (element.ariaLabel) lines.push(`- aria-label: ${element.ariaLabel}`) + if (element.text) lines.push(`- text: ${element.text.slice(0, TEXT_CAP)}`) + if (element.html) lines.push(`- html: ${element.html.slice(0, HTML_CAP)}`) + return lines.join('\n') +} + +export function formatElementContexts(elements: readonly PickedElement[]): string { + if (elements.length === 0) return '' + const body = elements.map((el, i) => `Element ${String(i + 1)}\n${formatElementContext(el)}`).join('\n\n') + return `\n${body}\n` +} + +export function elementChipLabel(element: PickedElement): string { + const text = element.text.trim().replace(/\s+/g, ' ') + if (text.length > 0) return text.length > 32 ? `${text.slice(0, 31)}…` : text + if (element.ariaLabel) return element.ariaLabel + return element.selector +} diff --git a/apps/desktop/src/main/browser-mcp.test.ts b/apps/desktop/src/main/browser-mcp.test.ts new file mode 100644 index 00000000..bf425fcf --- /dev/null +++ b/apps/desktop/src/main/browser-mcp.test.ts @@ -0,0 +1,79 @@ +import { mkdtemp, readFile } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { describe, expect, it, vi } from 'vitest' +import { BrowserService, type BrowserGuest } from './browser-service' +import { dispatchBrowserMcpTool, startBrowserMcpServer, writeBrowserMcpProxy } from './browser-mcp' + +function guest(): BrowserGuest { + let url = 'about:blank' + return { + loadURL: vi.fn(async (next: string) => { + url = next + }), + goBack: vi.fn(), + goForward: vi.fn(), + reload: vi.fn(), + canGoBack: () => false, + canGoForward: () => false, + getURL: () => url, + getTitle: () => 'Example', + isLoading: () => false, + executeJavaScript: vi.fn(async (code: string) => { + if (code.includes('el.click()') || code.includes('el.focus()')) return true + return { url, title: 'Example', text: 'hi', elements: [] } + }), + capturePage: vi.fn(async () => null), + setBounds: vi.fn(), + setVisible: vi.fn(), + destroy: vi.fn(), + } +} + +describe('dispatchBrowserMcpTool', () => { + it('navigates and snapshots the shared guest', async () => { + const service = new BrowserService(() => guest(), vi.fn()) + const nav = await dispatchBrowserMcpTool(service, 'browser_navigate', { url: 'example.com' }) + expect(nav.isError).toBeUndefined() + expect(nav.content[0]?.text).toContain('https://example.com') + const snap = await dispatchBrowserMcpTool(service, 'browser_snapshot', {}) + expect(snap.content[0]?.text).toContain('Example') + const bad = await dispatchBrowserMcpTool(service, 'browser_navigate', { url: 'file:///x' }) + expect(bad.isError).toBe(true) + }) +}) + +describe('writeBrowserMcpProxy', () => { + it('writes a stdio proxy the ACP agent can spawn', async () => { + const dir = await mkdtemp(join(tmpdir(), 'ari-mcp-')) + const path = await writeBrowserMcpProxy(dir) + const source = await readFile(path, 'utf8') + expect(source).toContain('ARI_BROWSER_MCP_URL') + expect(source).toContain('fetch(url') + }) +}) + +describe('startBrowserMcpServer', () => { + it('serves tools/list over authorized HTTP', async () => { + const service = new BrowserService(() => guest(), vi.fn()) + const handle = await startBrowserMcpServer(service) + try { + const denied = await fetch(handle.url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/list' }), + }) + expect(denied.status).toBe(401) + const ok = await fetch(handle.url, { + method: 'POST', + headers: { 'content-type': 'application/json', authorization: `Bearer ${handle.token}` }, + body: JSON.stringify({ jsonrpc: '2.0', id: 2, method: 'tools/list' }), + }) + expect(ok.status).toBe(200) + const body = (await ok.json()) as { result?: { tools?: { name: string }[] } } + expect(body.result?.tools?.map((t) => t.name)).toContain('browser_navigate') + } finally { + handle.close() + } + }) +}) diff --git a/apps/desktop/src/main/browser-mcp.ts b/apps/desktop/src/main/browser-mcp.ts new file mode 100644 index 00000000..0e58fdea --- /dev/null +++ b/apps/desktop/src/main/browser-mcp.ts @@ -0,0 +1,291 @@ +import { writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { createServer, type IncomingMessage, type ServerResponse } from 'node:http' +import { randomBytes } from 'node:crypto' +import { createLogger } from '@ari/shared/logger' +import type { BrowserService } from './browser-service' +import { resolveBrowserUrl } from './browser-url' + +const log = createLogger('desktop:browser-mcp') + +const DEFAULT_TAB = 'inspector' + +export interface BrowserMcpHandle { + url: string + token: string + close: () => void +} + +interface JsonRpc { + jsonrpc?: string + id?: number | string + method?: string + params?: unknown +} + +function json(res: ServerResponse, status: number, body: unknown): void { + const payload = JSON.stringify(body) + res.writeHead(status, { + 'content-type': 'application/json', + 'content-length': Buffer.byteLength(payload), + }) + res.end(payload) +} + +function asRecord(value: unknown): Record { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {} +} + +function textResult( + text: string, + isError = false, +): { content: { type: 'text'; text: string }[]; isError?: boolean } { + return { content: [{ type: 'text', text }], ...(isError ? { isError: true } : {}) } +} + +export const BROWSER_MCP_TOOLS = [ + { + name: 'browser_status', + description: + "URL, title, and loading state of Ari's visible in-app browser. Prefer this over any other browser.", + inputSchema: { type: 'object', properties: {}, additionalProperties: false }, + }, + { + name: 'browser_navigate', + description: + "Open a URL in Ari's visible in-app browser (http/https). The user sees this page. Use this instead of any other browser, web fetch, or computer-use tool.", + inputSchema: { + type: 'object', + properties: { url: { type: 'string', description: 'http(s) URL or host[:port]' } }, + required: ['url'], + additionalProperties: false, + }, + }, + { + name: 'browser_snapshot', + description: + 'Page text plus interactive elements (tag, selector, label) from the in-app browser.', + inputSchema: { type: 'object', properties: {}, additionalProperties: false }, + }, + { + name: 'browser_click', + description: 'Click the first element matching a CSS selector in the in-app browser.', + inputSchema: { + type: 'object', + properties: { selector: { type: 'string' } }, + required: ['selector'], + additionalProperties: false, + }, + }, + { + name: 'browser_type', + description: 'Type into the first element matching a CSS selector in the in-app browser.', + inputSchema: { + type: 'object', + properties: { selector: { type: 'string' }, text: { type: 'string' } }, + required: ['selector', 'text'], + additionalProperties: false, + }, + }, +] as const + +export async function dispatchBrowserMcpTool( + browsers: BrowserService, + name: string, + args: Record, + tabId = DEFAULT_TAB, +): Promise<{ content: { type: 'text'; text: string }[]; isError?: boolean }> { + try { + if (!browsers.has(tabId)) await browsers.open(tabId) + if (name === 'browser_status') { + return textResult(JSON.stringify(browsers.snapshot(tabId))) + } + if (name === 'browser_navigate') { + const url = typeof args['url'] === 'string' ? args['url'] : '' + const resolved = resolveBrowserUrl(url) + if (!resolved.ok) return textResult(resolved.error, true) + const result = await browsers.navigate(tabId, resolved.url) + browsers.reveal(tabId) + return result.ok ? textResult(JSON.stringify(result.tab)) : textResult(result.error, true) + } + if (name === 'browser_snapshot') { + const snap = await browsers.pageSnapshot(tabId) + return textResult(JSON.stringify(snap)) + } + if (name === 'browser_click') { + const selector = typeof args['selector'] === 'string' ? args['selector'] : '' + if (!selector) return textResult('selector is required', true) + const clicked = await browsers.runScript( + tabId, + `(() => { const el = document.querySelector(${JSON.stringify(selector)}); if (!el) return false; el.click(); return true })()`, + ) + return clicked === true + ? textResult('clicked') + : textResult(`no element for ${selector}`, true) + } + if (name === 'browser_type') { + const selector = typeof args['selector'] === 'string' ? args['selector'] : '' + const text = typeof args['text'] === 'string' ? args['text'] : '' + if (!selector) return textResult('selector is required', true) + const typed = await browsers.runScript( + tabId, + `(() => { const el = document.querySelector(${JSON.stringify(selector)}); if (!el) return false; el.focus(); if ('value' in el) el.value = ${JSON.stringify(text)}; el.dispatchEvent(new Event('input', { bubbles: true })); return true })()`, + ) + return typed === true ? textResult('typed') : textResult(`no element for ${selector}`, true) + } + return textResult(`unknown tool ${name}`, true) + } catch (error: unknown) { + return textResult(error instanceof Error ? error.message : String(error), true) + } +} + +function authorized(req: IncomingMessage, token: string): boolean { + const header = req.headers['authorization'] + if (typeof header === 'string' && header === `Bearer ${token}`) return true + const url = new URL(req.url ?? '/', 'http://127.0.0.1') + return url.searchParams.get('token') === token +} + +function readBody(req: IncomingMessage): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + req.on('data', (chunk: Buffer) => chunks.push(chunk)) + req.on('end', () => resolve(Buffer.concat(chunks).toString('utf8'))) + req.on('error', reject) + }) +} + +async function handleRpc( + browsers: BrowserService, + message: JsonRpc, +): Promise<{ + jsonrpc: '2.0' + id?: number | string + result?: unknown + error?: { code: number; message: string } +}> { + const id = message.id + const method = message.method ?? '' + if (method === 'initialize') { + return { + jsonrpc: '2.0', + id, + result: { + protocolVersion: '2024-11-05', + capabilities: { tools: {} }, + serverInfo: { name: 'ari-browser', version: '0.1.0' }, + instructions: + "You are connected to Ari's in-app browser. The user can see the page. Use browser_navigate, browser_snapshot, browser_click, and browser_type for every web task. Never say a browser is unavailable and do not use a separate browser or computer-use tool.", + }, + } + } + if (method === 'notifications/initialized' || method === 'notifications/cancelled') { + return { jsonrpc: '2.0', id } + } + if (method === 'ping' || method === 'tools/list') { + if (method === 'ping') return { jsonrpc: '2.0', id, result: {} } + return { jsonrpc: '2.0', id, result: { tools: BROWSER_MCP_TOOLS } } + } + if (method === 'tools/call') { + const params = asRecord(message.params) + const name = typeof params['name'] === 'string' ? params['name'] : '' + const args = asRecord(params['arguments']) + const result = await dispatchBrowserMcpTool(browsers, name, args) + return { jsonrpc: '2.0', id, result } + } + return { jsonrpc: '2.0', id, error: { code: -32601, message: `Method not found: ${method}` } } +} + +/** Loopback Streamable HTTP MCP for the in-app browser. */ +export function startBrowserMcpServer(browsers: BrowserService): Promise { + const token = randomBytes(24).toString('base64url') + const server = createServer((req, res) => { + void (async () => { + if (req.method === 'OPTIONS') { + res.writeHead(204) + res.end() + return + } + if (!authorized(req, token)) { + json(res, 401, { error: 'unauthorized' }) + return + } + if (req.method !== 'POST') { + json(res, 405, { error: 'POST required' }) + return + } + let parsed: JsonRpc + try { + parsed = JSON.parse(await readBody(req)) as JsonRpc + } catch { + json(res, 400, { error: 'invalid json' }) + return + } + if (parsed.id === undefined) { + void handleRpc(browsers, parsed) + res.writeHead(202) + res.end() + return + } + json(res, 200, await handleRpc(browsers, parsed)) + })().catch((error: unknown) => { + log.warn('browser mcp request failed', { error: String(error) }) + if (!res.headersSent) json(res, 500, { error: 'internal' }) + }) + }) + return new Promise((resolve, reject) => { + server.once('error', reject) + server.listen(0, '127.0.0.1', () => { + const address = server.address() + if (address === null || typeof address === 'string') { + reject(new Error('browser mcp failed to bind')) + return + } + resolve({ + url: `http://127.0.0.1:${String(address.port)}/mcp`, + token, + close: () => server.close(), + }) + }) + }) +} + +/** + * ACP requires stdio MCP. This proxy is spawned with ELECTRON_RUN_AS_NODE + * and forwards JSON-RPC lines to the loopback HTTP server. + */ +const MCP_PROXY_SOURCE = [ + "import { createInterface } from 'node:readline'", + "import { stdin, stdout } from 'node:process'", + "const url = process.env.ARI_BROWSER_MCP_URL", + "const token = process.env.ARI_BROWSER_MCP_TOKEN", + "async function rpc(msg) {", + " const res = await fetch(url, {", + " method: 'POST',", + " headers: { 'content-type': 'application/json', authorization: 'Bearer ' + token },", + " body: JSON.stringify(msg),", + " })", + " if (res.status === 202) return null", + " return await res.json()", + "}", + "const rl = createInterface({ input: stdin })", + "rl.on('line', (line) => {", + " if (!line.trim()) return", + " const msg = JSON.parse(line)", + " rpc(msg).then((out) => {", + " if (out && msg.id !== undefined) stdout.write(JSON.stringify(out) + '\\n')", + " }).catch((err) => {", + " if (msg.id !== undefined) {", + " stdout.write(JSON.stringify({ jsonrpc: '2.0', id: msg.id, error: { code: -32000, message: String(err) } }) + '\\n')", + " }", + " })", + "})", +].join('\n') + +export async function writeBrowserMcpProxy(dir: string): Promise { + const path = join(dir, 'browser-mcp-proxy.mjs') + await writeFile(path, MCP_PROXY_SOURCE, 'utf8') + return path +} diff --git a/apps/desktop/src/main/browser-pick-script.ts b/apps/desktop/src/main/browser-pick-script.ts new file mode 100644 index 00000000..349bcdb5 --- /dev/null +++ b/apps/desktop/src/main/browser-pick-script.ts @@ -0,0 +1,123 @@ +/** + * Injected into the guest page. Built from line arrays so the main bundle + * never carries a multi-line template literal (esbuild CJS shims split those). + */ +function script(lines: string[]): string { + return lines.join('\n') +} + +export const PICKER_SOURCE = script([ + 'new Promise((resolve) => {', + ' window.__ariCancelPick && window.__ariCancelPick()', + " const box = document.createElement('div')", + " box.setAttribute('data-ari-pick', '1')", + " box.style.cssText = 'position:fixed;z-index:2147483647;pointer-events:none;border:2px solid #7c6cf0;background:rgba(124,108,240,.12);border-radius:4px;'", + ' document.documentElement.appendChild(box)', + ' function cssPath(el) {', + ' if (el.id) {', + " const id = '#' + CSS.escape(el.id)", + ' try { if (document.querySelectorAll(id).length === 1) return id } catch (e) {}', + ' }', + ' const parts = []', + ' let cur = el', + ' for (let i = 0; i < 5 && cur && cur.nodeType === 1 && cur !== document.documentElement; i++) {', + ' let part = cur.tagName.toLowerCase()', + ' if (cur.classList && cur.classList.length) {', + " part += '.' + Array.from(cur.classList).slice(0, 2).map(function(c) { return CSS.escape(c) }).join('.')", + ' }', + ' const parent = cur.parentElement', + ' if (parent) {', + ' const same = Array.from(parent.children).filter(function(c) { return c.tagName === cur.tagName })', + " if (same.length > 1) part += ':nth-of-type(' + (same.indexOf(cur) + 1) + ')'", + ' }', + ' parts.unshift(part)', + ' cur = parent', + ' }', + " return parts.join(' > ')", + ' }', + ' function targetOf(event) {', + ' const el = document.elementFromPoint(event.clientX, event.clientY)', + ' if (!el || el === box) return null', + ' return el', + ' }', + ' function cleanup() {', + ' window.__ariCancelPick = undefined', + " window.removeEventListener('mousemove', onMove, true)", + " window.removeEventListener('click', onClick, true)", + " window.removeEventListener('keydown', onKey, true)", + ' box.remove()', + ' }', + ' function onMove(event) {', + ' const el = targetOf(event)', + ' if (!el) return', + ' const r = el.getBoundingClientRect()', + " box.style.left = r.left + 'px'", + " box.style.top = r.top + 'px'", + " box.style.width = r.width + 'px'", + " box.style.height = r.height + 'px'", + ' }', + ' function onClick(event) {', + ' event.preventDefault()', + ' event.stopPropagation()', + ' const el = targetOf(event)', + ' cleanup()', + ' if (!el) { resolve(null); return }', + ' const r = el.getBoundingClientRect()', + ' resolve({', + ' url: location.href,', + ' selector: cssPath(el),', + ' tag: el.tagName.toLowerCase(),', + " text: (el.innerText || el.value || '').trim().slice(0, 400),", + " role: el.getAttribute('role'),", + " ariaLabel: el.getAttribute('aria-label'),", + " html: (el.outerHTML || '').slice(0, 1500),", + ' x: r.x, y: r.y, width: r.width, height: r.height', + ' })', + ' }', + ' function onKey(event) {', + " if (event.key === 'Escape') { event.preventDefault(); cleanup(); resolve(null) }", + ' }', + ' window.__ariCancelPick = function() { cleanup(); resolve(null) }', + " window.addEventListener('mousemove', onMove, true)", + " window.addEventListener('click', onClick, true)", + " window.addEventListener('keydown', onKey, true)", + '})', +]) + +export const CANCEL_PICK_SOURCE = + 'void (window.__ariCancelPick && window.__ariCancelPick())' + +export const SNAPSHOT_SOURCE = script([ + '(function() {', + ' function cssPath(el) {', + ' if (el.id) {', + " const id = '#' + CSS.escape(el.id)", + ' try { if (document.querySelectorAll(id).length === 1) return id } catch (e) {}', + ' }', + ' const parts = []', + ' let cur = el', + ' for (let i = 0; i < 4 && cur && cur.nodeType === 1 && cur !== document.documentElement; i++) {', + ' let part = cur.tagName.toLowerCase()', + ' if (cur.classList && cur.classList.length) {', + " part += '.' + Array.from(cur.classList).slice(0, 2).map(function(c) { return CSS.escape(c) }).join('.')", + ' }', + ' parts.unshift(part)', + ' cur = cur.parentElement', + ' }', + " return parts.join(' > ')", + ' }', + " const nodes = Array.from(document.querySelectorAll('a,button,input,textarea,select,[role=button],[role=link]')).slice(0, 80)", + ' return {', + ' url: location.href,', + ' title: document.title,', + " text: (document.body && document.body.innerText ? document.body.innerText : '').slice(0, 4000),", + ' elements: nodes.map(function(el) {', + ' return {', + ' tag: el.tagName.toLowerCase(),', + ' selector: cssPath(el),', + " text: ((el.innerText || el.getAttribute('aria-label') || el.getAttribute('placeholder') || '') + '').trim().slice(0, 80)", + ' }', + ' })', + ' }', + '})()', +]) diff --git a/apps/desktop/src/main/browser-service.test.ts b/apps/desktop/src/main/browser-service.test.ts new file mode 100644 index 00000000..ab9f7a2c --- /dev/null +++ b/apps/desktop/src/main/browser-service.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, vi } from 'vitest' +import { BrowserService, type BrowserGuest } from './browser-service' + +function fakeGuest(): { + guest: BrowserGuest + loadURL: ReturnType + goBack: ReturnType + setBounds: ReturnType + setVisible: ReturnType + destroy: ReturnType +} { + let url = 'about:blank' + const loadURL = vi.fn(async (next: string) => { + url = next + }) + const goBack = vi.fn() + const setBounds = vi.fn() + const setVisible = vi.fn() + const destroy = vi.fn() + const guest: BrowserGuest = { + loadURL, + goBack, + goForward: vi.fn(), + reload: vi.fn(), + canGoBack: () => false, + canGoForward: () => false, + getURL: () => url, + getTitle: () => 'Example', + isLoading: () => false, + executeJavaScript: vi.fn(async () => null), + capturePage: vi.fn(async () => null), + setBounds, + setVisible, + destroy, + } + return { guest, loadURL, goBack, setBounds, setVisible, destroy } +} + +describe('BrowserService', () => { + it('opens a tab, navigates http(s), and refuses file URLs', async () => { + const { guest, loadURL } = fakeGuest() + const service = new BrowserService(() => guest, vi.fn()) + + const opened = await service.open('tab1') + expect(opened.url).toBe('about:blank') + + const ok = await service.navigate('tab1', 'example.com') + expect(ok.ok).toBe(true) + if (ok.ok) expect(ok.tab.url).toBe('https://example.com/') + expect(loadURL).toHaveBeenCalledWith('https://example.com/') + + const refused = await service.navigate('tab1', 'file:///etc/passwd') + expect(refused.ok).toBe(false) + expect(loadURL).toHaveBeenCalledTimes(1) + }) + + it('moves, hides, and destroys the guest', async () => { + const { guest, goBack, setBounds, setVisible, destroy } = fakeGuest() + const service = new BrowserService(() => guest, vi.fn()) + await service.open('tab1') + expect(service.go('tab1', 'back').id).toBe('tab1') + expect(goBack).toHaveBeenCalledOnce() + expect(service.layout('tab1', { x: 10, y: 20, width: 400, height: 300 }, true)).toBe(true) + expect(setVisible).toHaveBeenCalledWith(true) + expect(setBounds).toHaveBeenCalledWith({ x: 10, y: 20, width: 400, height: 300 }) + expect(service.close('tab1')).toBe(true) + expect(destroy).toHaveBeenCalledOnce() + expect(service.layout('tab1', { x: 0, y: 0, width: 1, height: 1 }, false)).toBe(false) + }) + + it('returns a picked element from the guest script', async () => { + const picked = { + url: 'https://example.com/', + selector: 'button', + tag: 'button', + text: 'Go', + role: 'button', + ariaLabel: null, + html: '', + x: 1, + y: 2, + width: 3, + height: 4, + } + const { guest } = fakeGuest() + guest.executeJavaScript = vi.fn(async (code: string) => + code.includes('__ariCancelPick') && !code.includes('new Promise') ? undefined : picked, + ) + guest.capturePage = vi.fn(async () => Buffer.from('png')) + const service = new BrowserService(() => guest, vi.fn()) + await service.open('tab1') + const result = await service.pick('tab1') + expect(result.ok).toBe(true) + if (result.ok) { + expect(result.element.selector).toBe('button') + expect(result.pngBase64).toBe(Buffer.from('png').toString('base64')) + } + }) +}) diff --git a/apps/desktop/src/main/browser-service.ts b/apps/desktop/src/main/browser-service.ts new file mode 100644 index 00000000..2a89b1f2 --- /dev/null +++ b/apps/desktop/src/main/browser-service.ts @@ -0,0 +1,195 @@ +import { isPickedElement, type PickedElement } from './browser-element' +import { CANCEL_PICK_SOURCE, PICKER_SOURCE, SNAPSHOT_SOURCE } from './browser-pick-script' +import { resolveBrowserUrl } from './browser-url' + +export interface BrowserBounds { + x: number + y: number + width: number + height: number +} + +export interface BrowserTabState { + id: string + url: string + title: string + canGoBack: boolean + canGoForward: boolean + loading: boolean + error: string | null + /** True when an agent just drove this tab — the shell should show the pane. */ + reveal?: boolean +} + +/** One Chromium guest the service positions and navigates. Injected in tests. */ +export interface BrowserGuest { + loadURL(url: string): Promise + goBack(): void + goForward(): void + reload(): void + canGoBack(): boolean + canGoForward(): boolean + getURL(): string + getTitle(): string + isLoading(): boolean + executeJavaScript(code: string): Promise + capturePage(rect?: BrowserBounds): Promise + setBounds(bounds: BrowserBounds): void + setVisible(visible: boolean): void + destroy(): void +} + +export type BrowserGuestFactory = (id: string, onUpdated: () => void) => BrowserGuest + +interface Tab { + guest: BrowserGuest + error: string | null +} + +/** + * Owns in-app browser tabs. The factory builds the Electron guest; this + * class is the navigation, layout, and snapshot policy so tests do not + * need Chromium. + */ +export class BrowserService { + readonly #tabs = new Map() + readonly #createGuest: BrowserGuestFactory + readonly #onUpdated: (state: BrowserTabState) => void + + constructor(createGuest: BrowserGuestFactory, onUpdated: (state: BrowserTabState) => void) { + this.#createGuest = createGuest + this.#onUpdated = onUpdated + } + + async open(id: string, url?: string): Promise { + if (this.#tabs.get(id) === undefined) { + const tab: Tab = { guest: this.#createGuest(id, () => this.#emit(id)), error: null } + this.#tabs.set(id, tab) + } + if (url !== undefined && url !== '') { + const result = await this.navigate(id, url) + if (result.ok) return result.tab + } + return this.snapshot(id) + } + + async navigate( + id: string, + input: string, + ): Promise<{ ok: true; tab: BrowserTabState } | { ok: false; error: string }> { + const tab = this.#tab(id) + const resolved = resolveBrowserUrl(input) + if (!resolved.ok) return resolved + tab.error = null + try { + await tab.guest.loadURL(resolved.url) + } catch (error: unknown) { + tab.error = error instanceof Error ? error.message : String(error) + } + this.#emit(id) + return { ok: true, tab: this.snapshot(id) } + } + + go(id: string, action: 'back' | 'forward' | 'reload'): BrowserTabState { + const tab = this.#tab(id) + if (action === 'back') tab.guest.goBack() + else if (action === 'forward') tab.guest.goForward() + else tab.guest.reload() + tab.error = null + this.#emit(id) + return this.snapshot(id) + } + + layout(id: string, bounds: BrowserBounds, visible: boolean): boolean { + const tab = this.#tabs.get(id) + if (tab === undefined) return false + tab.guest.setVisible(visible) + if (visible) tab.guest.setBounds(bounds) + return true + } + + close(id: string): boolean { + const tab = this.#tabs.get(id) + if (tab === undefined) return false + tab.guest.destroy() + this.#tabs.delete(id) + return true + } + + snapshot(id: string): BrowserTabState { + const tab = this.#tab(id) + const url = tab.guest.getURL() + return { + id, + url: url.length > 0 ? url : 'about:blank', + title: tab.guest.getTitle(), + canGoBack: tab.guest.canGoBack(), + canGoForward: tab.guest.canGoForward(), + loading: tab.guest.isLoading(), + error: tab.error, + } + } + + async pick( + id: string, + ): Promise< + { ok: true; element: PickedElement; pngBase64: string | null } | { ok: false; error: string } + > { + const tab = this.#tab(id) + await tab.guest.executeJavaScript(CANCEL_PICK_SOURCE) + const raw = await tab.guest.executeJavaScript(PICKER_SOURCE) + if (raw === null || raw === undefined) return { ok: false, error: 'cancelled' } + if (!isPickedElement(raw)) return { ok: false, error: 'pick returned no element' } + const png = await tab.guest.capturePage({ + x: Math.round(raw.x), + y: Math.round(raw.y), + width: Math.max(1, Math.round(raw.width)), + height: Math.max(1, Math.round(raw.height)), + }) + return { ok: true, element: raw, pngBase64: png ? png.toString('base64') : null } + } + + async cancelPick(id: string): Promise { + const tab = this.#tabs.get(id) + if (tab === undefined) return false + await tab.guest.executeJavaScript(CANCEL_PICK_SOURCE) + return true + } + + async pageSnapshot(id: string): Promise { + const tab = this.#tab(id) + return tab.guest.executeJavaScript(SNAPSHOT_SOURCE) + } + + async runScript(id: string, code: string): Promise { + const tab = this.#tab(id) + return tab.guest.executeJavaScript(code) + } + + async screenshot(id: string): Promise { + const tab = this.#tab(id) + return tab.guest.capturePage() + } + + reveal(id: string): void { + this.#onUpdated({ ...this.snapshot(id), reveal: true }) + } + + has(id: string): boolean { + return this.#tabs.has(id) + } + + dispose(): void { + for (const id of [...this.#tabs.keys()]) this.close(id) + } + + #tab(id: string): Tab { + const tab = this.#tabs.get(id) + if (tab === undefined) throw new Error(`unknown browser tab ${id}`) + return tab + } + + #emit(id: string): void { + this.#onUpdated(this.snapshot(id)) + } +} diff --git a/apps/desktop/src/main/browser-url.test.ts b/apps/desktop/src/main/browser-url.test.ts new file mode 100644 index 00000000..cb4a2131 --- /dev/null +++ b/apps/desktop/src/main/browser-url.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest' +import { isBrowserNavigable, resolveBrowserUrl } from './browser-url' + +describe('isBrowserNavigable', () => { + it('allows http, https, and about:blank', () => { + expect(isBrowserNavigable('https://example.com/x')).toBe(true) + expect(isBrowserNavigable('http://localhost:5173/')).toBe(true) + expect(isBrowserNavigable('about:blank')).toBe(true) + }) + + it('refuses file, javascript, data, and mailto', () => { + expect(isBrowserNavigable('file:///etc/passwd')).toBe(false) + expect(isBrowserNavigable('javascript:alert(1)')).toBe(false) + expect(isBrowserNavigable('data:text/html,hi')).toBe(false) + expect(isBrowserNavigable('mailto:hi@example.com')).toBe(false) + }) +}) + +describe('resolveBrowserUrl', () => { + it('keeps an empty bar as a blank tab', () => { + expect(resolveBrowserUrl('')).toEqual({ ok: true, url: 'about:blank' }) + expect(resolveBrowserUrl(' about:blank ')).toEqual({ ok: true, url: 'about:blank' }) + }) + + it('accepts http(s) URLs as written', () => { + expect(resolveBrowserUrl('https://example.com/a')).toEqual({ + ok: true, + url: 'https://example.com/a', + }) + expect(resolveBrowserUrl('http://127.0.0.1:3000')).toEqual({ + ok: true, + url: 'http://127.0.0.1:3000/', + }) + }) + + it('prefixes https for public hosts and http for loopback', () => { + expect(resolveBrowserUrl('example.com/docs')).toEqual({ + ok: true, + url: 'https://example.com/docs', + }) + expect(resolveBrowserUrl('localhost:5173')).toEqual({ + ok: true, + url: 'http://localhost:5173/', + }) + }) + + it('rejects schemes the guest must never load', () => { + expect(resolveBrowserUrl('file:///tmp/x')).toMatchObject({ ok: false }) + expect(resolveBrowserUrl('javascript:alert(1)')).toMatchObject({ ok: false }) + expect(resolveBrowserUrl('not a url')).toMatchObject({ ok: false }) + }) +}) diff --git a/apps/desktop/src/main/browser-url.ts b/apps/desktop/src/main/browser-url.ts new file mode 100644 index 00000000..bf4aa8db --- /dev/null +++ b/apps/desktop/src/main/browser-url.ts @@ -0,0 +1,57 @@ +/** + * What the in-app browser may load. http(s) only, plus about:blank as the + * empty tab. file:/javascript:/data: never reach the guest — those belong + * outside this view, or nowhere. + */ + +const NAVIGABLE = new Set(['http:', 'https:']) + +/** True when a fully-resolved URL may become the guest's document. */ +export function isBrowserNavigable(rawUrl: string): boolean { + if (rawUrl === 'about:blank') return true + try { + const parsed = new URL(rawUrl) + return NAVIGABLE.has(parsed.protocol) + } catch { + return false + } +} + +function isLoopbackHost(value: string): boolean { + const host = value.split('/')[0]?.split(':')[0]?.toLowerCase() ?? '' + return host === 'localhost' || host === '127.0.0.1' || host === '[::1]' || host === '::1' +} + +/** + * Turns an address-bar string into a loadable URL. Bare hosts get https, + * except loopback which stays http so local dev servers work. + */ +export function resolveBrowserUrl( + input: string, +): { ok: true; url: string } | { ok: false; error: string } { + const trimmed = input.trim() + if (trimmed === '' || trimmed.toLowerCase() === 'about:blank') { + return { ok: true, url: 'about:blank' } + } + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) { + try { + const parsed = new URL(trimmed) + if (!NAVIGABLE.has(parsed.protocol)) { + return { ok: false, error: `${parsed.protocol} URLs cannot open in the in-app browser` } + } + return { ok: true, url: parsed.href } + } catch { + return { ok: false, error: 'Enter an http(s) URL' } + } + } + const candidate = `${isLoopbackHost(trimmed) ? 'http' : 'https'}://${trimmed}` + try { + const parsed = new URL(candidate) + if (!NAVIGABLE.has(parsed.protocol) || parsed.hostname.length === 0) { + return { ok: false, error: 'Enter an http(s) URL' } + } + return { ok: true, url: parsed.href } + } catch { + return { ok: false, error: 'Enter an http(s) URL' } + } +} diff --git a/apps/desktop/src/main/extension-inventory.ts b/apps/desktop/src/main/extension-inventory.ts new file mode 100644 index 00000000..17c7369d --- /dev/null +++ b/apps/desktop/src/main/extension-inventory.ts @@ -0,0 +1,99 @@ +import { readFile } from 'node:fs/promises' +import { basename } from 'node:path' +import type { DriverKind } from '@ari/contracts/common' +import type { RpcResults } from '@ari/contracts/rpc' +import { discoverExtensions, type ExtensionRecord } from '@ari/providers/extensions' +import { realDetectEnvironment, type DetectEnvironment } from '@ari/providers/types' +import type { McpServerConfig } from '@ari/ari-core/mcp-servers' +import { sanitizeMcpSegment } from '@ari/ari-core/mcp-tools' +import type { AriSkillRecord } from '@ari/ari-core/skills' + +const READ_CAP = 64 * 1024 + +const BROWSER_SUMMARY = + 'Sent as HTTP only when the agent advertises mcpCapabilities.http. Otherwise the stdio proxy is sent. Never both.' + +export function browserExtensionRecord(provider: DriverKind): ExtensionRecord { + return { + kind: 'mcp', + scope: 'ari', + provider, + id: 'ari:browser', + name: 'ari-browser', + summary: BROWSER_SUMMARY, + sourcePath: null, + transport: 'http', + disabled: false, + delivery: 'injected', + } +} + +export async function buildExtensionInventory(input: { + kind: DriverKind + workspacePath: string | null + env?: DetectEnvironment + coreServers?: McpServerConfig[] + coreSkills?: AriSkillRecord[] +}): Promise { + if (input.kind === 'ari-core') { + const records = [ + ...coreMcpRecords(input.coreServers ?? []), + ...coreSkillRecords(input.coreSkills ?? []), + browserExtensionRecord('ari-core'), + ] + return { records, truncated: false } + } + const discovered = await discoverExtensions({ + provider: input.kind, + workspacePath: input.workspacePath, + env: input.env ?? realDetectEnvironment(), + }) + return { + records: [...discovered.records, browserExtensionRecord(input.kind)], + truncated: discovered.truncated, + } +} + +export async function readInventoriedSkill( + inventory: RpcResults['providers.extensionInventory'], + path: string, +): Promise { + const allowed = inventory.records.some( + (record) => record.kind === 'skill' && record.sourcePath === path, + ) + if (!allowed) throw new Error('that file is not a skill in this inventory') + const raw = await readFile(path, 'utf8') + const truncated = raw.length > READ_CAP + return { content: truncated ? raw.slice(0, READ_CAP) : raw, truncated } +} + +function coreMcpRecords(servers: McpServerConfig[]): ExtensionRecord[] { + return servers.map((server) => ({ + kind: 'mcp' as const, + scope: 'user' as const, + provider: 'ari-core' as const, + id: `ari-core:user:mcp:${server.id}:${server.name}`, + name: server.name, + sourcePath: null, + transport: 'stdio' as const, + command: basename(server.command), + disabled: server.disabled, + delivery: 'hosted' as const, + ...(sanitizeMcpSegment(server.name) === 'ari_browser' ? { problem: 'duplicate-name' as const } : {}), + })) +} + +function coreSkillRecords(skills: AriSkillRecord[]): ExtensionRecord[] { + return skills.map((skill) => ({ + kind: 'skill' as const, + scope: skill.scope, + provider: 'ari-core' as const, + id: `ari-core:${skill.scope}:skill:${skill.sourcePath}:${skill.name}`, + name: skill.name, + ...(skill.summary ? { summary: skill.summary } : {}), + sourcePath: skill.sourcePath, + disabled: false, + delivery: 'hosted' as const, + ...(skill.problem ? { problem: skill.problem } : {}), + })) +} diff --git a/apps/desktop/src/main/index.ts b/apps/desktop/src/main/index.ts index f39d3ce6..66234896 100644 --- a/apps/desktop/src/main/index.ts +++ b/apps/desktop/src/main/index.ts @@ -1,16 +1,15 @@ -import { app, BrowserWindow, shell } from 'electron' -import { join } from 'node:path' -import { pathToFileURL } from 'node:url' +import { app, BrowserWindow } from 'electron' import { isolateDevInstance } from './dev-instance' import { registerRpc, startAppUpdateChecks } from './rpc' import { createTray, type TrayHandle } from './tray' import { updateTrayStatus } from './tray-status' import { createMainWindow } from './window' -import { isAppUrl, isExternalOpenable } from './external-links' // The launch animation's signature sound is Web Audio; without this switch // Chromium blocks it until the first user gesture. app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required') +// Google properties (YouTube) often fail with net::ERR_FAILED under HTTP/3 in Electron. +app.commandLine.appendSwitch('disable-quic') // Unpackaged `pnpm dev` must take this identity before the lock: Electron keys // the mutex (and every userData store) off app name. Installed Ari stays on @@ -62,27 +61,4 @@ if (!gotLock) { app.on('window-all-closed', () => { if (process.platform !== 'darwin') app.quit() }) - - app.on('web-contents-created', (_event, contents) => { - // Popups (window.open / target=_blank) open in the OS browser. - contents.setWindowOpenHandler(({ url }) => { - if (isExternalOpenable(url)) void shell.openExternal(url) - return { action: 'deny' } - }) - // Same-window clicks (bare from transcript markdown) never - // navigate the ADE: the app entry stays, everything openable goes to the - // OS browser, everything else is dropped. Only the packaged renderer's - // own entry file counts as the app — never an arbitrary file:/data: URL. - const devServerUrl = process.env['ELECTRON_RENDERER_URL'] - const appFileUrl = devServerUrl - ? undefined - : pathToFileURL(join(import.meta.dirname, '../renderer/index.html')).href - const guardNavigation = (_navEvent: { preventDefault(): void }, url: string): void => { - if (isAppUrl(url, devServerUrl, appFileUrl)) return - _navEvent.preventDefault() - if (isExternalOpenable(url)) void shell.openExternal(url) - } - contents.on('will-navigate', guardNavigation) - contents.on('will-redirect', guardNavigation) - }) } diff --git a/apps/desktop/src/main/ipc-methods.ts b/apps/desktop/src/main/ipc-methods.ts index 4b49703c..29ba6314 100644 --- a/apps/desktop/src/main/ipc-methods.ts +++ b/apps/desktop/src/main/ipc-methods.ts @@ -49,6 +49,12 @@ export const IPC_METHODS = [ 'providers.configFiles', 'providers.readConfig', 'providers.writeConfig', + 'providers.extensionInventory', + 'providers.readExtensionFile', + 'ariCore.mcp.list', + 'ariCore.mcp.upsert', + 'ariCore.mcp.remove', + 'ariCore.skills.trust', 'window.minimize', 'window.toggleMaximize', 'window.close', @@ -57,6 +63,13 @@ export const IPC_METHODS = [ 'terminal.write', 'terminal.resize', 'terminal.kill', + 'browser.open', + 'browser.navigate', + 'browser.go', + 'browser.close', + 'browser.layout', + 'browser.pick', + 'browser.cancelPick', 'project.list', 'project.add', 'project.open', diff --git a/apps/desktop/src/main/rpc.ts b/apps/desktop/src/main/rpc.ts index 5c888103..3fbe4370 100644 --- a/apps/desktop/src/main/rpc.ts +++ b/apps/desktop/src/main/rpc.ts @@ -45,6 +45,10 @@ import { type PtyFactory, type PtyLike, } from './terminal-service' +import { BrowserService } from './browser-service' +import { createElectronBrowserGuest } from './browser-electron' +import { startBrowserMcpServer, writeBrowserMcpProxy } from './browser-mcp' +import { buildExtensionInventory, readInventoriedSkill } from './extension-inventory' import { ensureProjectWatched, getIndexedFiles, stopWatchingProject } from './watcher-bridge' import { createAppUpdater } from './updater' import type { UpdateController } from './update-controller' @@ -81,7 +85,7 @@ import { planFor } from '@ari/providers/package-manager' import { runInstall, type InstallHandle } from '@ari/providers/install' import { AcpDriver } from '@ari/providers/acp' import { resolveAcpLaunch, probeLaunch } from '@ari/providers/acp/launches' -import type { AcpLaunch } from '@ari/providers/acp/connection' +import type { AcpLaunch, AcpMcpServer } from '@ari/providers/acp/connection' import type { AcpTerminalLogin } from '@ari/providers/acp/protocol' import { ProviderAuthState, @@ -95,7 +99,12 @@ import { importPiSessionCandidate, listImportableSessions } from './session-impo import type { SessionImportDeps } from './session-import' import type { Driver } from '@ari/providers/driver' import { AriCoreDriver } from '@ari/ari-core/driver' +import { McpServerStore, mergeCoreMcp, publicMcpServer } from '@ari/ari-core/mcp-servers' +import { sanitizeMcpSegment } from '@ari/ari-core/mcp-tools' +import { listAriCoreSkills, readTrustedSkillRoots, setWorkspaceSkillTrust } from '@ari/ari-core/skills' +import { BUILT_IN_TOOLS } from '@ari/ari-core/tools' import { FileConversationStore } from '@ari/ari-core/conversation-store' +import type { McpServerConfig } from '@ari/ari-core/mcp-servers' import { todoFilenameFor } from '@ari/ari-core/todo' const log = createLogger('desktop:rpc') @@ -469,6 +478,9 @@ function publishAuthWall( } satisfies ProvidersUpdateFrame) } +let browserMcpServers: AcpMcpServer[] = [] +let browserCoreMcpServers: McpServerConfig[] = [] + /** * Registers each installed CLI driver as its detection resolves, preferring * the ACP transport (M16) with the legacy one-shot CLI driver as automatic @@ -497,8 +509,12 @@ function hydrateDrivers(registry: DriverRegistry): void { cliBinaryPath: detection.binaryPath, }) registry.register( - new AcpDriver(candidate.kind, launch, candidate.make(detection.binaryPath), (wall) => - publishAuthWall(candidate.kind, wall), + new AcpDriver( + candidate.kind, + launch, + candidate.make(detection.binaryPath), + (wall) => publishAuthWall(candidate.kind, wall), + () => browserMcpServers, ), ) log.info('driver registered', { @@ -579,13 +595,28 @@ export function registerRpc(contents: WebContents, options: RegisterRpcOptions = // The registry exists immediately with Ari Core attached; CLI drivers are // added as background detection completes. Nothing waits on that to answer. const driverRegistry = new DriverRegistry() + const coreDir = join(app.getPath('userData'), 'ari-core') + const mcpStore = new McpServerStore({ dir: coreDir }) + let mcpStoreReady: Promise | null = null + const ensureMcpStore = (): Promise => { + mcpStoreReady ??= mcpStore.load().then(() => undefined) + return mcpStoreReady + } + void ensureMcpStore() driverRegistry.register( new AriCoreDriver(getEndpointStore(), { // Ari Core owns its transcript (it has no provider-side thread to // resume), so conversation memory is persisted per session on disk. - conversations: new FileConversationStore( - join(app.getPath('userData'), 'ari-core', 'conversations'), - ), + conversations: new FileConversationStore(join(coreDir, 'conversations')), + skills: { homeDir: homedir(), trustDir: coreDir }, + mcpServers: () => { + for (const server of mcpStore.list()) { + if (!server.disabled && sanitizeMcpSegment(server.name) === 'ari_browser') { + log.warn('mcp server name is reserved', { server: server.name, reason: 'reserved-name' }) + } + } + return mergeCoreMcp(mcpStore.list(), browserCoreMcpServers) + }, }), ) driverRegistryRef = driverRegistry @@ -724,6 +755,58 @@ export function registerRpc(contents: WebContents, options: RegisterRpcOptions = }, ptyFactory, ) + const browsers = new BrowserService( + (_id, onUpdated) => { + const win = BrowserWindow.fromWebContents(contents) + if (win === null) throw new Error('browser host window is gone') + return createElectronBrowserGuest(win, onUpdated) + }, + (state) => rpcRegistry.publish('browser.updated', state), + ) + contents.once('destroyed', () => { + browsers.dispose() + browserMcpServers = [] + browserCoreMcpServers = [] + }) + void startBrowserMcpServer(browsers) + .then(async (handle) => { + const proxyPath = await writeBrowserMcpProxy(app.getPath('userData')) + const stdioEnv = [ + { name: 'ELECTRON_RUN_AS_NODE', value: '1' }, + { name: 'ARI_BROWSER_MCP_URL', value: handle.url }, + { name: 'ARI_BROWSER_MCP_TOKEN', value: handle.token }, + ] + browserMcpServers = [ + { + type: 'http', + name: 'ari-browser', + url: handle.url, + headers: [{ name: 'Authorization', value: `Bearer ${handle.token}` }], + }, + { + name: 'ari-browser', + command: process.execPath, + args: [proxyPath], + env: stdioEnv, + }, + ] + browserCoreMcpServers = [ + { + id: 'ari-browser', + name: 'ari-browser', + command: process.execPath, + args: [proxyPath], + env: { + ELECTRON_RUN_AS_NODE: '1', + ARI_BROWSER_MCP_URL: handle.url, + ARI_BROWSER_MCP_TOKEN: handle.token, + }, + disabled: false, + }, + ] + contents.once('destroyed', () => handle.close()) + }) + .catch((error: unknown) => log.warn('browser mcp failed to start', { error: String(error) })) const r = rpcRegistry r.register('ping', () => ({ pong: true, at: Date.now() })) @@ -1102,6 +1185,67 @@ export function registerRpc(contents: WebContents, options: RegisterRpcOptions = writeProviderConfig(params.kind, params.fileId, params.content), ) + const resolveInventoryWorkspace = async (workspacePath: string | null): Promise => { + if (workspacePath === null) return homedir() + return resolveInsideRoots(resolve(workspacePath), await collectFsRoots()) + } + + r.register('providers.extensionInventory', async (params) => { + const workspace = await resolveInventoryWorkspace(params.workspacePath) + if (params.kind === 'ari-core') { + await ensureMcpStore() + const trusted = await readTrustedSkillRoots(coreDir) + const skills = await listAriCoreSkills(workspace, trusted, { + homeDir: homedir(), + builtInNames: BUILT_IN_TOOLS.map((tool) => tool.name), + }) + return buildExtensionInventory({ + kind: 'ari-core', + workspacePath: workspace, + coreServers: mcpStore.list(), + coreSkills: skills, + }) + } + return buildExtensionInventory({ kind: params.kind, workspacePath: workspace }) + }) + + r.register('providers.readExtensionFile', async (params) => { + const workspace = await resolveInventoryWorkspace(params.workspacePath) + if (params.kind === 'ari-core') await ensureMcpStore() + const inventory = + params.kind === 'ari-core' + ? await buildExtensionInventory({ + kind: 'ari-core', + workspacePath: workspace, + coreServers: mcpStore.list(), + coreSkills: await listAriCoreSkills(workspace, await readTrustedSkillRoots(coreDir), { + homeDir: homedir(), + builtInNames: BUILT_IN_TOOLS.map((tool) => tool.name), + }), + }) + : await buildExtensionInventory({ kind: params.kind, workspacePath: workspace }) + return readInventoriedSkill(inventory, params.path) + }) + + r.register('ariCore.mcp.list', async () => { + await ensureMcpStore() + return { servers: mcpStore.list().map(publicMcpServer) } + }) + r.register('ariCore.mcp.upsert', async (params) => { + await ensureMcpStore() + const saved = await mcpStore.patch(params) + return publicMcpServer(saved) + }) + r.register('ariCore.mcp.remove', async (params) => { + await ensureMcpStore() + return { removed: await mcpStore.remove(params.id) } + }) + r.register('ariCore.skills.trust', async (params) => { + const workspace = await resolveInsideRoots(resolve(params.workspacePath), await collectFsRoots()) + const trusted = await setWorkspaceSkillTrust(coreDir, workspace, params.trusted) + return { trusted } + }) + // Merged model catalogs per kind: dynamic overlay → snapshot → static. r.register('providers.models', () => { void catalogService.refreshIfStale() @@ -1206,6 +1350,22 @@ export function registerRpc(contents: WebContents, options: RegisterRpcOptions = return { killed: true } }) + r.register('browser.open', async (params) => browsers.open(params.id, params.url)) + r.register('browser.navigate', async (params) => browsers.navigate(params.id, params.url)) + r.register('browser.go', (params) => browsers.go(params.id, params.action)) + r.register('browser.close', (params) => ({ closed: browsers.close(params.id) })) + r.register('browser.layout', (params) => ({ + applied: browsers.layout( + params.id, + { x: params.x, y: params.y, width: params.width, height: params.height }, + params.visible, + ), + })) + r.register('browser.pick', async (params) => browsers.pick(params.id)) + r.register('browser.cancelPick', async (params) => ({ + cancelled: await browsers.cancelPick(params.id), + })) + r.register('project.list', async () => getProjectStore().load()) r.register('project.add', async (params) => { diff --git a/apps/desktop/src/main/window.ts b/apps/desktop/src/main/window.ts index a7bf4f17..5afb3465 100644 --- a/apps/desktop/src/main/window.ts +++ b/apps/desktop/src/main/window.ts @@ -4,6 +4,7 @@ import { oklchToHex } from '@ari/ui/color' import { themeOf } from '@ari/ui/themes' import type { Theme } from '@ari/ui/themes' import { appDisplayName } from './dev-instance' +import { attachAppShellNavigationGuard } from './app-shell-guard' import { getSettingsStore } from './store' /** @@ -160,6 +161,8 @@ export function createMainWindow(): BrowserWindow { win.on('maximize', persistBounds) win.on('unmaximize', persistBounds) + attachAppShellNavigationGuard(win.webContents) + if (process.env['ELECTRON_RENDERER_URL']) { void win.loadURL(process.env['ELECTRON_RENDERER_URL']) } else { diff --git a/apps/desktop/src/renderer/src/App.tsx b/apps/desktop/src/renderer/src/App.tsx index 3c8385da..6dcd1272 100644 --- a/apps/desktop/src/renderer/src/App.tsx +++ b/apps/desktop/src/renderer/src/App.tsx @@ -6,7 +6,12 @@ import { ToastProvider, useToast } from '@ari/ui/toast' import { SessionImportDialog } from './features/providers' import { useUpdateToasts } from './features/providers/use-update-toasts' import { useAppUpdateToast } from './features/updates' -import type { RpcResults, SessionEventFrame, SessionSummary } from '@ari/contracts/rpc' +import type { + BrowserTabState, + RpcResults, + SessionEventFrame, + SessionSummary, +} from '@ari/contracts/rpc' import type { DriverKind, PermissionMode } from '@ari/contracts/common' import { createLogger } from '@ari/shared/logger' import { rpc } from './lib/rpc' @@ -21,6 +26,7 @@ import { sidebarOrder, } from './features/session/session-nav' import { descendantIds } from './features/session/session-tree' +import { BrowserPanel } from './features/browser/BrowserPanel' import { TerminalDock, TerminalPane } from './features/terminal' import { openTerminalTab, @@ -78,6 +84,7 @@ type InspectorId = Exclude /** Rail headings, and the accessible name of the rail itself. */ const INSPECTOR_TITLES: Record = { terminal: 'Terminal', + browser: 'Browser', changes: 'Changes', files: 'Files', usage: 'Usage', @@ -212,10 +219,20 @@ function Shell() { setInspector((prev) => (prev === 'terminal' ? null : 'terminal')) }, [leaveWorkspaceTool]) + useEffect(() => { + return rpc.subscribe('browser.updated', {}, (payload) => { + const next = payload as BrowserTabState + if (next.reveal !== true) return + leaveWorkspaceTool() + setFullPage(null) + setInspector('browser') + }) + }, [leaveWorkspaceTool]) + // Switching chats must not kill a running shell; every other rail still // yields to the session view the way it always has. const clearTransientInspector = useCallback(() => { - setInspector((prev) => (prev === 'terminal' ? prev : null)) + setInspector((prev) => (prev === 'terminal' || prev === 'browser' ? prev : null)) }, []) // Visiting a session lands on it and clears its settled badge — done/error @@ -389,6 +406,11 @@ function Shell() { setSettingsOpen(false) setFullPage(null) setInspector('terminal') + } else if (view === 'browser') { + setHubOpen(false) + setSettingsOpen(false) + setFullPage(null) + setInspector('browser') } else { setHubOpen(false) setSettingsOpen(false) @@ -753,6 +775,7 @@ function Shell() { section={settingsSection} onSectionChange={setSettingsSection} onBack={() => setSettingsOpen(false)} + workspacePath={sessionWorkspace?.path ?? null} onOpenTerminal={() => { setSettingsOpen(false) setFullPage(null) @@ -1065,6 +1088,12 @@ function Shell() { /> + ) : inspector === 'browser' ? ( +
+ + setInspector(null)} /> + +
) : ( <>
diff --git a/apps/desktop/src/renderer/src/features/browser/BrowserPanel.test.tsx b/apps/desktop/src/renderer/src/features/browser/BrowserPanel.test.tsx new file mode 100644 index 00000000..98ad73ce --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/BrowserPanel.test.tsx @@ -0,0 +1,69 @@ +import { render, screen } from '@testing-library/react' +import userEvent from '@testing-library/user-event' +import { beforeEach, describe, expect, it, vi } from 'vitest' +import { BrowserPanel } from './BrowserPanel' + +const { invokeFn, subscribeFn } = vi.hoisted(() => ({ + invokeFn: vi.fn(), + subscribeFn: vi.fn(() => () => undefined), +})) + +vi.mock('../../lib/rpc', () => ({ + rpc: { + invoke: invokeFn, + subscribe: subscribeFn, + }, +})) + +describe('BrowserPanel', () => { + beforeEach(() => { + invokeFn.mockReset() + subscribeFn.mockReset() + subscribeFn.mockReturnValue(() => undefined) + invokeFn.mockImplementation(async (method: string) => { + if (method === 'browser.open') { + return { + id: 'inspector', + url: 'about:blank', + title: '', + canGoBack: false, + canGoForward: false, + loading: false, + error: null, + } + } + if (method === 'browser.navigate') return { ok: true, tab: { id: 'inspector' } } + if (method === 'browser.go') return { id: 'inspector' } + if (method === 'browser.layout') return { applied: true } + if (method === 'browser.cancelPick') return { cancelled: true } + if (method === 'browser.pick') return { ok: false, error: 'cancelled' } + if (method === 'shell.openUrl') return { opened: true } + throw new Error(`unexpected method: ${method}`) + }) + }) + + it('opens a guest tab and navigates from the address bar', async () => { + const user = userEvent.setup() + render() + + expect(await screen.findByLabelText('Address')).toBeInTheDocument() + expect(screen.getByLabelText('Pick element for agent')).toBeDisabled() + expect(invokeFn).toHaveBeenCalledWith('browser.open', { id: 'inspector' }) + + await user.type(screen.getByLabelText('Address'), 'example.com') + await user.keyboard('{Enter}') + expect(invokeFn).toHaveBeenCalledWith('browser.navigate', { + id: 'inspector', + url: 'example.com', + }) + }) + + it('hides the guest when the panel unmounts', () => { + const { unmount } = render() + unmount() + expect(invokeFn).toHaveBeenCalledWith( + 'browser.layout', + expect.objectContaining({ id: 'inspector', visible: false }), + ) + }) +}) diff --git a/apps/desktop/src/renderer/src/features/browser/BrowserPanel.tsx b/apps/desktop/src/renderer/src/features/browser/BrowserPanel.tsx new file mode 100644 index 00000000..5eb44420 --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/BrowserPanel.tsx @@ -0,0 +1,222 @@ +import { useCallback, useEffect, useRef, useState } from 'react' +import { + ArrowLeft, + ArrowRight, + ExternalLink, + Globe, + MousePointer2, + RotateCw, + X, +} from 'lucide-react' +import { IconButton } from '@ari/ui/icon-button' +import { Input } from '@ari/ui/input' +import type { BrowserTabState } from '@ari/contracts/rpc' +import { rpc } from '../../lib/rpc' +import { addBrowserPick, fileFromPngBase64 } from './browser-picks' + +const TAB_ID = 'inspector' + +const EMPTY: BrowserTabState = { + id: TAB_ID, + url: 'about:blank', + title: '', + canGoBack: false, + canGoForward: false, + loading: false, + error: null, +} + +/** + * Inspector-rail browser: address chrome in the renderer, Chromium guest in + * the main process. The host div is only a bounds target — the page itself + * is a WebContentsView overlaid by main. + */ +export function BrowserPanel({ onClose }: { onClose?: () => void }) { + const hostRef = useRef(null) + const [tab, setTab] = useState(EMPTY) + const [draft, setDraft] = useState('') + const [picking, setPicking] = useState(false) + + const syncLayout = useCallback((visible: boolean) => { + const host = hostRef.current + if (host === null) return + const rect = host.getBoundingClientRect() + void rpc + .invoke('browser.layout', { + id: TAB_ID, + visible, + x: Math.round(rect.x), + y: Math.round(rect.y), + width: Math.round(rect.width), + height: Math.round(rect.height), + }) + .catch(() => undefined) + }, []) + + useEffect(() => { + void rpc + .invoke('browser.open', { id: TAB_ID }) + .then((next) => { + setTab(next) + if (next.url !== 'about:blank') setDraft(next.url) + }) + .catch(() => undefined) + const unsub = rpc.subscribe('browser.updated', { id: TAB_ID }, (payload) => { + const next = payload as BrowserTabState + if (next.id !== TAB_ID) return + setTab(next) + if (!next.loading) setDraft(next.url === 'about:blank' ? '' : next.url) + }) + return () => { + unsub() + void rpc.invoke('browser.cancelPick', { id: TAB_ID }) + void rpc.invoke('browser.layout', { + id: TAB_ID, + visible: false, + x: 0, + y: 0, + width: 0, + height: 0, + }) + } + }, []) + + const showGuest = tab.url !== 'about:blank' || tab.loading + + useEffect(() => { + const host = hostRef.current + if (host === null) return + const observer = new ResizeObserver(() => syncLayout(showGuest)) + observer.observe(host) + syncLayout(showGuest) + return () => observer.disconnect() + }, [syncLayout, showGuest]) + + const submit = (): void => { + void rpc + .invoke('browser.navigate', { id: TAB_ID, url: draft }) + .then((result) => { + if (!result.ok) setTab((prev) => ({ ...prev, error: result.error })) + }) + .catch(() => undefined) + } + + const go = (action: 'back' | 'forward' | 'reload'): void => { + void rpc.invoke('browser.go', { id: TAB_ID, action }).catch(() => undefined) + } + + const blank = !showGuest + + const openExternal = (): void => { + if (tab.url === 'about:blank') return + void rpc.invoke('shell.openUrl', { url: tab.url }).catch(() => undefined) + } + + const pickElement = (): void => { + if (blank || picking) return + setPicking(true) + void rpc + .invoke('browser.pick', { id: TAB_ID }) + .then((result) => { + if (result.ok) { + const image = + result.pngBase64 !== null + ? fileFromPngBase64(result.pngBase64, `${result.element.tag}.png`) + : null + addBrowserPick(result.element, image) + } + }) + .catch(() => undefined) + .finally(() => setPicking(false)) + } + + return ( +
+
+ + Browser +
+ {onClose !== undefined ? ( + } + aria-label="Close browser panel" + size="sm" + variant="ghost" + onClick={onClose} + /> + ) : null} +
+
+ } + aria-label="Back" + size="sm" + variant="ghost" + disabled={!tab.canGoBack} + onClick={() => go('back')} + /> + } + aria-label="Forward" + size="sm" + variant="ghost" + disabled={!tab.canGoForward} + onClick={() => go('forward')} + /> + } + aria-label="Reload" + size="sm" + variant="ghost" + onClick={() => go('reload')} + /> +
{ + event.preventDefault() + submit() + }} + > + setDraft(event.target.value)} + placeholder="Enter a URL" + aria-label="Address" + className="h-7 text-2xs" + invalid={tab.error !== null} + /> +
+ } + aria-label="Pick element for agent" + title="Pick an element to mention to the agent" + size="sm" + variant="ghost" + disabled={blank || picking} + onClick={pickElement} + /> + } + aria-label="Open in system browser" + size="sm" + variant="ghost" + disabled={blank} + onClick={openExternal} + /> +
+ {tab.error !== null ? ( +

{tab.error}

+ ) : null} +
+ {blank ? ( +
+

In-app browser

+

+ Type a URL, then pick an element to mention it to the agent. +

+
+ ) : null} +
+
+ ) +} diff --git a/apps/desktop/src/renderer/src/features/browser/ElementChips.tsx b/apps/desktop/src/renderer/src/features/browser/ElementChips.tsx new file mode 100644 index 00000000..9d8300f1 --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/ElementChips.tsx @@ -0,0 +1,34 @@ +import { MousePointer2, X } from 'lucide-react' +import { chipLabelFor, removeBrowserPick, type BrowserPick } from './browser-picks' + +export function ElementChips({ picks }: { picks: readonly BrowserPick[] }) { + if (picks.length === 0) return null + return ( +
+ + {picks.length} element{picks.length > 1 ? 's' : ''} with your next message: + + {picks.map((pick) => ( + + + {chipLabelFor(pick)} + + + ))} +
+ ) +} diff --git a/apps/desktop/src/renderer/src/features/browser/browser-picks.test.ts b/apps/desktop/src/renderer/src/features/browser/browser-picks.test.ts new file mode 100644 index 00000000..4783316f --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/browser-picks.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from 'vitest' +import { addBrowserPick, promptForPicks, takeBrowserPicks } from './browser-picks' +import type { PickedElement } from '@ari/contracts/rpc' + +const el: PickedElement = { + url: 'https://example.com/', + selector: 'h1', + tag: 'h1', + text: 'Hello', + role: null, + ariaLabel: null, + html: '

Hello

', + x: 0, + y: 0, + width: 10, + height: 10, +} + +describe('browser-picks', () => { + it('formats taken picks as element_context for the agent', () => { + takeBrowserPicks() + addBrowserPick(el, null) + const taken = takeBrowserPicks() + expect(taken).toHaveLength(1) + expect(promptForPicks(taken)).toContain('') + expect(promptForPicks(taken)).toContain('h1') + expect(takeBrowserPicks()).toHaveLength(0) + }) +}) diff --git a/apps/desktop/src/renderer/src/features/browser/browser-picks.ts b/apps/desktop/src/renderer/src/features/browser/browser-picks.ts new file mode 100644 index 00000000..ae29e357 --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/browser-picks.ts @@ -0,0 +1,64 @@ +import type { PickedElement } from '@ari/contracts/rpc' +import { formatElementContexts, elementChipLabel } from './element-context' + +export interface BrowserPick { + id: string + element: PickedElement + image: File | null +} + +let picks: BrowserPick[] = [] +const listeners = new Set<() => void>() +let seq = 0 + +function emit(): void { + for (const listener of listeners) listener() +} + +export function subscribeBrowserPicks(listener: () => void): () => void { + listeners.add(listener) + return () => listeners.delete(listener) +} + +export function browserPicksState(): readonly BrowserPick[] { + return picks +} + +export function addBrowserPick(element: PickedElement, image: File | null): BrowserPick { + const pick: BrowserPick = { id: `el_${++seq}`, element, image } + picks = [...picks, pick] + emit() + return pick +} + +export function removeBrowserPick(id: string): void { + picks = picks.filter((pick) => pick.id !== id) + emit() +} + +export function takeBrowserPicks(): BrowserPick[] { + const taken = picks + picks = [] + emit() + return taken +} + +export function restoreBrowserPicks(next: readonly BrowserPick[]): void { + picks = [...next] + emit() +} + +export function promptForPicks(items: readonly BrowserPick[]): string { + return formatElementContexts(items.map((pick) => pick.element)) +} + +export function chipLabelFor(pick: BrowserPick): string { + return elementChipLabel(pick.element) +} + +export function fileFromPngBase64(base64: string, name: string): File { + const binary = atob(base64) + const bytes = new Uint8Array(binary.length) + for (let i = 0; i < binary.length; i++) bytes[i] = binary.charCodeAt(i) + return new File([bytes], name, { type: 'image/png' }) +} diff --git a/apps/desktop/src/renderer/src/features/browser/element-context.ts b/apps/desktop/src/renderer/src/features/browser/element-context.ts new file mode 100644 index 00000000..8ae9ab6d --- /dev/null +++ b/apps/desktop/src/renderer/src/features/browser/element-context.ts @@ -0,0 +1,32 @@ +import type { PickedElement } from '@ari/contracts/rpc' + +const TEXT_CAP = 400 +const HTML_CAP = 1_500 + +export function formatElementContext(element: PickedElement): string { + const lines = [ + `Selected element on ${element.url}`, + `- tag: ${element.tag}`, + `- selector: ${element.selector}`, + ] + if (element.role) lines.push(`- role: ${element.role}`) + if (element.ariaLabel) lines.push(`- aria-label: ${element.ariaLabel}`) + if (element.text) lines.push(`- text: ${element.text.slice(0, TEXT_CAP)}`) + if (element.html) lines.push(`- html: ${element.html.slice(0, HTML_CAP)}`) + return lines.join('\n') +} + +export function formatElementContexts(elements: readonly PickedElement[]): string { + if (elements.length === 0) return '' + const body = elements + .map((el, i) => `Element ${String(i + 1)}\n${formatElementContext(el)}`) + .join('\n\n') + return `\n${body}\n` +} + +export function elementChipLabel(element: PickedElement): string { + const text = element.text.trim().replace(/\s+/g, ' ') + if (text.length > 0) return text.length > 32 ? `${text.slice(0, 31)}…` : text + if (element.ariaLabel) return element.ariaLabel + return element.selector +} diff --git a/apps/desktop/src/renderer/src/features/palette/useCommands.test.ts b/apps/desktop/src/renderer/src/features/palette/useCommands.test.ts index a7e26192..06e9fe42 100644 --- a/apps/desktop/src/renderer/src/features/palette/useCommands.test.ts +++ b/apps/desktop/src/renderer/src/features/palette/useCommands.test.ts @@ -20,6 +20,7 @@ describe('buildAppCommands', () => { for (const view of [ 'sessions', 'terminal', + 'browser', 'changes', 'settings', 'files', diff --git a/apps/desktop/src/renderer/src/features/palette/useCommands.ts b/apps/desktop/src/renderer/src/features/palette/useCommands.ts index 01a5472f..fa5db950 100644 --- a/apps/desktop/src/renderer/src/features/palette/useCommands.ts +++ b/apps/desktop/src/renderer/src/features/palette/useCommands.ts @@ -4,6 +4,7 @@ import { Folder, Gauge, GitPullRequest, + Globe, Images, Maximize2, MessageSquare, @@ -35,6 +36,7 @@ export interface PaletteCommand { export type NavigableView = | 'sessions' | 'terminal' + | 'browser' | 'changes' | 'settings' | 'files' @@ -123,6 +125,12 @@ export function buildAppCommands(ctx: CommandsContext): PaletteCommand[] { hint: 'Ctrl+`', run: () => ctx.onNavigate('terminal'), }, + { + id: 'nav.browser', + label: 'Go to Browser', + icon: Globe, + run: () => ctx.onNavigate('browser'), + }, { id: 'nav.changes', label: 'Go to Changes', diff --git a/apps/desktop/src/renderer/src/features/providers/AgentConfigSettings.test.tsx b/apps/desktop/src/renderer/src/features/providers/AgentConfigSettings.test.tsx index ba5e20e9..ab344a28 100644 --- a/apps/desktop/src/renderer/src/features/providers/AgentConfigSettings.test.tsx +++ b/apps/desktop/src/renderer/src/features/providers/AgentConfigSettings.test.tsx @@ -41,6 +41,7 @@ beforeEach(() => { if (method === 'providers.writeConfig') return { ok: true, bytesWritten: 12 } // The pi page also offers session import. if (method === 'sessions.importable') return [] + if (method === 'providers.extensionInventory') return { records: [], truncated: false } throw new Error(`unexpected ${method}`) }) }) @@ -89,6 +90,7 @@ describe('AgentConfigSettings', () => { return { content: '{}', exists: true, path: 'x', truncated: false } } if (method === 'sessions.importable') return [] + if (method === 'providers.extensionInventory') return { records: [], truncated: false } return { ok: false, error: 'settings.json is not valid JSON: Unexpected end of input' } }) render() @@ -110,9 +112,11 @@ describe('AgentConfigSettings', () => { }) it('says so plainly when Ari has no layout for an agent', async () => { - mocks.invoke.mockImplementation(async (method: string) => - method === 'sessions.importable' ? [] : { dir: null, files: [] }, - ) + mocks.invoke.mockImplementation(async (method: string) => { + if (method === 'sessions.importable') return [] + if (method === 'providers.extensionInventory') return { records: [], truncated: false } + return { dir: null, files: [] } + }) render() expect(await screen.findByText(/no confirmed config layout/i)).toBeInTheDocument() }) diff --git a/apps/desktop/src/renderer/src/features/providers/AgentConfigSettings.tsx b/apps/desktop/src/renderer/src/features/providers/AgentConfigSettings.tsx index cae63845..1691e299 100644 --- a/apps/desktop/src/renderer/src/features/providers/AgentConfigSettings.tsx +++ b/apps/desktop/src/renderer/src/features/providers/AgentConfigSettings.tsx @@ -8,6 +8,7 @@ import { Textarea } from '@ari/ui/textarea' import { createLogger } from '@ari/shared/logger' import { SettingsPage } from '../settings/SettingsPage' import { SessionImport } from './SessionImport' +import { ExtensionInventory } from './ExtensionInventory' import { rpc } from '../../lib/rpc' const log = createLogger('ui:agent-config') @@ -19,7 +20,7 @@ type ConfigFile = RpcResults['providers.configFiles']['files'][number] * process is the authority — an agent with no mapping answers with an empty * file list — but the picker needs names before any call returns. */ -const CONFIGURABLE: DriverKind[] = ['pi', 'claude', 'codex', 'opencode', 'grok'] +const CONFIGURABLE: DriverKind[] = ['pi', 'claude', 'codex', 'opencode', 'grok', 'ari-core'] const KIND_LABELS: Partial> = { pi: 'pi', @@ -27,6 +28,7 @@ const KIND_LABELS: Partial> = { codex: 'Codex', opencode: 'OpenCode', grok: 'Grok', + 'ari-core': 'Ari Core', } /** @@ -40,7 +42,7 @@ const KIND_LABELS: Partial> = { * refuses a JSON file it could not parse, which is the one mistake that loses * an agent's configuration silently. */ -export function AgentConfigSettings() { +export function AgentConfigSettings({ workspacePath = null }: { workspacePath?: string | null }) { const [kind, setKind] = useState('pi') const [dir, setDir] = useState(null) const [files, setFiles] = useState([]) @@ -156,7 +158,7 @@ export function AgentConfigSettings() { {loading ? ( - ) : files.length === 0 ? ( + ) : files.length === 0 && kind !== 'ari-core' ? (

Ari has no confirmed config layout for this agent, so it will not guess at a path.

@@ -230,6 +232,8 @@ export function AgentConfigSettings() { ) : null} + + {kind === 'pi' ? : null} ) diff --git a/apps/desktop/src/renderer/src/features/providers/ExtensionInventory.tsx b/apps/desktop/src/renderer/src/features/providers/ExtensionInventory.tsx new file mode 100644 index 00000000..9ab6dbfb --- /dev/null +++ b/apps/desktop/src/renderer/src/features/providers/ExtensionInventory.tsx @@ -0,0 +1,270 @@ +import { useCallback, useEffect, useState } from 'react' +import type { DriverKind } from '@ari/contracts/common' +import type { RpcResults } from '@ari/contracts/rpc' +import { Badge } from '@ari/ui/badge' +import { Button } from '@ari/ui/button' +import { Spinner } from '@ari/ui/spinner' +import { rpc } from '../../lib/rpc' + +type RecordRow = RpcResults['providers.extensionInventory']['records'][number] +type PublicServer = RpcResults['ariCore.mcp.list']['servers'][number] + +const DELIVERY: Record = { + delegated: 'Loaded by the agent', + injected: 'Ari browser', + hosted: 'Ari Core', +} + +const PROBLEM: Record, string> = { + 'missing-binary': 'missing binary', + unreadable: 'unreadable', + 'duplicate-name': 'name collides with ari-browser', + 'untrusted-project': 'not trusted', +} + +/** + * Skills and MCP servers the selected agent will see. Pass-through rows are + * read-only. Ari Core rows can be toggled; env values are write-only. + */ +export function ExtensionInventory({ + kind, + workspacePath, +}: { + kind: DriverKind + workspacePath: string | null +}) { + const [records, setRecords] = useState([]) + const [truncated, setTruncated] = useState(false) + const [loading, setLoading] = useState(true) + const [error, setError] = useState(null) + const [preview, setPreview] = useState(null) + const [servers, setServers] = useState([]) + + const refresh = useCallback(async () => { + setLoading(true) + setError(null) + try { + const listed = await rpc.invoke('providers.extensionInventory', { kind, workspacePath }) + setRecords(listed.records ?? []) + setTruncated(listed.truncated ?? false) + if (kind === 'ari-core') { + const mcp = await rpc.invoke('ariCore.mcp.list') + setServers(mcp.servers) + } else { + setServers([]) + } + } catch (caught: unknown) { + setRecords([]) + setError(caught instanceof Error ? caught.message : String(caught)) + } finally { + setLoading(false) + } + }, [kind, workspacePath]) + + useEffect(() => { + void refresh() + }, [refresh]) + + const untrusted = records.filter((row) => row.problem === 'untrusted-project').length + + return ( +
+

+ Skills and MCP +

+

+ {kind === 'ari-core' + ? 'Ari Core loads these itself. User skills apply immediately on the next message. Project skills stay off until you trust this folder.' + : 'Claude Code and Codex load these themselves. Ari only adds the in-app browser. Changes show up on the next message. If a resumed chat still lacks a server, start a new chat.'} +

+ {loading ? : null} + {error ? ( +

+ {error} +

+ ) : null} + {truncated ?

The list is truncated.

: null} + {!loading && records.length === 0 ? ( +

No skills or MCP servers found for this agent.

+ ) : ( +
    + {records.map((row) => ( +
  • +
    + {row.name} + {row.kind} + {row.scope} + + {row.provider === 'claude' + ? 'Loaded by Claude' + : row.provider === 'codex' + ? 'Loaded by Codex' + : DELIVERY[row.delivery]} + + {row.disabled ? disabled : null} + {row.problem ? {PROBLEM[row.problem]} : null} +
    + {row.summary ?

    {row.summary}

    : null} + {row.command ? ( +

    {row.command}

    + ) : null} + {row.problem === 'duplicate-name' ? ( +

    + This name collides with the in-app browser. Ari still sends its own ari-browser. +

    + ) : null} +
    + {row.kind === 'skill' && row.sourcePath ? ( + + ) : null} + {kind === 'ari-core' && row.kind === 'mcp' && row.scope === 'user' ? ( + + ) : null} +
    +
    • + ))} +
+ )} + {preview ? ( + 
+          {preview}
+
+ ) : null} + {kind === 'ari-core' && untrusted > 0 && workspacePath ? ( + + ) : null} + {kind === 'ari-core' ? void refresh()} /> : null} +
+ ) +} + +function CoreMcpForm({ servers, onSaved }: { servers: PublicServer[]; onSaved: () => void }) { + const [name, setName] = useState('') + const [command, setCommand] = useState('') + const [args, setArgs] = useState('') + const [env, setEnv] = useState('') + const [error, setError] = useState(null) + + return ( +
{ + event.preventDefault() + const envRecord: Record = {} + for (const line of env.split('\n')) { + const eq = line.indexOf('=') + if (eq <= 0) continue + envRecord[line.slice(0, eq).trim()] = line.slice(eq + 1) + } + void rpc + .invoke('ariCore.mcp.upsert', { + name: name.trim(), + command: command.trim(), + args: args.split('\n').map((line) => line.trim()).filter((line) => line.length > 0), + ...(env.trim().length > 0 ? { env: envRecord } : {}), + }) + .then(() => { + setName('') + setCommand('') + setArgs('') + setEnv('') + setError(null) + onSaved() + }) + .catch((caught: unknown) => setError(caught instanceof Error ? caught.message : String(caught))) + }} + > +

Add an Ari Core MCP server

+

+ {servers.length} saved. Env values are stored and never shown again. Leave env blank to keep + existing values when you disable a server above. +

+ + +