diff --git a/apps/desktop/src/ipc/methods/preview.ts b/apps/desktop/src/ipc/methods/preview.ts index 5fb7eff99fc6..61432a77e715 100644 --- a/apps/desktop/src/ipc/methods/preview.ts +++ b/apps/desktop/src/ipc/methods/preview.ts @@ -2,6 +2,7 @@ import { DesktopPreviewAnnotationThemeInputSchema, DesktopPreviewArtifactInputSchema, DesktopPreviewAutomationClickInputSchema, + DesktopPreviewAutomationSnapshotInputSchema, DesktopPreviewAutomationEvaluateInputSchema, DesktopPreviewAutomationPressInputSchema, DesktopPreviewAutomationScrollInputSchema, @@ -389,11 +390,11 @@ export const automationStatus = DesktopIpc.makeIpcMethod({ export const automationSnapshot = DesktopIpc.makeIpcMethod({ channel: IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, - payload: DesktopPreviewTabInputSchema, + payload: DesktopPreviewAutomationSnapshotInputSchema, result: PreviewAutomationSnapshot, - handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId }) { + handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId, include }) { const manager = yield* PreviewManager.PreviewManager; - return yield* manager.automationSnapshot(tabId); + return yield* manager.automationSnapshot(tabId, include ?? []); }), }); diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 685a9b1204db..27821ea81252 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -273,8 +273,11 @@ contextBridge.exposeInMainWorld("desktopBridge", { automation: { status: (tabId) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }), - snapshot: (tabId) => - ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId }), + snapshot: (tabId, include) => + ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { + tabId, + ...(include === undefined ? {} : { include }), + }), click: (tabId, input) => ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, { tabId, input }), type: (tabId, input) => diff --git a/apps/desktop/src/preview/Manager.test.ts b/apps/desktop/src/preview/Manager.test.ts index a7b3afabd3c3..b0de213068cd 100644 --- a/apps/desktop/src/preview/Manager.test.ts +++ b/apps/desktop/src/preview/Manager.test.ts @@ -140,6 +140,14 @@ vi.mock("electron", () => ({ }, nativeImage: { createFromPath, + createFromBuffer: (buffer: Buffer) => ({ + getSize: () => ({ width: buffer.length > 0 ? 1 : 0, height: buffer.length > 0 ? 1 : 0 }), + toPNG: () => buffer, + resize: () => ({ + getSize: () => ({ width: 1, height: 1 }), + toPNG: () => buffer, + }), + }), }, shell: { showItemInFolder, @@ -262,6 +270,7 @@ const makeTestPreviewWebContents = ( getURL: () => "https://example.com", getTitle: () => "Example", isLoading: () => false, + isDevToolsOpened: () => false, getZoomFactor: () => 1, setZoomFactor: vi.fn(), setAudioMuted: vi.fn(), @@ -3967,3 +3976,327 @@ describe("Preview automation diagnostics", () => { expect("locator" in error).toBe(false); }); }); + +describe("Preview automation snapshots", () => { + const pageValue = { + url: "https://example.com", + title: "Example", + loading: false, + visibleText: "Dashboard", + interactiveElements: [], + }; + + const snapshotImage = { + getSize: () => ({ width: 100, height: 80 }), + toPNG: () => Buffer.from("png"), + resize: () => ({ + getSize: () => ({ width: 100, height: 80 }), + toPNG: () => Buffer.from("png"), + }), + }; + + const mockAutomationWebContents = ( + sendCommand: ReturnType, + capturePage: () => Promise, + options: { + readonly id?: number; + readonly isDebuggerAttached?: () => boolean; + } = {}, + ) => + ({ + id: options.id ?? 42, + isDestroyed: () => false, + getType: () => "webview", + getURL: () => "https://example.com", + getTitle: () => "Example", + isLoading: () => false, + isDevToolsOpened: () => false, + getZoomFactor: () => 1, + setZoomFactor: vi.fn(), + setAudioMuted: vi.fn(), + on: vi.fn(), + off: vi.fn(), + ipc: { on: vi.fn(), off: vi.fn() }, + send: webviewSend, + navigationHistory: { canGoBack: () => false, canGoForward: () => false }, + setWindowOpenHandler: vi.fn(), + capturePage, + debugger: { + isAttached: options.isDebuggerAttached ?? (() => false), + attach: vi.fn(), + sendCommand, + on: vi.fn(), + off: vi.fn(), + }, + }) as never; + + effectIt.effect("finalizes the timeline when control-session acquisition fails", () => + withManager((manager) => + Effect.gen(function* () { + let debuggerAttached = true; + const sendCommand = vi.fn(async (method: string) => + method === "Runtime.evaluate" ? { result: { value: pageValue } } : undefined, + ); + fromId.mockReturnValue( + mockAutomationWebContents(sendCommand, async () => snapshotImage, { + isDebuggerAttached: () => debuggerAttached, + }), + ); + + yield* manager.createTab("tab_acquisition_failure"); + yield* manager.registerWebview("tab_acquisition_failure", 42); + const exit = yield* Effect.exit( + manager.automationEvaluate("tab_acquisition_failure", { expression: "location.href" }), + ); + expect(Exit.isFailure(exit)).toBe(true); + + debuggerAttached = false; + const snapshot = yield* manager.automationSnapshot("tab_acquisition_failure"); + expect(snapshot.actionTimeline).toContainEqual( + expect.objectContaining({ + action: "evaluate", + status: "failed", + completedAt: expect.any(String), + }), + ); + expect(snapshot.actionTimeline).not.toContainEqual( + expect.objectContaining({ action: "evaluate", status: "running" }), + ); + }), + ), + ); + + effectIt.effect("retries against the live webview when guests swap during acquisition", () => + withManager((manager) => + Effect.gen(function* () { + let release42: () => void = () => undefined; + let release43: () => void = () => undefined; + const diagnostic42 = new Promise((resolve) => { + release42 = resolve; + }); + const diagnostic43 = new Promise((resolve) => { + release43 = resolve; + }); + const makeSendCommand = (diagnostic: Promise | null) => + vi.fn(async (method: string) => { + if (method === "Accessibility.enable" && diagnostic) await diagnostic; + if (method === "Runtime.evaluate") return { result: { value: pageValue } }; + if (method === "Accessibility.getFullAXTree") return { nodes: [] }; + return undefined; + }); + const send42 = makeSendCommand(diagnostic42); + const send43 = makeSendCommand(diagnostic43); + const send44 = makeSendCommand(null); + const guests = new Map([ + [42, mockAutomationWebContents(send42, async () => snapshotImage, { id: 42 })], + [43, mockAutomationWebContents(send43, async () => snapshotImage, { id: 43 })], + [44, mockAutomationWebContents(send44, async () => snapshotImage, { id: 44 })], + ]); + fromId.mockImplementation((id) => guests.get(id ?? -1) ?? null); + + yield* manager.createTab("tab_webview_swap"); + yield* manager.registerWebview("tab_webview_swap", 42); + yield* Effect.yieldNow; + const snapshotFiber = yield* manager + .automationSnapshot("tab_webview_swap", ["ax"]) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + expect(send42).toHaveBeenCalledWith("Accessibility.enable"); + + yield* manager.registerWebview("tab_webview_swap", 43); + yield* Effect.yieldNow; + release42(); + yield* Effect.yieldNow; + expect(send43).toHaveBeenCalledWith("Accessibility.enable"); + + yield* manager.registerWebview("tab_webview_swap", 44); + yield* Effect.yieldNow; + release43(); + yield* Fiber.join(snapshotFiber); + + expect(send42).not.toHaveBeenCalledWith("Runtime.evaluate", expect.anything()); + expect(send43).not.toHaveBeenCalledWith("Runtime.evaluate", expect.anything()); + expect(send44).toHaveBeenCalledWith("Runtime.evaluate", expect.anything()); + }), + ), + ); + + effectIt.effect("does not clear agent control after a pipelined action starts", () => + withManager((manager) => + Effect.gen(function* () { + let releaseFirst: () => void = () => undefined; + let releaseSecond: () => void = () => undefined; + const firstEvaluation = new Promise((resolve) => { + releaseFirst = resolve; + }); + const secondEvaluation = new Promise((resolve) => { + releaseSecond = resolve; + }); + let evaluationCount = 0; + const sendCommand = vi.fn(async (method: string) => { + if (method !== "Runtime.evaluate") return undefined; + evaluationCount += 1; + await (evaluationCount === 1 ? firstEvaluation : secondEvaluation); + return { result: { value: null } }; + }); + fromId.mockReturnValue(mockAutomationWebContents(sendCommand, async () => snapshotImage)); + const controllers: string[] = []; + yield* manager.subscribeStateChanges((_tabId, state) => + Effect.sync(() => { + controllers.push(state.controller); + }), + ); + + yield* manager.createTab("tab_pipeline"); + yield* manager.registerWebview("tab_pipeline", 42); + yield* Effect.yieldNow; + const first = yield* manager + .automationEvaluate("tab_pipeline", { expression: "1" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + expect(evaluationCount).toBe(1); + + const second = yield* manager + .automationEvaluate("tab_pipeline", { expression: "2" }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.yieldNow; + expect(evaluationCount).toBe(1); + + releaseFirst(); + yield* Effect.yieldNow; + yield* Effect.yieldNow; + expect(evaluationCount).toBe(2); + expect(controllers.at(-1)).toBe("agent"); + + releaseSecond(); + yield* Fiber.join(first); + yield* Fiber.join(second); + expect(controllers.at(-1)).toBe("none"); + }), + ), + ); + + effectIt.effect("omits ax, console, and network unless include asks", () => + withManager((manager) => + Effect.gen(function* () { + const sendCommand = vi.fn(async (method: string, _params?: Record) => { + if (method === "Runtime.evaluate") { + return { result: { value: pageValue } }; + } + if (method === "Accessibility.getFullAXTree") { + return { nodes: [{ role: "main" }] }; + } + return undefined; + }); + fromId.mockReturnValue(mockAutomationWebContents(sendCommand, async () => snapshotImage)); + + yield* manager.createTab("tab_snapshot"); + yield* manager.registerWebview("tab_snapshot", 42); + const slim = yield* manager.automationSnapshot("tab_snapshot"); + expect(slim.accessibilityTree).toBeUndefined(); + expect(slim.consoleEntries).toEqual([]); + expect(slim.networkEntries).toEqual([]); + const slimMethods = sendCommand.mock.calls.map(([method]) => method); + expect(slimMethods).not.toContain("Accessibility.getFullAXTree"); + expect(slimMethods).not.toContain("Accessibility.enable"); + expect(slimMethods).not.toContain("Network.enable"); + expect(slimMethods).not.toContain("Log.enable"); + expect( + sendCommand.mock.calls.some( + ([method, params]) => + method === "Runtime.evaluate" && + typeof params === "object" && + params !== null && + "expression" in params && + typeof params.expression === "string" && + params.expression.includes('main, [role="main"]'), + ), + ).toBe(true); + + sendCommand.mockClear(); + const withAx = yield* manager.automationSnapshot("tab_snapshot", ["ax"]); + expect(withAx.accessibilityTree).toEqual({ nodes: [{ role: "main" }] }); + expect(sendCommand.mock.calls.map(([method]) => method)).toContain( + "Accessibility.getFullAXTree", + ); + expect(sendCommand.mock.calls.map(([method]) => method)).toContain("Accessibility.enable"); + + sendCommand.mockClear(); + yield* manager.automationSnapshot("tab_snapshot", ["console", "network"]); + expect(sendCommand.mock.calls.map(([method]) => method)).toEqual( + expect.arrayContaining(["Log.enable", "Network.enable"]), + ); + }), + ), + ); + + effectIt.effect("fails the snapshot when capturePage and CDP screenshot both miss", () => + withManager((manager) => + Effect.gen(function* () { + const sendCommand = vi.fn(async (method: string) => { + if (method === "Runtime.evaluate") { + return { result: { value: pageValue } }; + } + if (method === "Page.captureScreenshot") { + return {}; + } + return undefined; + }); + fromId.mockReturnValue( + mockAutomationWebContents(sendCommand, async () => { + throw new Error("capturePage failed"); + }), + ); + + yield* manager.createTab("tab_snapshot_fail"); + yield* manager.registerWebview("tab_snapshot_fail", 42); + const exit = yield* Effect.exit(manager.automationSnapshot("tab_snapshot_fail")); + expect(Exit.isFailure(exit)).toBe(true); + if (Exit.isSuccess(exit)) return; + expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({ + _tag: "PreviewOperationError", + operation: "automationSnapshot.capturePage", + }); + }), + ), + ); + + effectIt.effect("defaults waitFor text search to the main landmark", () => + withManager((manager) => + Effect.gen(function* () { + const expressions: string[] = []; + const sendCommand = vi.fn(async (method: string, params?: Record) => { + if (method === "Runtime.evaluate") { + expressions.push(String(params?.expression ?? "")); + return { result: { value: { matched: true } } }; + } + return undefined; + }); + fromId.mockReturnValue(mockAutomationWebContents(sendCommand, async () => snapshotImage)); + + yield* manager.createTab("tab_wait"); + yield* manager.registerWebview("tab_wait", 42); + yield* manager.automationWaitFor("tab_wait", { + locator: "text=Dashboard", + text: "Dashboard", + }); + expect(expressions.some((expression) => expression.includes('main, [role="main"]'))).toBe( + true, + ); + expect(expressions.some((expression) => expression.includes("searchRoots.some"))).toBe( + true, + ); + expect( + expressions.some((expression) => expression.includes('data-slot$="-viewport"')), + ).toBe(false); + expect( + expressions.some((expression) => expression.includes('slot === "dialog-trigger"')), + ).toBe(true); + expect( + expressions.some((expression) => expression.includes('slot.includes("trigger")')), + ).toBe(false); + expect(expressions.some((expression) => expression.includes("getRootNode"))).toBe(true); + }), + ), + ); +}); diff --git a/apps/desktop/src/preview/Manager.ts b/apps/desktop/src/preview/Manager.ts index 324b92034f36..2eaef1bf8230 100644 --- a/apps/desktop/src/preview/Manager.ts +++ b/apps/desktop/src/preview/Manager.ts @@ -27,6 +27,7 @@ import type { PreviewAutomationNetworkEntry, PreviewAutomationScrollInput, PreviewAutomationSnapshot, + PreviewAutomationSnapshotInclude, PreviewAutomationTypeInput, PreviewAutomationWaitForInput, } from "@t3tools/contracts"; @@ -445,6 +446,7 @@ interface BrowserControlSession { readonly debugger: Electron.Debugger; readonly semaphore: Semaphore.Semaphore; readonly scope: Scope.Closeable; + readonly diagnosticDomains: Ref.Ref>; readonly onMessage: ( event: Electron.Event, method: string, @@ -452,6 +454,12 @@ interface BrowserControlSession { ) => void; } +const DIAGNOSTIC_ENABLE_METHODS = { + ax: "Accessibility.enable", + console: "Log.enable", + network: "Network.enable", +} as const satisfies Record; + interface BrowserDiagnostics { readonly consoleEntries: ReadonlyArray; readonly networkEntries: ReadonlyArray; @@ -1016,7 +1024,15 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function webContentsId: number, method: string, params: Record, + diagnosticDomains: ReadonlySet, ) { + const capturesConsole = + diagnosticDomains.has("console") && + (method === "Runtime.consoleAPICalled" || + method === "Runtime.exceptionThrown" || + method === "Log.entryAdded"); + const capturesNetwork = diagnosticDomains.has("network") && method.startsWith("Network."); + if (!capturesConsole && !capturesNetwork) return; const timestamp = yield* currentIso; yield* Ref.update(diagnosticsRef, (allDiagnostics) => { const current = allDiagnostics.get(webContentsId); @@ -1160,10 +1176,45 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function ); }); + const enableDiagnosticDomains = Effect.fn("PreviewManager.enableDiagnosticDomains")(function* ( + control: BrowserControlSession, + requested: ReadonlySet, + ) { + if (requested.size === 0) return; + yield* control.semaphore.withPermit( + Effect.gen(function* () { + const current = (yield* SynchronizedRef.get(controlSessionsRef)).get(control.webContentsId); + if (current !== control) return; + const enabled = yield* Ref.get(control.diagnosticDomains); + const missing = [...requested].filter((domain) => !enabled.has(domain)); + if (missing.length === 0) return; + const currentWc = webContents.fromId(control.webContentsId); + if (!currentWc || currentWc.isDestroyed()) return; + yield* Effect.forEach( + missing, + (domain) => + Effect.gen(function* () { + const method = DIAGNOSTIC_ENABLE_METHODS[domain]; + yield* attemptPromise( + { operation: `enableDebugger.${method}`, webContentsId: currentWc.id }, + () => control.debugger.sendCommand(method), + ); + yield* Ref.update( + control.diagnosticDomains, + (current) => new Set([...current, domain]), + ); + }), + { discard: true }, + ); + }), + ); + }); + const ensureControlSession = Effect.fn("PreviewManager.ensureControlSession")(function* ( wc: Electron.WebContents, + requestedDiagnosticDomains: ReadonlySet = new Set(), ) { - return yield* SynchronizedRef.modifyEffect( + const control = yield* SynchronizedRef.modifyEffect( controlSessionsRef, ( sessions, @@ -1172,7 +1223,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function PreviewManagerError > => { const existing = sessions.get(wc.id); - if (existing) return Effect.succeed([existing, sessions] as const); + if (existing) { + return Effect.succeed([existing, sessions] as const); + } if (wc.isDevToolsOpened()) { return Effect.fail( new PreviewAutomationDevToolsOpenError({ @@ -1189,6 +1242,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } const createControlSession = Effect.fn("PreviewManager.createControlSession")(function* () { const semaphore = yield* Semaphore.make(1); + const diagnosticDomains = yield* Ref.make>( + new Set(requestedDiagnosticDomains), + ); const scope = yield* Scope.fork(parentScope, "sequential"); const wcDebugger = wc.debugger; const handleDebuggerMessage = Effect.fnUntraced(function* ( @@ -1236,7 +1292,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function } } } - yield* captureDiagnosticMessage(wc.id, method, params); + const activeDiagnosticDomains = yield* Ref.get(diagnosticDomains); + yield* captureDiagnosticMessage(wc.id, method, params, activeDiagnosticDomains); }); const onMessage: BrowserControlSession["onMessage"] = (_event, method, params) => { runFork(handleDebuggerMessage(method, params)); @@ -1263,6 +1320,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function debugger: wcDebugger, semaphore, scope, + diagnosticDomains, onMessage, }; const initialize = Effect.fn("PreviewManager.initializeControlSession")(function* () { @@ -1280,12 +1338,16 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function wcDebugger.attach("1.3"); }); yield* Effect.all( - ["Runtime.enable", "Accessibility.enable", "Network.enable", "Log.enable"].map( - (method) => - attemptPromise( - { operation: `initializeDebugger.${method}`, webContentsId: wc.id }, - () => wcDebugger.sendCommand(method), - ), + [ + "Runtime.enable", + ...[...requestedDiagnosticDomains].map( + (domain) => DIAGNOSTIC_ENABLE_METHODS[domain], + ), + ].map((method) => + attemptPromise( + { operation: `initializeDebugger.${method}`, webContentsId: wc.id }, + () => wcDebugger.sendCommand(method), + ), ), { concurrency: "unbounded", discard: true }, ); @@ -1303,6 +1365,8 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return createControlSession(); }, ); + yield* enableDiagnosticDomains(control, requestedDiagnosticDomains); + return control; }); const pushAction = (tabId: string, event: PreviewAutomationActionEvent) => @@ -1343,9 +1407,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const withControlSession = Effect.fn("PreviewManager.withControlSession")(function* ( tabId: string, - wc: Electron.WebContents, action: string, - use: (send: SendCommand, sendCleanup: SendCommand) => Effect.Effect, + use: ( + wc: Electron.WebContents, + send: SendCommand, + sendCleanup: SendCommand, + ) => Effect.Effect, + requestedDiagnosticDomains: ReadonlySet = new Set(), ) { const sequence = yield* nextCounter(actionSequenceRef); const startedAt = yield* currentIso; @@ -1358,8 +1426,10 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }; yield* pushAction(tabId, actionEvent); const epoch = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; - const control = yield* ensureControlSession(wc); - const execute = Effect.fn("PreviewManager.executeControlAction")(function* () { + const execute = Effect.fn("PreviewManager.executeControlAction")(function* ( + controlWc: Electron.WebContents, + control: BrowserControlSession, + ) { yield* update(tabId, { controller: "agent" }); const send: SendCommand = Effect.fn("PreviewManager.sendCommand")( function* (method, commandParams) { @@ -1368,11 +1438,11 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* new PreviewAutomationControlInterruptedError({ operation: action, tabId, - webContentsId: wc.id, + webContentsId: controlWc.id, }); } const result = yield* attemptPromise( - { operation: `${action}.${method}`, tabId, webContentsId: wc.id }, + { operation: `${action}.${method}`, tabId, webContentsId: controlWc.id }, () => control.debugger.sendCommand(method, commandParams), ); const after = (yield* Ref.get(controlEpochRef)).get(tabId) ?? 0; @@ -1380,7 +1450,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function return yield* new PreviewAutomationControlInterruptedError({ operation: action, tabId, - webContentsId: wc.id, + webContentsId: controlWc.id, }); } return result; @@ -1395,13 +1465,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function { operation: `${action}.cleanup.${method}`, tabId, - webContentsId: wc.id, + webContentsId: controlWc.id, }, () => control.debugger.sendCommand(method, commandParams), ); }, ); - return yield* use(send, sendCleanup); + return yield* use(controlWc, send, sendCleanup); }); const finalize = Effect.fn("PreviewManager.finalizeControlAction")(function* ( exit: Exit.Exit, @@ -1435,7 +1505,38 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const tabs = yield* SynchronizedRef.get(tabsRef); if (tabs.has(tabId)) yield* update(tabId, { controller: "none" }); }); - return yield* control.semaphore.withPermit(execute().pipe(Effect.onExit(finalize))); + const finalized = yield* Ref.make(false); + const finalizeOnce = Effect.fn("PreviewManager.finalizeControlActionOnce")(function* ( + exit: Exit.Exit, + ) { + const wasFinalized = yield* Ref.getAndSet(finalized, true); + if (!wasFinalized) yield* finalize(exit); + }); + const acquireAndExecute = (remainingAttempts: number): Effect.Effect => + Effect.gen(function* () { + const controlWc = yield* requireWebContents(tabId); + const control = yield* ensureControlSession(controlWc, requestedDiagnosticDomains); + const result = yield* control.semaphore.withPermit( + Effect.gen(function* () { + const liveWc = yield* requireWebContents(tabId); + const liveControl = (yield* SynchronizedRef.get(controlSessionsRef)).get(liveWc.id); + if (liveWc !== controlWc || liveControl !== control) return Option.none(); + return Option.some( + yield* execute(controlWc, control).pipe(Effect.onExit(finalizeOnce)), + ); + }), + ); + if (Option.isSome(result)) return result.value; + if (remainingAttempts > 1) { + return yield* Effect.suspend(() => acquireAndExecute(remainingAttempts - 1)); + } + return yield* new PreviewAutomationControlInterruptedError({ + operation: action, + tabId, + webContentsId: controlWc.id, + }); + }); + return yield* acquireAndExecute(3).pipe(Effect.onExit(finalizeOnce)); }); const evaluateWithDebugger = ( @@ -3487,11 +3588,16 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function }); const captureAutomationSnapshot = Effect.fn("PreviewManager.captureAutomationSnapshot")( - function* (tabId: string, wc: Electron.WebContents, send: SendCommand) { - yield* Effect.all([send("Runtime.enable"), send("Accessibility.enable")], { - concurrency: 2, - discard: true, - }); + function* ( + tabId: string, + wc: Electron.WebContents, + send: SendCommand, + include: ReadonlyArray = [], + ) { + const includeAx = include.includes("ax"); + const includeConsole = include.includes("console"); + const includeNetwork = include.includes("network"); + yield* send("Runtime.enable"); const page = yield* evaluateWithDebugger<{ url: string; title: string; @@ -3529,14 +3635,40 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const rect = element.getBoundingClientRect(); return style.visibility !== "hidden" && style.display !== "none" && rect.width > 0 && rect.height > 0; }; + const clickable = (element) => { + if (element.matches("a[href],button,input,textarea,select,[role],[tabindex]")) return true; + const role = element.getAttribute("role"); + if (role === "row" || role === "gridcell" || role === "option") return true; + if (element.tagName !== "TR" && element.tagName !== "TD" && element.tagName !== "DIV") return false; + const style = getComputedStyle(element); + if (style.cursor !== "pointer") return false; + // Cursor inherits. Keep the outermost harvested layout node + // (div/td/tr) so nested boxes do not fill the cap, but still + // list this node when the pointer owner is an unharvested + // wrapper such as li, section, label, or table. + let ancestor = element.parentElement; + while (ancestor) { + const harvested = + ancestor.tagName === "TR" || ancestor.tagName === "TD" || ancestor.tagName === "DIV"; + if (harvested && getComputedStyle(ancestor).cursor === "pointer") return false; + ancestor = ancestor.parentElement; + } + return true; + }; + const seen = new Set(); const elements = Array.from(document.querySelectorAll( - "a[href],button,input,textarea,select,[role],[tabindex]" - )).filter(visible).slice(0, ${MAX_INTERACTIVE_ELEMENTS}).map((element) => { + "a[href],button,input,textarea,select,[role],[tabindex],[role=row],tr,[role=gridcell],div,td" + )).filter((element) => { + if (!visible(element) || !clickable(element) || seen.has(element)) return false; + seen.add(element); + return true; + }).slice(0, ${MAX_INTERACTIVE_ELEMENTS}).map((element, index) => { const rect = element.getBoundingClientRect(); return { + id: "e" + (index + 1), tag: element.tagName.toLowerCase(), role: element.getAttribute("role"), - name: element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "", + name: (element.getAttribute("aria-label") || element.innerText || element.getAttribute("name") || "").slice(0, 160), selector: selectorFor(element), x: rect.x, y: rect.y, @@ -3544,29 +3676,43 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function height: rect.height }; }); + const main = document.querySelector('main, [role="main"]'); return { url: location.href, title: document.title, loading: document.readyState !== "complete", - visibleText: (document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH}), + visibleText: (main ? main.innerText : document.body?.innerText || "").slice(0, ${MAX_VISIBLE_TEXT_LENGTH}), interactiveElements: elements }; })()`, true, ); - const [accessibility, sourceImage, diagnostics, timelines] = yield* Effect.all([ - send("Accessibility.getFullAXTree"), - attemptPromise( - { - operation: "automationSnapshot.capturePage", - tabId, - webContentsId: wc.id, - }, - () => wc.capturePage(), + const accessibility = includeAx ? yield* send("Accessibility.getFullAXTree") : undefined; + const [diagnostics, timelines] = yield* Effect.all( + [Ref.get(diagnosticsRef), Ref.get(actionTimelineRef)], + { concurrency: 2 }, + ); + const sourceImage = yield* attemptPromise( + { + operation: "automationSnapshot.capturePage", + tabId, + webContentsId: wc.id, + }, + () => wc.capturePage(), + ).pipe( + Effect.catch((captureFailure) => + send("Page.captureScreenshot", { format: "png" }).pipe( + Effect.flatMap((result) => + result !== null && + typeof result === "object" && + "data" in result && + typeof result.data === "string" + ? Effect.succeed(nativeImage.createFromBuffer(Buffer.from(result.data, "base64"))) + : Effect.fail(captureFailure), + ), + ), ), - Ref.get(diagnosticsRef), - Ref.get(actionTimelineRef), - ]); + ); const sourceSize = sourceImage.getSize(); const image = sourceSize.width > MAX_SCREENSHOT_WIDTH @@ -3576,9 +3722,9 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const browserDiagnostics = diagnostics.get(wc.id); return { ...page, - accessibilityTree: accessibility, - consoleEntries: [...(browserDiagnostics?.consoleEntries ?? [])], - networkEntries: [...(browserDiagnostics?.networkEntries ?? [])], + ...(includeAx ? { accessibilityTree: accessibility } : {}), + consoleEntries: includeConsole ? [...(browserDiagnostics?.consoleEntries ?? [])] : [], + networkEntries: includeNetwork ? [...(browserDiagnostics?.networkEntries ?? [])] : [], actionTimeline: [...(timelines.get(tabId) ?? [])], screenshot: { mimeType: "image/png" as const, @@ -3592,10 +3738,13 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function const automationSnapshot = Effect.fn("PreviewManager.automationSnapshot")(function* ( tabId: string, + include: ReadonlyArray = [], ) { - const wc = yield* requireWebContents(tabId); - return yield* withControlSession(tabId, wc, "snapshot", (send) => - captureAutomationSnapshot(tabId, wc, send), + return yield* withControlSession( + tabId, + "snapshot", + (wc, send) => captureAutomationSnapshot(tabId, wc, send, include), + new Set(include), ); }); @@ -3727,8 +3876,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, input: PreviewAutomationClickInput, ) { - const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "click", (send) => + yield* withControlSession(tabId, "click", (_wc, send) => performAutomationClick(tabId, input, send), ); }); @@ -3853,8 +4001,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, input: PreviewAutomationTypeInput, ) { - const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "type", (send) => + yield* withControlSession(tabId, "type", (_wc, send) => performAutomationType(tabId, input, send), ); }); @@ -3915,8 +4062,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, input: PreviewAutomationPressInput, ) { - const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "press", (send, sendCleanup) => + yield* withControlSession(tabId, "press", (wc, send, sendCleanup) => performAutomationPress(tabId, wc, input, send, sendCleanup), ); }); @@ -3971,8 +4117,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, input: PreviewAutomationScrollInput, ) { - const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "scroll", (send) => + yield* withControlSession(tabId, "scroll", (_wc, send) => performAutomationScroll(tabId, input, send), ); }); @@ -4007,8 +4152,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, input: PreviewAutomationEvaluateInput, ) { - const wc = yield* requireWebContents(tabId); - return yield* withControlSession(tabId, wc, "evaluate", (send) => + return yield* withControlSession(tabId, "evaluate", (_wc, send) => performAutomationEvaluate(tabId, input, send), ); }); @@ -4022,7 +4166,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function yield* send("Runtime.enable"); const locator = automationLocator(input); if (locator) yield* ensurePlaywrightInjected(tabId, send); - const [locatorJson, textJson, urlIncludesJson] = yield* Effect.all([ + const [locatorJson, textJson, urlIncludesJson, scopeJson] = yield* Effect.all([ locator ? encodeJson({ operation: "automationWaitFor.encodeLocator", tabId }, locator) : Effect.succeed(null), @@ -4032,6 +4176,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function input.urlIncludes ? encodeJson({ operation: "automationWaitFor.encodeUrl", tabId }, input.urlIncludes) : Effect.succeed(null), + encodeJson({ operation: "automationWaitFor.encodeScope", tabId }, input.scope ?? "main"), ]); const deadline = (yield* currentMillis) + timeoutMs; while ((yield* currentMillis) <= deadline) { @@ -4042,9 +4187,59 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function send, `(() => { try { - const selectorMatched = ${locatorJson ? `(() => { const injected = globalThis.__t3PlaywrightInjected; return injected.querySelector(injected.parseSelector(${locatorJson}), document, false) !== null; })()` : "true"}; + const root = ${scopeJson} === "document" + ? document.documentElement + : (document.querySelector('main, [role="main"]') || document.documentElement); + const isWithinSearchRoot = (element, searchRoot) => { + let current = element; + while (current) { + if (current === searchRoot) return true; + if (current.parentElement) { + current = current.parentElement; + continue; + } + const rootNode = current.getRootNode(); + current = rootNode instanceof ShadowRoot ? rootNode.host : null; + } + return false; + }; + const searchRoots = ${scopeJson} === "document" + ? [root] + : [ + root, + ...Array.from(document.querySelectorAll( + '[aria-modal="true"], [role="dialog"], [role="alertdialog"], [data-slot$="-popup"], [data-slot$="-positioner"], [data-slot$="-portal"], [data-radix-portal], [data-radix-popper-content-wrapper]' + )), + ]; + const selectorMatched = ${ + locatorJson + ? `(() => { + const injected = globalThis.__t3PlaywrightInjected; + const parsed = injected.parseSelector(${locatorJson}); + const elements = injected.querySelectorAll(parsed, document); + return searchRoots.some((searchRoot) => { + return elements.some((element) => { + if (!isWithinSearchRoot(element, searchRoot)) return false; + const visible = injected.elementState(element, "visible"); + if (!visible.matches) return false; + const slot = element.getAttribute("data-slot") || ""; + if ( + slot === "dialog-trigger" || + slot === "alert-dialog-trigger" || + slot === "command-dialog-trigger" + ) { + return false; + } + return true; + }); + }); + })()` + : "true" + }; const textMatched = ${ - textJson ? `(document.body?.innerText || "").includes(${textJson})` : "true" + textJson + ? `searchRoots.some((searchRoot) => (searchRoot.innerText || "").includes(${textJson}))` + : "true" }; const urlMatched = ${ urlIncludesJson ? `location.href.includes(${urlIncludesJson})` : "true" @@ -4078,8 +4273,7 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function tabId: string, input: PreviewAutomationWaitForInput, ) { - const wc = yield* requireWebContents(tabId); - yield* withControlSession(tabId, wc, "waitFor", (send) => + yield* withControlSession(tabId, "waitFor", (_wc, send) => performAutomationWaitFor(tabId, input, send), ); }); @@ -4554,6 +4748,7 @@ export class PreviewManager extends Context.Service< ) => Effect.Effect; readonly automationSnapshot: ( tabId: string, + include?: ReadonlyArray, ) => Effect.Effect; readonly automationClick: ( tabId: string, diff --git a/apps/server/src/mcp/toolkits/preview/tools.ts b/apps/server/src/mcp/toolkits/preview/tools.ts index 33528d8bb38c..086c2e4ed3e5 100644 --- a/apps/server/src/mcp/toolkits/preview/tools.ts +++ b/apps/server/src/mcp/toolkits/preview/tools.ts @@ -13,6 +13,7 @@ import { PreviewAutomationSetColorSchemeInput, PreviewAutomationSetColorSchemeResult, PreviewAutomationSnapshot, + PreviewAutomationSnapshotInput, PreviewAutomationStatus, PreviewAutomationTabTargetInput, PreviewAutomationTypeInput, @@ -111,8 +112,8 @@ export const PreviewSetAppearanceTool = safeBrowserTool( export const PreviewSnapshotTool = readonlyBrowserTool( Tool.make("preview_snapshot", { description: - "Inspect a page before interacting. Pass tabId to inspect a specific tab; omit it to use this agent session's current tab. Returns page state, semantic elements, diagnostics, action history, and a PNG screenshot.", - parameters: PreviewAutomationTabTargetInput, + "Inspect a page before interacting. Default is a slim snapshot: URL, visible text, interactive elements, and a PNG. Pass include:['ax'], include:['console'], or include:['network'] only when you need those heavier slices. Hidden tabs still capture.", + parameters: PreviewAutomationSnapshotInput, success: PreviewAutomationSnapshot, failure: PreviewAutomationError, dependencies, @@ -177,7 +178,7 @@ export const PreviewEvaluateTool = browserTool( export const PreviewWaitForTool = readonlyBrowserTool( Tool.make("preview_wait_for", { description: - "Wait in the tab selected by tabId, or this agent session's current tab when omitted, until all supplied locator, selector, text, and URL conditions match.", + "Wait in the tab selected by tabId, or this agent session's current tab when omitted, until all supplied locator, selector, text, and URL conditions match. Text defaults to the main landmark so sidebar labels do not satisfy the wait. Locators must match a visible element.", parameters: PreviewAutomationWaitForInput, success: PreviewActionResult, failure: PreviewAutomationError, diff --git a/apps/web/src/components/preview/PreviewAutomationHosts.tsx b/apps/web/src/components/preview/PreviewAutomationHosts.tsx index 54c2e1d9cf68..9e25f98745ee 100644 --- a/apps/web/src/components/preview/PreviewAutomationHosts.tsx +++ b/apps/web/src/components/preview/PreviewAutomationHosts.tsx @@ -9,6 +9,7 @@ import { type PreviewAutomationNavigateInput, type PreviewAutomationOpenInput, type PreviewAutomationResizeInput, + type PreviewAutomationSnapshotInput, type PreviewAutomationResizeResult, type PreviewAutomationSetColorSchemeInput, type PreviewAutomationSetColorSchemeResult, @@ -635,7 +636,8 @@ function PreviewAutomationHost(props: { readonly environmentId: EnvironmentId }) } case "snapshot": { const ready = await requireReadyTab(); - return await ready.bridge.automation.snapshot(ready.runtimeTabId); + const input = request.input as PreviewAutomationSnapshotInput; + return await ready.bridge.automation.snapshot(ready.runtimeTabId, input.include); } case "click": { const ready = await requireReadyTab(); diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index 798d5a777d5d..cfe8983f5c41 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -71,6 +71,7 @@ import { PreviewAutomationResponse, PreviewAutomationScrollInput, PreviewAutomationSnapshot, + PreviewAutomationSnapshotInclude, PreviewAutomationStatus, PreviewAutomationStreamEvent, PreviewAutomationTypeInput, @@ -971,6 +972,11 @@ export interface DesktopPreviewTabDefaults { readonly colorScheme?: DesktopPreviewColorScheme | undefined; } +export const DesktopPreviewAutomationSnapshotInputSchema = Schema.Struct({ + tabId: DesktopPreviewTabIdSchema, + include: Schema.optional(Schema.Array(PreviewAutomationSnapshotInclude)), +}); + export const DesktopPreviewRegisterWebviewInputSchema = Schema.Struct({ tabId: DesktopPreviewTabIdSchema, webContentsId: Schema.Int.check(Schema.isGreaterThan(0)), @@ -1232,7 +1238,10 @@ export interface DesktopPreviewBridge { }; automation: { status: (tabId: string) => Promise; - snapshot: (tabId: string) => Promise; + snapshot: ( + tabId: string, + include?: ReadonlyArray, + ) => Promise; click: (tabId: string, input: PreviewAutomationClickInput) => Promise; type: (tabId: string, input: PreviewAutomationTypeInput) => Promise; press: (tabId: string, input: PreviewAutomationPressInput) => Promise; diff --git a/packages/contracts/src/preview.test.ts b/packages/contracts/src/preview.test.ts index 24f429745ef8..14cf135156d1 100644 --- a/packages/contracts/src/preview.test.ts +++ b/packages/contracts/src/preview.test.ts @@ -17,7 +17,9 @@ import { PreviewAutomationOpenInput, PreviewAutomationResizeInput, PreviewAutomationResizeResult, + PreviewAutomationSnapshotInput, PreviewAutomationStatus, + PreviewAutomationWaitForInput, } from "./previewAutomation.ts"; const decodePreviewEvent = Schema.decodeUnknownSync(PreviewEvent); @@ -32,6 +34,8 @@ const decodeResizeResult = Schema.decodeUnknownSync(PreviewAutomationResizeResul const decodeAutomationHost = Schema.decodeUnknownSync(PreviewAutomationHost); const decodeAutomationError = Schema.decodeUnknownSync(PreviewAutomationError); const decodeAutomationStatus = Schema.decodeUnknownSync(PreviewAutomationStatus); +const decodeSnapshotInput = Schema.decodeUnknownSync(PreviewAutomationSnapshotInput); +const decodeWaitForInput = Schema.decodeUnknownSync(PreviewAutomationWaitForInput); describe("PreviewAutomationOpenInput", () => { it("accepts the inline preview visibility flag", () => { @@ -223,6 +227,26 @@ describe("PreviewAutomationStatus", () => { }); }); +describe("PreviewAutomationSnapshotInput", () => { + it("defaults to a slim snapshot and accepts extra diagnostic slices", () => { + expect(decodeSnapshotInput({})).toEqual({}); + expect(decodeSnapshotInput({ include: ["ax", "console", "network"] }).include).toEqual([ + "ax", + "console", + "network", + ]); + expect(() => decodeSnapshotInput({ include: ["screenshot"] })).toThrow(); + }); +}); + +describe("PreviewAutomationWaitForInput", () => { + it("defaults text and locators to the main landmark", () => { + expect(decodeWaitForInput({ text: "Dashboard" })).toEqual({ text: "Dashboard" }); + expect(decodeWaitForInput({ text: "Dashboard", scope: "document" }).scope).toBe("document"); + expect(() => decodeWaitForInput({ scope: "main" })).toThrow(); + }); +}); + describe("PreviewEvent", () => { it("decodes opened", () => { const event = decodePreviewEvent({ diff --git a/packages/contracts/src/previewAutomation.ts b/packages/contracts/src/previewAutomation.ts index e33615fa4c05..1c421e16c205 100644 --- a/packages/contracts/src/previewAutomation.ts +++ b/packages/contracts/src/previewAutomation.ts @@ -63,6 +63,9 @@ const PreviewAutomationTabTargetFields = { export const PreviewAutomationTabTargetInput = Schema.Struct(PreviewAutomationTabTargetFields); export type PreviewAutomationTabTargetInput = typeof PreviewAutomationTabTargetInput.Type; +export const PreviewAutomationSnapshotInclude = Schema.Literals(["ax", "console", "network"]); +export type PreviewAutomationSnapshotInclude = typeof PreviewAutomationSnapshotInclude.Type; + export const PreviewAutomationStatus = Schema.Struct({ available: Schema.Boolean, visible: Schema.Boolean, @@ -465,6 +468,15 @@ export const PreviewAutomationWaitForInput = Schema.Struct({ description: "Substring that must appear in the current absolute URL.", }), ).annotate({ description: "Substring that must appear in the current absolute URL." }), + scope: Schema.optional( + Schema.Literals(["main", "document"]).annotate({ + description: + "Where text and locators are searched. 'main' uses the main landmark when present (default). 'document' searches the whole page, including sidebars.", + }), + ).annotate({ + description: + "Where text and locators are searched. Defaults to main so sidebar chrome does not satisfy the wait.", + }), timeoutMs: OptionalTimeoutMs, }) .check( @@ -488,6 +500,7 @@ export const PreviewAutomationWaitForInput = Schema.Struct({ export type PreviewAutomationWaitForInput = typeof PreviewAutomationWaitForInput.Type; export const PreviewAutomationElement = Schema.Struct({ + id: Schema.optional(Schema.String), tag: Schema.String, role: Schema.NullOr(Schema.String), name: Schema.String, @@ -527,13 +540,25 @@ export const PreviewAutomationActionEvent = Schema.Struct({ }); export type PreviewAutomationActionEvent = typeof PreviewAutomationActionEvent.Type; +export const PreviewAutomationSnapshotInput = Schema.Struct({ + ...PreviewAutomationTabTargetFields, + include: Schema.optional(Schema.Array(PreviewAutomationSnapshotInclude)).annotate({ + description: + "Optional extra snapshot slices. Default is screenshot, visible text, and interactive elements. Pass ax, console, and/or network when you need those heavier diagnostics.", + }), +}).annotate({ + description: + "Inspects the collaborative browser tab. Omit include for a slim snapshot; add ax, console, or network only when needed.", +}); +export type PreviewAutomationSnapshotInput = typeof PreviewAutomationSnapshotInput.Type; + export const PreviewAutomationSnapshot = Schema.Struct({ url: Schema.String, title: Schema.String, loading: Schema.Boolean, visibleText: Schema.String, interactiveElements: Schema.Array(PreviewAutomationElement), - accessibilityTree: Schema.Unknown, + accessibilityTree: Schema.optional(Schema.Unknown), consoleEntries: Schema.Array(PreviewAutomationConsoleEntry), networkEntries: Schema.Array(PreviewAutomationNetworkEntry), actionTimeline: Schema.Array(PreviewAutomationActionEvent),