fix(preview): keep snapshots slim unless the agent asks

apps/desktop/src/ipc/methods/preview.ts

@@ -2,6 +2,7 @@ import {
   DesktopPreviewAnnotationThemeInputSchema,
   DesktopPreviewArtifactInputSchema,
   DesktopPreviewAutomationClickInputSchema,
+  DesktopPreviewAutomationSnapshotInputSchema,
   DesktopPreviewAutomationEvaluateInputSchema,
   DesktopPreviewAutomationPressInputSchema,
   DesktopPreviewAutomationScrollInputSchema,
@@ -389,11 +390,11 @@ export const automationStatus = DesktopIpc.makeIpcMethod({
 
 export const automationSnapshot = DesktopIpc.makeIpcMethod({
   channel: IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL,
-  payload: DesktopPreviewTabInputSchema,
+  payload: DesktopPreviewAutomationSnapshotInputSchema,
   result: PreviewAutomationSnapshot,
-  handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId }) {
+  handler: Effect.fn("desktop.ipc.preview.automationSnapshot")(function* ({ tabId, include }) {
     const manager = yield* PreviewManager.PreviewManager;
-    return yield* manager.automationSnapshot(tabId);
+    return yield* manager.automationSnapshot(tabId, include ?? []);
   }),
 });
 

apps/desktop/src/preload.ts

@@ -273,8 +273,11 @@ contextBridge.exposeInMainWorld("desktopBridge", {
     automation: {
       status: (tabId) =>
         ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_STATUS_CHANNEL, { tabId }),
-      snapshot: (tabId) =>
-        ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, { tabId }),
+      snapshot: (tabId, include) =>
+        ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_SNAPSHOT_CHANNEL, {
+          tabId,
+          ...(include === undefined ? {} : { include }),
+        }),
       click: (tabId, input) =>
         ipcRenderer.invoke(IpcChannels.PREVIEW_AUTOMATION_CLICK_CHANNEL, { tabId, input }),
       type: (tabId, input) =>

apps/desktop/src/preview/Manager.test.ts

@@ -140,6 +140,14 @@ vi.mock("electron", () => ({
   },
   nativeImage: {
     createFromPath,
+    createFromBuffer: (buffer: Buffer) => ({
+      getSize: () => ({ width: buffer.length > 0 ? 1 : 0, height: buffer.length > 0 ? 1 : 0 }),
+      toPNG: () => buffer,
+      resize: () => ({
+        getSize: () => ({ width: 1, height: 1 }),
+        toPNG: () => buffer,
+      }),
+    }),
   },
   shell: {
     showItemInFolder,
@@ -262,6 +270,7 @@ const makeTestPreviewWebContents = (
     getURL: () => "https://example.com",
     getTitle: () => "Example",
     isLoading: () => false,
+    isDevToolsOpened: () => false,
     getZoomFactor: () => 1,
     setZoomFactor: vi.fn(),
     setAudioMuted: vi.fn(),
@@ -3967,3 +3976,327 @@ describe("Preview automation diagnostics", () => {
     expect("locator" in error).toBe(false);
   });
 });
+
+describe("Preview automation snapshots", () => {
+  const pageValue = {
+    url: "https://example.com",
+    title: "Example",
+    loading: false,
+    visibleText: "Dashboard",
+    interactiveElements: [],
+  };
+
+  const snapshotImage = {
+    getSize: () => ({ width: 100, height: 80 }),
+    toPNG: () => Buffer.from("png"),
+    resize: () => ({
+      getSize: () => ({ width: 100, height: 80 }),
+      toPNG: () => Buffer.from("png"),
+    }),
+  };
+
+  const mockAutomationWebContents = (
+    sendCommand: ReturnType<typeof vi.fn>,
+    capturePage: () => Promise<typeof snapshotImage>,
+    options: {
+      readonly id?: number;
+      readonly isDebuggerAttached?: () => boolean;
+    } = {},
+  ) =>
+    ({
+      id: options.id ?? 42,
+      isDestroyed: () => false,
+      getType: () => "webview",
+      getURL: () => "https://example.com",
+      getTitle: () => "Example",
+      isLoading: () => false,
+      isDevToolsOpened: () => false,
+      getZoomFactor: () => 1,
+      setZoomFactor: vi.fn(),
+      setAudioMuted: vi.fn(),
+      on: vi.fn(),
+      off: vi.fn(),
+      ipc: { on: vi.fn(), off: vi.fn() },
+      send: webviewSend,
+      navigationHistory: { canGoBack: () => false, canGoForward: () => false },
+      setWindowOpenHandler: vi.fn(),
+      capturePage,
+      debugger: {
+        isAttached: options.isDebuggerAttached ?? (() => false),
+        attach: vi.fn(),
+        sendCommand,
+        on: vi.fn(),
+        off: vi.fn(),
+      },
+    }) as never;
+
+  effectIt.effect("finalizes the timeline when control-session acquisition fails", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        let debuggerAttached = true;
+        const sendCommand = vi.fn(async (method: string) =>
+          method === "Runtime.evaluate" ? { result: { value: pageValue } } : undefined,
+        );
+        fromId.mockReturnValue(
+          mockAutomationWebContents(sendCommand, async () => snapshotImage, {
+            isDebuggerAttached: () => debuggerAttached,
+          }),
+        );
+
+        yield* manager.createTab("tab_acquisition_failure");
+        yield* manager.registerWebview("tab_acquisition_failure", 42);
+        const exit = yield* Effect.exit(
+          manager.automationEvaluate("tab_acquisition_failure", { expression: "location.href" }),
+        );
+        expect(Exit.isFailure(exit)).toBe(true);
+
+        debuggerAttached = false;
+        const snapshot = yield* manager.automationSnapshot("tab_acquisition_failure");
+        expect(snapshot.actionTimeline).toContainEqual(
+          expect.objectContaining({
+            action: "evaluate",
+            status: "failed",
+            completedAt: expect.any(String),
+          }),
+        );
+        expect(snapshot.actionTimeline).not.toContainEqual(
+          expect.objectContaining({ action: "evaluate", status: "running" }),
+        );
+      }),
+    ),
+  );
+
+  effectIt.effect("retries against the live webview when guests swap during acquisition", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        let release42: () => void = () => undefined;
+        let release43: () => void = () => undefined;
+        const diagnostic42 = new Promise<void>((resolve) => {
+          release42 = resolve;
+        });
+        const diagnostic43 = new Promise<void>((resolve) => {
+          release43 = resolve;
+        });
+        const makeSendCommand = (diagnostic: Promise<void> | null) =>
+          vi.fn(async (method: string) => {
+            if (method === "Accessibility.enable" && diagnostic) await diagnostic;
+            if (method === "Runtime.evaluate") return { result: { value: pageValue } };
+            if (method === "Accessibility.getFullAXTree") return { nodes: [] };
+            return undefined;
+          });
+        const send42 = makeSendCommand(diagnostic42);
+        const send43 = makeSendCommand(diagnostic43);
+        const send44 = makeSendCommand(null);
+        const guests = new Map([
+          [42, mockAutomationWebContents(send42, async () => snapshotImage, { id: 42 })],
+          [43, mockAutomationWebContents(send43, async () => snapshotImage, { id: 43 })],
+          [44, mockAutomationWebContents(send44, async () => snapshotImage, { id: 44 })],
+        ]);
+        fromId.mockImplementation((id) => guests.get(id ?? -1) ?? null);
+
+        yield* manager.createTab("tab_webview_swap");
+        yield* manager.registerWebview("tab_webview_swap", 42);
+        yield* Effect.yieldNow;
+        const snapshotFiber = yield* manager
+          .automationSnapshot("tab_webview_swap", ["ax"])
+          .pipe(Effect.forkChild({ startImmediately: true }));
+        yield* Effect.yieldNow;
+        expect(send42).toHaveBeenCalledWith("Accessibility.enable");
+
+        yield* manager.registerWebview("tab_webview_swap", 43);
+        yield* Effect.yieldNow;
+        release42();
+        yield* Effect.yieldNow;
+        expect(send43).toHaveBeenCalledWith("Accessibility.enable");
+
+        yield* manager.registerWebview("tab_webview_swap", 44);
+        yield* Effect.yieldNow;
+        release43();
+        yield* Fiber.join(snapshotFiber);
+
+        expect(send42).not.toHaveBeenCalledWith("Runtime.evaluate", expect.anything());
+        expect(send43).not.toHaveBeenCalledWith("Runtime.evaluate", expect.anything());
+        expect(send44).toHaveBeenCalledWith("Runtime.evaluate", expect.anything());
+      }),
+    ),
+  );
+
+  effectIt.effect("does not clear agent control after a pipelined action starts", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        let releaseFirst: () => void = () => undefined;
+        let releaseSecond: () => void = () => undefined;
+        const firstEvaluation = new Promise<void>((resolve) => {
+          releaseFirst = resolve;
+        });
+        const secondEvaluation = new Promise<void>((resolve) => {
+          releaseSecond = resolve;
+        });
+        let evaluationCount = 0;
+        const sendCommand = vi.fn(async (method: string) => {
+          if (method !== "Runtime.evaluate") return undefined;
+          evaluationCount += 1;
+          await (evaluationCount === 1 ? firstEvaluation : secondEvaluation);
+          return { result: { value: null } };
+        });
+        fromId.mockReturnValue(mockAutomationWebContents(sendCommand, async () => snapshotImage));
+        const controllers: string[] = [];
+        yield* manager.subscribeStateChanges((_tabId, state) =>
+          Effect.sync(() => {
+            controllers.push(state.controller);
+          }),
+        );
+
+        yield* manager.createTab("tab_pipeline");
+        yield* manager.registerWebview("tab_pipeline", 42);
+        yield* Effect.yieldNow;
+        const first = yield* manager
+          .automationEvaluate("tab_pipeline", { expression: "1" })
+          .pipe(Effect.forkChild({ startImmediately: true }));
+        yield* Effect.yieldNow;
+        expect(evaluationCount).toBe(1);
+
+        const second = yield* manager
+          .automationEvaluate("tab_pipeline", { expression: "2" })
+          .pipe(Effect.forkChild({ startImmediately: true }));
+        yield* Effect.yieldNow;
+        expect(evaluationCount).toBe(1);
+
+        releaseFirst();
+        yield* Effect.yieldNow;
+        yield* Effect.yieldNow;
+        expect(evaluationCount).toBe(2);
+        expect(controllers.at(-1)).toBe("agent");
+
+        releaseSecond();
+        yield* Fiber.join(first);
+        yield* Fiber.join(second);
+        expect(controllers.at(-1)).toBe("none");
+      }),
+    ),
+  );
+
+  effectIt.effect("omits ax, console, and network unless include asks", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        const sendCommand = vi.fn(async (method: string, _params?: Record<string, unknown>) => {
+          if (method === "Runtime.evaluate") {
+            return { result: { value: pageValue } };
+          }
+          if (method === "Accessibility.getFullAXTree") {
+            return { nodes: [{ role: "main" }] };
+          }
+          return undefined;
+        });
+        fromId.mockReturnValue(mockAutomationWebContents(sendCommand, async () => snapshotImage));
+
+        yield* manager.createTab("tab_snapshot");
+        yield* manager.registerWebview("tab_snapshot", 42);
+        const slim = yield* manager.automationSnapshot("tab_snapshot");
+        expect(slim.accessibilityTree).toBeUndefined();
+        expect(slim.consoleEntries).toEqual([]);
+        expect(slim.networkEntries).toEqual([]);
+        const slimMethods = sendCommand.mock.calls.map(([method]) => method);
+        expect(slimMethods).not.toContain("Accessibility.getFullAXTree");
+        expect(slimMethods).not.toContain("Accessibility.enable");
+        expect(slimMethods).not.toContain("Network.enable");
+        expect(slimMethods).not.toContain("Log.enable");
+        expect(
+          sendCommand.mock.calls.some(
+            ([method, params]) =>
+              method === "Runtime.evaluate" &&
+              typeof params === "object" &&
+              params !== null &&
+              "expression" in params &&
+              typeof params.expression === "string" &&
+              params.expression.includes('main, [role="main"]'),
+          ),
+        ).toBe(true);
+
+        sendCommand.mockClear();
+        const withAx = yield* manager.automationSnapshot("tab_snapshot", ["ax"]);
+        expect(withAx.accessibilityTree).toEqual({ nodes: [{ role: "main" }] });
+        expect(sendCommand.mock.calls.map(([method]) => method)).toContain(
+          "Accessibility.getFullAXTree",
+        );
+        expect(sendCommand.mock.calls.map(([method]) => method)).toContain("Accessibility.enable");
+
+        sendCommand.mockClear();
+        yield* manager.automationSnapshot("tab_snapshot", ["console", "network"]);
+        expect(sendCommand.mock.calls.map(([method]) => method)).toEqual(
+          expect.arrayContaining(["Log.enable", "Network.enable"]),
+        );
+      }),
+    ),
+  );
+
+  effectIt.effect("fails the snapshot when capturePage and CDP screenshot both miss", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        const sendCommand = vi.fn(async (method: string) => {
+          if (method === "Runtime.evaluate") {
+            return { result: { value: pageValue } };
+          }
+          if (method === "Page.captureScreenshot") {
+            return {};
+          }
+          return undefined;
+        });
+        fromId.mockReturnValue(
+          mockAutomationWebContents(sendCommand, async () => {
+            throw new Error("capturePage failed");
+          }),
+        );
+
+        yield* manager.createTab("tab_snapshot_fail");
+        yield* manager.registerWebview("tab_snapshot_fail", 42);
+        const exit = yield* Effect.exit(manager.automationSnapshot("tab_snapshot_fail"));
+        expect(Exit.isFailure(exit)).toBe(true);
+        if (Exit.isSuccess(exit)) return;
+        expect(Option.getOrThrow(Cause.findErrorOption(exit.cause))).toMatchObject({
+          _tag: "PreviewOperationError",
+          operation: "automationSnapshot.capturePage",
+        });
+      }),
+    ),
+  );
+
+  effectIt.effect("defaults waitFor text search to the main landmark", () =>
+    withManager((manager) =>
+      Effect.gen(function* () {
+        const expressions: string[] = [];
+        const sendCommand = vi.fn(async (method: string, params?: Record<string, unknown>) => {
+          if (method === "Runtime.evaluate") {
+            expressions.push(String(params?.expression ?? ""));
+            return { result: { value: { matched: true } } };
+          }
+          return undefined;
+        });
+        fromId.mockReturnValue(mockAutomationWebContents(sendCommand, async () => snapshotImage));
+
+        yield* manager.createTab("tab_wait");
+        yield* manager.registerWebview("tab_wait", 42);
+        yield* manager.automationWaitFor("tab_wait", {
+          locator: "text=Dashboard",
+          text: "Dashboard",
+        });
+        expect(expressions.some((expression) => expression.includes('main, [role="main"]'))).toBe(
+          true,
+        );
+        expect(expressions.some((expression) => expression.includes("searchRoots.some"))).toBe(
+          true,
+        );
+        expect(
+          expressions.some((expression) => expression.includes('data-slot$="-viewport"')),
+        ).toBe(false);
+        expect(
+          expressions.some((expression) => expression.includes('slot === "dialog-trigger"')),
+        ).toBe(true);
+        expect(
+          expressions.some((expression) => expression.includes('slot.includes("trigger")')),
+        ).toBe(false);
+        expect(expressions.some((expression) => expression.includes("getRootNode"))).toBe(true);
+      }),
+    ),
+  );
+});