From 422ff4dc540486f849f9d833d6b2a2c19a6d62dd Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Sat, 14 Nov 2026 00:32:21 -0800 Subject: [PATCH] fix(computer-use): stop reporting closed menu items past the cap as a truncated state --- apps/desktop/native/computer-use/ComputerUse.swift | 3 ++- apps/desktop/native/computer-use/tests/Fixture.swift | 8 +++++++- apps/desktop/native/computer-use/tests/live.py | 3 ++- 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index a723bb77532..a6afafb0f6f 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -354,8 +354,9 @@ final class Snapshot { if role == kAXMenuItemRole && !entry.menu && !entry.priority { queue.append(AXTraversalEntry(element: element, parent: entry.parent, depth: entry.depth, menu: true, priority: false)); continue } + // Closed menus past the cap hide no content: opening a menu lists its items as priority entries. if isMenu && !entry.priority { - guard menuCount < 80 else { truncated = true; continue } + guard menuCount < 80 else { continue } menuCount += 1 } let id = "e\(nodes.count)"; elements[id] = element; index.insert(element, id: id) diff --git a/apps/desktop/native/computer-use/tests/Fixture.swift b/apps/desktop/native/computer-use/tests/Fixture.swift index 69aab0234e6..32d4b189127 100644 --- a/apps/desktop/native/computer-use/tests/Fixture.swift +++ b/apps/desktop/native/computer-use/tests/Fixture.swift @@ -38,7 +38,13 @@ final class FixtureDelegate: NSObject, NSApplicationDelegate { let editItem = NSMenuItem(title: "Edit", action: nil, keyEquivalent: "") let editMenu = NSMenu(title: "Edit") editMenu.addItem(withTitle: "Select All", action: #selector(NSText.selectAll(_:)), keyEquivalent: "a") - editItem.submenu = editMenu; menu.addItem(editItem); NSApplication.shared.mainMenu = menu + editItem.submenu = editMenu; menu.addItem(editItem) + // More closed menu items than a snapshot lists, as most real apps have. + let manyItem = NSMenuItem(title: "Many", action: nil, keyEquivalent: "") + let manyMenu = NSMenu(title: "Many") + for index in 1...120 { manyMenu.addItem(withTitle: "Fixture command \(index)", action: nil, keyEquivalent: "") } + manyItem.submenu = manyMenu; menu.addItem(manyItem) + NSApplication.shared.mainMenu = menu window = NSWindow(contentRect: NSRect(x: 100, y: 100, width: 600, height: 720), styleMask: [.titled, .closable, .resizable], backing: .buffered, defer: false) window.title = "Mothership Computer Use Test Fixture" let stack = NSStackView(); stack.orientation = .vertical; stack.spacing = 16 diff --git a/apps/desktop/native/computer-use/tests/live.py b/apps/desktop/native/computer-use/tests/live.py index 8a4ac1a2fb1..34f4e9ff208 100644 --- a/apps/desktop/native/computer-use/tests/live.py +++ b/apps/desktop/native/computer-use/tests/live.py @@ -81,6 +81,7 @@ def coordinate(snapshot, node, fx=0.5, fy=0.5): time.sleep(1) snapshot = state(screenshot=True) assert 'fixture-only-secret' not in json.dumps(snapshot), 'Secure value leaked' + assert snapshot['truncated'] is False, 'Closed menu items past the snapshot cap marked the state truncated' infinite = find(snapshot, label='Fixture infinite frame') assert not {'x', 'y', 'width', 'height'} & infinite.keys(), f'An infinite frame reached the reply: {infinite}' image = snapshot.get('screenshot') @@ -211,7 +212,7 @@ def batch(snapshot, steps): assert error['dispatchState'] == 'not_started', error assert find(state(), role='AXTextField', label='Fixture click-focus field').get('value') == 'replaced', 'A background app was clicked or typed into' call('activate_app', bundleId=bundle) - print('PASS: real fixture state, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) + print('PASS: real fixture state, a large closed menu not reported as truncation, secure suppression, an element reported at infinity read without a frame, AX click/set/action, Unicode typing, keyboard chord, coordinate drag and scrolling, text refused by a slider, typing into a date picker, a window target typing into its focused editor, a title-bar click, a batch that applies, stops at a failing step, refuses before dispatch and stops when a sheet opens, click-to-focus for an editor that ignores Accessibility focus (inserting at the end, refusing an unconfirmed caret, and none in the background); every mutation verified in fresh state; screenshot=' + ('verified' if permissions['screenRecording'] else 'BLOCKED')) finally: if fixture: fixture.terminate()