Fix macOS action cursor updates and session-scoped cleanup

crates/computer-use-mcp/Cargo.toml

@@ -14,6 +14,7 @@ tokio = { version = "1", features = [
     "time",
     "sync",
 ] }
+tokio-util = { version = "0.7", features = ["rt"] }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 schemars = "1"
@@ -43,3 +44,7 @@ windows = { version = "=0.62.2", features = [
     "Win32_System_Threading",
     "Win32_UI_WindowsAndMessaging",
 ] }
+
+[[test]]
+name = "macos_overlay"
+harness = false

crates/computer-use-mcp/src/backend.rs

@@ -247,9 +247,39 @@ impl std::error::Error for BackendError {}
 
 #[cfg(target_os = "macos")]
 pub use macos::{observe, perform_action};
+
+pub(crate) fn set_feedback_enabled(enabled: bool) {
+    #[cfg(target_os = "macos")]
+    macos::set_feedback_enabled(enabled);
+    #[cfg(not(target_os = "macos"))]
+    let _ = enabled;
+}
+
+pub(crate) fn clear_feedback(owner: Option<u64>, action: Option<u64>) {
+    #[cfg(target_os = "macos")]
+    macos::clear_feedback(owner, action);
+    #[cfg(not(target_os = "macos"))]
+    let _ = (owner, action);
+}
 #[cfg(target_os = "windows")]
 pub use windows::{observe, perform_action};
 
+pub(crate) fn perform_action_with_feedback(
+    root: &RootInfo,
+    request: &ActionRequest,
+    feedback: &crate::feedback::FeedbackRun,
+) -> Result<ActionResult, BackendError> {
+    #[cfg(target_os = "macos")]
+    {
+        macos::perform_action_with_feedback(root, request, Some(feedback))
+    }
+    #[cfg(not(target_os = "macos"))]
+    {
+        let _ = feedback;
+        perform_action(root, request)
+    }
+}
+
 /// Enumerate desktop roots. The host process is never a root: in-process
 /// accessibility queries run the host's own accessibility callbacks on the
 /// calling (non-main) thread, which aborts the app.

crates/computer-use-mcp/src/backend/macos/mod.rs

@@ -126,7 +126,15 @@ pub fn perform_action(
     root: &RootInfo,
     request: &ActionRequest,
 ) -> Result<ActionResult, BackendError> {
-    reflect_overlay(root, request);
+    perform_action_with_feedback(root, request, None)
+}
+
+pub(crate) fn perform_action_with_feedback(
+    root: &RootInfo,
+    request: &ActionRequest,
+    feedback: Option<&crate::feedback::FeedbackRun>,
+) -> Result<ActionResult, BackendError> {
+    reflect_overlay(root, request, feedback);
     match request.kind {
         ActionKind::Press => {
             let target = target(root, request)?;
@@ -428,24 +436,37 @@ fn keyboard_result_with_optional_foreground(
     }
 }
 
-fn reflect_overlay(root: &RootInfo, request: &ActionRequest) {
-    let enabled = crate::config::get().show_agent_cursor;
-    overlay::set_enabled(enabled);
-    if !enabled {
+fn reflect_overlay(
+    root: &RootInfo,
+    request: &ActionRequest,
+    feedback: Option<&crate::feedback::FeedbackRun>,
+) {
+    let Some(publication) = overlay::begin_action(feedback.map(|run| run.ticket())) else {
         return;
-    }
+    };
     use overlay::OverlayActionKind as K;
     match request.kind {
         ActionKind::Drag => {
             if let Some(path) = request.path.as_ref()
                 && let (Some(first), Some(last)) = (path.first(), path.last())
             {
-                overlay::show_drag(root.pid, (first[0], first[1]), (last[0], last[1]));
+                overlay::show_drag(
+                    publication,
+                    root.pid,
+                    root.window_id,
+                    (first[0], first[1]),
+                    (last[0], last[1]),
+                );
             }
         }
         ActionKind::TypeText | ActionKind::SetText | ActionKind::Keypress => {
-            if let Ok(point) = action_point(root, request) {
-                overlay::show_action(root.pid, K::Keyboard, point);
+            // Untargeted typing/key chords are valid backend actions. Their
+            // feedback belongs to the current root, not the previous pointer.
+            let point = action_point(root, request)
+                .ok()
+                .or_else(|| root.frame.has_area().then(|| root.frame.center()));
+            if let Some(point) = point {
+                overlay::show_action(publication, root.pid, root.window_id, K::Keyboard, point);
             }
         }
         other => {
@@ -455,7 +476,7 @@ fn reflect_overlay(root: &RootInfo, request: &ActionRequest) {
                 _ => K::Click,
             };
             if let Ok(point) = action_point(root, request) {
-                overlay::show_action(root.pid, kind, point);
+                overlay::show_action(publication, root.pid, root.window_id, kind, point);
             }
         }
     }
@@ -538,3 +559,5 @@ fn window_bounds(dictionary: CFDictionaryRef) -> Option<crate::outline::Frame> {
         h: number("Height")?,
     })
 }
+
+pub(crate) use overlay::{clear as clear_feedback, set_enabled as set_feedback_enabled};

crates/computer-use-mcp/src/backend/macos/overlay/cursor.rs

@@ -6,7 +6,8 @@ use super::OverlayActionKind;
 use super::ffi::{
     Id, class, send_id, send_id_color, send_id_cstr, send_id_id, send_id_rect, send_id_window_init,
     send_void, send_void_bool, send_void_f32, send_void_f64, send_void_id, send_void_isize,
-    send_void_point, send_void_rect, send_void_size, send_void_usize, status_window_level,
+    send_void_point, send_void_rect, send_void_rect_bool, send_void_size, send_void_usize,
+    status_window_level,
 };
 use super::geometry::{DisplayGeometry, ax_screen_to_appkit};
 
@@ -108,11 +109,12 @@ impl CursorUi {
         self.set_kind(kind);
         let appkit_point = ax_screen_to_appkit(ax_point, display);
         if self.visible {
-            animate_window_origin(self.window, window_origin(appkit_point));
+            animate_window_origin(self.window, window_origin(appkit_point), ANIMATION_DURATION);
         } else {
             // A hidden panel has no on-screen position to slide from: land on
-            // the point, then reveal.
-            let _ = send_void_point(self.window, c"setFrameOrigin:", window_origin(appkit_point));
+            // the point, then reveal. A zero-duration frame animation also
+            // supersedes any animation still running when the panel was hidden.
+            animate_window_origin(self.window, window_origin(appkit_point), 0.0);
         }
         self.set_visible(visible);
     }
@@ -129,9 +131,9 @@ impl CursorUi {
         self.set_kind(OverlayActionKind::Drag);
         let from = ax_screen_to_appkit(from_ax, from_display);
         let to = ax_screen_to_appkit(to_ax, to_display);
-        let _ = send_void_point(self.window, c"setFrameOrigin:", window_origin(from));
+        animate_window_origin(self.window, window_origin(from), 0.0);
         self.set_visible(visible);
-        animate_window_origin(self.window, window_origin(to));
+        animate_window_origin(self.window, window_origin(to), ANIMATION_DURATION);
     }
 
     /// Must only be called from the process main queue.
@@ -196,7 +198,7 @@ fn cursor_path() -> Option<Id> {
     send_id(path, c"CGPath")
 }
 
-fn animate_window_origin(window: Id, origin: CGPoint) {
+fn animate_window_origin(window: Id, origin: CGPoint, duration: f64) {
     let Some(context_class) = class(c"NSAnimationContext") else {
         let _ = send_void_point(window, c"setFrameOrigin:", origin);
         return;
@@ -207,12 +209,20 @@ fn animate_window_origin(window: Id, origin: CGPoint) {
     }
 
     let animated = send_id(context_class, c"currentContext").is_some_and(|context| {
-        let duration_set = send_void_f64(context, c"setDuration:", ANIMATION_DURATION);
+        let duration_set = send_void_f64(context, c"setDuration:", duration);
         if let Some(timing) = timing_function() {
             let _ = send_void_id(context, c"setTimingFunction:", timing);
         }
-        let moved = send_id(window, c"animator")
-            .is_some_and(|animator| send_void_point(animator, c"setFrameOrigin:", origin));
+        // NSWindow animates its full frame. Its animator accepts setFrameOrigin:
+        // but leaves an already-visible panel at the old point on macOS.
+        let moved = send_id(window, c"animator").is_some_and(|animator| {
+            send_void_rect_bool(
+                animator,
+                c"setFrame:display:",
+                rect(origin.x, origin.y, CURSOR_SIZE, CURSOR_SIZE),
+                true,
+            )
+        });
         duration_set && moved
     });
     let _ = send_void(context_class, c"endGrouping");
@@ -249,3 +259,65 @@ fn window_origin(point: (f64, f64)) -> CGPoint {
 fn rect(x: f64, y: f64, width: f64, height: f64) -> CGRect {
     CGRect::new(&CGPoint::new(x, y), &CGSize::new(width, height))
 }
+
+// Called by the main-thread, opt-in test harness. Normal libtest workers cannot
+// construct AppKit windows, so this regression has a dedicated native runner.
+#[cfg(all(test, target_arch = "aarch64"))]
+#[allow(dead_code)] // Also compiled by ordinary libtest, which cannot run AppKit.
+pub(super) fn verify_native() -> u32 {
+    use std::time::{Duration, Instant};
+    fn pump() {
+        let end = Instant::now() + Duration::from_millis(400);
+        while Instant::now() < end {
+            // SAFETY: this runner owns the process main thread and its run loop.
+            unsafe {
+                core_foundation::runloop::CFRunLoopRunInMode(
+                    core_foundation::runloop::kCFRunLoopDefaultMode,
+                    0.01,
+                    0,
+                );
+            }
+        }
+    }
+    let display = super::ffi::display_frame_for_ax_point((100.0, 100.0)).expect("desktop display");
+    let mut cursor = CursorUi::new().expect("native cursor panel");
+    cursor.show(OverlayActionKind::Click, (100.0, 100.0), display, true);
+    pump();
+    let initial = super::ffi::native_panel_state(cursor.window).0;
+    cursor.show(OverlayActionKind::Move, (300.0, 200.0), display, true);
+    pump();
+    let (frame, visible, passthrough, number) = super::ffi::native_panel_state(cursor.window);
+    let expected = CGPoint::new(initial.origin.x + 200.0, initial.origin.y - 100.0);
+    assert!(
+        (frame.origin.x - expected.x).abs() < 1.0 && (frame.origin.y - expected.y).abs() < 1.0,
+        "latest move must reach its endpoint: {frame:?}, expected {expected:?}"
+    );
+    assert!(visible && passthrough);
+    cursor.show_drag((300.0, 200.0), (500.0, 300.0), display, display, true);
+    pump();
+    let (frame, _, _, _) = super::ffi::native_panel_state(cursor.window);
+    let expected = CGPoint::new(initial.origin.x + 400.0, initial.origin.y - 200.0);
+    assert!(
+        (frame.origin.x - expected.x).abs() < 1.0 && (frame.origin.y - expected.y).abs() < 1.0,
+        "drag must reach its final endpoint, not remain at its start"
+    );
+    cursor.show(OverlayActionKind::Move, (700.0, 100.0), display, true);
+    cursor.hide();
+    cursor.show(OverlayActionKind::Click, (200.0, 400.0), display, true);
+    pump();
+    let (frame, visible, _, _) = super::ffi::native_panel_state(cursor.window);
+    assert!(
+        visible
+            && (frame.origin.x - initial.origin.x - 100.0).abs() < 1.0
+            && (frame.origin.y - initial.origin.y + 300.0).abs() < 1.0,
+        "an older animation must not restore its endpoint after hide and new feedback: {frame:?}, initial {initial:?}, visible {visible}"
+    );
+    assert!(super::ffi::window_exists(number));
+    let _ = send_void(cursor.window, c"close");
+    pump();
+    assert!(
+        !super::ffi::window_exists(number),
+        "closed but retained native window must retire target feedback"
+    );
+    number
+}

crates/computer-use-mcp/src/backend/macos/overlay/ffi.rs

@@ -372,3 +372,49 @@ pub(super) fn status_window_level() -> isize {
     // SAFETY: STATUS_WINDOW_LEVEL_KEY is a documented CGWindowLevelKey value.
     unsafe { CGWindowLevelForKey(STATUS_WINDOW_LEVEL_KEY) as isize }
 }
+
+pub(super) fn window_exists(window_id: u32) -> bool {
+    use core_foundation::base::TCFType;
+    use core_foundation::boolean::CFBoolean;
+    use core_foundation::dictionary::{CFDictionaryGetValue, CFDictionaryRef};
+    use core_graphics::window::{
+        copy_window_info, kCGWindowIsOnscreen, kCGWindowListOptionIncludingWindow,
+    };
+
+    let Some(windows) = copy_window_info(kCGWindowListOptionIncludingWindow, window_id) else {
+        return false;
+    };
+    windows.iter().any(|window| {
+        // SAFETY: CGWindowListCopyWindowInfo returns an array of dictionaries;
+        // the borrowed Boolean remains owned by that array throughout the query.
+        unsafe {
+            let dictionary = *window as CFDictionaryRef;
+            let value = CFDictionaryGetValue(dictionary, kCGWindowIsOnscreen.cast());
+            !value.is_null()
+                && core_foundation::base::CFGetTypeID(value) == CFBoolean::type_id()
+                && bool::from(CFBoolean::wrap_under_get_rule(value.cast()))
+        }
+    })
+}
+
+pub(super) fn send_void_rect_bool(receiver: Id, name: &CStr, value: CGRect, display: bool) -> bool {
+    let Some(selector) = selector(name) else {
+        return false;
+    };
+    if !can_send(receiver, selector) {
+        return false;
+    }
+    invoke!((), receiver, selector, CGRect => value, i8 => i8::from(display));
+    true
+}
+
+// These getters inspect the actual AppKit panel in the opt-in native regression.
+// CGRect's objc_msgSend return ABI here is specific to the arm64 runner.
+#[cfg(all(test, target_arch = "aarch64"))]
+pub(super) fn native_panel_state(window: Id) -> (CGRect, bool, bool, u32) {
+    let frame = invoke!(CGRect, window, selector(c"frame").unwrap());
+    let visible = invoke!(i8, window, selector(c"isVisible").unwrap()) != 0;
+    let passthrough = invoke!(i8, window, selector(c"ignoresMouseEvents").unwrap()) != 0;
+    let number = invoke!(isize, window, selector(c"windowNumber").unwrap()) as u32;
+    (frame, visible, passthrough, number)
+}