Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
61 changes: 36 additions & 25 deletions apps/desktop/native/computer-use/ComputerUse.swift
Original file line number Diff line number Diff line change
Expand Up @@ -88,17 +88,15 @@ func remindersStore() throws -> EKEventStore {
}
/// A due date: `YYYY-MM-DD` is date-only; a date-time is ISO 8601, and one without an offset is this
/// Mac's local time, as typing it into the app would be.
func dueComponents(_ text: String) throws -> DateComponents {
func dueComponents(_ text: String, _ field: String = "Due") throws -> DateComponents {
var calendar = Calendar(identifier: .gregorian)
if text.range(of: #"^\d{4}-\d{2}-\d{2}$"#, options: .regularExpression) != nil {
if text.range(of: #"^[0-9]{4}-[0-9]{2}-[0-9]{2}$"#, options: .regularExpression) != nil {
let parts = text.split(separator: "-").compactMap { Int($0) }
let components = DateComponents(calendar: calendar, year: parts[0], month: parts[1], day: parts[2])
guard components.isValidDate else { throw ComputerError("invalid_arguments", "Due date is not a real date.", dispatchState: "not_started") }
guard components.isValidDate else { throw invalidCommand("\(field) is not a real date.") }
return DateComponents(year: parts[0], month: parts[1], day: parts[2])
}
guard let date = isoDateTime(text) else {
throw ComputerError("invalid_arguments", "Due must be YYYY-MM-DD or an ISO 8601 date-time.", dispatchState: "not_started")
}
guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be YYYY-MM-DD or a real ISO 8601 date-time.") }
calendar.timeZone = .current
return calendar.dateComponents([.year, .month, .day, .hour, .minute, .second, .timeZone], from: date)
}
Expand Down Expand Up @@ -134,29 +132,43 @@ func localGregorian() -> Calendar {
var calendar = Calendar(identifier: .gregorian); calendar.timeZone = .current
return calendar
}
/// An ISO 8601 date-time; without an offset it is this Mac's local time.
// [0-9], not \d: ICU's \d also matches other scripts' digits, which Int() cannot read.
let isoDateTimePattern = try! NSRegularExpression(pattern: #"^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2})(?::([0-9]{2})(?:\.[0-9]{1,9})?)?(?:(Z)|([+-])([0-9]{2}):([0-9]{2}))?$"#)
/// An ISO 8601 date-time; without an offset it is this Mac's local time. Fractions of a second do not
/// matter to a calendar item, so they are dropped. Every field must be in range: `ISO8601DateFormatter`
/// rolls Feb 30 into March and accepts any offset, so a zoned time is checked part by part.
func isoDateTime(_ text: String) -> Date? {
for options: ISO8601DateFormatter.Options in [[.withInternetDateTime], [.withInternetDateTime, .withFractionalSeconds]] {
let zoned = ISO8601DateFormatter(); zoned.formatOptions = options
if let date = zoned.date(from: text) { return date }
guard let match = isoDateTimePattern.firstMatch(in: text, range: NSRange(text.startIndex..., in: text)) else { return nil }
func part(_ index: Int) -> String? { Range(match.range(at: index), in: text).map { String(text[$0]) } }
func number(_ index: Int) -> Int? { part(index).flatMap { Int($0) } }
var zone: TimeZone?
if part(7) != nil { zone = TimeZone(secondsFromGMT: 0) }
if let sign = part(8), let hours = number(9), let minutes = number(10) {
guard minutes < 60, hours < 14 || (hours == 14 && minutes == 0) else { return nil }
zone = TimeZone(secondsFromGMT: (sign == "-" ? -1 : 1) * (hours * 3600 + minutes * 60))
}
guard let zone else {
// A local time keeps the formatter, which resolves a repeated hour at a daylight-saving change.
let whole = String(text.prefix(19))
let local = DateFormatter(); local.locale = Locale(identifier: "en_US_POSIX"); local.timeZone = .current
local.dateFormat = whole.count == 16 ? "yyyy-MM-dd'T'HH:mm" : "yyyy-MM-dd'T'HH:mm:ss"
local.isLenient = false
return local.date(from: whole)
}
// Fractions of a second do not matter to a calendar item, so a local time drops them.
guard text.range(of: #"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?$"#, options: .regularExpression) != nil else { return nil }
let whole = String(text.prefix(19))
let local = DateFormatter(); local.locale = Locale(identifier: "en_US_POSIX"); local.timeZone = .current
local.dateFormat = whole.count == 16 ? "yyyy-MM-dd'T'HH:mm" : "yyyy-MM-dd'T'HH:mm:ss"
local.isLenient = false
return local.date(from: whole)
var calendar = Calendar(identifier: .gregorian); calendar.timeZone = zone
let components = DateComponents(year: number(1), month: number(2), day: number(3), hour: number(4), minute: number(5), second: number(6) ?? 0)
guard components.isValidDate(in: calendar) else { return nil }
return calendar.date(from: components)
}
/// An event boundary: an ISO 8601 date-time (local without an offset), or `YYYY-MM-DD` for all-day events.
func eventDate(_ text: String?, _ field: String, allDay: Bool) throws -> Date {
let text = try commandText(text, field, max: 64)
if allDay {
let parts = try dueComponents(text)
let parts = try dueComponents(text, field)
guard parts.hour == nil, let date = localGregorian().date(from: parts) else { throw invalidCommand("All-day \(field) must be YYYY-MM-DD.") }
return date
}
guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be an ISO 8601 date-time.") }
guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be a real ISO 8601 date-time.") }
return date
}
func runAppCommand(_ command: AppCommand) throws -> [String: Any] {
Expand All @@ -168,11 +180,10 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] {
}
return ["kind": "command", "command": command.name, "items": lists, "truncated": allLists.count > 200]
case "create_reminder":
let title = (command.title ?? "").trimmingCharacters(in: .whitespacesAndNewlines)
guard !title.isEmpty, title.count <= 1024 else { throw ComputerError("invalid_arguments", "A reminder needs a title of at most 1024 characters.", dispatchState: "not_started") }
guard (command.notes?.count ?? 0) <= 8000 else { throw ComputerError("invalid_arguments", "Notes are limited to 8000 characters.", dispatchState: "not_started") }
guard [0, 1, 5, 9].contains(command.priority ?? 0) else { throw ComputerError("invalid_arguments", "Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).", dispatchState: "not_started") }
let due = try command.due.map(dueComponents)
let title = try commandText(command.title, "Title")
let notes = try optionalText(command.notes, "Notes", max: 8000)
guard [0, 1, 5, 9].contains(command.priority ?? 0) else { throw invalidCommand("Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).") }
let due = try command.due.map { try dueComponents($0) }
let store = try remindersStore()
let list: EKCalendar
if let listId = command.listId {
Expand All @@ -186,7 +197,7 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] {
}
guard list.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") }
let reminder = EKReminder(eventStore: store)
reminder.calendar = list; reminder.title = title; reminder.notes = command.notes
reminder.calendar = list; reminder.title = title; reminder.notes = notes
reminder.priority = command.priority ?? 0; reminder.dueDateComponents = due
try checkCancellation()
inputDispatched = true
Expand Down
5 changes: 4 additions & 1 deletion apps/desktop/native/computer-use/tests/NativeTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,12 @@ import AppKit
precondition(timed.hour != nil && timed.minute != nil && timed.second == 45)
let localDue = try dueComponents("2027-05-04T09:30")
precondition(localDue.hour == 9 && localDue.minute == 30)
for due in ["2027-02-30", "2027-05-04T25:30:00", "May 4", "2027-5-4"] {
for due in ["2027-02-30", "2027-05-04T25:30:00", "May 4", "2027-5-4", "2027-02-30T09:00:00Z", "2027-02-29T09:00:00.5+01:00", "2027-05-04T24:00:00Z", "2027-05-04T09:00:00+25:00", "2027-05-04T09:00:00+05:99", "2027-05-04T09:00:00+14:59", "2027-05-04T09:00:00-14:30", "2027-0٥-04", "2027-05-04T1٠:00Z"] {
expectError("invalid_arguments") { _ = try dueComponents(due) }
}
precondition(isoDateTime("2028-02-29T09:00Z") == Date(timeIntervalSince1970: 1_835_427_600))
precondition(isoDateTime("2027-05-04T09:00:00+05:30") == Date(timeIntervalSince1970: 1_809_401_400))
precondition(isoDateTime("2027-05-04T09:00:00+14:00") == Date(timeIntervalSince1970: 1_809_370_800))
expectError("unknown_command") { _ = try runAppCommand(AppCommand(name: "run_script")) }
expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: " ")) }
expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: "Pay rent", priority: 3)) }
Expand Down
110 changes: 82 additions & 28 deletions apps/desktop/src/main/computer-use/service.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,38 +20,40 @@ function setup(enabled = true, askForApps = true) {
config.set('computerUseEnabled', enabled)
config.set('computerUseAskForApps', askForApps)
let sequence = 0
const request = vi.fn(async (method: string): Promise<ComputerUseResult> => {
if (method === 'status' || method === 'request_permission')
return { kind: 'status', platform: 'darwin', accessibility: true, screenRecording: false }
if (method === 'list_apps')
const request = vi.fn(
async (method: string, _params?: Record<string, unknown>): Promise<ComputerUseResult> => {
if (method === 'status' || method === 'request_permission')
return { kind: 'status', platform: 'darwin', accessibility: true, screenRecording: false }
if (method === 'list_apps')
return {
kind: 'apps',
apps: [{ bundleId: 'com.example.Fixture', name: 'Fixture', pid: 42, isActive: false }],
}
if (method === 'app_command')
return {
kind: 'command',
command: 'create_reminder',
created: { id: 'r1', title: 'Pay rent', list: 'Reminders' },
}
if (method === 'get_app_state')
return {
kind: 'state',
bundleId: 'com.example.Fixture',
snapshotId: `s${++sequence}`,
windowId: '1',
windows: [],
nodes: [{ elementId: 'e1', role: 'AXTextArea', actions: [], windowId: '1' }],
truncated: false,
}
return {
kind: 'apps',
apps: [{ bundleId: 'com.example.Fixture', name: 'Fixture', pid: 42, isActive: false }],
}
if (method === 'app_command')
return {
kind: 'command',
command: 'create_reminder',
created: { id: 'r1', title: 'Pay rent', list: 'Reminders' },
}
if (method === 'get_app_state')
return {
kind: 'state',
kind: 'action',
action: 'click',
bundleId: 'com.example.Fixture',
snapshotId: `s${++sequence}`,
windowId: '1',
windows: [],
nodes: [{ elementId: 'e1', role: 'AXTextArea', actions: [], windowId: '1' }],
truncated: false,
dispatched: true,
verified: false,
}
return {
kind: 'action',
action: 'click',
bundleId: 'com.example.Fixture',
dispatched: true,
verified: false,
}
})
)
const native = { request, stop: vi.fn() }
const approveApp = vi.fn(async (): Promise<'always' | 'deny' | 'once'> => 'always')
const confirmAppCommand = vi.fn(async (_request: { appName: string; summary: string }) => true)
Expand Down Expand Up @@ -177,6 +179,58 @@ describe('native computer use authority and lifecycle', () => {
)
})

it('names the chosen list, calendar or group in the confirmation, not its id', async () => {
const { service, native, confirmAppCommand } = setup()
const lists = {
list_reminder_lists: 'Sprint 14',
list_calendars: 'Travel',
list_contact_groups: 'Hires Q1 2027',
}
let truncated = false
native.request.mockImplementation(async (method, params): Promise<ComputerUseResult> => {
if (method !== 'app_command') return { kind: 'apps', apps: [] }
const { name } = params?.command as { name: keyof typeof lists }
return {
kind: 'command',
command: name,
items: [{ id: 'id-1', title: lists[name], allowsModifications: true }],
truncated,
}
})
confirmAppCommand.mockResolvedValue(false)
for (const command of [
{ name: 'create_reminder', title: 'Pay rent', listId: 'id-1' },
{
name: 'create_event',
title: 'Flight',
start: '2027-06-09T07:15',
end: '2027-06-09T08:10',
calendarId: 'id-1',
},
{ name: 'create_contact', givenName: 'Ana', groupId: 'id-1' },
])
await expect(
service.execute(`named-${command.name}`, 'chat', { action: 'app_command', command })
).rejects.toMatchObject({ details: { code: 'user_declined' } })
expect(
confirmAppCommand.mock.calls.map(([request]) => request.summary.split('\n').at(-1))
).toEqual(['List: Sprint 14', 'Calendar: Travel', 'Group: Hires Q1 2027'])

const deleted = {
action: 'app_command',
command: { name: 'create_reminder', title: 'Pay rent', listId: 'deleted-list' },
}
await expect(service.execute('deleted', 'chat', deleted)).rejects.toMatchObject({
details: { code: 'list_unavailable', dispatchState: 'not_started' },
})
expect(confirmAppCommand).toHaveBeenCalledTimes(3)
truncated = true
await expect(service.execute('past-limit', 'chat', deleted)).rejects.toMatchObject({
details: { code: 'user_declined' },
})
expect(confirmAppCommand.mock.lastCall?.[0].summary).toContain('List: deleted-list')
})

it('refuses an app command when the user denies the app, before asking to confirm', async () => {
const { service, native, approveApp, confirmAppCommand } = setup()
approveApp.mockResolvedValue('deny')
Expand Down
55 changes: 49 additions & 6 deletions apps/desktop/src/main/computer-use/service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,24 @@ function workBundleId(input: ComputerUseWithAppCommandsInput): string | undefine
return 'bundleId' in input ? input.bundleId : undefined
}

/** The exact fields a write will save, for the user to confirm; null for a read, which needs none. */
function writeSummary(command: AppCommand): string | null {
/** The list, calendar or group a write names by id, and the read that lists its kind. */
function chosenDestination(
command: AppCommand
): { id: string; list: 'list_reminder_lists' | 'list_calendars' | 'list_contact_groups' } | null {
if (command.name === 'create_reminder' && command.listId)
return { id: command.listId, list: 'list_reminder_lists' }
if (command.name === 'create_event' && command.calendarId)
return { id: command.calendarId, list: 'list_calendars' }
if (command.name === 'create_contact' && command.groupId)
return { id: command.groupId, list: 'list_contact_groups' }
return null
}

/**
* The exact fields a write will save, for the user to confirm; null for a read, which needs none.
* `destination` is the chosen list, calendar or group by name, so the user can tell where it goes.
*/
function writeSummary(command: AppCommand, destination: string | undefined): string | null {
// Trimmed exactly as the helper trims before saving, so the user confirms the saved values.
const lines = (entries: Array<[string, string | number | undefined]>) =>
entries
Expand All @@ -83,7 +99,7 @@ function writeSummary(command: AppCommand): string | null {
['Reminder', command.title],
['Due', command.due],
['Priority', command.priority ? PRIORITY_NAMES[command.priority] : undefined],
['List', command.listId ?? 'default'],
['List', destination ?? 'default'],
['Notes', command.notes],
])
case 'create_event':
Expand All @@ -99,7 +115,7 @@ function writeSummary(command: AppCommand): string | null {
? `${command.alertMinutesBefore} minutes before`
: undefined,
],
['Calendar', command.calendarId ?? 'default'],
['Calendar', destination ?? 'default'],
['Notes', command.notes],
])
case 'create_contact_group':
Expand All @@ -111,7 +127,7 @@ function writeSummary(command: AppCommand): string | null {
['Job title', command.jobTitle],
['Emails', command.emails?.map((email) => email.trim()).join(', ')],
['Phones', command.phones?.map((phone) => phone.trim()).join(', ')],
['Group', command.groupId],
['Group', destination],
])
default:
return null
Expand Down Expand Up @@ -437,7 +453,7 @@ export class ComputerUseService {
this.activity = { ...this.activity, appName }
this.deps.onActivity(this.activity)
}
const summary = writeSummary(command)
const summary = writeSummary(command, await this.destinationName(command, work))
if (summary !== null) {
const confirmed = await this.deps.confirmAppCommand(
{ appName, summary },
Expand All @@ -452,12 +468,39 @@ export class ComputerUseService {
})
}
this.check(work)
return await this.appCommand(command)
}

/** Runs one app command in the helper and checks the reply is for that command. */
private async appCommand(
command: AppCommand
): Promise<Extract<ComputerUseResult, { kind: 'command' }>> {
const result = await this.deps.native.request('app_command', { command })
if (result.kind !== 'command' || result.command !== command.name)
throw new Error('Computer Use returned a result for a different command.')
return result
}

/**
* The name of the list, calendar or group a write chose by id, read when it is confirmed so the
* name is current. An id missing from a complete read is refused before the user is asked; past
* the read's limit the id itself is shown, and the helper checks it before saving.
*/
private async destinationName(command: AppCommand, work: ToolWork): Promise<string | undefined> {
const destination = chosenDestination(command)
if (!destination) return undefined
const lists = await this.appCommand({ name: destination.list })
this.check(work)
const chosen = lists.items?.find((item) => item.id === destination.id)
if (chosen) return chosen.title
if (lists.truncated) return destination.id
throw new ComputerUseError({
code: 'list_unavailable',
message: `No list, calendar or group has this id; call ${destination.list}.`,
dispatchState: 'not_started',
})
}

private rememberSnapshot(work: ToolWork, snapshot: ComputerUseSnapshot, windowId?: string): void {
if (!('bundleId' in work.input) || snapshot.bundleId !== work.input.bundleId)
throw new Error('Computer Use returned state for a different app.')
Expand Down
Loading
Loading