diff --git a/apps/desktop/native/computer-use/ComputerUse.swift b/apps/desktop/native/computer-use/ComputerUse.swift index 2b4a7acb02b..a55a3fde106 100644 --- a/apps/desktop/native/computer-use/ComputerUse.swift +++ b/apps/desktop/native/computer-use/ComputerUse.swift @@ -88,17 +88,15 @@ func remindersStore() throws -> EKEventStore { } /// A due date: `YYYY-MM-DD` is date-only; a date-time is ISO 8601, and one without an offset is this /// Mac's local time, as typing it into the app would be. -func dueComponents(_ text: String) throws -> DateComponents { +func dueComponents(_ text: String, _ field: String = "Due") throws -> DateComponents { var calendar = Calendar(identifier: .gregorian) - if text.range(of: #"^\d{4}-\d{2}-\d{2}$"#, options: .regularExpression) != nil { + if text.range(of: #"^[0-9]{4}-[0-9]{2}-[0-9]{2}$"#, options: .regularExpression) != nil { let parts = text.split(separator: "-").compactMap { Int($0) } let components = DateComponents(calendar: calendar, year: parts[0], month: parts[1], day: parts[2]) - guard components.isValidDate else { throw ComputerError("invalid_arguments", "Due date is not a real date.", dispatchState: "not_started") } + guard components.isValidDate else { throw invalidCommand("\(field) is not a real date.") } return DateComponents(year: parts[0], month: parts[1], day: parts[2]) } - guard let date = isoDateTime(text) else { - throw ComputerError("invalid_arguments", "Due must be YYYY-MM-DD or an ISO 8601 date-time.", dispatchState: "not_started") - } + guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be YYYY-MM-DD or a real ISO 8601 date-time.") } calendar.timeZone = .current return calendar.dateComponents([.year, .month, .day, .hour, .minute, .second, .timeZone], from: date) } @@ -134,29 +132,43 @@ func localGregorian() -> Calendar { var calendar = Calendar(identifier: .gregorian); calendar.timeZone = .current return calendar } -/// An ISO 8601 date-time; without an offset it is this Mac's local time. +// [0-9], not \d: ICU's \d also matches other scripts' digits, which Int() cannot read. +let isoDateTimePattern = try! NSRegularExpression(pattern: #"^([0-9]{4})-([0-9]{2})-([0-9]{2})T([0-9]{2}):([0-9]{2})(?::([0-9]{2})(?:\.[0-9]{1,9})?)?(?:(Z)|([+-])([0-9]{2}):([0-9]{2}))?$"#) +/// An ISO 8601 date-time; without an offset it is this Mac's local time. Fractions of a second do not +/// matter to a calendar item, so they are dropped. Every field must be in range: `ISO8601DateFormatter` +/// rolls Feb 30 into March and accepts any offset, so a zoned time is checked part by part. func isoDateTime(_ text: String) -> Date? { - for options: ISO8601DateFormatter.Options in [[.withInternetDateTime], [.withInternetDateTime, .withFractionalSeconds]] { - let zoned = ISO8601DateFormatter(); zoned.formatOptions = options - if let date = zoned.date(from: text) { return date } + guard let match = isoDateTimePattern.firstMatch(in: text, range: NSRange(text.startIndex..., in: text)) else { return nil } + func part(_ index: Int) -> String? { Range(match.range(at: index), in: text).map { String(text[$0]) } } + func number(_ index: Int) -> Int? { part(index).flatMap { Int($0) } } + var zone: TimeZone? + if part(7) != nil { zone = TimeZone(secondsFromGMT: 0) } + if let sign = part(8), let hours = number(9), let minutes = number(10) { + guard minutes < 60, hours < 14 || (hours == 14 && minutes == 0) else { return nil } + zone = TimeZone(secondsFromGMT: (sign == "-" ? -1 : 1) * (hours * 3600 + minutes * 60)) + } + guard let zone else { + // A local time keeps the formatter, which resolves a repeated hour at a daylight-saving change. + let whole = String(text.prefix(19)) + let local = DateFormatter(); local.locale = Locale(identifier: "en_US_POSIX"); local.timeZone = .current + local.dateFormat = whole.count == 16 ? "yyyy-MM-dd'T'HH:mm" : "yyyy-MM-dd'T'HH:mm:ss" + local.isLenient = false + return local.date(from: whole) } - // Fractions of a second do not matter to a calendar item, so a local time drops them. - guard text.range(of: #"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?$"#, options: .regularExpression) != nil else { return nil } - let whole = String(text.prefix(19)) - let local = DateFormatter(); local.locale = Locale(identifier: "en_US_POSIX"); local.timeZone = .current - local.dateFormat = whole.count == 16 ? "yyyy-MM-dd'T'HH:mm" : "yyyy-MM-dd'T'HH:mm:ss" - local.isLenient = false - return local.date(from: whole) + var calendar = Calendar(identifier: .gregorian); calendar.timeZone = zone + let components = DateComponents(year: number(1), month: number(2), day: number(3), hour: number(4), minute: number(5), second: number(6) ?? 0) + guard components.isValidDate(in: calendar) else { return nil } + return calendar.date(from: components) } /// An event boundary: an ISO 8601 date-time (local without an offset), or `YYYY-MM-DD` for all-day events. func eventDate(_ text: String?, _ field: String, allDay: Bool) throws -> Date { let text = try commandText(text, field, max: 64) if allDay { - let parts = try dueComponents(text) + let parts = try dueComponents(text, field) guard parts.hour == nil, let date = localGregorian().date(from: parts) else { throw invalidCommand("All-day \(field) must be YYYY-MM-DD.") } return date } - guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be an ISO 8601 date-time.") } + guard let date = isoDateTime(text) else { throw invalidCommand("\(field) must be a real ISO 8601 date-time.") } return date } func runAppCommand(_ command: AppCommand) throws -> [String: Any] { @@ -168,11 +180,10 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] { } return ["kind": "command", "command": command.name, "items": lists, "truncated": allLists.count > 200] case "create_reminder": - let title = (command.title ?? "").trimmingCharacters(in: .whitespacesAndNewlines) - guard !title.isEmpty, title.count <= 1024 else { throw ComputerError("invalid_arguments", "A reminder needs a title of at most 1024 characters.", dispatchState: "not_started") } - guard (command.notes?.count ?? 0) <= 8000 else { throw ComputerError("invalid_arguments", "Notes are limited to 8000 characters.", dispatchState: "not_started") } - guard [0, 1, 5, 9].contains(command.priority ?? 0) else { throw ComputerError("invalid_arguments", "Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).", dispatchState: "not_started") } - let due = try command.due.map(dueComponents) + let title = try commandText(command.title, "Title") + let notes = try optionalText(command.notes, "Notes", max: 8000) + guard [0, 1, 5, 9].contains(command.priority ?? 0) else { throw invalidCommand("Priority must be 0 (none), 1 (high), 5 (medium) or 9 (low).") } + let due = try command.due.map { try dueComponents($0) } let store = try remindersStore() let list: EKCalendar if let listId = command.listId { @@ -186,7 +197,7 @@ func runAppCommand(_ command: AppCommand) throws -> [String: Any] { } guard list.allowsContentModifications else { throw ComputerError("list_read_only", "This reminder list is read-only.", dispatchState: "not_started") } let reminder = EKReminder(eventStore: store) - reminder.calendar = list; reminder.title = title; reminder.notes = command.notes + reminder.calendar = list; reminder.title = title; reminder.notes = notes reminder.priority = command.priority ?? 0; reminder.dueDateComponents = due try checkCancellation() inputDispatched = true diff --git a/apps/desktop/native/computer-use/tests/NativeTests.swift b/apps/desktop/native/computer-use/tests/NativeTests.swift index 269fc6bda39..bd29405fe49 100644 --- a/apps/desktop/native/computer-use/tests/NativeTests.swift +++ b/apps/desktop/native/computer-use/tests/NativeTests.swift @@ -42,9 +42,12 @@ import AppKit precondition(timed.hour != nil && timed.minute != nil && timed.second == 45) let localDue = try dueComponents("2027-05-04T09:30") precondition(localDue.hour == 9 && localDue.minute == 30) - for due in ["2027-02-30", "2027-05-04T25:30:00", "May 4", "2027-5-4"] { + for due in ["2027-02-30", "2027-05-04T25:30:00", "May 4", "2027-5-4", "2027-02-30T09:00:00Z", "2027-02-29T09:00:00.5+01:00", "2027-05-04T24:00:00Z", "2027-05-04T09:00:00+25:00", "2027-05-04T09:00:00+05:99", "2027-05-04T09:00:00+14:59", "2027-05-04T09:00:00-14:30", "2027-0٥-04", "2027-05-04T1٠:00Z"] { expectError("invalid_arguments") { _ = try dueComponents(due) } } + precondition(isoDateTime("2028-02-29T09:00Z") == Date(timeIntervalSince1970: 1_835_427_600)) + precondition(isoDateTime("2027-05-04T09:00:00+05:30") == Date(timeIntervalSince1970: 1_809_401_400)) + precondition(isoDateTime("2027-05-04T09:00:00+14:00") == Date(timeIntervalSince1970: 1_809_370_800)) expectError("unknown_command") { _ = try runAppCommand(AppCommand(name: "run_script")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: " ")) } expectError("invalid_arguments") { _ = try runAppCommand(AppCommand(name: "create_reminder", title: "Pay rent", priority: 3)) } diff --git a/apps/desktop/src/main/computer-use/service.test.ts b/apps/desktop/src/main/computer-use/service.test.ts index 3fd3b00e3b8..4684e5c7f20 100644 --- a/apps/desktop/src/main/computer-use/service.test.ts +++ b/apps/desktop/src/main/computer-use/service.test.ts @@ -20,38 +20,40 @@ function setup(enabled = true, askForApps = true) { config.set('computerUseEnabled', enabled) config.set('computerUseAskForApps', askForApps) let sequence = 0 - const request = vi.fn(async (method: string): Promise => { - if (method === 'status' || method === 'request_permission') - return { kind: 'status', platform: 'darwin', accessibility: true, screenRecording: false } - if (method === 'list_apps') + const request = vi.fn( + async (method: string, _params?: Record): Promise => { + if (method === 'status' || method === 'request_permission') + return { kind: 'status', platform: 'darwin', accessibility: true, screenRecording: false } + if (method === 'list_apps') + return { + kind: 'apps', + apps: [{ bundleId: 'com.example.Fixture', name: 'Fixture', pid: 42, isActive: false }], + } + if (method === 'app_command') + return { + kind: 'command', + command: 'create_reminder', + created: { id: 'r1', title: 'Pay rent', list: 'Reminders' }, + } + if (method === 'get_app_state') + return { + kind: 'state', + bundleId: 'com.example.Fixture', + snapshotId: `s${++sequence}`, + windowId: '1', + windows: [], + nodes: [{ elementId: 'e1', role: 'AXTextArea', actions: [], windowId: '1' }], + truncated: false, + } return { - kind: 'apps', - apps: [{ bundleId: 'com.example.Fixture', name: 'Fixture', pid: 42, isActive: false }], - } - if (method === 'app_command') - return { - kind: 'command', - command: 'create_reminder', - created: { id: 'r1', title: 'Pay rent', list: 'Reminders' }, - } - if (method === 'get_app_state') - return { - kind: 'state', + kind: 'action', + action: 'click', bundleId: 'com.example.Fixture', - snapshotId: `s${++sequence}`, - windowId: '1', - windows: [], - nodes: [{ elementId: 'e1', role: 'AXTextArea', actions: [], windowId: '1' }], - truncated: false, + dispatched: true, + verified: false, } - return { - kind: 'action', - action: 'click', - bundleId: 'com.example.Fixture', - dispatched: true, - verified: false, } - }) + ) const native = { request, stop: vi.fn() } const approveApp = vi.fn(async (): Promise<'always' | 'deny' | 'once'> => 'always') const confirmAppCommand = vi.fn(async (_request: { appName: string; summary: string }) => true) @@ -177,6 +179,58 @@ describe('native computer use authority and lifecycle', () => { ) }) + it('names the chosen list, calendar or group in the confirmation, not its id', async () => { + const { service, native, confirmAppCommand } = setup() + const lists = { + list_reminder_lists: 'Sprint 14', + list_calendars: 'Travel', + list_contact_groups: 'Hires Q1 2027', + } + let truncated = false + native.request.mockImplementation(async (method, params): Promise => { + if (method !== 'app_command') return { kind: 'apps', apps: [] } + const { name } = params?.command as { name: keyof typeof lists } + return { + kind: 'command', + command: name, + items: [{ id: 'id-1', title: lists[name], allowsModifications: true }], + truncated, + } + }) + confirmAppCommand.mockResolvedValue(false) + for (const command of [ + { name: 'create_reminder', title: 'Pay rent', listId: 'id-1' }, + { + name: 'create_event', + title: 'Flight', + start: '2027-06-09T07:15', + end: '2027-06-09T08:10', + calendarId: 'id-1', + }, + { name: 'create_contact', givenName: 'Ana', groupId: 'id-1' }, + ]) + await expect( + service.execute(`named-${command.name}`, 'chat', { action: 'app_command', command }) + ).rejects.toMatchObject({ details: { code: 'user_declined' } }) + expect( + confirmAppCommand.mock.calls.map(([request]) => request.summary.split('\n').at(-1)) + ).toEqual(['List: Sprint 14', 'Calendar: Travel', 'Group: Hires Q1 2027']) + + const deleted = { + action: 'app_command', + command: { name: 'create_reminder', title: 'Pay rent', listId: 'deleted-list' }, + } + await expect(service.execute('deleted', 'chat', deleted)).rejects.toMatchObject({ + details: { code: 'list_unavailable', dispatchState: 'not_started' }, + }) + expect(confirmAppCommand).toHaveBeenCalledTimes(3) + truncated = true + await expect(service.execute('past-limit', 'chat', deleted)).rejects.toMatchObject({ + details: { code: 'user_declined' }, + }) + expect(confirmAppCommand.mock.lastCall?.[0].summary).toContain('List: deleted-list') + }) + it('refuses an app command when the user denies the app, before asking to confirm', async () => { const { service, native, approveApp, confirmAppCommand } = setup() approveApp.mockResolvedValue('deny') diff --git a/apps/desktop/src/main/computer-use/service.ts b/apps/desktop/src/main/computer-use/service.ts index 08b4139de13..d142a91572c 100644 --- a/apps/desktop/src/main/computer-use/service.ts +++ b/apps/desktop/src/main/computer-use/service.ts @@ -68,8 +68,24 @@ function workBundleId(input: ComputerUseWithAppCommandsInput): string | undefine return 'bundleId' in input ? input.bundleId : undefined } -/** The exact fields a write will save, for the user to confirm; null for a read, which needs none. */ -function writeSummary(command: AppCommand): string | null { +/** The list, calendar or group a write names by id, and the read that lists its kind. */ +function chosenDestination( + command: AppCommand +): { id: string; list: 'list_reminder_lists' | 'list_calendars' | 'list_contact_groups' } | null { + if (command.name === 'create_reminder' && command.listId) + return { id: command.listId, list: 'list_reminder_lists' } + if (command.name === 'create_event' && command.calendarId) + return { id: command.calendarId, list: 'list_calendars' } + if (command.name === 'create_contact' && command.groupId) + return { id: command.groupId, list: 'list_contact_groups' } + return null +} + +/** + * The exact fields a write will save, for the user to confirm; null for a read, which needs none. + * `destination` is the chosen list, calendar or group by name, so the user can tell where it goes. + */ +function writeSummary(command: AppCommand, destination: string | undefined): string | null { // Trimmed exactly as the helper trims before saving, so the user confirms the saved values. const lines = (entries: Array<[string, string | number | undefined]>) => entries @@ -83,7 +99,7 @@ function writeSummary(command: AppCommand): string | null { ['Reminder', command.title], ['Due', command.due], ['Priority', command.priority ? PRIORITY_NAMES[command.priority] : undefined], - ['List', command.listId ?? 'default'], + ['List', destination ?? 'default'], ['Notes', command.notes], ]) case 'create_event': @@ -99,7 +115,7 @@ function writeSummary(command: AppCommand): string | null { ? `${command.alertMinutesBefore} minutes before` : undefined, ], - ['Calendar', command.calendarId ?? 'default'], + ['Calendar', destination ?? 'default'], ['Notes', command.notes], ]) case 'create_contact_group': @@ -111,7 +127,7 @@ function writeSummary(command: AppCommand): string | null { ['Job title', command.jobTitle], ['Emails', command.emails?.map((email) => email.trim()).join(', ')], ['Phones', command.phones?.map((phone) => phone.trim()).join(', ')], - ['Group', command.groupId], + ['Group', destination], ]) default: return null @@ -437,7 +453,7 @@ export class ComputerUseService { this.activity = { ...this.activity, appName } this.deps.onActivity(this.activity) } - const summary = writeSummary(command) + const summary = writeSummary(command, await this.destinationName(command, work)) if (summary !== null) { const confirmed = await this.deps.confirmAppCommand( { appName, summary }, @@ -452,12 +468,39 @@ export class ComputerUseService { }) } this.check(work) + return await this.appCommand(command) + } + + /** Runs one app command in the helper and checks the reply is for that command. */ + private async appCommand( + command: AppCommand + ): Promise> { const result = await this.deps.native.request('app_command', { command }) if (result.kind !== 'command' || result.command !== command.name) throw new Error('Computer Use returned a result for a different command.') return result } + /** + * The name of the list, calendar or group a write chose by id, read when it is confirmed so the + * name is current. An id missing from a complete read is refused before the user is asked; past + * the read's limit the id itself is shown, and the helper checks it before saving. + */ + private async destinationName(command: AppCommand, work: ToolWork): Promise { + const destination = chosenDestination(command) + if (!destination) return undefined + const lists = await this.appCommand({ name: destination.list }) + this.check(work) + const chosen = lists.items?.find((item) => item.id === destination.id) + if (chosen) return chosen.title + if (lists.truncated) return destination.id + throw new ComputerUseError({ + code: 'list_unavailable', + message: `No list, calendar or group has this id; call ${destination.list}.`, + dispatchState: 'not_started', + }) + } + private rememberSnapshot(work: ToolWork, snapshot: ComputerUseSnapshot, windowId?: string): void { if (!('bundleId' in work.input) || snapshot.bundleId !== work.input.bundleId) throw new Error('Computer Use returned state for a different app.') diff --git a/packages/desktop-bridge/src/computer-use.generated.ts b/packages/desktop-bridge/src/computer-use.generated.ts index 2c0dddfebe4..69fe3e86753 100644 --- a/packages/desktop-bridge/src/computer-use.generated.ts +++ b/packages/desktop-bridge/src/computer-use.generated.ts @@ -234,15 +234,26 @@ const CommandTitle = z .string() .max(1024) .refine((title) => title.trim().length > 0, "A title must not be blank"); -/** `YYYY-MM-DD`, or a date-time with optional seconds, fraction and offset; the helper checks it is a real date. */ +/** `YYYY-MM-DD`, or a date-time with optional seconds, fraction and offset. */ const ISO_DATE_OR_TIME = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:\d{2})?)?$/; -const CommandNotes = z.string().max(8000).optional(); -const EventBoundary = z +const RealDate = z.iso.date(); +// Zod requires seconds beside a zone unless precision is minutes, and the shape allows either. +const RealDateTime = z.union([ + z.iso.datetime({ local: true, offset: true }), + z.iso.datetime({ local: true, offset: true, precision: -1 }), +]); +/** The shape the model sees, refined to a day and time that exist, so Feb 30 never reaches the user's confirmation. */ +const DateOrDateTime = z .string() .regex(ISO_DATE_OR_TIME, "Use YYYY-MM-DD or an ISO 8601 date-time") - .describe( - "ISO 8601 date-time, or YYYY-MM-DD when allDay is true. Without an offset it is this Mac's local time, as typing it into Calendar would be.", + .refine( + (value) => RealDate.safeParse(value).success || RealDateTime.safeParse(value).success, + "Use a real date and time", ); +const CommandNotes = z.string().max(8000).optional(); +const EventBoundary = DateOrDateTime.describe( + "ISO 8601 date-time, or YYYY-MM-DD when allDay is true. Without an offset it is this Mac's local time, as typing it into Calendar would be.", +); /** Typed app commands: one allowlisted create or list read through the OS's own data frameworks, never a script. */ export const AppCommandSchema = z.discriminatedUnion("name", [ @@ -251,13 +262,9 @@ export const AppCommandSchema = z.discriminatedUnion("name", [ name: z.literal("create_reminder"), title: CommandTitle, notes: CommandNotes, - due: z - .string() - .regex(ISO_DATE_OR_TIME, "Use YYYY-MM-DD or an ISO 8601 date-time") - .optional() - .describe( - "YYYY-MM-DD for a date-only reminder, or an ISO 8601 date-time (this Mac's local time without an offset).", - ), + due: DateOrDateTime.optional().describe( + "YYYY-MM-DD for a date-only reminder, or an ISO 8601 date-time (this Mac's local time without an offset).", + ), listId: z .string() .min(1)