Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions apps/desktop/src/preload/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,7 @@ const api: SimDesktopApi = {
...(process.platform === 'darwin'
? {
computerUse: {
stableElementIds: true,
getActivity: () => ipcRenderer.invoke('computer-use:get-activity'),
getStatus: () => ipcRenderer.invoke('computer-use:status'),
setEnabled: (enabled: boolean) => ipcRenderer.invoke('computer-use:set-enabled', enabled),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { getErrorMessage } from '@sim/utils/errors'
import { omit } from '@sim/utils/object'
import { backoffWithJitter } from '@sim/utils/retry'
import { cancelComputerUseTool, executeComputerUseTool } from '@/lib/computer-use/transport'
import { getDesktopBridge } from '@/lib/desktop'
import {
ASYNC_TOOL_CONFIRMATION_STATUS,
type AsyncConfirmationStatus,
Expand Down Expand Up @@ -245,7 +246,10 @@ export async function executeComputerToolOnClient(
: result.kind === 'action' && !result.verified
? 'Input was dispatched. Inspect the returned observation or the app to verify its effect.'
: 'Computer observation completed',
data: computerToolResultForModel(result),
data: computerToolResultForModel(result, {
baseSnapshotId: 'snapshotId' in parsed.data ? parsed.data.snapshotId : undefined,
stableElementIds: getDesktopBridge()?.computerUse?.stableElementIds === true,
}),
}
} else
execution.completion = {
Expand Down
148 changes: 148 additions & 0 deletions apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -299,3 +299,151 @@ describe('native computer model projection', () => {
expect(computerToolResultForModel(result)).toEqual(result)
})
})

describe('change-only observations after an action', () => {
const buttons = (snapshotId: string, labels: string[], windowId = '1'): ComputerUseSnapshot => ({
kind: 'state',
bundleId: 'com.example.Fixture',
snapshotId,
windowId,
windows: [{ windowId, title: 'Fixture', x: 0, y: 0, width: 400, height: 300 }],
nodes: labels.map((label, index) => ({
elementId: `e${index}`,
role: 'AXButton',
label,
actions: ['AXPress'],
})),
truncated: false,
})
const click = (observation: ComputerUseSnapshot): ComputerUseResult => ({
kind: 'action',
action: 'click',
bundleId: observation.bundleId,
dispatched: true,
verified: false,
observation,
})
const tree = (result: unknown) =>
(result as { observation: { accessibilityTree: string } }).observation.accessibilityTree
const field = (result: unknown, name: string) =>
(result as { observation: Record<string, unknown> }).observation[name]
const labels = ['One', 'Two', 'Three', 'Four', 'Five', 'Six']
const stable = { stableElementIds: true }

it('sends whole trees when the desktop cannot keep ids stable', () => {
computerToolResultForModel(buttons('a1', labels))
const result = computerToolResultForModel(
click(buttons('a2', [...labels.slice(0, 5), 'Changed'])),
{
baseSnapshotId: 'a1',
}
)
expect(field(result, 'changedSince')).toBeUndefined()
expect(tree(result).split('\n')).toHaveLength(6)
})

it('sends a whole tree when the snapshot the action used is unknown', () => {
const result = computerToolResultForModel(click(buttons('b2', labels)), {
...stable,
baseSnapshotId: 'never-seen',
})
expect(field(result, 'changedSince')).toBeUndefined()
expect(tree(result).split('\n')).toHaveLength(6)
})

it('lists only changed and new rows and the ids that disappeared', () => {
computerToolResultForModel(buttons('c1', labels))
const next = buttons('c2', ['One', 'Two', 'Three', 'Four', 'Renamed'])
next.nodes.push({ elementId: 'e9', role: 'AXButton', label: 'Added', actions: ['AXPress'] })
const result = computerToolResultForModel(click(next), { ...stable, baseSnapshotId: 'c1' })
expect(field(result, 'changedSince')).toBe('c1')
expect(tree(result)).toContain('label="Renamed"')
expect(tree(result)).toContain('e9 AXButton label="Added"')
expect(tree(result)).not.toContain('label="One"')
expect(field(result, 'removedElementIds')).toEqual(['e5'])
})

it('folds earlier change lists into what the next action is compared against', () => {
computerToolResultForModel(buttons('d1', labels))
const renamed = [...labels.slice(0, 5), 'Renamed']
computerToolResultForModel(click(buttons('d2', renamed)), { ...stable, baseSnapshotId: 'd1' })
const result = computerToolResultForModel(click(buttons('d3', ['Uno', ...renamed.slice(1)])), {
...stable,
baseSnapshotId: 'd2',
})
expect(field(result, 'changedSince')).toBe('d2')
expect(tree(result)).toContain('label="Uno"')
expect(tree(result)).not.toContain('label="Renamed"')
})

it('sends a whole tree when an action changed nothing visible', () => {
computerToolResultForModel(buttons('k1', labels))
const result = computerToolResultForModel(click(buttons('k2', labels)), {
...stable,
baseSnapshotId: 'k1',
})
expect(field(result, 'changedSince')).toBeUndefined()
expect(tree(result).split('\n')).toHaveLength(6)
})

it('sends a whole tree when a window, sheet or popover opens', () => {
computerToolResultForModel(buttons('m1', labels))
const next = buttons('m2', [...labels.slice(0, 5), 'Renamed'])
next.windows.push({ windowId: '9', title: 'Sheet', x: 0, y: 0, width: 200, height: 100 })
const result = computerToolResultForModel(click(next), { ...stable, baseSnapshotId: 'm1' })
expect(field(result, 'changedSince')).toBeUndefined()
})

it('reports a shown row that folded into a layout wrapper as no longer shown', () => {
const pane = (enabled: boolean, snapshotId: string): ComputerUseSnapshot => {
const state = buttons(snapshotId, [...labels.slice(0, 5), enabled ? 'Ready' : 'Busy'])
state.nodes.push({ elementId: 'e8', role: 'AXGroup', actions: [], enabled })
return state
}
computerToolResultForModel(pane(false, 'n1'))
const result = computerToolResultForModel(click(pane(true, 'n2')), {
...stable,
baseSnapshotId: 'n1',
})
expect(field(result, 'changedSince')).toBe('n1')
expect(field(result, 'removedElementIds')).toEqual(['e8'])
})

it('sends a whole tree when the native read was cut short', () => {
computerToolResultForModel(buttons('p1', labels))
const partial = { ...buttons('p2', labels.slice(0, 4)), truncated: true }
const result = computerToolResultForModel(click(partial), { ...stable, baseSnapshotId: 'p1' })
expect(field(result, 'changedSince')).toBeUndefined()
expect(field(result, 'removedElementIds')).toBeUndefined()
})

it('returns to a whole tree after a run of change lists', () => {
computerToolResultForModel(buttons('f0', labels))
let result: unknown
for (let step = 1; step <= 5; step++)
result = computerToolResultForModel(
click(buttons(`f${step}`, [...labels.slice(0, 5), `Step ${step}`])),
{
...stable,
baseSnapshotId: `f${step - 1}`,
}
)
expect(field(result, 'changedSince')).toBeUndefined()
expect(tree(result).split('\n')).toHaveLength(6)
})

it('sends a whole tree for another window or when most of the tree changed', () => {
computerToolResultForModel(buttons('g1', labels))
const otherWindow = computerToolResultForModel(click(buttons('g2', labels, '2')), {
...stable,
baseSnapshotId: 'g1',
})
expect(field(otherWindow, 'changedSince')).toBeUndefined()
computerToolResultForModel(buttons('h1', labels))
const rewritten = computerToolResultForModel(
click(buttons('h2', ['A', 'B', 'C', 'D', 'Five', 'Six'])),
{ ...stable, baseSnapshotId: 'h1' }
)
expect(field(rewritten, 'changedSince')).toBeUndefined()
})
})
148 changes: 134 additions & 14 deletions apps/sim/lib/mothership/tools/client/computer-tool-result.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import type { ComputerUseResult, ComputerUseSnapshot } from '@sim/desktop-bridge/computer-use'
import { truncateAtCodePoint } from '@sim/utils/string'
import { LRUCache } from 'lru-cache'

// Keep encoded pixels below the 10 MiB completion body cap, with room for state and its envelope.
const MAX_SCREENSHOT_BASE64_BYTES = 9 * 1024 * 1024
Expand All @@ -8,18 +9,56 @@ const MAX_SCREENSHOT_BASE64_BYTES = 9 * 1024 * 1024
const MAX_STATE_BYTES = 22 * 1024
const encoder = new TextEncoder()

/** Change-only observations in a row before a full tree again, so a condensed context recovers. */
const MAX_CHAINED_CHANGES = 4

/** What the model has been shown of a snapshot: its rows, folding in any change lists before it. */
interface KnownTree {
snapshotId: string
bundleId: string
windowId: string
/** The app's windows when shown, so a window, sheet or popover that opens or closes gets a whole tree. */
windowIds: string[]
rows: Map<string, string>
changesSinceFull: number
}

const knownTrees = new LRUCache<string, KnownTree>({ max: 64 })

export interface ComputerToolResultContext {
/** The snapshot the action consumed; its observation then lists only rows that changed since it. */
baseSnapshotId?: string
/** Whether the desktop keeps an element's id across observations; positional ids get whole trees. */
stableElementIds?: boolean
}

/** Screenshots are image observations; a bounded readable tree keeps actionable references inline. */
export function computerToolResultForModel(result: ComputerUseResult) {
if (result.kind === 'state') return snapshotForModel(result)
export function computerToolResultForModel(
result: ComputerUseResult,
context: ComputerToolResultContext = {}
) {
if (result.kind === 'state') return remember(snapshotForModel(result))
if (result.kind !== 'action' || !result.observation) return result
const { observation: raw, ...action } = result
const { observations, ...observation } = snapshotForModel(
raw,
jsonBytes({ ...action, observation: null })
const base =
context.stableElementIds && context.baseSnapshotId
? knownTrees.get(context.baseSnapshotId)
: undefined
const { observations, ...observation } = remember(
snapshotForModel(raw, jsonBytes({ ...action, observation: null }), base)
)
return { ...action, observation, ...(observations ? { observations } : {}) }
}

/** Records what the model now knows of this snapshot and returns only what it sees. */
function remember<T extends { snapshotId: string }>({
known,
...projected
}: T & { known: KnownTree }) {
knownTrees.set(projected.snapshotId, known)
return projected
}

function jsonBytes(value: unknown): number {
return encoder.encode(JSON.stringify(value)).length
}
Expand Down Expand Up @@ -77,7 +116,7 @@ function nodeRow(
return fields.join(' ')
}

function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0, base?: KnownTree) {
const { screenshot, nodes, windows: allWindows, ...state } = result
const image =
screenshot && screenshot.base64.length <= MAX_SCREENSHOT_BASE64_BYTES ? screenshot : undefined
Expand Down Expand Up @@ -139,6 +178,50 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
nodeRow(node, shownParent.get(node.elementId), state.windowId),
])
)
const clipped =
nodes.some(
(node) =>
(node.label?.length ?? 0) > 512 ||
(node.value?.length ?? 0) > 512 ||
(node.placeholder?.length ?? 0) > 256 ||
node.actions.length > 16
) || allWindows.some((window) => window.title.length > 256)
const changes = base && changesSince(base, result, shown, rows)
if (changes) {
const changed = {
...model,
treeFormat: `${model.treeFormat} Only rows added or changed since snapshot ${base.snapshotId} are listed: every other row of that snapshot is unchanged and keeps its elementId, and removedElementIds are no longer shown. Use get_app_state for the whole tree.`,
changedSince: base.snapshotId,
accessibilityTree: changes.ids.map((id) => rows.get(id)).join('\n'),
removedElementIds: changes.removed,
omittedNodeCount: 0,
}
if (jsonBytes(changed) + envelopeBytes <= MAX_STATE_BYTES) {
const known = new Map(base.rows)
for (const id of changes.removed) known.delete(id)
for (const id of changes.ids) known.set(id, rows.get(id) ?? '')
return {
...changed,
truncated:
state.truncated || windows.length < allWindows.length || clipped || screenshotOmitted,
...(image
? {
observations: [
{ name: 'Computer screenshot', mediaType: image.mimeType, data: image.base64 },
],
}
: {}),
known: {
snapshotId: state.snapshotId,
bundleId: state.bundleId,
windowId: state.windowId,
windowIds: allWindows.map((window) => window.windowId),
rows: known,
changesSinceFull: base.changesSinceFull + 1,
},
}
}
}
const priority = (node: SnapshotNode) => {
const visible =
selectedWindow &&
Expand Down Expand Up @@ -197,14 +280,6 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
/** Keep the editor itself even if an unusually deep ancestry cannot fit. */
if (!add(ancestry.reverse())) add([node.elementId])
}
const clipped =
nodes.some(
(node) =>
(node.label?.length ?? 0) > 512 ||
(node.value?.length ?? 0) > 512 ||
(node.placeholder?.length ?? 0) > 256 ||
node.actions.length > 16
) || allWindows.some((window) => window.title.length > 256)
return {
...model,
accessibilityTree: shown
Expand All @@ -225,5 +300,50 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) {
],
}
: {}),
known: {
snapshotId: state.snapshotId,
bundleId: state.bundleId,
windowId: state.windowId,
windowIds: allWindows.map((window) => window.windowId),
/** Exactly the rows sent, so a parent link left out here is sent once that parent shows. */
rows: new Map(
shown
.filter((node) => selected.has(node.elementId))
.map((node) => [
node.elementId,
nodeRow(node, shownParent.get(node.elementId), state.windowId, selected),
])
),
changesSinceFull: 0,
},
}
}

/**
* The rows that differ from what the model was shown of `base`, or undefined when a whole tree is
* clearer: another app or window, a window, sheet or popover opening or closing, a partial native
* read (an element it missed may still exist), a long chain of change lists, most of the tree
* changing, or nothing visible changing at all (an empty list reads as a failed action, while the
* whole tree shows where things stand).
*/
function changesSince(
base: KnownTree,
result: ComputerUseSnapshot,
shown: SnapshotNode[],
rows: Map<string, string>
): { ids: string[]; removed: string[] } | undefined {
if (
base.bundleId !== result.bundleId ||
base.windowId !== result.windowId ||
result.truncated ||
base.changesSinceFull >= MAX_CHAINED_CHANGES ||
base.windowIds.join() !== result.windows.map((window) => window.windowId).join()
)
return undefined
const ids = shown.map((node) => node.elementId).filter((id) => rows.get(id) !== base.rows.get(id))
if (ids.length * 2 > shown.length) return undefined
/** Gone or folded into a layout wrapper: either way the model should stop relying on its row. */
const removed = [...base.rows.keys()].filter((id) => !rows.has(id))
if (ids.length === 0 && removed.length === 0) return undefined
return { ids, removed }
}
5 changes: 5 additions & 0 deletions packages/desktop-bridge/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,11 @@ export class ComputerUseError extends Error {

/** Optional so a new web deployment remains compatible with older desktop shells. */
interface SimDesktopComputerUseApi {
/**
* An element keeps its id across observations of the same app process, so an action's observation
* can list only what changed. Absent in older shells, whose ids are positional.
*/
stableElementIds?: true
/** Current activity without starting the helper; absent in older shells. */
getActivity?(): Promise<ComputerUseActivity | null>
getStatus(): Promise<ComputerUseStatus>
Expand Down
Loading