From 1478e1ca729fc6b8b969756cf45138214c197770 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Mon, 16 Nov 2026 13:44:33 -0800 Subject: [PATCH 1/3] improvement(computer-use): list only what changed in the observation after an action --- apps/desktop/src/preload/index.ts | 1 + .../tools/client/computer-tool-execution.ts | 6 +- .../tools/client/computer-tool-result.test.ts | 103 ++++++++++++++ .../tools/client/computer-tool-result.ts | 130 ++++++++++++++++-- packages/desktop-bridge/src/index.ts | 5 + 5 files changed, 230 insertions(+), 15 deletions(-) diff --git a/apps/desktop/src/preload/index.ts b/apps/desktop/src/preload/index.ts index e4a0c216116..873b880fba2 100644 --- a/apps/desktop/src/preload/index.ts +++ b/apps/desktop/src/preload/index.ts @@ -127,6 +127,7 @@ const api: SimDesktopApi = { ...(process.platform === 'darwin' ? { computerUse: { + stableElementIds: true, getActivity: () => ipcRenderer.invoke('computer-use:get-activity'), getStatus: () => ipcRenderer.invoke('computer-use:status'), setEnabled: (enabled: boolean) => ipcRenderer.invoke('computer-use:set-enabled', enabled), diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts index e4a38a1eb9f..4539b3e6f8d 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-execution.ts @@ -5,6 +5,7 @@ import { getErrorMessage } from '@sim/utils/errors' import { omit } from '@sim/utils/object' import { backoffWithJitter } from '@sim/utils/retry' import { cancelComputerUseTool, executeComputerUseTool } from '@/lib/computer-use/transport' +import { getDesktopBridge } from '@/lib/desktop' import { ASYNC_TOOL_CONFIRMATION_STATUS, type AsyncConfirmationStatus, @@ -245,7 +246,10 @@ export async function executeComputerToolOnClient( : result.kind === 'action' && !result.verified ? 'Input was dispatched. Inspect the returned observation or the app to verify its effect.' : 'Computer observation completed', - data: computerToolResultForModel(result), + data: computerToolResultForModel(result, { + baseSnapshotId: 'snapshotId' in parsed.data ? parsed.data.snapshotId : undefined, + stableElementIds: getDesktopBridge()?.computerUse?.stableElementIds === true, + }), } } else execution.completion = { diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts index 0e9fb9d563c..597586ca24c 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts @@ -299,3 +299,106 @@ describe('native computer model projection', () => { expect(computerToolResultForModel(result)).toEqual(result) }) }) + +describe('change-only observations after an action', () => { + const buttons = (snapshotId: string, labels: string[], windowId = '1'): ComputerUseSnapshot => ({ + kind: 'state', + bundleId: 'com.example.Fixture', + snapshotId, + windowId, + windows: [{ windowId, title: 'Fixture', x: 0, y: 0, width: 400, height: 300 }], + nodes: labels.map((label, index) => ({ + elementId: `e${index}`, + role: 'AXButton', + label, + actions: ['AXPress'], + })), + truncated: false, + }) + const click = (observation: ComputerUseSnapshot): ComputerUseResult => ({ + kind: 'action', + action: 'click', + bundleId: observation.bundleId, + dispatched: true, + verified: false, + observation, + }) + const tree = (result: unknown) => + (result as { observation: { accessibilityTree: string } }).observation.accessibilityTree + const field = (result: unknown, name: string) => + (result as { observation: Record }).observation[name] + const labels = ['One', 'Two', 'Three', 'Four', 'Five', 'Six'] + const stable = { stableElementIds: true } + + it('sends whole trees when the desktop cannot keep ids stable', () => { + computerToolResultForModel(buttons('a1', labels)) + const result = computerToolResultForModel( + click(buttons('a2', [...labels.slice(0, 5), 'Changed'])), + { + baseSnapshotId: 'a1', + } + ) + expect(field(result, 'changedSince')).toBeUndefined() + expect(tree(result).split('\n')).toHaveLength(6) + }) + + it('sends a whole tree when the snapshot the action used is unknown', () => { + const result = computerToolResultForModel(click(buttons('b2', labels)), { + ...stable, + baseSnapshotId: 'never-seen', + }) + expect(field(result, 'changedSince')).toBeUndefined() + expect(tree(result).split('\n')).toHaveLength(6) + }) + + it('lists only changed and new rows and the ids that disappeared', () => { + computerToolResultForModel(buttons('c1', labels)) + const next = buttons('c2', ['One', 'Two', 'Three', 'Four', 'Renamed']) + next.nodes.push({ elementId: 'e9', role: 'AXButton', label: 'Added', actions: ['AXPress'] }) + const result = computerToolResultForModel(click(next), { ...stable, baseSnapshotId: 'c1' }) + expect(field(result, 'changedSince')).toBe('c1') + expect(tree(result)).toContain('label="Renamed"') + expect(tree(result)).toContain('e9 AXButton label="Added"') + expect(tree(result)).not.toContain('label="One"') + expect(field(result, 'removedElementIds')).toEqual(['e5']) + }) + + it('folds earlier change lists into what the next action is compared against', () => { + computerToolResultForModel(buttons('d1', labels)) + const renamed = [...labels.slice(0, 5), 'Renamed'] + computerToolResultForModel(click(buttons('d2', renamed)), { ...stable, baseSnapshotId: 'd1' }) + const result = computerToolResultForModel(click(buttons('d3', renamed)), { + ...stable, + baseSnapshotId: 'd2', + }) + expect(field(result, 'changedSince')).toBe('d2') + expect(tree(result)).toBe('') + }) + + it('returns to a whole tree after a run of change lists', () => { + computerToolResultForModel(buttons('f0', labels)) + let result: unknown + for (let step = 1; step <= 5; step++) + result = computerToolResultForModel(click(buttons(`f${step}`, labels)), { + ...stable, + baseSnapshotId: `f${step - 1}`, + }) + expect(field(result, 'changedSince')).toBeUndefined() + expect(tree(result).split('\n')).toHaveLength(6) + }) + + it('sends a whole tree for another window or when most of the tree changed', () => { + computerToolResultForModel(buttons('g1', labels)) + const otherWindow = computerToolResultForModel(click(buttons('g2', labels, '2')), { + ...stable, + baseSnapshotId: 'g1', + }) + expect(field(otherWindow, 'changedSince')).toBeUndefined() + computerToolResultForModel(buttons('h1', labels)) + const rewritten = computerToolResultForModel( + click(buttons('h2', ['A', 'B', 'C', 'D', 'Five', 'Six'])), + { ...stable, baseSnapshotId: 'h1' } + ) + expect(field(rewritten, 'changedSince')).toBeUndefined() + }) +}) diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts index 72db13ab9cf..1d7a4a2b75d 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -1,5 +1,6 @@ import type { ComputerUseResult, ComputerUseSnapshot } from '@sim/desktop-bridge/computer-use' import { truncateAtCodePoint } from '@sim/utils/string' +import { LRUCache } from 'lru-cache' // Keep encoded pixels below the 10 MiB completion body cap, with room for state and its envelope. const MAX_SCREENSHOT_BASE64_BYTES = 9 * 1024 * 1024 @@ -8,18 +9,54 @@ const MAX_SCREENSHOT_BASE64_BYTES = 9 * 1024 * 1024 const MAX_STATE_BYTES = 22 * 1024 const encoder = new TextEncoder() +/** Change-only observations in a row before a full tree again, so a condensed context recovers. */ +const MAX_CHAINED_CHANGES = 4 + +/** What the model has been shown of a snapshot: its rows, folding in any change lists before it. */ +interface KnownTree { + snapshotId: string + bundleId: string + windowId: string + rows: Map + changesSinceFull: number +} + +const knownTrees = new LRUCache({ max: 64 }) + +export interface ComputerToolResultContext { + /** The snapshot the action consumed; its observation then lists only rows that changed since it. */ + baseSnapshotId?: string + /** Whether the desktop keeps an element's id across observations; positional ids get whole trees. */ + stableElementIds?: boolean +} + /** Screenshots are image observations; a bounded readable tree keeps actionable references inline. */ -export function computerToolResultForModel(result: ComputerUseResult) { - if (result.kind === 'state') return snapshotForModel(result) +export function computerToolResultForModel( + result: ComputerUseResult, + context: ComputerToolResultContext = {} +) { + if (result.kind === 'state') return remember(snapshotForModel(result)) if (result.kind !== 'action' || !result.observation) return result const { observation: raw, ...action } = result - const { observations, ...observation } = snapshotForModel( - raw, - jsonBytes({ ...action, observation: null }) + const base = + context.stableElementIds && context.baseSnapshotId + ? knownTrees.get(context.baseSnapshotId) + : undefined + const { observations, ...observation } = remember( + snapshotForModel(raw, jsonBytes({ ...action, observation: null }), base) ) return { ...action, observation, ...(observations ? { observations } : {}) } } +/** Records what the model now knows of this snapshot and returns only what it sees. */ +function remember({ + known, + ...projected +}: T & { known: KnownTree }) { + knownTrees.set(projected.snapshotId, known) + return projected +} + function jsonBytes(value: unknown): number { return encoder.encode(JSON.stringify(value)).length } @@ -77,7 +114,7 @@ function nodeRow( return fields.join(' ') } -function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { +function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0, base?: KnownTree) { const { screenshot, nodes, windows: allWindows, ...state } = result const image = screenshot && screenshot.base64.length <= MAX_SCREENSHOT_BASE64_BYTES ? screenshot : undefined @@ -139,6 +176,49 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { nodeRow(node, shownParent.get(node.elementId), state.windowId), ]) ) + const clipped = + nodes.some( + (node) => + (node.label?.length ?? 0) > 512 || + (node.value?.length ?? 0) > 512 || + (node.placeholder?.length ?? 0) > 256 || + node.actions.length > 16 + ) || allWindows.some((window) => window.title.length > 256) + const changes = base && changesSince(base, result, shown, rows) + if (changes) { + const changed = { + ...model, + treeFormat: `${model.treeFormat} Only rows added or changed since snapshot ${base.snapshotId} are listed: every other row of that snapshot is unchanged and keeps its elementId, and removedElementIds no longer exist. Use get_app_state for the whole tree.`, + changedSince: base.snapshotId, + accessibilityTree: changes.ids.map((id) => rows.get(id)).join('\n'), + removedElementIds: changes.removed, + omittedNodeCount: 0, + } + if (jsonBytes(changed) + envelopeBytes <= MAX_STATE_BYTES) { + const known = new Map(base.rows) + for (const id of changes.removed) known.delete(id) + for (const id of changes.ids) known.set(id, rows.get(id) ?? '') + return { + ...changed, + truncated: + state.truncated || windows.length < allWindows.length || clipped || screenshotOmitted, + ...(image + ? { + observations: [ + { name: 'Computer screenshot', mediaType: image.mimeType, data: image.base64 }, + ], + } + : {}), + known: { + snapshotId: state.snapshotId, + bundleId: state.bundleId, + windowId: state.windowId, + rows: known, + changesSinceFull: base.changesSinceFull + 1, + }, + } + } + } const priority = (node: SnapshotNode) => { const visible = selectedWindow && @@ -197,14 +277,6 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { /** Keep the editor itself even if an unusually deep ancestry cannot fit. */ if (!add(ancestry.reverse())) add([node.elementId]) } - const clipped = - nodes.some( - (node) => - (node.label?.length ?? 0) > 512 || - (node.value?.length ?? 0) > 512 || - (node.placeholder?.length ?? 0) > 256 || - node.actions.length > 16 - ) || allWindows.some((window) => window.title.length > 256) return { ...model, accessibilityTree: shown @@ -225,5 +297,35 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0) { ], } : {}), + known: { + snapshotId: state.snapshotId, + bundleId: state.bundleId, + windowId: state.windowId, + rows: new Map([...selected].map((id) => [id, rows.get(id) ?? ''])), + changesSinceFull: 0, + }, } } + +/** + * The rows that differ from what the model was shown of `base`, or undefined when a whole tree is + * clearer: another app or window, a long chain of change lists, or most of the tree changing. + */ +function changesSince( + base: KnownTree, + result: ComputerUseSnapshot, + shown: SnapshotNode[], + rows: Map +): { ids: string[]; removed: string[] } | undefined { + if ( + base.bundleId !== result.bundleId || + base.windowId !== result.windowId || + base.changesSinceFull >= MAX_CHAINED_CHANGES + ) + return undefined + const ids = shown.map((node) => node.elementId).filter((id) => rows.get(id) !== base.rows.get(id)) + if (ids.length * 2 > shown.length) return undefined + const present = new Set(result.nodes.map((node) => node.elementId)) + const removed = [...base.rows.keys()].filter((id) => !present.has(id)) + return { ids, removed } +} diff --git a/packages/desktop-bridge/src/index.ts b/packages/desktop-bridge/src/index.ts index 6b715aaa083..4ec4b1bf27c 100644 --- a/packages/desktop-bridge/src/index.ts +++ b/packages/desktop-bridge/src/index.ts @@ -85,6 +85,11 @@ export class ComputerUseError extends Error { /** Optional so a new web deployment remains compatible with older desktop shells. */ interface SimDesktopComputerUseApi { + /** + * An element keeps its id across observations of the same app process, so an action's observation + * can list only what changed. Absent in older shells, whose ids are positional. + */ + stableElementIds?: true /** Current activity without starting the helper; absent in older shells. */ getActivity?(): Promise getStatus(): Promise From dba0779a04d5f2ee708ed147a5599dbdb49eb590 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 17 Nov 2026 00:26:31 -0800 Subject: [PATCH 2/3] send a whole tree when nothing visible changed or a window opened or closed --- .../tools/client/computer-tool-result.test.ts | 34 +++++++++++++++---- .../tools/client/computer-tool-result.ts | 12 +++++-- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts index 597586ca24c..5e6ff4b1700 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts @@ -367,22 +367,44 @@ describe('change-only observations after an action', () => { computerToolResultForModel(buttons('d1', labels)) const renamed = [...labels.slice(0, 5), 'Renamed'] computerToolResultForModel(click(buttons('d2', renamed)), { ...stable, baseSnapshotId: 'd1' }) - const result = computerToolResultForModel(click(buttons('d3', renamed)), { + const result = computerToolResultForModel(click(buttons('d3', ['Uno', ...renamed.slice(1)])), { ...stable, baseSnapshotId: 'd2', }) expect(field(result, 'changedSince')).toBe('d2') - expect(tree(result)).toBe('') + expect(tree(result)).toContain('label="Uno"') + expect(tree(result)).not.toContain('label="Renamed"') + }) + + it('sends a whole tree when an action changed nothing visible', () => { + computerToolResultForModel(buttons('k1', labels)) + const result = computerToolResultForModel(click(buttons('k2', labels)), { + ...stable, + baseSnapshotId: 'k1', + }) + expect(field(result, 'changedSince')).toBeUndefined() + expect(tree(result).split('\n')).toHaveLength(6) + }) + + it('sends a whole tree when a window, sheet or popover opens', () => { + computerToolResultForModel(buttons('m1', labels)) + const next = buttons('m2', [...labels.slice(0, 5), 'Renamed']) + next.windows.push({ windowId: '9', title: 'Sheet', x: 0, y: 0, width: 200, height: 100 }) + const result = computerToolResultForModel(click(next), { ...stable, baseSnapshotId: 'm1' }) + expect(field(result, 'changedSince')).toBeUndefined() }) it('returns to a whole tree after a run of change lists', () => { computerToolResultForModel(buttons('f0', labels)) let result: unknown for (let step = 1; step <= 5; step++) - result = computerToolResultForModel(click(buttons(`f${step}`, labels)), { - ...stable, - baseSnapshotId: `f${step - 1}`, - }) + result = computerToolResultForModel( + click(buttons(`f${step}`, [...labels.slice(0, 5), `Step ${step}`])), + { + ...stable, + baseSnapshotId: `f${step - 1}`, + } + ) expect(field(result, 'changedSince')).toBeUndefined() expect(tree(result).split('\n')).toHaveLength(6) }) diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts index 1d7a4a2b75d..95df9001ef5 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -17,6 +17,8 @@ interface KnownTree { snapshotId: string bundleId: string windowId: string + /** The app's windows when shown, so a window, sheet or popover that opens or closes gets a whole tree. */ + windowIds: string[] rows: Map changesSinceFull: number } @@ -213,6 +215,7 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0, base?: snapshotId: state.snapshotId, bundleId: state.bundleId, windowId: state.windowId, + windowIds: allWindows.map((window) => window.windowId), rows: known, changesSinceFull: base.changesSinceFull + 1, }, @@ -301,6 +304,7 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0, base?: snapshotId: state.snapshotId, bundleId: state.bundleId, windowId: state.windowId, + windowIds: allWindows.map((window) => window.windowId), rows: new Map([...selected].map((id) => [id, rows.get(id) ?? ''])), changesSinceFull: 0, }, @@ -309,7 +313,9 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0, base?: /** * The rows that differ from what the model was shown of `base`, or undefined when a whole tree is - * clearer: another app or window, a long chain of change lists, or most of the tree changing. + * clearer: another app or window, a window, sheet or popover opening or closing, a long chain of + * change lists, most of the tree changing, or nothing visible changing at all (an empty list reads + * as a failed action, while the whole tree shows where things stand). */ function changesSince( base: KnownTree, @@ -320,12 +326,14 @@ function changesSince( if ( base.bundleId !== result.bundleId || base.windowId !== result.windowId || - base.changesSinceFull >= MAX_CHAINED_CHANGES + base.changesSinceFull >= MAX_CHAINED_CHANGES || + base.windowIds.join() !== result.windows.map((window) => window.windowId).join() ) return undefined const ids = shown.map((node) => node.elementId).filter((id) => rows.get(id) !== base.rows.get(id)) if (ids.length * 2 > shown.length) return undefined const present = new Set(result.nodes.map((node) => node.elementId)) const removed = [...base.rows.keys()].filter((id) => !present.has(id)) + if (ids.length === 0 && removed.length === 0) return undefined return { ids, removed } } From e40542bba5d733eead52a08e5d7545205b829dd1 Mon Sep 17 00:00:00 2001 From: Waleed Latif Date: Tue, 17 Nov 2026 05:52:56 -0800 Subject: [PATCH 3/3] report folded rows, send whole trees for partial reads, and cache the rows exactly as sent --- .../tools/client/computer-tool-result.test.ts | 23 ++++++++++++++++++ .../tools/client/computer-tool-result.ts | 24 +++++++++++++------ 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts index 5e6ff4b1700..a12d8287896 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.test.ts @@ -394,6 +394,29 @@ describe('change-only observations after an action', () => { expect(field(result, 'changedSince')).toBeUndefined() }) + it('reports a shown row that folded into a layout wrapper as no longer shown', () => { + const pane = (enabled: boolean, snapshotId: string): ComputerUseSnapshot => { + const state = buttons(snapshotId, [...labels.slice(0, 5), enabled ? 'Ready' : 'Busy']) + state.nodes.push({ elementId: 'e8', role: 'AXGroup', actions: [], enabled }) + return state + } + computerToolResultForModel(pane(false, 'n1')) + const result = computerToolResultForModel(click(pane(true, 'n2')), { + ...stable, + baseSnapshotId: 'n1', + }) + expect(field(result, 'changedSince')).toBe('n1') + expect(field(result, 'removedElementIds')).toEqual(['e8']) + }) + + it('sends a whole tree when the native read was cut short', () => { + computerToolResultForModel(buttons('p1', labels)) + const partial = { ...buttons('p2', labels.slice(0, 4)), truncated: true } + const result = computerToolResultForModel(click(partial), { ...stable, baseSnapshotId: 'p1' }) + expect(field(result, 'changedSince')).toBeUndefined() + expect(field(result, 'removedElementIds')).toBeUndefined() + }) + it('returns to a whole tree after a run of change lists', () => { computerToolResultForModel(buttons('f0', labels)) let result: unknown diff --git a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts index 95df9001ef5..2be5ccd5657 100644 --- a/apps/sim/lib/mothership/tools/client/computer-tool-result.ts +++ b/apps/sim/lib/mothership/tools/client/computer-tool-result.ts @@ -190,7 +190,7 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0, base?: if (changes) { const changed = { ...model, - treeFormat: `${model.treeFormat} Only rows added or changed since snapshot ${base.snapshotId} are listed: every other row of that snapshot is unchanged and keeps its elementId, and removedElementIds no longer exist. Use get_app_state for the whole tree.`, + treeFormat: `${model.treeFormat} Only rows added or changed since snapshot ${base.snapshotId} are listed: every other row of that snapshot is unchanged and keeps its elementId, and removedElementIds are no longer shown. Use get_app_state for the whole tree.`, changedSince: base.snapshotId, accessibilityTree: changes.ids.map((id) => rows.get(id)).join('\n'), removedElementIds: changes.removed, @@ -305,7 +305,15 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0, base?: bundleId: state.bundleId, windowId: state.windowId, windowIds: allWindows.map((window) => window.windowId), - rows: new Map([...selected].map((id) => [id, rows.get(id) ?? ''])), + /** Exactly the rows sent, so a parent link left out here is sent once that parent shows. */ + rows: new Map( + shown + .filter((node) => selected.has(node.elementId)) + .map((node) => [ + node.elementId, + nodeRow(node, shownParent.get(node.elementId), state.windowId, selected), + ]) + ), changesSinceFull: 0, }, } @@ -313,9 +321,10 @@ function snapshotForModel(result: ComputerUseSnapshot, envelopeBytes = 0, base?: /** * The rows that differ from what the model was shown of `base`, or undefined when a whole tree is - * clearer: another app or window, a window, sheet or popover opening or closing, a long chain of - * change lists, most of the tree changing, or nothing visible changing at all (an empty list reads - * as a failed action, while the whole tree shows where things stand). + * clearer: another app or window, a window, sheet or popover opening or closing, a partial native + * read (an element it missed may still exist), a long chain of change lists, most of the tree + * changing, or nothing visible changing at all (an empty list reads as a failed action, while the + * whole tree shows where things stand). */ function changesSince( base: KnownTree, @@ -326,14 +335,15 @@ function changesSince( if ( base.bundleId !== result.bundleId || base.windowId !== result.windowId || + result.truncated || base.changesSinceFull >= MAX_CHAINED_CHANGES || base.windowIds.join() !== result.windows.map((window) => window.windowId).join() ) return undefined const ids = shown.map((node) => node.elementId).filter((id) => rows.get(id) !== base.rows.get(id)) if (ids.length * 2 > shown.length) return undefined - const present = new Set(result.nodes.map((node) => node.elementId)) - const removed = [...base.rows.keys()].filter((id) => !present.has(id)) + /** Gone or folded into a layout wrapper: either way the model should stop relying on its row. */ + const removed = [...base.rows.keys()].filter((id) => !rows.has(id)) if (ids.length === 0 && removed.length === 0) return undefined return { ids, removed } }