From 158474f70a9eb9cbcbd9499b6f0c8c8f8f19b2b1 Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Tue, 14 Jul 2026 02:43:49 +0800 Subject: [PATCH 1/2] fix(cu): harden merged executor boundaries --- docs/computer-use-executor-hardening.md | 56 +++++ .../src/__tests__/cua-driver-backend.test.ts | 101 +++++++-- .../computer-use/src/cua-driver-backend.ts | 195 +++++++++++++----- 3 files changed, 282 insertions(+), 70 deletions(-) create mode 100644 docs/computer-use-executor-hardening.md diff --git a/docs/computer-use-executor-hardening.md b/docs/computer-use-executor-hardening.md new file mode 100644 index 0000000000..ec0eb8e419 --- /dev/null +++ b/docs/computer-use-executor-hardening.md @@ -0,0 +1,56 @@ +# Computer Use Executor Hardening + +This note records the post-merge review of PR #893 against the current +`cua-driver` executor and the local Codex Computer Use reverse-engineering +evidence. + +## Fixed In This Follow-Up + +- Semantic refetch now requires one unique role/label/value candidate and then + verifies that its frame, depth, and value still match the observed control. + A same-label replacement or ambiguous candidate set fails closed. +- Native content fingerprints include label and value, so a control changing + meaning in the same structural slot invalidates coordinate actions. +- Window screenshots use the same compression threshold and 8 MiB cap as + desktop screenshots. +- Unconsumed observations are bounded to 16 per session and evicted FIFO. +- Keyboard ownership is invalidated when the bound PID or window no longer + matches the click-established target. +- Delivered but unverifiable Electron pointer actions and delivered text writes + preserve `outcome_unknown` instead of becoming retryable `capture_failed` + results. +- `select_text` and `secondary_action` fail closed because the pinned driver + registry does not expose their claimed tools. + +## Deliberately Not Changed + +- Coordinate click, scroll, drag, and key dispatch remain disabled by default. + Re-enabling the compatibility CGEvent path would restore the physical-input + interference found during real-machine testing. +- The physical-input callback remains optional at this executor layer because + semantic AX/CDP operations are also used by non-Desktop hosts. Desktop wiring + must supply the guard before advertising concurrent-user safety. +- The process-wide operation queue remains global. Maka currently owns one + action-child stdio connection, and the fresh-snapshot/action pair must remain + atomic across that shared connection. The Codex native service supports + concurrency through separate connections while serializing one connection + and one application instance. Removing the queue without introducing + separate service connections broke the existing ordering contract. + +## Remaining Work + +- If cross-session concurrency becomes necessary, create isolated driver + connections or per-target service instances and preserve snapshot/action + transactions explicitly. +- Initial `observeApp` failures cannot return the full typed capture result + through the current observation-only interface. The backend still enforces + the screenshot cap, but the Runtime interface needs a result-bearing + observation contract to preserve `sensitivity_blocked` end to end. +- Continue real-provider model-loop testing with coordinate actions fail closed + until an isolated native event executor exists. + +## Verification + +The focused `@maka/computer-use` suite passes 111 tests, including semantic +replacement, registry mismatch, observation eviction, window compression, +shared-client ordering, lifecycle error, and keyboard-target regression cases. diff --git a/packages/computer-use/src/__tests__/cua-driver-backend.test.ts b/packages/computer-use/src/__tests__/cua-driver-backend.test.ts index a4c107f1ae..af17f472e0 100644 --- a/packages/computer-use/src/__tests__/cua-driver-backend.test.ts +++ b/packages/computer-use/src/__tests__/cua-driver-backend.test.ts @@ -208,6 +208,13 @@ function handle(msg) { }; const refetchedElements = WINDOW_STATE_CALLS === 2 && REFETCH_MODE === 'replacement' ? [{ ...baseElement, element_index: 9, element_token: 'snapshot:9' }] + : WINDOW_STATE_CALLS === 2 && REFETCH_MODE === 'moved' + ? [{ + ...baseElement, + element_index: 9, + element_token: 'snapshot:9', + frame: { ...snapshotFrame, x: snapshotFrame.x + 40 }, + }] : WINDOW_STATE_CALLS === 2 && REFETCH_MODE === 'missing' ? [] : WINDOW_STATE_CALLS === 2 && REFETCH_MODE === 'ambiguous' @@ -222,7 +229,11 @@ function handle(msg) { ] : [baseElement]; setTimeout(() => reply(id, { - content: [{ type: 'image', data: PNG, mimeType: 'image/png' }], + content: [{ + type: 'image', + data: BIG_IMG || PNG, + mimeType: 'image/png', + }], structuredContent: { screenshot_width: 1200, screenshot_height: 800, @@ -446,7 +457,7 @@ function makeBackend(opts: { resolvePageDocumentFingerprint?: CuaDriverBackendOptions['resolvePageDocumentFingerprint']; resolveContentFingerprint?: CuaDriverBackendOptions['resolveContentFingerprint']; semanticPointerResult?: Record; - refetchMode?: 'replacement' | 'missing' | 'ambiguous'; + refetchMode?: 'replacement' | 'moved' | 'missing' | 'ambiguous'; resolveDisplays?: CuaDriverBackendOptions['resolveDisplays']; physicalInputRecentlyActive?: CuaDriverBackendOptions['physicalInputRecentlyActive']; onTrace?: CuaDriverBackendOptions['onTrace']; @@ -796,6 +807,30 @@ describe('cua-driver backend', () => { assert.equal(click?.element_token, 'snapshot:9'); }); + it('rejects a same-label replacement that moved before semantic dispatch', async () => { + const { backend, logPath } = makeBackend({ + axRole: 'AXButton', + axLabel: 'Continue', + refetchMode: 'moved', + }); + const signal = new AbortController().signal; + const context = { sessionId: 's1', turnId: 't1', toolCallId: 'semantic' }; + const observation = await backend.observeApp!({ + app: 'Fixture Window', + includeScreenshot: true, + }, signal, context); + const result = await backend.runSemantic!({ + type: 'click_element', + observationId: observation.observationId, + elementId: '7', + elementIdentity: observation.elements[0]!.identity, + }, signal, { ...context, boundAction: boundElementAction(observation, '7') }); + + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) assert.equal(result.outcome.error, 'stale_frame'); + assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0); + }); + it('refetches an unlabeled element by unique structural identity', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton', @@ -872,11 +907,10 @@ describe('cua-driver backend', () => { }]); }); - it('runs select_text, secondary action, and press_key with full fresh observations', async () => { + it('fails closed for semantic actions absent from the pinned driver registry', async () => { for (const action of [ { type: 'select_text', text: 'target' }, { type: 'secondary_action', action: 'Increment' }, - { type: 'press_key', key: 'Tab' }, ] as const) { const { backend, logPath } = makeBackend({ axRole: 'AXTextField' }); const context = { @@ -888,13 +922,7 @@ describe('cua-driver backend', () => { app: 'Fixture Window', includeScreenshot: true, }, new AbortController().signal, context); - const semanticAction: CuSemanticAction = action.type === 'press_key' - ? { - type: 'press_key', - observationId: observation.observationId, - key: action.key, - } - : action.type === 'select_text' + const semanticAction: CuSemanticAction = action.type === 'select_text' ? { type: 'select_text', observationId: observation.observationId, @@ -914,14 +942,32 @@ describe('cua-driver backend', () => { boundAction: boundElementAction(observation, '7'), }); - assert.equal(result.outcome.ok, true); - assert.ok(result.observation?.observationId); - assert.ok(result.screenshot); - const tool = action.type === 'secondary_action' - ? 'perform_secondary_action' - : action.type; - assert.equal(toolCalls(await readRecords(logPath), tool).length, 1); + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) assert.equal(result.outcome.error, 'unsupported_action'); + assert.equal(toolCalls(await readRecords(logPath), 'select_text').length, 0); + assert.equal(toolCalls(await readRecords(logPath), 'perform_secondary_action').length, 0); + } + }); + + it('evicts old unconsumed observations within a long-lived session', async () => { + const { backend } = makeBackend({ axRole: 'AXButton' }); + const signal = new AbortController().signal; + const context = { sessionId: 'long-session', turnId: 't1', toolCallId: 'observe' }; + const observations: CuObservation[] = []; + for (let index = 0; index < 17; index += 1) { + observations.push(await backend.observeApp!({ + app: 'Fixture Window', + includeScreenshot: false, + }, signal, { ...context, toolCallId: `observe-${index}` })); } + + const result = await backend.runSemantic!({ + type: 'click_element', + observationId: observations[0]!.observationId, + elementId: '7', + }, signal, { ...context, toolCallId: 'old-action' }); + assert.equal(result.outcome.ok, false); + if (!result.outcome.ok) assert.equal(result.outcome.error, 'stale_frame'); }); it('window_id disambiguates multiple visible windows from the same app', async () => { @@ -976,6 +1022,25 @@ describe('cua-driver backend', () => { assert.equal(smallRes.screenshot!.mimeType, 'image/png'); }); + it('applies the same compression policy to window observations', async () => { + let calls = 0; + const { backend } = makeBackend({ + bigImage: true, + compressFrame: () => { + calls += 1; + return { base64: 'anVzdGpwZWc=', mimeType: 'image/jpeg' }; + }, + }); + const observation = await backend.observeApp!({ + app: 'Fixture Window', + includeScreenshot: true, + }, new AbortController().signal, DEFAULT_RUN_CONTEXT); + + assert.equal(calls, 1); + assert.equal(observation.screenshot?.mimeType, 'image/jpeg'); + assert.equal(observation.screenshot?.base64, 'anVzdGpwZWc='); + }); + it('click on an app window with no AX element → same-snapshot pixel path, NEVER scope:desktop', async () => { const { backend, logPath } = makeBackend({ emptyAx: true }); const sig = new AbortController().signal; diff --git a/packages/computer-use/src/cua-driver-backend.ts b/packages/computer-use/src/cua-driver-backend.ts index 2d3181c0a8..1b3e5792b9 100644 --- a/packages/computer-use/src/cua-driver-backend.ts +++ b/packages/computer-use/src/cua-driver-backend.ts @@ -80,11 +80,22 @@ import { // crisp PNGs (simple screens) pass through untouched. const COMPRESS_FRAME_THRESHOLD = 1.5 * 1024 * 1024; const CUA_DRIVER_FRAME_MAX_BYTES = 8 * 1024 * 1024; +const MAX_OBSERVATIONS_PER_SESSION = 16; function exceedsCuaDriverFrameCap(byteLength: number): boolean { return byteLength > CUA_DRIVER_FRAME_MAX_BYTES; } +type CuaDriverCaptureFailure = CuRunResult & { + outcome: Extract; +}; + +class CuaDriverCaptureError extends Error { + constructor(readonly result: CuaDriverCaptureFailure) { + super(result.outcome.message); + } +} + export interface CuaDriverBackendOptions { /** Absolute path to the bundled `cua-driver` binary. */ binaryPath: string; @@ -303,6 +314,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc displays?: ComputerUseDisplayIdentity[]; } const observations = new Map(); + const observationIdsBySession = new Map(); let operationQueue = Promise.resolve(); let disposed = false; @@ -341,6 +353,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc const structural = [ ...new Set([...elements].map((element) => JSON.stringify({ role: element.role, + label: element.label, + value: element.value, frame: element.frame, depth: element.depth, }))), @@ -350,11 +364,75 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc .digest('hex'); } + function storeObservation( + observationId: string, + observation: StoredObservation, + ): void { + const sessionId = observation.context.sessionId; + const ids = observationIdsBySession.get(sessionId) ?? []; + while (ids.length >= MAX_OBSERVATIONS_PER_SESSION) { + observations.delete(ids.shift()!); + } + ids.push(observationId); + observationIdsBySession.set(sessionId, ids); + observations.set(observationId, observation); + } + + function deleteObservation(observationId: string): void { + const observation = observations.get(observationId); + if (!observation) return; + observations.delete(observationId); + const sessionId = observation.context.sessionId; + const ids = observationIdsBySession.get(sessionId); + if (!ids) return; + const index = ids.indexOf(observationId); + if (index >= 0) ids.splice(index, 1); + if (ids.length === 0) observationIdsBySession.delete(sessionId); + } + + function normalizeScreenshot( + image: { data?: string; mimeType?: string } | undefined, + widthPx: number, + heightPx: number, + label: string, + ): CuScreenshot | CuaDriverCaptureFailure { + if (!image?.data) { + return { + outcome: { + ok: false, + error: 'capture_failed', + message: `${label} returned no image`, + }, + }; + } + let base64 = image.data; + let mimeType: 'image/png' | 'image/jpeg' = + image.mimeType === 'image/jpeg' ? 'image/jpeg' : 'image/png'; + let byteLength = Buffer.from(base64, 'base64').byteLength; + if (opts.compressFrame && byteLength > COMPRESS_FRAME_THRESHOLD) { + const compressed = opts.compressFrame(base64, mimeType); + base64 = compressed.base64; + mimeType = compressed.mimeType; + byteLength = Buffer.from(base64, 'base64').byteLength; + } + if (exceedsCuaDriverFrameCap(byteLength)) { + return { + outcome: { + ok: false, + error: 'sensitivity_blocked', + message: `${label} ${byteLength}B exceeds cap`, + }, + }; + } + return { base64, mimeType, widthPx, heightPx }; + } + function clearLocalSession(sessionId: string): void { targetsBySession.delete(sessionId); - for (const [id, observation] of observations) { - if (observation.context.sessionId === sessionId) observations.delete(id); + for (const id of observationIdsBySession.get(sessionId) ?? []) { + observations.delete(id); } + observationIdsBySession.delete(sessionId); sessionGenerations.set( sessionId, (sessionGenerations.get(sessionId) ?? 0) + 1, @@ -852,7 +930,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc const axContentFingerprint = opts.resolveContentFingerprint ? opts.resolveContentFingerprint([...elements.values()]) : contentFingerprint(elements.values()); - observations.set(observationId, { + storeObservation(observationId, { context: { sessionId: context.sessionId, turnId: context.turnId }, appId, window, @@ -866,14 +944,18 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc const image = includeScreenshot ? state?.content?.find((content) => content.type === 'image' && typeof content.data === 'string') : undefined; - const screenshot = image?.data - ? { - base64: image.data, - mimeType: image.mimeType === 'image/png' ? 'image/png' as const : 'image/jpeg' as const, - widthPx: Number(structured.screenshot_width) || 0, - heightPx: Number(structured.screenshot_height) || 0, - } + const normalizedScreenshot = includeScreenshot + ? normalizeScreenshot( + image, + Number(structured.screenshot_width) || 0, + Number(structured.screenshot_height) || 0, + 'window frame', + ) : undefined; + if (normalizedScreenshot && 'outcome' in normalizedScreenshot) { + throw new CuaDriverCaptureError(normalizedScreenshot); + } + const screenshot = normalizedScreenshot; return { observationId, appId, @@ -919,8 +1001,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc if (!identity) return false; if (element.role !== identity.role) return false; const label = identity.label?.trim(); - if (label) return element.label === identity.label; - return !!original + return (!label || element.label === identity.label) + && !!original && element.depth === original.depth && element.frame.x === original.frame.x && element.frame.y === original.frame.y @@ -1354,7 +1436,23 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }, }; } - const matches = fresh.filter( + const identityMatches = fresh.filter((candidate) => + candidate.role === identity.role + && (!identity.label?.trim() || candidate.label === identity.label) + && ( + identity.value === undefined + || candidate.value === identity.value + )); + if (identityMatches.length > 1) { + return { + outcome: { + ok: false, + error: 'stale_frame', + message: 'semantic element identity is ambiguous in the fresh observation', + }, + }; + } + const matches = identityMatches.filter( (candidate) => elementMatchesIdentity(candidate, identity, original), ); if (matches.length === 1) return matches[0]!; @@ -1362,9 +1460,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc outcome: { ok: false, error: 'stale_frame', - message: matches.length === 0 - ? 'semantic element is missing from the fresh observation' - : 'semantic element identity is ambiguous in the fresh observation', + message: 'semantic element is missing from the fresh observation', }, }; } @@ -1372,7 +1468,14 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc function targetForContext(context: CuRunContext): KeyboardTarget | undefined { const state = targetsBySession.get(context.sessionId); if (!state) return undefined; - if (state.turnId !== context.turnId) { + const boundTarget = context.boundAction?.target; + if ( + state.turnId !== context.turnId + || (boundTarget && ( + boundTarget.pid !== state.target.window.pid + || boundTarget.windowId !== state.target.window.windowId + )) + ) { targetsBySession.delete(context.sessionId); return undefined; } @@ -1710,7 +1813,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc } : { ok: false, - error: 'capture_failed', + error: 'outcome_unknown', message: `semantic pointer action did not verify (${result.reason ?? result.kind ?? action.type})`, evidence: { path: 'cdp', effect: 'unverifiable' }, }; @@ -1780,7 +1883,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc try { return await withOperationQueue(signal, async () => { const observation = observations.get(action.observationId); - observations.delete(action.observationId); + deleteObservation(action.observationId); if (!observation) { return { outcome: { ok: false, error: 'stale_frame', message: 'observation is missing or already consumed' } }; } @@ -1817,6 +1920,15 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc ...(fresh.screenshot ? { screenshot: fresh.screenshot } : {}), }; } + if (action.type === 'select_text' || action.type === 'secondary_action') { + return { + outcome: { + ok: false, + error: 'unsupported_action', + message: `semantic action '${action.type}' is not exposed by the pinned cua-driver registry`, + }, + }; + } const refetched = await refetchSemanticElement(observation, action, signal); if ('outcome' in refetched) return refetched; const visibilityFailure = await validateSemanticElementVisibility( @@ -1837,18 +1949,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc ? await actionClient.callTool('click', args, signal) : action.type === 'set_value' ? await actionClient.callTool('set_value', { ...args, value: action.value }, signal) - : action.type === 'select_text' - ? await actionClient.callTool('select_text', { - ...args, - text: action.text, - selection_type: 'text', - }, signal) - : action.type === 'secondary_action' - ? await actionClient.callTool('perform_secondary_action', { - ...args, - action: action.action, - }, signal) - : undefined; + : undefined; if (!result) { return { outcome: { ok: false, error: 'unsupported_action', message: `semantic action '${action.type}' is not supported by cua-driver` } }; } @@ -1867,6 +1968,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }; }, context.sessionId); } catch (error) { + if (error instanceof CuaDriverCaptureError) return error.result; const failure = lifecycleFailure(error); if (failure) return failure; throw error; @@ -1906,33 +2008,19 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc case 'screenshot': { const r = await captureClient.callTool('get_desktop_state', {}, signal); const img = r?.content?.find((c) => c.type === 'image'); - if (!img?.data) return { outcome: { ok: false, error: 'capture_failed', message: 'no image returned' } }; - let base64 = img.data; - let mimeType: 'image/png' | 'image/jpeg' = img.mimeType === 'image/jpeg' ? 'image/jpeg' : 'image/png'; - let byteLength = Buffer.from(base64, 'base64').byteLength; - // Compress large frames (native res, coords unchanged) so a Retina - // full-display PNG doesn't balloon past the cap / the provider's limit. - if (opts.compressFrame && byteLength > COMPRESS_FRAME_THRESHOLD) { - const c = opts.compressFrame(base64, mimeType); - base64 = c.base64; - mimeType = c.mimeType; - byteLength = Buffer.from(base64, 'base64').byteLength; - } - if (exceedsCuaDriverFrameCap(byteLength)) { - return { outcome: { ok: false, error: 'sensitivity_blocked', message: `frame ${byteLength}B exceeds cap` } }; - } const sc = r?.structuredContent ?? {}; // Remember the device frame width so getScale() can derive the true // device/logical ratio (see getScale — scale_factor is unreliable). if (typeof sc.screenshot_width === 'number' && sc.screenshot_width > 0) { lastFrameWidthPx = sc.screenshot_width; } - const screenshot: CuScreenshot = { - base64, - mimeType, - widthPx: typeof sc.screenshot_width === 'number' ? sc.screenshot_width : 0, - heightPx: typeof sc.screenshot_height === 'number' ? sc.screenshot_height : 0, - }; + const screenshot = normalizeScreenshot( + img, + typeof sc.screenshot_width === 'number' ? sc.screenshot_width : 0, + typeof sc.screenshot_height === 'number' ? sc.screenshot_height : 0, + 'frame', + ); + if ('outcome' in screenshot) return screenshot; return { outcome: { ok: true, tier: 'coordinate-background' }, screenshot }; } case 'left_click': @@ -2413,6 +2501,8 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc try { return { outcome: await fillEditableTarget(target, action.text, signal) }; } catch (error) { + const failure = lifecycleFailure(error); + if (failure) return failure; return { outcome: { ok: false, @@ -2484,6 +2574,7 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc disposed = true; targetsBySession.clear(); observations.clear(); + observationIdsBySession.clear(); sessionGenerations.clear(); const errors: unknown[] = []; for (const client of [actionClient, captureClient]) { From dae389670126a8569d638efc96ce9b8d8359e64c Mon Sep 17 00:00:00 2001 From: hqhq1025 <1506751656@qq.com> Date: Tue, 14 Jul 2026 03:04:07 +0800 Subject: [PATCH 2/2] fix(cu): preserve semantic paths behind input fence --- .../src/__tests__/cua-driver-backend.test.ts | 79 +++++++++++++++++++ .../computer-use/src/cua-driver-backend.ts | 17 +++- 2 files changed, 92 insertions(+), 4 deletions(-) diff --git a/packages/computer-use/src/__tests__/cua-driver-backend.test.ts b/packages/computer-use/src/__tests__/cua-driver-backend.test.ts index af17f472e0..b319b0a194 100644 --- a/packages/computer-use/src/__tests__/cua-driver-backend.test.ts +++ b/packages/computer-use/src/__tests__/cua-driver-backend.test.ts @@ -1407,6 +1407,85 @@ describe('cua-driver backend', () => { assert.equal(toolCalls(records, 'drag').length, 0); }); + it('keeps Electron semantic click available while compatibility input is disabled', async () => { + const traces: CuaDriverTraceEvent[] = []; + const { backend, logPath } = makeBackend({ + allowCompatibilityInputDispatch: false, + processKind: 'electron', + pageTarget: testPageTarget(), + semanticPointerResult: { + supported: true, + ok: true, + kind: 'left_click', + editable: true, + tagName: 'input', + focusChanged: true, + }, + onTrace: (event) => traces.push(event), + }); + + const result = await backend.run( + { type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction, + new AbortController().signal, + ); + + assert.equal(result.outcome.ok, true); + if (result.outcome.ok) assert.equal(result.outcome.tier, 'semantic-background'); + const records = await readRecords(logPath); + assert.equal(businessPageCalls(records).length, 1); + assert.equal(toolCalls(records, 'click').length, 0); + assert.equal( + traces.some((event) => + event.type === 'dispatch' + && event.tool === 'page' + && event.address === 'semantic'), + true, + ); + }); + + it('traces native semantic dispatch without exposing element content', async () => { + const traces: CuaDriverTraceEvent[] = []; + const { backend } = makeBackend({ + axRole: 'AXTextField', + axLabel: 'Private field label', + onTrace: (event) => traces.push(event), + }); + const signal = new AbortController().signal; + const context = { + sessionId: 'trace-session', + turnId: 'trace-turn', + toolCallId: 'native-set-value', + }; + const observed = await backend.observeApp!({ + app: 'Fixture Window', + includeScreenshot: false, + }, signal, context); + + await backend.runSemantic!({ + type: 'set_value', + observationId: observed.observationId, + elementId: '7', + value: 'private value', + elementIdentity: observed.elements[0]!.identity, + }, signal, { + ...context, + boundAction: boundElementAction(observed, '7'), + }); + + const dispatch = traces.find((event) => + event.type === 'dispatch' && event.toolCallId === 'native-set-value'); + assert.deepEqual(dispatch, { + type: 'dispatch', + toolCallId: 'native-set-value', + actionType: 'set_value', + tool: 'set_value', + pid: 4242, + windowId: 77, + address: 'ax', + }); + assert.doesNotMatch(JSON.stringify(traces), /Private field label|private value/); + }); + it('fails closed when the physical-input guard cannot be read', async () => { const { backend, logPath } = makeBackend({ axRole: 'AXButton', diff --git a/packages/computer-use/src/cua-driver-backend.ts b/packages/computer-use/src/cua-driver-backend.ts index 1b3e5792b9..5d1a355805 100644 --- a/packages/computer-use/src/cua-driver-backend.ts +++ b/packages/computer-use/src/cua-driver-backend.ts @@ -195,7 +195,7 @@ export type CuaDriverTraceEvent = | { type: 'dispatch'; toolCallId?: string; - actionType: CuAction['type']; + actionType: CuAction['type'] | CuSemanticAction['type']; tool: string; pid?: number; windowId?: number; @@ -1945,6 +1945,15 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc }; const intervention = await physicalInputFailure(); if (intervention) return intervention; + trace({ + type: 'dispatch', + toolCallId: context.toolCallId, + actionType: action.type, + tool: action.type === 'click_element' ? 'click' : 'set_value', + pid: validated.pid, + windowId: validated.windowId, + address: 'ax', + }); const result = action.type === 'click_element' ? await actionClient.callTool('click', args, signal) : action.type === 'set_value' @@ -2028,9 +2037,6 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc case 'middle_click': case 'double_click': case 'triple_click': { - if (opts.allowCompatibilityInputDispatch !== true) { - return compatibilityInputBlocked(action.type); - } // Resolve the window under the point and click via pid+window_id, which // forces cua-driver's click_at_xy_with_window_local → CGEventPostToPid / // SLEventPostToPid — NO cursor warp (the forbidden pid-less path would @@ -2091,6 +2097,9 @@ export function createCuaDriverBackend(opts: CuaDriverBackendOptions): CuDispatc return { outcome: semantic.outcome, resolvedScreenPoint: win.screenPoint }; } } + if (opts.allowCompatibilityInputDispatch !== true) { + return compatibilityInputBlocked(action.type); + } { let snapshot: TargetSnapshot; try {