fix(cu): harden merged executor boundaries

docs/computer-use-executor-hardening.md

@@ -0,0 +1,56 @@
+# Computer Use Executor Hardening
+
+This note records the post-merge review of PR #893 against the current
+`cua-driver` executor and the local Codex Computer Use reverse-engineering
+evidence.
+
+## Fixed In This Follow-Up
+
+- Semantic refetch now requires one unique role/label/value candidate and then
+  verifies that its frame, depth, and value still match the observed control.
+  A same-label replacement or ambiguous candidate set fails closed.
+- Native content fingerprints include label and value, so a control changing
+  meaning in the same structural slot invalidates coordinate actions.
+- Window screenshots use the same compression threshold and 8 MiB cap as
+  desktop screenshots.
+- Unconsumed observations are bounded to 16 per session and evicted FIFO.
+- Keyboard ownership is invalidated when the bound PID or window no longer
+  matches the click-established target.
+- Delivered but unverifiable Electron pointer actions and delivered text writes
+  preserve `outcome_unknown` instead of becoming retryable `capture_failed`
+  results.
+- `select_text` and `secondary_action` fail closed because the pinned driver
+  registry does not expose their claimed tools.
+
+## Deliberately Not Changed
+
+- Coordinate click, scroll, drag, and key dispatch remain disabled by default.
+  Re-enabling the compatibility CGEvent path would restore the physical-input
+  interference found during real-machine testing.
+- The physical-input callback remains optional at this executor layer because
+  semantic AX/CDP operations are also used by non-Desktop hosts. Desktop wiring
+  must supply the guard before advertising concurrent-user safety.
+- The process-wide operation queue remains global. Maka currently owns one
+  action-child stdio connection, and the fresh-snapshot/action pair must remain
+  atomic across that shared connection. The Codex native service supports
+  concurrency through separate connections while serializing one connection
+  and one application instance. Removing the queue without introducing
+  separate service connections broke the existing ordering contract.
+
+## Remaining Work
+
+- If cross-session concurrency becomes necessary, create isolated driver
+  connections or per-target service instances and preserve snapshot/action
+  transactions explicitly.
+- Initial `observeApp` failures cannot return the full typed capture result
+  through the current observation-only interface. The backend still enforces
+  the screenshot cap, but the Runtime interface needs a result-bearing
+  observation contract to preserve `sensitivity_blocked` end to end.
+- Continue real-provider model-loop testing with coordinate actions fail closed
+  until an isolated native event executor exists.
+
+## Verification
+
+The focused `@maka/computer-use` suite passes 111 tests, including semantic
+replacement, registry mismatch, observation eviction, window compression,
+shared-client ordering, lifecycle error, and keyboard-target regression cases.

packages/computer-use/src/__tests__/cua-driver-backend.test.ts

@@ -208,6 +208,13 @@ function handle(msg) {
         };
         const refetchedElements = WINDOW_STATE_CALLS === 2 && REFETCH_MODE === 'replacement'
           ? [{ ...baseElement, element_index: 9, element_token: 'snapshot:9' }]
+          : WINDOW_STATE_CALLS === 2 && REFETCH_MODE === 'moved'
+            ? [{
+                ...baseElement,
+                element_index: 9,
+                element_token: 'snapshot:9',
+                frame: { ...snapshotFrame, x: snapshotFrame.x + 40 },
+              }]
           : WINDOW_STATE_CALLS === 2 && REFETCH_MODE === 'missing'
             ? []
             : WINDOW_STATE_CALLS === 2 && REFETCH_MODE === 'ambiguous'
@@ -222,7 +229,11 @@ function handle(msg) {
                 ]
               : [baseElement];
         setTimeout(() => reply(id, {
-            content: [{ type: 'image', data: PNG, mimeType: 'image/png' }],
+            content: [{
+              type: 'image',
+              data: BIG_IMG || PNG,
+              mimeType: 'image/png',
+            }],
             structuredContent: {
               screenshot_width: 1200,
               screenshot_height: 800,
@@ -446,7 +457,7 @@ function makeBackend(opts: {
   resolvePageDocumentFingerprint?: CuaDriverBackendOptions['resolvePageDocumentFingerprint'];
   resolveContentFingerprint?: CuaDriverBackendOptions['resolveContentFingerprint'];
   semanticPointerResult?: Record<string, unknown>;
-  refetchMode?: 'replacement' | 'missing' | 'ambiguous';
+  refetchMode?: 'replacement' | 'moved' | 'missing' | 'ambiguous';
   resolveDisplays?: CuaDriverBackendOptions['resolveDisplays'];
   physicalInputRecentlyActive?: CuaDriverBackendOptions['physicalInputRecentlyActive'];
   onTrace?: CuaDriverBackendOptions['onTrace'];
@@ -796,6 +807,30 @@ describe('cua-driver backend', () => {
     assert.equal(click?.element_token, 'snapshot:9');
   });
 
+  it('rejects a same-label replacement that moved before semantic dispatch', async () => {
+    const { backend, logPath } = makeBackend({
+      axRole: 'AXButton',
+      axLabel: 'Continue',
+      refetchMode: 'moved',
+    });
+    const signal = new AbortController().signal;
+    const context = { sessionId: 's1', turnId: 't1', toolCallId: 'semantic' };
+    const observation = await backend.observeApp!({
+      app: 'Fixture Window',
+      includeScreenshot: true,
+    }, signal, context);
+    const result = await backend.runSemantic!({
+      type: 'click_element',
+      observationId: observation.observationId,
+      elementId: '7',
+      elementIdentity: observation.elements[0]!.identity,
+    }, signal, { ...context, boundAction: boundElementAction(observation, '7') });
+
+    assert.equal(result.outcome.ok, false);
+    if (!result.outcome.ok) assert.equal(result.outcome.error, 'stale_frame');
+    assert.equal(toolCalls(await readRecords(logPath), 'click').length, 0);
+  });
+
   it('refetches an unlabeled element by unique structural identity', async () => {
     const { backend, logPath } = makeBackend({
       axRole: 'AXButton',
@@ -872,11 +907,10 @@ describe('cua-driver backend', () => {
     }]);
   });
 
-  it('runs select_text, secondary action, and press_key with full fresh observations', async () => {
+  it('fails closed for semantic actions absent from the pinned driver registry', async () => {
     for (const action of [
       { type: 'select_text', text: 'target' },
       { type: 'secondary_action', action: 'Increment' },
-      { type: 'press_key', key: 'Tab' },
     ] as const) {
       const { backend, logPath } = makeBackend({ axRole: 'AXTextField' });
       const context = {
@@ -888,13 +922,7 @@ describe('cua-driver backend', () => {
         app: 'Fixture Window',
         includeScreenshot: true,
       }, new AbortController().signal, context);
-      const semanticAction: CuSemanticAction = action.type === 'press_key'
-        ? {
-            type: 'press_key',
-            observationId: observation.observationId,
-            key: action.key,
-          }
-        : action.type === 'select_text'
+      const semanticAction: CuSemanticAction = action.type === 'select_text'
           ? {
               type: 'select_text',
               observationId: observation.observationId,
@@ -914,14 +942,32 @@ describe('cua-driver backend', () => {
         boundAction: boundElementAction(observation, '7'),
       });
 
-      assert.equal(result.outcome.ok, true);
-      assert.ok(result.observation?.observationId);
-      assert.ok(result.screenshot);
-      const tool = action.type === 'secondary_action'
-        ? 'perform_secondary_action'
-        : action.type;
-      assert.equal(toolCalls(await readRecords(logPath), tool).length, 1);
+      assert.equal(result.outcome.ok, false);
+      if (!result.outcome.ok) assert.equal(result.outcome.error, 'unsupported_action');
+      assert.equal(toolCalls(await readRecords(logPath), 'select_text').length, 0);
+      assert.equal(toolCalls(await readRecords(logPath), 'perform_secondary_action').length, 0);
+    }
+  });
+
+  it('evicts old unconsumed observations within a long-lived session', async () => {
+    const { backend } = makeBackend({ axRole: 'AXButton' });
+    const signal = new AbortController().signal;
+    const context = { sessionId: 'long-session', turnId: 't1', toolCallId: 'observe' };
+    const observations: CuObservation[] = [];
+    for (let index = 0; index < 17; index += 1) {
+      observations.push(await backend.observeApp!({
+        app: 'Fixture Window',
+        includeScreenshot: false,
+      }, signal, { ...context, toolCallId: `observe-${index}` }));
     }
+
+    const result = await backend.runSemantic!({
+      type: 'click_element',
+      observationId: observations[0]!.observationId,
+      elementId: '7',
+    }, signal, { ...context, toolCallId: 'old-action' });
+    assert.equal(result.outcome.ok, false);
+    if (!result.outcome.ok) assert.equal(result.outcome.error, 'stale_frame');
   });
 
   it('window_id disambiguates multiple visible windows from the same app', async () => {
@@ -976,6 +1022,25 @@ describe('cua-driver backend', () => {
     assert.equal(smallRes.screenshot!.mimeType, 'image/png');
   });
 
+  it('applies the same compression policy to window observations', async () => {
+    let calls = 0;
+    const { backend } = makeBackend({
+      bigImage: true,
+      compressFrame: () => {
+        calls += 1;
+        return { base64: 'anVzdGpwZWc=', mimeType: 'image/jpeg' };
+      },
+    });
+    const observation = await backend.observeApp!({
+      app: 'Fixture Window',
+      includeScreenshot: true,
+    }, new AbortController().signal, DEFAULT_RUN_CONTEXT);
+
+    assert.equal(calls, 1);
+    assert.equal(observation.screenshot?.mimeType, 'image/jpeg');
+    assert.equal(observation.screenshot?.base64, 'anVzdGpwZWc=');
+  });
+
   it('click on an app window with no AX element → same-snapshot pixel path, NEVER scope:desktop', async () => {
     const { backend, logPath } = makeBackend({ emptyAx: true });
     const sig = new AbortController().signal;
@@ -1342,6 +1407,85 @@ describe('cua-driver backend', () => {
     assert.equal(toolCalls(records, 'drag').length, 0);
   });
 
+  it('keeps Electron semantic click available while compatibility input is disabled', async () => {
+    const traces: CuaDriverTraceEvent[] = [];
+    const { backend, logPath } = makeBackend({
+      allowCompatibilityInputDispatch: false,
+      processKind: 'electron',
+      pageTarget: testPageTarget(),
+      semanticPointerResult: {
+        supported: true,
+        ok: true,
+        kind: 'left_click',
+        editable: true,
+        tagName: 'input',
+        focusChanged: true,
+      },
+      onTrace: (event) => traces.push(event),
+    });
+
+    const result = await backend.run(
+      { type: 'left_click', coordinate: { x: 600, y: 400 } } as CuAction,
+      new AbortController().signal,
+    );
+
+    assert.equal(result.outcome.ok, true);
+    if (result.outcome.ok) assert.equal(result.outcome.tier, 'semantic-background');
+    const records = await readRecords(logPath);
+    assert.equal(businessPageCalls(records).length, 1);
+    assert.equal(toolCalls(records, 'click').length, 0);
+    assert.equal(
+      traces.some((event) =>
+        event.type === 'dispatch'
+        && event.tool === 'page'
+        && event.address === 'semantic'),
+      true,
+    );
+  });
+
+  it('traces native semantic dispatch without exposing element content', async () => {
+    const traces: CuaDriverTraceEvent[] = [];
+    const { backend } = makeBackend({
+      axRole: 'AXTextField',
+      axLabel: 'Private field label',
+      onTrace: (event) => traces.push(event),
+    });
+    const signal = new AbortController().signal;
+    const context = {
+      sessionId: 'trace-session',
+      turnId: 'trace-turn',
+      toolCallId: 'native-set-value',
+    };
+    const observed = await backend.observeApp!({
+      app: 'Fixture Window',
+      includeScreenshot: false,
+    }, signal, context);
+
+    await backend.runSemantic!({
+      type: 'set_value',
+      observationId: observed.observationId,
+      elementId: '7',
+      value: 'private value',
+      elementIdentity: observed.elements[0]!.identity,
+    }, signal, {
+      ...context,
+      boundAction: boundElementAction(observed, '7'),
+    });
+
+    const dispatch = traces.find((event) =>
+      event.type === 'dispatch' && event.toolCallId === 'native-set-value');
+    assert.deepEqual(dispatch, {
+      type: 'dispatch',
+      toolCallId: 'native-set-value',
+      actionType: 'set_value',
+      tool: 'set_value',
+      pid: 4242,
+      windowId: 77,
+      address: 'ax',
+    });
+    assert.doesNotMatch(JSON.stringify(traces), /Private field label|private value/);
+  });
+
   it('fails closed when the physical-input guard cannot be read', async () => {
     const { backend, logPath } = makeBackend({
       axRole: 'AXButton',