fix(computer-use): enforce cursor regression and package gates

Route native-only changes through macOS checks and verify relocated cursor resources in final packages. Reject resources that escape the app and exercise visible click feedback against a disposable native receiver. Connect the regressions to required checks and capture shell fixture output through temporary files.
2026-10-10 03:43:11 +08:00 · 2026-09-11 15:53:22 +08:00
parent a0a8fd4b45
commit 0bcbfe6921
11 changed files with 441 additions and 32 deletions
@@ -38,6 +38,10 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
        try await verifyPublishedControl(mismatchedWindowTitle: false, exerciseDrags: true, wideWindow: true)
    }

+    func testVisibleCursorSurvivesRepeatedClicksAndTracksExposedBackgroundWindow() async throws {
+        try await verifyPublishedControl(mismatchedWindowTitle: false, exerciseVisualClicks: true)
+    }
+
    func testTextScrollAndSecondaryMethodsReachTheSameOfficialReceiver() async throws {
        if ProcessInfo.processInfo.environment[Self.fixtureFlag] == "1" {
            try await runFixtureProcess(mismatchedWindowTitle: false, regularActivation: true, wideWindow: false)
@@ -174,7 +178,7 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
        try await waitUntil(description: "method fixture exit") { process.isTerminated }
    }

-    private func verifyPublishedControl(mismatchedWindowTitle: Bool, exerciseDrags: Bool = false, exerciseKeys: Bool = false, wideWindow: Bool = false) async throws {
+    private func verifyPublishedControl(mismatchedWindowTitle: Bool, exerciseDrags: Bool = false, exerciseKeys: Bool = false, wideWindow: Bool = false, exerciseVisualClicks: Bool = false) async throws {
        if ProcessInfo.processInfo.environment[Self.fixtureFlag] == "1" {
            try await runFixtureProcess(
                mismatchedWindowTitle: ProcessInfo.processInfo.environment[Self.fixtureMismatchedTitle] == "1",
@@ -188,7 +192,7 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
            AXIsProcessTrusted(),
            "Live AX publication requires Accessibility permission for the test runner"
        )
-        if mismatchedWindowTitle || exerciseDrags {
+        if mismatchedWindowTitle || exerciseDrags || exerciseVisualClicks {
            try XCTSkipUnless(
                Capture.hasScreenRecordingPermission(),
                "Coordinate publication requires Screen Recording permission for the test runner"
@@ -220,7 +224,7 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
            Self.fixtureGesturePath: gestures.path,
            // A full-suite child enters the first test, so fixture modes must
            // travel with this launch rather than that test method's defaults.
-            Self.fixtureRegularActivation: exerciseKeys ? "1" : "0",
+            Self.fixtureRegularActivation: exerciseKeys || exerciseVisualClicks ? "1" : "0",
            Self.fixtureWideWindow: wideWindow ? "1" : "0",
        ]) { _, fixture in fixture }
        configuration.activates = false
@@ -273,6 +277,10 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
        let (_, clickedLine) = try publishedHandle(label: "Bold", state: clickedState)
        XCTAssertTrue(clickedLine.contains("Value: 1"), clickedLine)

+        if exerciseVisualClicks {
+            try await verifyVisibleClicks(cursor: cursor, router: router, process: process, gestures: gestures)
+        }
+
        if exerciseKeys {
            let (canvasHandle, _) = try publishedHandle(label: "Drag fixture", state: clickedState)
            var phase = "canvas click"
@@ -435,6 +443,114 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
        try await waitUntil(description: "fixture exit") { process.isTerminated }
    }

+    private func verifyVisibleClicks(
+        cursor: VirtualCursor, router: CommandRouter, process: NSRunningApplication, gestures: URL
+    ) async throws {
+        let pid = process.processIdentifier
+        let captured = try await router.handle(cmd: "get_app_state", payload: .object([
+            "pid": .int(Int(pid)), "disableDiff": .bool(true),
+        ]))
+        let shot = try XCTUnwrap(captured["screenshot"])
+        let observed = try await AXTree.appState(pid: pid, disableDiff: true)
+        let (handle, _) = try publishedHandle(label: "Drag fixture", state: observed)
+        let frame = try XCTUnwrap(AXTree.record(pid: pid, index: handle.index)?.frameGlobal)
+        let point = CGPoint(x: frame.x + frame.w / 2, y: frame.y + frame.h / 2)
+        let x = (point.x - (try XCTUnwrap(shot["originX"]?.asDouble)))
+            * Double(try XCTUnwrap(shot["width"]?.asInt)) / (try XCTUnwrap(shot["pointWidth"]?.asDouble))
+        let y = (point.y - (try XCTUnwrap(shot["originY"]?.asDouble)))
+            * Double(try XCTUnwrap(shot["height"]?.asInt)) / (try XCTUnwrap(shot["pointHeight"]?.asDouble))
+
+        // Observe real AppKit layers, not a mock callback or just a successful
+        // input result. A non-headless cursor that was never shown used to let
+        // every integration test skip the exact showClick branch that crashed.
+        // The first indexed action already preloaded this cursor's windows.
+        let overlays = NSApplication.shared.windows.filter { $0.ignoresMouseEvents && $0.level.rawValue == Int(CGShieldingWindowLevel()) }
+        XCTAssertFalse(overlays.isEmpty)
+        func ripples() -> [CALayer] {
+            overlays.flatMap { $0.contentView?.layer?.sublayers ?? [] }.filter {
+                $0.animation(forKey: "ripple") != nil || $0.animation(forKey: "rasterRipple") != nil
+            }
+        }
+        func click(button: String = "left", count: Int = 1) async throws {
+            let result = try await router.handle(cmd: "click", payload: .object([
+                "pid": .int(Int(pid)), "x": .double(x), "y": .double(y),
+                "mouse_button": .string(button), "click_count": .int(count),
+            ]))
+            XCTAssertEqual(result, .bool(true))
+        }
+        cursor.show()
+        XCTAssertTrue(process.activate(options: []))
+        try await waitUntil(description: "disposable receiver is foreground") {
+            NSWorkspace.shared.frontmostApplication?.processIdentifier == pid
+        }
+        for step in 0..<12 {
+            // Retain the layers across await: a removed ripple's address can
+            // otherwise be reused by the next layer and look like no new ring.
+            let before = ripples()
+            try await click(button: step % 3 == 2 ? "right" : "left", count: step % 3 == 1 ? 2 : 1)
+            XCTAssertTrue(overlays.contains { $0.isVisible })
+            XCTAssertTrue(ripples().contains { layer in !before.contains { $0 === layer } }, "Click \(step) must create visible feedback")
+        }
+        try await waitUntil(description: "16 mouse-up receipts from 12 single/double/right click commands") {
+            guard let data = try? Data(contentsOf: gestures),
+                  let receipt = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return false }
+            return receipt["completed"] as? Int == 16
+        }
+        let receipt = try XCTUnwrap(try JSONSerialization.jsonObject(with: Data(contentsOf: gestures)) as? [String: Any])
+        let events = try XCTUnwrap(receipt["events"] as? [[String: Any]])
+        XCTAssertEqual(events.filter { $0["type"] as? UInt == NSEvent.EventType.rightMouseUp.rawValue }.count, 4)
+        XCTAssertEqual(receipt["unpaired"] as? Int, 0)
+
+        // Put an independent host window to the right of the background
+        // receiver, then cover it, then uncover it. WindowServer supplies the
+        // actual occlusion evidence, as in the user's two-app layout.
+        let app = NSApplication.shared
+        let previousPolicy = app.activationPolicy()
+        app.setActivationPolicy(.regular)
+        let host = NSWindow(contentRect: NSRect(x: 660, y: 200, width: 240, height: 200), styleMask: [.titled], backing: .buffered, defer: false)
+        host.isReleasedWhenClosed = false
+        host.collectionBehavior = [.canJoinAllApplications]
+        host.title = "Disposable Computer Use host"
+        defer {
+            host.orderOut(nil)
+            host.close()
+            app.setActivationPolicy(previousPolicy)
+            WindowExposure.resetForTests()
+        }
+        host.makeKeyAndOrderFront(nil)
+        app.activate(ignoringOtherApps: true)
+        try await waitUntil(description: "disposable host is foreground") {
+            NSWorkspace.shared.frontmostApplication?.processIdentifier == getpid()
+        }
+        func expectFeedback(exposed: Bool) async throws {
+            try await waitUntil(description: exposed ? "target exposed" : "target covered") {
+                WindowExposure.resetForTests()
+                return WindowExposure.targetWindowExposed(at: point, targetPid: pid) == exposed
+            }
+            cursor.hide()
+            cursor.show()
+            try await click()
+            XCTAssertEqual(overlays.contains { $0.isVisible }, exposed)
+            XCTAssertEqual(!ripples().isEmpty, exposed)
+            XCTAssertEqual(NSWorkspace.shared.frontmostApplication?.processIdentifier, getpid())
+        }
+        try await expectFeedback(exposed: true)
+        host.setFrame(NSRect(x: point.x - 40, y: CGDisplayBounds(CGMainDisplayID()).height - point.y - 40, width: 240, height: 200), display: true)
+        try await expectFeedback(exposed: false)
+        host.setFrame(NSRect(x: 660, y: 200, width: 240, height: 200), display: true)
+        try await expectFeedback(exposed: true)
+        try await waitUntil(description: "three background clicks received, including covered target") {
+            guard let data = try? Data(contentsOf: gestures),
+                  let receipt = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return false }
+            return receipt["completed"] as? Int == 19
+        }
+        cursor.hide()
+        cursor.showClick(at: point, kind: .single)
+        XCTAssertTrue(ripples().isEmpty, "Turn end must clear feedback")
+        XCTAssertFalse(overlays.contains { $0.isVisible })
+        print("[native-visible-click-smoke] observed 15 commands, 19 received clicks; checked foreground/exposed/covered/re-exposed/hidden feedback")
+    }
+
    private func runFixtureProcess(mismatchedWindowTitle: Bool, regularActivation: Bool, wideWindow: Bool) async throws {
        let environment = ProcessInfo.processInfo.environment
        let readyPath = try XCTUnwrap(environment[Self.fixtureReadyPath])
@@ -637,6 +753,9 @@ private final class DragReceiptView: NSView {
        record(event)
    }

+    override func rightMouseDown(with event: NSEvent) { mouseDown(with: event) }
+    override func rightMouseUp(with event: NSEvent) { mouseUp(with: event) }
+
    private func record(_ event: NSEvent) {
        events.append([
            "type": event.type.rawValue,
@@ -517,6 +517,12 @@ verify_relocated_cursor_resources() (
  local expected_directory="$probe_app/Contents/Resources/cu-helper_cc-haha-computer-use.bundle/LensSequence"
  [ -d "$expected_directory" ] || die "Cursor resource probe package is missing $expected_directory"
  expected_directory="$(cd "$expected_directory" && pwd -P)"
+  local canonical_app
+  canonical_app="$(cd "$probe_app" && pwd -P)"
+  case "$expected_directory" in
+    "$canonical_app"/*) ;;
+    *) die "Cursor resource probe found resources outside relocated package: $expected_directory" ;;
+  esac
  [ "$resource_directory" = "$expected_directory" ] \
    || die "Cursor resource probe loaded '$resource_directory' instead of relocated package '$expected_directory'"
  log "verified: relocated cursor resources ($resource_directory)"
@@ -1,5 +1,5 @@
 import { afterEach, describe, expect, test } from 'bun:test'
-import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'
+import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'
 import { tmpdir } from 'node:os'
 import path from 'node:path'

@@ -8,11 +8,26 @@ const productIcon = path.resolve(import.meta.dirname, '../../desktop/src-tauri/i
 const fixtureDirectories: string[] = []
 const resourceBundleName = 'cu-helper_cc-haha-computer-use.bundle'

+function runFixtureCommand(command: string[], options: { cwd?: string, env?: Record<string, string | undefined> } = {}) {
+  // File-backed output also works on Bun versions where test subprocesses
+  // receive closed pipe descriptors (even /bin/echo exits 1 with no output).
+  // Keep the real shell result and diagnostics; never turn that failure into a skip.
+  const directory = mkdtempSync(path.join(tmpdir(), 'cu-helper-fixture-output-'))
+  fixtureDirectories.push(directory)
+  const stdout = path.join(directory, 'stdout')
+  const stderr = path.join(directory, 'stderr')
+  const result = Bun.spawnSync(command, {
+    ...options,
+    stdin: 'ignore', stdout: Bun.file(stdout), stderr: Bun.file(stderr),
+  })
+  return { exitCode: result.exitCode, stdout: readFileSync(stdout), stderr: readFileSync(stderr) }
+}
+
 function resolveArchitectureSpecificBuildPaths(arch: 'arm64' | 'x86_64') {
  const directory = mkdtempSync(path.join(tmpdir(), 'cu-helper-build-path-'))
  fixtureDirectories.push(directory)
  const binDir = path.join(directory, arch, `${arch}-apple-macosx`, 'release')
-  const result = Bun.spawnSync([
+  const result = runFixtureCommand([
    'bash',
    '-c',
    `
@@ -58,7 +73,7 @@ function wrapFixtureApp(options: { missingIcon?: boolean, missingResources?: boo
    writeFileSync(path.join(resourceBundle, 'LensSequence', 'README.md'), 'optional frames fixture')
  }

-  const result = Bun.spawnSync([
+  const result = runFixtureCommand([
    'bash',
    '-c',
    `
@@ -96,7 +111,7 @@ wrap_app
  }
 }

-function probeFixtureApp(mode: 'packaged' | 'build-path' | 'invalid-json' | 'crash', crossArch = false) {
+function probeFixtureApp(mode: 'packaged' | 'build-path' | 'external-symlink' | 'invalid-json' | 'crash', crossArch = false) {
  const directory = mkdtempSync(path.join(tmpdir(), 'cu-helper-resource-probe-'))
  fixtureDirectories.push(directory)
  const app = path.join(directory, 'source.app')
@@ -106,6 +121,11 @@ function probeFixtureApp(mode: 'packaged' | 'build-path' | 'invalid-json' | 'cra
  writeFileSync(path.join(app, 'Contents', 'Resources', resourceBundleName, 'LensSequence', 'README.md'), 'optional frames fixture')
  const buildTreeResources = path.join(directory, 'build-tree', resourceBundleName, 'LensSequence')
  mkdirSync(buildTreeResources, { recursive: true })
+  if (mode === 'external-symlink') {
+    const lensDirectory = path.join(app, 'Contents', 'Resources', resourceBundleName, 'LensSequence')
+    rmSync(lensDirectory, { recursive: true })
+    symlinkSync(buildTreeResources, lensDirectory)
+  }
  writeFileSync(path.join(app, 'Contents', 'Resources', 'outside-resource-path'), buildTreeResources)
  writeFileSync(binary, `#!/bin/bash
 set -eu
@@ -119,7 +139,7 @@ esac
 printf '{"resourceDirectory":"%s","frameCount":0,"proceduralFallback":true}\\n' "$resource_dir"
 `)
  chmodSync(binary, 0o755)
-  const result = Bun.spawnSync([
+  const result = runFixtureCommand([
    'bash', '-c', `
 source "$1"
 APP_PATH="$2"
@@ -140,11 +160,12 @@ verify_relocated_cursor_resources
 }

 function resolveTimestampArgument(identity: string, mode = 'auto') {
-  const result = Bun.spawnSync([
+  const result = runFixtureCommand([
    'bash',
    '-c',
    [
      'source "$1"',
+      'security() { return 1; }',
      'SIGN_IDENTITY="$2"',
      'CU_HELPER_TIMESTAMP_MODE="$3"',
      'resolve_timestamp_mode',
@@ -163,7 +184,7 @@ function resolveTimestampArgument(identity: string, mode = 'auto') {
 }

 function resolveIdentityWithOnlyDeveloperId() {
-  const result = Bun.spawnSync([
+  const result = runFixtureCommand([
    'bash',
    '-c',
    [
@@ -239,7 +260,7 @@ describe.skipIf(process.platform !== 'darwin')('cu-helper permission-list app ic
    const result = wrapFixtureApp()
    expect(result.exitCode).toBe(0)

-    const plist = Bun.spawnSync([
+    const plist = runFixtureCommand([
      '/usr/bin/plutil', '-convert', 'json', '-o', '-',
      path.join(result.contents, 'Info.plist'),
    ])
@@ -292,11 +313,12 @@ describe.skipIf(process.platform !== 'darwin')('cu-helper relocated resource pro
    expect(result.leftovers).toEqual([])
  })

-  test.each(['build-path', 'invalid-json', 'crash'] as const)('rejects %s instead of accepting a false resource-load success', mode => {
+  test.each(['build-path', 'external-symlink', 'invalid-json', 'crash'] as const)('rejects %s instead of accepting a false resource-load success', mode => {
    const result = probeFixtureApp(mode)
    expect(result.exitCode).not.toBe(0)
    expect(result.stderr).toContain('Cursor resource probe')
    if (mode === 'build-path') expect(result.stderr).toContain('instead of relocated package')
+    if (mode === 'external-symlink') expect(result.stderr).toContain('outside relocated package')
    expect(result.leftovers).toEqual([])
  })