From df0a566ed2c0087ef0e4e62e1f5f7ae42fb27cc6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?=
 =?UTF-8?q?=29?= <relakkes@gmail.com>
Date: Wed, 2 Sep 2026 08:48:52 +0800
Subject: [PATCH] fix(computer-use): restore offscreen macOS targets

---
 .../Sources/cu-helper/AXAction.swift          |   5 +-
 .../Sources/cu-helper/CommandRouter.swift     |   5 +
 .../cu-helper/TargetWindowRecovery.swift      |  89 +++++++
 .../TargetWindowRecoveryTests.swift           | 217 ++++++++++++++++++
 4 files changed, 313 insertions(+), 3 deletions(-)
 create mode 100644 native/cu-helper/Sources/cu-helper/TargetWindowRecovery.swift
 create mode 100644 native/cu-helper/Tests/CuHelperTests/TargetWindowRecoveryTests.swift

diff --git a/native/cu-helper/Sources/cu-helper/AXAction.swift b/native/cu-helper/Sources/cu-helper/AXAction.swift
index d3e4949a..e15cb7a6 100644
--- a/native/cu-helper/Sources/cu-helper/AXAction.swift
+++ b/native/cu-helper/Sources/cu-helper/AXAction.swift
@@ -605,9 +605,8 @@ public enum AXAction {
                 """
                 The target app has no window on screen, so no click or keystroke \
                 can reach it. It is minimized, hidden, or on another Space. \
-                Screenshots still work for such a window, so the state you were \
-                shown is real — it just cannot be acted on. Ask the user to \
-                bring the app back on screen.
+                Call get_app_state so Computer Use can restore the explicitly \
+                selected app before retrying the action.
                 """
             )
         case .failure(.pointOutsideWindows):
diff --git a/native/cu-helper/Sources/cu-helper/CommandRouter.swift b/native/cu-helper/Sources/cu-helper/CommandRouter.swift
index 52acc42c..f0a28653 100644
--- a/native/cu-helper/Sources/cu-helper/CommandRouter.swift
+++ b/native/cu-helper/Sources/cu-helper/CommandRouter.swift
@@ -711,6 +711,11 @@ public final class CommandRouter {
         // target), not whatever happens to be frontmost.
         setResolvedTarget(target)
 
+        // A hidden, minimized, or other-Space window has no on-screen geometry
+        // that later input can bind to. Recover only this explicit, authorized
+        // target; ordinary background windows remain background-readable.
+        _ = try await TargetWindowRecovery.recoverIfNeeded(target: target)
+
         let result = try await AXTree.appState(pid: pid, disableDiff: disableDiff)
         var object = try encode(result).asObject ?? [:]
         guard let snapshotEvidence = AXTree.snapshotEvidence(pid: pid) else {
diff --git a/native/cu-helper/Sources/cu-helper/TargetWindowRecovery.swift b/native/cu-helper/Sources/cu-helper/TargetWindowRecovery.swift
new file mode 100644
index 00000000..575b2d13
--- /dev/null
+++ b/native/cu-helper/Sources/cu-helper/TargetWindowRecovery.swift
@@ -0,0 +1,89 @@
+import AppKit
+
+/// Makes an explicitly authorized app actionable when macOS has removed all of
+/// its windows from the current on-screen window list.
+///
+/// Ordinary background windows stay untouched. Recovery is reserved for the
+/// three states that cannot be reached by window-bound input: a hidden app, a
+/// minimized window, or a window on another Space. `activateAllWindows` is the
+/// public native equivalent of the user choosing the exact app in the Dock; it
+/// can unhide/restore the app and move macOS to its Space without a shell or
+/// AppleScript fallback.
+@MainActor
+enum TargetWindowRecovery {
+    enum Outcome: Equatable {
+        case alreadyOnScreen
+        case recovered
+    }
+
+    struct Runtime {
+        var currentIdentity: (pid_t) -> AXTreeProcessIdentity?
+        var hasOnScreenWindow: (pid_t) -> Bool
+        var activateAllWindows: (pid_t) -> Bool
+        var pause: () async throws -> Void
+        var attempts: Int = 20
+
+        @MainActor static let live = Runtime(
+            currentIdentity: { AXTree.currentProcessIdentity(pid: $0) },
+            hasOnScreenWindow: { WindowGeometry.hasWindowOnScreen(pid: $0) },
+            activateAllWindows: { pid in
+                guard let application = NSRunningApplication(processIdentifier: pid),
+                      !application.isTerminated else { return false }
+                // `activate` normally unhides as part of activation. Sending an
+                // explicit unhide first also covers apps whose reopen handling is
+                // delayed; the on-screen verification below is still authoritative.
+                if application.isHidden { _ = application.unhide() }
+                return application.activate(options: [.activateAllWindows])
+            },
+            pause: { try await Task.sleep(for: .milliseconds(100)) }
+        )
+    }
+
+    static func recoverIfNeeded(
+        target: ProvenProcessTarget,
+        runtime: Runtime = .live
+    ) async throws -> Outcome {
+        try validateIdentity(target, runtime: runtime)
+        if runtime.hasOnScreenWindow(target.pid) {
+            return .alreadyOnScreen
+        }
+
+        let activationAccepted = runtime.activateAllWindows(target.pid)
+
+        // The request is asynchronous and AppKit explicitly says acceptance is
+        // not proof of activation. Check once immediately, then yield the main
+        // run loop for a bounded Space/window transition.
+        try validateIdentity(target, runtime: runtime)
+        if runtime.hasOnScreenWindow(target.pid) {
+            return .recovered
+        }
+        for _ in 0..<max(0, runtime.attempts) {
+            try Task.checkCancellation()
+            try await runtime.pause()
+            try validateIdentity(target, runtime: runtime)
+            if runtime.hasOnScreenWindow(target.pid) {
+                return .recovered
+            }
+        }
+
+        let reason = activationAccepted
+            ? "macOS accepted activation, but no target window appeared on screen"
+            : "macOS refused the target app activation request"
+        throw CUError(
+            "target_window_offscreen",
+            "Computer Use could not restore the explicitly selected app: \(reason). No state snapshot was published."
+        )
+    }
+
+    private static func validateIdentity(
+        _ target: ProvenProcessTarget,
+        runtime: Runtime
+    ) throws {
+        guard runtime.currentIdentity(target.pid) == target.identity else {
+            throw CUError(
+                "stale_process",
+                "The target process changed while restoring its window. No state snapshot was published."
+            )
+        }
+    }
+}
diff --git a/native/cu-helper/Tests/CuHelperTests/TargetWindowRecoveryTests.swift b/native/cu-helper/Tests/CuHelperTests/TargetWindowRecoveryTests.swift
new file mode 100644
index 00000000..5fc515c1
--- /dev/null
+++ b/native/cu-helper/Tests/CuHelperTests/TargetWindowRecoveryTests.swift
@@ -0,0 +1,217 @@
+import XCTest
+
+@testable import cc_haha_computer_use
+
+@MainActor
+final class TargetWindowRecoveryTests: XCTestCase {
+    private let pid: pid_t = 72_041
+    private let identity = AXTreeProcessIdentity(
+        bundleID: "com.apple.TextEdit",
+        executablePath: "/System/Applications/TextEdit.app/Contents/MacOS/TextEdit",
+        launchTime: 44
+    )
+
+    func testHiddenMinimizedAndOtherSpaceTargetsTransitionBackOnScreen() async throws {
+        for placement in RecoveryHarness.Placement.unavailableCases {
+            let harness = RecoveryHarness(pid: pid, identity: identity, placement: placement)
+
+            let result = try await TargetWindowRecovery.recoverIfNeeded(
+                target: try target(),
+                runtime: harness.runtime()
+            )
+
+            XCTAssertEqual(result, .recovered, "placement: \(placement)")
+            XCTAssertEqual(harness.activationPIDs, [pid], "placement: \(placement)")
+            XCTAssertEqual(harness.placement, .onScreen, "placement: \(placement)")
+        }
+    }
+
+    func testAlreadyOnScreenBackgroundTargetIsNotActivated() async throws {
+        let harness = RecoveryHarness(pid: pid, identity: identity, placement: .onScreen)
+
+        let result = try await TargetWindowRecovery.recoverIfNeeded(
+            target: try target(),
+            runtime: harness.runtime()
+        )
+
+        XCTAssertEqual(result, .alreadyOnScreen)
+        XCTAssertTrue(harness.activationPIDs.isEmpty)
+        XCTAssertEqual(harness.pauseCount, 0)
+    }
+
+    func testAcceptedActivationThatNeverProducesAWindowFailsClosed() async throws {
+        let harness = RecoveryHarness(
+            pid: pid,
+            identity: identity,
+            placement: .otherSpace,
+            transitionAfterPauses: nil,
+            activationAccepted: true
+        )
+
+        do {
+            _ = try await TargetWindowRecovery.recoverIfNeeded(
+                target: try target(),
+                runtime: harness.runtime(attempts: 2)
+            )
+            XCTFail("an accepted activation is not proof that a window became available")
+        } catch let error as CUError {
+            XCTAssertEqual(error.code, "target_window_offscreen")
+        }
+
+        XCTAssertEqual(harness.activationPIDs, [pid])
+        XCTAssertEqual(harness.pauseCount, 2)
+        XCTAssertEqual(harness.placement, .otherSpace)
+    }
+
+    func testRejectedActivationNeverReportsRecovery() async throws {
+        let harness = RecoveryHarness(
+            pid: pid,
+            identity: identity,
+            placement: .minimized,
+            transitionAfterPauses: nil,
+            activationAccepted: false
+        )
+
+        do {
+            _ = try await TargetWindowRecovery.recoverIfNeeded(
+                target: try target(),
+                runtime: harness.runtime(attempts: 1)
+            )
+            XCTFail("a rejected activation must fail")
+        } catch let error as CUError {
+            XCTAssertEqual(error.code, "target_window_offscreen")
+            XCTAssertTrue(error.message.contains("refused"))
+        }
+
+        XCTAssertEqual(harness.activationPIDs, [pid])
+        XCTAssertEqual(harness.placement, .minimized)
+    }
+
+    func testProcessReplacementDuringRecoveryCannotInheritSuccess() async throws {
+        let replacement = AXTreeProcessIdentity(
+            bundleID: identity.bundleID,
+            executablePath: identity.executablePath,
+            launchTime: 45
+        )
+        let harness = RecoveryHarness(
+            pid: pid,
+            identity: identity,
+            placement: .hidden,
+            transitionAfterPauses: 1,
+            replacementAfterPauses: (1, replacement)
+        )
+
+        do {
+            _ = try await TargetWindowRecovery.recoverIfNeeded(
+                target: try target(),
+                runtime: harness.runtime()
+            )
+            XCTFail("a replacement process must not inherit the original recovery")
+        } catch let error as CUError {
+            XCTAssertEqual(error.code, "stale_process")
+        }
+    }
+
+    func testGetAppStateJoinsRecoveryBeforePublishingANewSnapshot() throws {
+        let source = try String(
+            contentsOf: sourceURL("CommandRouter.swift"),
+            encoding: .utf8
+        )
+        let start = try XCTUnwrap(source.range(of: "private func handleGetAppState"))
+        let body = String(source[start.lowerBound...].prefix(8_000))
+        let recovery = try XCTUnwrap(body.range(of: "TargetWindowRecovery.recoverIfNeeded"))
+        let snapshot = try XCTUnwrap(body.range(of: "AXTree.appState"))
+
+        XCTAssertLessThan(
+            recovery.lowerBound,
+            snapshot.lowerBound,
+            "the explicit target must be recoverable before AX publishes window identity and geometry"
+        )
+    }
+
+    private func target() throws -> ProvenProcessTarget {
+        try XCTUnwrap(ProvenProcessTarget(pid: pid, identity: identity))
+    }
+
+    private func sourceURL(_ name: String) -> URL {
+        URL(fileURLWithPath: #filePath)
+            .deletingLastPathComponent()
+            .deletingLastPathComponent()
+            .deletingLastPathComponent()
+            .appendingPathComponent("Sources/cu-helper")
+            .appendingPathComponent(name)
+    }
+}
+
+@MainActor
+private final class RecoveryHarness {
+    enum Placement: CaseIterable {
+        case onScreen
+        case hidden
+        case minimized
+        case otherSpace
+
+        static let unavailableCases: [Placement] = [.hidden, .minimized, .otherSpace]
+    }
+
+    let pid: pid_t
+    var identity: AXTreeProcessIdentity
+    var placement: Placement
+    var activationPIDs: [pid_t] = []
+    var pauseCount = 0
+
+    private let transitionAfterPauses: Int?
+    private let activationAccepted: Bool
+    private let replacementAfterPauses: (count: Int, identity: AXTreeProcessIdentity)?
+    private let originalIdentity: AXTreeProcessIdentity
+    private var recoveryRequested = false
+
+    init(
+        pid: pid_t,
+        identity: AXTreeProcessIdentity,
+        placement: Placement,
+        transitionAfterPauses: Int? = 1,
+        activationAccepted: Bool = true,
+        replacementAfterPauses: (Int, AXTreeProcessIdentity)? = nil
+    ) {
+        self.pid = pid
+        self.identity = identity
+        self.placement = placement
+        self.transitionAfterPauses = transitionAfterPauses
+        self.activationAccepted = activationAccepted
+        self.replacementAfterPauses = replacementAfterPauses
+        self.originalIdentity = identity
+    }
+
+    func runtime(attempts: Int = 20) -> TargetWindowRecovery.Runtime {
+        TargetWindowRecovery.Runtime(
+            currentIdentity: { [self] requestedPID in
+                XCTAssertEqual(requestedPID, pid)
+                return identity
+            },
+            hasOnScreenWindow: { [self] requestedPID in
+                XCTAssertEqual(requestedPID, pid)
+                return placement == .onScreen
+            },
+            activateAllWindows: { [self] requestedPID in
+                activationPIDs.append(requestedPID)
+                recoveryRequested = true
+                return activationAccepted
+            },
+            pause: { [self] in
+                pauseCount += 1
+                if let replacementAfterPauses,
+                   pauseCount == replacementAfterPauses.count {
+                    identity = replacementAfterPauses.identity
+                }
+                if recoveryRequested,
+                   let transitionAfterPauses,
+                   pauseCount == transitionAfterPauses,
+                   identity == originalIdentity {
+                    placement = .onScreen
+                }
+            },
+            attempts: attempts
+        )
+    }
+}