From ad7321f7cc3ff5407153f8fb610c922b85dc9b30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Tue, 1 Sep 2026 17:31:17 +0800 Subject: [PATCH] fix(computer-use): align long-running macOS automation with Codex --- native/cu-helper/Info.plist | 2 + .../Sources/cu-helper/AXAction.swift | 47 +++++- .../Sources/cu-helper/ClientAttestation.swift | 4 +- .../cu-helper/ClipboardPasteReceipt.swift | 45 +++++- .../Sources/cu-helper/CommandRouter.swift | 47 +++++- .../cu-helper/Sources/cu-helper/Daemon.swift | 47 ++++-- .../Sources/cu-helper/DaemonProtocol.swift | 99 +++++++++++++ .../cu-helper/DisplaySleepAssertion.swift | 50 +++++++ .../Sources/cu-helper/ForegroundLease.swift | 3 +- .../Sources/cu-helper/JSONValue.swift | 21 ++- .../Sources/cu-helper/UISettlePolicy.swift | 58 ++++++-- .../cu-helper/WindowCaptureStream.swift | 21 ++- .../ClientAttestationTests.swift | 4 +- .../ClipboardPasteReceiptTests.swift | 23 +++ .../CuHelperTests/DaemonProtocolTests.swift | 116 +++++++++++++++ .../DisplaySleepAssertionTests.swift | 39 +++++ .../CuHelperTests/UISettlePolicyTests.swift | 24 ++- .../WindowCaptureStreamTests.swift | 67 ++++++++- native/cu-helper/build.sh | 4 + native/cu-helper/build.test.ts | 86 ++++++++++- src/skills/bundled/computerUse.test.ts | 15 +- src/skills/bundled/computerUse.ts | 16 +- src/utils/computerUse/cuHelperDaemon.test.ts | 139 +++++++++++++++++- src/utils/computerUse/cuHelperDaemon.ts | 79 +++++++++- .../computerUse/executor.codexEngine.test.ts | 9 ++ src/utils/computerUse/executor.ts | 14 +- src/vendor/computer-use-mcp/executor.ts | 2 +- src/vendor/computer-use-mcp/instructions.ts | 8 +- .../computer-use-mcp/platformRouting.test.ts | 3 +- src/vendor/computer-use-mcp/toolCalls.test.ts | 49 +++++- src/vendor/computer-use-mcp/toolCalls.ts | 21 ++- src/vendor/computer-use-mcp/tools.ts | 37 ++++- 32 files changed, 1100 insertions(+), 99 deletions(-) create mode 100644 native/cu-helper/Sources/cu-helper/DaemonProtocol.swift create mode 100644 native/cu-helper/Sources/cu-helper/DisplaySleepAssertion.swift create mode 100644 native/cu-helper/Tests/CuHelperTests/DaemonProtocolTests.swift create mode 100644 native/cu-helper/Tests/CuHelperTests/DisplaySleepAssertionTests.swift diff --git a/native/cu-helper/Info.plist b/native/cu-helper/Info.plist index 70cc5ca2..f3e5a3fd 100644 --- a/native/cu-helper/Info.plist +++ b/native/cu-helper/Info.plist @@ -21,6 +21,8 @@ cc-haha-computer-use CFBundleDisplayName cc-haha-computer-use + CFBundleIconFile + icon.icns CFBundlePackageType APPL CFBundleShortVersionString diff --git a/native/cu-helper/Sources/cu-helper/AXAction.swift b/native/cu-helper/Sources/cu-helper/AXAction.swift index 3b5d4a01..d3e4949a 100644 --- a/native/cu-helper/Sources/cu-helper/AXAction.swift +++ b/native/cu-helper/Sources/cu-helper/AXAction.swift @@ -237,10 +237,18 @@ public final class ClipboardLease { } func writeTemporaryStringWithReceipt(_ text: String) throws -> ClipboardPasteReceipt { + try writeTemporaryContentWithReceipt(text, format: .text) + } + + func writeTemporaryContentWithReceipt( + _ text: String, + format: ClipboardPasteFormat + ) throws -> ClipboardPasteReceipt { if let captureError { throw captureError } - let receipt = ClipboardPasteReceipt(text: text) + let receipt = ClipboardPasteReceipt(text: text, format: format) let item = NSPasteboardItem() - guard item.setDataProvider(receipt, forTypes: [.string]) else { + let types = Array(format.promisedData(for: text).keys) + guard item.setDataProvider(receipt, forTypes: types) else { throw CUError("clipboard_write_failed", "Could not register temporary pasteboard data") } pasteboard.clearContents() @@ -926,13 +934,35 @@ public enum AXAction { try await typeViaClipboard(target: target, text) } + /// Explicit Codex-compatible paste path. Unlike `typeText`, this does not + /// first try AX value mutation or Unicode key events; it targets the app's + /// in-process focused field with Command-V and restores the user's original + /// pasteboard after observing a real promised-data read. + static func pasteText( + pid: pid_t, + _ text: String, + format: ClipboardPasteFormat + ) async throws { + guard !text.isEmpty else { return } + let target = try Injection.authorizeResolvedTarget(pid: pid) + try await typeViaClipboard(target: target, text, format: format) + } + /// Paste `text` into whatever holds in-app keyboard focus in `pid`, via the /// system clipboard + ⌘V (Codex's approach for CEF/Chromium text fields). /// The read receipt confirms supplied clipboard bytes, not the reader's /// PID or the field's final value; the next screenshot still verifies UI. - private static func typeViaClipboard(target: ProvenProcessTarget, _ text: String) async throws { + private static func typeViaClipboard( + target: ProvenProcessTarget, + _ text: String, + format: ClipboardPasteFormat = .text + ) async throws { let pid = target.pid - try await ClipboardPasteReceipt.perform(text: text, lease: ClipboardLease()) { validate in + try await ClipboardPasteReceipt.perform( + text: text, + format: format, + lease: ClipboardLease() + ) { validate in try await pressKey(pid: pid, "super+v", validateBeforePosting: { _ = try Injection.validateAuthorizedTarget(target) try validate() @@ -1609,10 +1639,11 @@ public enum AXAction { return CFEqual(lhs, rhs) } - /// Mark that a mutation just landed. Does NOT block: `get_app_state` reads - /// this stamp and waits out whatever settle time is still owed, so the cost - /// is paid once, and only when someone is actually about to look. + /// Settlement is recorded once at `ForegroundMutationRunner`, which covers + /// every input path and knows the target PID. AX-local success receipts must + /// not create a second, unscoped marker that can leak across apps. private static func settle() { - MutationClock.recordMutation() + // Intentionally empty; retained at the AX call sites as a semantic + // marker for successful mutations. } } diff --git a/native/cu-helper/Sources/cu-helper/ClientAttestation.swift b/native/cu-helper/Sources/cu-helper/ClientAttestation.swift index 41ce2667..95eed7b8 100644 --- a/native/cu-helper/Sources/cu-helper/ClientAttestation.swift +++ b/native/cu-helper/Sources/cu-helper/ClientAttestation.swift @@ -30,9 +30,9 @@ enum HelperClientPolicy { // read-only authorization preflight used before the ten public tools. "list_apps", "resolve_app_target", "get_app_state", "click", "set_value", "select_text", "perform_secondary_action", "scroll", - "drag", "press_key", "type_text", + "drag", "press_key", "type_text", "paste", // Private lifecycle / visible-overlay / permission and input diagnostics. - "ping", "shutdown", "overlay_show", "overlay_hide", + "ping", "shutdown", "overlay_show", "overlay_hide", "turn_end", "check_permissions", "input_monitor_state", "focus_monitor_state", "held_input_state", "last_injection_state", ] diff --git a/native/cu-helper/Sources/cu-helper/ClipboardPasteReceipt.swift b/native/cu-helper/Sources/cu-helper/ClipboardPasteReceipt.swift index fd89abfa..95a8699f 100644 --- a/native/cu-helper/Sources/cu-helper/ClipboardPasteReceipt.swift +++ b/native/cu-helper/Sources/cu-helper/ClipboardPasteReceipt.swift @@ -2,6 +2,32 @@ import AppKit import Foundation import os +enum ClipboardPasteFormat: String, Sendable { + case text + case md + case html + + func promisedData(for text: String) -> [NSPasteboard.PasteboardType: Data] { + switch self { + case .text, .md: + return [.string: Data(text.utf8)] + case .html: + let html = Data(text.utf8) + let plain = ( + try? NSAttributedString( + data: html, + options: [ + .documentType: NSAttributedString.DocumentType.html, + .characterEncoding: String.Encoding.utf8.rawValue, + ], + documentAttributes: nil + ).string + ) ?? text + return [.html: html, .string: Data(plain.utf8)] + } + } +} + /// Confirms that this pasteboard item's promised bytes were requested and /// supplied. AppKit does not identify the reader: this is not proof that the /// intended application's focused field accepted the text. @@ -25,16 +51,26 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch } @MainActor private(set) static var lastDiagnostic: Diagnostic? - private let data: Data + private let promisedData: [NSPasteboard.PasteboardType: Data] private let state = OSAllocatedUnfairLock(initialState: State()) init(text: String) { - data = Data(text.utf8) + promisedData = ClipboardPasteFormat.text.promisedData(for: text) super.init() } + init(text: String, format: ClipboardPasteFormat) { + promisedData = format.promisedData(for: text) + super.init() + } + + @MainActor + static func resetForTurn() { + lastDiagnostic = nil + } + func pasteboard(_ pasteboard: NSPasteboard?, item: NSPasteboardItem, provideDataForType type: NSPasteboard.PasteboardType) { - guard type == .string else { return } + guard let data = promisedData[type] else { return } state.withLock { $0.requested = true } guard item.setData(data, forType: type) else { return } state.withLock { value in @@ -50,6 +86,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch @MainActor static func perform( text: String, + format: ClipboardPasteFormat = .text, lease: ClipboardLease, timeout: Duration = .seconds(2), sendPaste: @MainActor (_ validateBeforePosting: @MainActor () throws -> Void) async throws -> Void @@ -79,7 +116,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch } do { try Task.checkCancellation() - let written = try lease.writeTemporaryStringWithReceipt(text) + let written = try lease.writeTemporaryContentWithReceipt(text, format: format) receipt = written try await Task.sleep(for: .milliseconds(40)) let validate: @MainActor () throws -> Void = { diff --git a/native/cu-helper/Sources/cu-helper/CommandRouter.swift b/native/cu-helper/Sources/cu-helper/CommandRouter.swift index 1a881f81..44d53a08 100644 --- a/native/cu-helper/Sources/cu-helper/CommandRouter.swift +++ b/native/cu-helper/Sources/cu-helper/CommandRouter.swift @@ -104,6 +104,7 @@ public final class CommandRouter { Self.lastShotTransform.removeAll() Self.lastCaptureDigest.removeAll() MutationClock.reset() + ClipboardPasteReceipt.resetForTurn() windowCaptureProvider?.invalidate() // Apps we told they were focused must be told they are not, or the // belief outlives the session that needed it. @@ -267,6 +268,9 @@ public final class CommandRouter { case "type_text": return try await handleTypeText(payload) + case "paste": + return try await handlePaste(payload) + case "press_key": return try await handlePressKey(payload) @@ -728,18 +732,19 @@ public final class CommandRouter { if #available(macOS 14.0, *) { // Let the UI finish whatever the last action started before we // photograph it, otherwise the model reasons about a half-drawn - // frame. Costs nothing when no action is pending. The rendered tree - // doubles as the busy signal — a progress indicator in it means the - // app is still working, so we allow a longer window. + // frame. Costs nothing when no action is pending, applies only to + // this target PID, and is consumed by this single capture. + let pendingMutation = MutationClock.takeMutation(pid: pid) let streamedShot = await Self.captureSettledWindowShot( - appIsBusy: result.axText.contains("progress indicator") + appIsBusy: result.axText.contains("progress indicator"), + lastMutationAt: pendingMutation ) { await windowCaptureProvider?.windowShot( pid: pid, processIdentity: snapshotEvidence.processIdentity, preferredWindowID: snapshotEvidence.keyWindowID, scale: 0.5, - newerThanUptime: MutationClock.lastMutation() + newerThanUptime: pendingMutation ) } guard TargetVisibilityPolicy.captureTargetStillMatches( @@ -875,9 +880,13 @@ public final class CommandRouter { /// action-to-screenshot regression can exercise both without live AX/TCC. static func captureSettledWindowShot( appIsBusy: Bool, + lastMutationAt: TimeInterval?, capture: () async -> WindowShot? ) async -> WindowShot? { - await MutationClock.awaitSettle(appIsBusy: appIsBusy) + await MutationClock.awaitSettle( + lastMutationAt: lastMutationAt, + appIsBusy: appIsBusy + ) return await capture() } @@ -1438,6 +1447,31 @@ public final class CommandRouter { } } + /// `paste`: bypass AX/Unicode typing and use the target app's in-process + /// paste focus. This is the reliable recovery path for Chromium/CEF fields + /// such as NeteaseMusic's search box. The clipboard lease restores every + /// original pasteboard item unless the user copied something meanwhile. + private func handlePaste(_ payload: JSONValue) async throws -> JSONValue { + try requireAXTrusted() + guard let text = payload["text"]?.asString else { + throw CUError("bad_payload", "paste requires a 'text' string") + } + guard let rawFormat = payload["format"]?.asString, + let format = ClipboardPasteFormat(rawValue: rawFormat) else { + throw CUError("bad_payload", "paste format must be 'text', 'md', or 'html'") + } + let expected = try Self.expectedProcessTarget(payload) + let target = try resolveTargetForMutation(payload) + setResolvedTarget(target) + try requireSnapshotProcess(target: target, expected: expected) + return try await withForegroundLease(command: "paste", target: target) { + _ = try Injection.validateAuthorizedTarget(target) + try self.requireSnapshotProcess(target: target, expected: expected) + try await AXAction.pasteText(pid: target.pid, text, format: format) + return .bool(true) + } + } + /// `press_key`: send an xdotool-style key spec (e.g. "super+c", "Return", /// "Tab", "KP_0", "Prior") to the target app via postToPid. private func handlePressKey(_ payload: JSONValue) async throws -> JSONValue { @@ -1566,6 +1600,7 @@ public final class CommandRouter { ) return try await ForegroundMutationRunner.run( lease: lease, + targetPID: target.pid, action: action ) } diff --git a/native/cu-helper/Sources/cu-helper/Daemon.swift b/native/cu-helper/Sources/cu-helper/Daemon.swift index 89377848..351c75fa 100644 --- a/native/cu-helper/Sources/cu-helper/Daemon.swift +++ b/native/cu-helper/Sources/cu-helper/Daemon.swift @@ -38,21 +38,21 @@ enum DaemonOverlayTargetResolver { // `cursor_position`, implicit-`from` drags and decomposed `mouse_down`/`mouse_up` // behave correctly across commands. // -// Transport: a private AF_UNIX SOCK_STREAM socket carrying NDJSON (one JSON -// object + '\n' per request, same per response). A GUI/AppKit process leaks +// Transport: a private AF_UNIX SOCK_STREAM socket carrying versioned NDJSON +// (one JSON object + '\n' per request, same per response). A GUI/AppKit process leaks // os_log / CoreGraphics chatter to stdout/stderr, which would corrupt the TS // bridge's `JSON.parse`; the daemon therefore reserves stdout for exactly ONE // readiness line and serves the request/response stream over the socket. // -// readiness : {"ready":true,"pid":,"proto":1}\n (stdout, once) -// request : {"id":"","cmd":"","payload":{...}}\n +// readiness : {"ready":true,"pid":,"protocolVersion":"…"}\n +// request : {"id":"","requestId":"","cmd":"",…}\n // response : {"id":"","ok":true,"result":}\n // {"id":"","ok":false,"error":{"message":"…","code":"…"}}\n // // Control verbs handled in-daemon (never reach CommandRouter): // overlay_show -> cursor.show() ; result true -// overlay_hide -> cursor.hide() ; result true -// ping -> result "pong" +// turn_end -> release all turn-owned state ; result true +// ping -> version/capability hello // shutdown -> result true, then NSApp.terminate(nil) // Every other cmd is forwarded to the shared CommandRouter — the exact same // dispatcher the one-shot CLI path uses, so each command has one implementation. @@ -63,6 +63,7 @@ public final class Daemon { private let cursor: VirtualCursor private let inputMonitor: PhysicalInputEpochMonitor private let router: CommandRouter + private let displaySleepAssertion = ComputerUseDisplaySleepAssertion() /// Listening socket fd (AF_UNIX SOCK_STREAM). -1 until `bindAndListen`. private var listenFD: Int32 = -1 @@ -78,11 +79,12 @@ public final class Daemon { private var connection: Connection? private var connectionToken: DaemonSessionToken? private var sessionGate = DaemonSessionGate() + private var turnGate = DaemonTurnGate() /// Set once the run loop is live; guards against double `run()`. private var didStartRunLoop = false - /// True between `overlay_show` and `overlay_hide` (CU active this turn). The + /// True between `overlay_show` and `turn_end` (CU active this turn). The /// cursor re-aims at the actual injection target while this holds. private var overlayActive = false private var explicitOverlayTarget: ProvenProcessTarget? @@ -220,6 +222,7 @@ public final class Daemon { // immediately by exit() (shutdown verb + signal handlers). Injection.releaseAllHeldSync() router.resetSessionState() + displaySleepAssertion.release() inputMonitor.stop() } @@ -435,6 +438,8 @@ public final class Daemon { stopOverlaySession() Injection.releaseAllHeldSync() router.resetSessionState() + turnGate.reset() + displaySleepAssertion.release() } /// A LaunchServices child is reparented to launchd, so the peer socket is @@ -479,7 +484,16 @@ public final class Daemon { let payload = request.payload ?? .object([:]) do { + let metadata = try ComputerUseDaemonProtocol.validate(request) + let opensTurn = ComputerUseDaemonProtocol.isTurnScoped(command: request.cmd) + && turnGate.active == nil + try turnGate.admit(metadata, command: request.cmd) + if opensTurn { displaySleepAssertion.acquire() } let result = try await route(cmd: request.cmd, payload: payload) + if request.cmd == "turn_end" || request.cmd == "overlay_hide" { + try turnGate.finish(metadata) + displaySleepAssertion.release() + } conn.send(encodeResponse(DaemonResponse(id: id, ok: true, result: result, error: nil))) } catch let cuError as CUError { conn.send(encodeResponse(DaemonResponse( @@ -508,12 +522,12 @@ public final class Daemon { try showOverlay(payload: payload) return .bool(true) - case "overlay_hide": - stopOverlaySession() + case "overlay_hide", "turn_end": + endTurn() return .bool(true) case "ping": - return .string("pong") + return ComputerUseDaemonProtocol.hello() case "check_permissions": // Usable in both modes; the daemon exposes it for onboarding / @@ -592,6 +606,15 @@ public final class Daemon { router.invalidateWindowCaptureStream() } + /// Codex-parity turn boundary. The helper process stays warm, but no AX + /// snapshot, coordinate transform, focus belief, held input, mutation clock, + /// clipboard diagnostic, or capture stream may leak into the next turn. + private func endTurn() { + stopOverlaySession() + Injection.releaseAllHeldSync() + router.resetSessionState() + } + /// The app the cursor should follow: ONLY the app the last injection / /// get_app_state actually resolved (`Injection.lastResolvedTargetPid`). We /// deliberately do NOT fall back to the frontmost app — at turn start, @@ -763,9 +786,9 @@ private final class Connection: @unchecked Sendable { private var inBuffer = Data() /// Capped to avoid unbounded growth from a malformed/never-newline client. - /// 16 MiB comfortably exceeds the largest legitimate request (a base64 + /// 8 MiB comfortably exceeds the largest legitimate request (a base64 /// screenshot travels in *responses*, not requests). - private let maxBufferBytes = 16 * 1024 * 1024 + private let maxBufferBytes = ComputerUseDaemonProtocol.maxFrameBytes private var onRequest: (@Sendable (Data) -> Void)? private var onClose: (@Sendable () -> Void)? diff --git a/native/cu-helper/Sources/cu-helper/DaemonProtocol.swift b/native/cu-helper/Sources/cu-helper/DaemonProtocol.swift new file mode 100644 index 00000000..243c24fe --- /dev/null +++ b/native/cu-helper/Sources/cu-helper/DaemonProtocol.swift @@ -0,0 +1,99 @@ +import Foundation + +/// Versioned request contract for the authenticated daemon socket. +/// +/// Code signature attestation proves who is connected; this handshake proves +/// that both sides agree on request shape and lifecycle semantics. Every request +/// also carries an absolute deadline plus session/turn identity so a command +/// queued behind a slow capture cannot execute after its caller has moved on. +enum ComputerUseDaemonProtocol { + static let version = "CCHahaComputerUseIPC-2" + static let maxFrameBytes = 8 * 1024 * 1024 + private static let connectionScopedCommands: Set = [ + "ping", "check_permissions", "shutdown", + ] + + static func isTurnScoped(command: String) -> Bool { + !connectionScopedCommands.contains(command) + } + + struct Metadata: Equatable, Sendable { + let sessionId: String + let turnId: String + } + + static func validate( + _ request: Request, + nowUnixMilliseconds: Int64 = Int64(Date().timeIntervalSince1970 * 1_000) + ) throws -> Metadata { + guard request.clientApiVersion == version else { + throw CUError( + "protocol_mismatch", + "Computer Use client/helper protocol mismatch; restart the app after updating" + ) + } + guard let id = request.id, !id.isEmpty, + request.requestId == id else { + throw CUError("bad_request_id", "Computer Use request identity is missing or inconsistent") + } + guard let deadline = request.deadlineUnixMilliseconds else { + throw CUError("missing_deadline", "Computer Use request is missing its absolute deadline") + } + guard deadline > nowUnixMilliseconds else { + throw CUError("deadline_exceeded", "Computer Use request expired before execution") + } + guard let sessionId = request.sessionId?.trimmingCharacters(in: .whitespacesAndNewlines), + !sessionId.isEmpty, + let turnId = request.turnId?.trimmingCharacters(in: .whitespacesAndNewlines), + !turnId.isEmpty else { + throw CUError("missing_turn_metadata", "Computer Use request is missing session/turn identity") + } + return Metadata(sessionId: sessionId, turnId: turnId) + } + + static func hello() -> JSONValue { + .object([ + "protocolVersion": .string(version), + "supportsAbsoluteDeadlines": .bool(true), + "supportsTurnEnd": .bool(true), + "supportsCaptureDiagnostics": .bool(true), + ]) + } +} + +/// Enforces one explicit turn at a time on the single authenticated connection. +/// `ping` negotiates the protocol without opening a turn. Every other request +/// must keep the same identity until `turn_end` releases all turn-owned state. +struct DaemonTurnGate { + private(set) var active: ComputerUseDaemonProtocol.Metadata? + + mutating func admit( + _ metadata: ComputerUseDaemonProtocol.Metadata, + command: String + ) throws { + if !ComputerUseDaemonProtocol.isTurnScoped(command: command) { return } + if let active { + guard active == metadata else { + throw CUError( + "turn_mismatch", + "A different Computer Use turn is still active; finish it before starting another" + ) + } + return + } + active = metadata + } + + mutating func finish( + _ metadata: ComputerUseDaemonProtocol.Metadata + ) throws { + guard active == metadata else { + throw CUError("turn_mismatch", "Computer Use turn_end did not match the active turn") + } + active = nil + } + + mutating func reset() { + active = nil + } +} diff --git a/native/cu-helper/Sources/cu-helper/DisplaySleepAssertion.swift b/native/cu-helper/Sources/cu-helper/DisplaySleepAssertion.swift new file mode 100644 index 00000000..a1a987fd --- /dev/null +++ b/native/cu-helper/Sources/cu-helper/DisplaySleepAssertion.swift @@ -0,0 +1,50 @@ +import Foundation +import IOKit.pwr_mgt + +/// Keeps the display compositor awake while a Computer Use turn is active. +/// Covered-window ScreenCaptureKit consumers can otherwise stop receiving +/// useful updates once macOS enters user-idle display sleep during a long task. +@MainActor +final class ComputerUseDisplaySleepAssertion { + typealias AssertionID = IOPMAssertionID + + private let create: () -> AssertionID? + private let releaseAssertion: (AssertionID) -> Void + private var assertionID: AssertionID? + + init( + create: @escaping () -> AssertionID? = { + var assertionID: IOPMAssertionID = 0 + let result = IOPMAssertionCreateWithDescription( + kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString, + "Claude Code Haha Computer Use interaction" as CFString, + "Computer Use turn is controlling a background application" as CFString, + "Keeping the display awake while Computer Use is active" as CFString, + nil, + 0, + nil, + &assertionID + ) + return result == kIOReturnSuccess ? assertionID : nil + }, + release: @escaping (AssertionID) -> Void = { assertionID in + _ = IOPMAssertionRelease(assertionID) + } + ) { + self.create = create + self.releaseAssertion = release + } + + func acquire() { + guard assertionID == nil else { return } + assertionID = create() + } + + func release() { + guard let assertionID else { return } + self.assertionID = nil + releaseAssertion(assertionID) + } + + var isHeldForTesting: Bool { assertionID != nil } +} diff --git a/native/cu-helper/Sources/cu-helper/ForegroundLease.swift b/native/cu-helper/Sources/cu-helper/ForegroundLease.swift index 1a250fc9..b3fbe5ef 100644 --- a/native/cu-helper/Sources/cu-helper/ForegroundLease.swift +++ b/native/cu-helper/Sources/cu-helper/ForegroundLease.swift @@ -394,12 +394,13 @@ final class ForegroundLease { enum ForegroundMutationRunner { static func run( lease: ForegroundLease, + targetPID: pid_t? = nil, action: () async throws -> T ) async throws -> T { // Every input path crosses this boundary, including synthetic events // that never call AXAction.settle(). A throw can follow a partial // delivery, so neither success nor failure may reuse pre-action pixels. - defer { MutationClock.recordMutation() } + defer { MutationClock.recordMutation(pid: targetPID) } let result: Result do { result = .success(try await action()) diff --git a/native/cu-helper/Sources/cu-helper/JSONValue.swift b/native/cu-helper/Sources/cu-helper/JSONValue.swift index 2afadc59..d5beaf03 100644 --- a/native/cu-helper/Sources/cu-helper/JSONValue.swift +++ b/native/cu-helper/Sources/cu-helper/JSONValue.swift @@ -439,11 +439,30 @@ public struct Request: Decodable, Sendable { public let id: String? public let cmd: String public let payload: JSONValue? + public let clientApiVersion: String? + public let deadlineUnixMilliseconds: Int64? + public let sessionId: String? + public let turnId: String? + public let requestId: String? - public init(id: String?, cmd: String, payload: JSONValue?) { + public init( + id: String?, + cmd: String, + payload: JSONValue?, + clientApiVersion: String? = nil, + deadlineUnixMilliseconds: Int64? = nil, + sessionId: String? = nil, + turnId: String? = nil, + requestId: String? = nil + ) { self.id = id self.cmd = cmd self.payload = payload + self.clientApiVersion = clientApiVersion + self.deadlineUnixMilliseconds = deadlineUnixMilliseconds + self.sessionId = sessionId + self.turnId = turnId + self.requestId = requestId } /// The payload, defaulting to an empty object when omitted/null — every diff --git a/native/cu-helper/Sources/cu-helper/UISettlePolicy.swift b/native/cu-helper/Sources/cu-helper/UISettlePolicy.swift index 486b950a..2d34fdf4 100644 --- a/native/cu-helper/Sources/cu-helper/UISettlePolicy.swift +++ b/native/cu-helper/Sources/cu-helper/UISettlePolicy.swift @@ -14,18 +14,19 @@ import Foundation /// which it no longer does. Instead the wait happens once, at the moment of /// capture, and only if an action happened recently enough to still be settling. /// -/// This mirrors Codex's `needsUISettleBeforeSkyshot`: wait about a second after -/// a recent action, and extend that when the app is visibly still working. +/// This mirrors Codex's one-shot `needsUISettleBeforeSkyshot`: the next capture +/// for that target waits about 250ms after a recent action. A different app and +/// later captures do not inherit the wait. /// /// Pure and clock-injected so the decision is testable without sleeping. enum UISettlePolicy { /// How long after a mutation the UI is presumed to still be settling. - static let postActionWindow: TimeInterval = 1.0 + static let postActionWindow: TimeInterval = 0.25 - /// Ceiling while the app still shows a busy/progress indicator. Generous - /// because "still loading" is a real signal, but bounded so a permanently - /// spinning indicator (some apps never stop one) cannot hang the capture. - static let busyWindow: TimeInterval = 5.0 + /// Kept as a named compatibility seam for callers that already compute a + /// busy signal. Codex does not turn that signal into a multi-second sleep; + /// freshness comes from subsequent on-demand captures instead. + static let busyWindow: TimeInterval = postActionWindow /// Never wait less than this once we've decided to wait at all — a delay /// too short to cover a frame boundary is just latency for nothing. @@ -57,28 +58,52 @@ enum UISettlePolicy { } } -/// Records when the last mutating action completed, so the capture path can ask -/// "was something just changed?" without the action path having to block. +/// Records one pending capture settle per target process, so an action in one +/// app cannot delay a screenshot of another app. /// /// `@MainActor` because every mutation already runs there; this keeps the state /// single-threaded without a lock. @MainActor enum MutationClock { - private static var lastMutationAt: TimeInterval? + private static var pendingByPID: [pid_t: TimeInterval] = [:] + private static var unscopedMutationAt: TimeInterval? /// Called by the action path after a mutation lands. Does not sleep. - static func recordMutation(at time: TimeInterval = ProcessInfo.processInfo.systemUptime) { - lastMutationAt = time + static func recordMutation( + pid: pid_t? = nil, + at time: TimeInterval = ProcessInfo.processInfo.systemUptime + ) { + if let pid { + pendingByPID[pid] = time + } else { + unscopedMutationAt = time + } } - static func lastMutation() -> TimeInterval? { lastMutationAt } + static func lastMutation(pid: pid_t? = nil) -> TimeInterval? { + guard let pid else { return unscopedMutationAt } + return pendingByPID[pid] + } + + /// Consume the one-shot marker before waiting. Another capture cannot pay + /// the same action's settle cost a second time. + static func takeMutation(pid: pid_t? = nil) -> TimeInterval? { + guard let pid else { + defer { unscopedMutationAt = nil } + return unscopedMutationAt + } + return pendingByPID.removeValue(forKey: pid) + } /// Wait out whatever settle time the last mutation still owes. /// /// Uses `Task.sleep`, not `Thread.sleep`: this runs on the main actor, where /// blocking would starve the overlay's display link and freeze the virtual /// cursor animation mid-glide. - static func awaitSettle(appIsBusy: Bool) async { + static func awaitSettle( + lastMutationAt: TimeInterval?, + appIsBusy: Bool + ) async { let delay = UISettlePolicy.delay( now: ProcessInfo.processInfo.systemUptime, lastMutationAt: lastMutationAt, @@ -88,7 +113,10 @@ enum MutationClock { try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) } - static func reset() { lastMutationAt = nil } + static func reset() { + pendingByPID.removeAll() + unscopedMutationAt = nil + } static func resetForTests() { reset() } } diff --git a/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift b/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift index fd8d844c..4a5b94a5 100644 --- a/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift +++ b/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift @@ -172,7 +172,11 @@ final class WindowCaptureStreamManager: WindowCaptureProviding { invalidate() return nil } - guard let shot = await captureSnapshot(for: target, scale: scale) else { + guard let shot = await captureSnapshot( + for: target, + scale: scale, + newerThanUptime: newerThanUptime + ) else { return nil } if let preferredWindowID, @@ -213,7 +217,20 @@ final class WindowCaptureStreamManager: WindowCaptureProviding { /// Match the reference's two separate lifetimes: SCStream remains a /// consumer while covered; every state read runs an on-demand Skyshot/SCK /// capture. An idle stream's cached frame is not evidence of the current UI. - func captureSnapshot(for target: WindowCaptureStreamTarget, scale: Double) async -> WindowShot? { + func captureSnapshot( + for target: WindowCaptureStreamTarget, + scale: Double, + newerThanUptime: TimeInterval? = nil + ) async -> WindowShot? { + if let newerThanUptime { + // The stream is a long-lived render/freshness consumer, not the + // model screenshot source. Before the post-action Skyshot, observe + // a stream frame newer than the action when possible. `frame` + // performs one bounded rebuild for a silently starved stream; a + // static/no-op UI may legitimately emit no changed frame, so the + // authoritative on-demand screenshot still runs after the bound. + _ = await frame(for: target, newerThanUptime: newerThanUptime) + } for _ in 0..<2 { guard let source = await source(for: target) else { continue } if source.hasFailed { diff --git a/native/cu-helper/Tests/CuHelperTests/ClientAttestationTests.swift b/native/cu-helper/Tests/CuHelperTests/ClientAttestationTests.swift index cc81a871..54ac2b51 100644 --- a/native/cu-helper/Tests/CuHelperTests/ClientAttestationTests.swift +++ b/native/cu-helper/Tests/CuHelperTests/ClientAttestationTests.swift @@ -365,8 +365,8 @@ struct ClientAttestationTests { let allowed = [ "list_apps", "resolve_app_target", "get_app_state", "click", "set_value", "select_text", "perform_secondary_action", "scroll", - "drag", "press_key", "type_text", "ping", "shutdown", - "overlay_show", "overlay_hide", "check_permissions", + "drag", "press_key", "type_text", "paste", "ping", "shutdown", + "overlay_show", "overlay_hide", "turn_end", "check_permissions", "input_monitor_state", "held_input_state", ] for command in allowed { diff --git a/native/cu-helper/Tests/CuHelperTests/ClipboardPasteReceiptTests.swift b/native/cu-helper/Tests/CuHelperTests/ClipboardPasteReceiptTests.swift index 7338b560..c30fd8fa 100644 --- a/native/cu-helper/Tests/CuHelperTests/ClipboardPasteReceiptTests.swift +++ b/native/cu-helper/Tests/CuHelperTests/ClipboardPasteReceiptTests.swift @@ -5,6 +5,29 @@ import XCTest @testable import cc_haha_computer_use final class ClipboardPasteReceiptTests: XCTestCase { + @MainActor + func testHtmlPastePromisesRichAndPlainRepresentationsThenRestoresClipboard() async throws { + let fixture = PasteReceiptFixture() + defer { fixture.close() } + let lease = ClipboardLease(pasteboard: fixture.board) + + try await ClipboardPasteReceipt.perform( + text: "Hello world", + format: .html, + lease: lease + ) { validate in + try await fixture.sendPaste(validate) + XCTAssertEqual( + String(data: fixture.board.data(forType: .html) ?? Data(), encoding: .utf8), + "Hello world" + ) + XCTAssertTrue((fixture.board.string(forType: .string) ?? "").contains("Hello world")) + } + + XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.status, "completed") + XCTAssertEqual(fixture.board.string(forType: .string), "original") + } + @MainActor func testPasteWaitsForARealReadBeyondTheOld180MillisecondWindow() async throws { let fixture = PasteReceiptFixture() diff --git a/native/cu-helper/Tests/CuHelperTests/DaemonProtocolTests.swift b/native/cu-helper/Tests/CuHelperTests/DaemonProtocolTests.swift new file mode 100644 index 00000000..0adcd6e1 --- /dev/null +++ b/native/cu-helper/Tests/CuHelperTests/DaemonProtocolTests.swift @@ -0,0 +1,116 @@ +import XCTest +@testable import cc_haha_computer_use + +final class DaemonProtocolTests: XCTestCase { + private func request( + version: String? = ComputerUseDaemonProtocol.version, + deadline: Int64? = 2_000, + sessionId: String? = "session-a", + turnId: String? = "turn-a", + id: String? = "request-a", + requestId: String? = "request-a", + command: String = "get_app_state" + ) -> Request { + Request( + id: id, + cmd: command, + payload: .object([:]), + clientApiVersion: version, + deadlineUnixMilliseconds: deadline, + sessionId: sessionId, + turnId: turnId, + requestId: requestId + ) + } + + func testValidRequestCarriesVersionDeadlineAndTurnIdentity() throws { + let metadata = try ComputerUseDaemonProtocol.validate( + request(), + nowUnixMilliseconds: 1_000 + ) + + XCTAssertEqual(metadata.sessionId, "session-a") + XCTAssertEqual(metadata.turnId, "turn-a") + } + + func testProtocolMismatchAndExpiredDeadlineFailClosed() { + XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate( + request(version: "CCHahaComputerUseIPC-1"), + nowUnixMilliseconds: 1_000 + )) { error in + XCTAssertEqual((error as? CUError)?.code, "protocol_mismatch") + } + + XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate( + request(deadline: 1_000), + nowUnixMilliseconds: 1_000 + )) { error in + XCTAssertEqual((error as? CUError)?.code, "deadline_exceeded") + } + } + + func testRequestIdentityAndTurnMetadataCannotBeOmitted() { + XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate( + request(requestId: "different"), + nowUnixMilliseconds: 1_000 + )) { error in + XCTAssertEqual((error as? CUError)?.code, "bad_request_id") + } + + XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate( + request(turnId: " "), + nowUnixMilliseconds: 1_000 + )) { error in + XCTAssertEqual((error as? CUError)?.code, "missing_turn_metadata") + } + } + + func testTurnGateRejectsCrossTurnStateUntilMatchingEnd() throws { + let first = ComputerUseDaemonProtocol.Metadata( + sessionId: "session-a", + turnId: "turn-a" + ) + let second = ComputerUseDaemonProtocol.Metadata( + sessionId: "session-a", + turnId: "turn-b" + ) + var gate = DaemonTurnGate() + + try gate.admit(first, command: "get_app_state") + try gate.admit(first, command: "click") + XCTAssertThrowsError(try gate.admit(second, command: "get_app_state")) { + XCTAssertEqual(($0 as? CUError)?.code, "turn_mismatch") + } + try gate.finish(first) + try gate.admit(second, command: "get_app_state") + XCTAssertEqual(gate.active, second) + } + + func testPingNegotiatesWithoutOpeningATurn() throws { + let metadata = ComputerUseDaemonProtocol.Metadata( + sessionId: "session-a", + turnId: "handshake-1" + ) + var gate = DaemonTurnGate() + + try gate.admit(metadata, command: "ping") + + XCTAssertNil(gate.active) + XCTAssertEqual( + ComputerUseDaemonProtocol.hello()["protocolVersion"]?.asString, + ComputerUseDaemonProtocol.version + ) + } + + func testPermissionProbeIsConnectionScopedAndDoesNotOpenATurn() throws { + let metadata = ComputerUseDaemonProtocol.Metadata( + sessionId: "session-a", + turnId: "probe" + ) + var gate = DaemonTurnGate() + + try gate.admit(metadata, command: "check_permissions") + + XCTAssertNil(gate.active) + } +} diff --git a/native/cu-helper/Tests/CuHelperTests/DisplaySleepAssertionTests.swift b/native/cu-helper/Tests/CuHelperTests/DisplaySleepAssertionTests.swift new file mode 100644 index 00000000..0145eb28 --- /dev/null +++ b/native/cu-helper/Tests/CuHelperTests/DisplaySleepAssertionTests.swift @@ -0,0 +1,39 @@ +import XCTest +@testable import cc_haha_computer_use + +final class DisplaySleepAssertionTests: XCTestCase { + @MainActor + func testTurnAssertionIsAcquiredAndReleasedExactlyOnce() { + var creates = 0 + var releases: [UInt32] = [] + let assertion = ComputerUseDisplaySleepAssertion( + create: { + creates += 1 + return 42 + }, + release: { releases.append($0) } + ) + + assertion.acquire() + assertion.acquire() + XCTAssertTrue(assertion.isHeldForTesting) + XCTAssertEqual(creates, 1) + + assertion.release() + assertion.release() + XCTAssertFalse(assertion.isHeldForTesting) + XCTAssertEqual(releases, [42]) + + assertion.acquire() + XCTAssertEqual(creates, 2) + } + + @MainActor + func testFailedPowerAssertionDoesNotPretendItIsHeld() { + let assertion = ComputerUseDisplaySleepAssertion(create: { nil }) + + assertion.acquire() + + XCTAssertFalse(assertion.isHeldForTesting) + } +} diff --git a/native/cu-helper/Tests/CuHelperTests/UISettlePolicyTests.swift b/native/cu-helper/Tests/CuHelperTests/UISettlePolicyTests.swift index 2c0417bc..66ab352f 100644 --- a/native/cu-helper/Tests/CuHelperTests/UISettlePolicyTests.swift +++ b/native/cu-helper/Tests/CuHelperTests/UISettlePolicyTests.swift @@ -5,7 +5,7 @@ import XCTest final class UISettlePolicyTests: XCTestCase { /// Nothing has been mutated, so nothing is mid-transition on our account. /// This is the common case for the first `get_app_state` of a session and - /// must not cost the model a second of latency. + /// must not add capture latency. func testNoMutationMeansNoWait() { XCTAssertEqual( UISettlePolicy.delay(now: 100, lastMutationAt: nil, appIsBusy: false), @@ -33,12 +33,12 @@ final class UISettlePolicyTests: XCTestCase { ) } - /// A visible progress indicator extends the window — but only to a bound, - /// so an app that spins forever cannot hang the capture. - func testBusyAppGetsTheLongerWindowButStaysBounded() { + /// A permanently spinning progress indicator must not turn the one-shot + /// settle into a multi-second stall. + func testBusyAppUsesTheSameBoundedOneShotWindow() { let busy = UISettlePolicy.delay(now: 100, lastMutationAt: 98.9, appIsBusy: true) - XCTAssertGreaterThan(busy, 0) - XCTAssertLessThanOrEqual(busy, UISettlePolicy.busyWindow) + XCTAssertEqual(busy, 0) + XCTAssertEqual(UISettlePolicy.busyWindow, UISettlePolicy.postActionWindow) XCTAssertEqual( UISettlePolicy.delay( @@ -67,4 +67,16 @@ final class UISettlePolicyTests: XCTestCase { XCTAssertLessThanOrEqual(delay, UISettlePolicy.postActionWindow) XCTAssertGreaterThanOrEqual(delay, 0) } + + @MainActor + func testMutationMarkerIsScopedByPIDAndConsumedOnce() { + MutationClock.resetForTests() + defer { MutationClock.resetForTests() } + + MutationClock.recordMutation(pid: 101, at: 42) + + XCTAssertNil(MutationClock.takeMutation(pid: 202)) + XCTAssertEqual(MutationClock.takeMutation(pid: 101), 42) + XCTAssertNil(MutationClock.takeMutation(pid: 101)) + } } diff --git a/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift b/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift index c1ed0627..4c406a0a 100644 --- a/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift +++ b/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift @@ -132,6 +132,35 @@ final class WindowCaptureStreamTests: XCTestCase { XCTAssertEqual(factory.sources[0].latestReadCount, 0, "A cached stream frame must not become the model's screenshot") } + func testPostMutationSnapshotConsumesFreshStreamWatermarkBeforeSkyshot() async throws { + let target = makeTarget(windowID: 87) + let factory = FakeWindowCaptureStreamFactory { source, _ in + source.startFrame = makeFrame( + for: source.targetKey, + sequence: 1, + uptime: 12, + byte: 7 + ) + } + var captures = 0 + let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in + captures += 1 + return self.makeSnapshot(target, pixels: "fresh-skyshot") + }) + + let shot = await manager.captureSnapshot( + for: target, + scale: 0.5, + newerThanUptime: 11 + ) + + XCTAssertEqual(shot?.base64, "fresh-skyshot") + XCTAssertEqual(captures, 1) + XCTAssertEqual(factory.sources.count, 1) + XCTAssertGreaterThan(factory.sources[0].latestReadCount, 0) + XCTAssertEqual(factory.sources[0].retireCount, 0) + } + func testSnapshotFailureDoesNotFallBackToCachedStreamPixels() async { let target = makeTarget(windowID: 85) let factory = FakeWindowCaptureStreamFactory { source, _ in @@ -171,9 +200,10 @@ final class WindowCaptureStreamTests: XCTestCase { let target = makeTarget(windowID: 81) let factory = FakeWindowCaptureStreamFactory { _, _ in } var captures = 0 + var captureTimes: [TimeInterval] = [] let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in captures += 1 - self.assertMutationHasSettledBeforeCapture() + captureTimes.append(ProcessInfo.processInfo.systemUptime) return self.makeSnapshot(target, pixels: "unchanged-pixels") }) var previousMutation: TimeInterval? @@ -183,11 +213,20 @@ final class WindowCaptureStreamTests: XCTestCase { XCTAssertNotNil(MutationClock.lastMutation()) XCTAssertNotEqual(MutationClock.lastMutation(), previousMutation) previousMutation = MutationClock.lastMutation() - let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) { + let pendingMutation = MutationClock.takeMutation() + let shot = await CommandRouter.captureSettledWindowShot( + appIsBusy: false, + lastMutationAt: pendingMutation + ) { await manager.captureSnapshot(for: target, scale: 0.5) } XCTAssertEqual(shot?.base64, "unchanged-pixels") XCTAssertEqual(captures, expectedCaptureCount, "Identical pixels still require a new capture") + self.assertMutationHasSettledBeforeCapture( + pendingMutation, + capturedAt: captureTimes[expectedCaptureCount - 1] + ) + XCTAssertNil(MutationClock.lastMutation(), "The settle marker is one-shot") } XCTAssertEqual(factory.sources.count, 1) XCTAssertEqual(factory.sources[0].latestReadCount, 0) @@ -198,8 +237,9 @@ final class WindowCaptureStreamTests: XCTestCase { defer { MutationClock.resetForTests() } let target = makeTarget(windowID: 83) let factory = FakeWindowCaptureStreamFactory { _, _ in } + var capturedAt: TimeInterval? let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in - self.assertMutationHasSettledBeforeCapture() + capturedAt = ProcessInfo.processInfo.systemUptime return self.makeSnapshot(target, pixels: "partial-action-state") }) do { @@ -211,18 +251,29 @@ final class WindowCaptureStreamTests: XCTestCase { XCTAssertEqual(error.code, "partial_action") } XCTAssertNotNil(MutationClock.lastMutation()) - let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) { + let pendingMutation = MutationClock.takeMutation() + let shot = await CommandRouter.captureSettledWindowShot( + appIsBusy: false, + lastMutationAt: pendingMutation + ) { await manager.captureSnapshot(for: target, scale: 0.5) } XCTAssertEqual(shot?.base64, "partial-action-state") + self.assertMutationHasSettledBeforeCapture( + pendingMutation, + capturedAt: try XCTUnwrap(capturedAt) + ) XCTAssertEqual(factory.sources[0].latestReadCount, 0) } - private func assertMutationHasSettledBeforeCapture() { - XCTAssertNotNil(MutationClock.lastMutation()) + private func assertMutationHasSettledBeforeCapture( + _ mutationAt: TimeInterval?, + capturedAt: TimeInterval + ) { + XCTAssertNotNil(mutationAt) XCTAssertEqual(UISettlePolicy.delay( - now: ProcessInfo.processInfo.systemUptime, - lastMutationAt: MutationClock.lastMutation(), + now: capturedAt, + lastMutationAt: mutationAt, appIsBusy: false ), 0, "The actual screenshot callback must run after the action's settle deadline") } diff --git a/native/cu-helper/build.sh b/native/cu-helper/build.sh index 81b2991d..7f6ced2e 100755 --- a/native/cu-helper/build.sh +++ b/native/cu-helper/build.sh @@ -57,6 +57,8 @@ BIN_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use" # Accessibility tolerates a bare binary (works), Screen Recording does NOT. So # the shipped/dragged artifact is the .app; the inner binary is what we spawn. APP_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use.app" +# Reuse the desktop brand asset so both Privacy lists show the product logo. +APP_ICON_PATH="$PKG_DIR/../../desktop/src-tauri/icons/icon.icns" # Records the (identity, identifier) actually used, so we can detect rotation # across rebuilds and warn that TCC grants will have been dropped. SIGN_STAMP="$BUILD_DIR/.cu-helper.signid" @@ -407,6 +409,7 @@ verify() { # makes the inner binary's TCC identity a proper app bundle. # --------------------------------------------------------------------------- wrap_app() { + [ -s "$APP_ICON_PATH" ] || die "App icon not found at $APP_ICON_PATH (needed for the Privacy lists)." log "" log "==> wrap .app bundle: $APP_PATH" rm -rf "$APP_PATH" @@ -416,6 +419,7 @@ wrap_app() { [ -f "$PKG_DIR/Info.plist" ] || die "Info.plist not found at $PKG_DIR/Info.plist (needed for the .app bundle)." cp "$PKG_DIR/Info.plist" "$APP_PATH/Contents/Info.plist" + cp "$APP_ICON_PATH" "$APP_PATH/Contents/Resources/icon.icns" # SwiftPM resource bundle (LensSequence overlay), loaded via Bundle.module. # Standard .app location is Contents/Resources/ (Bundle.main.resourceURL). Do diff --git a/native/cu-helper/build.test.ts b/native/cu-helper/build.test.ts index 64435d60..c7921115 100644 --- a/native/cu-helper/build.test.ts +++ b/native/cu-helper/build.test.ts @@ -1,7 +1,59 @@ -import { describe, expect, test } from 'bun:test' +import { afterEach, describe, expect, test } from 'bun:test' +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' import path from 'node:path' const buildScript = path.resolve(import.meta.dirname, 'build.sh') +const productIcon = path.resolve(import.meta.dirname, '../../desktop/src-tauri/icons/icon.icns') +const fixtureDirectories: string[] = [] + +afterEach(() => { + for (const directory of fixtureDirectories.splice(0)) { + rmSync(directory, { recursive: true, force: true }) + } +}) + +function wrapFixtureApp(missingIcon = false) { + const directory = mkdtempSync(path.join(tmpdir(), 'cu-helper-app-icon-')) + fixtureDirectories.push(directory) + writeFileSync(path.join(directory, 'fixture-binary'), 'fixture executable') + + const result = Bun.spawnSync([ + 'bash', + '-c', + ` +source "$1" +TEST_BUNDLE_DIR="$2" +BUILD_DIR="$TEST_BUNDLE_DIR/build" +BIN_PATH="$TEST_BUNDLE_DIR/fixture-binary" +APP_PATH="$TEST_BUNDLE_DIR/cc-haha-computer-use.app" +BUNDLE_ID="dev.cchaha.cu-helper" +SIGN_IDENTITY="fixture-only" +RESOLVED_TIMESTAMP_MODE="none" +if [ "$3" = "missing" ]; then + APP_ICON_PATH="$TEST_BUNDLE_DIR/missing.icns" +fi +codesign() { + case "$1" in + --force) cp -R "$APP_PATH/Contents" "$TEST_BUNDLE_DIR/contents-at-sign" ;; + -dv) printf 'Identifier=%s\\n' "$BUNDLE_ID" ;; + esac +} +wrap_app +`, + 'cu-helper-app-icon-test', + buildScript, + directory, + missingIcon ? 'missing' : 'present', + ], { cwd: directory }) + + return { + directory, + contents: path.join(directory, 'cc-haha-computer-use.app', 'Contents'), + exitCode: result.exitCode, + stderr: result.stderr.toString(), + } +} function resolveTimestampArgument(identity: string, mode = 'auto') { const result = Bun.spawnSync([ @@ -74,3 +126,35 @@ describe('cu-helper build signing identity', () => { expect(result.stdout).toBe('Developer ID Application: Example Corp (TEAM123456)') }) }) + +describe.skipIf(process.platform !== 'darwin')('cu-helper permission-list app icon', () => { + test('declares and bundles the product icon before signing the helper app', () => { + const result = wrapFixtureApp() + expect(result.exitCode).toBe(0) + + const plist = Bun.spawnSync([ + '/usr/bin/plutil', '-convert', 'json', '-o', '-', + path.join(result.contents, 'Info.plist'), + ]) + expect(plist.exitCode).toBe(0) + const info = JSON.parse(plist.stdout.toString()) + expect(info.CFBundleIdentifier).toBe('dev.cchaha.cu-helper') + expect(info.CFBundleExecutable).toBe('cc-haha-computer-use') + expect(info.CFBundleIconFile).toBe('icon.icns') + + const expectedIcon = readFileSync(productIcon) + expect(expectedIcon.subarray(0, 4).toString()).toBe('icns') + expect(readFileSync(path.join(result.contents, 'Resources', info.CFBundleIconFile))) + .toEqual(expectedIcon) + expect(readFileSync(path.join(result.directory, 'contents-at-sign', 'Resources', info.CFBundleIconFile))) + .toEqual(expectedIcon) + }) + + test('refuses to sign an app when the required product icon is missing', () => { + const result = wrapFixtureApp(true) + + expect(result.exitCode).not.toBe(0) + expect(result.stderr).toContain('App icon not found') + expect(existsSync(path.join(result.directory, 'contents-at-sign'))).toBe(false) + }) +}) diff --git a/src/skills/bundled/computerUse.test.ts b/src/skills/bundled/computerUse.test.ts index 1e5df566..cd383734 100644 --- a/src/skills/bundled/computerUse.test.ts +++ b/src/skills/bundled/computerUse.test.ts @@ -9,7 +9,7 @@ import { /** * Asserting on prose is unusual, but this prose is load-bearing twice over: it - * is the only thing that carries the operating procedure the ten tools assume, + * is the only thing that carries the operating procedure the tool set assumes, * and it is the only place that says which clicks must not be made without * asking. Each test names the failure it prevents so anyone trimming a line can * see what it was buying. @@ -24,22 +24,29 @@ describe('computer-use skill content', () => { // success, and reported the task done while the app had not changed. const prompt = await computerUsePrompt() expect(prompt).toContain('dispatched') - expect(prompt).toContain('no change in the accessibility tree') + expect(prompt).toContain('AX diff stays empty') + expect(prompt).toContain('Judge the screenshot') }) - test('names the four tools that still work on a dead tree', async () => { + test('names the tools that still work on a dead tree', async () => { // Observed: on a Chromium app whose tree is a bare shell, the model clicked // element handles fifteen times and never tried the screenshot coordinates // it already had. "The tree is empty" alone is not actionable — the escape // hatch has to be enumerated. const prompt = await computerUsePrompt() expect(prompt).toContain('will never fill in') - for (const tool of ['click', 'drag', 'press_key', 'type_text']) { + for (const tool of ['click', 'drag', 'press_key', 'type_text', 'paste']) { expect(prompt).toContain(tool) } expect(prompt).toContain('menu bar') }) + test('treats a timed-out paste as result-unknown and refreshes before retry', async () => { + const prompt = await computerUsePrompt() + expect(prompt).toContain('may have consumed the paste late') + expect(prompt).toContain('get_app_state') + }) + test('caps repetition and closes the shell escape hatch', async () => { // Observed: the same failing click repeated many times, then the model // abandoned the toolset for osascript and Python, burning the session. diff --git a/src/skills/bundled/computerUse.ts b/src/skills/bundled/computerUse.ts index c8ec2b14..d9402acf 100644 --- a/src/skills/bundled/computerUse.ts +++ b/src/skills/bundled/computerUse.ts @@ -13,6 +13,7 @@ const MAC_COMPUTER_USE_TOOLS = [ 'drag', 'press_key', 'type_text', + 'paste', ].map(name => `mcp__computer-use__${name}`) export function getComputerUseToolAllowlist( @@ -71,9 +72,9 @@ Switch to \`x\`/\`y\` read off the screenshot when either is true: - **the tree is a dead end.** Many Chromium/Electron apps expose only their window frame and menu bar. \`get_app_state\` says so explicitly when it detects - this. That is not a slow-loading tree — it will never fill in. Only four tools + this. That is not a slow-loading tree — it will never fill in. Five tools still work there: \`click\` with x/y, \`drag\` with x/y, \`press_key\`, and - \`type_text\`. Everything else needs a handle it cannot get. The menu bar stays + \`type_text\`/\`paste\`. Everything else needs a handle it cannot get. The menu bar stays fully addressable, so a menu path is often the shortest route. - **element actions run but the UI does not change.** @@ -84,11 +85,10 @@ them as-is; do not convert them. Mutating tools return a fixed receipt. The receipt means the action was **dispatched**, never that it had the intended effect. Only the next -\`get_app_state\` tells you what actually happened — and an empty diff, or the -line "There has been no change in the accessibility tree", means your action did -nothing. +\`get_app_state\` tells you what actually happened. Judge the screenshot as well +as AX text: Chromium/CEF content can visibly change while the AX diff stays empty. -If two consecutive attempts leave the state unchanged, the approach is wrong. +If two consecutive screenshots leave the relevant UI unchanged, the approach is wrong. Change something real: switch from handle to coordinates, target a different element, re-read the full tree with \`disableDiff: true\`, or take a different route through the UI such as the menu bar. Repeating the same call a third time @@ -109,6 +109,10 @@ session. element in the tree. Do not guess action names. - \`press_key\` and \`type_text\` are delivered to the named app, so they cannot trigger global system shortcuts. +- If \`type_text\` does not visibly change a Chromium/CEF field, use + \`paste({ app, text, format: "text" })\`. It restores the user's prior clipboard. + If it times out after dispatch, call \`get_app_state\` before retrying: the app + may have consumed the paste late. - \`press_key\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c". - \`select_text\` works inside editable elements; use \`prefix\`/\`suffix\` to disambiguate repeated matches. diff --git a/src/utils/computerUse/cuHelperDaemon.test.ts b/src/utils/computerUse/cuHelperDaemon.test.ts index 52dcc56a..7b953f57 100644 --- a/src/utils/computerUse/cuHelperDaemon.test.ts +++ b/src/utils/computerUse/cuHelperDaemon.test.ts @@ -7,8 +7,10 @@ import { __prepareDaemonSocketDirectoryForTests, __resetDaemonClientForTests, __daemonStartCountForTests, + __negotiateDaemonProtocolForTests, __reapStaleDaemonsForTests, __setDaemonSocketForTests, + CU_HELPER_PROTOCOL_VERSION, callDaemon, DaemonCommandTimeoutError, DaemonCommandResultUnknownError, @@ -185,6 +187,76 @@ describe('cu-helper daemon failure classification', () => { expect(error.message).toBe('grant_flag_required') }) + test('every request carries negotiated protocol, deadline, and stable turn identity', async () => { + const socket = new FakeSocket() + __setDaemonSocketForTests(socket as never, 100) + + const first = callDaemon('get_app_state', { app: 'TextEdit' }) + await waitForWriteCount(socket, 1) + const firstEnvelope = JSON.parse(socket.writes[0]!) + const firstTurnId = firstEnvelope.turnId + expect(firstEnvelope).toMatchObject({ + clientApiVersion: CU_HELPER_PROTOCOL_VERSION, + requestId: firstEnvelope.id, + sessionId: expect.any(String), + turnId: expect.any(String), + }) + expect(firstEnvelope.deadlineUnixMilliseconds).toBeGreaterThan(Date.now()) + reply(socket, 0, { ok: true, result: {} }) + await first + + const second = callDaemon('click', { app: 'TextEdit', x: 1, y: 1 }) + await waitForWriteCount(socket, 2) + const secondEnvelope = JSON.parse(socket.writes[1]!) + expect(secondEnvelope.turnId).toBe(firstTurnId) + reply(socket, 1, { ok: true, result: true }) + await second + }) + + test('accepts only a daemon hello with the complete negotiated capability set', async () => { + const socket = new FakeSocket() + __setDaemonSocketForTests(socket as never, 100) + + const negotiation = __negotiateDaemonProtocolForTests() + await waitForWrite(socket) + expect(JSON.parse(socket.writes[0]!)).toMatchObject({ + cmd: 'ping', + clientApiVersion: CU_HELPER_PROTOCOL_VERSION, + }) + reply(socket, 0, { + ok: true, + result: { + protocolVersion: CU_HELPER_PROTOCOL_VERSION, + supportsAbsoluteDeadlines: true, + supportsTurnEnd: true, + }, + }) + + await expect(negotiation).resolves.toBeUndefined() + expect(socket.destroyed).toBe(false) + }) + + test('rejects and retires a daemon that reports an incomplete hello', async () => { + const socket = new FakeSocket() + __setDaemonSocketForTests(socket as never, 100) + + const negotiation = __negotiateDaemonProtocolForTests().catch(err => err) + await waitForWrite(socket) + reply(socket, 0, { + ok: true, + result: { + protocolVersion: CU_HELPER_PROTOCOL_VERSION, + supportsAbsoluteDeadlines: true, + supportsTurnEnd: false, + }, + }) + + const error = await negotiation + expect(error).toBeInstanceOf(DaemonUnavailableError) + expect(error.message).toMatch(/protocol negotiation failed.*unexpected hello/i) + expect(socket.destroyed).toBe(true) + }) + test('post-dispatch timeout is ambiguous and must not be classified as replayable infrastructure', async () => { const socket = new FakeSocket() __setDaemonSocketForTests(socket as never, 5) @@ -279,13 +351,20 @@ describe('cu-helper overlay reconciliation', () => { reply(socket, 0, { ok: true, result: true }) await waitForWriteCount(socket, 2) expect(JSON.parse(socket.writes[1]!)).toMatchObject({ - cmd: 'overlay_hide', + cmd: 'turn_end', payload: {}, }) reply(socket, 1, { ok: true, result: true }) await Promise.all([show, hide]) expect(isOverlayShown()).toBe(false) + + const next = callDaemon('get_app_state', { app: 'TextEdit' }) + await waitForWriteCount(socket, 3) + expect(JSON.parse(socket.writes[2]!).turnId) + .not.toBe(JSON.parse(socket.writes[0]!).turnId) + reply(socket, 2, { ok: true, result: {} }) + await next }) test('a target change while show is pending serially retargets to the latest app', async () => { @@ -318,10 +397,68 @@ describe('cu-helper overlay reconciliation', () => { await show expect(isOverlayShown()).toBe(false) + const hide = overlayHide() + await waitForWriteCount(socket, 2) + expect(JSON.parse(socket.writes[1]!)).toMatchObject({ cmd: 'turn_end' }) + reply(socket, 1, { ok: true, result: true }) + await hide + }) + + test('cleanup ends a read-only turn even when no overlay was shown', async () => { + const socket = new FakeSocket() + __setDaemonSocketForTests(socket as never) + + const state = callDaemon('get_app_state', { app: 'TextEdit' }) + await waitForWriteCount(socket, 1) + const stateEnvelope = JSON.parse(socket.writes[0]!) + reply(socket, 0, { ok: true, result: {} }) + await state + + const hide = overlayHide() + await waitForWriteCount(socket, 2) + const endEnvelope = JSON.parse(socket.writes[1]!) + expect(endEnvelope).toMatchObject({ cmd: 'turn_end', turnId: stateEnvelope.turnId }) + reply(socket, 1, { ok: true, result: true }) + await hide + + const next = callDaemon('get_app_state', { app: 'TextEdit' }) + await waitForWriteCount(socket, 3) + expect(JSON.parse(socket.writes[2]!).turnId).not.toBe(stateEnvelope.turnId) + reply(socket, 2, { ok: true, result: {} }) + await next + }) + + test('connection-scoped permission checks do not manufacture a turn to clean up', async () => { + const socket = new FakeSocket() + __setDaemonSocketForTests(socket as never) + + const check = callDaemon('check_permissions', {}) + await waitForWriteCount(socket, 1) + expect(JSON.parse(socket.writes[0]!).turnId).toMatch(/^connection-/) + reply(socket, 0, { ok: true, result: {} }) + await check + await overlayHide() expect(socket.writes).toHaveLength(1) }) + test('a rejected turn_end retires the daemon instead of poisoning the next turn', async () => { + const socket = new FakeSocket() + __setDaemonSocketForTests(socket as never) + + const state = callDaemon('get_app_state', { app: 'TextEdit' }) + await waitForWriteCount(socket, 1) + reply(socket, 0, { ok: true, result: {} }) + await state + + const hide = overlayHide() + await waitForWriteCount(socket, 2) + reply(socket, 1, { ok: false, error: { message: 'deadline_exceeded' } }) + await hide + + expect(socket.destroyed).toBe(true) + }) + test('cleanup with no daemon does not start one', async () => { expect(__daemonStartCountForTests()).toBe(0) await overlayHide() diff --git a/src/utils/computerUse/cuHelperDaemon.ts b/src/utils/computerUse/cuHelperDaemon.ts index 659e00aa..271c3cb2 100644 --- a/src/utils/computerUse/cuHelperDaemon.ts +++ b/src/utils/computerUse/cuHelperDaemon.ts @@ -1,7 +1,9 @@ import { spawn, spawnSync, type ChildProcess } from 'node:child_process' +import { randomUUID } from 'node:crypto' import fs from 'node:fs' import net from 'node:net' import path from 'node:path' +import { getSessionId } from '../../bootstrap/state.js' import { logForDebugging } from '../debug.js' import { ensureInstalledHelper } from './cuHelperInstall.js' import { attestDaemonSocketPeer } from './cuHelperPeerAttestation.js' @@ -13,10 +15,10 @@ import { getRuntimePaths } from './pythonBridge.js' * The daemon owns the main run loop that the animated virtual cursor and the * animated virtual cursor needs, and holds the virtual cursor's position + * held-input state across commands. We spawn ONE daemon per CLI process, keep - * an AF_UNIX socket open to it, and speak the NDJSON request/response protocol: + * an AF_UNIX socket open to it, and speak a versioned NDJSON protocol: * - * readiness : {"ready":true,"pid":N,"proto":1}\n (the daemon's stdout, once) - * request : {"id":"","cmd":"","payload":{...}}\n + * readiness : {"ready":true,"pid":N,"protocolVersion":"..."}\n + * request : {"id":"","requestId":"","cmd":"",...}\n * response : {"id":"","ok":true,"result":...}\n | {"id":"","ok":false,"error":{...}}\n * * Routing through the daemon (instead of one-shot CLI) is what makes execution @@ -31,6 +33,7 @@ import { getRuntimePaths } from './pythonBridge.js' const REQUEST_TIMEOUT_MS = 20_000 const READINESS_TIMEOUT_MS = 8_000 +export const CU_HELPER_PROTOCOL_VERSION = 'CCHahaComputerUseIPC-2' /** * Thrown ONLY for daemon INFRASTRUCTURE failures known to happen before the @@ -107,6 +110,7 @@ let overlayActualKey: string | undefined let overlayRevision = 0 let overlayReconcilePromise: Promise | undefined let requestTimeoutMs = REQUEST_TIMEOUT_MS +let activeTurnId: string | undefined function socketPath(generation: number): string { const { runtimeStateRoot } = getRuntimePaths() @@ -379,6 +383,7 @@ function resetState(reason: string, expectedGeneration?: number): void { overlayActualVisible = false overlayActualKey = undefined overlayRevision++ + activeTurnId = undefined const p = statePromise statePromise = undefined activeDaemonGeneration = undefined @@ -513,6 +518,8 @@ async function startDaemon(generation: number): Promise { attachSocketHandlers(state) + await negotiateDaemonProtocol(state) + return state } @@ -542,6 +549,31 @@ function attachSocketHandlers(state: DaemonState): void { state.socket.on('error', err => resetState(`socket error: ${String(err)}`, state.generation)) } +async function negotiateDaemonProtocol(state: DaemonState): Promise { + // Socket ownership + code-signature attestation identify the peer. The + // versioned hello additionally proves that the installed helper understands + // deadlines and explicit turn cleanup before we dispatch any real command. + try { + const hello = await dispatchDaemonCommand<{ + protocolVersion?: string + supportsAbsoluteDeadlines?: boolean + supportsTurnEnd?: boolean + }>(state, 'ping', {}) + if ( + hello.protocolVersion !== CU_HELPER_PROTOCOL_VERSION + || hello.supportsAbsoluteDeadlines !== true + || hello.supportsTurnEnd !== true + ) { + throw new Error(`unexpected hello ${JSON.stringify(hello)}`) + } + } catch (err) { + state.socket.destroy() + throw new DaemonUnavailableError( + `cu-helper protocol negotiation failed: ${err instanceof Error ? err.message : String(err)}`, + ) + } +} + async function ensureDaemon(): Promise { if (!statePromise) { const generation = ++daemonGeneration @@ -573,6 +605,19 @@ function dispatchDaemonCommand( )) } const id = String(++state.nextId) + const isTurnScoped = !['ping', 'check_permissions', 'shutdown'].includes(command) + const turnId = activeTurnId + ?? (isTurnScoped ? (activeTurnId = randomUUID()) : `connection-${state.generation}`) + const request = { + id, + requestId: id, + cmd: command, + payload, + clientApiVersion: CU_HELPER_PROTOCOL_VERSION, + deadlineUnixMilliseconds: Date.now() + requestTimeoutMs, + sessionId: getSessionId(), + turnId, + } return new Promise((resolve, reject) => { const timer = setTimeout(() => { state.pending.delete(id) @@ -588,7 +633,7 @@ function dispatchDaemonCommand( }, requestTimeoutMs) state.pending.set(id, { resolve: resolve as (v: unknown) => void, reject, timer }) try { - state.socket.write(`${JSON.stringify({ id, cmd: command, payload })}\n`) + state.socket.write(`${JSON.stringify(request)}\n`) } catch (err) { clearTimeout(timer) state.pending.delete(id) @@ -597,6 +642,10 @@ function dispatchDaemonCommand( reject(new DaemonUnavailableError(err instanceof Error ? err.message : String(err))) resetState(`socket write failed: ${String(err)}`, state.generation) } + }).finally(() => { + if ((command === 'turn_end' || command === 'overlay_hide') && activeTurnId === turnId) { + activeTurnId = undefined + } }) } @@ -627,7 +676,10 @@ async function callExistingDaemon( } function needsOverlayReconciliation(): boolean { - if (!overlayDesiredVisible) return overlayActualVisible + // A turn may contain only get_app_state and therefore never show the overlay. + // Its native state (capture stream, AX baseline, held-input guard and display + // sleep assertion) still needs an explicit turn_end at host cleanup. + if (!overlayDesiredVisible) return overlayActualVisible || activeTurnId !== undefined return !overlayActualVisible || overlayActualKey !== overlayDesiredKey } @@ -662,16 +714,21 @@ async function reconcileOverlay(): Promise { // A turn-end cleanup must never launch a daemon just to hide it. When a // pending show completes this branch sees the already-owned daemon and - // serially sends the compensating hide. + // serially ends the turn. This also covers read-only turns whose overlay + // was never visible. if (!statePromise || activeDaemonGeneration === undefined) { overlayActualVisible = false overlayActualKey = undefined return } try { - await callExistingDaemon('overlay_hide', {}) + await callExistingDaemon('turn_end', {}) } catch (err) { - logForDebugging(`cu-helper overlay_hide failed: ${String(err)}`, { level: 'debug' }) + logForDebugging(`cu-helper turn_end failed: ${String(err)}`, { level: 'debug' }) + // If the helper rejected/lost turn_end it may still own the old turn. + // Retire it now so the next user turn cannot inherit stale native state + // or fail forever with turn_mismatch. + resetState('turn_end failed') } overlayActualVisible = false overlayActualKey = undefined @@ -740,6 +797,7 @@ export function __resetDaemonClientForTests(): void { overlayReconcilePromise = undefined daemonStartCount = 0 requestTimeoutMs = REQUEST_TIMEOUT_MS + activeTurnId = undefined } /** Focused proof that a no-daemon cleanup did not enter the startup path. */ @@ -775,3 +833,8 @@ export function __setDaemonSocketForTests( attachSocketHandlers(state) statePromise = Promise.resolve(state) } + +/** Drive the same hello negotiation used by production against an installed test socket. */ +export async function __negotiateDaemonProtocolForTests(): Promise { + await negotiateDaemonProtocol(await ensureDaemon()) +} diff --git a/src/utils/computerUse/executor.codexEngine.test.ts b/src/utils/computerUse/executor.codexEngine.test.ts index 1dbe900d..1eb6e804 100644 --- a/src/utils/computerUse/executor.codexEngine.test.ts +++ b/src/utils/computerUse/executor.codexEngine.test.ts @@ -264,6 +264,15 @@ describe('CLI executor Codex engine — daemon payload alignment', () => { expect(lastCall()).toEqual({ command: 'type_text', payload: { text: 'hello' } }) }) + itEngine('paste sends explicit content format', async () => { + const exec = await loadExecutor() + await exec.engine!.paste({ target: { app: 'NeteaseMusic' }, text: '喜欢你', format: 'text' }) + expect(lastCall()).toEqual({ + command: 'paste', + payload: { app: 'NeteaseMusic', text: '喜欢你', format: 'text' }, + }) + }) + itEngine('selectText sends text range parameters', async () => { const exec = await loadExecutor() await exec.engine!.selectText({ diff --git a/src/utils/computerUse/executor.ts b/src/utils/computerUse/executor.ts index cbcbabd2..834fc376 100644 --- a/src/utils/computerUse/executor.ts +++ b/src/utils/computerUse/executor.ts @@ -91,7 +91,7 @@ async function writeClipboard(text: string): Promise { // ---------------------------------------------------------------------------- // Codex semantic engine (blueprint §4–§7) // -// Maps the ten `CodexComputerEngine` methods onto `callHelper(, payload)` +// Maps the Codex-compatible `CodexComputerEngine` methods onto `callHelper(, payload)` // round-trips against the native daemon's `CommandRouter`. Each command's // payload key names mirror what `CommandRouter` decodes (see CommandRouter.swift): // - target: `pid` | `bundleId` | `app` (resolveTargetPid precedence) @@ -287,6 +287,18 @@ export function createCodexEngine(): CodexComputerEngine { text: args.text, }) }, + + async paste(args: { + target: AppTarget + text: string + format: 'text' | 'md' | 'html' + }): Promise { + await callHelper('paste', { + ...appTargetPayload(args.target), + text: args.text, + format: args.format, + }) + }, } } diff --git a/src/vendor/computer-use-mcp/executor.ts b/src/vendor/computer-use-mcp/executor.ts index 54ea4bda..a2c72ba9 100644 --- a/src/vendor/computer-use-mcp/executor.ts +++ b/src/vendor/computer-use-mcp/executor.ts @@ -206,7 +206,7 @@ export interface CodexComputerEngine { */ pressKey(args: { target: AppTarget; key: string; systemKeyCombos: boolean }): Promise /** Type literal text (append to focused value; Unicode keyboard fallback). */ - typeText(args: { target: AppTarget; text: string }): Promise + typeText(args: { target: AppTarget; text: string }): Promise; paste(args: { target: AppTarget; text: string; format: 'text' | 'md' | 'html' }): Promise } export interface InstalledApp { diff --git a/src/vendor/computer-use-mcp/instructions.ts b/src/vendor/computer-use-mcp/instructions.ts index 6fa88a52..401e0a09 100644 --- a/src/vendor/computer-use-mcp/instructions.ts +++ b/src/vendor/computer-use-mcp/instructions.ts @@ -67,7 +67,9 @@ Mutating tools return a fixed receipt. The receipt means "the action was dispatched", NOT "it had the intended effect" — you must look at the next \`get_app_state\` to know. -If two consecutive attempts leave the state unchanged, the approach is wrong. +Judge success from the screenshot as well as the AX text. An empty AX diff does +not mean a Chromium/CEF interface stayed unchanged. If two consecutive screenshots +leave the relevant UI unchanged, the approach is wrong. Change something real: switch from element handle to coordinates, target a different element, re-read the full tree with \`disableDiff: true\`, or take a different route through the UI. Repeating the same call a third time never helps. @@ -86,6 +88,10 @@ waste the user's time. element in the tree. Do not guess action names. - \`press_key\` and \`type_text\` are delivered to the named app, so they cannot trigger global system shortcuts. +- If \`type_text\` does not visually change a Chromium/CEF field, use + \`paste({ app, text, format: "text" })\`; it restores the user's prior clipboard. + If paste times out after dispatch, treat the result as unknown and call + \`get_app_state\` before retrying, because the target may have consumed it late. - \`press_key\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c". - \`select_text\` works inside editable elements; use \`prefix\`/\`suffix\` to disambiguate repeated matches. diff --git a/src/vendor/computer-use-mcp/platformRouting.test.ts b/src/vendor/computer-use-mcp/platformRouting.test.ts index 5696a5a6..7005c5de 100644 --- a/src/vendor/computer-use-mcp/platformRouting.test.ts +++ b/src/vendor/computer-use-mcp/platformRouting.test.ts @@ -19,6 +19,7 @@ const DARWIN_TOOL_NAMES = [ 'get_app_state', 'list_apps', 'perform_secondary_action', + 'paste', 'press_key', 'scroll', 'select_text', @@ -288,7 +289,7 @@ function makeWindowsAdapter(calls: string[]): ComputerUseHostAdapter { } describe('Computer Use platform routing', () => { - test('darwin ListTools advertises exactly the ten semantic tools', async () => { + test('darwin ListTools advertises the current semantic tools', async () => { const connection = await connect(makeDarwinAdapter()) try { const result = await connection.client.listTools() diff --git a/src/vendor/computer-use-mcp/toolCalls.test.ts b/src/vendor/computer-use-mcp/toolCalls.test.ts index 592b88d1..bac77d96 100644 --- a/src/vendor/computer-use-mcp/toolCalls.test.ts +++ b/src/vendor/computer-use-mcp/toolCalls.test.ts @@ -17,6 +17,7 @@ import { resetMouseButtonHeld, } from './toolCalls.js' import { buildComputerUseTools } from './tools.js' +import { COMPUTER_USE_INSTRUCTIONS } from './instructions.js' import { isSystemKeyCombo } from './keyBlocklist.js' import type { ComputerUseHostAdapter, @@ -128,6 +129,7 @@ function makeEngine( drag: record('drag', async () => {}), pressKey: record('pressKey', async () => {}), typeText: record('typeText', async () => {}), + paste: record('paste', async () => {}), selectText: record('selectText', async () => {}), } return { engine, calls } @@ -208,10 +210,10 @@ function imageBlocks(result: { content: unknown }): Array<{ data?: string; mimeT // Tool schema // --------------------------------------------------------------------------- -describe('buildComputerUseTools — Codex 10-tool face', () => { +describe('buildComputerUseTools — current Codex tool face', () => { const tools = buildComputerUseTools() - test('exposes exactly the ten Codex tools, verbatim names', () => { + test('exposes the current Codex tools, including explicit paste', () => { expect(tools.map(t => t.name).sort()).toEqual( [ 'click', @@ -219,6 +221,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => { 'get_app_state', 'list_apps', 'perform_secondary_action', + 'paste', 'press_key', 'scroll', 'select_text', @@ -318,6 +321,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => { 'drag', 'press_key', 'type_text', + 'paste', ]) { const description = tools.find(t => t.name === name)!.description ?? '' expect(description).toContain('get_app_state') @@ -433,7 +437,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => { 'normalized_0_100', ['Finder', 'Slack'], ) - expect(withArgs).toHaveLength(10) + expect(withArgs).toHaveLength(11) }) }) @@ -1575,6 +1579,45 @@ describe('handleToolCall — tool dispatch', () => { expect(calls.find(c => c.method === 'typeText')!.args).toMatchObject({ target: { pid: 1234 }, text: 'Codex' }) }) + test('paste passes text and format as an explicit recovery action', async () => { + const { engine, calls } = makeEngine() + await handleToolCall( + makeAdapter({ engine }), + 'paste', + { app: 'Activity Monitor', text: '喜欢你', format: 'text' }, + baseOverrides(), + ) + expect(calls.find(c => c.method === 'paste')!.args).toMatchObject({ + target: { pid: 1234 }, + text: '喜欢你', + format: 'text', + }) + }) + + test('paste rejects missing text and unknown formats before target resolution', async () => { + for (const args of [ + { app: 'Activity Monitor', format: 'text' }, + { app: 'Activity Monitor', text: 'hello', format: 'rtf' }, + ]) { + const { engine, calls } = makeEngine() + const result = await handleToolCall( + makeAdapter({ engine }), + 'paste', + args, + baseOverrides(), + ) + expect(result.isError).toBe(true) + expect(result.telemetry?.error_kind).toBe('bad_args') + expect(calls).toHaveLength(0) + } + }) + + test('server guidance treats AX diffs and timed-out paste as non-authoritative', () => { + expect(COMPUTER_USE_INSTRUCTIONS).toContain('An empty AX diff does') + expect(COMPUTER_USE_INSTRUCTIONS).toContain('paste({ app, text, format: "text" })') + expect(COMPUTER_USE_INSTRUCTIONS).toContain('treat the result as unknown') + }) + test('select_text passes the opaque handle, context, and selection mode', async () => { const { engine, calls } = makeEngine() await handleToolCall( diff --git a/src/vendor/computer-use-mcp/toolCalls.ts b/src/vendor/computer-use-mcp/toolCalls.ts index ba759221..1ec7c9d2 100644 --- a/src/vendor/computer-use-mcp/toolCalls.ts +++ b/src/vendor/computer-use-mcp/toolCalls.ts @@ -6,7 +6,7 @@ * commands): * * list_apps, get_app_state, click, perform_secondary_action, set_value, - * select_text, scroll, drag, press_key, type_text + * select_text, scroll, drag, press_key, type_text, paste * * ## Three properties this file exists to hold * @@ -565,6 +565,7 @@ const MUTATING_TOOLS: ReadonlySet = new Set([ "drag", "press_key", "type_text", + "paste", ]); const KNOWN_TOOLS: ReadonlySet = new Set([ @@ -801,6 +802,24 @@ function parseRequest( }; } + case "paste": { + const text = args.text; + if (typeof text !== "string") { + throw new BadArgs('Missing required string "text"'); + } + const format = requiredString(args, "format"); + if (format !== "text" && format !== "md" && format !== "html") { + throw new BadArgs('paste format must be "text", "md", or "html"'); + } + return { + ...base, + run: async (engine, t) => { + await engine.paste({ target: t, text, format }); + return okText(MUTATION_RECEIPT); + }, + }; + } + default: throw new BadArgs(`Unknown computer-use tool "${name}".`); } diff --git a/src/vendor/computer-use-mcp/tools.ts b/src/vendor/computer-use-mcp/tools.ts index c4c979f6..cadc93d6 100644 --- a/src/vendor/computer-use-mcp/tools.ts +++ b/src/vendor/computer-use-mcp/tools.ts @@ -1,14 +1,14 @@ /** * MCP tool schemas for the computer-use server — Codex-compatible semantic - * face (blueprint §7). Exactly TEN tools, named verbatim after Codex's public + * face (blueprint §7). Eleven tools, named verbatim after Codex's public * computer-use MCP: * * list_apps, get_app_state, click, perform_secondary_action, set_value, - * select_text, scroll, drag, press_key, type_text + * select_text, scroll, drag, press_key, type_text, paste * * This replaces the prior 27-tool pixel face. The legacy `coordinateMode` / * `screenshotFiltering` parameters are accepted for call-site compatibility - * but no longer influence the schema — the ten tool shapes are static. + * but no longer influence the schema — the tool shapes are static. * * Param names mirror Codex exactly (verified against iFurySt's reverse- * engineered ToolDefinitions.swift and the 2026-04-17 tool-call samples): @@ -83,7 +83,7 @@ function coordinateProp(axis: "x" | "y", role: string) { } /** - * Build the ten-tool Codex computer-use face. + * Build the Codex computer-use face. * * Signature is preserved from the legacy pixel builder so existing call sites * (`setup.ts`, host `mcpServer.ts`) keep compiling. All three parameters are @@ -415,5 +415,34 @@ export function buildComputerUseTools( required: ["app", "text"], }, }, + + { + name: "paste", + annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false }, + description: + "Paste content into whatever currently has keyboard focus in the target " + + "app. Prefer this when type_text did not change a Chromium/CEF field, for " + + "Chinese text, formatted content, or multiline text. The user's previous " + + "clipboard is restored unless they copy something during the operation. " + + "A timeout after Command-V is result-unknown: call get_app_state before " + + "deciding whether to retry. Call get_app_state afterwards to see the result.", + inputSchema: { + type: "object" as const, + additionalProperties: false, + properties: { + app: APP_PROP, + text: { + type: "string", + description: "The content to paste.", + }, + format: { + type: "string", + enum: ["text", "md", "html"], + description: "Pasteboard representation to use.", + }, + }, + required: ["app", "text", "format"], + }, + }, ]; }