diff --git a/native/cu-helper/Info.plist b/native/cu-helper/Info.plist
index 70cc5ca2..f3e5a3fd 100644
--- a/native/cu-helper/Info.plist
+++ b/native/cu-helper/Info.plist
@@ -21,6 +21,8 @@
cc-haha-computer-use
CFBundleDisplayName
cc-haha-computer-use
+ CFBundleIconFile
+ icon.icns
CFBundlePackageType
APPL
CFBundleShortVersionString
diff --git a/native/cu-helper/Sources/cu-helper/AXAction.swift b/native/cu-helper/Sources/cu-helper/AXAction.swift
index 3b5d4a01..d3e4949a 100644
--- a/native/cu-helper/Sources/cu-helper/AXAction.swift
+++ b/native/cu-helper/Sources/cu-helper/AXAction.swift
@@ -237,10 +237,18 @@ public final class ClipboardLease {
}
func writeTemporaryStringWithReceipt(_ text: String) throws -> ClipboardPasteReceipt {
+ try writeTemporaryContentWithReceipt(text, format: .text)
+ }
+
+ func writeTemporaryContentWithReceipt(
+ _ text: String,
+ format: ClipboardPasteFormat
+ ) throws -> ClipboardPasteReceipt {
if let captureError { throw captureError }
- let receipt = ClipboardPasteReceipt(text: text)
+ let receipt = ClipboardPasteReceipt(text: text, format: format)
let item = NSPasteboardItem()
- guard item.setDataProvider(receipt, forTypes: [.string]) else {
+ let types = Array(format.promisedData(for: text).keys)
+ guard item.setDataProvider(receipt, forTypes: types) else {
throw CUError("clipboard_write_failed", "Could not register temporary pasteboard data")
}
pasteboard.clearContents()
@@ -926,13 +934,35 @@ public enum AXAction {
try await typeViaClipboard(target: target, text)
}
+ /// Explicit Codex-compatible paste path. Unlike `typeText`, this does not
+ /// first try AX value mutation or Unicode key events; it targets the app's
+ /// in-process focused field with Command-V and restores the user's original
+ /// pasteboard after observing a real promised-data read.
+ static func pasteText(
+ pid: pid_t,
+ _ text: String,
+ format: ClipboardPasteFormat
+ ) async throws {
+ guard !text.isEmpty else { return }
+ let target = try Injection.authorizeResolvedTarget(pid: pid)
+ try await typeViaClipboard(target: target, text, format: format)
+ }
+
/// Paste `text` into whatever holds in-app keyboard focus in `pid`, via the
/// system clipboard + ⌘V (Codex's approach for CEF/Chromium text fields).
/// The read receipt confirms supplied clipboard bytes, not the reader's
/// PID or the field's final value; the next screenshot still verifies UI.
- private static func typeViaClipboard(target: ProvenProcessTarget, _ text: String) async throws {
+ private static func typeViaClipboard(
+ target: ProvenProcessTarget,
+ _ text: String,
+ format: ClipboardPasteFormat = .text
+ ) async throws {
let pid = target.pid
- try await ClipboardPasteReceipt.perform(text: text, lease: ClipboardLease()) { validate in
+ try await ClipboardPasteReceipt.perform(
+ text: text,
+ format: format,
+ lease: ClipboardLease()
+ ) { validate in
try await pressKey(pid: pid, "super+v", validateBeforePosting: {
_ = try Injection.validateAuthorizedTarget(target)
try validate()
@@ -1609,10 +1639,11 @@ public enum AXAction {
return CFEqual(lhs, rhs)
}
- /// Mark that a mutation just landed. Does NOT block: `get_app_state` reads
- /// this stamp and waits out whatever settle time is still owed, so the cost
- /// is paid once, and only when someone is actually about to look.
+ /// Settlement is recorded once at `ForegroundMutationRunner`, which covers
+ /// every input path and knows the target PID. AX-local success receipts must
+ /// not create a second, unscoped marker that can leak across apps.
private static func settle() {
- MutationClock.recordMutation()
+ // Intentionally empty; retained at the AX call sites as a semantic
+ // marker for successful mutations.
}
}
diff --git a/native/cu-helper/Sources/cu-helper/ClientAttestation.swift b/native/cu-helper/Sources/cu-helper/ClientAttestation.swift
index 41ce2667..95eed7b8 100644
--- a/native/cu-helper/Sources/cu-helper/ClientAttestation.swift
+++ b/native/cu-helper/Sources/cu-helper/ClientAttestation.swift
@@ -30,9 +30,9 @@ enum HelperClientPolicy {
// read-only authorization preflight used before the ten public tools.
"list_apps", "resolve_app_target", "get_app_state", "click",
"set_value", "select_text", "perform_secondary_action", "scroll",
- "drag", "press_key", "type_text",
+ "drag", "press_key", "type_text", "paste",
// Private lifecycle / visible-overlay / permission and input diagnostics.
- "ping", "shutdown", "overlay_show", "overlay_hide",
+ "ping", "shutdown", "overlay_show", "overlay_hide", "turn_end",
"check_permissions", "input_monitor_state", "focus_monitor_state", "held_input_state",
"last_injection_state",
]
diff --git a/native/cu-helper/Sources/cu-helper/ClipboardPasteReceipt.swift b/native/cu-helper/Sources/cu-helper/ClipboardPasteReceipt.swift
index fd89abfa..95a8699f 100644
--- a/native/cu-helper/Sources/cu-helper/ClipboardPasteReceipt.swift
+++ b/native/cu-helper/Sources/cu-helper/ClipboardPasteReceipt.swift
@@ -2,6 +2,32 @@ import AppKit
import Foundation
import os
+enum ClipboardPasteFormat: String, Sendable {
+ case text
+ case md
+ case html
+
+ func promisedData(for text: String) -> [NSPasteboard.PasteboardType: Data] {
+ switch self {
+ case .text, .md:
+ return [.string: Data(text.utf8)]
+ case .html:
+ let html = Data(text.utf8)
+ let plain = (
+ try? NSAttributedString(
+ data: html,
+ options: [
+ .documentType: NSAttributedString.DocumentType.html,
+ .characterEncoding: String.Encoding.utf8.rawValue,
+ ],
+ documentAttributes: nil
+ ).string
+ ) ?? text
+ return [.html: html, .string: Data(plain.utf8)]
+ }
+ }
+}
+
/// Confirms that this pasteboard item's promised bytes were requested and
/// supplied. AppKit does not identify the reader: this is not proof that the
/// intended application's focused field accepted the text.
@@ -25,16 +51,26 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
}
@MainActor private(set) static var lastDiagnostic: Diagnostic?
- private let data: Data
+ private let promisedData: [NSPasteboard.PasteboardType: Data]
private let state = OSAllocatedUnfairLock(initialState: State())
init(text: String) {
- data = Data(text.utf8)
+ promisedData = ClipboardPasteFormat.text.promisedData(for: text)
super.init()
}
+ init(text: String, format: ClipboardPasteFormat) {
+ promisedData = format.promisedData(for: text)
+ super.init()
+ }
+
+ @MainActor
+ static func resetForTurn() {
+ lastDiagnostic = nil
+ }
+
func pasteboard(_ pasteboard: NSPasteboard?, item: NSPasteboardItem, provideDataForType type: NSPasteboard.PasteboardType) {
- guard type == .string else { return }
+ guard let data = promisedData[type] else { return }
state.withLock { $0.requested = true }
guard item.setData(data, forType: type) else { return }
state.withLock { value in
@@ -50,6 +86,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
@MainActor
static func perform(
text: String,
+ format: ClipboardPasteFormat = .text,
lease: ClipboardLease,
timeout: Duration = .seconds(2),
sendPaste: @MainActor (_ validateBeforePosting: @MainActor () throws -> Void) async throws -> Void
@@ -79,7 +116,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
}
do {
try Task.checkCancellation()
- let written = try lease.writeTemporaryStringWithReceipt(text)
+ let written = try lease.writeTemporaryContentWithReceipt(text, format: format)
receipt = written
try await Task.sleep(for: .milliseconds(40))
let validate: @MainActor () throws -> Void = {
diff --git a/native/cu-helper/Sources/cu-helper/CommandRouter.swift b/native/cu-helper/Sources/cu-helper/CommandRouter.swift
index 1a881f81..44d53a08 100644
--- a/native/cu-helper/Sources/cu-helper/CommandRouter.swift
+++ b/native/cu-helper/Sources/cu-helper/CommandRouter.swift
@@ -104,6 +104,7 @@ public final class CommandRouter {
Self.lastShotTransform.removeAll()
Self.lastCaptureDigest.removeAll()
MutationClock.reset()
+ ClipboardPasteReceipt.resetForTurn()
windowCaptureProvider?.invalidate()
// Apps we told they were focused must be told they are not, or the
// belief outlives the session that needed it.
@@ -267,6 +268,9 @@ public final class CommandRouter {
case "type_text":
return try await handleTypeText(payload)
+ case "paste":
+ return try await handlePaste(payload)
+
case "press_key":
return try await handlePressKey(payload)
@@ -728,18 +732,19 @@ public final class CommandRouter {
if #available(macOS 14.0, *) {
// Let the UI finish whatever the last action started before we
// photograph it, otherwise the model reasons about a half-drawn
- // frame. Costs nothing when no action is pending. The rendered tree
- // doubles as the busy signal — a progress indicator in it means the
- // app is still working, so we allow a longer window.
+ // frame. Costs nothing when no action is pending, applies only to
+ // this target PID, and is consumed by this single capture.
+ let pendingMutation = MutationClock.takeMutation(pid: pid)
let streamedShot = await Self.captureSettledWindowShot(
- appIsBusy: result.axText.contains("progress indicator")
+ appIsBusy: result.axText.contains("progress indicator"),
+ lastMutationAt: pendingMutation
) {
await windowCaptureProvider?.windowShot(
pid: pid,
processIdentity: snapshotEvidence.processIdentity,
preferredWindowID: snapshotEvidence.keyWindowID,
scale: 0.5,
- newerThanUptime: MutationClock.lastMutation()
+ newerThanUptime: pendingMutation
)
}
guard TargetVisibilityPolicy.captureTargetStillMatches(
@@ -875,9 +880,13 @@ public final class CommandRouter {
/// action-to-screenshot regression can exercise both without live AX/TCC.
static func captureSettledWindowShot(
appIsBusy: Bool,
+ lastMutationAt: TimeInterval?,
capture: () async -> WindowShot?
) async -> WindowShot? {
- await MutationClock.awaitSettle(appIsBusy: appIsBusy)
+ await MutationClock.awaitSettle(
+ lastMutationAt: lastMutationAt,
+ appIsBusy: appIsBusy
+ )
return await capture()
}
@@ -1438,6 +1447,31 @@ public final class CommandRouter {
}
}
+ /// `paste`: bypass AX/Unicode typing and use the target app's in-process
+ /// paste focus. This is the reliable recovery path for Chromium/CEF fields
+ /// such as NeteaseMusic's search box. The clipboard lease restores every
+ /// original pasteboard item unless the user copied something meanwhile.
+ private func handlePaste(_ payload: JSONValue) async throws -> JSONValue {
+ try requireAXTrusted()
+ guard let text = payload["text"]?.asString else {
+ throw CUError("bad_payload", "paste requires a 'text' string")
+ }
+ guard let rawFormat = payload["format"]?.asString,
+ let format = ClipboardPasteFormat(rawValue: rawFormat) else {
+ throw CUError("bad_payload", "paste format must be 'text', 'md', or 'html'")
+ }
+ let expected = try Self.expectedProcessTarget(payload)
+ let target = try resolveTargetForMutation(payload)
+ setResolvedTarget(target)
+ try requireSnapshotProcess(target: target, expected: expected)
+ return try await withForegroundLease(command: "paste", target: target) {
+ _ = try Injection.validateAuthorizedTarget(target)
+ try self.requireSnapshotProcess(target: target, expected: expected)
+ try await AXAction.pasteText(pid: target.pid, text, format: format)
+ return .bool(true)
+ }
+ }
+
/// `press_key`: send an xdotool-style key spec (e.g. "super+c", "Return",
/// "Tab", "KP_0", "Prior") to the target app via postToPid.
private func handlePressKey(_ payload: JSONValue) async throws -> JSONValue {
@@ -1566,6 +1600,7 @@ public final class CommandRouter {
)
return try await ForegroundMutationRunner.run(
lease: lease,
+ targetPID: target.pid,
action: action
)
}
diff --git a/native/cu-helper/Sources/cu-helper/Daemon.swift b/native/cu-helper/Sources/cu-helper/Daemon.swift
index 89377848..351c75fa 100644
--- a/native/cu-helper/Sources/cu-helper/Daemon.swift
+++ b/native/cu-helper/Sources/cu-helper/Daemon.swift
@@ -38,21 +38,21 @@ enum DaemonOverlayTargetResolver {
// `cursor_position`, implicit-`from` drags and decomposed `mouse_down`/`mouse_up`
// behave correctly across commands.
//
-// Transport: a private AF_UNIX SOCK_STREAM socket carrying NDJSON (one JSON
-// object + '\n' per request, same per response). A GUI/AppKit process leaks
+// Transport: a private AF_UNIX SOCK_STREAM socket carrying versioned NDJSON
+// (one JSON object + '\n' per request, same per response). A GUI/AppKit process leaks
// os_log / CoreGraphics chatter to stdout/stderr, which would corrupt the TS
// bridge's `JSON.parse`; the daemon therefore reserves stdout for exactly ONE
// readiness line and serves the request/response stream over the socket.
//
-// readiness : {"ready":true,"pid":,"proto":1}\n (stdout, once)
-// request : {"id":"","cmd":"","payload":{...}}\n
+// readiness : {"ready":true,"pid":,"protocolVersion":"…"}\n
+// request : {"id":"","requestId":"","cmd":"",…}\n
// response : {"id":"","ok":true,"result":}\n
// {"id":"","ok":false,"error":{"message":"…","code":"…"}}\n
//
// Control verbs handled in-daemon (never reach CommandRouter):
// overlay_show -> cursor.show() ; result true
-// overlay_hide -> cursor.hide() ; result true
-// ping -> result "pong"
+// turn_end -> release all turn-owned state ; result true
+// ping -> version/capability hello
// shutdown -> result true, then NSApp.terminate(nil)
// Every other cmd is forwarded to the shared CommandRouter — the exact same
// dispatcher the one-shot CLI path uses, so each command has one implementation.
@@ -63,6 +63,7 @@ public final class Daemon {
private let cursor: VirtualCursor
private let inputMonitor: PhysicalInputEpochMonitor
private let router: CommandRouter
+ private let displaySleepAssertion = ComputerUseDisplaySleepAssertion()
/// Listening socket fd (AF_UNIX SOCK_STREAM). -1 until `bindAndListen`.
private var listenFD: Int32 = -1
@@ -78,11 +79,12 @@ public final class Daemon {
private var connection: Connection?
private var connectionToken: DaemonSessionToken?
private var sessionGate = DaemonSessionGate()
+ private var turnGate = DaemonTurnGate()
/// Set once the run loop is live; guards against double `run()`.
private var didStartRunLoop = false
- /// True between `overlay_show` and `overlay_hide` (CU active this turn). The
+ /// True between `overlay_show` and `turn_end` (CU active this turn). The
/// cursor re-aims at the actual injection target while this holds.
private var overlayActive = false
private var explicitOverlayTarget: ProvenProcessTarget?
@@ -220,6 +222,7 @@ public final class Daemon {
// immediately by exit() (shutdown verb + signal handlers).
Injection.releaseAllHeldSync()
router.resetSessionState()
+ displaySleepAssertion.release()
inputMonitor.stop()
}
@@ -435,6 +438,8 @@ public final class Daemon {
stopOverlaySession()
Injection.releaseAllHeldSync()
router.resetSessionState()
+ turnGate.reset()
+ displaySleepAssertion.release()
}
/// A LaunchServices child is reparented to launchd, so the peer socket is
@@ -479,7 +484,16 @@ public final class Daemon {
let payload = request.payload ?? .object([:])
do {
+ let metadata = try ComputerUseDaemonProtocol.validate(request)
+ let opensTurn = ComputerUseDaemonProtocol.isTurnScoped(command: request.cmd)
+ && turnGate.active == nil
+ try turnGate.admit(metadata, command: request.cmd)
+ if opensTurn { displaySleepAssertion.acquire() }
let result = try await route(cmd: request.cmd, payload: payload)
+ if request.cmd == "turn_end" || request.cmd == "overlay_hide" {
+ try turnGate.finish(metadata)
+ displaySleepAssertion.release()
+ }
conn.send(encodeResponse(DaemonResponse(id: id, ok: true, result: result, error: nil)))
} catch let cuError as CUError {
conn.send(encodeResponse(DaemonResponse(
@@ -508,12 +522,12 @@ public final class Daemon {
try showOverlay(payload: payload)
return .bool(true)
- case "overlay_hide":
- stopOverlaySession()
+ case "overlay_hide", "turn_end":
+ endTurn()
return .bool(true)
case "ping":
- return .string("pong")
+ return ComputerUseDaemonProtocol.hello()
case "check_permissions":
// Usable in both modes; the daemon exposes it for onboarding /
@@ -592,6 +606,15 @@ public final class Daemon {
router.invalidateWindowCaptureStream()
}
+ /// Codex-parity turn boundary. The helper process stays warm, but no AX
+ /// snapshot, coordinate transform, focus belief, held input, mutation clock,
+ /// clipboard diagnostic, or capture stream may leak into the next turn.
+ private func endTurn() {
+ stopOverlaySession()
+ Injection.releaseAllHeldSync()
+ router.resetSessionState()
+ }
+
/// The app the cursor should follow: ONLY the app the last injection /
/// get_app_state actually resolved (`Injection.lastResolvedTargetPid`). We
/// deliberately do NOT fall back to the frontmost app — at turn start,
@@ -763,9 +786,9 @@ private final class Connection: @unchecked Sendable {
private var inBuffer = Data()
/// Capped to avoid unbounded growth from a malformed/never-newline client.
- /// 16 MiB comfortably exceeds the largest legitimate request (a base64
+ /// 8 MiB comfortably exceeds the largest legitimate request (a base64
/// screenshot travels in *responses*, not requests).
- private let maxBufferBytes = 16 * 1024 * 1024
+ private let maxBufferBytes = ComputerUseDaemonProtocol.maxFrameBytes
private var onRequest: (@Sendable (Data) -> Void)?
private var onClose: (@Sendable () -> Void)?
diff --git a/native/cu-helper/Sources/cu-helper/DaemonProtocol.swift b/native/cu-helper/Sources/cu-helper/DaemonProtocol.swift
new file mode 100644
index 00000000..243c24fe
--- /dev/null
+++ b/native/cu-helper/Sources/cu-helper/DaemonProtocol.swift
@@ -0,0 +1,99 @@
+import Foundation
+
+/// Versioned request contract for the authenticated daemon socket.
+///
+/// Code signature attestation proves who is connected; this handshake proves
+/// that both sides agree on request shape and lifecycle semantics. Every request
+/// also carries an absolute deadline plus session/turn identity so a command
+/// queued behind a slow capture cannot execute after its caller has moved on.
+enum ComputerUseDaemonProtocol {
+ static let version = "CCHahaComputerUseIPC-2"
+ static let maxFrameBytes = 8 * 1024 * 1024
+ private static let connectionScopedCommands: Set = [
+ "ping", "check_permissions", "shutdown",
+ ]
+
+ static func isTurnScoped(command: String) -> Bool {
+ !connectionScopedCommands.contains(command)
+ }
+
+ struct Metadata: Equatable, Sendable {
+ let sessionId: String
+ let turnId: String
+ }
+
+ static func validate(
+ _ request: Request,
+ nowUnixMilliseconds: Int64 = Int64(Date().timeIntervalSince1970 * 1_000)
+ ) throws -> Metadata {
+ guard request.clientApiVersion == version else {
+ throw CUError(
+ "protocol_mismatch",
+ "Computer Use client/helper protocol mismatch; restart the app after updating"
+ )
+ }
+ guard let id = request.id, !id.isEmpty,
+ request.requestId == id else {
+ throw CUError("bad_request_id", "Computer Use request identity is missing or inconsistent")
+ }
+ guard let deadline = request.deadlineUnixMilliseconds else {
+ throw CUError("missing_deadline", "Computer Use request is missing its absolute deadline")
+ }
+ guard deadline > nowUnixMilliseconds else {
+ throw CUError("deadline_exceeded", "Computer Use request expired before execution")
+ }
+ guard let sessionId = request.sessionId?.trimmingCharacters(in: .whitespacesAndNewlines),
+ !sessionId.isEmpty,
+ let turnId = request.turnId?.trimmingCharacters(in: .whitespacesAndNewlines),
+ !turnId.isEmpty else {
+ throw CUError("missing_turn_metadata", "Computer Use request is missing session/turn identity")
+ }
+ return Metadata(sessionId: sessionId, turnId: turnId)
+ }
+
+ static func hello() -> JSONValue {
+ .object([
+ "protocolVersion": .string(version),
+ "supportsAbsoluteDeadlines": .bool(true),
+ "supportsTurnEnd": .bool(true),
+ "supportsCaptureDiagnostics": .bool(true),
+ ])
+ }
+}
+
+/// Enforces one explicit turn at a time on the single authenticated connection.
+/// `ping` negotiates the protocol without opening a turn. Every other request
+/// must keep the same identity until `turn_end` releases all turn-owned state.
+struct DaemonTurnGate {
+ private(set) var active: ComputerUseDaemonProtocol.Metadata?
+
+ mutating func admit(
+ _ metadata: ComputerUseDaemonProtocol.Metadata,
+ command: String
+ ) throws {
+ if !ComputerUseDaemonProtocol.isTurnScoped(command: command) { return }
+ if let active {
+ guard active == metadata else {
+ throw CUError(
+ "turn_mismatch",
+ "A different Computer Use turn is still active; finish it before starting another"
+ )
+ }
+ return
+ }
+ active = metadata
+ }
+
+ mutating func finish(
+ _ metadata: ComputerUseDaemonProtocol.Metadata
+ ) throws {
+ guard active == metadata else {
+ throw CUError("turn_mismatch", "Computer Use turn_end did not match the active turn")
+ }
+ active = nil
+ }
+
+ mutating func reset() {
+ active = nil
+ }
+}
diff --git a/native/cu-helper/Sources/cu-helper/DisplaySleepAssertion.swift b/native/cu-helper/Sources/cu-helper/DisplaySleepAssertion.swift
new file mode 100644
index 00000000..a1a987fd
--- /dev/null
+++ b/native/cu-helper/Sources/cu-helper/DisplaySleepAssertion.swift
@@ -0,0 +1,50 @@
+import Foundation
+import IOKit.pwr_mgt
+
+/// Keeps the display compositor awake while a Computer Use turn is active.
+/// Covered-window ScreenCaptureKit consumers can otherwise stop receiving
+/// useful updates once macOS enters user-idle display sleep during a long task.
+@MainActor
+final class ComputerUseDisplaySleepAssertion {
+ typealias AssertionID = IOPMAssertionID
+
+ private let create: () -> AssertionID?
+ private let releaseAssertion: (AssertionID) -> Void
+ private var assertionID: AssertionID?
+
+ init(
+ create: @escaping () -> AssertionID? = {
+ var assertionID: IOPMAssertionID = 0
+ let result = IOPMAssertionCreateWithDescription(
+ kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString,
+ "Claude Code Haha Computer Use interaction" as CFString,
+ "Computer Use turn is controlling a background application" as CFString,
+ "Keeping the display awake while Computer Use is active" as CFString,
+ nil,
+ 0,
+ nil,
+ &assertionID
+ )
+ return result == kIOReturnSuccess ? assertionID : nil
+ },
+ release: @escaping (AssertionID) -> Void = { assertionID in
+ _ = IOPMAssertionRelease(assertionID)
+ }
+ ) {
+ self.create = create
+ self.releaseAssertion = release
+ }
+
+ func acquire() {
+ guard assertionID == nil else { return }
+ assertionID = create()
+ }
+
+ func release() {
+ guard let assertionID else { return }
+ self.assertionID = nil
+ releaseAssertion(assertionID)
+ }
+
+ var isHeldForTesting: Bool { assertionID != nil }
+}
diff --git a/native/cu-helper/Sources/cu-helper/ForegroundLease.swift b/native/cu-helper/Sources/cu-helper/ForegroundLease.swift
index 1a250fc9..b3fbe5ef 100644
--- a/native/cu-helper/Sources/cu-helper/ForegroundLease.swift
+++ b/native/cu-helper/Sources/cu-helper/ForegroundLease.swift
@@ -394,12 +394,13 @@ final class ForegroundLease {
enum ForegroundMutationRunner {
static func run(
lease: ForegroundLease,
+ targetPID: pid_t? = nil,
action: () async throws -> T
) async throws -> T {
// Every input path crosses this boundary, including synthetic events
// that never call AXAction.settle(). A throw can follow a partial
// delivery, so neither success nor failure may reuse pre-action pixels.
- defer { MutationClock.recordMutation() }
+ defer { MutationClock.recordMutation(pid: targetPID) }
let result: Result
do {
result = .success(try await action())
diff --git a/native/cu-helper/Sources/cu-helper/JSONValue.swift b/native/cu-helper/Sources/cu-helper/JSONValue.swift
index 2afadc59..d5beaf03 100644
--- a/native/cu-helper/Sources/cu-helper/JSONValue.swift
+++ b/native/cu-helper/Sources/cu-helper/JSONValue.swift
@@ -439,11 +439,30 @@ public struct Request: Decodable, Sendable {
public let id: String?
public let cmd: String
public let payload: JSONValue?
+ public let clientApiVersion: String?
+ public let deadlineUnixMilliseconds: Int64?
+ public let sessionId: String?
+ public let turnId: String?
+ public let requestId: String?
- public init(id: String?, cmd: String, payload: JSONValue?) {
+ public init(
+ id: String?,
+ cmd: String,
+ payload: JSONValue?,
+ clientApiVersion: String? = nil,
+ deadlineUnixMilliseconds: Int64? = nil,
+ sessionId: String? = nil,
+ turnId: String? = nil,
+ requestId: String? = nil
+ ) {
self.id = id
self.cmd = cmd
self.payload = payload
+ self.clientApiVersion = clientApiVersion
+ self.deadlineUnixMilliseconds = deadlineUnixMilliseconds
+ self.sessionId = sessionId
+ self.turnId = turnId
+ self.requestId = requestId
}
/// The payload, defaulting to an empty object when omitted/null — every
diff --git a/native/cu-helper/Sources/cu-helper/UISettlePolicy.swift b/native/cu-helper/Sources/cu-helper/UISettlePolicy.swift
index 486b950a..2d34fdf4 100644
--- a/native/cu-helper/Sources/cu-helper/UISettlePolicy.swift
+++ b/native/cu-helper/Sources/cu-helper/UISettlePolicy.swift
@@ -14,18 +14,19 @@ import Foundation
/// which it no longer does. Instead the wait happens once, at the moment of
/// capture, and only if an action happened recently enough to still be settling.
///
-/// This mirrors Codex's `needsUISettleBeforeSkyshot`: wait about a second after
-/// a recent action, and extend that when the app is visibly still working.
+/// This mirrors Codex's one-shot `needsUISettleBeforeSkyshot`: the next capture
+/// for that target waits about 250ms after a recent action. A different app and
+/// later captures do not inherit the wait.
///
/// Pure and clock-injected so the decision is testable without sleeping.
enum UISettlePolicy {
/// How long after a mutation the UI is presumed to still be settling.
- static let postActionWindow: TimeInterval = 1.0
+ static let postActionWindow: TimeInterval = 0.25
- /// Ceiling while the app still shows a busy/progress indicator. Generous
- /// because "still loading" is a real signal, but bounded so a permanently
- /// spinning indicator (some apps never stop one) cannot hang the capture.
- static let busyWindow: TimeInterval = 5.0
+ /// Kept as a named compatibility seam for callers that already compute a
+ /// busy signal. Codex does not turn that signal into a multi-second sleep;
+ /// freshness comes from subsequent on-demand captures instead.
+ static let busyWindow: TimeInterval = postActionWindow
/// Never wait less than this once we've decided to wait at all — a delay
/// too short to cover a frame boundary is just latency for nothing.
@@ -57,28 +58,52 @@ enum UISettlePolicy {
}
}
-/// Records when the last mutating action completed, so the capture path can ask
-/// "was something just changed?" without the action path having to block.
+/// Records one pending capture settle per target process, so an action in one
+/// app cannot delay a screenshot of another app.
///
/// `@MainActor` because every mutation already runs there; this keeps the state
/// single-threaded without a lock.
@MainActor
enum MutationClock {
- private static var lastMutationAt: TimeInterval?
+ private static var pendingByPID: [pid_t: TimeInterval] = [:]
+ private static var unscopedMutationAt: TimeInterval?
/// Called by the action path after a mutation lands. Does not sleep.
- static func recordMutation(at time: TimeInterval = ProcessInfo.processInfo.systemUptime) {
- lastMutationAt = time
+ static func recordMutation(
+ pid: pid_t? = nil,
+ at time: TimeInterval = ProcessInfo.processInfo.systemUptime
+ ) {
+ if let pid {
+ pendingByPID[pid] = time
+ } else {
+ unscopedMutationAt = time
+ }
}
- static func lastMutation() -> TimeInterval? { lastMutationAt }
+ static func lastMutation(pid: pid_t? = nil) -> TimeInterval? {
+ guard let pid else { return unscopedMutationAt }
+ return pendingByPID[pid]
+ }
+
+ /// Consume the one-shot marker before waiting. Another capture cannot pay
+ /// the same action's settle cost a second time.
+ static func takeMutation(pid: pid_t? = nil) -> TimeInterval? {
+ guard let pid else {
+ defer { unscopedMutationAt = nil }
+ return unscopedMutationAt
+ }
+ return pendingByPID.removeValue(forKey: pid)
+ }
/// Wait out whatever settle time the last mutation still owes.
///
/// Uses `Task.sleep`, not `Thread.sleep`: this runs on the main actor, where
/// blocking would starve the overlay's display link and freeze the virtual
/// cursor animation mid-glide.
- static func awaitSettle(appIsBusy: Bool) async {
+ static func awaitSettle(
+ lastMutationAt: TimeInterval?,
+ appIsBusy: Bool
+ ) async {
let delay = UISettlePolicy.delay(
now: ProcessInfo.processInfo.systemUptime,
lastMutationAt: lastMutationAt,
@@ -88,7 +113,10 @@ enum MutationClock {
try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
}
- static func reset() { lastMutationAt = nil }
+ static func reset() {
+ pendingByPID.removeAll()
+ unscopedMutationAt = nil
+ }
static func resetForTests() { reset() }
}
diff --git a/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift b/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift
index fd8d844c..4a5b94a5 100644
--- a/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift
+++ b/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift
@@ -172,7 +172,11 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
invalidate()
return nil
}
- guard let shot = await captureSnapshot(for: target, scale: scale) else {
+ guard let shot = await captureSnapshot(
+ for: target,
+ scale: scale,
+ newerThanUptime: newerThanUptime
+ ) else {
return nil
}
if let preferredWindowID,
@@ -213,7 +217,20 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
/// Match the reference's two separate lifetimes: SCStream remains a
/// consumer while covered; every state read runs an on-demand Skyshot/SCK
/// capture. An idle stream's cached frame is not evidence of the current UI.
- func captureSnapshot(for target: WindowCaptureStreamTarget, scale: Double) async -> WindowShot? {
+ func captureSnapshot(
+ for target: WindowCaptureStreamTarget,
+ scale: Double,
+ newerThanUptime: TimeInterval? = nil
+ ) async -> WindowShot? {
+ if let newerThanUptime {
+ // The stream is a long-lived render/freshness consumer, not the
+ // model screenshot source. Before the post-action Skyshot, observe
+ // a stream frame newer than the action when possible. `frame`
+ // performs one bounded rebuild for a silently starved stream; a
+ // static/no-op UI may legitimately emit no changed frame, so the
+ // authoritative on-demand screenshot still runs after the bound.
+ _ = await frame(for: target, newerThanUptime: newerThanUptime)
+ }
for _ in 0..<2 {
guard let source = await source(for: target) else { continue }
if source.hasFailed {
diff --git a/native/cu-helper/Tests/CuHelperTests/ClientAttestationTests.swift b/native/cu-helper/Tests/CuHelperTests/ClientAttestationTests.swift
index cc81a871..54ac2b51 100644
--- a/native/cu-helper/Tests/CuHelperTests/ClientAttestationTests.swift
+++ b/native/cu-helper/Tests/CuHelperTests/ClientAttestationTests.swift
@@ -365,8 +365,8 @@ struct ClientAttestationTests {
let allowed = [
"list_apps", "resolve_app_target", "get_app_state", "click",
"set_value", "select_text", "perform_secondary_action", "scroll",
- "drag", "press_key", "type_text", "ping", "shutdown",
- "overlay_show", "overlay_hide", "check_permissions",
+ "drag", "press_key", "type_text", "paste", "ping", "shutdown",
+ "overlay_show", "overlay_hide", "turn_end", "check_permissions",
"input_monitor_state", "held_input_state",
]
for command in allowed {
diff --git a/native/cu-helper/Tests/CuHelperTests/ClipboardPasteReceiptTests.swift b/native/cu-helper/Tests/CuHelperTests/ClipboardPasteReceiptTests.swift
index 7338b560..c30fd8fa 100644
--- a/native/cu-helper/Tests/CuHelperTests/ClipboardPasteReceiptTests.swift
+++ b/native/cu-helper/Tests/CuHelperTests/ClipboardPasteReceiptTests.swift
@@ -5,6 +5,29 @@ import XCTest
@testable import cc_haha_computer_use
final class ClipboardPasteReceiptTests: XCTestCase {
+ @MainActor
+ func testHtmlPastePromisesRichAndPlainRepresentationsThenRestoresClipboard() async throws {
+ let fixture = PasteReceiptFixture()
+ defer { fixture.close() }
+ let lease = ClipboardLease(pasteboard: fixture.board)
+
+ try await ClipboardPasteReceipt.perform(
+ text: "Hello world",
+ format: .html,
+ lease: lease
+ ) { validate in
+ try await fixture.sendPaste(validate)
+ XCTAssertEqual(
+ String(data: fixture.board.data(forType: .html) ?? Data(), encoding: .utf8),
+ "Hello world"
+ )
+ XCTAssertTrue((fixture.board.string(forType: .string) ?? "").contains("Hello world"))
+ }
+
+ XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.status, "completed")
+ XCTAssertEqual(fixture.board.string(forType: .string), "original")
+ }
+
@MainActor
func testPasteWaitsForARealReadBeyondTheOld180MillisecondWindow() async throws {
let fixture = PasteReceiptFixture()
diff --git a/native/cu-helper/Tests/CuHelperTests/DaemonProtocolTests.swift b/native/cu-helper/Tests/CuHelperTests/DaemonProtocolTests.swift
new file mode 100644
index 00000000..0adcd6e1
--- /dev/null
+++ b/native/cu-helper/Tests/CuHelperTests/DaemonProtocolTests.swift
@@ -0,0 +1,116 @@
+import XCTest
+@testable import cc_haha_computer_use
+
+final class DaemonProtocolTests: XCTestCase {
+ private func request(
+ version: String? = ComputerUseDaemonProtocol.version,
+ deadline: Int64? = 2_000,
+ sessionId: String? = "session-a",
+ turnId: String? = "turn-a",
+ id: String? = "request-a",
+ requestId: String? = "request-a",
+ command: String = "get_app_state"
+ ) -> Request {
+ Request(
+ id: id,
+ cmd: command,
+ payload: .object([:]),
+ clientApiVersion: version,
+ deadlineUnixMilliseconds: deadline,
+ sessionId: sessionId,
+ turnId: turnId,
+ requestId: requestId
+ )
+ }
+
+ func testValidRequestCarriesVersionDeadlineAndTurnIdentity() throws {
+ let metadata = try ComputerUseDaemonProtocol.validate(
+ request(),
+ nowUnixMilliseconds: 1_000
+ )
+
+ XCTAssertEqual(metadata.sessionId, "session-a")
+ XCTAssertEqual(metadata.turnId, "turn-a")
+ }
+
+ func testProtocolMismatchAndExpiredDeadlineFailClosed() {
+ XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
+ request(version: "CCHahaComputerUseIPC-1"),
+ nowUnixMilliseconds: 1_000
+ )) { error in
+ XCTAssertEqual((error as? CUError)?.code, "protocol_mismatch")
+ }
+
+ XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
+ request(deadline: 1_000),
+ nowUnixMilliseconds: 1_000
+ )) { error in
+ XCTAssertEqual((error as? CUError)?.code, "deadline_exceeded")
+ }
+ }
+
+ func testRequestIdentityAndTurnMetadataCannotBeOmitted() {
+ XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
+ request(requestId: "different"),
+ nowUnixMilliseconds: 1_000
+ )) { error in
+ XCTAssertEqual((error as? CUError)?.code, "bad_request_id")
+ }
+
+ XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
+ request(turnId: " "),
+ nowUnixMilliseconds: 1_000
+ )) { error in
+ XCTAssertEqual((error as? CUError)?.code, "missing_turn_metadata")
+ }
+ }
+
+ func testTurnGateRejectsCrossTurnStateUntilMatchingEnd() throws {
+ let first = ComputerUseDaemonProtocol.Metadata(
+ sessionId: "session-a",
+ turnId: "turn-a"
+ )
+ let second = ComputerUseDaemonProtocol.Metadata(
+ sessionId: "session-a",
+ turnId: "turn-b"
+ )
+ var gate = DaemonTurnGate()
+
+ try gate.admit(first, command: "get_app_state")
+ try gate.admit(first, command: "click")
+ XCTAssertThrowsError(try gate.admit(second, command: "get_app_state")) {
+ XCTAssertEqual(($0 as? CUError)?.code, "turn_mismatch")
+ }
+ try gate.finish(first)
+ try gate.admit(second, command: "get_app_state")
+ XCTAssertEqual(gate.active, second)
+ }
+
+ func testPingNegotiatesWithoutOpeningATurn() throws {
+ let metadata = ComputerUseDaemonProtocol.Metadata(
+ sessionId: "session-a",
+ turnId: "handshake-1"
+ )
+ var gate = DaemonTurnGate()
+
+ try gate.admit(metadata, command: "ping")
+
+ XCTAssertNil(gate.active)
+ XCTAssertEqual(
+ ComputerUseDaemonProtocol.hello()["protocolVersion"]?.asString,
+ ComputerUseDaemonProtocol.version
+ )
+ }
+
+ func testPermissionProbeIsConnectionScopedAndDoesNotOpenATurn() throws {
+ let metadata = ComputerUseDaemonProtocol.Metadata(
+ sessionId: "session-a",
+ turnId: "probe"
+ )
+ var gate = DaemonTurnGate()
+
+ try gate.admit(metadata, command: "check_permissions")
+
+ XCTAssertNil(gate.active)
+ }
+}
diff --git a/native/cu-helper/Tests/CuHelperTests/DisplaySleepAssertionTests.swift b/native/cu-helper/Tests/CuHelperTests/DisplaySleepAssertionTests.swift
new file mode 100644
index 00000000..0145eb28
--- /dev/null
+++ b/native/cu-helper/Tests/CuHelperTests/DisplaySleepAssertionTests.swift
@@ -0,0 +1,39 @@
+import XCTest
+@testable import cc_haha_computer_use
+
+final class DisplaySleepAssertionTests: XCTestCase {
+ @MainActor
+ func testTurnAssertionIsAcquiredAndReleasedExactlyOnce() {
+ var creates = 0
+ var releases: [UInt32] = []
+ let assertion = ComputerUseDisplaySleepAssertion(
+ create: {
+ creates += 1
+ return 42
+ },
+ release: { releases.append($0) }
+ )
+
+ assertion.acquire()
+ assertion.acquire()
+ XCTAssertTrue(assertion.isHeldForTesting)
+ XCTAssertEqual(creates, 1)
+
+ assertion.release()
+ assertion.release()
+ XCTAssertFalse(assertion.isHeldForTesting)
+ XCTAssertEqual(releases, [42])
+
+ assertion.acquire()
+ XCTAssertEqual(creates, 2)
+ }
+
+ @MainActor
+ func testFailedPowerAssertionDoesNotPretendItIsHeld() {
+ let assertion = ComputerUseDisplaySleepAssertion(create: { nil })
+
+ assertion.acquire()
+
+ XCTAssertFalse(assertion.isHeldForTesting)
+ }
+}
diff --git a/native/cu-helper/Tests/CuHelperTests/UISettlePolicyTests.swift b/native/cu-helper/Tests/CuHelperTests/UISettlePolicyTests.swift
index 2c0417bc..66ab352f 100644
--- a/native/cu-helper/Tests/CuHelperTests/UISettlePolicyTests.swift
+++ b/native/cu-helper/Tests/CuHelperTests/UISettlePolicyTests.swift
@@ -5,7 +5,7 @@ import XCTest
final class UISettlePolicyTests: XCTestCase {
/// Nothing has been mutated, so nothing is mid-transition on our account.
/// This is the common case for the first `get_app_state` of a session and
- /// must not cost the model a second of latency.
+ /// must not add capture latency.
func testNoMutationMeansNoWait() {
XCTAssertEqual(
UISettlePolicy.delay(now: 100, lastMutationAt: nil, appIsBusy: false),
@@ -33,12 +33,12 @@ final class UISettlePolicyTests: XCTestCase {
)
}
- /// A visible progress indicator extends the window — but only to a bound,
- /// so an app that spins forever cannot hang the capture.
- func testBusyAppGetsTheLongerWindowButStaysBounded() {
+ /// A permanently spinning progress indicator must not turn the one-shot
+ /// settle into a multi-second stall.
+ func testBusyAppUsesTheSameBoundedOneShotWindow() {
let busy = UISettlePolicy.delay(now: 100, lastMutationAt: 98.9, appIsBusy: true)
- XCTAssertGreaterThan(busy, 0)
- XCTAssertLessThanOrEqual(busy, UISettlePolicy.busyWindow)
+ XCTAssertEqual(busy, 0)
+ XCTAssertEqual(UISettlePolicy.busyWindow, UISettlePolicy.postActionWindow)
XCTAssertEqual(
UISettlePolicy.delay(
@@ -67,4 +67,16 @@ final class UISettlePolicyTests: XCTestCase {
XCTAssertLessThanOrEqual(delay, UISettlePolicy.postActionWindow)
XCTAssertGreaterThanOrEqual(delay, 0)
}
+
+ @MainActor
+ func testMutationMarkerIsScopedByPIDAndConsumedOnce() {
+ MutationClock.resetForTests()
+ defer { MutationClock.resetForTests() }
+
+ MutationClock.recordMutation(pid: 101, at: 42)
+
+ XCTAssertNil(MutationClock.takeMutation(pid: 202))
+ XCTAssertEqual(MutationClock.takeMutation(pid: 101), 42)
+ XCTAssertNil(MutationClock.takeMutation(pid: 101))
+ }
}
diff --git a/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift b/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift
index c1ed0627..4c406a0a 100644
--- a/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift
+++ b/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift
@@ -132,6 +132,35 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertEqual(factory.sources[0].latestReadCount, 0, "A cached stream frame must not become the model's screenshot")
}
+ func testPostMutationSnapshotConsumesFreshStreamWatermarkBeforeSkyshot() async throws {
+ let target = makeTarget(windowID: 87)
+ let factory = FakeWindowCaptureStreamFactory { source, _ in
+ source.startFrame = makeFrame(
+ for: source.targetKey,
+ sequence: 1,
+ uptime: 12,
+ byte: 7
+ )
+ }
+ var captures = 0
+ let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
+ captures += 1
+ return self.makeSnapshot(target, pixels: "fresh-skyshot")
+ })
+
+ let shot = await manager.captureSnapshot(
+ for: target,
+ scale: 0.5,
+ newerThanUptime: 11
+ )
+
+ XCTAssertEqual(shot?.base64, "fresh-skyshot")
+ XCTAssertEqual(captures, 1)
+ XCTAssertEqual(factory.sources.count, 1)
+ XCTAssertGreaterThan(factory.sources[0].latestReadCount, 0)
+ XCTAssertEqual(factory.sources[0].retireCount, 0)
+ }
+
func testSnapshotFailureDoesNotFallBackToCachedStreamPixels() async {
let target = makeTarget(windowID: 85)
let factory = FakeWindowCaptureStreamFactory { source, _ in
@@ -171,9 +200,10 @@ final class WindowCaptureStreamTests: XCTestCase {
let target = makeTarget(windowID: 81)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var captures = 0
+ var captureTimes: [TimeInterval] = []
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
captures += 1
- self.assertMutationHasSettledBeforeCapture()
+ captureTimes.append(ProcessInfo.processInfo.systemUptime)
return self.makeSnapshot(target, pixels: "unchanged-pixels")
})
var previousMutation: TimeInterval?
@@ -183,11 +213,20 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertNotNil(MutationClock.lastMutation())
XCTAssertNotEqual(MutationClock.lastMutation(), previousMutation)
previousMutation = MutationClock.lastMutation()
- let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
+ let pendingMutation = MutationClock.takeMutation()
+ let shot = await CommandRouter.captureSettledWindowShot(
+ appIsBusy: false,
+ lastMutationAt: pendingMutation
+ ) {
await manager.captureSnapshot(for: target, scale: 0.5)
}
XCTAssertEqual(shot?.base64, "unchanged-pixels")
XCTAssertEqual(captures, expectedCaptureCount, "Identical pixels still require a new capture")
+ self.assertMutationHasSettledBeforeCapture(
+ pendingMutation,
+ capturedAt: captureTimes[expectedCaptureCount - 1]
+ )
+ XCTAssertNil(MutationClock.lastMutation(), "The settle marker is one-shot")
}
XCTAssertEqual(factory.sources.count, 1)
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
@@ -198,8 +237,9 @@ final class WindowCaptureStreamTests: XCTestCase {
defer { MutationClock.resetForTests() }
let target = makeTarget(windowID: 83)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
+ var capturedAt: TimeInterval?
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
- self.assertMutationHasSettledBeforeCapture()
+ capturedAt = ProcessInfo.processInfo.systemUptime
return self.makeSnapshot(target, pixels: "partial-action-state")
})
do {
@@ -211,18 +251,29 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertEqual(error.code, "partial_action")
}
XCTAssertNotNil(MutationClock.lastMutation())
- let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
+ let pendingMutation = MutationClock.takeMutation()
+ let shot = await CommandRouter.captureSettledWindowShot(
+ appIsBusy: false,
+ lastMutationAt: pendingMutation
+ ) {
await manager.captureSnapshot(for: target, scale: 0.5)
}
XCTAssertEqual(shot?.base64, "partial-action-state")
+ self.assertMutationHasSettledBeforeCapture(
+ pendingMutation,
+ capturedAt: try XCTUnwrap(capturedAt)
+ )
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
}
- private func assertMutationHasSettledBeforeCapture() {
- XCTAssertNotNil(MutationClock.lastMutation())
+ private func assertMutationHasSettledBeforeCapture(
+ _ mutationAt: TimeInterval?,
+ capturedAt: TimeInterval
+ ) {
+ XCTAssertNotNil(mutationAt)
XCTAssertEqual(UISettlePolicy.delay(
- now: ProcessInfo.processInfo.systemUptime,
- lastMutationAt: MutationClock.lastMutation(),
+ now: capturedAt,
+ lastMutationAt: mutationAt,
appIsBusy: false
), 0, "The actual screenshot callback must run after the action's settle deadline")
}
diff --git a/native/cu-helper/build.sh b/native/cu-helper/build.sh
index 81b2991d..7f6ced2e 100755
--- a/native/cu-helper/build.sh
+++ b/native/cu-helper/build.sh
@@ -57,6 +57,8 @@ BIN_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use"
# Accessibility tolerates a bare binary (works), Screen Recording does NOT. So
# the shipped/dragged artifact is the .app; the inner binary is what we spawn.
APP_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use.app"
+# Reuse the desktop brand asset so both Privacy lists show the product logo.
+APP_ICON_PATH="$PKG_DIR/../../desktop/src-tauri/icons/icon.icns"
# Records the (identity, identifier) actually used, so we can detect rotation
# across rebuilds and warn that TCC grants will have been dropped.
SIGN_STAMP="$BUILD_DIR/.cu-helper.signid"
@@ -407,6 +409,7 @@ verify() {
# makes the inner binary's TCC identity a proper app bundle.
# ---------------------------------------------------------------------------
wrap_app() {
+ [ -s "$APP_ICON_PATH" ] || die "App icon not found at $APP_ICON_PATH (needed for the Privacy lists)."
log ""
log "==> wrap .app bundle: $APP_PATH"
rm -rf "$APP_PATH"
@@ -416,6 +419,7 @@ wrap_app() {
[ -f "$PKG_DIR/Info.plist" ] || die "Info.plist not found at $PKG_DIR/Info.plist (needed for the .app bundle)."
cp "$PKG_DIR/Info.plist" "$APP_PATH/Contents/Info.plist"
+ cp "$APP_ICON_PATH" "$APP_PATH/Contents/Resources/icon.icns"
# SwiftPM resource bundle (LensSequence overlay), loaded via Bundle.module.
# Standard .app location is Contents/Resources/ (Bundle.main.resourceURL). Do
diff --git a/native/cu-helper/build.test.ts b/native/cu-helper/build.test.ts
index 64435d60..c7921115 100644
--- a/native/cu-helper/build.test.ts
+++ b/native/cu-helper/build.test.ts
@@ -1,7 +1,59 @@
-import { describe, expect, test } from 'bun:test'
+import { afterEach, describe, expect, test } from 'bun:test'
+import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
+import { tmpdir } from 'node:os'
import path from 'node:path'
const buildScript = path.resolve(import.meta.dirname, 'build.sh')
+const productIcon = path.resolve(import.meta.dirname, '../../desktop/src-tauri/icons/icon.icns')
+const fixtureDirectories: string[] = []
+
+afterEach(() => {
+ for (const directory of fixtureDirectories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true })
+ }
+})
+
+function wrapFixtureApp(missingIcon = false) {
+ const directory = mkdtempSync(path.join(tmpdir(), 'cu-helper-app-icon-'))
+ fixtureDirectories.push(directory)
+ writeFileSync(path.join(directory, 'fixture-binary'), 'fixture executable')
+
+ const result = Bun.spawnSync([
+ 'bash',
+ '-c',
+ `
+source "$1"
+TEST_BUNDLE_DIR="$2"
+BUILD_DIR="$TEST_BUNDLE_DIR/build"
+BIN_PATH="$TEST_BUNDLE_DIR/fixture-binary"
+APP_PATH="$TEST_BUNDLE_DIR/cc-haha-computer-use.app"
+BUNDLE_ID="dev.cchaha.cu-helper"
+SIGN_IDENTITY="fixture-only"
+RESOLVED_TIMESTAMP_MODE="none"
+if [ "$3" = "missing" ]; then
+ APP_ICON_PATH="$TEST_BUNDLE_DIR/missing.icns"
+fi
+codesign() {
+ case "$1" in
+ --force) cp -R "$APP_PATH/Contents" "$TEST_BUNDLE_DIR/contents-at-sign" ;;
+ -dv) printf 'Identifier=%s\\n' "$BUNDLE_ID" ;;
+ esac
+}
+wrap_app
+`,
+ 'cu-helper-app-icon-test',
+ buildScript,
+ directory,
+ missingIcon ? 'missing' : 'present',
+ ], { cwd: directory })
+
+ return {
+ directory,
+ contents: path.join(directory, 'cc-haha-computer-use.app', 'Contents'),
+ exitCode: result.exitCode,
+ stderr: result.stderr.toString(),
+ }
+}
function resolveTimestampArgument(identity: string, mode = 'auto') {
const result = Bun.spawnSync([
@@ -74,3 +126,35 @@ describe('cu-helper build signing identity', () => {
expect(result.stdout).toBe('Developer ID Application: Example Corp (TEAM123456)')
})
})
+
+describe.skipIf(process.platform !== 'darwin')('cu-helper permission-list app icon', () => {
+ test('declares and bundles the product icon before signing the helper app', () => {
+ const result = wrapFixtureApp()
+ expect(result.exitCode).toBe(0)
+
+ const plist = Bun.spawnSync([
+ '/usr/bin/plutil', '-convert', 'json', '-o', '-',
+ path.join(result.contents, 'Info.plist'),
+ ])
+ expect(plist.exitCode).toBe(0)
+ const info = JSON.parse(plist.stdout.toString())
+ expect(info.CFBundleIdentifier).toBe('dev.cchaha.cu-helper')
+ expect(info.CFBundleExecutable).toBe('cc-haha-computer-use')
+ expect(info.CFBundleIconFile).toBe('icon.icns')
+
+ const expectedIcon = readFileSync(productIcon)
+ expect(expectedIcon.subarray(0, 4).toString()).toBe('icns')
+ expect(readFileSync(path.join(result.contents, 'Resources', info.CFBundleIconFile)))
+ .toEqual(expectedIcon)
+ expect(readFileSync(path.join(result.directory, 'contents-at-sign', 'Resources', info.CFBundleIconFile)))
+ .toEqual(expectedIcon)
+ })
+
+ test('refuses to sign an app when the required product icon is missing', () => {
+ const result = wrapFixtureApp(true)
+
+ expect(result.exitCode).not.toBe(0)
+ expect(result.stderr).toContain('App icon not found')
+ expect(existsSync(path.join(result.directory, 'contents-at-sign'))).toBe(false)
+ })
+})
diff --git a/src/skills/bundled/computerUse.test.ts b/src/skills/bundled/computerUse.test.ts
index 1e5df566..cd383734 100644
--- a/src/skills/bundled/computerUse.test.ts
+++ b/src/skills/bundled/computerUse.test.ts
@@ -9,7 +9,7 @@ import {
/**
* Asserting on prose is unusual, but this prose is load-bearing twice over: it
- * is the only thing that carries the operating procedure the ten tools assume,
+ * is the only thing that carries the operating procedure the tool set assumes,
* and it is the only place that says which clicks must not be made without
* asking. Each test names the failure it prevents so anyone trimming a line can
* see what it was buying.
@@ -24,22 +24,29 @@ describe('computer-use skill content', () => {
// success, and reported the task done while the app had not changed.
const prompt = await computerUsePrompt()
expect(prompt).toContain('dispatched')
- expect(prompt).toContain('no change in the accessibility tree')
+ expect(prompt).toContain('AX diff stays empty')
+ expect(prompt).toContain('Judge the screenshot')
})
- test('names the four tools that still work on a dead tree', async () => {
+ test('names the tools that still work on a dead tree', async () => {
// Observed: on a Chromium app whose tree is a bare shell, the model clicked
// element handles fifteen times and never tried the screenshot coordinates
// it already had. "The tree is empty" alone is not actionable — the escape
// hatch has to be enumerated.
const prompt = await computerUsePrompt()
expect(prompt).toContain('will never fill in')
- for (const tool of ['click', 'drag', 'press_key', 'type_text']) {
+ for (const tool of ['click', 'drag', 'press_key', 'type_text', 'paste']) {
expect(prompt).toContain(tool)
}
expect(prompt).toContain('menu bar')
})
+ test('treats a timed-out paste as result-unknown and refreshes before retry', async () => {
+ const prompt = await computerUsePrompt()
+ expect(prompt).toContain('may have consumed the paste late')
+ expect(prompt).toContain('get_app_state')
+ })
+
test('caps repetition and closes the shell escape hatch', async () => {
// Observed: the same failing click repeated many times, then the model
// abandoned the toolset for osascript and Python, burning the session.
diff --git a/src/skills/bundled/computerUse.ts b/src/skills/bundled/computerUse.ts
index c8ec2b14..d9402acf 100644
--- a/src/skills/bundled/computerUse.ts
+++ b/src/skills/bundled/computerUse.ts
@@ -13,6 +13,7 @@ const MAC_COMPUTER_USE_TOOLS = [
'drag',
'press_key',
'type_text',
+ 'paste',
].map(name => `mcp__computer-use__${name}`)
export function getComputerUseToolAllowlist(
@@ -71,9 +72,9 @@ Switch to \`x\`/\`y\` read off the screenshot when either is true:
- **the tree is a dead end.** Many Chromium/Electron apps expose only their
window frame and menu bar. \`get_app_state\` says so explicitly when it detects
- this. That is not a slow-loading tree — it will never fill in. Only four tools
+ this. That is not a slow-loading tree — it will never fill in. Five tools
still work there: \`click\` with x/y, \`drag\` with x/y, \`press_key\`, and
- \`type_text\`. Everything else needs a handle it cannot get. The menu bar stays
+ \`type_text\`/\`paste\`. Everything else needs a handle it cannot get. The menu bar stays
fully addressable, so a menu path is often the shortest route.
- **element actions run but the UI does not change.**
@@ -84,11 +85,10 @@ them as-is; do not convert them.
Mutating tools return a fixed receipt. The receipt means the action was
**dispatched**, never that it had the intended effect. Only the next
-\`get_app_state\` tells you what actually happened — and an empty diff, or the
-line "There has been no change in the accessibility tree", means your action did
-nothing.
+\`get_app_state\` tells you what actually happened. Judge the screenshot as well
+as AX text: Chromium/CEF content can visibly change while the AX diff stays empty.
-If two consecutive attempts leave the state unchanged, the approach is wrong.
+If two consecutive screenshots leave the relevant UI unchanged, the approach is wrong.
Change something real: switch from handle to coordinates, target a different
element, re-read the full tree with \`disableDiff: true\`, or take a different
route through the UI such as the menu bar. Repeating the same call a third time
@@ -109,6 +109,10 @@ session.
element in the tree. Do not guess action names.
- \`press_key\` and \`type_text\` are delivered to the named app, so they cannot
trigger global system shortcuts.
+- If \`type_text\` does not visibly change a Chromium/CEF field, use
+ \`paste({ app, text, format: "text" })\`. It restores the user's prior clipboard.
+ If it times out after dispatch, call \`get_app_state\` before retrying: the app
+ may have consumed the paste late.
- \`press_key\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c".
- \`select_text\` works inside editable elements; use \`prefix\`/\`suffix\` to
disambiguate repeated matches.
diff --git a/src/utils/computerUse/cuHelperDaemon.test.ts b/src/utils/computerUse/cuHelperDaemon.test.ts
index 52dcc56a..7b953f57 100644
--- a/src/utils/computerUse/cuHelperDaemon.test.ts
+++ b/src/utils/computerUse/cuHelperDaemon.test.ts
@@ -7,8 +7,10 @@ import {
__prepareDaemonSocketDirectoryForTests,
__resetDaemonClientForTests,
__daemonStartCountForTests,
+ __negotiateDaemonProtocolForTests,
__reapStaleDaemonsForTests,
__setDaemonSocketForTests,
+ CU_HELPER_PROTOCOL_VERSION,
callDaemon,
DaemonCommandTimeoutError,
DaemonCommandResultUnknownError,
@@ -185,6 +187,76 @@ describe('cu-helper daemon failure classification', () => {
expect(error.message).toBe('grant_flag_required')
})
+ test('every request carries negotiated protocol, deadline, and stable turn identity', async () => {
+ const socket = new FakeSocket()
+ __setDaemonSocketForTests(socket as never, 100)
+
+ const first = callDaemon('get_app_state', { app: 'TextEdit' })
+ await waitForWriteCount(socket, 1)
+ const firstEnvelope = JSON.parse(socket.writes[0]!)
+ const firstTurnId = firstEnvelope.turnId
+ expect(firstEnvelope).toMatchObject({
+ clientApiVersion: CU_HELPER_PROTOCOL_VERSION,
+ requestId: firstEnvelope.id,
+ sessionId: expect.any(String),
+ turnId: expect.any(String),
+ })
+ expect(firstEnvelope.deadlineUnixMilliseconds).toBeGreaterThan(Date.now())
+ reply(socket, 0, { ok: true, result: {} })
+ await first
+
+ const second = callDaemon('click', { app: 'TextEdit', x: 1, y: 1 })
+ await waitForWriteCount(socket, 2)
+ const secondEnvelope = JSON.parse(socket.writes[1]!)
+ expect(secondEnvelope.turnId).toBe(firstTurnId)
+ reply(socket, 1, { ok: true, result: true })
+ await second
+ })
+
+ test('accepts only a daemon hello with the complete negotiated capability set', async () => {
+ const socket = new FakeSocket()
+ __setDaemonSocketForTests(socket as never, 100)
+
+ const negotiation = __negotiateDaemonProtocolForTests()
+ await waitForWrite(socket)
+ expect(JSON.parse(socket.writes[0]!)).toMatchObject({
+ cmd: 'ping',
+ clientApiVersion: CU_HELPER_PROTOCOL_VERSION,
+ })
+ reply(socket, 0, {
+ ok: true,
+ result: {
+ protocolVersion: CU_HELPER_PROTOCOL_VERSION,
+ supportsAbsoluteDeadlines: true,
+ supportsTurnEnd: true,
+ },
+ })
+
+ await expect(negotiation).resolves.toBeUndefined()
+ expect(socket.destroyed).toBe(false)
+ })
+
+ test('rejects and retires a daemon that reports an incomplete hello', async () => {
+ const socket = new FakeSocket()
+ __setDaemonSocketForTests(socket as never, 100)
+
+ const negotiation = __negotiateDaemonProtocolForTests().catch(err => err)
+ await waitForWrite(socket)
+ reply(socket, 0, {
+ ok: true,
+ result: {
+ protocolVersion: CU_HELPER_PROTOCOL_VERSION,
+ supportsAbsoluteDeadlines: true,
+ supportsTurnEnd: false,
+ },
+ })
+
+ const error = await negotiation
+ expect(error).toBeInstanceOf(DaemonUnavailableError)
+ expect(error.message).toMatch(/protocol negotiation failed.*unexpected hello/i)
+ expect(socket.destroyed).toBe(true)
+ })
+
test('post-dispatch timeout is ambiguous and must not be classified as replayable infrastructure', async () => {
const socket = new FakeSocket()
__setDaemonSocketForTests(socket as never, 5)
@@ -279,13 +351,20 @@ describe('cu-helper overlay reconciliation', () => {
reply(socket, 0, { ok: true, result: true })
await waitForWriteCount(socket, 2)
expect(JSON.parse(socket.writes[1]!)).toMatchObject({
- cmd: 'overlay_hide',
+ cmd: 'turn_end',
payload: {},
})
reply(socket, 1, { ok: true, result: true })
await Promise.all([show, hide])
expect(isOverlayShown()).toBe(false)
+
+ const next = callDaemon('get_app_state', { app: 'TextEdit' })
+ await waitForWriteCount(socket, 3)
+ expect(JSON.parse(socket.writes[2]!).turnId)
+ .not.toBe(JSON.parse(socket.writes[0]!).turnId)
+ reply(socket, 2, { ok: true, result: {} })
+ await next
})
test('a target change while show is pending serially retargets to the latest app', async () => {
@@ -318,10 +397,68 @@ describe('cu-helper overlay reconciliation', () => {
await show
expect(isOverlayShown()).toBe(false)
+ const hide = overlayHide()
+ await waitForWriteCount(socket, 2)
+ expect(JSON.parse(socket.writes[1]!)).toMatchObject({ cmd: 'turn_end' })
+ reply(socket, 1, { ok: true, result: true })
+ await hide
+ })
+
+ test('cleanup ends a read-only turn even when no overlay was shown', async () => {
+ const socket = new FakeSocket()
+ __setDaemonSocketForTests(socket as never)
+
+ const state = callDaemon('get_app_state', { app: 'TextEdit' })
+ await waitForWriteCount(socket, 1)
+ const stateEnvelope = JSON.parse(socket.writes[0]!)
+ reply(socket, 0, { ok: true, result: {} })
+ await state
+
+ const hide = overlayHide()
+ await waitForWriteCount(socket, 2)
+ const endEnvelope = JSON.parse(socket.writes[1]!)
+ expect(endEnvelope).toMatchObject({ cmd: 'turn_end', turnId: stateEnvelope.turnId })
+ reply(socket, 1, { ok: true, result: true })
+ await hide
+
+ const next = callDaemon('get_app_state', { app: 'TextEdit' })
+ await waitForWriteCount(socket, 3)
+ expect(JSON.parse(socket.writes[2]!).turnId).not.toBe(stateEnvelope.turnId)
+ reply(socket, 2, { ok: true, result: {} })
+ await next
+ })
+
+ test('connection-scoped permission checks do not manufacture a turn to clean up', async () => {
+ const socket = new FakeSocket()
+ __setDaemonSocketForTests(socket as never)
+
+ const check = callDaemon('check_permissions', {})
+ await waitForWriteCount(socket, 1)
+ expect(JSON.parse(socket.writes[0]!).turnId).toMatch(/^connection-/)
+ reply(socket, 0, { ok: true, result: {} })
+ await check
+
await overlayHide()
expect(socket.writes).toHaveLength(1)
})
+ test('a rejected turn_end retires the daemon instead of poisoning the next turn', async () => {
+ const socket = new FakeSocket()
+ __setDaemonSocketForTests(socket as never)
+
+ const state = callDaemon('get_app_state', { app: 'TextEdit' })
+ await waitForWriteCount(socket, 1)
+ reply(socket, 0, { ok: true, result: {} })
+ await state
+
+ const hide = overlayHide()
+ await waitForWriteCount(socket, 2)
+ reply(socket, 1, { ok: false, error: { message: 'deadline_exceeded' } })
+ await hide
+
+ expect(socket.destroyed).toBe(true)
+ })
+
test('cleanup with no daemon does not start one', async () => {
expect(__daemonStartCountForTests()).toBe(0)
await overlayHide()
diff --git a/src/utils/computerUse/cuHelperDaemon.ts b/src/utils/computerUse/cuHelperDaemon.ts
index 659e00aa..271c3cb2 100644
--- a/src/utils/computerUse/cuHelperDaemon.ts
+++ b/src/utils/computerUse/cuHelperDaemon.ts
@@ -1,7 +1,9 @@
import { spawn, spawnSync, type ChildProcess } from 'node:child_process'
+import { randomUUID } from 'node:crypto'
import fs from 'node:fs'
import net from 'node:net'
import path from 'node:path'
+import { getSessionId } from '../../bootstrap/state.js'
import { logForDebugging } from '../debug.js'
import { ensureInstalledHelper } from './cuHelperInstall.js'
import { attestDaemonSocketPeer } from './cuHelperPeerAttestation.js'
@@ -13,10 +15,10 @@ import { getRuntimePaths } from './pythonBridge.js'
* The daemon owns the main run loop that the animated virtual cursor and the
* animated virtual cursor needs, and holds the virtual cursor's position +
* held-input state across commands. We spawn ONE daemon per CLI process, keep
- * an AF_UNIX socket open to it, and speak the NDJSON request/response protocol:
+ * an AF_UNIX socket open to it, and speak a versioned NDJSON protocol:
*
- * readiness : {"ready":true,"pid":N,"proto":1}\n (the daemon's stdout, once)
- * request : {"id":"","cmd":"","payload":{...}}\n
+ * readiness : {"ready":true,"pid":N,"protocolVersion":"..."}\n
+ * request : {"id":"","requestId":"","cmd":"",...}\n
* response : {"id":"","ok":true,"result":...}\n | {"id":"","ok":false,"error":{...}}\n
*
* Routing through the daemon (instead of one-shot CLI) is what makes execution
@@ -31,6 +33,7 @@ import { getRuntimePaths } from './pythonBridge.js'
const REQUEST_TIMEOUT_MS = 20_000
const READINESS_TIMEOUT_MS = 8_000
+export const CU_HELPER_PROTOCOL_VERSION = 'CCHahaComputerUseIPC-2'
/**
* Thrown ONLY for daemon INFRASTRUCTURE failures known to happen before the
@@ -107,6 +110,7 @@ let overlayActualKey: string | undefined
let overlayRevision = 0
let overlayReconcilePromise: Promise | undefined
let requestTimeoutMs = REQUEST_TIMEOUT_MS
+let activeTurnId: string | undefined
function socketPath(generation: number): string {
const { runtimeStateRoot } = getRuntimePaths()
@@ -379,6 +383,7 @@ function resetState(reason: string, expectedGeneration?: number): void {
overlayActualVisible = false
overlayActualKey = undefined
overlayRevision++
+ activeTurnId = undefined
const p = statePromise
statePromise = undefined
activeDaemonGeneration = undefined
@@ -513,6 +518,8 @@ async function startDaemon(generation: number): Promise {
attachSocketHandlers(state)
+ await negotiateDaemonProtocol(state)
+
return state
}
@@ -542,6 +549,31 @@ function attachSocketHandlers(state: DaemonState): void {
state.socket.on('error', err => resetState(`socket error: ${String(err)}`, state.generation))
}
+async function negotiateDaemonProtocol(state: DaemonState): Promise {
+ // Socket ownership + code-signature attestation identify the peer. The
+ // versioned hello additionally proves that the installed helper understands
+ // deadlines and explicit turn cleanup before we dispatch any real command.
+ try {
+ const hello = await dispatchDaemonCommand<{
+ protocolVersion?: string
+ supportsAbsoluteDeadlines?: boolean
+ supportsTurnEnd?: boolean
+ }>(state, 'ping', {})
+ if (
+ hello.protocolVersion !== CU_HELPER_PROTOCOL_VERSION
+ || hello.supportsAbsoluteDeadlines !== true
+ || hello.supportsTurnEnd !== true
+ ) {
+ throw new Error(`unexpected hello ${JSON.stringify(hello)}`)
+ }
+ } catch (err) {
+ state.socket.destroy()
+ throw new DaemonUnavailableError(
+ `cu-helper protocol negotiation failed: ${err instanceof Error ? err.message : String(err)}`,
+ )
+ }
+}
+
async function ensureDaemon(): Promise {
if (!statePromise) {
const generation = ++daemonGeneration
@@ -573,6 +605,19 @@ function dispatchDaemonCommand(
))
}
const id = String(++state.nextId)
+ const isTurnScoped = !['ping', 'check_permissions', 'shutdown'].includes(command)
+ const turnId = activeTurnId
+ ?? (isTurnScoped ? (activeTurnId = randomUUID()) : `connection-${state.generation}`)
+ const request = {
+ id,
+ requestId: id,
+ cmd: command,
+ payload,
+ clientApiVersion: CU_HELPER_PROTOCOL_VERSION,
+ deadlineUnixMilliseconds: Date.now() + requestTimeoutMs,
+ sessionId: getSessionId(),
+ turnId,
+ }
return new Promise((resolve, reject) => {
const timer = setTimeout(() => {
state.pending.delete(id)
@@ -588,7 +633,7 @@ function dispatchDaemonCommand(
}, requestTimeoutMs)
state.pending.set(id, { resolve: resolve as (v: unknown) => void, reject, timer })
try {
- state.socket.write(`${JSON.stringify({ id, cmd: command, payload })}\n`)
+ state.socket.write(`${JSON.stringify(request)}\n`)
} catch (err) {
clearTimeout(timer)
state.pending.delete(id)
@@ -597,6 +642,10 @@ function dispatchDaemonCommand(
reject(new DaemonUnavailableError(err instanceof Error ? err.message : String(err)))
resetState(`socket write failed: ${String(err)}`, state.generation)
}
+ }).finally(() => {
+ if ((command === 'turn_end' || command === 'overlay_hide') && activeTurnId === turnId) {
+ activeTurnId = undefined
+ }
})
}
@@ -627,7 +676,10 @@ async function callExistingDaemon(
}
function needsOverlayReconciliation(): boolean {
- if (!overlayDesiredVisible) return overlayActualVisible
+ // A turn may contain only get_app_state and therefore never show the overlay.
+ // Its native state (capture stream, AX baseline, held-input guard and display
+ // sleep assertion) still needs an explicit turn_end at host cleanup.
+ if (!overlayDesiredVisible) return overlayActualVisible || activeTurnId !== undefined
return !overlayActualVisible || overlayActualKey !== overlayDesiredKey
}
@@ -662,16 +714,21 @@ async function reconcileOverlay(): Promise {
// A turn-end cleanup must never launch a daemon just to hide it. When a
// pending show completes this branch sees the already-owned daemon and
- // serially sends the compensating hide.
+ // serially ends the turn. This also covers read-only turns whose overlay
+ // was never visible.
if (!statePromise || activeDaemonGeneration === undefined) {
overlayActualVisible = false
overlayActualKey = undefined
return
}
try {
- await callExistingDaemon('overlay_hide', {})
+ await callExistingDaemon('turn_end', {})
} catch (err) {
- logForDebugging(`cu-helper overlay_hide failed: ${String(err)}`, { level: 'debug' })
+ logForDebugging(`cu-helper turn_end failed: ${String(err)}`, { level: 'debug' })
+ // If the helper rejected/lost turn_end it may still own the old turn.
+ // Retire it now so the next user turn cannot inherit stale native state
+ // or fail forever with turn_mismatch.
+ resetState('turn_end failed')
}
overlayActualVisible = false
overlayActualKey = undefined
@@ -740,6 +797,7 @@ export function __resetDaemonClientForTests(): void {
overlayReconcilePromise = undefined
daemonStartCount = 0
requestTimeoutMs = REQUEST_TIMEOUT_MS
+ activeTurnId = undefined
}
/** Focused proof that a no-daemon cleanup did not enter the startup path. */
@@ -775,3 +833,8 @@ export function __setDaemonSocketForTests(
attachSocketHandlers(state)
statePromise = Promise.resolve(state)
}
+
+/** Drive the same hello negotiation used by production against an installed test socket. */
+export async function __negotiateDaemonProtocolForTests(): Promise {
+ await negotiateDaemonProtocol(await ensureDaemon())
+}
diff --git a/src/utils/computerUse/executor.codexEngine.test.ts b/src/utils/computerUse/executor.codexEngine.test.ts
index 1dbe900d..1eb6e804 100644
--- a/src/utils/computerUse/executor.codexEngine.test.ts
+++ b/src/utils/computerUse/executor.codexEngine.test.ts
@@ -264,6 +264,15 @@ describe('CLI executor Codex engine — daemon payload alignment', () => {
expect(lastCall()).toEqual({ command: 'type_text', payload: { text: 'hello' } })
})
+ itEngine('paste sends explicit content format', async () => {
+ const exec = await loadExecutor()
+ await exec.engine!.paste({ target: { app: 'NeteaseMusic' }, text: '喜欢你', format: 'text' })
+ expect(lastCall()).toEqual({
+ command: 'paste',
+ payload: { app: 'NeteaseMusic', text: '喜欢你', format: 'text' },
+ })
+ })
+
itEngine('selectText sends text range parameters', async () => {
const exec = await loadExecutor()
await exec.engine!.selectText({
diff --git a/src/utils/computerUse/executor.ts b/src/utils/computerUse/executor.ts
index cbcbabd2..834fc376 100644
--- a/src/utils/computerUse/executor.ts
+++ b/src/utils/computerUse/executor.ts
@@ -91,7 +91,7 @@ async function writeClipboard(text: string): Promise {
// ----------------------------------------------------------------------------
// Codex semantic engine (blueprint §4–§7)
//
-// Maps the ten `CodexComputerEngine` methods onto `callHelper(, payload)`
+// Maps the Codex-compatible `CodexComputerEngine` methods onto `callHelper(, payload)`
// round-trips against the native daemon's `CommandRouter`. Each command's
// payload key names mirror what `CommandRouter` decodes (see CommandRouter.swift):
// - target: `pid` | `bundleId` | `app` (resolveTargetPid precedence)
@@ -287,6 +287,18 @@ export function createCodexEngine(): CodexComputerEngine {
text: args.text,
})
},
+
+ async paste(args: {
+ target: AppTarget
+ text: string
+ format: 'text' | 'md' | 'html'
+ }): Promise {
+ await callHelper('paste', {
+ ...appTargetPayload(args.target),
+ text: args.text,
+ format: args.format,
+ })
+ },
}
}
diff --git a/src/vendor/computer-use-mcp/executor.ts b/src/vendor/computer-use-mcp/executor.ts
index 54ea4bda..a2c72ba9 100644
--- a/src/vendor/computer-use-mcp/executor.ts
+++ b/src/vendor/computer-use-mcp/executor.ts
@@ -206,7 +206,7 @@ export interface CodexComputerEngine {
*/
pressKey(args: { target: AppTarget; key: string; systemKeyCombos: boolean }): Promise
/** Type literal text (append to focused value; Unicode keyboard fallback). */
- typeText(args: { target: AppTarget; text: string }): Promise
+ typeText(args: { target: AppTarget; text: string }): Promise; paste(args: { target: AppTarget; text: string; format: 'text' | 'md' | 'html' }): Promise
}
export interface InstalledApp {
diff --git a/src/vendor/computer-use-mcp/instructions.ts b/src/vendor/computer-use-mcp/instructions.ts
index 6fa88a52..401e0a09 100644
--- a/src/vendor/computer-use-mcp/instructions.ts
+++ b/src/vendor/computer-use-mcp/instructions.ts
@@ -67,7 +67,9 @@ Mutating tools return a fixed receipt. The receipt means "the action was
dispatched", NOT "it had the intended effect" — you must look at the next
\`get_app_state\` to know.
-If two consecutive attempts leave the state unchanged, the approach is wrong.
+Judge success from the screenshot as well as the AX text. An empty AX diff does
+not mean a Chromium/CEF interface stayed unchanged. If two consecutive screenshots
+leave the relevant UI unchanged, the approach is wrong.
Change something real: switch from element handle to coordinates, target a
different element, re-read the full tree with \`disableDiff: true\`, or take a
different route through the UI. Repeating the same call a third time never helps.
@@ -86,6 +88,10 @@ waste the user's time.
element in the tree. Do not guess action names.
- \`press_key\` and \`type_text\` are delivered to the named app, so they cannot
trigger global system shortcuts.
+- If \`type_text\` does not visually change a Chromium/CEF field, use
+ \`paste({ app, text, format: "text" })\`; it restores the user's prior clipboard.
+ If paste times out after dispatch, treat the result as unknown and call
+ \`get_app_state\` before retrying, because the target may have consumed it late.
- \`press_key\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c".
- \`select_text\` works inside editable elements; use \`prefix\`/\`suffix\` to
disambiguate repeated matches.
diff --git a/src/vendor/computer-use-mcp/platformRouting.test.ts b/src/vendor/computer-use-mcp/platformRouting.test.ts
index 5696a5a6..7005c5de 100644
--- a/src/vendor/computer-use-mcp/platformRouting.test.ts
+++ b/src/vendor/computer-use-mcp/platformRouting.test.ts
@@ -19,6 +19,7 @@ const DARWIN_TOOL_NAMES = [
'get_app_state',
'list_apps',
'perform_secondary_action',
+ 'paste',
'press_key',
'scroll',
'select_text',
@@ -288,7 +289,7 @@ function makeWindowsAdapter(calls: string[]): ComputerUseHostAdapter {
}
describe('Computer Use platform routing', () => {
- test('darwin ListTools advertises exactly the ten semantic tools', async () => {
+ test('darwin ListTools advertises the current semantic tools', async () => {
const connection = await connect(makeDarwinAdapter())
try {
const result = await connection.client.listTools()
diff --git a/src/vendor/computer-use-mcp/toolCalls.test.ts b/src/vendor/computer-use-mcp/toolCalls.test.ts
index 592b88d1..bac77d96 100644
--- a/src/vendor/computer-use-mcp/toolCalls.test.ts
+++ b/src/vendor/computer-use-mcp/toolCalls.test.ts
@@ -17,6 +17,7 @@ import {
resetMouseButtonHeld,
} from './toolCalls.js'
import { buildComputerUseTools } from './tools.js'
+import { COMPUTER_USE_INSTRUCTIONS } from './instructions.js'
import { isSystemKeyCombo } from './keyBlocklist.js'
import type {
ComputerUseHostAdapter,
@@ -128,6 +129,7 @@ function makeEngine(
drag: record('drag', async () => {}),
pressKey: record('pressKey', async () => {}),
typeText: record('typeText', async () => {}),
+ paste: record('paste', async () => {}),
selectText: record('selectText', async () => {}),
}
return { engine, calls }
@@ -208,10 +210,10 @@ function imageBlocks(result: { content: unknown }): Array<{ data?: string; mimeT
// Tool schema
// ---------------------------------------------------------------------------
-describe('buildComputerUseTools — Codex 10-tool face', () => {
+describe('buildComputerUseTools — current Codex tool face', () => {
const tools = buildComputerUseTools()
- test('exposes exactly the ten Codex tools, verbatim names', () => {
+ test('exposes the current Codex tools, including explicit paste', () => {
expect(tools.map(t => t.name).sort()).toEqual(
[
'click',
@@ -219,6 +221,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => {
'get_app_state',
'list_apps',
'perform_secondary_action',
+ 'paste',
'press_key',
'scroll',
'select_text',
@@ -318,6 +321,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => {
'drag',
'press_key',
'type_text',
+ 'paste',
]) {
const description = tools.find(t => t.name === name)!.description ?? ''
expect(description).toContain('get_app_state')
@@ -433,7 +437,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => {
'normalized_0_100',
['Finder', 'Slack'],
)
- expect(withArgs).toHaveLength(10)
+ expect(withArgs).toHaveLength(11)
})
})
@@ -1575,6 +1579,45 @@ describe('handleToolCall — tool dispatch', () => {
expect(calls.find(c => c.method === 'typeText')!.args).toMatchObject({ target: { pid: 1234 }, text: 'Codex' })
})
+ test('paste passes text and format as an explicit recovery action', async () => {
+ const { engine, calls } = makeEngine()
+ await handleToolCall(
+ makeAdapter({ engine }),
+ 'paste',
+ { app: 'Activity Monitor', text: '喜欢你', format: 'text' },
+ baseOverrides(),
+ )
+ expect(calls.find(c => c.method === 'paste')!.args).toMatchObject({
+ target: { pid: 1234 },
+ text: '喜欢你',
+ format: 'text',
+ })
+ })
+
+ test('paste rejects missing text and unknown formats before target resolution', async () => {
+ for (const args of [
+ { app: 'Activity Monitor', format: 'text' },
+ { app: 'Activity Monitor', text: 'hello', format: 'rtf' },
+ ]) {
+ const { engine, calls } = makeEngine()
+ const result = await handleToolCall(
+ makeAdapter({ engine }),
+ 'paste',
+ args,
+ baseOverrides(),
+ )
+ expect(result.isError).toBe(true)
+ expect(result.telemetry?.error_kind).toBe('bad_args')
+ expect(calls).toHaveLength(0)
+ }
+ })
+
+ test('server guidance treats AX diffs and timed-out paste as non-authoritative', () => {
+ expect(COMPUTER_USE_INSTRUCTIONS).toContain('An empty AX diff does')
+ expect(COMPUTER_USE_INSTRUCTIONS).toContain('paste({ app, text, format: "text" })')
+ expect(COMPUTER_USE_INSTRUCTIONS).toContain('treat the result as unknown')
+ })
+
test('select_text passes the opaque handle, context, and selection mode', async () => {
const { engine, calls } = makeEngine()
await handleToolCall(
diff --git a/src/vendor/computer-use-mcp/toolCalls.ts b/src/vendor/computer-use-mcp/toolCalls.ts
index ba759221..1ec7c9d2 100644
--- a/src/vendor/computer-use-mcp/toolCalls.ts
+++ b/src/vendor/computer-use-mcp/toolCalls.ts
@@ -6,7 +6,7 @@
* commands):
*
* list_apps, get_app_state, click, perform_secondary_action, set_value,
- * select_text, scroll, drag, press_key, type_text
+ * select_text, scroll, drag, press_key, type_text, paste
*
* ## Three properties this file exists to hold
*
@@ -565,6 +565,7 @@ const MUTATING_TOOLS: ReadonlySet = new Set([
"drag",
"press_key",
"type_text",
+ "paste",
]);
const KNOWN_TOOLS: ReadonlySet = new Set([
@@ -801,6 +802,24 @@ function parseRequest(
};
}
+ case "paste": {
+ const text = args.text;
+ if (typeof text !== "string") {
+ throw new BadArgs('Missing required string "text"');
+ }
+ const format = requiredString(args, "format");
+ if (format !== "text" && format !== "md" && format !== "html") {
+ throw new BadArgs('paste format must be "text", "md", or "html"');
+ }
+ return {
+ ...base,
+ run: async (engine, t) => {
+ await engine.paste({ target: t, text, format });
+ return okText(MUTATION_RECEIPT);
+ },
+ };
+ }
+
default:
throw new BadArgs(`Unknown computer-use tool "${name}".`);
}
diff --git a/src/vendor/computer-use-mcp/tools.ts b/src/vendor/computer-use-mcp/tools.ts
index c4c979f6..cadc93d6 100644
--- a/src/vendor/computer-use-mcp/tools.ts
+++ b/src/vendor/computer-use-mcp/tools.ts
@@ -1,14 +1,14 @@
/**
* MCP tool schemas for the computer-use server — Codex-compatible semantic
- * face (blueprint §7). Exactly TEN tools, named verbatim after Codex's public
+ * face (blueprint §7). Eleven tools, named verbatim after Codex's public
* computer-use MCP:
*
* list_apps, get_app_state, click, perform_secondary_action, set_value,
- * select_text, scroll, drag, press_key, type_text
+ * select_text, scroll, drag, press_key, type_text, paste
*
* This replaces the prior 27-tool pixel face. The legacy `coordinateMode` /
* `screenshotFiltering` parameters are accepted for call-site compatibility
- * but no longer influence the schema — the ten tool shapes are static.
+ * but no longer influence the schema — the tool shapes are static.
*
* Param names mirror Codex exactly (verified against iFurySt's reverse-
* engineered ToolDefinitions.swift and the 2026-04-17 tool-call samples):
@@ -83,7 +83,7 @@ function coordinateProp(axis: "x" | "y", role: string) {
}
/**
- * Build the ten-tool Codex computer-use face.
+ * Build the Codex computer-use face.
*
* Signature is preserved from the legacy pixel builder so existing call sites
* (`setup.ts`, host `mcpServer.ts`) keep compiling. All three parameters are
@@ -415,5 +415,34 @@ export function buildComputerUseTools(
required: ["app", "text"],
},
},
+
+ {
+ name: "paste",
+ annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false },
+ description:
+ "Paste content into whatever currently has keyboard focus in the target " +
+ "app. Prefer this when type_text did not change a Chromium/CEF field, for " +
+ "Chinese text, formatted content, or multiline text. The user's previous " +
+ "clipboard is restored unless they copy something during the operation. " +
+ "A timeout after Command-V is result-unknown: call get_app_state before " +
+ "deciding whether to retry. Call get_app_state afterwards to see the result.",
+ inputSchema: {
+ type: "object" as const,
+ additionalProperties: false,
+ properties: {
+ app: APP_PROP,
+ text: {
+ type: "string",
+ description: "The content to paste.",
+ },
+ format: {
+ type: "string",
+ enum: ["text", "md", "html"],
+ description: "Pasteboard representation to use.",
+ },
+ },
+ required: ["app", "text", "format"],
+ },
+ },
];
}