fix(computer-use): align long-running macOS automation with Codex

This commit is contained in:
程序员阿江(Relakkes)
2026-09-01 17:31:17 +08:00
parent 8751e558e5
commit ad7321f7cc
32 changed files with 1100 additions and 99 deletions
+2
View File
@@ -21,6 +21,8 @@
<string>cc-haha-computer-use</string>
<key>CFBundleDisplayName</key>
<string>cc-haha-computer-use</string>
<key>CFBundleIconFile</key>
<string>icon.icns</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
@@ -237,10 +237,18 @@ public final class ClipboardLease {
}
func writeTemporaryStringWithReceipt(_ text: String) throws -> ClipboardPasteReceipt {
try writeTemporaryContentWithReceipt(text, format: .text)
}
func writeTemporaryContentWithReceipt(
_ text: String,
format: ClipboardPasteFormat
) throws -> ClipboardPasteReceipt {
if let captureError { throw captureError }
let receipt = ClipboardPasteReceipt(text: text)
let receipt = ClipboardPasteReceipt(text: text, format: format)
let item = NSPasteboardItem()
guard item.setDataProvider(receipt, forTypes: [.string]) else {
let types = Array(format.promisedData(for: text).keys)
guard item.setDataProvider(receipt, forTypes: types) else {
throw CUError("clipboard_write_failed", "Could not register temporary pasteboard data")
}
pasteboard.clearContents()
@@ -926,13 +934,35 @@ public enum AXAction {
try await typeViaClipboard(target: target, text)
}
/// Explicit Codex-compatible paste path. Unlike `typeText`, this does not
/// first try AX value mutation or Unicode key events; it targets the app's
/// in-process focused field with Command-V and restores the user's original
/// pasteboard after observing a real promised-data read.
static func pasteText(
pid: pid_t,
_ text: String,
format: ClipboardPasteFormat
) async throws {
guard !text.isEmpty else { return }
let target = try Injection.authorizeResolvedTarget(pid: pid)
try await typeViaClipboard(target: target, text, format: format)
}
/// Paste `text` into whatever holds in-app keyboard focus in `pid`, via the
/// system clipboard + ⌘V (Codex's approach for CEF/Chromium text fields).
/// The read receipt confirms supplied clipboard bytes, not the reader's
/// PID or the field's final value; the next screenshot still verifies UI.
private static func typeViaClipboard(target: ProvenProcessTarget, _ text: String) async throws {
private static func typeViaClipboard(
target: ProvenProcessTarget,
_ text: String,
format: ClipboardPasteFormat = .text
) async throws {
let pid = target.pid
try await ClipboardPasteReceipt.perform(text: text, lease: ClipboardLease()) { validate in
try await ClipboardPasteReceipt.perform(
text: text,
format: format,
lease: ClipboardLease()
) { validate in
try await pressKey(pid: pid, "super+v", validateBeforePosting: {
_ = try Injection.validateAuthorizedTarget(target)
try validate()
@@ -1609,10 +1639,11 @@ public enum AXAction {
return CFEqual(lhs, rhs)
}
/// Mark that a mutation just landed. Does NOT block: `get_app_state` reads
/// this stamp and waits out whatever settle time is still owed, so the cost
/// is paid once, and only when someone is actually about to look.
/// Settlement is recorded once at `ForegroundMutationRunner`, which covers
/// every input path and knows the target PID. AX-local success receipts must
/// not create a second, unscoped marker that can leak across apps.
private static func settle() {
MutationClock.recordMutation()
// Intentionally empty; retained at the AX call sites as a semantic
// marker for successful mutations.
}
}
@@ -30,9 +30,9 @@ enum HelperClientPolicy {
// read-only authorization preflight used before the ten public tools.
"list_apps", "resolve_app_target", "get_app_state", "click",
"set_value", "select_text", "perform_secondary_action", "scroll",
"drag", "press_key", "type_text",
"drag", "press_key", "type_text", "paste",
// Private lifecycle / visible-overlay / permission and input diagnostics.
"ping", "shutdown", "overlay_show", "overlay_hide",
"ping", "shutdown", "overlay_show", "overlay_hide", "turn_end",
"check_permissions", "input_monitor_state", "focus_monitor_state", "held_input_state",
"last_injection_state",
]
@@ -2,6 +2,32 @@ import AppKit
import Foundation
import os
enum ClipboardPasteFormat: String, Sendable {
case text
case md
case html
func promisedData(for text: String) -> [NSPasteboard.PasteboardType: Data] {
switch self {
case .text, .md:
return [.string: Data(text.utf8)]
case .html:
let html = Data(text.utf8)
let plain = (
try? NSAttributedString(
data: html,
options: [
.documentType: NSAttributedString.DocumentType.html,
.characterEncoding: String.Encoding.utf8.rawValue,
],
documentAttributes: nil
).string
) ?? text
return [.html: html, .string: Data(plain.utf8)]
}
}
}
/// Confirms that this pasteboard item's promised bytes were requested and
/// supplied. AppKit does not identify the reader: this is not proof that the
/// intended application's focused field accepted the text.
@@ -25,16 +51,26 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
}
@MainActor private(set) static var lastDiagnostic: Diagnostic?
private let data: Data
private let promisedData: [NSPasteboard.PasteboardType: Data]
private let state = OSAllocatedUnfairLock(initialState: State())
init(text: String) {
data = Data(text.utf8)
promisedData = ClipboardPasteFormat.text.promisedData(for: text)
super.init()
}
init(text: String, format: ClipboardPasteFormat) {
promisedData = format.promisedData(for: text)
super.init()
}
@MainActor
static func resetForTurn() {
lastDiagnostic = nil
}
func pasteboard(_ pasteboard: NSPasteboard?, item: NSPasteboardItem, provideDataForType type: NSPasteboard.PasteboardType) {
guard type == .string else { return }
guard let data = promisedData[type] else { return }
state.withLock { $0.requested = true }
guard item.setData(data, forType: type) else { return }
state.withLock { value in
@@ -50,6 +86,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
@MainActor
static func perform(
text: String,
format: ClipboardPasteFormat = .text,
lease: ClipboardLease,
timeout: Duration = .seconds(2),
sendPaste: @MainActor (_ validateBeforePosting: @MainActor () throws -> Void) async throws -> Void
@@ -79,7 +116,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
}
do {
try Task.checkCancellation()
let written = try lease.writeTemporaryStringWithReceipt(text)
let written = try lease.writeTemporaryContentWithReceipt(text, format: format)
receipt = written
try await Task.sleep(for: .milliseconds(40))
let validate: @MainActor () throws -> Void = {
@@ -104,6 +104,7 @@ public final class CommandRouter {
Self.lastShotTransform.removeAll()
Self.lastCaptureDigest.removeAll()
MutationClock.reset()
ClipboardPasteReceipt.resetForTurn()
windowCaptureProvider?.invalidate()
// Apps we told they were focused must be told they are not, or the
// belief outlives the session that needed it.
@@ -267,6 +268,9 @@ public final class CommandRouter {
case "type_text":
return try await handleTypeText(payload)
case "paste":
return try await handlePaste(payload)
case "press_key":
return try await handlePressKey(payload)
@@ -728,18 +732,19 @@ public final class CommandRouter {
if #available(macOS 14.0, *) {
// Let the UI finish whatever the last action started before we
// photograph it, otherwise the model reasons about a half-drawn
// frame. Costs nothing when no action is pending. The rendered tree
// doubles as the busy signal — a progress indicator in it means the
// app is still working, so we allow a longer window.
// frame. Costs nothing when no action is pending, applies only to
// this target PID, and is consumed by this single capture.
let pendingMutation = MutationClock.takeMutation(pid: pid)
let streamedShot = await Self.captureSettledWindowShot(
appIsBusy: result.axText.contains("progress indicator")
appIsBusy: result.axText.contains("progress indicator"),
lastMutationAt: pendingMutation
) {
await windowCaptureProvider?.windowShot(
pid: pid,
processIdentity: snapshotEvidence.processIdentity,
preferredWindowID: snapshotEvidence.keyWindowID,
scale: 0.5,
newerThanUptime: MutationClock.lastMutation()
newerThanUptime: pendingMutation
)
}
guard TargetVisibilityPolicy.captureTargetStillMatches(
@@ -875,9 +880,13 @@ public final class CommandRouter {
/// action-to-screenshot regression can exercise both without live AX/TCC.
static func captureSettledWindowShot(
appIsBusy: Bool,
lastMutationAt: TimeInterval?,
capture: () async -> WindowShot?
) async -> WindowShot? {
await MutationClock.awaitSettle(appIsBusy: appIsBusy)
await MutationClock.awaitSettle(
lastMutationAt: lastMutationAt,
appIsBusy: appIsBusy
)
return await capture()
}
@@ -1438,6 +1447,31 @@ public final class CommandRouter {
}
}
/// `paste`: bypass AX/Unicode typing and use the target app's in-process
/// paste focus. This is the reliable recovery path for Chromium/CEF fields
/// such as NeteaseMusic's search box. The clipboard lease restores every
/// original pasteboard item unless the user copied something meanwhile.
private func handlePaste(_ payload: JSONValue) async throws -> JSONValue {
try requireAXTrusted()
guard let text = payload["text"]?.asString else {
throw CUError("bad_payload", "paste requires a 'text' string")
}
guard let rawFormat = payload["format"]?.asString,
let format = ClipboardPasteFormat(rawValue: rawFormat) else {
throw CUError("bad_payload", "paste format must be 'text', 'md', or 'html'")
}
let expected = try Self.expectedProcessTarget(payload)
let target = try resolveTargetForMutation(payload)
setResolvedTarget(target)
try requireSnapshotProcess(target: target, expected: expected)
return try await withForegroundLease(command: "paste", target: target) {
_ = try Injection.validateAuthorizedTarget(target)
try self.requireSnapshotProcess(target: target, expected: expected)
try await AXAction.pasteText(pid: target.pid, text, format: format)
return .bool(true)
}
}
/// `press_key`: send an xdotool-style key spec (e.g. "super+c", "Return",
/// "Tab", "KP_0", "Prior") to the target app via postToPid.
private func handlePressKey(_ payload: JSONValue) async throws -> JSONValue {
@@ -1566,6 +1600,7 @@ public final class CommandRouter {
)
return try await ForegroundMutationRunner.run(
lease: lease,
targetPID: target.pid,
action: action
)
}
+35 -12
View File
@@ -38,21 +38,21 @@ enum DaemonOverlayTargetResolver {
// `cursor_position`, implicit-`from` drags and decomposed `mouse_down`/`mouse_up`
// behave correctly across commands.
//
// Transport: a private AF_UNIX SOCK_STREAM socket carrying NDJSON (one JSON
// object + '\n' per request, same per response). A GUI/AppKit process leaks
// Transport: a private AF_UNIX SOCK_STREAM socket carrying versioned NDJSON
// (one JSON object + '\n' per request, same per response). A GUI/AppKit process leaks
// os_log / CoreGraphics chatter to stdout/stderr, which would corrupt the TS
// bridge's `JSON.parse`; the daemon therefore reserves stdout for exactly ONE
// readiness line and serves the request/response stream over the socket.
//
// readiness : {"ready":true,"pid":<N>,"proto":1}\n (stdout, once)
// request : {"id":"<opaque>","cmd":"<verb>","payload":{...}}\n
// readiness : {"ready":true,"pid":<N>,"protocolVersion":"…"}\n
// request : {"id":"<opaque>","requestId":"<opaque>","cmd":"<verb>",…}\n
// response : {"id":"<opaque>","ok":true,"result":<json>}\n
// {"id":"<opaque>","ok":false,"error":{"message":"…","code":"…"}}\n
//
// Control verbs handled in-daemon (never reach CommandRouter):
// overlay_show -> cursor.show() ; result true
// overlay_hide -> cursor.hide() ; result true
// ping -> result "pong"
// turn_end -> release all turn-owned state ; result true
// ping -> version/capability hello
// shutdown -> result true, then NSApp.terminate(nil)
// Every other cmd is forwarded to the shared CommandRouter — the exact same
// dispatcher the one-shot CLI path uses, so each command has one implementation.
@@ -63,6 +63,7 @@ public final class Daemon {
private let cursor: VirtualCursor
private let inputMonitor: PhysicalInputEpochMonitor
private let router: CommandRouter
private let displaySleepAssertion = ComputerUseDisplaySleepAssertion()
/// Listening socket fd (AF_UNIX SOCK_STREAM). -1 until `bindAndListen`.
private var listenFD: Int32 = -1
@@ -78,11 +79,12 @@ public final class Daemon {
private var connection: Connection?
private var connectionToken: DaemonSessionToken?
private var sessionGate = DaemonSessionGate()
private var turnGate = DaemonTurnGate()
/// Set once the run loop is live; guards against double `run()`.
private var didStartRunLoop = false
/// True between `overlay_show` and `overlay_hide` (CU active this turn). The
/// True between `overlay_show` and `turn_end` (CU active this turn). The
/// cursor re-aims at the actual injection target while this holds.
private var overlayActive = false
private var explicitOverlayTarget: ProvenProcessTarget?
@@ -220,6 +222,7 @@ public final class Daemon {
// immediately by exit() (shutdown verb + signal handlers).
Injection.releaseAllHeldSync()
router.resetSessionState()
displaySleepAssertion.release()
inputMonitor.stop()
}
@@ -435,6 +438,8 @@ public final class Daemon {
stopOverlaySession()
Injection.releaseAllHeldSync()
router.resetSessionState()
turnGate.reset()
displaySleepAssertion.release()
}
/// A LaunchServices child is reparented to launchd, so the peer socket is
@@ -479,7 +484,16 @@ public final class Daemon {
let payload = request.payload ?? .object([:])
do {
let metadata = try ComputerUseDaemonProtocol.validate(request)
let opensTurn = ComputerUseDaemonProtocol.isTurnScoped(command: request.cmd)
&& turnGate.active == nil
try turnGate.admit(metadata, command: request.cmd)
if opensTurn { displaySleepAssertion.acquire() }
let result = try await route(cmd: request.cmd, payload: payload)
if request.cmd == "turn_end" || request.cmd == "overlay_hide" {
try turnGate.finish(metadata)
displaySleepAssertion.release()
}
conn.send(encodeResponse(DaemonResponse(id: id, ok: true, result: result, error: nil)))
} catch let cuError as CUError {
conn.send(encodeResponse(DaemonResponse(
@@ -508,12 +522,12 @@ public final class Daemon {
try showOverlay(payload: payload)
return .bool(true)
case "overlay_hide":
stopOverlaySession()
case "overlay_hide", "turn_end":
endTurn()
return .bool(true)
case "ping":
return .string("pong")
return ComputerUseDaemonProtocol.hello()
case "check_permissions":
// Usable in both modes; the daemon exposes it for onboarding /
@@ -592,6 +606,15 @@ public final class Daemon {
router.invalidateWindowCaptureStream()
}
/// Codex-parity turn boundary. The helper process stays warm, but no AX
/// snapshot, coordinate transform, focus belief, held input, mutation clock,
/// clipboard diagnostic, or capture stream may leak into the next turn.
private func endTurn() {
stopOverlaySession()
Injection.releaseAllHeldSync()
router.resetSessionState()
}
/// The app the cursor should follow: ONLY the app the last injection /
/// get_app_state actually resolved (`Injection.lastResolvedTargetPid`). We
/// deliberately do NOT fall back to the frontmost app — at turn start,
@@ -763,9 +786,9 @@ private final class Connection: @unchecked Sendable {
private var inBuffer = Data()
/// Capped to avoid unbounded growth from a malformed/never-newline client.
/// 16 MiB comfortably exceeds the largest legitimate request (a base64
/// 8 MiB comfortably exceeds the largest legitimate request (a base64
/// screenshot travels in *responses*, not requests).
private let maxBufferBytes = 16 * 1024 * 1024
private let maxBufferBytes = ComputerUseDaemonProtocol.maxFrameBytes
private var onRequest: (@Sendable (Data) -> Void)?
private var onClose: (@Sendable () -> Void)?
@@ -0,0 +1,99 @@
import Foundation
/// Versioned request contract for the authenticated daemon socket.
///
/// Code signature attestation proves who is connected; this handshake proves
/// that both sides agree on request shape and lifecycle semantics. Every request
/// also carries an absolute deadline plus session/turn identity so a command
/// queued behind a slow capture cannot execute after its caller has moved on.
enum ComputerUseDaemonProtocol {
static let version = "CCHahaComputerUseIPC-2"
static let maxFrameBytes = 8 * 1024 * 1024
private static let connectionScopedCommands: Set<String> = [
"ping", "check_permissions", "shutdown",
]
static func isTurnScoped(command: String) -> Bool {
!connectionScopedCommands.contains(command)
}
struct Metadata: Equatable, Sendable {
let sessionId: String
let turnId: String
}
static func validate(
_ request: Request,
nowUnixMilliseconds: Int64 = Int64(Date().timeIntervalSince1970 * 1_000)
) throws -> Metadata {
guard request.clientApiVersion == version else {
throw CUError(
"protocol_mismatch",
"Computer Use client/helper protocol mismatch; restart the app after updating"
)
}
guard let id = request.id, !id.isEmpty,
request.requestId == id else {
throw CUError("bad_request_id", "Computer Use request identity is missing or inconsistent")
}
guard let deadline = request.deadlineUnixMilliseconds else {
throw CUError("missing_deadline", "Computer Use request is missing its absolute deadline")
}
guard deadline > nowUnixMilliseconds else {
throw CUError("deadline_exceeded", "Computer Use request expired before execution")
}
guard let sessionId = request.sessionId?.trimmingCharacters(in: .whitespacesAndNewlines),
!sessionId.isEmpty,
let turnId = request.turnId?.trimmingCharacters(in: .whitespacesAndNewlines),
!turnId.isEmpty else {
throw CUError("missing_turn_metadata", "Computer Use request is missing session/turn identity")
}
return Metadata(sessionId: sessionId, turnId: turnId)
}
static func hello() -> JSONValue {
.object([
"protocolVersion": .string(version),
"supportsAbsoluteDeadlines": .bool(true),
"supportsTurnEnd": .bool(true),
"supportsCaptureDiagnostics": .bool(true),
])
}
}
/// Enforces one explicit turn at a time on the single authenticated connection.
/// `ping` negotiates the protocol without opening a turn. Every other request
/// must keep the same identity until `turn_end` releases all turn-owned state.
struct DaemonTurnGate {
private(set) var active: ComputerUseDaemonProtocol.Metadata?
mutating func admit(
_ metadata: ComputerUseDaemonProtocol.Metadata,
command: String
) throws {
if !ComputerUseDaemonProtocol.isTurnScoped(command: command) { return }
if let active {
guard active == metadata else {
throw CUError(
"turn_mismatch",
"A different Computer Use turn is still active; finish it before starting another"
)
}
return
}
active = metadata
}
mutating func finish(
_ metadata: ComputerUseDaemonProtocol.Metadata
) throws {
guard active == metadata else {
throw CUError("turn_mismatch", "Computer Use turn_end did not match the active turn")
}
active = nil
}
mutating func reset() {
active = nil
}
}
@@ -0,0 +1,50 @@
import Foundation
import IOKit.pwr_mgt
/// Keeps the display compositor awake while a Computer Use turn is active.
/// Covered-window ScreenCaptureKit consumers can otherwise stop receiving
/// useful updates once macOS enters user-idle display sleep during a long task.
@MainActor
final class ComputerUseDisplaySleepAssertion {
typealias AssertionID = IOPMAssertionID
private let create: () -> AssertionID?
private let releaseAssertion: (AssertionID) -> Void
private var assertionID: AssertionID?
init(
create: @escaping () -> AssertionID? = {
var assertionID: IOPMAssertionID = 0
let result = IOPMAssertionCreateWithDescription(
kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString,
"Claude Code Haha Computer Use interaction" as CFString,
"Computer Use turn is controlling a background application" as CFString,
"Keeping the display awake while Computer Use is active" as CFString,
nil,
0,
nil,
&assertionID
)
return result == kIOReturnSuccess ? assertionID : nil
},
release: @escaping (AssertionID) -> Void = { assertionID in
_ = IOPMAssertionRelease(assertionID)
}
) {
self.create = create
self.releaseAssertion = release
}
func acquire() {
guard assertionID == nil else { return }
assertionID = create()
}
func release() {
guard let assertionID else { return }
self.assertionID = nil
releaseAssertion(assertionID)
}
var isHeldForTesting: Bool { assertionID != nil }
}
@@ -394,12 +394,13 @@ final class ForegroundLease {
enum ForegroundMutationRunner {
static func run<T>(
lease: ForegroundLease,
targetPID: pid_t? = nil,
action: () async throws -> T
) async throws -> T {
// Every input path crosses this boundary, including synthetic events
// that never call AXAction.settle(). A throw can follow a partial
// delivery, so neither success nor failure may reuse pre-action pixels.
defer { MutationClock.recordMutation() }
defer { MutationClock.recordMutation(pid: targetPID) }
let result: Result<T, Error>
do {
result = .success(try await action())
@@ -439,11 +439,30 @@ public struct Request: Decodable, Sendable {
public let id: String?
public let cmd: String
public let payload: JSONValue?
public let clientApiVersion: String?
public let deadlineUnixMilliseconds: Int64?
public let sessionId: String?
public let turnId: String?
public let requestId: String?
public init(id: String?, cmd: String, payload: JSONValue?) {
public init(
id: String?,
cmd: String,
payload: JSONValue?,
clientApiVersion: String? = nil,
deadlineUnixMilliseconds: Int64? = nil,
sessionId: String? = nil,
turnId: String? = nil,
requestId: String? = nil
) {
self.id = id
self.cmd = cmd
self.payload = payload
self.clientApiVersion = clientApiVersion
self.deadlineUnixMilliseconds = deadlineUnixMilliseconds
self.sessionId = sessionId
self.turnId = turnId
self.requestId = requestId
}
/// The payload, defaulting to an empty object when omitted/null — every
@@ -14,18 +14,19 @@ import Foundation
/// which it no longer does. Instead the wait happens once, at the moment of
/// capture, and only if an action happened recently enough to still be settling.
///
/// This mirrors Codex's `needsUISettleBeforeSkyshot`: wait about a second after
/// a recent action, and extend that when the app is visibly still working.
/// This mirrors Codex's one-shot `needsUISettleBeforeSkyshot`: the next capture
/// for that target waits about 250ms after a recent action. A different app and
/// later captures do not inherit the wait.
///
/// Pure and clock-injected so the decision is testable without sleeping.
enum UISettlePolicy {
/// How long after a mutation the UI is presumed to still be settling.
static let postActionWindow: TimeInterval = 1.0
static let postActionWindow: TimeInterval = 0.25
/// Ceiling while the app still shows a busy/progress indicator. Generous
/// because "still loading" is a real signal, but bounded so a permanently
/// spinning indicator (some apps never stop one) cannot hang the capture.
static let busyWindow: TimeInterval = 5.0
/// Kept as a named compatibility seam for callers that already compute a
/// busy signal. Codex does not turn that signal into a multi-second sleep;
/// freshness comes from subsequent on-demand captures instead.
static let busyWindow: TimeInterval = postActionWindow
/// Never wait less than this once we've decided to wait at all — a delay
/// too short to cover a frame boundary is just latency for nothing.
@@ -57,28 +58,52 @@ enum UISettlePolicy {
}
}
/// Records when the last mutating action completed, so the capture path can ask
/// "was something just changed?" without the action path having to block.
/// Records one pending capture settle per target process, so an action in one
/// app cannot delay a screenshot of another app.
///
/// `@MainActor` because every mutation already runs there; this keeps the state
/// single-threaded without a lock.
@MainActor
enum MutationClock {
private static var lastMutationAt: TimeInterval?
private static var pendingByPID: [pid_t: TimeInterval] = [:]
private static var unscopedMutationAt: TimeInterval?
/// Called by the action path after a mutation lands. Does not sleep.
static func recordMutation(at time: TimeInterval = ProcessInfo.processInfo.systemUptime) {
lastMutationAt = time
static func recordMutation(
pid: pid_t? = nil,
at time: TimeInterval = ProcessInfo.processInfo.systemUptime
) {
if let pid {
pendingByPID[pid] = time
} else {
unscopedMutationAt = time
}
}
static func lastMutation() -> TimeInterval? { lastMutationAt }
static func lastMutation(pid: pid_t? = nil) -> TimeInterval? {
guard let pid else { return unscopedMutationAt }
return pendingByPID[pid]
}
/// Consume the one-shot marker before waiting. Another capture cannot pay
/// the same action's settle cost a second time.
static func takeMutation(pid: pid_t? = nil) -> TimeInterval? {
guard let pid else {
defer { unscopedMutationAt = nil }
return unscopedMutationAt
}
return pendingByPID.removeValue(forKey: pid)
}
/// Wait out whatever settle time the last mutation still owes.
///
/// Uses `Task.sleep`, not `Thread.sleep`: this runs on the main actor, where
/// blocking would starve the overlay's display link and freeze the virtual
/// cursor animation mid-glide.
static func awaitSettle(appIsBusy: Bool) async {
static func awaitSettle(
lastMutationAt: TimeInterval?,
appIsBusy: Bool
) async {
let delay = UISettlePolicy.delay(
now: ProcessInfo.processInfo.systemUptime,
lastMutationAt: lastMutationAt,
@@ -88,7 +113,10 @@ enum MutationClock {
try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
}
static func reset() { lastMutationAt = nil }
static func reset() {
pendingByPID.removeAll()
unscopedMutationAt = nil
}
static func resetForTests() { reset() }
}
@@ -172,7 +172,11 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
invalidate()
return nil
}
guard let shot = await captureSnapshot(for: target, scale: scale) else {
guard let shot = await captureSnapshot(
for: target,
scale: scale,
newerThanUptime: newerThanUptime
) else {
return nil
}
if let preferredWindowID,
@@ -213,7 +217,20 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
/// Match the reference's two separate lifetimes: SCStream remains a
/// consumer while covered; every state read runs an on-demand Skyshot/SCK
/// capture. An idle stream's cached frame is not evidence of the current UI.
func captureSnapshot(for target: WindowCaptureStreamTarget, scale: Double) async -> WindowShot? {
func captureSnapshot(
for target: WindowCaptureStreamTarget,
scale: Double,
newerThanUptime: TimeInterval? = nil
) async -> WindowShot? {
if let newerThanUptime {
// The stream is a long-lived render/freshness consumer, not the
// model screenshot source. Before the post-action Skyshot, observe
// a stream frame newer than the action when possible. `frame`
// performs one bounded rebuild for a silently starved stream; a
// static/no-op UI may legitimately emit no changed frame, so the
// authoritative on-demand screenshot still runs after the bound.
_ = await frame(for: target, newerThanUptime: newerThanUptime)
}
for _ in 0..<2 {
guard let source = await source(for: target) else { continue }
if source.hasFailed {
@@ -365,8 +365,8 @@ struct ClientAttestationTests {
let allowed = [
"list_apps", "resolve_app_target", "get_app_state", "click",
"set_value", "select_text", "perform_secondary_action", "scroll",
"drag", "press_key", "type_text", "ping", "shutdown",
"overlay_show", "overlay_hide", "check_permissions",
"drag", "press_key", "type_text", "paste", "ping", "shutdown",
"overlay_show", "overlay_hide", "turn_end", "check_permissions",
"input_monitor_state", "held_input_state",
]
for command in allowed {
@@ -5,6 +5,29 @@ import XCTest
@testable import cc_haha_computer_use
final class ClipboardPasteReceiptTests: XCTestCase {
@MainActor
func testHtmlPastePromisesRichAndPlainRepresentationsThenRestoresClipboard() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
let lease = ClipboardLease(pasteboard: fixture.board)
try await ClipboardPasteReceipt.perform(
text: "<strong>Hello</strong> world",
format: .html,
lease: lease
) { validate in
try await fixture.sendPaste(validate)
XCTAssertEqual(
String(data: fixture.board.data(forType: .html) ?? Data(), encoding: .utf8),
"<strong>Hello</strong> world"
)
XCTAssertTrue((fixture.board.string(forType: .string) ?? "").contains("Hello world"))
}
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.status, "completed")
XCTAssertEqual(fixture.board.string(forType: .string), "original")
}
@MainActor
func testPasteWaitsForARealReadBeyondTheOld180MillisecondWindow() async throws {
let fixture = PasteReceiptFixture()
@@ -0,0 +1,116 @@
import XCTest
@testable import cc_haha_computer_use
final class DaemonProtocolTests: XCTestCase {
private func request(
version: String? = ComputerUseDaemonProtocol.version,
deadline: Int64? = 2_000,
sessionId: String? = "session-a",
turnId: String? = "turn-a",
id: String? = "request-a",
requestId: String? = "request-a",
command: String = "get_app_state"
) -> Request {
Request(
id: id,
cmd: command,
payload: .object([:]),
clientApiVersion: version,
deadlineUnixMilliseconds: deadline,
sessionId: sessionId,
turnId: turnId,
requestId: requestId
)
}
func testValidRequestCarriesVersionDeadlineAndTurnIdentity() throws {
let metadata = try ComputerUseDaemonProtocol.validate(
request(),
nowUnixMilliseconds: 1_000
)
XCTAssertEqual(metadata.sessionId, "session-a")
XCTAssertEqual(metadata.turnId, "turn-a")
}
func testProtocolMismatchAndExpiredDeadlineFailClosed() {
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
request(version: "CCHahaComputerUseIPC-1"),
nowUnixMilliseconds: 1_000
)) { error in
XCTAssertEqual((error as? CUError)?.code, "protocol_mismatch")
}
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
request(deadline: 1_000),
nowUnixMilliseconds: 1_000
)) { error in
XCTAssertEqual((error as? CUError)?.code, "deadline_exceeded")
}
}
func testRequestIdentityAndTurnMetadataCannotBeOmitted() {
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
request(requestId: "different"),
nowUnixMilliseconds: 1_000
)) { error in
XCTAssertEqual((error as? CUError)?.code, "bad_request_id")
}
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
request(turnId: " "),
nowUnixMilliseconds: 1_000
)) { error in
XCTAssertEqual((error as? CUError)?.code, "missing_turn_metadata")
}
}
func testTurnGateRejectsCrossTurnStateUntilMatchingEnd() throws {
let first = ComputerUseDaemonProtocol.Metadata(
sessionId: "session-a",
turnId: "turn-a"
)
let second = ComputerUseDaemonProtocol.Metadata(
sessionId: "session-a",
turnId: "turn-b"
)
var gate = DaemonTurnGate()
try gate.admit(first, command: "get_app_state")
try gate.admit(first, command: "click")
XCTAssertThrowsError(try gate.admit(second, command: "get_app_state")) {
XCTAssertEqual(($0 as? CUError)?.code, "turn_mismatch")
}
try gate.finish(first)
try gate.admit(second, command: "get_app_state")
XCTAssertEqual(gate.active, second)
}
func testPingNegotiatesWithoutOpeningATurn() throws {
let metadata = ComputerUseDaemonProtocol.Metadata(
sessionId: "session-a",
turnId: "handshake-1"
)
var gate = DaemonTurnGate()
try gate.admit(metadata, command: "ping")
XCTAssertNil(gate.active)
XCTAssertEqual(
ComputerUseDaemonProtocol.hello()["protocolVersion"]?.asString,
ComputerUseDaemonProtocol.version
)
}
func testPermissionProbeIsConnectionScopedAndDoesNotOpenATurn() throws {
let metadata = ComputerUseDaemonProtocol.Metadata(
sessionId: "session-a",
turnId: "probe"
)
var gate = DaemonTurnGate()
try gate.admit(metadata, command: "check_permissions")
XCTAssertNil(gate.active)
}
}
@@ -0,0 +1,39 @@
import XCTest
@testable import cc_haha_computer_use
final class DisplaySleepAssertionTests: XCTestCase {
@MainActor
func testTurnAssertionIsAcquiredAndReleasedExactlyOnce() {
var creates = 0
var releases: [UInt32] = []
let assertion = ComputerUseDisplaySleepAssertion(
create: {
creates += 1
return 42
},
release: { releases.append($0) }
)
assertion.acquire()
assertion.acquire()
XCTAssertTrue(assertion.isHeldForTesting)
XCTAssertEqual(creates, 1)
assertion.release()
assertion.release()
XCTAssertFalse(assertion.isHeldForTesting)
XCTAssertEqual(releases, [42])
assertion.acquire()
XCTAssertEqual(creates, 2)
}
@MainActor
func testFailedPowerAssertionDoesNotPretendItIsHeld() {
let assertion = ComputerUseDisplaySleepAssertion(create: { nil })
assertion.acquire()
XCTAssertFalse(assertion.isHeldForTesting)
}
}
@@ -5,7 +5,7 @@ import XCTest
final class UISettlePolicyTests: XCTestCase {
/// Nothing has been mutated, so nothing is mid-transition on our account.
/// This is the common case for the first `get_app_state` of a session and
/// must not cost the model a second of latency.
/// must not add capture latency.
func testNoMutationMeansNoWait() {
XCTAssertEqual(
UISettlePolicy.delay(now: 100, lastMutationAt: nil, appIsBusy: false),
@@ -33,12 +33,12 @@ final class UISettlePolicyTests: XCTestCase {
)
}
/// A visible progress indicator extends the window — but only to a bound,
/// so an app that spins forever cannot hang the capture.
func testBusyAppGetsTheLongerWindowButStaysBounded() {
/// A permanently spinning progress indicator must not turn the one-shot
/// settle into a multi-second stall.
func testBusyAppUsesTheSameBoundedOneShotWindow() {
let busy = UISettlePolicy.delay(now: 100, lastMutationAt: 98.9, appIsBusy: true)
XCTAssertGreaterThan(busy, 0)
XCTAssertLessThanOrEqual(busy, UISettlePolicy.busyWindow)
XCTAssertEqual(busy, 0)
XCTAssertEqual(UISettlePolicy.busyWindow, UISettlePolicy.postActionWindow)
XCTAssertEqual(
UISettlePolicy.delay(
@@ -67,4 +67,16 @@ final class UISettlePolicyTests: XCTestCase {
XCTAssertLessThanOrEqual(delay, UISettlePolicy.postActionWindow)
XCTAssertGreaterThanOrEqual(delay, 0)
}
@MainActor
func testMutationMarkerIsScopedByPIDAndConsumedOnce() {
MutationClock.resetForTests()
defer { MutationClock.resetForTests() }
MutationClock.recordMutation(pid: 101, at: 42)
XCTAssertNil(MutationClock.takeMutation(pid: 202))
XCTAssertEqual(MutationClock.takeMutation(pid: 101), 42)
XCTAssertNil(MutationClock.takeMutation(pid: 101))
}
}
@@ -132,6 +132,35 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertEqual(factory.sources[0].latestReadCount, 0, "A cached stream frame must not become the model's screenshot")
}
func testPostMutationSnapshotConsumesFreshStreamWatermarkBeforeSkyshot() async throws {
let target = makeTarget(windowID: 87)
let factory = FakeWindowCaptureStreamFactory { source, _ in
source.startFrame = makeFrame(
for: source.targetKey,
sequence: 1,
uptime: 12,
byte: 7
)
}
var captures = 0
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
captures += 1
return self.makeSnapshot(target, pixels: "fresh-skyshot")
})
let shot = await manager.captureSnapshot(
for: target,
scale: 0.5,
newerThanUptime: 11
)
XCTAssertEqual(shot?.base64, "fresh-skyshot")
XCTAssertEqual(captures, 1)
XCTAssertEqual(factory.sources.count, 1)
XCTAssertGreaterThan(factory.sources[0].latestReadCount, 0)
XCTAssertEqual(factory.sources[0].retireCount, 0)
}
func testSnapshotFailureDoesNotFallBackToCachedStreamPixels() async {
let target = makeTarget(windowID: 85)
let factory = FakeWindowCaptureStreamFactory { source, _ in
@@ -171,9 +200,10 @@ final class WindowCaptureStreamTests: XCTestCase {
let target = makeTarget(windowID: 81)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var captures = 0
var captureTimes: [TimeInterval] = []
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
captures += 1
self.assertMutationHasSettledBeforeCapture()
captureTimes.append(ProcessInfo.processInfo.systemUptime)
return self.makeSnapshot(target, pixels: "unchanged-pixels")
})
var previousMutation: TimeInterval?
@@ -183,11 +213,20 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertNotNil(MutationClock.lastMutation())
XCTAssertNotEqual(MutationClock.lastMutation(), previousMutation)
previousMutation = MutationClock.lastMutation()
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
let pendingMutation = MutationClock.takeMutation()
let shot = await CommandRouter.captureSettledWindowShot(
appIsBusy: false,
lastMutationAt: pendingMutation
) {
await manager.captureSnapshot(for: target, scale: 0.5)
}
XCTAssertEqual(shot?.base64, "unchanged-pixels")
XCTAssertEqual(captures, expectedCaptureCount, "Identical pixels still require a new capture")
self.assertMutationHasSettledBeforeCapture(
pendingMutation,
capturedAt: captureTimes[expectedCaptureCount - 1]
)
XCTAssertNil(MutationClock.lastMutation(), "The settle marker is one-shot")
}
XCTAssertEqual(factory.sources.count, 1)
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
@@ -198,8 +237,9 @@ final class WindowCaptureStreamTests: XCTestCase {
defer { MutationClock.resetForTests() }
let target = makeTarget(windowID: 83)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var capturedAt: TimeInterval?
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
self.assertMutationHasSettledBeforeCapture()
capturedAt = ProcessInfo.processInfo.systemUptime
return self.makeSnapshot(target, pixels: "partial-action-state")
})
do {
@@ -211,18 +251,29 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertEqual(error.code, "partial_action")
}
XCTAssertNotNil(MutationClock.lastMutation())
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
let pendingMutation = MutationClock.takeMutation()
let shot = await CommandRouter.captureSettledWindowShot(
appIsBusy: false,
lastMutationAt: pendingMutation
) {
await manager.captureSnapshot(for: target, scale: 0.5)
}
XCTAssertEqual(shot?.base64, "partial-action-state")
self.assertMutationHasSettledBeforeCapture(
pendingMutation,
capturedAt: try XCTUnwrap(capturedAt)
)
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
}
private func assertMutationHasSettledBeforeCapture() {
XCTAssertNotNil(MutationClock.lastMutation())
private func assertMutationHasSettledBeforeCapture(
_ mutationAt: TimeInterval?,
capturedAt: TimeInterval
) {
XCTAssertNotNil(mutationAt)
XCTAssertEqual(UISettlePolicy.delay(
now: ProcessInfo.processInfo.systemUptime,
lastMutationAt: MutationClock.lastMutation(),
now: capturedAt,
lastMutationAt: mutationAt,
appIsBusy: false
), 0, "The actual screenshot callback must run after the action's settle deadline")
}
+4
View File
@@ -57,6 +57,8 @@ BIN_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use"
# Accessibility tolerates a bare binary (works), Screen Recording does NOT. So
# the shipped/dragged artifact is the .app; the inner binary is what we spawn.
APP_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use.app"
# Reuse the desktop brand asset so both Privacy lists show the product logo.
APP_ICON_PATH="$PKG_DIR/../../desktop/src-tauri/icons/icon.icns"
# Records the (identity, identifier) actually used, so we can detect rotation
# across rebuilds and warn that TCC grants will have been dropped.
SIGN_STAMP="$BUILD_DIR/.cu-helper.signid"
@@ -407,6 +409,7 @@ verify() {
# makes the inner binary's TCC identity a proper app bundle.
# ---------------------------------------------------------------------------
wrap_app() {
[ -s "$APP_ICON_PATH" ] || die "App icon not found at $APP_ICON_PATH (needed for the Privacy lists)."
log ""
log "==> wrap .app bundle: $APP_PATH"
rm -rf "$APP_PATH"
@@ -416,6 +419,7 @@ wrap_app() {
[ -f "$PKG_DIR/Info.plist" ] || die "Info.plist not found at $PKG_DIR/Info.plist (needed for the .app bundle)."
cp "$PKG_DIR/Info.plist" "$APP_PATH/Contents/Info.plist"
cp "$APP_ICON_PATH" "$APP_PATH/Contents/Resources/icon.icns"
# SwiftPM resource bundle (LensSequence overlay), loaded via Bundle.module.
# Standard .app location is Contents/Resources/ (Bundle.main.resourceURL). Do
+85 -1
View File
@@ -1,7 +1,59 @@
import { describe, expect, test } from 'bun:test'
import { afterEach, describe, expect, test } from 'bun:test'
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import path from 'node:path'
const buildScript = path.resolve(import.meta.dirname, 'build.sh')
const productIcon = path.resolve(import.meta.dirname, '../../desktop/src-tauri/icons/icon.icns')
const fixtureDirectories: string[] = []
afterEach(() => {
for (const directory of fixtureDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true })
}
})
function wrapFixtureApp(missingIcon = false) {
const directory = mkdtempSync(path.join(tmpdir(), 'cu-helper-app-icon-'))
fixtureDirectories.push(directory)
writeFileSync(path.join(directory, 'fixture-binary'), 'fixture executable')
const result = Bun.spawnSync([
'bash',
'-c',
`
source "$1"
TEST_BUNDLE_DIR="$2"
BUILD_DIR="$TEST_BUNDLE_DIR/build"
BIN_PATH="$TEST_BUNDLE_DIR/fixture-binary"
APP_PATH="$TEST_BUNDLE_DIR/cc-haha-computer-use.app"
BUNDLE_ID="dev.cchaha.cu-helper"
SIGN_IDENTITY="fixture-only"
RESOLVED_TIMESTAMP_MODE="none"
if [ "$3" = "missing" ]; then
APP_ICON_PATH="$TEST_BUNDLE_DIR/missing.icns"
fi
codesign() {
case "$1" in
--force) cp -R "$APP_PATH/Contents" "$TEST_BUNDLE_DIR/contents-at-sign" ;;
-dv) printf 'Identifier=%s\\n' "$BUNDLE_ID" ;;
esac
}
wrap_app
`,
'cu-helper-app-icon-test',
buildScript,
directory,
missingIcon ? 'missing' : 'present',
], { cwd: directory })
return {
directory,
contents: path.join(directory, 'cc-haha-computer-use.app', 'Contents'),
exitCode: result.exitCode,
stderr: result.stderr.toString(),
}
}
function resolveTimestampArgument(identity: string, mode = 'auto') {
const result = Bun.spawnSync([
@@ -74,3 +126,35 @@ describe('cu-helper build signing identity', () => {
expect(result.stdout).toBe('Developer ID Application: Example Corp (TEAM123456)')
})
})
describe.skipIf(process.platform !== 'darwin')('cu-helper permission-list app icon', () => {
test('declares and bundles the product icon before signing the helper app', () => {
const result = wrapFixtureApp()
expect(result.exitCode).toBe(0)
const plist = Bun.spawnSync([
'/usr/bin/plutil', '-convert', 'json', '-o', '-',
path.join(result.contents, 'Info.plist'),
])
expect(plist.exitCode).toBe(0)
const info = JSON.parse(plist.stdout.toString())
expect(info.CFBundleIdentifier).toBe('dev.cchaha.cu-helper')
expect(info.CFBundleExecutable).toBe('cc-haha-computer-use')
expect(info.CFBundleIconFile).toBe('icon.icns')
const expectedIcon = readFileSync(productIcon)
expect(expectedIcon.subarray(0, 4).toString()).toBe('icns')
expect(readFileSync(path.join(result.contents, 'Resources', info.CFBundleIconFile)))
.toEqual(expectedIcon)
expect(readFileSync(path.join(result.directory, 'contents-at-sign', 'Resources', info.CFBundleIconFile)))
.toEqual(expectedIcon)
})
test('refuses to sign an app when the required product icon is missing', () => {
const result = wrapFixtureApp(true)
expect(result.exitCode).not.toBe(0)
expect(result.stderr).toContain('App icon not found')
expect(existsSync(path.join(result.directory, 'contents-at-sign'))).toBe(false)
})
})
+11 -4
View File
@@ -9,7 +9,7 @@ import {
/**
* Asserting on prose is unusual, but this prose is load-bearing twice over: it
* is the only thing that carries the operating procedure the ten tools assume,
* is the only thing that carries the operating procedure the tool set assumes,
* and it is the only place that says which clicks must not be made without
* asking. Each test names the failure it prevents so anyone trimming a line can
* see what it was buying.
@@ -24,22 +24,29 @@ describe('computer-use skill content', () => {
// success, and reported the task done while the app had not changed.
const prompt = await computerUsePrompt()
expect(prompt).toContain('dispatched')
expect(prompt).toContain('no change in the accessibility tree')
expect(prompt).toContain('AX diff stays empty')
expect(prompt).toContain('Judge the screenshot')
})
test('names the four tools that still work on a dead tree', async () => {
test('names the tools that still work on a dead tree', async () => {
// Observed: on a Chromium app whose tree is a bare shell, the model clicked
// element handles fifteen times and never tried the screenshot coordinates
// it already had. "The tree is empty" alone is not actionable — the escape
// hatch has to be enumerated.
const prompt = await computerUsePrompt()
expect(prompt).toContain('will never fill in')
for (const tool of ['click', 'drag', 'press_key', 'type_text']) {
for (const tool of ['click', 'drag', 'press_key', 'type_text', 'paste']) {
expect(prompt).toContain(tool)
}
expect(prompt).toContain('menu bar')
})
test('treats a timed-out paste as result-unknown and refreshes before retry', async () => {
const prompt = await computerUsePrompt()
expect(prompt).toContain('may have consumed the paste late')
expect(prompt).toContain('get_app_state')
})
test('caps repetition and closes the shell escape hatch', async () => {
// Observed: the same failing click repeated many times, then the model
// abandoned the toolset for osascript and Python, burning the session.
+10 -6
View File
@@ -13,6 +13,7 @@ const MAC_COMPUTER_USE_TOOLS = [
'drag',
'press_key',
'type_text',
'paste',
].map(name => `mcp__computer-use__${name}`)
export function getComputerUseToolAllowlist(
@@ -71,9 +72,9 @@ Switch to \`x\`/\`y\` read off the screenshot when either is true:
- **the tree is a dead end.** Many Chromium/Electron apps expose only their
window frame and menu bar. \`get_app_state\` says so explicitly when it detects
this. That is not a slow-loading tree — it will never fill in. Only four tools
this. That is not a slow-loading tree — it will never fill in. Five tools
still work there: \`click\` with x/y, \`drag\` with x/y, \`press_key\`, and
\`type_text\`. Everything else needs a handle it cannot get. The menu bar stays
\`type_text\`/\`paste\`. Everything else needs a handle it cannot get. The menu bar stays
fully addressable, so a menu path is often the shortest route.
- **element actions run but the UI does not change.**
@@ -84,11 +85,10 @@ them as-is; do not convert them.
Mutating tools return a fixed receipt. The receipt means the action was
**dispatched**, never that it had the intended effect. Only the next
\`get_app_state\` tells you what actually happened — and an empty diff, or the
line "There has been no change in the accessibility tree", means your action did
nothing.
\`get_app_state\` tells you what actually happened. Judge the screenshot as well
as AX text: Chromium/CEF content can visibly change while the AX diff stays empty.
If two consecutive attempts leave the state unchanged, the approach is wrong.
If two consecutive screenshots leave the relevant UI unchanged, the approach is wrong.
Change something real: switch from handle to coordinates, target a different
element, re-read the full tree with \`disableDiff: true\`, or take a different
route through the UI such as the menu bar. Repeating the same call a third time
@@ -109,6 +109,10 @@ session.
element in the tree. Do not guess action names.
- \`press_key\` and \`type_text\` are delivered to the named app, so they cannot
trigger global system shortcuts.
- If \`type_text\` does not visibly change a Chromium/CEF field, use
\`paste({ app, text, format: "text" })\`. It restores the user's prior clipboard.
If it times out after dispatch, call \`get_app_state\` before retrying: the app
may have consumed the paste late.
- \`press_key\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c".
- \`select_text\` works inside editable elements; use \`prefix\`/\`suffix\` to
disambiguate repeated matches.
+138 -1
View File
@@ -7,8 +7,10 @@ import {
__prepareDaemonSocketDirectoryForTests,
__resetDaemonClientForTests,
__daemonStartCountForTests,
__negotiateDaemonProtocolForTests,
__reapStaleDaemonsForTests,
__setDaemonSocketForTests,
CU_HELPER_PROTOCOL_VERSION,
callDaemon,
DaemonCommandTimeoutError,
DaemonCommandResultUnknownError,
@@ -185,6 +187,76 @@ describe('cu-helper daemon failure classification', () => {
expect(error.message).toBe('grant_flag_required')
})
test('every request carries negotiated protocol, deadline, and stable turn identity', async () => {
const socket = new FakeSocket()
__setDaemonSocketForTests(socket as never, 100)
const first = callDaemon('get_app_state', { app: 'TextEdit' })
await waitForWriteCount(socket, 1)
const firstEnvelope = JSON.parse(socket.writes[0]!)
const firstTurnId = firstEnvelope.turnId
expect(firstEnvelope).toMatchObject({
clientApiVersion: CU_HELPER_PROTOCOL_VERSION,
requestId: firstEnvelope.id,
sessionId: expect.any(String),
turnId: expect.any(String),
})
expect(firstEnvelope.deadlineUnixMilliseconds).toBeGreaterThan(Date.now())
reply(socket, 0, { ok: true, result: {} })
await first
const second = callDaemon('click', { app: 'TextEdit', x: 1, y: 1 })
await waitForWriteCount(socket, 2)
const secondEnvelope = JSON.parse(socket.writes[1]!)
expect(secondEnvelope.turnId).toBe(firstTurnId)
reply(socket, 1, { ok: true, result: true })
await second
})
test('accepts only a daemon hello with the complete negotiated capability set', async () => {
const socket = new FakeSocket()
__setDaemonSocketForTests(socket as never, 100)
const negotiation = __negotiateDaemonProtocolForTests()
await waitForWrite(socket)
expect(JSON.parse(socket.writes[0]!)).toMatchObject({
cmd: 'ping',
clientApiVersion: CU_HELPER_PROTOCOL_VERSION,
})
reply(socket, 0, {
ok: true,
result: {
protocolVersion: CU_HELPER_PROTOCOL_VERSION,
supportsAbsoluteDeadlines: true,
supportsTurnEnd: true,
},
})
await expect(negotiation).resolves.toBeUndefined()
expect(socket.destroyed).toBe(false)
})
test('rejects and retires a daemon that reports an incomplete hello', async () => {
const socket = new FakeSocket()
__setDaemonSocketForTests(socket as never, 100)
const negotiation = __negotiateDaemonProtocolForTests().catch(err => err)
await waitForWrite(socket)
reply(socket, 0, {
ok: true,
result: {
protocolVersion: CU_HELPER_PROTOCOL_VERSION,
supportsAbsoluteDeadlines: true,
supportsTurnEnd: false,
},
})
const error = await negotiation
expect(error).toBeInstanceOf(DaemonUnavailableError)
expect(error.message).toMatch(/protocol negotiation failed.*unexpected hello/i)
expect(socket.destroyed).toBe(true)
})
test('post-dispatch timeout is ambiguous and must not be classified as replayable infrastructure', async () => {
const socket = new FakeSocket()
__setDaemonSocketForTests(socket as never, 5)
@@ -279,13 +351,20 @@ describe('cu-helper overlay reconciliation', () => {
reply(socket, 0, { ok: true, result: true })
await waitForWriteCount(socket, 2)
expect(JSON.parse(socket.writes[1]!)).toMatchObject({
cmd: 'overlay_hide',
cmd: 'turn_end',
payload: {},
})
reply(socket, 1, { ok: true, result: true })
await Promise.all([show, hide])
expect(isOverlayShown()).toBe(false)
const next = callDaemon('get_app_state', { app: 'TextEdit' })
await waitForWriteCount(socket, 3)
expect(JSON.parse(socket.writes[2]!).turnId)
.not.toBe(JSON.parse(socket.writes[0]!).turnId)
reply(socket, 2, { ok: true, result: {} })
await next
})
test('a target change while show is pending serially retargets to the latest app', async () => {
@@ -318,10 +397,68 @@ describe('cu-helper overlay reconciliation', () => {
await show
expect(isOverlayShown()).toBe(false)
const hide = overlayHide()
await waitForWriteCount(socket, 2)
expect(JSON.parse(socket.writes[1]!)).toMatchObject({ cmd: 'turn_end' })
reply(socket, 1, { ok: true, result: true })
await hide
})
test('cleanup ends a read-only turn even when no overlay was shown', async () => {
const socket = new FakeSocket()
__setDaemonSocketForTests(socket as never)
const state = callDaemon('get_app_state', { app: 'TextEdit' })
await waitForWriteCount(socket, 1)
const stateEnvelope = JSON.parse(socket.writes[0]!)
reply(socket, 0, { ok: true, result: {} })
await state
const hide = overlayHide()
await waitForWriteCount(socket, 2)
const endEnvelope = JSON.parse(socket.writes[1]!)
expect(endEnvelope).toMatchObject({ cmd: 'turn_end', turnId: stateEnvelope.turnId })
reply(socket, 1, { ok: true, result: true })
await hide
const next = callDaemon('get_app_state', { app: 'TextEdit' })
await waitForWriteCount(socket, 3)
expect(JSON.parse(socket.writes[2]!).turnId).not.toBe(stateEnvelope.turnId)
reply(socket, 2, { ok: true, result: {} })
await next
})
test('connection-scoped permission checks do not manufacture a turn to clean up', async () => {
const socket = new FakeSocket()
__setDaemonSocketForTests(socket as never)
const check = callDaemon('check_permissions', {})
await waitForWriteCount(socket, 1)
expect(JSON.parse(socket.writes[0]!).turnId).toMatch(/^connection-/)
reply(socket, 0, { ok: true, result: {} })
await check
await overlayHide()
expect(socket.writes).toHaveLength(1)
})
test('a rejected turn_end retires the daemon instead of poisoning the next turn', async () => {
const socket = new FakeSocket()
__setDaemonSocketForTests(socket as never)
const state = callDaemon('get_app_state', { app: 'TextEdit' })
await waitForWriteCount(socket, 1)
reply(socket, 0, { ok: true, result: {} })
await state
const hide = overlayHide()
await waitForWriteCount(socket, 2)
reply(socket, 1, { ok: false, error: { message: 'deadline_exceeded' } })
await hide
expect(socket.destroyed).toBe(true)
})
test('cleanup with no daemon does not start one', async () => {
expect(__daemonStartCountForTests()).toBe(0)
await overlayHide()
+71 -8
View File
@@ -1,7 +1,9 @@
import { spawn, spawnSync, type ChildProcess } from 'node:child_process'
import { randomUUID } from 'node:crypto'
import fs from 'node:fs'
import net from 'node:net'
import path from 'node:path'
import { getSessionId } from '../../bootstrap/state.js'
import { logForDebugging } from '../debug.js'
import { ensureInstalledHelper } from './cuHelperInstall.js'
import { attestDaemonSocketPeer } from './cuHelperPeerAttestation.js'
@@ -13,10 +15,10 @@ import { getRuntimePaths } from './pythonBridge.js'
* The daemon owns the main run loop that the animated virtual cursor and the
* animated virtual cursor needs, and holds the virtual cursor's position +
* held-input state across commands. We spawn ONE daemon per CLI process, keep
* an AF_UNIX socket open to it, and speak the NDJSON request/response protocol:
* an AF_UNIX socket open to it, and speak a versioned NDJSON protocol:
*
* readiness : {"ready":true,"pid":N,"proto":1}\n (the daemon's stdout, once)
* request : {"id":"<n>","cmd":"<verb>","payload":{...}}\n
* readiness : {"ready":true,"pid":N,"protocolVersion":"..."}\n
* request : {"id":"<n>","requestId":"<n>","cmd":"<verb>",...}\n
* response : {"id":"<n>","ok":true,"result":...}\n | {"id":"<n>","ok":false,"error":{...}}\n
*
* Routing through the daemon (instead of one-shot CLI) is what makes execution
@@ -31,6 +33,7 @@ import { getRuntimePaths } from './pythonBridge.js'
const REQUEST_TIMEOUT_MS = 20_000
const READINESS_TIMEOUT_MS = 8_000
export const CU_HELPER_PROTOCOL_VERSION = 'CCHahaComputerUseIPC-2'
/**
* Thrown ONLY for daemon INFRASTRUCTURE failures known to happen before the
@@ -107,6 +110,7 @@ let overlayActualKey: string | undefined
let overlayRevision = 0
let overlayReconcilePromise: Promise<void> | undefined
let requestTimeoutMs = REQUEST_TIMEOUT_MS
let activeTurnId: string | undefined
function socketPath(generation: number): string {
const { runtimeStateRoot } = getRuntimePaths()
@@ -379,6 +383,7 @@ function resetState(reason: string, expectedGeneration?: number): void {
overlayActualVisible = false
overlayActualKey = undefined
overlayRevision++
activeTurnId = undefined
const p = statePromise
statePromise = undefined
activeDaemonGeneration = undefined
@@ -513,6 +518,8 @@ async function startDaemon(generation: number): Promise<DaemonState> {
attachSocketHandlers(state)
await negotiateDaemonProtocol(state)
return state
}
@@ -542,6 +549,31 @@ function attachSocketHandlers(state: DaemonState): void {
state.socket.on('error', err => resetState(`socket error: ${String(err)}`, state.generation))
}
async function negotiateDaemonProtocol(state: DaemonState): Promise<void> {
// Socket ownership + code-signature attestation identify the peer. The
// versioned hello additionally proves that the installed helper understands
// deadlines and explicit turn cleanup before we dispatch any real command.
try {
const hello = await dispatchDaemonCommand<{
protocolVersion?: string
supportsAbsoluteDeadlines?: boolean
supportsTurnEnd?: boolean
}>(state, 'ping', {})
if (
hello.protocolVersion !== CU_HELPER_PROTOCOL_VERSION
|| hello.supportsAbsoluteDeadlines !== true
|| hello.supportsTurnEnd !== true
) {
throw new Error(`unexpected hello ${JSON.stringify(hello)}`)
}
} catch (err) {
state.socket.destroy()
throw new DaemonUnavailableError(
`cu-helper protocol negotiation failed: ${err instanceof Error ? err.message : String(err)}`,
)
}
}
async function ensureDaemon(): Promise<DaemonState> {
if (!statePromise) {
const generation = ++daemonGeneration
@@ -573,6 +605,19 @@ function dispatchDaemonCommand<T>(
))
}
const id = String(++state.nextId)
const isTurnScoped = !['ping', 'check_permissions', 'shutdown'].includes(command)
const turnId = activeTurnId
?? (isTurnScoped ? (activeTurnId = randomUUID()) : `connection-${state.generation}`)
const request = {
id,
requestId: id,
cmd: command,
payload,
clientApiVersion: CU_HELPER_PROTOCOL_VERSION,
deadlineUnixMilliseconds: Date.now() + requestTimeoutMs,
sessionId: getSessionId(),
turnId,
}
return new Promise<T>((resolve, reject) => {
const timer = setTimeout(() => {
state.pending.delete(id)
@@ -588,7 +633,7 @@ function dispatchDaemonCommand<T>(
}, requestTimeoutMs)
state.pending.set(id, { resolve: resolve as (v: unknown) => void, reject, timer })
try {
state.socket.write(`${JSON.stringify({ id, cmd: command, payload })}\n`)
state.socket.write(`${JSON.stringify(request)}\n`)
} catch (err) {
clearTimeout(timer)
state.pending.delete(id)
@@ -597,6 +642,10 @@ function dispatchDaemonCommand<T>(
reject(new DaemonUnavailableError(err instanceof Error ? err.message : String(err)))
resetState(`socket write failed: ${String(err)}`, state.generation)
}
}).finally(() => {
if ((command === 'turn_end' || command === 'overlay_hide') && activeTurnId === turnId) {
activeTurnId = undefined
}
})
}
@@ -627,7 +676,10 @@ async function callExistingDaemon<T>(
}
function needsOverlayReconciliation(): boolean {
if (!overlayDesiredVisible) return overlayActualVisible
// A turn may contain only get_app_state and therefore never show the overlay.
// Its native state (capture stream, AX baseline, held-input guard and display
// sleep assertion) still needs an explicit turn_end at host cleanup.
if (!overlayDesiredVisible) return overlayActualVisible || activeTurnId !== undefined
return !overlayActualVisible || overlayActualKey !== overlayDesiredKey
}
@@ -662,16 +714,21 @@ async function reconcileOverlay(): Promise<void> {
// A turn-end cleanup must never launch a daemon just to hide it. When a
// pending show completes this branch sees the already-owned daemon and
// serially sends the compensating hide.
// serially ends the turn. This also covers read-only turns whose overlay
// was never visible.
if (!statePromise || activeDaemonGeneration === undefined) {
overlayActualVisible = false
overlayActualKey = undefined
return
}
try {
await callExistingDaemon('overlay_hide', {})
await callExistingDaemon('turn_end', {})
} catch (err) {
logForDebugging(`cu-helper overlay_hide failed: ${String(err)}`, { level: 'debug' })
logForDebugging(`cu-helper turn_end failed: ${String(err)}`, { level: 'debug' })
// If the helper rejected/lost turn_end it may still own the old turn.
// Retire it now so the next user turn cannot inherit stale native state
// or fail forever with turn_mismatch.
resetState('turn_end failed')
}
overlayActualVisible = false
overlayActualKey = undefined
@@ -740,6 +797,7 @@ export function __resetDaemonClientForTests(): void {
overlayReconcilePromise = undefined
daemonStartCount = 0
requestTimeoutMs = REQUEST_TIMEOUT_MS
activeTurnId = undefined
}
/** Focused proof that a no-daemon cleanup did not enter the startup path. */
@@ -775,3 +833,8 @@ export function __setDaemonSocketForTests(
attachSocketHandlers(state)
statePromise = Promise.resolve(state)
}
/** Drive the same hello negotiation used by production against an installed test socket. */
export async function __negotiateDaemonProtocolForTests(): Promise<void> {
await negotiateDaemonProtocol(await ensureDaemon())
}
@@ -264,6 +264,15 @@ describe('CLI executor Codex engine — daemon payload alignment', () => {
expect(lastCall()).toEqual({ command: 'type_text', payload: { text: 'hello' } })
})
itEngine('paste sends explicit content format', async () => {
const exec = await loadExecutor()
await exec.engine!.paste({ target: { app: 'NeteaseMusic' }, text: '喜欢你', format: 'text' })
expect(lastCall()).toEqual({
command: 'paste',
payload: { app: 'NeteaseMusic', text: '喜欢你', format: 'text' },
})
})
itEngine('selectText sends text range parameters', async () => {
const exec = await loadExecutor()
await exec.engine!.selectText({
+13 -1
View File
@@ -91,7 +91,7 @@ async function writeClipboard(text: string): Promise<void> {
// ----------------------------------------------------------------------------
// Codex semantic engine (blueprint §4–§7)
//
// Maps the ten `CodexComputerEngine` methods onto `callHelper(<cmd>, payload)`
// Maps the Codex-compatible `CodexComputerEngine` methods onto `callHelper(<cmd>, payload)`
// round-trips against the native daemon's `CommandRouter`. Each command's
// payload key names mirror what `CommandRouter` decodes (see CommandRouter.swift):
// - target: `pid` | `bundleId` | `app` (resolveTargetPid precedence)
@@ -287,6 +287,18 @@ export function createCodexEngine(): CodexComputerEngine {
text: args.text,
})
},
async paste(args: {
target: AppTarget
text: string
format: 'text' | 'md' | 'html'
}): Promise<void> {
await callHelper('paste', {
...appTargetPayload(args.target),
text: args.text,
format: args.format,
})
},
}
}
+1 -1
View File
@@ -206,7 +206,7 @@ export interface CodexComputerEngine {
*/
pressKey(args: { target: AppTarget; key: string; systemKeyCombos: boolean }): Promise<void>
/** Type literal text (append to focused value; Unicode keyboard fallback). */
typeText(args: { target: AppTarget; text: string }): Promise<void>
typeText(args: { target: AppTarget; text: string }): Promise<void>; paste(args: { target: AppTarget; text: string; format: 'text' | 'md' | 'html' }): Promise<void>
}
export interface InstalledApp {
+7 -1
View File
@@ -67,7 +67,9 @@ Mutating tools return a fixed receipt. The receipt means "the action was
dispatched", NOT "it had the intended effect" — you must look at the next
\`get_app_state\` to know.
If two consecutive attempts leave the state unchanged, the approach is wrong.
Judge success from the screenshot as well as the AX text. An empty AX diff does
not mean a Chromium/CEF interface stayed unchanged. If two consecutive screenshots
leave the relevant UI unchanged, the approach is wrong.
Change something real: switch from element handle to coordinates, target a
different element, re-read the full tree with \`disableDiff: true\`, or take a
different route through the UI. Repeating the same call a third time never helps.
@@ -86,6 +88,10 @@ waste the user's time.
element in the tree. Do not guess action names.
- \`press_key\` and \`type_text\` are delivered to the named app, so they cannot
trigger global system shortcuts.
- If \`type_text\` does not visually change a Chromium/CEF field, use
\`paste({ app, text, format: "text" })\`; it restores the user's prior clipboard.
If paste times out after dispatch, treat the result as unknown and call
\`get_app_state\` before retrying, because the target may have consumed it late.
- \`press_key\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c".
- \`select_text\` works inside editable elements; use \`prefix\`/\`suffix\` to
disambiguate repeated matches.
+2 -1
View File
@@ -19,6 +19,7 @@ const DARWIN_TOOL_NAMES = [
'get_app_state',
'list_apps',
'perform_secondary_action',
'paste',
'press_key',
'scroll',
'select_text',
@@ -288,7 +289,7 @@ function makeWindowsAdapter(calls: string[]): ComputerUseHostAdapter {
}
describe('Computer Use platform routing', () => {
test('darwin ListTools advertises exactly the ten semantic tools', async () => {
test('darwin ListTools advertises the current semantic tools', async () => {
const connection = await connect(makeDarwinAdapter())
try {
const result = await connection.client.listTools()
+46 -3
View File
@@ -17,6 +17,7 @@ import {
resetMouseButtonHeld,
} from './toolCalls.js'
import { buildComputerUseTools } from './tools.js'
import { COMPUTER_USE_INSTRUCTIONS } from './instructions.js'
import { isSystemKeyCombo } from './keyBlocklist.js'
import type {
ComputerUseHostAdapter,
@@ -128,6 +129,7 @@ function makeEngine(
drag: record('drag', async () => {}),
pressKey: record('pressKey', async () => {}),
typeText: record('typeText', async () => {}),
paste: record('paste', async () => {}),
selectText: record('selectText', async () => {}),
}
return { engine, calls }
@@ -208,10 +210,10 @@ function imageBlocks(result: { content: unknown }): Array<{ data?: string; mimeT
// Tool schema
// ---------------------------------------------------------------------------
describe('buildComputerUseTools — Codex 10-tool face', () => {
describe('buildComputerUseTools — current Codex tool face', () => {
const tools = buildComputerUseTools()
test('exposes exactly the ten Codex tools, verbatim names', () => {
test('exposes the current Codex tools, including explicit paste', () => {
expect(tools.map(t => t.name).sort()).toEqual(
[
'click',
@@ -219,6 +221,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => {
'get_app_state',
'list_apps',
'perform_secondary_action',
'paste',
'press_key',
'scroll',
'select_text',
@@ -318,6 +321,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => {
'drag',
'press_key',
'type_text',
'paste',
]) {
const description = tools.find(t => t.name === name)!.description ?? ''
expect(description).toContain('get_app_state')
@@ -433,7 +437,7 @@ describe('buildComputerUseTools — Codex 10-tool face', () => {
'normalized_0_100',
['Finder', 'Slack'],
)
expect(withArgs).toHaveLength(10)
expect(withArgs).toHaveLength(11)
})
})
@@ -1575,6 +1579,45 @@ describe('handleToolCall — tool dispatch', () => {
expect(calls.find(c => c.method === 'typeText')!.args).toMatchObject({ target: { pid: 1234 }, text: 'Codex' })
})
test('paste passes text and format as an explicit recovery action', async () => {
const { engine, calls } = makeEngine()
await handleToolCall(
makeAdapter({ engine }),
'paste',
{ app: 'Activity Monitor', text: '喜欢你', format: 'text' },
baseOverrides(),
)
expect(calls.find(c => c.method === 'paste')!.args).toMatchObject({
target: { pid: 1234 },
text: '喜欢你',
format: 'text',
})
})
test('paste rejects missing text and unknown formats before target resolution', async () => {
for (const args of [
{ app: 'Activity Monitor', format: 'text' },
{ app: 'Activity Monitor', text: 'hello', format: 'rtf' },
]) {
const { engine, calls } = makeEngine()
const result = await handleToolCall(
makeAdapter({ engine }),
'paste',
args,
baseOverrides(),
)
expect(result.isError).toBe(true)
expect(result.telemetry?.error_kind).toBe('bad_args')
expect(calls).toHaveLength(0)
}
})
test('server guidance treats AX diffs and timed-out paste as non-authoritative', () => {
expect(COMPUTER_USE_INSTRUCTIONS).toContain('An empty AX diff does')
expect(COMPUTER_USE_INSTRUCTIONS).toContain('paste({ app, text, format: "text" })')
expect(COMPUTER_USE_INSTRUCTIONS).toContain('treat the result as unknown')
})
test('select_text passes the opaque handle, context, and selection mode', async () => {
const { engine, calls } = makeEngine()
await handleToolCall(
+20 -1
View File
@@ -6,7 +6,7 @@
* commands):
*
* list_apps, get_app_state, click, perform_secondary_action, set_value,
* select_text, scroll, drag, press_key, type_text
* select_text, scroll, drag, press_key, type_text, paste
*
* ## Three properties this file exists to hold
*
@@ -565,6 +565,7 @@ const MUTATING_TOOLS: ReadonlySet<string> = new Set([
"drag",
"press_key",
"type_text",
"paste",
]);
const KNOWN_TOOLS: ReadonlySet<string> = new Set([
@@ -801,6 +802,24 @@ function parseRequest(
};
}
case "paste": {
const text = args.text;
if (typeof text !== "string") {
throw new BadArgs('Missing required string "text"');
}
const format = requiredString(args, "format");
if (format !== "text" && format !== "md" && format !== "html") {
throw new BadArgs('paste format must be "text", "md", or "html"');
}
return {
...base,
run: async (engine, t) => {
await engine.paste({ target: t, text, format });
return okText(MUTATION_RECEIPT);
},
};
}
default:
throw new BadArgs(`Unknown computer-use tool "${name}".`);
}
+33 -4
View File
@@ -1,14 +1,14 @@
/**
* MCP tool schemas for the computer-use server — Codex-compatible semantic
* face (blueprint §7). Exactly TEN tools, named verbatim after Codex's public
* face (blueprint §7). Eleven tools, named verbatim after Codex's public
* computer-use MCP:
*
* list_apps, get_app_state, click, perform_secondary_action, set_value,
* select_text, scroll, drag, press_key, type_text
* select_text, scroll, drag, press_key, type_text, paste
*
* This replaces the prior 27-tool pixel face. The legacy `coordinateMode` /
* `screenshotFiltering` parameters are accepted for call-site compatibility
* but no longer influence the schema — the ten tool shapes are static.
* but no longer influence the schema — the tool shapes are static.
*
* Param names mirror Codex exactly (verified against iFurySt's reverse-
* engineered ToolDefinitions.swift and the 2026-04-17 tool-call samples):
@@ -83,7 +83,7 @@ function coordinateProp(axis: "x" | "y", role: string) {
}
/**
* Build the ten-tool Codex computer-use face.
* Build the Codex computer-use face.
*
* Signature is preserved from the legacy pixel builder so existing call sites
* (`setup.ts`, host `mcpServer.ts`) keep compiling. All three parameters are
@@ -415,5 +415,34 @@ export function buildComputerUseTools(
required: ["app", "text"],
},
},
{
name: "paste",
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: false, openWorldHint: false },
description:
"Paste content into whatever currently has keyboard focus in the target " +
"app. Prefer this when type_text did not change a Chromium/CEF field, for " +
"Chinese text, formatted content, or multiline text. The user's previous " +
"clipboard is restored unless they copy something during the operation. " +
"A timeout after Command-V is result-unknown: call get_app_state before " +
"deciding whether to retry. Call get_app_state afterwards to see the result.",
inputSchema: {
type: "object" as const,
additionalProperties: false,
properties: {
app: APP_PROP,
text: {
type: "string",
description: "The content to paste.",
},
format: {
type: "string",
enum: ["text", "md", "html"],
description: "Pasteboard representation to use.",
},
},
required: ["app", "text", "format"],
},
},
];
}