fix(computer-use): align long-running macOS automation with Codex

This commit is contained in:
程序员阿江(Relakkes)
2026-09-01 17:31:17 +08:00
parent 8751e558e5
commit ad7321f7cc
32 changed files with 1100 additions and 99 deletions
+2
View File
@@ -21,6 +21,8 @@
<string>cc-haha-computer-use</string>
<key>CFBundleDisplayName</key>
<string>cc-haha-computer-use</string>
<key>CFBundleIconFile</key>
<string>icon.icns</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleShortVersionString</key>
@@ -237,10 +237,18 @@ public final class ClipboardLease {
}
func writeTemporaryStringWithReceipt(_ text: String) throws -> ClipboardPasteReceipt {
try writeTemporaryContentWithReceipt(text, format: .text)
}
func writeTemporaryContentWithReceipt(
_ text: String,
format: ClipboardPasteFormat
) throws -> ClipboardPasteReceipt {
if let captureError { throw captureError }
let receipt = ClipboardPasteReceipt(text: text)
let receipt = ClipboardPasteReceipt(text: text, format: format)
let item = NSPasteboardItem()
guard item.setDataProvider(receipt, forTypes: [.string]) else {
let types = Array(format.promisedData(for: text).keys)
guard item.setDataProvider(receipt, forTypes: types) else {
throw CUError("clipboard_write_failed", "Could not register temporary pasteboard data")
}
pasteboard.clearContents()
@@ -926,13 +934,35 @@ public enum AXAction {
try await typeViaClipboard(target: target, text)
}
/// Explicit Codex-compatible paste path. Unlike `typeText`, this does not
/// first try AX value mutation or Unicode key events; it targets the app's
/// in-process focused field with Command-V and restores the user's original
/// pasteboard after observing a real promised-data read.
static func pasteText(
pid: pid_t,
_ text: String,
format: ClipboardPasteFormat
) async throws {
guard !text.isEmpty else { return }
let target = try Injection.authorizeResolvedTarget(pid: pid)
try await typeViaClipboard(target: target, text, format: format)
}
/// Paste `text` into whatever holds in-app keyboard focus in `pid`, via the
/// system clipboard + ⌘V (Codex's approach for CEF/Chromium text fields).
/// The read receipt confirms supplied clipboard bytes, not the reader's
/// PID or the field's final value; the next screenshot still verifies UI.
private static func typeViaClipboard(target: ProvenProcessTarget, _ text: String) async throws {
private static func typeViaClipboard(
target: ProvenProcessTarget,
_ text: String,
format: ClipboardPasteFormat = .text
) async throws {
let pid = target.pid
try await ClipboardPasteReceipt.perform(text: text, lease: ClipboardLease()) { validate in
try await ClipboardPasteReceipt.perform(
text: text,
format: format,
lease: ClipboardLease()
) { validate in
try await pressKey(pid: pid, "super+v", validateBeforePosting: {
_ = try Injection.validateAuthorizedTarget(target)
try validate()
@@ -1609,10 +1639,11 @@ public enum AXAction {
return CFEqual(lhs, rhs)
}
/// Mark that a mutation just landed. Does NOT block: `get_app_state` reads
/// this stamp and waits out whatever settle time is still owed, so the cost
/// is paid once, and only when someone is actually about to look.
/// Settlement is recorded once at `ForegroundMutationRunner`, which covers
/// every input path and knows the target PID. AX-local success receipts must
/// not create a second, unscoped marker that can leak across apps.
private static func settle() {
MutationClock.recordMutation()
// Intentionally empty; retained at the AX call sites as a semantic
// marker for successful mutations.
}
}
@@ -30,9 +30,9 @@ enum HelperClientPolicy {
// read-only authorization preflight used before the ten public tools.
"list_apps", "resolve_app_target", "get_app_state", "click",
"set_value", "select_text", "perform_secondary_action", "scroll",
"drag", "press_key", "type_text",
"drag", "press_key", "type_text", "paste",
// Private lifecycle / visible-overlay / permission and input diagnostics.
"ping", "shutdown", "overlay_show", "overlay_hide",
"ping", "shutdown", "overlay_show", "overlay_hide", "turn_end",
"check_permissions", "input_monitor_state", "focus_monitor_state", "held_input_state",
"last_injection_state",
]
@@ -2,6 +2,32 @@ import AppKit
import Foundation
import os
enum ClipboardPasteFormat: String, Sendable {
case text
case md
case html
func promisedData(for text: String) -> [NSPasteboard.PasteboardType: Data] {
switch self {
case .text, .md:
return [.string: Data(text.utf8)]
case .html:
let html = Data(text.utf8)
let plain = (
try? NSAttributedString(
data: html,
options: [
.documentType: NSAttributedString.DocumentType.html,
.characterEncoding: String.Encoding.utf8.rawValue,
],
documentAttributes: nil
).string
) ?? text
return [.html: html, .string: Data(plain.utf8)]
}
}
}
/// Confirms that this pasteboard item's promised bytes were requested and
/// supplied. AppKit does not identify the reader: this is not proof that the
/// intended application's focused field accepted the text.
@@ -25,16 +51,26 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
}
@MainActor private(set) static var lastDiagnostic: Diagnostic?
private let data: Data
private let promisedData: [NSPasteboard.PasteboardType: Data]
private let state = OSAllocatedUnfairLock(initialState: State())
init(text: String) {
data = Data(text.utf8)
promisedData = ClipboardPasteFormat.text.promisedData(for: text)
super.init()
}
init(text: String, format: ClipboardPasteFormat) {
promisedData = format.promisedData(for: text)
super.init()
}
@MainActor
static func resetForTurn() {
lastDiagnostic = nil
}
func pasteboard(_ pasteboard: NSPasteboard?, item: NSPasteboardItem, provideDataForType type: NSPasteboard.PasteboardType) {
guard type == .string else { return }
guard let data = promisedData[type] else { return }
state.withLock { $0.requested = true }
guard item.setData(data, forType: type) else { return }
state.withLock { value in
@@ -50,6 +86,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
@MainActor
static func perform(
text: String,
format: ClipboardPasteFormat = .text,
lease: ClipboardLease,
timeout: Duration = .seconds(2),
sendPaste: @MainActor (_ validateBeforePosting: @MainActor () throws -> Void) async throws -> Void
@@ -79,7 +116,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
}
do {
try Task.checkCancellation()
let written = try lease.writeTemporaryStringWithReceipt(text)
let written = try lease.writeTemporaryContentWithReceipt(text, format: format)
receipt = written
try await Task.sleep(for: .milliseconds(40))
let validate: @MainActor () throws -> Void = {
@@ -104,6 +104,7 @@ public final class CommandRouter {
Self.lastShotTransform.removeAll()
Self.lastCaptureDigest.removeAll()
MutationClock.reset()
ClipboardPasteReceipt.resetForTurn()
windowCaptureProvider?.invalidate()
// Apps we told they were focused must be told they are not, or the
// belief outlives the session that needed it.
@@ -267,6 +268,9 @@ public final class CommandRouter {
case "type_text":
return try await handleTypeText(payload)
case "paste":
return try await handlePaste(payload)
case "press_key":
return try await handlePressKey(payload)
@@ -728,18 +732,19 @@ public final class CommandRouter {
if #available(macOS 14.0, *) {
// Let the UI finish whatever the last action started before we
// photograph it, otherwise the model reasons about a half-drawn
// frame. Costs nothing when no action is pending. The rendered tree
// doubles as the busy signal — a progress indicator in it means the
// app is still working, so we allow a longer window.
// frame. Costs nothing when no action is pending, applies only to
// this target PID, and is consumed by this single capture.
let pendingMutation = MutationClock.takeMutation(pid: pid)
let streamedShot = await Self.captureSettledWindowShot(
appIsBusy: result.axText.contains("progress indicator")
appIsBusy: result.axText.contains("progress indicator"),
lastMutationAt: pendingMutation
) {
await windowCaptureProvider?.windowShot(
pid: pid,
processIdentity: snapshotEvidence.processIdentity,
preferredWindowID: snapshotEvidence.keyWindowID,
scale: 0.5,
newerThanUptime: MutationClock.lastMutation()
newerThanUptime: pendingMutation
)
}
guard TargetVisibilityPolicy.captureTargetStillMatches(
@@ -875,9 +880,13 @@ public final class CommandRouter {
/// action-to-screenshot regression can exercise both without live AX/TCC.
static func captureSettledWindowShot(
appIsBusy: Bool,
lastMutationAt: TimeInterval?,
capture: () async -> WindowShot?
) async -> WindowShot? {
await MutationClock.awaitSettle(appIsBusy: appIsBusy)
await MutationClock.awaitSettle(
lastMutationAt: lastMutationAt,
appIsBusy: appIsBusy
)
return await capture()
}
@@ -1438,6 +1447,31 @@ public final class CommandRouter {
}
}
/// `paste`: bypass AX/Unicode typing and use the target app's in-process
/// paste focus. This is the reliable recovery path for Chromium/CEF fields
/// such as NeteaseMusic's search box. The clipboard lease restores every
/// original pasteboard item unless the user copied something meanwhile.
private func handlePaste(_ payload: JSONValue) async throws -> JSONValue {
try requireAXTrusted()
guard let text = payload["text"]?.asString else {
throw CUError("bad_payload", "paste requires a 'text' string")
}
guard let rawFormat = payload["format"]?.asString,
let format = ClipboardPasteFormat(rawValue: rawFormat) else {
throw CUError("bad_payload", "paste format must be 'text', 'md', or 'html'")
}
let expected = try Self.expectedProcessTarget(payload)
let target = try resolveTargetForMutation(payload)
setResolvedTarget(target)
try requireSnapshotProcess(target: target, expected: expected)
return try await withForegroundLease(command: "paste", target: target) {
_ = try Injection.validateAuthorizedTarget(target)
try self.requireSnapshotProcess(target: target, expected: expected)
try await AXAction.pasteText(pid: target.pid, text, format: format)
return .bool(true)
}
}
/// `press_key`: send an xdotool-style key spec (e.g. "super+c", "Return",
/// "Tab", "KP_0", "Prior") to the target app via postToPid.
private func handlePressKey(_ payload: JSONValue) async throws -> JSONValue {
@@ -1566,6 +1600,7 @@ public final class CommandRouter {
)
return try await ForegroundMutationRunner.run(
lease: lease,
targetPID: target.pid,
action: action
)
}
+35 -12
View File
@@ -38,21 +38,21 @@ enum DaemonOverlayTargetResolver {
// `cursor_position`, implicit-`from` drags and decomposed `mouse_down`/`mouse_up`
// behave correctly across commands.
//
// Transport: a private AF_UNIX SOCK_STREAM socket carrying NDJSON (one JSON
// object + '\n' per request, same per response). A GUI/AppKit process leaks
// Transport: a private AF_UNIX SOCK_STREAM socket carrying versioned NDJSON
// (one JSON object + '\n' per request, same per response). A GUI/AppKit process leaks
// os_log / CoreGraphics chatter to stdout/stderr, which would corrupt the TS
// bridge's `JSON.parse`; the daemon therefore reserves stdout for exactly ONE
// readiness line and serves the request/response stream over the socket.
//
// readiness : {"ready":true,"pid":<N>,"proto":1}\n (stdout, once)
// request : {"id":"<opaque>","cmd":"<verb>","payload":{...}}\n
// readiness : {"ready":true,"pid":<N>,"protocolVersion":"…"}\n
// request : {"id":"<opaque>","requestId":"<opaque>","cmd":"<verb>",…}\n
// response : {"id":"<opaque>","ok":true,"result":<json>}\n
// {"id":"<opaque>","ok":false,"error":{"message":"…","code":"…"}}\n
//
// Control verbs handled in-daemon (never reach CommandRouter):
// overlay_show -> cursor.show() ; result true
// overlay_hide -> cursor.hide() ; result true
// ping -> result "pong"
// turn_end -> release all turn-owned state ; result true
// ping -> version/capability hello
// shutdown -> result true, then NSApp.terminate(nil)
// Every other cmd is forwarded to the shared CommandRouter — the exact same
// dispatcher the one-shot CLI path uses, so each command has one implementation.
@@ -63,6 +63,7 @@ public final class Daemon {
private let cursor: VirtualCursor
private let inputMonitor: PhysicalInputEpochMonitor
private let router: CommandRouter
private let displaySleepAssertion = ComputerUseDisplaySleepAssertion()
/// Listening socket fd (AF_UNIX SOCK_STREAM). -1 until `bindAndListen`.
private var listenFD: Int32 = -1
@@ -78,11 +79,12 @@ public final class Daemon {
private var connection: Connection?
private var connectionToken: DaemonSessionToken?
private var sessionGate = DaemonSessionGate()
private var turnGate = DaemonTurnGate()
/// Set once the run loop is live; guards against double `run()`.
private var didStartRunLoop = false
/// True between `overlay_show` and `overlay_hide` (CU active this turn). The
/// True between `overlay_show` and `turn_end` (CU active this turn). The
/// cursor re-aims at the actual injection target while this holds.
private var overlayActive = false
private var explicitOverlayTarget: ProvenProcessTarget?
@@ -220,6 +222,7 @@ public final class Daemon {
// immediately by exit() (shutdown verb + signal handlers).
Injection.releaseAllHeldSync()
router.resetSessionState()
displaySleepAssertion.release()
inputMonitor.stop()
}
@@ -435,6 +438,8 @@ public final class Daemon {
stopOverlaySession()
Injection.releaseAllHeldSync()
router.resetSessionState()
turnGate.reset()
displaySleepAssertion.release()
}
/// A LaunchServices child is reparented to launchd, so the peer socket is
@@ -479,7 +484,16 @@ public final class Daemon {
let payload = request.payload ?? .object([:])
do {
let metadata = try ComputerUseDaemonProtocol.validate(request)
let opensTurn = ComputerUseDaemonProtocol.isTurnScoped(command: request.cmd)
&& turnGate.active == nil
try turnGate.admit(metadata, command: request.cmd)
if opensTurn { displaySleepAssertion.acquire() }
let result = try await route(cmd: request.cmd, payload: payload)
if request.cmd == "turn_end" || request.cmd == "overlay_hide" {
try turnGate.finish(metadata)
displaySleepAssertion.release()
}
conn.send(encodeResponse(DaemonResponse(id: id, ok: true, result: result, error: nil)))
} catch let cuError as CUError {
conn.send(encodeResponse(DaemonResponse(
@@ -508,12 +522,12 @@ public final class Daemon {
try showOverlay(payload: payload)
return .bool(true)
case "overlay_hide":
stopOverlaySession()
case "overlay_hide", "turn_end":
endTurn()
return .bool(true)
case "ping":
return .string("pong")
return ComputerUseDaemonProtocol.hello()
case "check_permissions":
// Usable in both modes; the daemon exposes it for onboarding /
@@ -592,6 +606,15 @@ public final class Daemon {
router.invalidateWindowCaptureStream()
}
/// Codex-parity turn boundary. The helper process stays warm, but no AX
/// snapshot, coordinate transform, focus belief, held input, mutation clock,
/// clipboard diagnostic, or capture stream may leak into the next turn.
private func endTurn() {
stopOverlaySession()
Injection.releaseAllHeldSync()
router.resetSessionState()
}
/// The app the cursor should follow: ONLY the app the last injection /
/// get_app_state actually resolved (`Injection.lastResolvedTargetPid`). We
/// deliberately do NOT fall back to the frontmost app — at turn start,
@@ -763,9 +786,9 @@ private final class Connection: @unchecked Sendable {
private var inBuffer = Data()
/// Capped to avoid unbounded growth from a malformed/never-newline client.
/// 16 MiB comfortably exceeds the largest legitimate request (a base64
/// 8 MiB comfortably exceeds the largest legitimate request (a base64
/// screenshot travels in *responses*, not requests).
private let maxBufferBytes = 16 * 1024 * 1024
private let maxBufferBytes = ComputerUseDaemonProtocol.maxFrameBytes
private var onRequest: (@Sendable (Data) -> Void)?
private var onClose: (@Sendable () -> Void)?
@@ -0,0 +1,99 @@
import Foundation
/// Versioned request contract for the authenticated daemon socket.
///
/// Code signature attestation proves who is connected; this handshake proves
/// that both sides agree on request shape and lifecycle semantics. Every request
/// also carries an absolute deadline plus session/turn identity so a command
/// queued behind a slow capture cannot execute after its caller has moved on.
enum ComputerUseDaemonProtocol {
static let version = "CCHahaComputerUseIPC-2"
static let maxFrameBytes = 8 * 1024 * 1024
private static let connectionScopedCommands: Set<String> = [
"ping", "check_permissions", "shutdown",
]
static func isTurnScoped(command: String) -> Bool {
!connectionScopedCommands.contains(command)
}
struct Metadata: Equatable, Sendable {
let sessionId: String
let turnId: String
}
static func validate(
_ request: Request,
nowUnixMilliseconds: Int64 = Int64(Date().timeIntervalSince1970 * 1_000)
) throws -> Metadata {
guard request.clientApiVersion == version else {
throw CUError(
"protocol_mismatch",
"Computer Use client/helper protocol mismatch; restart the app after updating"
)
}
guard let id = request.id, !id.isEmpty,
request.requestId == id else {
throw CUError("bad_request_id", "Computer Use request identity is missing or inconsistent")
}
guard let deadline = request.deadlineUnixMilliseconds else {
throw CUError("missing_deadline", "Computer Use request is missing its absolute deadline")
}
guard deadline > nowUnixMilliseconds else {
throw CUError("deadline_exceeded", "Computer Use request expired before execution")
}
guard let sessionId = request.sessionId?.trimmingCharacters(in: .whitespacesAndNewlines),
!sessionId.isEmpty,
let turnId = request.turnId?.trimmingCharacters(in: .whitespacesAndNewlines),
!turnId.isEmpty else {
throw CUError("missing_turn_metadata", "Computer Use request is missing session/turn identity")
}
return Metadata(sessionId: sessionId, turnId: turnId)
}
static func hello() -> JSONValue {
.object([
"protocolVersion": .string(version),
"supportsAbsoluteDeadlines": .bool(true),
"supportsTurnEnd": .bool(true),
"supportsCaptureDiagnostics": .bool(true),
])
}
}
/// Enforces one explicit turn at a time on the single authenticated connection.
/// `ping` negotiates the protocol without opening a turn. Every other request
/// must keep the same identity until `turn_end` releases all turn-owned state.
struct DaemonTurnGate {
private(set) var active: ComputerUseDaemonProtocol.Metadata?
mutating func admit(
_ metadata: ComputerUseDaemonProtocol.Metadata,
command: String
) throws {
if !ComputerUseDaemonProtocol.isTurnScoped(command: command) { return }
if let active {
guard active == metadata else {
throw CUError(
"turn_mismatch",
"A different Computer Use turn is still active; finish it before starting another"
)
}
return
}
active = metadata
}
mutating func finish(
_ metadata: ComputerUseDaemonProtocol.Metadata
) throws {
guard active == metadata else {
throw CUError("turn_mismatch", "Computer Use turn_end did not match the active turn")
}
active = nil
}
mutating func reset() {
active = nil
}
}
@@ -0,0 +1,50 @@
import Foundation
import IOKit.pwr_mgt
/// Keeps the display compositor awake while a Computer Use turn is active.
/// Covered-window ScreenCaptureKit consumers can otherwise stop receiving
/// useful updates once macOS enters user-idle display sleep during a long task.
@MainActor
final class ComputerUseDisplaySleepAssertion {
typealias AssertionID = IOPMAssertionID
private let create: () -> AssertionID?
private let releaseAssertion: (AssertionID) -> Void
private var assertionID: AssertionID?
init(
create: @escaping () -> AssertionID? = {
var assertionID: IOPMAssertionID = 0
let result = IOPMAssertionCreateWithDescription(
kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString,
"Claude Code Haha Computer Use interaction" as CFString,
"Computer Use turn is controlling a background application" as CFString,
"Keeping the display awake while Computer Use is active" as CFString,
nil,
0,
nil,
&assertionID
)
return result == kIOReturnSuccess ? assertionID : nil
},
release: @escaping (AssertionID) -> Void = { assertionID in
_ = IOPMAssertionRelease(assertionID)
}
) {
self.create = create
self.releaseAssertion = release
}
func acquire() {
guard assertionID == nil else { return }
assertionID = create()
}
func release() {
guard let assertionID else { return }
self.assertionID = nil
releaseAssertion(assertionID)
}
var isHeldForTesting: Bool { assertionID != nil }
}
@@ -394,12 +394,13 @@ final class ForegroundLease {
enum ForegroundMutationRunner {
static func run<T>(
lease: ForegroundLease,
targetPID: pid_t? = nil,
action: () async throws -> T
) async throws -> T {
// Every input path crosses this boundary, including synthetic events
// that never call AXAction.settle(). A throw can follow a partial
// delivery, so neither success nor failure may reuse pre-action pixels.
defer { MutationClock.recordMutation() }
defer { MutationClock.recordMutation(pid: targetPID) }
let result: Result<T, Error>
do {
result = .success(try await action())
@@ -439,11 +439,30 @@ public struct Request: Decodable, Sendable {
public let id: String?
public let cmd: String
public let payload: JSONValue?
public let clientApiVersion: String?
public let deadlineUnixMilliseconds: Int64?
public let sessionId: String?
public let turnId: String?
public let requestId: String?
public init(id: String?, cmd: String, payload: JSONValue?) {
public init(
id: String?,
cmd: String,
payload: JSONValue?,
clientApiVersion: String? = nil,
deadlineUnixMilliseconds: Int64? = nil,
sessionId: String? = nil,
turnId: String? = nil,
requestId: String? = nil
) {
self.id = id
self.cmd = cmd
self.payload = payload
self.clientApiVersion = clientApiVersion
self.deadlineUnixMilliseconds = deadlineUnixMilliseconds
self.sessionId = sessionId
self.turnId = turnId
self.requestId = requestId
}
/// The payload, defaulting to an empty object when omitted/null — every
@@ -14,18 +14,19 @@ import Foundation
/// which it no longer does. Instead the wait happens once, at the moment of
/// capture, and only if an action happened recently enough to still be settling.
///
/// This mirrors Codex's `needsUISettleBeforeSkyshot`: wait about a second after
/// a recent action, and extend that when the app is visibly still working.
/// This mirrors Codex's one-shot `needsUISettleBeforeSkyshot`: the next capture
/// for that target waits about 250ms after a recent action. A different app and
/// later captures do not inherit the wait.
///
/// Pure and clock-injected so the decision is testable without sleeping.
enum UISettlePolicy {
/// How long after a mutation the UI is presumed to still be settling.
static let postActionWindow: TimeInterval = 1.0
static let postActionWindow: TimeInterval = 0.25
/// Ceiling while the app still shows a busy/progress indicator. Generous
/// because "still loading" is a real signal, but bounded so a permanently
/// spinning indicator (some apps never stop one) cannot hang the capture.
static let busyWindow: TimeInterval = 5.0
/// Kept as a named compatibility seam for callers that already compute a
/// busy signal. Codex does not turn that signal into a multi-second sleep;
/// freshness comes from subsequent on-demand captures instead.
static let busyWindow: TimeInterval = postActionWindow
/// Never wait less than this once we've decided to wait at all — a delay
/// too short to cover a frame boundary is just latency for nothing.
@@ -57,28 +58,52 @@ enum UISettlePolicy {
}
}
/// Records when the last mutating action completed, so the capture path can ask
/// "was something just changed?" without the action path having to block.
/// Records one pending capture settle per target process, so an action in one
/// app cannot delay a screenshot of another app.
///
/// `@MainActor` because every mutation already runs there; this keeps the state
/// single-threaded without a lock.
@MainActor
enum MutationClock {
private static var lastMutationAt: TimeInterval?
private static var pendingByPID: [pid_t: TimeInterval] = [:]
private static var unscopedMutationAt: TimeInterval?
/// Called by the action path after a mutation lands. Does not sleep.
static func recordMutation(at time: TimeInterval = ProcessInfo.processInfo.systemUptime) {
lastMutationAt = time
static func recordMutation(
pid: pid_t? = nil,
at time: TimeInterval = ProcessInfo.processInfo.systemUptime
) {
if let pid {
pendingByPID[pid] = time
} else {
unscopedMutationAt = time
}
}
static func lastMutation() -> TimeInterval? { lastMutationAt }
static func lastMutation(pid: pid_t? = nil) -> TimeInterval? {
guard let pid else { return unscopedMutationAt }
return pendingByPID[pid]
}
/// Consume the one-shot marker before waiting. Another capture cannot pay
/// the same action's settle cost a second time.
static func takeMutation(pid: pid_t? = nil) -> TimeInterval? {
guard let pid else {
defer { unscopedMutationAt = nil }
return unscopedMutationAt
}
return pendingByPID.removeValue(forKey: pid)
}
/// Wait out whatever settle time the last mutation still owes.
///
/// Uses `Task.sleep`, not `Thread.sleep`: this runs on the main actor, where
/// blocking would starve the overlay's display link and freeze the virtual
/// cursor animation mid-glide.
static func awaitSettle(appIsBusy: Bool) async {
static func awaitSettle(
lastMutationAt: TimeInterval?,
appIsBusy: Bool
) async {
let delay = UISettlePolicy.delay(
now: ProcessInfo.processInfo.systemUptime,
lastMutationAt: lastMutationAt,
@@ -88,7 +113,10 @@ enum MutationClock {
try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
}
static func reset() { lastMutationAt = nil }
static func reset() {
pendingByPID.removeAll()
unscopedMutationAt = nil
}
static func resetForTests() { reset() }
}
@@ -172,7 +172,11 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
invalidate()
return nil
}
guard let shot = await captureSnapshot(for: target, scale: scale) else {
guard let shot = await captureSnapshot(
for: target,
scale: scale,
newerThanUptime: newerThanUptime
) else {
return nil
}
if let preferredWindowID,
@@ -213,7 +217,20 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
/// Match the reference's two separate lifetimes: SCStream remains a
/// consumer while covered; every state read runs an on-demand Skyshot/SCK
/// capture. An idle stream's cached frame is not evidence of the current UI.
func captureSnapshot(for target: WindowCaptureStreamTarget, scale: Double) async -> WindowShot? {
func captureSnapshot(
for target: WindowCaptureStreamTarget,
scale: Double,
newerThanUptime: TimeInterval? = nil
) async -> WindowShot? {
if let newerThanUptime {
// The stream is a long-lived render/freshness consumer, not the
// model screenshot source. Before the post-action Skyshot, observe
// a stream frame newer than the action when possible. `frame`
// performs one bounded rebuild for a silently starved stream; a
// static/no-op UI may legitimately emit no changed frame, so the
// authoritative on-demand screenshot still runs after the bound.
_ = await frame(for: target, newerThanUptime: newerThanUptime)
}
for _ in 0..<2 {
guard let source = await source(for: target) else { continue }
if source.hasFailed {
@@ -365,8 +365,8 @@ struct ClientAttestationTests {
let allowed = [
"list_apps", "resolve_app_target", "get_app_state", "click",
"set_value", "select_text", "perform_secondary_action", "scroll",
"drag", "press_key", "type_text", "ping", "shutdown",
"overlay_show", "overlay_hide", "check_permissions",
"drag", "press_key", "type_text", "paste", "ping", "shutdown",
"overlay_show", "overlay_hide", "turn_end", "check_permissions",
"input_monitor_state", "held_input_state",
]
for command in allowed {
@@ -5,6 +5,29 @@ import XCTest
@testable import cc_haha_computer_use
final class ClipboardPasteReceiptTests: XCTestCase {
@MainActor
func testHtmlPastePromisesRichAndPlainRepresentationsThenRestoresClipboard() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
let lease = ClipboardLease(pasteboard: fixture.board)
try await ClipboardPasteReceipt.perform(
text: "<strong>Hello</strong> world",
format: .html,
lease: lease
) { validate in
try await fixture.sendPaste(validate)
XCTAssertEqual(
String(data: fixture.board.data(forType: .html) ?? Data(), encoding: .utf8),
"<strong>Hello</strong> world"
)
XCTAssertTrue((fixture.board.string(forType: .string) ?? "").contains("Hello world"))
}
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.status, "completed")
XCTAssertEqual(fixture.board.string(forType: .string), "original")
}
@MainActor
func testPasteWaitsForARealReadBeyondTheOld180MillisecondWindow() async throws {
let fixture = PasteReceiptFixture()
@@ -0,0 +1,116 @@
import XCTest
@testable import cc_haha_computer_use
final class DaemonProtocolTests: XCTestCase {
private func request(
version: String? = ComputerUseDaemonProtocol.version,
deadline: Int64? = 2_000,
sessionId: String? = "session-a",
turnId: String? = "turn-a",
id: String? = "request-a",
requestId: String? = "request-a",
command: String = "get_app_state"
) -> Request {
Request(
id: id,
cmd: command,
payload: .object([:]),
clientApiVersion: version,
deadlineUnixMilliseconds: deadline,
sessionId: sessionId,
turnId: turnId,
requestId: requestId
)
}
func testValidRequestCarriesVersionDeadlineAndTurnIdentity() throws {
let metadata = try ComputerUseDaemonProtocol.validate(
request(),
nowUnixMilliseconds: 1_000
)
XCTAssertEqual(metadata.sessionId, "session-a")
XCTAssertEqual(metadata.turnId, "turn-a")
}
func testProtocolMismatchAndExpiredDeadlineFailClosed() {
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
request(version: "CCHahaComputerUseIPC-1"),
nowUnixMilliseconds: 1_000
)) { error in
XCTAssertEqual((error as? CUError)?.code, "protocol_mismatch")
}
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
request(deadline: 1_000),
nowUnixMilliseconds: 1_000
)) { error in
XCTAssertEqual((error as? CUError)?.code, "deadline_exceeded")
}
}
func testRequestIdentityAndTurnMetadataCannotBeOmitted() {
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
request(requestId: "different"),
nowUnixMilliseconds: 1_000
)) { error in
XCTAssertEqual((error as? CUError)?.code, "bad_request_id")
}
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
request(turnId: " "),
nowUnixMilliseconds: 1_000
)) { error in
XCTAssertEqual((error as? CUError)?.code, "missing_turn_metadata")
}
}
func testTurnGateRejectsCrossTurnStateUntilMatchingEnd() throws {
let first = ComputerUseDaemonProtocol.Metadata(
sessionId: "session-a",
turnId: "turn-a"
)
let second = ComputerUseDaemonProtocol.Metadata(
sessionId: "session-a",
turnId: "turn-b"
)
var gate = DaemonTurnGate()
try gate.admit(first, command: "get_app_state")
try gate.admit(first, command: "click")
XCTAssertThrowsError(try gate.admit(second, command: "get_app_state")) {
XCTAssertEqual(($0 as? CUError)?.code, "turn_mismatch")
}
try gate.finish(first)
try gate.admit(second, command: "get_app_state")
XCTAssertEqual(gate.active, second)
}
func testPingNegotiatesWithoutOpeningATurn() throws {
let metadata = ComputerUseDaemonProtocol.Metadata(
sessionId: "session-a",
turnId: "handshake-1"
)
var gate = DaemonTurnGate()
try gate.admit(metadata, command: "ping")
XCTAssertNil(gate.active)
XCTAssertEqual(
ComputerUseDaemonProtocol.hello()["protocolVersion"]?.asString,
ComputerUseDaemonProtocol.version
)
}
func testPermissionProbeIsConnectionScopedAndDoesNotOpenATurn() throws {
let metadata = ComputerUseDaemonProtocol.Metadata(
sessionId: "session-a",
turnId: "probe"
)
var gate = DaemonTurnGate()
try gate.admit(metadata, command: "check_permissions")
XCTAssertNil(gate.active)
}
}
@@ -0,0 +1,39 @@
import XCTest
@testable import cc_haha_computer_use
final class DisplaySleepAssertionTests: XCTestCase {
@MainActor
func testTurnAssertionIsAcquiredAndReleasedExactlyOnce() {
var creates = 0
var releases: [UInt32] = []
let assertion = ComputerUseDisplaySleepAssertion(
create: {
creates += 1
return 42
},
release: { releases.append($0) }
)
assertion.acquire()
assertion.acquire()
XCTAssertTrue(assertion.isHeldForTesting)
XCTAssertEqual(creates, 1)
assertion.release()
assertion.release()
XCTAssertFalse(assertion.isHeldForTesting)
XCTAssertEqual(releases, [42])
assertion.acquire()
XCTAssertEqual(creates, 2)
}
@MainActor
func testFailedPowerAssertionDoesNotPretendItIsHeld() {
let assertion = ComputerUseDisplaySleepAssertion(create: { nil })
assertion.acquire()
XCTAssertFalse(assertion.isHeldForTesting)
}
}
@@ -5,7 +5,7 @@ import XCTest
final class UISettlePolicyTests: XCTestCase {
/// Nothing has been mutated, so nothing is mid-transition on our account.
/// This is the common case for the first `get_app_state` of a session and
/// must not cost the model a second of latency.
/// must not add capture latency.
func testNoMutationMeansNoWait() {
XCTAssertEqual(
UISettlePolicy.delay(now: 100, lastMutationAt: nil, appIsBusy: false),
@@ -33,12 +33,12 @@ final class UISettlePolicyTests: XCTestCase {
)
}
/// A visible progress indicator extends the window — but only to a bound,
/// so an app that spins forever cannot hang the capture.
func testBusyAppGetsTheLongerWindowButStaysBounded() {
/// A permanently spinning progress indicator must not turn the one-shot
/// settle into a multi-second stall.
func testBusyAppUsesTheSameBoundedOneShotWindow() {
let busy = UISettlePolicy.delay(now: 100, lastMutationAt: 98.9, appIsBusy: true)
XCTAssertGreaterThan(busy, 0)
XCTAssertLessThanOrEqual(busy, UISettlePolicy.busyWindow)
XCTAssertEqual(busy, 0)
XCTAssertEqual(UISettlePolicy.busyWindow, UISettlePolicy.postActionWindow)
XCTAssertEqual(
UISettlePolicy.delay(
@@ -67,4 +67,16 @@ final class UISettlePolicyTests: XCTestCase {
XCTAssertLessThanOrEqual(delay, UISettlePolicy.postActionWindow)
XCTAssertGreaterThanOrEqual(delay, 0)
}
@MainActor
func testMutationMarkerIsScopedByPIDAndConsumedOnce() {
MutationClock.resetForTests()
defer { MutationClock.resetForTests() }
MutationClock.recordMutation(pid: 101, at: 42)
XCTAssertNil(MutationClock.takeMutation(pid: 202))
XCTAssertEqual(MutationClock.takeMutation(pid: 101), 42)
XCTAssertNil(MutationClock.takeMutation(pid: 101))
}
}
@@ -132,6 +132,35 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertEqual(factory.sources[0].latestReadCount, 0, "A cached stream frame must not become the model's screenshot")
}
func testPostMutationSnapshotConsumesFreshStreamWatermarkBeforeSkyshot() async throws {
let target = makeTarget(windowID: 87)
let factory = FakeWindowCaptureStreamFactory { source, _ in
source.startFrame = makeFrame(
for: source.targetKey,
sequence: 1,
uptime: 12,
byte: 7
)
}
var captures = 0
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
captures += 1
return self.makeSnapshot(target, pixels: "fresh-skyshot")
})
let shot = await manager.captureSnapshot(
for: target,
scale: 0.5,
newerThanUptime: 11
)
XCTAssertEqual(shot?.base64, "fresh-skyshot")
XCTAssertEqual(captures, 1)
XCTAssertEqual(factory.sources.count, 1)
XCTAssertGreaterThan(factory.sources[0].latestReadCount, 0)
XCTAssertEqual(factory.sources[0].retireCount, 0)
}
func testSnapshotFailureDoesNotFallBackToCachedStreamPixels() async {
let target = makeTarget(windowID: 85)
let factory = FakeWindowCaptureStreamFactory { source, _ in
@@ -171,9 +200,10 @@ final class WindowCaptureStreamTests: XCTestCase {
let target = makeTarget(windowID: 81)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var captures = 0
var captureTimes: [TimeInterval] = []
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
captures += 1
self.assertMutationHasSettledBeforeCapture()
captureTimes.append(ProcessInfo.processInfo.systemUptime)
return self.makeSnapshot(target, pixels: "unchanged-pixels")
})
var previousMutation: TimeInterval?
@@ -183,11 +213,20 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertNotNil(MutationClock.lastMutation())
XCTAssertNotEqual(MutationClock.lastMutation(), previousMutation)
previousMutation = MutationClock.lastMutation()
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
let pendingMutation = MutationClock.takeMutation()
let shot = await CommandRouter.captureSettledWindowShot(
appIsBusy: false,
lastMutationAt: pendingMutation
) {
await manager.captureSnapshot(for: target, scale: 0.5)
}
XCTAssertEqual(shot?.base64, "unchanged-pixels")
XCTAssertEqual(captures, expectedCaptureCount, "Identical pixels still require a new capture")
self.assertMutationHasSettledBeforeCapture(
pendingMutation,
capturedAt: captureTimes[expectedCaptureCount - 1]
)
XCTAssertNil(MutationClock.lastMutation(), "The settle marker is one-shot")
}
XCTAssertEqual(factory.sources.count, 1)
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
@@ -198,8 +237,9 @@ final class WindowCaptureStreamTests: XCTestCase {
defer { MutationClock.resetForTests() }
let target = makeTarget(windowID: 83)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var capturedAt: TimeInterval?
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
self.assertMutationHasSettledBeforeCapture()
capturedAt = ProcessInfo.processInfo.systemUptime
return self.makeSnapshot(target, pixels: "partial-action-state")
})
do {
@@ -211,18 +251,29 @@ final class WindowCaptureStreamTests: XCTestCase {
XCTAssertEqual(error.code, "partial_action")
}
XCTAssertNotNil(MutationClock.lastMutation())
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
let pendingMutation = MutationClock.takeMutation()
let shot = await CommandRouter.captureSettledWindowShot(
appIsBusy: false,
lastMutationAt: pendingMutation
) {
await manager.captureSnapshot(for: target, scale: 0.5)
}
XCTAssertEqual(shot?.base64, "partial-action-state")
self.assertMutationHasSettledBeforeCapture(
pendingMutation,
capturedAt: try XCTUnwrap(capturedAt)
)
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
}
private func assertMutationHasSettledBeforeCapture() {
XCTAssertNotNil(MutationClock.lastMutation())
private func assertMutationHasSettledBeforeCapture(
_ mutationAt: TimeInterval?,
capturedAt: TimeInterval
) {
XCTAssertNotNil(mutationAt)
XCTAssertEqual(UISettlePolicy.delay(
now: ProcessInfo.processInfo.systemUptime,
lastMutationAt: MutationClock.lastMutation(),
now: capturedAt,
lastMutationAt: mutationAt,
appIsBusy: false
), 0, "The actual screenshot callback must run after the action's settle deadline")
}
+4
View File
@@ -57,6 +57,8 @@ BIN_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use"
# Accessibility tolerates a bare binary (works), Screen Recording does NOT. So
# the shipped/dragged artifact is the .app; the inner binary is what we spawn.
APP_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use.app"
# Reuse the desktop brand asset so both Privacy lists show the product logo.
APP_ICON_PATH="$PKG_DIR/../../desktop/src-tauri/icons/icon.icns"
# Records the (identity, identifier) actually used, so we can detect rotation
# across rebuilds and warn that TCC grants will have been dropped.
SIGN_STAMP="$BUILD_DIR/.cu-helper.signid"
@@ -407,6 +409,7 @@ verify() {
# makes the inner binary's TCC identity a proper app bundle.
# ---------------------------------------------------------------------------
wrap_app() {
[ -s "$APP_ICON_PATH" ] || die "App icon not found at $APP_ICON_PATH (needed for the Privacy lists)."
log ""
log "==> wrap .app bundle: $APP_PATH"
rm -rf "$APP_PATH"
@@ -416,6 +419,7 @@ wrap_app() {
[ -f "$PKG_DIR/Info.plist" ] || die "Info.plist not found at $PKG_DIR/Info.plist (needed for the .app bundle)."
cp "$PKG_DIR/Info.plist" "$APP_PATH/Contents/Info.plist"
cp "$APP_ICON_PATH" "$APP_PATH/Contents/Resources/icon.icns"
# SwiftPM resource bundle (LensSequence overlay), loaded via Bundle.module.
# Standard .app location is Contents/Resources/ (Bundle.main.resourceURL). Do
+85 -1
View File
@@ -1,7 +1,59 @@
import { describe, expect, test } from 'bun:test'
import { afterEach, describe, expect, test } from 'bun:test'
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
import { tmpdir } from 'node:os'
import path from 'node:path'
const buildScript = path.resolve(import.meta.dirname, 'build.sh')
const productIcon = path.resolve(import.meta.dirname, '../../desktop/src-tauri/icons/icon.icns')
const fixtureDirectories: string[] = []
afterEach(() => {
for (const directory of fixtureDirectories.splice(0)) {
rmSync(directory, { recursive: true, force: true })
}
})
function wrapFixtureApp(missingIcon = false) {
const directory = mkdtempSync(path.join(tmpdir(), 'cu-helper-app-icon-'))
fixtureDirectories.push(directory)
writeFileSync(path.join(directory, 'fixture-binary'), 'fixture executable')
const result = Bun.spawnSync([
'bash',
'-c',
`
source "$1"
TEST_BUNDLE_DIR="$2"
BUILD_DIR="$TEST_BUNDLE_DIR/build"
BIN_PATH="$TEST_BUNDLE_DIR/fixture-binary"
APP_PATH="$TEST_BUNDLE_DIR/cc-haha-computer-use.app"
BUNDLE_ID="dev.cchaha.cu-helper"
SIGN_IDENTITY="fixture-only"
RESOLVED_TIMESTAMP_MODE="none"
if [ "$3" = "missing" ]; then
APP_ICON_PATH="$TEST_BUNDLE_DIR/missing.icns"
fi
codesign() {
case "$1" in
--force) cp -R "$APP_PATH/Contents" "$TEST_BUNDLE_DIR/contents-at-sign" ;;
-dv) printf 'Identifier=%s\\n' "$BUNDLE_ID" ;;
esac
}
wrap_app
`,
'cu-helper-app-icon-test',
buildScript,
directory,
missingIcon ? 'missing' : 'present',
], { cwd: directory })
return {
directory,
contents: path.join(directory, 'cc-haha-computer-use.app', 'Contents'),
exitCode: result.exitCode,
stderr: result.stderr.toString(),
}
}
function resolveTimestampArgument(identity: string, mode = 'auto') {
const result = Bun.spawnSync([
@@ -74,3 +126,35 @@ describe('cu-helper build signing identity', () => {
expect(result.stdout).toBe('Developer ID Application: Example Corp (TEAM123456)')
})
})
describe.skipIf(process.platform !== 'darwin')('cu-helper permission-list app icon', () => {
test('declares and bundles the product icon before signing the helper app', () => {
const result = wrapFixtureApp()
expect(result.exitCode).toBe(0)
const plist = Bun.spawnSync([
'/usr/bin/plutil', '-convert', 'json', '-o', '-',
path.join(result.contents, 'Info.plist'),
])
expect(plist.exitCode).toBe(0)
const info = JSON.parse(plist.stdout.toString())
expect(info.CFBundleIdentifier).toBe('dev.cchaha.cu-helper')
expect(info.CFBundleExecutable).toBe('cc-haha-computer-use')
expect(info.CFBundleIconFile).toBe('icon.icns')
const expectedIcon = readFileSync(productIcon)
expect(expectedIcon.subarray(0, 4).toString()).toBe('icns')
expect(readFileSync(path.join(result.contents, 'Resources', info.CFBundleIconFile)))
.toEqual(expectedIcon)
expect(readFileSync(path.join(result.directory, 'contents-at-sign', 'Resources', info.CFBundleIconFile)))
.toEqual(expectedIcon)
})
test('refuses to sign an app when the required product icon is missing', () => {
const result = wrapFixtureApp(true)
expect(result.exitCode).not.toBe(0)
expect(result.stderr).toContain('App icon not found')
expect(existsSync(path.join(result.directory, 'contents-at-sign'))).toBe(false)
})
})