mirror of
https://github.com/NanmiCoder/claude-code-haha.git
synced 2026-10-10 03:43:11 +08:00
fix(computer-use): align long-running macOS automation with Codex
This commit is contained in:
@@ -21,6 +21,8 @@
|
||||
<string>cc-haha-computer-use</string>
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>cc-haha-computer-use</string>
|
||||
<key>CFBundleIconFile</key>
|
||||
<string>icon.icns</string>
|
||||
<key>CFBundlePackageType</key>
|
||||
<string>APPL</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
|
||||
@@ -237,10 +237,18 @@ public final class ClipboardLease {
|
||||
}
|
||||
|
||||
func writeTemporaryStringWithReceipt(_ text: String) throws -> ClipboardPasteReceipt {
|
||||
try writeTemporaryContentWithReceipt(text, format: .text)
|
||||
}
|
||||
|
||||
func writeTemporaryContentWithReceipt(
|
||||
_ text: String,
|
||||
format: ClipboardPasteFormat
|
||||
) throws -> ClipboardPasteReceipt {
|
||||
if let captureError { throw captureError }
|
||||
let receipt = ClipboardPasteReceipt(text: text)
|
||||
let receipt = ClipboardPasteReceipt(text: text, format: format)
|
||||
let item = NSPasteboardItem()
|
||||
guard item.setDataProvider(receipt, forTypes: [.string]) else {
|
||||
let types = Array(format.promisedData(for: text).keys)
|
||||
guard item.setDataProvider(receipt, forTypes: types) else {
|
||||
throw CUError("clipboard_write_failed", "Could not register temporary pasteboard data")
|
||||
}
|
||||
pasteboard.clearContents()
|
||||
@@ -926,13 +934,35 @@ public enum AXAction {
|
||||
try await typeViaClipboard(target: target, text)
|
||||
}
|
||||
|
||||
/// Explicit Codex-compatible paste path. Unlike `typeText`, this does not
|
||||
/// first try AX value mutation or Unicode key events; it targets the app's
|
||||
/// in-process focused field with Command-V and restores the user's original
|
||||
/// pasteboard after observing a real promised-data read.
|
||||
static func pasteText(
|
||||
pid: pid_t,
|
||||
_ text: String,
|
||||
format: ClipboardPasteFormat
|
||||
) async throws {
|
||||
guard !text.isEmpty else { return }
|
||||
let target = try Injection.authorizeResolvedTarget(pid: pid)
|
||||
try await typeViaClipboard(target: target, text, format: format)
|
||||
}
|
||||
|
||||
/// Paste `text` into whatever holds in-app keyboard focus in `pid`, via the
|
||||
/// system clipboard + ⌘V (Codex's approach for CEF/Chromium text fields).
|
||||
/// The read receipt confirms supplied clipboard bytes, not the reader's
|
||||
/// PID or the field's final value; the next screenshot still verifies UI.
|
||||
private static func typeViaClipboard(target: ProvenProcessTarget, _ text: String) async throws {
|
||||
private static func typeViaClipboard(
|
||||
target: ProvenProcessTarget,
|
||||
_ text: String,
|
||||
format: ClipboardPasteFormat = .text
|
||||
) async throws {
|
||||
let pid = target.pid
|
||||
try await ClipboardPasteReceipt.perform(text: text, lease: ClipboardLease()) { validate in
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: text,
|
||||
format: format,
|
||||
lease: ClipboardLease()
|
||||
) { validate in
|
||||
try await pressKey(pid: pid, "super+v", validateBeforePosting: {
|
||||
_ = try Injection.validateAuthorizedTarget(target)
|
||||
try validate()
|
||||
@@ -1609,10 +1639,11 @@ public enum AXAction {
|
||||
return CFEqual(lhs, rhs)
|
||||
}
|
||||
|
||||
/// Mark that a mutation just landed. Does NOT block: `get_app_state` reads
|
||||
/// this stamp and waits out whatever settle time is still owed, so the cost
|
||||
/// is paid once, and only when someone is actually about to look.
|
||||
/// Settlement is recorded once at `ForegroundMutationRunner`, which covers
|
||||
/// every input path and knows the target PID. AX-local success receipts must
|
||||
/// not create a second, unscoped marker that can leak across apps.
|
||||
private static func settle() {
|
||||
MutationClock.recordMutation()
|
||||
// Intentionally empty; retained at the AX call sites as a semantic
|
||||
// marker for successful mutations.
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,9 +30,9 @@ enum HelperClientPolicy {
|
||||
// read-only authorization preflight used before the ten public tools.
|
||||
"list_apps", "resolve_app_target", "get_app_state", "click",
|
||||
"set_value", "select_text", "perform_secondary_action", "scroll",
|
||||
"drag", "press_key", "type_text",
|
||||
"drag", "press_key", "type_text", "paste",
|
||||
// Private lifecycle / visible-overlay / permission and input diagnostics.
|
||||
"ping", "shutdown", "overlay_show", "overlay_hide",
|
||||
"ping", "shutdown", "overlay_show", "overlay_hide", "turn_end",
|
||||
"check_permissions", "input_monitor_state", "focus_monitor_state", "held_input_state",
|
||||
"last_injection_state",
|
||||
]
|
||||
|
||||
@@ -2,6 +2,32 @@ import AppKit
|
||||
import Foundation
|
||||
import os
|
||||
|
||||
enum ClipboardPasteFormat: String, Sendable {
|
||||
case text
|
||||
case md
|
||||
case html
|
||||
|
||||
func promisedData(for text: String) -> [NSPasteboard.PasteboardType: Data] {
|
||||
switch self {
|
||||
case .text, .md:
|
||||
return [.string: Data(text.utf8)]
|
||||
case .html:
|
||||
let html = Data(text.utf8)
|
||||
let plain = (
|
||||
try? NSAttributedString(
|
||||
data: html,
|
||||
options: [
|
||||
.documentType: NSAttributedString.DocumentType.html,
|
||||
.characterEncoding: String.Encoding.utf8.rawValue,
|
||||
],
|
||||
documentAttributes: nil
|
||||
).string
|
||||
) ?? text
|
||||
return [.html: html, .string: Data(plain.utf8)]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Confirms that this pasteboard item's promised bytes were requested and
|
||||
/// supplied. AppKit does not identify the reader: this is not proof that the
|
||||
/// intended application's focused field accepted the text.
|
||||
@@ -25,16 +51,26 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
}
|
||||
|
||||
@MainActor private(set) static var lastDiagnostic: Diagnostic?
|
||||
private let data: Data
|
||||
private let promisedData: [NSPasteboard.PasteboardType: Data]
|
||||
private let state = OSAllocatedUnfairLock(initialState: State())
|
||||
|
||||
init(text: String) {
|
||||
data = Data(text.utf8)
|
||||
promisedData = ClipboardPasteFormat.text.promisedData(for: text)
|
||||
super.init()
|
||||
}
|
||||
|
||||
init(text: String, format: ClipboardPasteFormat) {
|
||||
promisedData = format.promisedData(for: text)
|
||||
super.init()
|
||||
}
|
||||
|
||||
@MainActor
|
||||
static func resetForTurn() {
|
||||
lastDiagnostic = nil
|
||||
}
|
||||
|
||||
func pasteboard(_ pasteboard: NSPasteboard?, item: NSPasteboardItem, provideDataForType type: NSPasteboard.PasteboardType) {
|
||||
guard type == .string else { return }
|
||||
guard let data = promisedData[type] else { return }
|
||||
state.withLock { $0.requested = true }
|
||||
guard item.setData(data, forType: type) else { return }
|
||||
state.withLock { value in
|
||||
@@ -50,6 +86,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
@MainActor
|
||||
static func perform(
|
||||
text: String,
|
||||
format: ClipboardPasteFormat = .text,
|
||||
lease: ClipboardLease,
|
||||
timeout: Duration = .seconds(2),
|
||||
sendPaste: @MainActor (_ validateBeforePosting: @MainActor () throws -> Void) async throws -> Void
|
||||
@@ -79,7 +116,7 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
}
|
||||
do {
|
||||
try Task.checkCancellation()
|
||||
let written = try lease.writeTemporaryStringWithReceipt(text)
|
||||
let written = try lease.writeTemporaryContentWithReceipt(text, format: format)
|
||||
receipt = written
|
||||
try await Task.sleep(for: .milliseconds(40))
|
||||
let validate: @MainActor () throws -> Void = {
|
||||
|
||||
@@ -104,6 +104,7 @@ public final class CommandRouter {
|
||||
Self.lastShotTransform.removeAll()
|
||||
Self.lastCaptureDigest.removeAll()
|
||||
MutationClock.reset()
|
||||
ClipboardPasteReceipt.resetForTurn()
|
||||
windowCaptureProvider?.invalidate()
|
||||
// Apps we told they were focused must be told they are not, or the
|
||||
// belief outlives the session that needed it.
|
||||
@@ -267,6 +268,9 @@ public final class CommandRouter {
|
||||
case "type_text":
|
||||
return try await handleTypeText(payload)
|
||||
|
||||
case "paste":
|
||||
return try await handlePaste(payload)
|
||||
|
||||
case "press_key":
|
||||
return try await handlePressKey(payload)
|
||||
|
||||
@@ -728,18 +732,19 @@ public final class CommandRouter {
|
||||
if #available(macOS 14.0, *) {
|
||||
// Let the UI finish whatever the last action started before we
|
||||
// photograph it, otherwise the model reasons about a half-drawn
|
||||
// frame. Costs nothing when no action is pending. The rendered tree
|
||||
// doubles as the busy signal — a progress indicator in it means the
|
||||
// app is still working, so we allow a longer window.
|
||||
// frame. Costs nothing when no action is pending, applies only to
|
||||
// this target PID, and is consumed by this single capture.
|
||||
let pendingMutation = MutationClock.takeMutation(pid: pid)
|
||||
let streamedShot = await Self.captureSettledWindowShot(
|
||||
appIsBusy: result.axText.contains("progress indicator")
|
||||
appIsBusy: result.axText.contains("progress indicator"),
|
||||
lastMutationAt: pendingMutation
|
||||
) {
|
||||
await windowCaptureProvider?.windowShot(
|
||||
pid: pid,
|
||||
processIdentity: snapshotEvidence.processIdentity,
|
||||
preferredWindowID: snapshotEvidence.keyWindowID,
|
||||
scale: 0.5,
|
||||
newerThanUptime: MutationClock.lastMutation()
|
||||
newerThanUptime: pendingMutation
|
||||
)
|
||||
}
|
||||
guard TargetVisibilityPolicy.captureTargetStillMatches(
|
||||
@@ -875,9 +880,13 @@ public final class CommandRouter {
|
||||
/// action-to-screenshot regression can exercise both without live AX/TCC.
|
||||
static func captureSettledWindowShot(
|
||||
appIsBusy: Bool,
|
||||
lastMutationAt: TimeInterval?,
|
||||
capture: () async -> WindowShot?
|
||||
) async -> WindowShot? {
|
||||
await MutationClock.awaitSettle(appIsBusy: appIsBusy)
|
||||
await MutationClock.awaitSettle(
|
||||
lastMutationAt: lastMutationAt,
|
||||
appIsBusy: appIsBusy
|
||||
)
|
||||
return await capture()
|
||||
}
|
||||
|
||||
@@ -1438,6 +1447,31 @@ public final class CommandRouter {
|
||||
}
|
||||
}
|
||||
|
||||
/// `paste`: bypass AX/Unicode typing and use the target app's in-process
|
||||
/// paste focus. This is the reliable recovery path for Chromium/CEF fields
|
||||
/// such as NeteaseMusic's search box. The clipboard lease restores every
|
||||
/// original pasteboard item unless the user copied something meanwhile.
|
||||
private func handlePaste(_ payload: JSONValue) async throws -> JSONValue {
|
||||
try requireAXTrusted()
|
||||
guard let text = payload["text"]?.asString else {
|
||||
throw CUError("bad_payload", "paste requires a 'text' string")
|
||||
}
|
||||
guard let rawFormat = payload["format"]?.asString,
|
||||
let format = ClipboardPasteFormat(rawValue: rawFormat) else {
|
||||
throw CUError("bad_payload", "paste format must be 'text', 'md', or 'html'")
|
||||
}
|
||||
let expected = try Self.expectedProcessTarget(payload)
|
||||
let target = try resolveTargetForMutation(payload)
|
||||
setResolvedTarget(target)
|
||||
try requireSnapshotProcess(target: target, expected: expected)
|
||||
return try await withForegroundLease(command: "paste", target: target) {
|
||||
_ = try Injection.validateAuthorizedTarget(target)
|
||||
try self.requireSnapshotProcess(target: target, expected: expected)
|
||||
try await AXAction.pasteText(pid: target.pid, text, format: format)
|
||||
return .bool(true)
|
||||
}
|
||||
}
|
||||
|
||||
/// `press_key`: send an xdotool-style key spec (e.g. "super+c", "Return",
|
||||
/// "Tab", "KP_0", "Prior") to the target app via postToPid.
|
||||
private func handlePressKey(_ payload: JSONValue) async throws -> JSONValue {
|
||||
@@ -1566,6 +1600,7 @@ public final class CommandRouter {
|
||||
)
|
||||
return try await ForegroundMutationRunner.run(
|
||||
lease: lease,
|
||||
targetPID: target.pid,
|
||||
action: action
|
||||
)
|
||||
}
|
||||
|
||||
@@ -38,21 +38,21 @@ enum DaemonOverlayTargetResolver {
|
||||
// `cursor_position`, implicit-`from` drags and decomposed `mouse_down`/`mouse_up`
|
||||
// behave correctly across commands.
|
||||
//
|
||||
// Transport: a private AF_UNIX SOCK_STREAM socket carrying NDJSON (one JSON
|
||||
// object + '\n' per request, same per response). A GUI/AppKit process leaks
|
||||
// Transport: a private AF_UNIX SOCK_STREAM socket carrying versioned NDJSON
|
||||
// (one JSON object + '\n' per request, same per response). A GUI/AppKit process leaks
|
||||
// os_log / CoreGraphics chatter to stdout/stderr, which would corrupt the TS
|
||||
// bridge's `JSON.parse`; the daemon therefore reserves stdout for exactly ONE
|
||||
// readiness line and serves the request/response stream over the socket.
|
||||
//
|
||||
// readiness : {"ready":true,"pid":<N>,"proto":1}\n (stdout, once)
|
||||
// request : {"id":"<opaque>","cmd":"<verb>","payload":{...}}\n
|
||||
// readiness : {"ready":true,"pid":<N>,"protocolVersion":"…"}\n
|
||||
// request : {"id":"<opaque>","requestId":"<opaque>","cmd":"<verb>",…}\n
|
||||
// response : {"id":"<opaque>","ok":true,"result":<json>}\n
|
||||
// {"id":"<opaque>","ok":false,"error":{"message":"…","code":"…"}}\n
|
||||
//
|
||||
// Control verbs handled in-daemon (never reach CommandRouter):
|
||||
// overlay_show -> cursor.show() ; result true
|
||||
// overlay_hide -> cursor.hide() ; result true
|
||||
// ping -> result "pong"
|
||||
// turn_end -> release all turn-owned state ; result true
|
||||
// ping -> version/capability hello
|
||||
// shutdown -> result true, then NSApp.terminate(nil)
|
||||
// Every other cmd is forwarded to the shared CommandRouter — the exact same
|
||||
// dispatcher the one-shot CLI path uses, so each command has one implementation.
|
||||
@@ -63,6 +63,7 @@ public final class Daemon {
|
||||
private let cursor: VirtualCursor
|
||||
private let inputMonitor: PhysicalInputEpochMonitor
|
||||
private let router: CommandRouter
|
||||
private let displaySleepAssertion = ComputerUseDisplaySleepAssertion()
|
||||
|
||||
/// Listening socket fd (AF_UNIX SOCK_STREAM). -1 until `bindAndListen`.
|
||||
private var listenFD: Int32 = -1
|
||||
@@ -78,11 +79,12 @@ public final class Daemon {
|
||||
private var connection: Connection?
|
||||
private var connectionToken: DaemonSessionToken?
|
||||
private var sessionGate = DaemonSessionGate()
|
||||
private var turnGate = DaemonTurnGate()
|
||||
|
||||
/// Set once the run loop is live; guards against double `run()`.
|
||||
private var didStartRunLoop = false
|
||||
|
||||
/// True between `overlay_show` and `overlay_hide` (CU active this turn). The
|
||||
/// True between `overlay_show` and `turn_end` (CU active this turn). The
|
||||
/// cursor re-aims at the actual injection target while this holds.
|
||||
private var overlayActive = false
|
||||
private var explicitOverlayTarget: ProvenProcessTarget?
|
||||
@@ -220,6 +222,7 @@ public final class Daemon {
|
||||
// immediately by exit() (shutdown verb + signal handlers).
|
||||
Injection.releaseAllHeldSync()
|
||||
router.resetSessionState()
|
||||
displaySleepAssertion.release()
|
||||
inputMonitor.stop()
|
||||
}
|
||||
|
||||
@@ -435,6 +438,8 @@ public final class Daemon {
|
||||
stopOverlaySession()
|
||||
Injection.releaseAllHeldSync()
|
||||
router.resetSessionState()
|
||||
turnGate.reset()
|
||||
displaySleepAssertion.release()
|
||||
}
|
||||
|
||||
/// A LaunchServices child is reparented to launchd, so the peer socket is
|
||||
@@ -479,7 +484,16 @@ public final class Daemon {
|
||||
let payload = request.payload ?? .object([:])
|
||||
|
||||
do {
|
||||
let metadata = try ComputerUseDaemonProtocol.validate(request)
|
||||
let opensTurn = ComputerUseDaemonProtocol.isTurnScoped(command: request.cmd)
|
||||
&& turnGate.active == nil
|
||||
try turnGate.admit(metadata, command: request.cmd)
|
||||
if opensTurn { displaySleepAssertion.acquire() }
|
||||
let result = try await route(cmd: request.cmd, payload: payload)
|
||||
if request.cmd == "turn_end" || request.cmd == "overlay_hide" {
|
||||
try turnGate.finish(metadata)
|
||||
displaySleepAssertion.release()
|
||||
}
|
||||
conn.send(encodeResponse(DaemonResponse(id: id, ok: true, result: result, error: nil)))
|
||||
} catch let cuError as CUError {
|
||||
conn.send(encodeResponse(DaemonResponse(
|
||||
@@ -508,12 +522,12 @@ public final class Daemon {
|
||||
try showOverlay(payload: payload)
|
||||
return .bool(true)
|
||||
|
||||
case "overlay_hide":
|
||||
stopOverlaySession()
|
||||
case "overlay_hide", "turn_end":
|
||||
endTurn()
|
||||
return .bool(true)
|
||||
|
||||
case "ping":
|
||||
return .string("pong")
|
||||
return ComputerUseDaemonProtocol.hello()
|
||||
|
||||
case "check_permissions":
|
||||
// Usable in both modes; the daemon exposes it for onboarding /
|
||||
@@ -592,6 +606,15 @@ public final class Daemon {
|
||||
router.invalidateWindowCaptureStream()
|
||||
}
|
||||
|
||||
/// Codex-parity turn boundary. The helper process stays warm, but no AX
|
||||
/// snapshot, coordinate transform, focus belief, held input, mutation clock,
|
||||
/// clipboard diagnostic, or capture stream may leak into the next turn.
|
||||
private func endTurn() {
|
||||
stopOverlaySession()
|
||||
Injection.releaseAllHeldSync()
|
||||
router.resetSessionState()
|
||||
}
|
||||
|
||||
/// The app the cursor should follow: ONLY the app the last injection /
|
||||
/// get_app_state actually resolved (`Injection.lastResolvedTargetPid`). We
|
||||
/// deliberately do NOT fall back to the frontmost app — at turn start,
|
||||
@@ -763,9 +786,9 @@ private final class Connection: @unchecked Sendable {
|
||||
private var inBuffer = Data()
|
||||
|
||||
/// Capped to avoid unbounded growth from a malformed/never-newline client.
|
||||
/// 16 MiB comfortably exceeds the largest legitimate request (a base64
|
||||
/// 8 MiB comfortably exceeds the largest legitimate request (a base64
|
||||
/// screenshot travels in *responses*, not requests).
|
||||
private let maxBufferBytes = 16 * 1024 * 1024
|
||||
private let maxBufferBytes = ComputerUseDaemonProtocol.maxFrameBytes
|
||||
|
||||
private var onRequest: (@Sendable (Data) -> Void)?
|
||||
private var onClose: (@Sendable () -> Void)?
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
import Foundation
|
||||
|
||||
/// Versioned request contract for the authenticated daemon socket.
|
||||
///
|
||||
/// Code signature attestation proves who is connected; this handshake proves
|
||||
/// that both sides agree on request shape and lifecycle semantics. Every request
|
||||
/// also carries an absolute deadline plus session/turn identity so a command
|
||||
/// queued behind a slow capture cannot execute after its caller has moved on.
|
||||
enum ComputerUseDaemonProtocol {
|
||||
static let version = "CCHahaComputerUseIPC-2"
|
||||
static let maxFrameBytes = 8 * 1024 * 1024
|
||||
private static let connectionScopedCommands: Set<String> = [
|
||||
"ping", "check_permissions", "shutdown",
|
||||
]
|
||||
|
||||
static func isTurnScoped(command: String) -> Bool {
|
||||
!connectionScopedCommands.contains(command)
|
||||
}
|
||||
|
||||
struct Metadata: Equatable, Sendable {
|
||||
let sessionId: String
|
||||
let turnId: String
|
||||
}
|
||||
|
||||
static func validate(
|
||||
_ request: Request,
|
||||
nowUnixMilliseconds: Int64 = Int64(Date().timeIntervalSince1970 * 1_000)
|
||||
) throws -> Metadata {
|
||||
guard request.clientApiVersion == version else {
|
||||
throw CUError(
|
||||
"protocol_mismatch",
|
||||
"Computer Use client/helper protocol mismatch; restart the app after updating"
|
||||
)
|
||||
}
|
||||
guard let id = request.id, !id.isEmpty,
|
||||
request.requestId == id else {
|
||||
throw CUError("bad_request_id", "Computer Use request identity is missing or inconsistent")
|
||||
}
|
||||
guard let deadline = request.deadlineUnixMilliseconds else {
|
||||
throw CUError("missing_deadline", "Computer Use request is missing its absolute deadline")
|
||||
}
|
||||
guard deadline > nowUnixMilliseconds else {
|
||||
throw CUError("deadline_exceeded", "Computer Use request expired before execution")
|
||||
}
|
||||
guard let sessionId = request.sessionId?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!sessionId.isEmpty,
|
||||
let turnId = request.turnId?.trimmingCharacters(in: .whitespacesAndNewlines),
|
||||
!turnId.isEmpty else {
|
||||
throw CUError("missing_turn_metadata", "Computer Use request is missing session/turn identity")
|
||||
}
|
||||
return Metadata(sessionId: sessionId, turnId: turnId)
|
||||
}
|
||||
|
||||
static func hello() -> JSONValue {
|
||||
.object([
|
||||
"protocolVersion": .string(version),
|
||||
"supportsAbsoluteDeadlines": .bool(true),
|
||||
"supportsTurnEnd": .bool(true),
|
||||
"supportsCaptureDiagnostics": .bool(true),
|
||||
])
|
||||
}
|
||||
}
|
||||
|
||||
/// Enforces one explicit turn at a time on the single authenticated connection.
|
||||
/// `ping` negotiates the protocol without opening a turn. Every other request
|
||||
/// must keep the same identity until `turn_end` releases all turn-owned state.
|
||||
struct DaemonTurnGate {
|
||||
private(set) var active: ComputerUseDaemonProtocol.Metadata?
|
||||
|
||||
mutating func admit(
|
||||
_ metadata: ComputerUseDaemonProtocol.Metadata,
|
||||
command: String
|
||||
) throws {
|
||||
if !ComputerUseDaemonProtocol.isTurnScoped(command: command) { return }
|
||||
if let active {
|
||||
guard active == metadata else {
|
||||
throw CUError(
|
||||
"turn_mismatch",
|
||||
"A different Computer Use turn is still active; finish it before starting another"
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
active = metadata
|
||||
}
|
||||
|
||||
mutating func finish(
|
||||
_ metadata: ComputerUseDaemonProtocol.Metadata
|
||||
) throws {
|
||||
guard active == metadata else {
|
||||
throw CUError("turn_mismatch", "Computer Use turn_end did not match the active turn")
|
||||
}
|
||||
active = nil
|
||||
}
|
||||
|
||||
mutating func reset() {
|
||||
active = nil
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import Foundation
|
||||
import IOKit.pwr_mgt
|
||||
|
||||
/// Keeps the display compositor awake while a Computer Use turn is active.
|
||||
/// Covered-window ScreenCaptureKit consumers can otherwise stop receiving
|
||||
/// useful updates once macOS enters user-idle display sleep during a long task.
|
||||
@MainActor
|
||||
final class ComputerUseDisplaySleepAssertion {
|
||||
typealias AssertionID = IOPMAssertionID
|
||||
|
||||
private let create: () -> AssertionID?
|
||||
private let releaseAssertion: (AssertionID) -> Void
|
||||
private var assertionID: AssertionID?
|
||||
|
||||
init(
|
||||
create: @escaping () -> AssertionID? = {
|
||||
var assertionID: IOPMAssertionID = 0
|
||||
let result = IOPMAssertionCreateWithDescription(
|
||||
kIOPMAssertionTypePreventUserIdleDisplaySleep as CFString,
|
||||
"Claude Code Haha Computer Use interaction" as CFString,
|
||||
"Computer Use turn is controlling a background application" as CFString,
|
||||
"Keeping the display awake while Computer Use is active" as CFString,
|
||||
nil,
|
||||
0,
|
||||
nil,
|
||||
&assertionID
|
||||
)
|
||||
return result == kIOReturnSuccess ? assertionID : nil
|
||||
},
|
||||
release: @escaping (AssertionID) -> Void = { assertionID in
|
||||
_ = IOPMAssertionRelease(assertionID)
|
||||
}
|
||||
) {
|
||||
self.create = create
|
||||
self.releaseAssertion = release
|
||||
}
|
||||
|
||||
func acquire() {
|
||||
guard assertionID == nil else { return }
|
||||
assertionID = create()
|
||||
}
|
||||
|
||||
func release() {
|
||||
guard let assertionID else { return }
|
||||
self.assertionID = nil
|
||||
releaseAssertion(assertionID)
|
||||
}
|
||||
|
||||
var isHeldForTesting: Bool { assertionID != nil }
|
||||
}
|
||||
@@ -394,12 +394,13 @@ final class ForegroundLease {
|
||||
enum ForegroundMutationRunner {
|
||||
static func run<T>(
|
||||
lease: ForegroundLease,
|
||||
targetPID: pid_t? = nil,
|
||||
action: () async throws -> T
|
||||
) async throws -> T {
|
||||
// Every input path crosses this boundary, including synthetic events
|
||||
// that never call AXAction.settle(). A throw can follow a partial
|
||||
// delivery, so neither success nor failure may reuse pre-action pixels.
|
||||
defer { MutationClock.recordMutation() }
|
||||
defer { MutationClock.recordMutation(pid: targetPID) }
|
||||
let result: Result<T, Error>
|
||||
do {
|
||||
result = .success(try await action())
|
||||
|
||||
@@ -439,11 +439,30 @@ public struct Request: Decodable, Sendable {
|
||||
public let id: String?
|
||||
public let cmd: String
|
||||
public let payload: JSONValue?
|
||||
public let clientApiVersion: String?
|
||||
public let deadlineUnixMilliseconds: Int64?
|
||||
public let sessionId: String?
|
||||
public let turnId: String?
|
||||
public let requestId: String?
|
||||
|
||||
public init(id: String?, cmd: String, payload: JSONValue?) {
|
||||
public init(
|
||||
id: String?,
|
||||
cmd: String,
|
||||
payload: JSONValue?,
|
||||
clientApiVersion: String? = nil,
|
||||
deadlineUnixMilliseconds: Int64? = nil,
|
||||
sessionId: String? = nil,
|
||||
turnId: String? = nil,
|
||||
requestId: String? = nil
|
||||
) {
|
||||
self.id = id
|
||||
self.cmd = cmd
|
||||
self.payload = payload
|
||||
self.clientApiVersion = clientApiVersion
|
||||
self.deadlineUnixMilliseconds = deadlineUnixMilliseconds
|
||||
self.sessionId = sessionId
|
||||
self.turnId = turnId
|
||||
self.requestId = requestId
|
||||
}
|
||||
|
||||
/// The payload, defaulting to an empty object when omitted/null — every
|
||||
|
||||
@@ -14,18 +14,19 @@ import Foundation
|
||||
/// which it no longer does. Instead the wait happens once, at the moment of
|
||||
/// capture, and only if an action happened recently enough to still be settling.
|
||||
///
|
||||
/// This mirrors Codex's `needsUISettleBeforeSkyshot`: wait about a second after
|
||||
/// a recent action, and extend that when the app is visibly still working.
|
||||
/// This mirrors Codex's one-shot `needsUISettleBeforeSkyshot`: the next capture
|
||||
/// for that target waits about 250ms after a recent action. A different app and
|
||||
/// later captures do not inherit the wait.
|
||||
///
|
||||
/// Pure and clock-injected so the decision is testable without sleeping.
|
||||
enum UISettlePolicy {
|
||||
/// How long after a mutation the UI is presumed to still be settling.
|
||||
static let postActionWindow: TimeInterval = 1.0
|
||||
static let postActionWindow: TimeInterval = 0.25
|
||||
|
||||
/// Ceiling while the app still shows a busy/progress indicator. Generous
|
||||
/// because "still loading" is a real signal, but bounded so a permanently
|
||||
/// spinning indicator (some apps never stop one) cannot hang the capture.
|
||||
static let busyWindow: TimeInterval = 5.0
|
||||
/// Kept as a named compatibility seam for callers that already compute a
|
||||
/// busy signal. Codex does not turn that signal into a multi-second sleep;
|
||||
/// freshness comes from subsequent on-demand captures instead.
|
||||
static let busyWindow: TimeInterval = postActionWindow
|
||||
|
||||
/// Never wait less than this once we've decided to wait at all — a delay
|
||||
/// too short to cover a frame boundary is just latency for nothing.
|
||||
@@ -57,28 +58,52 @@ enum UISettlePolicy {
|
||||
}
|
||||
}
|
||||
|
||||
/// Records when the last mutating action completed, so the capture path can ask
|
||||
/// "was something just changed?" without the action path having to block.
|
||||
/// Records one pending capture settle per target process, so an action in one
|
||||
/// app cannot delay a screenshot of another app.
|
||||
///
|
||||
/// `@MainActor` because every mutation already runs there; this keeps the state
|
||||
/// single-threaded without a lock.
|
||||
@MainActor
|
||||
enum MutationClock {
|
||||
private static var lastMutationAt: TimeInterval?
|
||||
private static var pendingByPID: [pid_t: TimeInterval] = [:]
|
||||
private static var unscopedMutationAt: TimeInterval?
|
||||
|
||||
/// Called by the action path after a mutation lands. Does not sleep.
|
||||
static func recordMutation(at time: TimeInterval = ProcessInfo.processInfo.systemUptime) {
|
||||
lastMutationAt = time
|
||||
static func recordMutation(
|
||||
pid: pid_t? = nil,
|
||||
at time: TimeInterval = ProcessInfo.processInfo.systemUptime
|
||||
) {
|
||||
if let pid {
|
||||
pendingByPID[pid] = time
|
||||
} else {
|
||||
unscopedMutationAt = time
|
||||
}
|
||||
}
|
||||
|
||||
static func lastMutation() -> TimeInterval? { lastMutationAt }
|
||||
static func lastMutation(pid: pid_t? = nil) -> TimeInterval? {
|
||||
guard let pid else { return unscopedMutationAt }
|
||||
return pendingByPID[pid]
|
||||
}
|
||||
|
||||
/// Consume the one-shot marker before waiting. Another capture cannot pay
|
||||
/// the same action's settle cost a second time.
|
||||
static func takeMutation(pid: pid_t? = nil) -> TimeInterval? {
|
||||
guard let pid else {
|
||||
defer { unscopedMutationAt = nil }
|
||||
return unscopedMutationAt
|
||||
}
|
||||
return pendingByPID.removeValue(forKey: pid)
|
||||
}
|
||||
|
||||
/// Wait out whatever settle time the last mutation still owes.
|
||||
///
|
||||
/// Uses `Task.sleep`, not `Thread.sleep`: this runs on the main actor, where
|
||||
/// blocking would starve the overlay's display link and freeze the virtual
|
||||
/// cursor animation mid-glide.
|
||||
static func awaitSettle(appIsBusy: Bool) async {
|
||||
static func awaitSettle(
|
||||
lastMutationAt: TimeInterval?,
|
||||
appIsBusy: Bool
|
||||
) async {
|
||||
let delay = UISettlePolicy.delay(
|
||||
now: ProcessInfo.processInfo.systemUptime,
|
||||
lastMutationAt: lastMutationAt,
|
||||
@@ -88,7 +113,10 @@ enum MutationClock {
|
||||
try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
|
||||
}
|
||||
|
||||
static func reset() { lastMutationAt = nil }
|
||||
static func reset() {
|
||||
pendingByPID.removeAll()
|
||||
unscopedMutationAt = nil
|
||||
}
|
||||
|
||||
static func resetForTests() { reset() }
|
||||
}
|
||||
|
||||
@@ -172,7 +172,11 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
invalidate()
|
||||
return nil
|
||||
}
|
||||
guard let shot = await captureSnapshot(for: target, scale: scale) else {
|
||||
guard let shot = await captureSnapshot(
|
||||
for: target,
|
||||
scale: scale,
|
||||
newerThanUptime: newerThanUptime
|
||||
) else {
|
||||
return nil
|
||||
}
|
||||
if let preferredWindowID,
|
||||
@@ -213,7 +217,20 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
/// Match the reference's two separate lifetimes: SCStream remains a
|
||||
/// consumer while covered; every state read runs an on-demand Skyshot/SCK
|
||||
/// capture. An idle stream's cached frame is not evidence of the current UI.
|
||||
func captureSnapshot(for target: WindowCaptureStreamTarget, scale: Double) async -> WindowShot? {
|
||||
func captureSnapshot(
|
||||
for target: WindowCaptureStreamTarget,
|
||||
scale: Double,
|
||||
newerThanUptime: TimeInterval? = nil
|
||||
) async -> WindowShot? {
|
||||
if let newerThanUptime {
|
||||
// The stream is a long-lived render/freshness consumer, not the
|
||||
// model screenshot source. Before the post-action Skyshot, observe
|
||||
// a stream frame newer than the action when possible. `frame`
|
||||
// performs one bounded rebuild for a silently starved stream; a
|
||||
// static/no-op UI may legitimately emit no changed frame, so the
|
||||
// authoritative on-demand screenshot still runs after the bound.
|
||||
_ = await frame(for: target, newerThanUptime: newerThanUptime)
|
||||
}
|
||||
for _ in 0..<2 {
|
||||
guard let source = await source(for: target) else { continue }
|
||||
if source.hasFailed {
|
||||
|
||||
@@ -365,8 +365,8 @@ struct ClientAttestationTests {
|
||||
let allowed = [
|
||||
"list_apps", "resolve_app_target", "get_app_state", "click",
|
||||
"set_value", "select_text", "perform_secondary_action", "scroll",
|
||||
"drag", "press_key", "type_text", "ping", "shutdown",
|
||||
"overlay_show", "overlay_hide", "check_permissions",
|
||||
"drag", "press_key", "type_text", "paste", "ping", "shutdown",
|
||||
"overlay_show", "overlay_hide", "turn_end", "check_permissions",
|
||||
"input_monitor_state", "held_input_state",
|
||||
]
|
||||
for command in allowed {
|
||||
|
||||
@@ -5,6 +5,29 @@ import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class ClipboardPasteReceiptTests: XCTestCase {
|
||||
@MainActor
|
||||
func testHtmlPastePromisesRichAndPlainRepresentationsThenRestoresClipboard() async throws {
|
||||
let fixture = PasteReceiptFixture()
|
||||
defer { fixture.close() }
|
||||
let lease = ClipboardLease(pasteboard: fixture.board)
|
||||
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: "<strong>Hello</strong> world",
|
||||
format: .html,
|
||||
lease: lease
|
||||
) { validate in
|
||||
try await fixture.sendPaste(validate)
|
||||
XCTAssertEqual(
|
||||
String(data: fixture.board.data(forType: .html) ?? Data(), encoding: .utf8),
|
||||
"<strong>Hello</strong> world"
|
||||
)
|
||||
XCTAssertTrue((fixture.board.string(forType: .string) ?? "").contains("Hello world"))
|
||||
}
|
||||
|
||||
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.status, "completed")
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "original")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testPasteWaitsForARealReadBeyondTheOld180MillisecondWindow() async throws {
|
||||
let fixture = PasteReceiptFixture()
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class DaemonProtocolTests: XCTestCase {
|
||||
private func request(
|
||||
version: String? = ComputerUseDaemonProtocol.version,
|
||||
deadline: Int64? = 2_000,
|
||||
sessionId: String? = "session-a",
|
||||
turnId: String? = "turn-a",
|
||||
id: String? = "request-a",
|
||||
requestId: String? = "request-a",
|
||||
command: String = "get_app_state"
|
||||
) -> Request {
|
||||
Request(
|
||||
id: id,
|
||||
cmd: command,
|
||||
payload: .object([:]),
|
||||
clientApiVersion: version,
|
||||
deadlineUnixMilliseconds: deadline,
|
||||
sessionId: sessionId,
|
||||
turnId: turnId,
|
||||
requestId: requestId
|
||||
)
|
||||
}
|
||||
|
||||
func testValidRequestCarriesVersionDeadlineAndTurnIdentity() throws {
|
||||
let metadata = try ComputerUseDaemonProtocol.validate(
|
||||
request(),
|
||||
nowUnixMilliseconds: 1_000
|
||||
)
|
||||
|
||||
XCTAssertEqual(metadata.sessionId, "session-a")
|
||||
XCTAssertEqual(metadata.turnId, "turn-a")
|
||||
}
|
||||
|
||||
func testProtocolMismatchAndExpiredDeadlineFailClosed() {
|
||||
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
|
||||
request(version: "CCHahaComputerUseIPC-1"),
|
||||
nowUnixMilliseconds: 1_000
|
||||
)) { error in
|
||||
XCTAssertEqual((error as? CUError)?.code, "protocol_mismatch")
|
||||
}
|
||||
|
||||
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
|
||||
request(deadline: 1_000),
|
||||
nowUnixMilliseconds: 1_000
|
||||
)) { error in
|
||||
XCTAssertEqual((error as? CUError)?.code, "deadline_exceeded")
|
||||
}
|
||||
}
|
||||
|
||||
func testRequestIdentityAndTurnMetadataCannotBeOmitted() {
|
||||
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
|
||||
request(requestId: "different"),
|
||||
nowUnixMilliseconds: 1_000
|
||||
)) { error in
|
||||
XCTAssertEqual((error as? CUError)?.code, "bad_request_id")
|
||||
}
|
||||
|
||||
XCTAssertThrowsError(try ComputerUseDaemonProtocol.validate(
|
||||
request(turnId: " "),
|
||||
nowUnixMilliseconds: 1_000
|
||||
)) { error in
|
||||
XCTAssertEqual((error as? CUError)?.code, "missing_turn_metadata")
|
||||
}
|
||||
}
|
||||
|
||||
func testTurnGateRejectsCrossTurnStateUntilMatchingEnd() throws {
|
||||
let first = ComputerUseDaemonProtocol.Metadata(
|
||||
sessionId: "session-a",
|
||||
turnId: "turn-a"
|
||||
)
|
||||
let second = ComputerUseDaemonProtocol.Metadata(
|
||||
sessionId: "session-a",
|
||||
turnId: "turn-b"
|
||||
)
|
||||
var gate = DaemonTurnGate()
|
||||
|
||||
try gate.admit(first, command: "get_app_state")
|
||||
try gate.admit(first, command: "click")
|
||||
XCTAssertThrowsError(try gate.admit(second, command: "get_app_state")) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "turn_mismatch")
|
||||
}
|
||||
try gate.finish(first)
|
||||
try gate.admit(second, command: "get_app_state")
|
||||
XCTAssertEqual(gate.active, second)
|
||||
}
|
||||
|
||||
func testPingNegotiatesWithoutOpeningATurn() throws {
|
||||
let metadata = ComputerUseDaemonProtocol.Metadata(
|
||||
sessionId: "session-a",
|
||||
turnId: "handshake-1"
|
||||
)
|
||||
var gate = DaemonTurnGate()
|
||||
|
||||
try gate.admit(metadata, command: "ping")
|
||||
|
||||
XCTAssertNil(gate.active)
|
||||
XCTAssertEqual(
|
||||
ComputerUseDaemonProtocol.hello()["protocolVersion"]?.asString,
|
||||
ComputerUseDaemonProtocol.version
|
||||
)
|
||||
}
|
||||
|
||||
func testPermissionProbeIsConnectionScopedAndDoesNotOpenATurn() throws {
|
||||
let metadata = ComputerUseDaemonProtocol.Metadata(
|
||||
sessionId: "session-a",
|
||||
turnId: "probe"
|
||||
)
|
||||
var gate = DaemonTurnGate()
|
||||
|
||||
try gate.admit(metadata, command: "check_permissions")
|
||||
|
||||
XCTAssertNil(gate.active)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class DisplaySleepAssertionTests: XCTestCase {
|
||||
@MainActor
|
||||
func testTurnAssertionIsAcquiredAndReleasedExactlyOnce() {
|
||||
var creates = 0
|
||||
var releases: [UInt32] = []
|
||||
let assertion = ComputerUseDisplaySleepAssertion(
|
||||
create: {
|
||||
creates += 1
|
||||
return 42
|
||||
},
|
||||
release: { releases.append($0) }
|
||||
)
|
||||
|
||||
assertion.acquire()
|
||||
assertion.acquire()
|
||||
XCTAssertTrue(assertion.isHeldForTesting)
|
||||
XCTAssertEqual(creates, 1)
|
||||
|
||||
assertion.release()
|
||||
assertion.release()
|
||||
XCTAssertFalse(assertion.isHeldForTesting)
|
||||
XCTAssertEqual(releases, [42])
|
||||
|
||||
assertion.acquire()
|
||||
XCTAssertEqual(creates, 2)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testFailedPowerAssertionDoesNotPretendItIsHeld() {
|
||||
let assertion = ComputerUseDisplaySleepAssertion(create: { nil })
|
||||
|
||||
assertion.acquire()
|
||||
|
||||
XCTAssertFalse(assertion.isHeldForTesting)
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import XCTest
|
||||
final class UISettlePolicyTests: XCTestCase {
|
||||
/// Nothing has been mutated, so nothing is mid-transition on our account.
|
||||
/// This is the common case for the first `get_app_state` of a session and
|
||||
/// must not cost the model a second of latency.
|
||||
/// must not add capture latency.
|
||||
func testNoMutationMeansNoWait() {
|
||||
XCTAssertEqual(
|
||||
UISettlePolicy.delay(now: 100, lastMutationAt: nil, appIsBusy: false),
|
||||
@@ -33,12 +33,12 @@ final class UISettlePolicyTests: XCTestCase {
|
||||
)
|
||||
}
|
||||
|
||||
/// A visible progress indicator extends the window — but only to a bound,
|
||||
/// so an app that spins forever cannot hang the capture.
|
||||
func testBusyAppGetsTheLongerWindowButStaysBounded() {
|
||||
/// A permanently spinning progress indicator must not turn the one-shot
|
||||
/// settle into a multi-second stall.
|
||||
func testBusyAppUsesTheSameBoundedOneShotWindow() {
|
||||
let busy = UISettlePolicy.delay(now: 100, lastMutationAt: 98.9, appIsBusy: true)
|
||||
XCTAssertGreaterThan(busy, 0)
|
||||
XCTAssertLessThanOrEqual(busy, UISettlePolicy.busyWindow)
|
||||
XCTAssertEqual(busy, 0)
|
||||
XCTAssertEqual(UISettlePolicy.busyWindow, UISettlePolicy.postActionWindow)
|
||||
|
||||
XCTAssertEqual(
|
||||
UISettlePolicy.delay(
|
||||
@@ -67,4 +67,16 @@ final class UISettlePolicyTests: XCTestCase {
|
||||
XCTAssertLessThanOrEqual(delay, UISettlePolicy.postActionWindow)
|
||||
XCTAssertGreaterThanOrEqual(delay, 0)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testMutationMarkerIsScopedByPIDAndConsumedOnce() {
|
||||
MutationClock.resetForTests()
|
||||
defer { MutationClock.resetForTests() }
|
||||
|
||||
MutationClock.recordMutation(pid: 101, at: 42)
|
||||
|
||||
XCTAssertNil(MutationClock.takeMutation(pid: 202))
|
||||
XCTAssertEqual(MutationClock.takeMutation(pid: 101), 42)
|
||||
XCTAssertNil(MutationClock.takeMutation(pid: 101))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -132,6 +132,35 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
XCTAssertEqual(factory.sources[0].latestReadCount, 0, "A cached stream frame must not become the model's screenshot")
|
||||
}
|
||||
|
||||
func testPostMutationSnapshotConsumesFreshStreamWatermarkBeforeSkyshot() async throws {
|
||||
let target = makeTarget(windowID: 87)
|
||||
let factory = FakeWindowCaptureStreamFactory { source, _ in
|
||||
source.startFrame = makeFrame(
|
||||
for: source.targetKey,
|
||||
sequence: 1,
|
||||
uptime: 12,
|
||||
byte: 7
|
||||
)
|
||||
}
|
||||
var captures = 0
|
||||
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
|
||||
captures += 1
|
||||
return self.makeSnapshot(target, pixels: "fresh-skyshot")
|
||||
})
|
||||
|
||||
let shot = await manager.captureSnapshot(
|
||||
for: target,
|
||||
scale: 0.5,
|
||||
newerThanUptime: 11
|
||||
)
|
||||
|
||||
XCTAssertEqual(shot?.base64, "fresh-skyshot")
|
||||
XCTAssertEqual(captures, 1)
|
||||
XCTAssertEqual(factory.sources.count, 1)
|
||||
XCTAssertGreaterThan(factory.sources[0].latestReadCount, 0)
|
||||
XCTAssertEqual(factory.sources[0].retireCount, 0)
|
||||
}
|
||||
|
||||
func testSnapshotFailureDoesNotFallBackToCachedStreamPixels() async {
|
||||
let target = makeTarget(windowID: 85)
|
||||
let factory = FakeWindowCaptureStreamFactory { source, _ in
|
||||
@@ -171,9 +200,10 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
let target = makeTarget(windowID: 81)
|
||||
let factory = FakeWindowCaptureStreamFactory { _, _ in }
|
||||
var captures = 0
|
||||
var captureTimes: [TimeInterval] = []
|
||||
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
|
||||
captures += 1
|
||||
self.assertMutationHasSettledBeforeCapture()
|
||||
captureTimes.append(ProcessInfo.processInfo.systemUptime)
|
||||
return self.makeSnapshot(target, pixels: "unchanged-pixels")
|
||||
})
|
||||
var previousMutation: TimeInterval?
|
||||
@@ -183,11 +213,20 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
XCTAssertNotNil(MutationClock.lastMutation())
|
||||
XCTAssertNotEqual(MutationClock.lastMutation(), previousMutation)
|
||||
previousMutation = MutationClock.lastMutation()
|
||||
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
|
||||
let pendingMutation = MutationClock.takeMutation()
|
||||
let shot = await CommandRouter.captureSettledWindowShot(
|
||||
appIsBusy: false,
|
||||
lastMutationAt: pendingMutation
|
||||
) {
|
||||
await manager.captureSnapshot(for: target, scale: 0.5)
|
||||
}
|
||||
XCTAssertEqual(shot?.base64, "unchanged-pixels")
|
||||
XCTAssertEqual(captures, expectedCaptureCount, "Identical pixels still require a new capture")
|
||||
self.assertMutationHasSettledBeforeCapture(
|
||||
pendingMutation,
|
||||
capturedAt: captureTimes[expectedCaptureCount - 1]
|
||||
)
|
||||
XCTAssertNil(MutationClock.lastMutation(), "The settle marker is one-shot")
|
||||
}
|
||||
XCTAssertEqual(factory.sources.count, 1)
|
||||
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
|
||||
@@ -198,8 +237,9 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
defer { MutationClock.resetForTests() }
|
||||
let target = makeTarget(windowID: 83)
|
||||
let factory = FakeWindowCaptureStreamFactory { _, _ in }
|
||||
var capturedAt: TimeInterval?
|
||||
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
|
||||
self.assertMutationHasSettledBeforeCapture()
|
||||
capturedAt = ProcessInfo.processInfo.systemUptime
|
||||
return self.makeSnapshot(target, pixels: "partial-action-state")
|
||||
})
|
||||
do {
|
||||
@@ -211,18 +251,29 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
XCTAssertEqual(error.code, "partial_action")
|
||||
}
|
||||
XCTAssertNotNil(MutationClock.lastMutation())
|
||||
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
|
||||
let pendingMutation = MutationClock.takeMutation()
|
||||
let shot = await CommandRouter.captureSettledWindowShot(
|
||||
appIsBusy: false,
|
||||
lastMutationAt: pendingMutation
|
||||
) {
|
||||
await manager.captureSnapshot(for: target, scale: 0.5)
|
||||
}
|
||||
XCTAssertEqual(shot?.base64, "partial-action-state")
|
||||
self.assertMutationHasSettledBeforeCapture(
|
||||
pendingMutation,
|
||||
capturedAt: try XCTUnwrap(capturedAt)
|
||||
)
|
||||
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
|
||||
}
|
||||
|
||||
private func assertMutationHasSettledBeforeCapture() {
|
||||
XCTAssertNotNil(MutationClock.lastMutation())
|
||||
private func assertMutationHasSettledBeforeCapture(
|
||||
_ mutationAt: TimeInterval?,
|
||||
capturedAt: TimeInterval
|
||||
) {
|
||||
XCTAssertNotNil(mutationAt)
|
||||
XCTAssertEqual(UISettlePolicy.delay(
|
||||
now: ProcessInfo.processInfo.systemUptime,
|
||||
lastMutationAt: MutationClock.lastMutation(),
|
||||
now: capturedAt,
|
||||
lastMutationAt: mutationAt,
|
||||
appIsBusy: false
|
||||
), 0, "The actual screenshot callback must run after the action's settle deadline")
|
||||
}
|
||||
|
||||
@@ -57,6 +57,8 @@ BIN_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use"
|
||||
# Accessibility tolerates a bare binary (works), Screen Recording does NOT. So
|
||||
# the shipped/dragged artifact is the .app; the inner binary is what we spawn.
|
||||
APP_PATH="$BUILD_DIR/$BUILD_CONFIG/cc-haha-computer-use.app"
|
||||
# Reuse the desktop brand asset so both Privacy lists show the product logo.
|
||||
APP_ICON_PATH="$PKG_DIR/../../desktop/src-tauri/icons/icon.icns"
|
||||
# Records the (identity, identifier) actually used, so we can detect rotation
|
||||
# across rebuilds and warn that TCC grants will have been dropped.
|
||||
SIGN_STAMP="$BUILD_DIR/.cu-helper.signid"
|
||||
@@ -407,6 +409,7 @@ verify() {
|
||||
# makes the inner binary's TCC identity a proper app bundle.
|
||||
# ---------------------------------------------------------------------------
|
||||
wrap_app() {
|
||||
[ -s "$APP_ICON_PATH" ] || die "App icon not found at $APP_ICON_PATH (needed for the Privacy lists)."
|
||||
log ""
|
||||
log "==> wrap .app bundle: $APP_PATH"
|
||||
rm -rf "$APP_PATH"
|
||||
@@ -416,6 +419,7 @@ wrap_app() {
|
||||
|
||||
[ -f "$PKG_DIR/Info.plist" ] || die "Info.plist not found at $PKG_DIR/Info.plist (needed for the .app bundle)."
|
||||
cp "$PKG_DIR/Info.plist" "$APP_PATH/Contents/Info.plist"
|
||||
cp "$APP_ICON_PATH" "$APP_PATH/Contents/Resources/icon.icns"
|
||||
|
||||
# SwiftPM resource bundle (LensSequence overlay), loaded via Bundle.module.
|
||||
# Standard .app location is Contents/Resources/ (Bundle.main.resourceURL). Do
|
||||
|
||||
@@ -1,7 +1,59 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { afterEach, describe, expect, test } from 'bun:test'
|
||||
import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
const buildScript = path.resolve(import.meta.dirname, 'build.sh')
|
||||
const productIcon = path.resolve(import.meta.dirname, '../../desktop/src-tauri/icons/icon.icns')
|
||||
const fixtureDirectories: string[] = []
|
||||
|
||||
afterEach(() => {
|
||||
for (const directory of fixtureDirectories.splice(0)) {
|
||||
rmSync(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
function wrapFixtureApp(missingIcon = false) {
|
||||
const directory = mkdtempSync(path.join(tmpdir(), 'cu-helper-app-icon-'))
|
||||
fixtureDirectories.push(directory)
|
||||
writeFileSync(path.join(directory, 'fixture-binary'), 'fixture executable')
|
||||
|
||||
const result = Bun.spawnSync([
|
||||
'bash',
|
||||
'-c',
|
||||
`
|
||||
source "$1"
|
||||
TEST_BUNDLE_DIR="$2"
|
||||
BUILD_DIR="$TEST_BUNDLE_DIR/build"
|
||||
BIN_PATH="$TEST_BUNDLE_DIR/fixture-binary"
|
||||
APP_PATH="$TEST_BUNDLE_DIR/cc-haha-computer-use.app"
|
||||
BUNDLE_ID="dev.cchaha.cu-helper"
|
||||
SIGN_IDENTITY="fixture-only"
|
||||
RESOLVED_TIMESTAMP_MODE="none"
|
||||
if [ "$3" = "missing" ]; then
|
||||
APP_ICON_PATH="$TEST_BUNDLE_DIR/missing.icns"
|
||||
fi
|
||||
codesign() {
|
||||
case "$1" in
|
||||
--force) cp -R "$APP_PATH/Contents" "$TEST_BUNDLE_DIR/contents-at-sign" ;;
|
||||
-dv) printf 'Identifier=%s\\n' "$BUNDLE_ID" ;;
|
||||
esac
|
||||
}
|
||||
wrap_app
|
||||
`,
|
||||
'cu-helper-app-icon-test',
|
||||
buildScript,
|
||||
directory,
|
||||
missingIcon ? 'missing' : 'present',
|
||||
], { cwd: directory })
|
||||
|
||||
return {
|
||||
directory,
|
||||
contents: path.join(directory, 'cc-haha-computer-use.app', 'Contents'),
|
||||
exitCode: result.exitCode,
|
||||
stderr: result.stderr.toString(),
|
||||
}
|
||||
}
|
||||
|
||||
function resolveTimestampArgument(identity: string, mode = 'auto') {
|
||||
const result = Bun.spawnSync([
|
||||
@@ -74,3 +126,35 @@ describe('cu-helper build signing identity', () => {
|
||||
expect(result.stdout).toBe('Developer ID Application: Example Corp (TEAM123456)')
|
||||
})
|
||||
})
|
||||
|
||||
describe.skipIf(process.platform !== 'darwin')('cu-helper permission-list app icon', () => {
|
||||
test('declares and bundles the product icon before signing the helper app', () => {
|
||||
const result = wrapFixtureApp()
|
||||
expect(result.exitCode).toBe(0)
|
||||
|
||||
const plist = Bun.spawnSync([
|
||||
'/usr/bin/plutil', '-convert', 'json', '-o', '-',
|
||||
path.join(result.contents, 'Info.plist'),
|
||||
])
|
||||
expect(plist.exitCode).toBe(0)
|
||||
const info = JSON.parse(plist.stdout.toString())
|
||||
expect(info.CFBundleIdentifier).toBe('dev.cchaha.cu-helper')
|
||||
expect(info.CFBundleExecutable).toBe('cc-haha-computer-use')
|
||||
expect(info.CFBundleIconFile).toBe('icon.icns')
|
||||
|
||||
const expectedIcon = readFileSync(productIcon)
|
||||
expect(expectedIcon.subarray(0, 4).toString()).toBe('icns')
|
||||
expect(readFileSync(path.join(result.contents, 'Resources', info.CFBundleIconFile)))
|
||||
.toEqual(expectedIcon)
|
||||
expect(readFileSync(path.join(result.directory, 'contents-at-sign', 'Resources', info.CFBundleIconFile)))
|
||||
.toEqual(expectedIcon)
|
||||
})
|
||||
|
||||
test('refuses to sign an app when the required product icon is missing', () => {
|
||||
const result = wrapFixtureApp(true)
|
||||
|
||||
expect(result.exitCode).not.toBe(0)
|
||||
expect(result.stderr).toContain('App icon not found')
|
||||
expect(existsSync(path.join(result.directory, 'contents-at-sign'))).toBe(false)
|
||||
})
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user