fix(computer-use): preserve background input across focus changes

Track real and synthetic focus with process-bound monitor receipts.
Bind keyboard activation to the snapshot window and align targeted
pointer and keyboard bursts with the reference event sequence.
Wait for pasteboard data delivery before restoring the clipboard,
and add focus/stream diagnostics with transition regression coverage.

Validated with check:native and six covered NetEase focus cycles.
This commit is contained in:
程序员阿江(Relakkes)
2026-08-31 23:03:54 +08:00
parent 217867fff1
commit 8751e558e5
26 changed files with 4995 additions and 928 deletions
+205 -184
View File
@@ -7,9 +7,9 @@
// (or, for click/scroll/drag, a window-relative point), preferring real
// Accessibility actions over synthesized input. Only when no usable AX action
// exists does it fall back to a synthetic pointer/keyboard event — and every such
// event is posted with `CGEvent.postToPid(pid)` + a `.combinedSessionState`
// source, NEVER `.cghidEventTap`/`.hidSystemState`, so the user's real pointer is
// never hijacked.
// event is posted with `CGEvent.postToPid(pid)`, never `.cghidEventTap`, so the
// user's real pointer is never hijacked. A keyboard event's `.hidSystemState`
// source describes its input state; it does not change this PID-only routing.
//
// Why a gradient, not a single AXPress (the v1 mistake):
// • Finder sidebars / Activity Monitor / System Settings rows are "clicked" by
@@ -236,6 +236,22 @@ public final class ClipboardLease {
temporaryChangeCount = pasteboard.changeCount
}
func writeTemporaryStringWithReceipt(_ text: String) throws -> ClipboardPasteReceipt {
if let captureError { throw captureError }
let receipt = ClipboardPasteReceipt(text: text)
let item = NSPasteboardItem()
guard item.setDataProvider(receipt, forTypes: [.string]) else {
throw CUError("clipboard_write_failed", "Could not register temporary pasteboard data")
}
pasteboard.clearContents()
temporaryChangeCount = pasteboard.changeCount
guard pasteboard.writeObjects([item]) else {
throw CUError("clipboard_write_failed", "Could not write temporary pasteboard data")
}
temporaryChangeCount = pasteboard.changeCount
return receipt
}
/// Whether the temporary text is still the latest pasteboard write. Check
/// this immediately before sending Command-V so a concurrent user copy is
/// never pasted into the agent's target application.
@@ -308,12 +324,6 @@ public enum AXAction {
}
}
private struct KeyBurstSpec {
let keyCode: CGKeyCode
let keyDown: Bool
let flags: CGEventFlags
}
// MARK: - Tunables
/// Synthetic event source — combined session state so events inherit the real
@@ -359,7 +369,7 @@ public enum AXAction {
index: Int,
clickCount: Int = 1,
button: MouseButton = .left
) throws -> String {
) async throws -> String {
let element = try resolveElement(pid: pid, index: index)
let record = AXTree.record(pid: pid, index: index)
let reps = max(1, clickCount)
@@ -371,7 +381,7 @@ public enum AXAction {
}
// Fall through to a synthetic right-click at the element center.
if let p = centerGlobal(record) {
try clickPoint(pid: pid, x: p.x, y: p.y, clickCount: reps, button: .right)
try await clickPoint(pid: pid, x: p.x, y: p.y, clickCount: reps, button: .right)
return "synthetic:point"
}
throw CUError("no_action", "Element \(index) exposes no right-click (AXShowMenu) and has no frame to click")
@@ -381,7 +391,7 @@ public enum AXAction {
// Middle/back/forward have no AX analogue; go straight to a
// synthetic point click using their exact CoreGraphics button id.
if let p = centerGlobal(record) {
try clickPoint(pid: pid, x: p.x, y: p.y, clickCount: reps, button: button)
try await clickPoint(pid: pid, x: p.x, y: p.y, clickCount: reps, button: button)
return "synthetic:point"
}
throw CUError("no_action", "Element \(index) has no frame for a \(button.rawValue) click")
@@ -428,7 +438,7 @@ public enum AXAction {
// ── ⑥ Last resort: synthetic pointer click via postToPid. ─────────────
if let center = centerGlobal(record) {
try clickPoint(pid: pid, x: center.x, y: center.y, clickCount: reps, button: button)
try await clickPoint(pid: pid, x: center.x, y: center.y, clickCount: reps, button: button)
return "synthetic:point"
}
@@ -438,46 +448,15 @@ public enum AXAction {
)
}
/// Let the target's run loop process a synthetic focus notification. Short
/// because nothing comes to the foreground — but not zero, since the target
/// has to actually dequeue the event before the burst arrives.
private static let syntheticFocusSettleSeconds: TimeInterval = 0.12
/// Put the target in a state where it will act on synthesized input, and
/// wait long enough for that to be true.
///
/// This posts a CPS focus notification and nothing else. The target is NOT
/// brought to the foreground: driving an app while the user works in a
/// different one is the entire feature, and a click that costs them their
/// foreground has not delivered it.
///
/// WHY THE FOREGROUND GRANT THAT USED TO LIVE HERE IS GONE
/// -------------------------------------------------------
/// It was added after the notification alone appeared to fail: in one
/// session 24 mutating actions produced 1 effect, and in another nine
/// window-bound clicks were discarded while the target's traffic lights
/// stayed fully coloured.
///
/// That second session is what voids the conclusion. The app was active and
/// its window was key — precisely the state a foreground grant exists to
/// produce — and the clicks were dropped anyway. Focus was not the variable.
///
/// What was actually broken has since been fixed. Every click those sessions
/// sent carried a leading move claiming `clickState 1`, and a press and a
/// release stamped with two different event numbers, so AppKit had no reason
/// to read the pair as one click (see `MouseClickStateTests`). Single clicks
/// landed as hover; double clicks worked, because the second pair got
/// through. A foreground grant plus an 800ms settle made a malformed click
/// likelier to survive, which is why it read as the cure.
///
/// If background actuation does regress, `WindowKeyFocus.grantIfNeeded` is
/// still there to be called from here and from `Injection.focusForClick` —
/// but measure it with a SINGLE click on a control that needs a complete
/// one. A text field focuses on the press alone and cannot tell the two
/// implementations apart.
private static func ensureTargetAcceptsInput(pid: pid_t) {
SyntheticWindowFocus.enforceActiveState(pid: pid)
Thread.sleep(forTimeInterval: syntheticFocusSettleSeconds)
/// Shared preparation for every synthetic action. The actor is yielded
/// while focus is established, so lifecycle notifications are not delayed.
@MainActor
@discardableResult
private static func ensureTargetAcceptsInput(
pid: pid_t, window: WindowGeometry.Window? = nil,
beforeFocus: (@MainActor (FocusEventMonitor.RegistrationReceipt) async throws -> Void)? = nil
) async throws -> FocusEventMonitor.RegistrationReceipt {
try await SyntheticWindowFocus.prepareInput(pid: pid, window: window, beforeFocus: beforeFocus)
}
/// Synthetic pointer click at a GLOBAL (Quartz, top-left) point, posted to the
@@ -490,9 +469,8 @@ public enum AXAction {
y: Double,
clickCount: Int = 1,
button: MouseButton = .left
) throws {
) async throws {
let point = CGPoint(x: x, y: y)
let reps = max(1, clickCount)
guard let src = eventSource else {
throw CUError(CUError.Code.eventAlloc, "Failed to allocate a combined-session event source for a synthetic click")
}
@@ -501,20 +479,40 @@ public enum AXAction {
// named. Every event in the burst is bound to this same window, so the
// burst can no longer half-succeed against a window that moved.
let window = try requireBindableWindow(at: point, pid: pid)
let target = try Injection.authorizeResolvedTarget(pid: pid)
ensureTargetAcceptsInput(pid: pid)
let focusReceipt = try await ensureTargetAcceptsInput(pid: pid, window: window, beforeFocus: { receipt in
try await movePointerBeforeFocus(
at: point, window: window,
validate: {
_ = try Injection.validateAuthorizedTarget(target)
try SyntheticWindowFocus.validate(receipt)
guard WindowGeometry.window(id: window.id, pid: pid) == window else {
throw CUError("stale_window", "The pointer target window moved or closed. Read its current state before retrying.")
}
},
post: { WindowTargetedEvent.post($0, to: pid) }
)
})
let events = try clickEvents(
at: point, clickCount: clickCount, button: button,
source: src, pid: pid, window: window
)
try await postMouseBurst(
events, target: target, window: window, button: button,
focusReceipt: focusReceipt, pause: nil
)
}
// Allocate the complete move + click sequence before posting its first
// event, so allocation failure can never strand a down stroke.
// The move carries clickState 0: it delivers the pointer so hover-only
// affordances appear, but it is not part of the click that follows.
var specs = [MouseBurstSpec(
type: .mouseMoved,
point: point,
button: button,
clickState: mouseClickState(for: .mouseMoved, click: 1),
eventNumber: WindowTargetedEvent.nextEventNumber()
)]
static func clickEvents(
at point: CGPoint, clickCount: Int, button: MouseButton,
source: CGEventSource, pid: pid_t, window: WindowGeometry.Window
) throws -> [CGEvent] {
let reps = max(1, clickCount)
// Ordinary clicks are only down/up pairs. Pointer movement is a
// separate action; inserting it here changes the reference protocol.
// Allocate every pair before the first post so failure cannot strand a down.
var specs: [MouseBurstSpec] = []
for i in 1...reps {
// One number per press/release pair: that pairing is what makes the
// two events read as a single click rather than two loose halves.
@@ -534,15 +532,43 @@ public enum AXAction {
eventNumber: clickNumber
))
}
let events: [CGEvent] = try EventBurst.allocateAll(
return try EventBurst.allocateAll(
specs: specs
) { spec in
makeMouse(spec, source: src, targetPid: pid, window: window)
makeMouse(spec, source: source, targetPid: pid, window: window)
}
for event in events {
WindowTargetedEvent.post(event, to: pid)
Thread.sleep(forTimeInterval: 0.03)
}
/// The reference controller sends this hover before preparing app focus;
/// moving the visual cursor alone does not notify the target application.
static func movePointerBeforeFocus(
at point: CGPoint, window: WindowGeometry.Window,
validate: @MainActor () throws -> Void, post: @MainActor (CGEvent) -> Void,
pause: @MainActor (Duration) async throws -> Void = { try await Task.sleep(for: $0) },
makeEvent: @MainActor (CGPoint, WindowGeometry.Window) -> CGEvent? = pointerMoveEvent
) async throws {
try Task.checkCancellation()
try validate()
guard let event = makeEvent(point, window) else {
throw CUError(CUError.Code.eventAlloc, "Failed to allocate a window-targeted pointer move")
}
try Task.checkCancellation()
try validate()
try Task.checkCancellation()
post(event)
try await pause(.milliseconds(10))
try Task.checkCancellation()
try validate()
}
static func pointerMoveEvent(at point: CGPoint, window: WindowGeometry.Window) -> CGEvent? {
// This standalone hover has clickCount 1 in the reference protocol;
// the separate drag-movement rule remains unchanged.
WindowTargetedEvent.makeMouseEvent(
type: .mouseMoved, nsType: .mouseMoved, point: point, button: .left,
clickCount: 1, windowID: window.id, windowBounds: window.bounds,
eventNumber: WindowTargetedEvent.nextEventNumber()
)
}
/// The window a coordinate action will name, or a refusal explaining why no
@@ -611,7 +637,7 @@ public enum AXAction {
y: Double,
clickCount: Int = 1,
button: MouseButton = .left
) throws -> String {
) async throws -> String {
let point = CGPoint(x: x, y: y)
let reps = max(1, clickCount)
@@ -631,7 +657,7 @@ public enum AXAction {
}
// No AX element answered (or a non-left button): synthetic pointer click.
try clickPoint(pid: pid, x: x, y: y, clickCount: reps, button: button)
try await clickPoint(pid: pid, x: x, y: y, clickCount: reps, button: button)
return "synthetic:point"
}
@@ -755,7 +781,7 @@ public enum AXAction {
y: Double? = nil,
direction: String,
pages: Double = 1
) throws {
) async throws {
let dir = direction.lowercased()
guard ["up", "down", "left", "right"].contains(dir) else {
throw CUError("bad_payload", "Invalid scroll direction: \(direction)")
@@ -782,7 +808,7 @@ public enum AXAction {
if actionNames(element).contains(pageAction) {
for _ in 0..<wholePages {
_ = AXUIElementPerformAction(element, pageAction as CFString)
Thread.sleep(forTimeInterval: 0.05)
try await Task.sleep(for: .milliseconds(50))
}
settle()
return
@@ -793,6 +819,14 @@ public enum AXAction {
guard let point = center else {
throw CUError("no_target", "scroll: element \(index.map(String.init) ?? "?") has no frame and no x/y point was given")
}
let window = try requireBindableWindow(at: point, pid: pid)
let target = try Injection.authorizeResolvedTarget(pid: pid)
let focusReceipt = try await ensureTargetAcceptsInput(pid: pid, window: window)
_ = try Injection.validateAuthorizedTarget(target)
try SyntheticWindowFocus.validate(focusReceipt)
guard WindowGeometry.window(id: window.id, pid: pid) == window else {
throw CUError("stale_window", "The scroll target moved or closed. Read its current state before retrying.")
}
try scrollWheel(at: point, direction: dir, pages: pages, pid: pid)
settle()
}
@@ -803,16 +837,17 @@ public enum AXAction {
/// UTF-16 kAXSelectedTextRange (or inserts at the UTF-16 end when unavailable),
/// then places the caret immediately after the inserted text. If AX editing is
/// unavailable, a PID-directed Unicode event path precedes clipboard paste.
public static func typeText(pid: pid_t, _ text: String) throws {
public static func typeText(pid: pid_t, _ text: String) async throws {
guard !text.isEmpty else { return }
let target = try Injection.authorizeResolvedTarget(pid: pid)
let window = try keyboardWindow(pid: pid)
// Typing needs the same acceptance a click does, and cannot inherit it.
// Each MCP call is a separate request seconds apart: the click that
// focused the field made the app active at the time, and by the time
// `type_text` arrives the user has usually clicked away. That gap is
// where nine consecutive type_text calls went nowhere while every one
// of them returned "Action completed".
ensureTargetAcceptsInput(pid: pid)
// Validate the lifetime without resetting a still-focused field. An
// observed loss requires preparation again before keyboard input.
let focusReceipt = try await ensureTargetAcceptsInput(pid: pid, window: window)
_ = try Injection.validateAuthorizedTarget(target)
try SyntheticWindowFocus.validate(focusReceipt)
try validateKeyboardWindow(window, pid: pid)
let focused = focusedElement(of: pid)
@@ -871,6 +906,7 @@ public enum AXAction {
// ── Fallback 1: Unicode keyboard, only into an actual text field/area
// (reliable for native AppKit fields). ───────────────────────────────
if let focused, isTextEntry(focused) {
try SyntheticWindowFocus.validate(focusReceipt)
do {
try postUnicodeText(pid: pid, text: text)
return
@@ -887,105 +923,64 @@ public enum AXAction {
// postToPid without the app being frontmost. Save + restore the user's
// clipboard so we don't clobber it. (This replaces the old hard
// `no_focus` throw that stranded every CEF text-input task.)
try typeViaClipboard(pid: pid, text)
try await typeViaClipboard(target: target, text)
}
/// Paste `text` into whatever holds in-app keyboard focus in `pid`, via the
/// system clipboard + ⌘V (Codex's approach for CEF/Chromium text fields).
/// Best-effort: if nothing is focused the paste simply goes nowhere and the
/// next get_app_state screenshot shows the model it didn't take.
private static func typeViaClipboard(pid: pid_t, _ text: String) throws {
let lease = ClipboardLease()
defer { lease.restoreIfUnchanged() }
try lease.writeTemporaryString(text)
Thread.sleep(forTimeInterval: 0.04) // let the pasteboard commit before ⌘V
guard lease.temporaryWriteIsCurrent() else {
throw CUError(
"clipboard_changed",
"The user copied new clipboard content before paste; no paste was sent"
)
/// The read receipt confirms supplied clipboard bytes, not the reader's
/// PID or the field's final value; the next screenshot still verifies UI.
private static func typeViaClipboard(target: ProvenProcessTarget, _ text: String) async throws {
let pid = target.pid
try await ClipboardPasteReceipt.perform(text: text, lease: ClipboardLease()) { validate in
try await pressKey(pid: pid, "super+v", validateBeforePosting: {
_ = try Injection.validateAuthorizedTarget(target)
try validate()
})
}
try pressKey(pid: pid, "super+v")
Thread.sleep(forTimeInterval: 0.18) // let the app consume the paste
}
// MARK: - Press key
/// Press an xdotool-style key sequence (`"super+c"`, `"Return"`, `"shift+Tab"`,
/// `"KP_0"`, `"Prior"`) against `pid`. `KeyMapping.parse` yields one or more
/// chords (keyCode + folded modifier flags); each is posted as
/// modifier-down(s) → key down → key up → modifier-up(s), all via `postToPid`
/// so the keystroke lands in the target without touching the HID tap.
public static func pressKey(pid: pid_t, _ key: String) throws {
ensureTargetAcceptsInput(pid: pid)
/// Deliver modifier transitions as flagsChanged events, including for a
/// bare Return after a shortcut. Restore the captured session baseline
/// without posting anything to the HID tap.
public static func pressKey(
pid: pid_t, _ key: String,
validateBeforePosting: () throws -> Void = {}
) async throws {
let chords = try KeyMapping.parse(key)
guard !chords.isEmpty else {
throw CUError(CUError.Code.unknownKey, "Empty key sequence: \"\(key)\"")
}
guard let src = eventSource else {
throw CUError(CUError.Code.eventAlloc, "Failed to allocate a combined-session event source for a key press")
throw CUError(CUError.Code.unknownKey, "Empty key sequence")
}
let target = try Injection.authorizeResolvedTarget(pid: pid)
let window = try keyboardWindow(pid: pid)
var focusReceipt: FocusEventMonitor.RegistrationReceipt?
try await KeyboardEventBurst.dispatch(
chords: chords,
prepare: { focusReceipt = try await ensureTargetAcceptsInput(pid: pid, window: window) },
validateBeforePosting: {
_ = try Injection.validateAuthorizedTarget(target)
try SyntheticWindowFocus.validate(focusReceipt)
try validateKeyboardWindow(window, pid: pid)
try validateBeforePosting()
},
post: { WindowTargetedEvent.post($0, to: pid) }
)
}
var specs: [KeyBurstSpec] = []
var chordEventCounts: [Int] = []
for chord in chords {
let modifiers = modifierKeyCodes(for: chord.flags)
let startCount = specs.count
private static func keyboardWindow(pid: pid_t) throws -> WindowGeometry.Window {
try SnapshotKeyboardWindow.resolve(
pid: pid,
snapshot: AXTree.snapshotEvidence(pid: pid),
currentIdentity: AXTree.currentProcessIdentity(pid: pid),
windowForID: { WindowGeometry.window(id: $0, pid: $1) }
)
}
// Modifier down(s), accumulating the flag mask.
var active: CGEventFlags = []
for mod in modifiers {
active.insert(mod.flag)
specs.append(KeyBurstSpec(
keyCode: mod.keyCode,
keyDown: true,
flags: active
))
}
// The key itself, carrying the full modifier mask.
specs.append(KeyBurstSpec(
keyCode: chord.keyCode,
keyDown: true,
flags: chord.flags
))
specs.append(KeyBurstSpec(
keyCode: chord.keyCode,
keyDown: false,
flags: chord.flags
))
// Modifier up(s) in reverse, peeling the mask back down.
for mod in modifiers.reversed() {
active.remove(mod.flag)
specs.append(KeyBurstSpec(
keyCode: mod.keyCode,
keyDown: false,
flags: active
))
}
chordEventCounts.append(specs.count - startCount)
}
let events: [CGEvent] = try EventBurst.allocateAll(
specs: specs
) { spec in
guard let event = CGEvent(
keyboardEventSource: src,
virtualKey: spec.keyCode,
keyDown: spec.keyDown
) else { return nil }
event.flags = spec.flags
return event
}
var offset = 0
for count in chordEventCounts {
for event in events[offset..<(offset + count)] {
WindowTargetedEvent.post(event, to: pid)
}
offset += count
Thread.sleep(forTimeInterval: 0.04)
private static func validateKeyboardWindow(_ window: WindowGeometry.Window, pid: pid_t) throws {
guard WindowGeometry.window(id: window.id, pid: pid) == window else {
throw CUError("stale_window", "The keyboard target window moved or closed. Read its current state before retrying.")
}
}
@@ -995,7 +990,7 @@ public enum AXAction {
/// `from`, ten interpolated dragged steps, up at `to`. Posted to the window
/// explicitly resolved target via `postToPid`. Coordinate-only by contract —
/// the model drives drags by pixel, not by element index.
public static func drag(pid: pid_t, from: CGPoint, to: CGPoint, button: MouseButton = .left) throws {
public static func drag(pid: pid_t, from: CGPoint, to: CGPoint, button: MouseButton = .left) async throws {
guard let src = eventSource else {
throw CUError(CUError.Code.eventAlloc, "Failed to allocate a combined-session event source for a drag")
}
@@ -1005,6 +1000,8 @@ public enum AXAction {
// of a list), and re-binding mid-gesture would send the tail of the
// drag to a different window.
let dragWindow = try requireBindableWindow(at: from, pid: pid)
let target = try Injection.authorizeResolvedTarget(pid: pid)
let focusReceipt = try await ensureTargetAcceptsInput(pid: pid, window: dragWindow)
// Movement carries clickState 0 (both the leading move and every
// dragged step); only the press and the release belong to the click.
@@ -1051,10 +1048,45 @@ public enum AXAction {
) { spec in
makeMouse(spec, source: src, targetPid: pid, window: dragWindow)
}
for event in events {
WindowTargetedEvent.post(event, to: pid)
Thread.sleep(forTimeInterval: 0.03)
try await postMouseBurst(
events, target: target, window: dragWindow, button: button, focusReceipt: focusReceipt
)
}
private static func postMouseBurst(
_ events: [CGEvent], target: ProvenProcessTarget,
window: WindowGeometry.Window, button: MouseButton,
focusReceipt: FocusEventMonitor.RegistrationReceipt,
pause: (@MainActor () async throws -> Void)? = {
try await Task.sleep(for: .milliseconds(30))
}
) async throws {
try await MouseEventBurstDelivery.deliver(
events: events,
validate: {
_ = try Injection.validateAuthorizedTarget(target)
try SyntheticWindowFocus.validate(focusReceipt)
guard WindowGeometry.window(id: window.id, pid: target.pid) == window else {
throw CUError("stale_window", "The target window moved or closed. Read its current state before retrying.")
}
},
post: { WindowTargetedEvent.post($0, to: target.pid) },
release: { down, point in
// A canceled gesture still needs its up, but never at a new
// process that happened to reuse the original PID.
_ = try Injection.validateAuthorizedTarget(target)
guard let liveWindow = WindowGeometry.window(id: window.id, pid: target.pid),
let nsType = Injection.nsEventType(for: button.up),
let up = WindowTargetedEvent.makeMouseEvent(
type: button.up, nsType: nsType, point: point, button: button,
clickCount: Int(down.getIntegerValueField(.mouseEventClickState)),
windowID: liveWindow.id, windowBounds: liveWindow.bounds,
eventNumber: Int(down.getIntegerValueField(.mouseEventNumber))
) else { return }
WindowTargetedEvent.post(up, to: target.pid)
},
pause: pause
)
}
// ════════════════════════════════════════════════════════════════════════
@@ -1361,17 +1393,6 @@ public enum AXAction {
}
}
/// Decompose a folded `CGEventFlags` mask into the discrete modifier keys we
/// must press/release around the main key, each with its own keyCode + flag.
private static func modifierKeyCodes(for flags: CGEventFlags) -> [(keyCode: CGKeyCode, flag: CGEventFlags)] {
var mods: [(CGKeyCode, CGEventFlags)] = []
if flags.contains(.maskCommand) { mods.append((CGKeyCode(0x37), .maskCommand)) } // kVK_Command
if flags.contains(.maskShift) { mods.append((CGKeyCode(0x38), .maskShift)) } // kVK_Shift
if flags.contains(.maskAlternate) { mods.append((CGKeyCode(0x3A), .maskAlternate)) } // kVK_Option
if flags.contains(.maskControl) { mods.append((CGKeyCode(0x3B), .maskControl)) } // kVK_Control
return mods.map { (keyCode: $0.0, flag: $0.1) }
}
// MARK: Mouse synthesis
/// Allocate one mouse event without posting it. Callers allocate their full
@@ -192,7 +192,7 @@ public enum AXTree {
// MARK: - Public entry (contract)
private static func processIdentity(
nonisolated private static func processIdentity(
for running: NSRunningApplication?
) -> AXTreeProcessIdentity {
AXTreeProcessIdentity(
@@ -202,7 +202,7 @@ public enum AXTree {
)
}
static func currentProcessIdentity(pid: pid_t) -> AXTreeProcessIdentity? {
nonisolated static func currentProcessIdentity(pid: pid_t) -> AXTreeProcessIdentity? {
guard let running = NSRunningApplication(processIdentifier: pid) else {
return nil
}
@@ -526,6 +526,18 @@ public enum AXTree {
)
}
/// Reuse the snapshot's proven window identity and the ordinary fresh AX
/// refetch path, rather than independently guessing a window from its frame.
static func snapshotWindowElement(pid: pid_t, windowID: CGWindowID) throws -> AXUIElement {
let roots = sessions[pid]?.locators.filter {
$0.value.root.windowID == windowID && $0.value.path?.isEmpty == true
} ?? [:]
guard windowID != kCGNullWindowID, roots.count == 1, let index = roots.keys.first else {
throw CUError("stale_window", "No proven snapshot window; call get_app_state before acting")
}
return try refetch(pid: pid, index: index)
}
/// Resolve the live AX key-window to its current Window Server identity.
/// This performs a fresh lookup and never trusts the snapshot-time window
/// array position, allowing coordinate actions to reject a window switch.
@@ -33,7 +33,7 @@ enum HelperClientPolicy {
"drag", "press_key", "type_text",
// Private lifecycle / visible-overlay / permission and input diagnostics.
"ping", "shutdown", "overlay_show", "overlay_hide",
"check_permissions", "input_monitor_state", "held_input_state",
"check_permissions", "input_monitor_state", "focus_monitor_state", "held_input_state",
"last_injection_state",
]
@@ -0,0 +1,144 @@
import AppKit
import Foundation
import os
/// Confirms that this pasteboard item's promised bytes were requested and
/// supplied. AppKit does not identify the reader: this is not proof that the
/// intended application's focused field accepted the text.
final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unchecked Sendable {
struct Diagnostic: Sendable {
let status: String
let pastePosted: Bool
let dataRequested: Bool
let dataSupplied: Bool
let providerFinished: Bool
let readElapsedMilliseconds: Double?
let elapsedMilliseconds: Double
let ownedBeforeRestore: Bool
let restored: Bool
}
private struct State {
var requested = false
var suppliedAt: ContinuousClock.Instant?
var finished = false
}
@MainActor private(set) static var lastDiagnostic: Diagnostic?
private let data: Data
private let state = OSAllocatedUnfairLock(initialState: State())
init(text: String) {
data = Data(text.utf8)
super.init()
}
func pasteboard(_ pasteboard: NSPasteboard?, item: NSPasteboardItem, provideDataForType type: NSPasteboard.PasteboardType) {
guard type == .string else { return }
state.withLock { $0.requested = true }
guard item.setData(data, forType: type) else { return }
state.withLock { value in
if value.suppliedAt == nil { value.suppliedAt = .now }
}
}
func pasteboardFinishedWithDataProvider(_ pasteboard: NSPasteboard) {
// Ownership loss also invokes this callback. It is never a read ack.
state.withLock { $0.finished = true }
}
@MainActor
static func perform(
text: String,
lease: ClipboardLease,
timeout: Duration = .seconds(2),
sendPaste: @MainActor (_ validateBeforePosting: @MainActor () throws -> Void) async throws -> Void
) async throws {
lastDiagnostic = nil
let started = ContinuousClock.now
var receipt: ClipboardPasteReceipt?
var posted = false
var status = "failed"
defer {
let owned = lease.temporaryWriteIsCurrent()
let observed = receipt?.state.withLock { $0 }
let restored = lease.restoreIfUnchanged()
lastDiagnostic = Diagnostic(
status: status,
pastePosted: posted,
dataRequested: observed?.requested ?? false,
dataSupplied: observed?.suppliedAt != nil,
providerFinished: observed?.finished ?? false,
readElapsedMilliseconds: observed?.suppliedAt.map {
milliseconds(started.duration(to: $0))
},
elapsedMilliseconds: milliseconds(started.duration(to: .now)),
ownedBeforeRestore: owned,
restored: restored
)
}
do {
try Task.checkCancellation()
let written = try lease.writeTemporaryStringWithReceipt(text)
receipt = written
try await Task.sleep(for: .milliseconds(40))
let validate: @MainActor () throws -> Void = {
guard lease.temporaryWriteIsCurrent() else {
throw CUError("clipboard_changed", "The clipboard changed before paste; no further paste was sent")
}
}
try validate()
try await sendPaste(validate)
posted = true
try await written.waitForRead(timeout: timeout, ownsClipboard: lease.temporaryWriteIsCurrent)
// CEF often exposes no AX text/selection evidence. As in the
// reference's no-AX branch, allow a short processing window after
// actual data delivery; this is not a claim of field acceptance.
await pause(for: .milliseconds(100))
guard lease.temporaryWriteIsCurrent() else {
throw CUError("clipboard_changed", "The clipboard changed after paste data was supplied")
}
try Task.checkCancellation()
status = "completed"
} catch {
status = error is CancellationError ? "cancelled" : (error as? CUError)?.code ?? "failed"
throw error
}
}
/// Once Command-V was sent, cancellation must not restore the previous
/// clipboard while the target can still be reading this one. Finish the
/// bounded read/settle window, then surface cancellation to the caller.
@MainActor
func waitForRead(timeout: Duration, ownsClipboard: @MainActor () -> Bool) async throws {
let deadline = ContinuousClock.now.advanced(by: timeout)
while true {
guard ownsClipboard() else {
throw CUError("clipboard_changed", "The clipboard changed while waiting for paste consumption")
}
if state.withLock({ $0.suppliedAt != nil }) { return }
let remaining = ContinuousClock.now.duration(to: deadline)
guard remaining > .zero else {
try Task.checkCancellation()
throw CUError("clipboard_read_timeout", "No pasteboard data read was observed within the paste deadline; inspect the target before retrying")
}
await Self.pause(for: min(.milliseconds(10), remaining))
}
}
@MainActor
private static func pause(for duration: Duration) async {
let seconds = milliseconds(duration) / 1_000
guard seconds > 0 else { return }
await withCheckedContinuation { (continuation: CheckedContinuation<Void, Never>) in
DispatchQueue.main.asyncAfter(deadline: .now() + seconds) {
continuation.resume()
}
}
}
private static func milliseconds(_ duration: Duration) -> Double {
let parts = duration.components
return Double(parts.seconds) * 1_000 + Double(parts.attoseconds) / 1e15
}
}
@@ -288,6 +288,75 @@ public final class CommandRouter {
),
])
// No key contents are collected: this exposes the focus protocol and
// continuity evidence needed to distinguish a dispatched input from
// an application that can actually receive it.
case "focus_monitor_state":
let monitor = FocusEventMonitor.shared
let diagnostic = monitor.diagnostic
var fields: [String: JSONValue] = [
"available": .bool(diagnostic.available),
"reason": .string(diagnostic.reason),
"continuityGeneration": .string(String(diagnostic.continuityGeneration)),
"frontmostPID": .int(Int(NSWorkspace.shared.frontmostApplication?.processIdentifier ?? 0)),
"targets": .array(SyntheticWindowFocus.beliefs.sorted { $0.key < $1.key }.map { pid, belief in
.object([
"pid": .int(Int(pid)),
"observedActive": .bool(belief.applicationIsActive),
"believesActive": .bool(belief.applicationBelievesItIsActive),
"believesFocused": .bool(belief.applicationBelievesItHasFocus),
"generation": .string(String(belief.generation)),
])
}),
]
if let event = diagnostic.lastEvent {
fields["lastEvent"] = .object([
"type": .int(Int(event.type)),
"subtype": .string(String(event.subtype)),
"sourcePID": .int(Int(event.sourcePID)),
"targetPID": .int(Int(event.targetPID)),
"focusPID": .int(Int(event.focusPID)),
"focusToken": .string(String(event.focusToken)),
])
}
if let prepared = SyntheticWindowFocus.lastPreparedWindow {
fields["lastPreparation"] = .object([
"pid": .int(Int(prepared.pid)),
"windowID": .int(Int(prepared.window?.id ?? 0)),
"activationPointAvailable": .bool(prepared.window.map { $0.activationPoint != nil } ?? false),
"activationPoint": prepared.window?.resolvedActivationPoint.map {
.object(["x": .double($0.x), "y": .double($0.y)])
} ?? .null,
])
}
if let capture = windowCaptureProvider as? WindowCaptureStreamManager {
let stream = capture.diagnostic()
var streamFields: [String: JSONValue] = ["generation": .string(String(stream.generation))]
streamFields["pid"] = stream.activeKey.map { JSONValue.int(Int($0.pid)) } ?? .null
streamFields["windowID"] = stream.activeKey.map { JSONValue.int(Int($0.windowID)) } ?? .null
streamFields["hasFailed"] = stream.hasFailed.map(JSONValue.bool) ?? .null
streamFields["latestFrameSequence"] = stream.latestFrameSequence.map { JSONValue.string(String($0)) } ?? .null
streamFields["latestFrameAgeSeconds"] = stream.latestFrameAgeSeconds.map(JSONValue.double) ?? .null
streamFields["sampleCount"] = stream.sampleCount.map { JSONValue.string(String($0)) } ?? .null
streamFields["latestSampleStatus"] = stream.latestSampleStatus.map { JSONValue.int(Int($0)) } ?? .null
streamFields["latestSampleAgeSeconds"] = stream.latestSampleAgeSeconds.map(JSONValue.double) ?? .null
fields["windowStream"] = .object(streamFields)
}
if let paste = ClipboardPasteReceipt.lastDiagnostic {
fields["lastPaste"] = .object([
"status": .string(paste.status),
"pastePosted": .bool(paste.pastePosted),
"dataRequested": .bool(paste.dataRequested),
"dataSupplied": .bool(paste.dataSupplied),
"providerFinished": .bool(paste.providerFinished),
"readElapsedMilliseconds": paste.readElapsedMilliseconds.map(JSONValue.double) ?? .null,
"elapsedMilliseconds": .double(paste.elapsedMilliseconds),
"ownedBeforeRestore": .bool(paste.ownedBeforeRestore),
"restored": .bool(paste.restored),
])
}
return .object(fields)
// Internal smoke/diagnostic command. Not advertised through MCP.
// Answers "did the last click actually get bound to a window?" — the
// window-bound path degrades silently to the old broken behaviour, so
@@ -1109,7 +1178,7 @@ public final class CommandRouter {
// Coordinate clicks PREFER AX (hit-test the element under the point and
// press it) so Chromium/CEF apps — whose tree we can't traverse — still
// click; the synthetic postToPid click is the fallback.
let tag = try AXAction.clickAtPoint(
let tag = try await AXAction.clickAtPoint(
pid: target.pid,
x: g.x,
y: g.y,
@@ -1167,7 +1236,7 @@ public final class CommandRouter {
try guardStaleness(pid: target.pid, handle: handle)
},
mutate: {
try AXAction.click(
try await AXAction.click(
pid: target.pid,
index: index,
clickCount: clickCount,
@@ -1335,7 +1404,7 @@ public final class CommandRouter {
x = point.x
y = point.y
}
try AXAction.scroll(
try await AXAction.scroll(
pid: target.pid,
index: index,
x: x,
@@ -1364,7 +1433,7 @@ public final class CommandRouter {
) {
_ = try Injection.validateAuthorizedTarget(target)
try self.requireSnapshotProcess(target: target, expected: expected)
try AXAction.typeText(pid: target.pid, text)
try await AXAction.typeText(pid: target.pid, text)
return .bool(true)
}
}
@@ -1393,7 +1462,7 @@ public final class CommandRouter {
) {
_ = try Injection.validateAuthorizedTarget(target)
try self.requireSnapshotProcess(target: target, expected: expected)
try AXAction.pressKey(pid: target.pid, key)
try await AXAction.pressKey(pid: target.pid, key)
return .bool(true)
}
}
@@ -1451,7 +1520,7 @@ public final class CommandRouter {
pid: target.pid
)
_ = try Injection.validateAuthorizedTarget(target)
try AXAction.drag(
try await AXAction.drag(
pid: target.pid,
from: from,
to: to,
@@ -82,10 +82,10 @@ enum CursorIndexedActionGate {
static func perform<Result>(
moveForAction: () async -> Void,
recheckStaleness: () throws -> Void,
mutate: () throws -> Result
mutate: () async throws -> Result
) async rethrows -> Result {
await moveForAction()
try recheckStaleness()
return try mutate()
return try await mutate()
}
}
@@ -0,0 +1,780 @@
import AppKit
import Carbon
import CoreGraphics
import Darwin
import Foundation
/// Focus protection for regular/accessory processes whose PID is the owner.
/// The reference normalizes activationPolicy.prohibited / ViewBridge-owned
/// processes through AX metadata. That branch is not implemented here, so
/// prohibited targets are rejected and unknown notification shapes pass through.
final class FocusEventMonitor: @unchecked Sendable {
typealias FocusChanged = @Sendable (Bool) -> Void
static let shared = FocusEventMonitor()
struct Event: Equatable, Sendable {
let type: UInt32
let subtype: Int64
let sourcePID: pid_t
let targetPID: pid_t
let focusPID: pid_t
let focusToken: Int64
}
struct Diagnostic: Equatable, Sendable {
var available = false
var reason = "not_started"
var continuityGeneration: UInt64 = 0
var lastEvent: Event?
}
struct ProcessIdentity: Equatable, Sendable {
let executablePath: String
let launchTime: TimeInterval
}
struct RegistrationReceipt: Equatable, Sendable {
let pid: pid_t
fileprivate let generation: UInt64
fileprivate let identity: ProcessIdentity
}
enum Disposition: Equatable, Sendable {
case pass
case suppress
case redirect(pid_t)
}
struct ProtectionPolicy: Sendable {
struct Pending: Equatable, Sendable {
let thief: pid_t
let victim: pid_t
var released = false
var returnedSeen = false
}
struct Effect: Sendable {
var disposition: Disposition = .pass
var releaseToken: UInt32?
var focusChanges: [(pid_t, Bool)] = []
}
var focusedPID: pid_t?
private(set) var pending: Pending?
mutating func consume(
_ event: Event, helperPID: pid_t, protectedPIDs: Set<pid_t>,
realFrontmostPID: pid_t?, isSystemObserver: Bool
) -> Effect {
var effect = Effect()
// A real user activation wins over every synthetic focus lease.
if let pending, realFrontmostPID != pending.victim { self.pending = nil }
if [UInt32(10), 11, 12].contains(event.type) {
guard event.sourcePID != helperPID else { return effect }
if let pending, event.targetPID == pending.thief,
protectedPIDs.contains(pending.thief), realFrontmostPID == pending.victim {
effect.disposition = .redirect(pending.victim)
}
return effect
}
guard event.type == 21 else { return effect }
// CPS-generated system notifications can retain our source PID.
// Their recipient/transaction identifies them; the self-source
// exemption belongs only to the PID-directed keyboard tap above.
switch event.subtype {
case 0x4000:
guard protectedPIDs.contains(event.focusPID), event.focusPID != focusedPID,
event.targetPID == focusedPID, event.targetPID == realFrontmostPID,
event.targetPID > 0, event.focusPID != realFrontmostPID else {
pending = nil
return effect
}
pending = Pending(thief: event.focusPID, victim: event.targetPID)
effect.disposition = .suppress
case 0x8000:
if var pending, pending.victim == event.targetPID, !pending.returnedSeen {
pending.returnedSeen = true
self.pending = pending
effect.disposition = .suppress
}
case 0xf102:
guard isSystemObserver, event.focusPID > 0 else { return effect }
let previous = focusedPID
focusedPID = event.focusPID
if previous != event.focusPID {
if let previous, previous != pending?.thief {
effect.focusChanges.append((previous, false))
}
if event.focusPID != pending?.thief {
effect.focusChanges.append((event.focusPID, true))
}
}
if var pending {
if event.focusPID == pending.thief, !pending.released {
pending.released = true
self.pending = pending
// Zero is not a known transaction. Invalid tokens fail
// the controller rather than call an undocumented API.
effect.releaseToken = UInt32(exactly: event.focusToken) ?? 0
} else if event.focusPID != pending.thief && event.focusPID != pending.victim {
self.pending = nil
}
}
case 2:
if isSystemObserver, event.focusPID == pending?.thief { pending = nil }
default:
break
}
return effect
}
mutating func invalidate() { focusedPID = nil; pending = nil }
}
protocol Stream: AnyObject, Sendable {
func start(
receive: @escaping @Sendable (Event) -> Disposition,
interrupted: @escaping @Sendable (String) -> Void
) -> Bool
func addProtectedPID(_ pid: pid_t) -> Bool
func stop()
}
private struct KeyboardRecovery: Equatable {
let thief: pid_t
let victim: pid_t
let thiefIdentity: ProcessIdentity
let victimIdentity: ProcessIdentity
var canForward = true
}
private struct State {
var callbacks: [pid_t: FocusChanged] = [:]
var identities: [pid_t: ProcessIdentity] = [:]
var policy = ProtectionPolicy()
var pendingIdentity: KeyboardRecovery?
var recovery: KeyboardRecovery?
var stream: (any Stream)?
var generation: UInt64 = 0
var diagnostic = Diagnostic()
}
private let lock = NSLock()
private var state = State()
private let helperPID: pid_t
private let readInitialFocus: @Sendable () -> pid_t?
private let isFocusObserver: @Sendable (pid_t) -> Bool
private let makeStream: @Sendable () -> any Stream
private let releaseFocus: (@Sendable (UInt32) -> Bool)?
private let readRealFrontmost: @Sendable () -> pid_t?
private let isOrdinaryApp: @Sendable (pid_t) -> Bool
private let readProcessIdentity: @Sendable (pid_t) -> ProcessIdentity?
init(
helperPID: pid_t = getpid(),
readInitialFocus: @escaping @Sendable () -> pid_t? = FocusEventMonitor.systemFocusPID,
isFocusObserver: @escaping @Sendable (pid_t) -> Bool = FocusEventMonitor.isViewBridge,
makeStream: @escaping @Sendable () -> any Stream = { FocusNotificationStream() },
releaseFocus: (@Sendable (UInt32) -> Bool)? = FocusEventMonitor.systemReleaseFocus,
readRealFrontmost: @escaping @Sendable () -> pid_t? = {
NSWorkspace.shared.frontmostApplication?.processIdentifier
},
isOrdinaryApp: @escaping @Sendable (pid_t) -> Bool = {
guard let app = NSRunningApplication(processIdentifier: $0) else { return false }
return app.activationPolicy != .prohibited
},
readProcessIdentity: @escaping @Sendable (pid_t) -> ProcessIdentity? = {
guard let app = NSRunningApplication(processIdentifier: $0),
let executable = app.executableURL?.path, let launch = app.launchDate else { return nil }
return ProcessIdentity(executablePath: executable, launchTime: launch.timeIntervalSince1970)
}
) {
self.helperPID = helperPID
self.readInitialFocus = readInitialFocus
self.isFocusObserver = isFocusObserver
self.makeStream = makeStream
self.releaseFocus = releaseFocus
self.readRealFrontmost = readRealFrontmost
self.isOrdinaryApp = isOrdinaryApp
self.readProcessIdentity = readProcessIdentity
}
deinit { state.stream?.stop() }
var diagnostic: Diagnostic { lock.withLock { state.diagnostic } }
/// Requires both cancellation SPI and a working PID keyboard tap before
/// enabling suppression for this target. A failed cancellation blocks new
/// input while the existing transaction's keyboard routing drains safely.
@discardableResult
func register(pid: pid_t, onFocusChanged: @escaping FocusChanged) -> Bool {
_ = recoveryDisposition(nil, realFrontmost: readRealFrontmost())
guard lock.withLock({ state.recovery == nil }) else { return false }
guard pid > 0, isOrdinaryApp(pid), releaseFocus != nil,
let identity = readProcessIdentity(pid) else {
lock.withLock { state.diagnostic.reason = "focus_protection_unsupported" }
return false
}
let previousIdentity = lock.withLock { (state.identities[pid], state.generation) }
if let previous = previousIdentity.0, previous != identity {
interrupt("target_process_identity_changed", generation: previousIdentity.1)
}
let existing = lock.withLock {
state.diagnostic.available ? state.stream.map { ($0, state.generation) } : nil
}
if let (existing, generation) = existing {
guard existing.addProtectedPID(pid) else {
interrupt("pid_keyboard_tap_unavailable", generation: generation)
return false
}
guard readProcessIdentity(pid) == identity else {
interrupt("target_process_identity_changed", generation: generation)
return false
}
return lock.withLock {
guard state.generation == generation, state.diagnostic.available,
state.stream === existing else { return false }
state.callbacks[pid] = onFocusChanged
state.identities[pid] = identity
return true
}
}
let startup = lock.withLock { () -> ((any Stream)?, UInt64)? in
guard state.recovery == nil else { return nil }
state.callbacks[pid] = onFocusChanged
state.identities[pid] = identity
if state.diagnostic.available { return nil }
let previous = state.stream
state.generation &+= 1
state.diagnostic.continuityGeneration &+= 1
state.stream = nil
state.policy.invalidate()
state.pendingIdentity = nil
state.diagnostic.reason = "starting"
return (previous, state.generation)
}
guard let (previous, generation) = startup else {
return lock.withLock {
state.diagnostic.available && state.identities[pid] == identity
&& state.callbacks[pid] != nil
}
}
previous?.stop()
let initialFocus = readInitialFocus()
let stream = makeStream()
let installed = lock.withLock { () -> Bool in
guard state.generation == generation else { return false }
state.stream = stream
state.policy.focusedPID = initialFocus
return true
}
guard installed else { stream.stop(); return false }
let started = stream.start(
receive: { [weak self] event in self?.receive(event, generation: generation) ?? .pass },
interrupted: { [weak self] reason in self?.interrupt(reason, generation: generation) }
)
let identities = lock.withLock { state.identities }
let expired = identities.filter { readProcessIdentity($0.key) != $0.value }
let pids = lock.withLock { () -> [pid_t] in
guard state.generation == generation else { return [] }
for (expiredPID, expected) in expired where state.identities[expiredPID] == expected {
state.identities.removeValue(forKey: expiredPID)
state.callbacks.removeValue(forKey: expiredPID)
}
return Array(state.callbacks.keys)
}
let keyboardReady = started && pids.allSatisfy { stream.addProtectedPID($0) }
let confirmedInitialFocus = readInitialFocus()
let confirmedTargetIdentity = readProcessIdentity(pid)
let available = lock.withLock {
guard state.generation == generation else { return false }
// An interruption during startup must not be overwritten as healthy.
guard keyboardReady, confirmedInitialFocus != nil, confirmedTargetIdentity == identity,
state.identities[pid] == identity,
state.diagnostic.reason == "starting" else {
if state.diagnostic.reason == "starting" {
state.diagnostic.reason = "event_tap_unavailable"
state.policy.invalidate()
}
return false
}
state.policy.focusedPID = confirmedInitialFocus
state.diagnostic.available = true
state.diagnostic.reason = "protecting_ordinary_apps"
return true
}
if !available { stream.stop() }
return available
}
func isAppCurrentlyFocused(pid: pid_t) -> Bool {
lock.withLock { state.diagnostic.available && state.policy.focusedPID == pid }
}
func registrationReceipt(pid: pid_t) -> RegistrationReceipt? {
guard let identity = readProcessIdentity(pid) else { return nil }
return lock.withLock {
guard state.diagnostic.available, state.identities[pid] == identity,
state.callbacks[pid] != nil else { return nil }
return RegistrationReceipt(pid: pid, generation: state.generation, identity: identity)
}
}
func isRegistrationCurrent(_ receipt: RegistrationReceipt) -> Bool {
guard readProcessIdentity(receipt.pid) == receipt.identity else { return false }
return lock.withLock {
state.diagnostic.available && state.generation == receipt.generation
&& state.identities[receipt.pid] == receipt.identity && state.callbacks[receipt.pid] != nil
}
}
/// A synchronous workspace activation observer can invalidate a pending
/// redirect immediately; the event callback also checks the live front PID.
func observeRealFrontmost(pid: pid_t?) {
if recoveryDisposition(nil, realFrontmost: pid) != nil { return }
lock.withLock {
guard state.policy.pending != nil else { return }
_ = state.policy.consume(
Event(type: 0, subtype: 0, sourcePID: helperPID,
targetPID: 0, focusPID: 0, focusToken: 0),
helperPID: helperPID, protectedPIDs: Set(state.callbacks.keys),
realFrontmostPID: pid, isSystemObserver: false
)
if state.policy.pending == nil { state.pendingIdentity = nil }
}
}
func unregisterAll() {
let stream = lock.withLock { () -> (any Stream)? in
let previous = state.stream
// Cancellation runs outside this lock. Teardown during that call
// must not dismantle the only route back to the user's keyboard.
if state.recovery == nil, state.policy.pending?.released == true,
state.policy.focusedPID == state.pendingIdentity?.thief {
state.recovery = state.pendingIdentity
}
state.callbacks.removeAll()
state.identities.removeAll()
state.policy.invalidate()
state.pendingIdentity = nil
state.diagnostic.available = false
state.diagnostic.continuityGeneration &+= 1
if state.recovery != nil {
state.diagnostic.reason = state.recovery?.canForward == true
? "stopped_waiting_for_keyboard_recovery"
: "stopped_keyboard_safety_forwarding_unavailable"
return nil
}
state.generation &+= 1
state.stream = nil
state.diagnostic.reason = "stopped"
return previous
}
stream?.stop()
}
private func receive(_ event: Event, generation: UInt64) -> Disposition {
let realFrontmost = readRealFrontmost()
let observer = event.type == 21 && [Int64(0xf102), 2].contains(event.subtype)
&& isFocusObserver(event.targetPID)
if let disposition = recoveryDisposition(
event, generation: generation, realFrontmost: realFrontmost, observer: observer
) { return disposition }
let candidate = event.type == 21 ? event.focusPID : event.targetPID
if let expected = lock.withLock({ state.identities[candidate] }),
readProcessIdentity(candidate) != expected {
interrupt("target_process_identity_changed", generation: generation)
return .pass
}
let victimIdentity = event.type == 21 && event.subtype == 0x4000
? readProcessIdentity(event.targetPID) : nil
let result = lock.withLock { () -> (ProtectionPolicy.Effect, [(FocusChanged, Bool)]) in
guard state.generation == generation, state.diagnostic.available else { return (.init(), []) }
state.diagnostic.lastEvent = event
var effect = state.policy.consume(
event, helperPID: helperPID, protectedPIDs: Set(state.callbacks.keys),
realFrontmostPID: realFrontmost, isSystemObserver: observer
)
if event.type == 21, event.subtype == 0x4000,
let pending = state.policy.pending {
if let thiefIdentity = state.identities[pending.thief], let victimIdentity {
state.pendingIdentity = KeyboardRecovery(
thief: pending.thief, victim: pending.victim,
thiefIdentity: thiefIdentity, victimIdentity: victimIdentity
)
} else {
// Without both identities, suppression would create a
// transaction that cannot be safely routed after failure.
state.policy = ProtectionPolicy(focusedPID: state.policy.focusedPID)
effect = .init()
}
}
if state.policy.pending == nil { state.pendingIdentity = nil }
let callbacks = effect.focusChanges.compactMap { pid, value in
state.callbacks[pid].map { ($0, value) }
}
return (effect, callbacks)
}
guard lock.withLock({ state.generation == generation && state.diagnostic.available }) else { return .pass }
if let token = result.0.releaseToken,
token == 0 || releaseFocus?(token) != true {
retainKeyboardRecovery(generation: generation)
return .pass
}
// Never invoke arbitrary caller code under the state lock.
for (callback, focused) in result.1 {
guard lock.withLock({ state.generation == generation && state.diagnostic.available }) else { return .pass }
callback(focused)
}
return lock.withLock {
state.generation == generation && state.diagnostic.available ? result.0.disposition : .pass
}
}
/// A cancellation error invalidates automation, not the already-proven
/// keyboard route. Only that route survives; system notifications all pass.
private func retainKeyboardRecovery(generation: UInt64) {
let callbacks = lock.withLock { () -> [FocusChanged] in
guard state.generation == generation else { return [] }
if state.recovery == nil { state.recovery = state.pendingIdentity }
state.diagnostic.available = false
state.diagnostic.reason = state.recovery?.canForward == true
? "release_key_focus_failed_waiting_for_keyboard_recovery"
: "release_key_focus_failed_keyboard_safety_forwarding_unavailable"
state.diagnostic.continuityGeneration &+= 1
state.policy.invalidate()
state.pendingIdentity = nil
return Array(state.callbacks.values)
}
for callback in callbacks { callback(false) }
}
/// nil means no recovery owns this event. A nil front PID/identity is not
/// evidence of restoration: stop routing for that event, but keep input
/// blocked until a positive foreground/focus/identity transition is seen.
private func recoveryDisposition(
_ event: Event?, generation: UInt64? = nil,
realFrontmost: pid_t?, observer: Bool = false
) -> Disposition? {
let snapshot = lock.withLock { () -> (KeyboardRecovery, UInt64)? in
guard generation == nil || state.generation == generation,
let recovery = state.recovery else { return nil }
return (recovery, state.generation)
}
guard let (recovery, ownerGeneration) = snapshot else { return nil }
let thiefIdentity = readProcessIdentity(recovery.thief)
let victimIdentity = readProcessIdentity(recovery.victim)
let frontChanged = realFrontmost.map { $0 > 0 && $0 != recovery.victim } ?? false
let identityChanged = thiefIdentity.map { $0 != recovery.thiefIdentity } == true
|| victimIdentity.map { $0 != recovery.victimIdentity } == true
let focusRestored = event.map {
observer && $0.type == 21 && $0.subtype == 0xf102
&& $0.focusPID > 0 && $0.focusPID != recovery.thief
} ?? false
let result = lock.withLock { () -> (Disposition, (any Stream)?) in
guard state.generation == ownerGeneration, state.recovery == recovery else { return (.pass, nil) }
if let event { state.diagnostic.lastEvent = event }
if frontChanged || identityChanged || focusRestored {
let previous = state.stream
state.stream = nil
state.recovery = nil
state.pendingIdentity = nil
state.policy.invalidate()
state.generation &+= 1
state.diagnostic.continuityGeneration &+= 1
state.diagnostic.reason = identityChanged
? "keyboard_recovery_process_identity_changed" : "keyboard_focus_recovery_observed"
return (.pass, previous)
}
guard recovery.canForward, let event, [UInt32(10), 11, 12].contains(event.type),
event.sourcePID != helperPID, event.targetPID == recovery.thief,
realFrontmost == recovery.victim,
thiefIdentity == recovery.thiefIdentity,
victimIdentity == recovery.victimIdentity else { return (.pass, nil) }
return (.redirect(recovery.victim), nil)
}
result.1?.stop()
return result.0
}
private func interrupt(_ reason: String, generation: UInt64) {
let result = lock.withLock { () -> ([FocusChanged], (any Stream)?) in
guard state.generation == generation else { return ([], nil) }
state.diagnostic.available = false
if state.recovery == nil, state.policy.pending?.released == true {
state.recovery = state.pendingIdentity
}
// Once macOS disables a tap we cannot promise keyboard delivery.
// Keep new automation blocked until the unsafe focus window ends.
state.recovery?.canForward = false
state.diagnostic.reason = state.recovery == nil
? reason : "\(reason)_keyboard_safety_forwarding_unavailable"
state.diagnostic.continuityGeneration &+= 1
state.policy.invalidate()
state.pendingIdentity = nil
return (Array(state.callbacks.values), state.stream)
}
result.1?.stop()
for callback in result.0 { callback(false) }
}
private typealias GetKeyFocus = @convention(c) (
UnsafeMutablePointer<ProcessSerialNumber>, UnsafeMutablePointer<DarwinBoolean>
) -> OSStatus
private static let getKeyFocus: GetKeyFocus? = {
guard let handle = dlopen(nil, RTLD_LAZY),
let symbol = dlsym(handle, "CPSGetKeyFocusProcess") else { return nil }
return unsafeBitCast(symbol, to: GetKeyFocus.self)
}()
private typealias GetPID = @convention(c) (
UnsafePointer<ProcessSerialNumber>, UnsafeMutablePointer<pid_t>
) -> OSStatus
private static let getPID: GetPID? = {
guard let handle = dlopen(nil, RTLD_LAZY),
let symbol = dlsym(handle, "GetProcessPID") else { return nil }
return unsafeBitCast(symbol, to: GetPID.self)
}()
private typealias ReleaseFocus = @convention(c) (UInt32) -> OSStatus
private static let systemReleaseFocus: (@Sendable (UInt32) -> Bool)? = {
guard let handle = dlopen(nil, RTLD_LAZY),
let symbol = dlsym(handle, "CPSReleaseKeyFocusWithID") else { return nil }
let release = unsafeBitCast(symbol, to: ReleaseFocus.self)
return { release($0) == noErr }
}()
private static func systemFocusPID() -> pid_t? {
var process = ProcessSerialNumber()
var focused = DarwinBoolean(false)
var pid: pid_t = 0
if let getKeyFocus, let getPID, getKeyFocus(&process, &focused) == noErr,
getPID(&process, &pid) == noErr, pid > 0 { return pid }
// Older systems can lack the SPI. Public frontmost process is an initial
// fallback only; synthetic per-process notifications never overwrite it.
return NSWorkspace.shared.frontmostApplication?.processIdentifier
}
private static func isViewBridge(_ pid: pid_t) -> Bool {
guard pid > 0 else { return false }
var name = [CChar](repeating: 0, count: 256)
if proc_name(pid, &name, UInt32(name.count)) > 0 {
return name.withUnsafeBufferPointer { buffer in
guard let base = buffer.baseAddress else { return false }
return isViewBridgeProcess(name: String(cString: base), executablePath: nil)
}
}
// A root-owned ViewBridge is visible to proc_pidpath even when
// proc_name is unreadable. Accept only the observed system executable,
// not another app or bundle with the same last path component.
var path = [CChar](repeating: 0, count: 4096)
guard proc_pidpath(pid, &path, UInt32(path.count)) > 0 else { return false }
return path.withUnsafeBufferPointer { buffer in
guard let base = buffer.baseAddress else { return false }
return isViewBridgeProcess(name: nil, executablePath: String(cString: base))
}
}
static func isViewBridgeProcess(name: String?, executablePath: String?) -> Bool {
if let name { return name == "ViewBridgeAuxiliary" }
return executablePath == "/System/Library/PrivateFrameworks/ViewBridge.framework/Versions/A/XPCServices/ViewBridgeAuxiliary.xpc/Contents/MacOS/ViewBridgeAuxiliary"
}
}
final class FocusNotificationStream: FocusEventMonitor.Stream, @unchecked Sendable {
private let lock = NSLock()
private var cancelled = false
private var runLoop: CFRunLoop?
private var tap: CFMachPort?
private var keyboardTaps: [pid_t: CFMachPort] = [:]
private var keyboardSources: [CFRunLoopSource] = []
private var receive: (@Sendable (FocusEventMonitor.Event) -> FocusEventMonitor.Disposition)?
private var interrupted: (@Sendable (String) -> Void)?
private typealias CreatePIDTap = @convention(c) (
pid_t, UInt32, UInt32, CGEventMask, CGEventTapCallBack, UnsafeMutableRawPointer?
) -> Unmanaged<CFMachPort>?
private static let createPIDTap: CreatePIDTap? = {
guard let handle = dlopen(nil, RTLD_LAZY),
let symbol = dlsym(handle, "CGEventTapCreateForPid") else { return nil }
return unsafeBitCast(symbol, to: CreatePIDTap.self)
}()
private static let callback: CGEventTapCallBack = { _, type, event, context in
guard let context else { return Unmanaged.passUnretained(event) }
let stream = Unmanaged<FocusNotificationStream>.fromOpaque(context).takeUnretainedValue()
switch stream.handle(type: type, event: event) {
case .pass: return Unmanaged.passUnretained(event)
case .suppress: return nil
case .redirect(let victim):
event.postToPid(victim)
return nil
}
}
func start(
receive: @escaping @Sendable (FocusEventMonitor.Event) -> FocusEventMonitor.Disposition,
interrupted: @escaping @Sendable (String) -> Void
) -> Bool {
lock.withLock { self.receive = receive; self.interrupted = interrupted }
let ready = DispatchSemaphore(value: 0)
let thread = Thread { [self] in run(ready: ready) }
thread.name = "computer-use-focus-observer"
thread.start()
guard ready.wait(timeout: .now() + 1) == .success else {
interrupted("event_tap_start_timeout")
stop()
return false
}
return lock.withLock { !cancelled && tap != nil }
}
func addProtectedPID(_ pid: pid_t) -> Bool {
guard Self.createPIDTap != nil else { return false }
let loop = lock.withLock { cancelled ? nil : runLoop }
guard let loop else { return false }
guard Self.perform(on: loop, operation: { [weak self] in
self?.installKeyboardTap(pid, loop: CFRunLoopGetCurrent())
}) else {
interrupted?("pid_keyboard_tap_start_timeout")
stop()
return false
}
return lock.withLock { !cancelled && keyboardTaps[pid] != nil }
}
/// A callback already executes on this run loop; waiting for a queued block
/// there would block the very worker responsible for acknowledging it.
static func perform(on loop: CFRunLoop, operation: @escaping @Sendable () -> Void) -> Bool {
if CFEqual(CFRunLoopGetCurrent(), loop) { operation(); return true }
let ready = DispatchSemaphore(value: 0)
CFRunLoopPerformBlock(loop, CFRunLoopMode.commonModes.rawValue) {
operation()
ready.signal()
}
CFRunLoopWakeUp(loop)
return ready.wait(timeout: .now() + 1) == .success
}
static func runWhileActive(
isCancelled: () -> Bool,
runOnce: (TimeInterval) -> CFRunLoopRunResult = {
CFRunLoopRunInMode(.defaultMode, $0, false)
}
) {
while !isCancelled() {
if runOnce(0.1) == .finished { break }
}
}
private func installKeyboardTap(_ pid: pid_t, loop: CFRunLoop) {
guard lock.withLock({ !cancelled && keyboardTaps[pid] == nil }),
let create = Self.createPIDTap,
let port = create(
pid, CGEventTapPlacement.tailAppendEventTap.rawValue,
CGEventTapOptions.defaultTap.rawValue, 0x1c00, Self.callback,
Unmanaged.passUnretained(self).toOpaque()
)?.takeRetainedValue() else { return }
guard let source = CFMachPortCreateRunLoopSource(nil, port, 0) else {
CFMachPortInvalidate(port)
return
}
CFRunLoopAddSource(loop, source, .commonModes)
CGEvent.tapEnable(tap: port, enable: true)
let installed = lock.withLock { () -> Bool in
guard !cancelled, CGEvent.tapIsEnabled(tap: port) else { return false }
keyboardTaps[pid] = port
keyboardSources.append(source)
return true
}
if !installed {
CFRunLoopRemoveSource(loop, source, .commonModes)
CFMachPortInvalidate(port)
}
}
func stop() {
let loop = lock.withLock { () -> CFRunLoop? in
cancelled = true
return runLoop
}
if let loop { CFRunLoopStop(loop); CFRunLoopWakeUp(loop) }
}
private func run(ready: DispatchSemaphore) {
guard let port = CGEvent.tapCreate(
tap: .cgAnnotatedSessionEventTap,
place: .tailAppendEventTap,
options: .defaultTap,
eventsOfInterest: CGEventMask(1) << 21,
callback: Self.callback,
userInfo: Unmanaged.passUnretained(self).toOpaque()
) else {
interrupted?("event_tap_creation_failed_permission_or_unsupported")
ready.signal()
return
}
guard let source = CFMachPortCreateRunLoopSource(nil, port, 0) else {
CFMachPortInvalidate(port)
interrupted?("event_tap_run_loop_source_failed")
ready.signal()
return
}
let loop = CFRunLoopGetCurrent()
CFRunLoopAddSource(loop, source, .commonModes)
CGEvent.tapEnable(tap: port, enable: true)
let shouldRun = lock.withLock { () -> Bool in
guard !cancelled, CGEvent.tapIsEnabled(tap: port) else { return false }
tap = port
runLoop = loop
return true
}
ready.signal()
if shouldRun {
// Stop can race the gap before RunInMode enters. A bounded run keeps
// that lost wakeup from leaving the worker and its taps alive forever.
Self.runWhileActive(isCancelled: { lock.withLock { cancelled } })
}
let keyboards = lock.withLock { () -> ([CFMachPort], [CFRunLoopSource]) in
let owned = (Array(keyboardTaps.values), keyboardSources)
keyboardTaps.removeAll()
keyboardSources.removeAll()
return owned
}
for source in keyboards.1 { CFRunLoopRemoveSource(loop, source, .commonModes) }
for port in keyboards.0 { CFMachPortInvalidate(port) }
CGEvent.tapEnable(tap: port, enable: false)
CFRunLoopRemoveSource(loop, source, .commonModes)
CFMachPortInvalidate(port)
let unexpected = lock.withLock { () -> Bool in
tap = nil
runLoop = nil
return !cancelled
}
if unexpected { interrupted?("event_tap_run_loop_stopped") }
}
private func handle(type: CGEventType, event: CGEvent) -> FocusEventMonitor.Disposition {
if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
interrupted?(type == .tapDisabledByTimeout ? "event_tap_timeout" : "event_tap_disabled")
// The next registration creates a fresh stream and fresh initial
// focus. Re-enabling here would pretend the missed interval was safe.
stop()
return .pass
}
guard [UInt32(21), 10, 11, 12].contains(type.rawValue),
let subtypeField = CGEventField(rawValue: 64),
let focusField = CGEventField(rawValue: 73),
let tokenField = CGEventField(rawValue: 71) else { return .pass }
return receive?(FocusEventMonitor.Event(
type: type.rawValue,
subtype: event.getIntegerValueField(subtypeField),
sourcePID: pid_t(truncatingIfNeeded: event.getIntegerValueField(.eventSourceUnixProcessID)),
targetPID: pid_t(truncatingIfNeeded: event.getIntegerValueField(.eventTargetUnixProcessID)),
focusPID: pid_t(truncatingIfNeeded: event.getIntegerValueField(focusField)),
focusToken: event.getIntegerValueField(tokenField)
)) ?? .pass
}
}
@@ -234,25 +234,9 @@ public enum Injection {
/// carries the same "human click interval" between down and up.
private static let pressHoldMs: UInt64 = 24
/// How long to let a synthetic focus notification reach the target's run
/// loop before the click burst arrives. Short, because nothing came to the
/// foreground — but not zero, since the target has to dequeue the event.
/// Mirrors `AXAction.syntheticFocusSettleSeconds`.
private static let focusSettleMs: UInt64 = 120
/// Tell the target it holds keyboard focus, and let it act on that before
/// clicking. The user's foreground is never touched.
///
/// This used to bring the target's window forward with a CPS grant and wait
/// 800ms for the switch to settle — see the note on
/// `AXAction.ensureTargetAcceptsInput` for why that was neither necessary
/// nor what it appeared to be. Note that this path did not send the
/// notification at all: it relied on the foreground change entirely, so
/// removing the grant without adding the notification would leave it with
/// nothing.
private static func focusForClick(pid: pid_t) async {
guard SyntheticWindowFocus.enforceActiveState(pid: pid) else { return }
await sleepMs(focusSettleMs)
/// Share the semantic input path's focus lifecycle and acknowledgement.
private static func focusForClick(pid: pid_t) async throws {
try await SyntheticWindowFocus.prepareInput(pid: pid)
}
private static let interKeyGapMs: UInt64 = 6 // between down/up of a single key
private static let interGraphemeGapMs: UInt64 = 4 // between typed characters
@@ -550,7 +534,7 @@ public enum Injection {
let clicks = max(1, count)
let flags = KeySym.flags(for: modifiers)
await focusForClick(pid: targetPid)
try await focusForClick(pid: targetPid)
let specs = (1...clicks).flatMap { clickState in
[
@@ -617,7 +601,7 @@ public enum Injection {
// A press starts an interaction, so the target has to believe it holds
// focus before it arrives — otherwise the press is discarded and the
// matching release lands on nothing.
await focusForClick(pid: target.pid)
try await focusForClick(pid: target.pid)
let event = try makeMouse(button.down, at: p, button: button, clickState: 1, targetPid: target.pid)
event.postToPid(target.pid)
HeldState.buttons.append(
@@ -700,7 +684,7 @@ public enum Injection {
try ensurePostable(targetPid)
let n = max(1, steps)
await focusForClick(pid: targetPid)
try await focusForClick(pid: targetPid)
var specs = [
MouseBurstSpec(
@@ -0,0 +1,106 @@
import CoreGraphics
/// Allocates a complete PID-targeted keyboard burst before caller-supplied posting.
/// Modifier state is communicated with flagsChanged, not synthetic modifier-key
/// presses. Allocation uses an explicit source and modifier baseline; production
/// dispatch captures that baseline independently without changing physical input.
enum KeyboardEventBurst {
enum Specification {
case flagsChanged(CGEventFlags)
case keyDown(CGKeyCode, CGEventFlags)
case keyUp(CGKeyCode, CGEventFlags)
}
/// Match the reference keyboard source lifetime and state domain. HID is
/// the source state only; the caller still posts exclusively to its PID.
@MainActor
static func makeSource() throws -> CGEventSource {
guard let source = HelperEventMarker.mark(CGEventSource(stateID: .hidSystemState)) else {
throw CUError(CUError.Code.eventAlloc, "Failed to allocate a keyboard event source")
}
return source
}
/// A key command owns a fresh source. Restore physical modifiers from the
/// independent combined-session domain, after focus preparation completes.
@MainActor
static func dispatch(
chords: [KeyMapping.Chord],
prepare: @MainActor () async throws -> Void,
makeSource: @MainActor () throws -> CGEventSource = { try KeyboardEventBurst.makeSource() },
readFlagsState: @MainActor (CGEventSourceStateID) -> CGEventFlags = { CGEventSource.flagsState($0) },
validateBeforePosting: @MainActor () throws -> Void,
post: @MainActor (CGEvent) -> Void
) async throws {
try Task.checkCancellation()
let source = try makeSource()
try await dispatch(
chords: chords, source: source, prepare: prepare,
restoringFlags: { readFlagsState(.combinedSessionState) },
validateBeforePosting: validateBeforePosting, post: post
)
}
/// Prepare may yield while target focus is established. Validate the
/// caller's clipboard lease only after that wait and allocation, then keep
/// validation and the complete burst in the same main-actor turn.
@MainActor
static func dispatch(
chords: [KeyMapping.Chord],
source: CGEventSource,
prepare: @MainActor () async throws -> Void,
restoringFlags: @MainActor () -> CGEventFlags,
validateBeforePosting: @MainActor () throws -> Void,
post: @MainActor (CGEvent) -> Void
) async throws {
try Task.checkCancellation()
try await prepare()
let events = try allocate(chords: chords, source: source, restoringFlags: restoringFlags())
try Task.checkCancellation()
try validateBeforePosting()
for event in events { post(event) }
}
static func allocate(
chords: [KeyMapping.Chord],
source: CGEventSource,
restoringFlags: CGEventFlags,
allocateEvent: (Specification, CGEventSource) throws -> CGEvent? = {
makeEvent($0, source: $1)
}
) throws -> [CGEvent] {
let specs: [Specification] = chords.flatMap { chord in
[
.flagsChanged(chord.flags),
.keyDown(chord.keyCode, chord.flags),
// Restore the captured baseline before key-up. The key-up
// itself retains its chord flags, matching the key-down.
.flagsChanged(restoringFlags),
.keyUp(chord.keyCode, chord.flags),
]
}
return try EventBurst.allocateAll(specs: specs) { spec in
try allocateEvent(spec, source)
}
}
static func makeEvent(_ spec: Specification, source: CGEventSource) -> CGEvent? {
let event: CGEvent?
let flags: CGEventFlags
switch spec {
case let .flagsChanged(value):
event = CGEvent(source: source)
event?.type = .flagsChanged
flags = value
case let .keyDown(keyCode, value):
event = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: true)
flags = value
case let .keyUp(keyCode, value):
event = CGEvent(keyboardEventSource: source, virtualKey: keyCode, keyDown: false)
flags = value
}
event?.flags = flags
return event
}
}
@@ -0,0 +1,51 @@
import CoreGraphics
/// Delivers an already allocated mouse gesture. An optional pause lets drags
/// process lifecycle work; ordinary clicks stay in one main-actor turn without
/// yielding between their down/up pairs. Cancellation stops further input; only an outstanding down
/// may be released, at the last point actually delivered rather than the end of
/// an unfinished drag. The caller must validate identity again inside release.
@MainActor
enum MouseEventBurstDelivery {
static func deliver(
events: [CGEvent],
validate: @MainActor () throws -> Void,
post: @MainActor (CGEvent) -> Void,
release: @MainActor (CGEvent, CGPoint) throws -> Void,
pause: (@MainActor () async throws -> Void)? = nil
) async throws {
var heldDown: CGEvent?
var lastPoint = CGPoint.zero
do {
for event in events {
try Task.checkCancellation()
try validate()
try Task.checkCancellation()
post(event)
lastPoint = event.location
switch event.type {
case .leftMouseDown, .rightMouseDown, .otherMouseDown:
heldDown = event
case .leftMouseUp where heldDown?.type == .leftMouseDown,
.rightMouseUp where heldDown?.type == .rightMouseDown,
.otherMouseUp where heldDown?.type == .otherMouseDown:
heldDown = nil
default:
break
}
if let pause { try await pause() }
// A caller's pause may return normally despite cancellation.
// This check also covers cancellation after the final mouse-up.
try Task.checkCancellation()
}
} catch {
if let heldDown {
// Cleanup failure must not hide the original interruption or
// restart the gesture. The caller owns safe release routing.
try? release(heldDown, lastPoint)
}
throw error
}
}
}
@@ -0,0 +1,25 @@
import CoreGraphics
/// Keyboard focus belongs to the window published by get_app_state, not the
/// first layer-zero CG window (which can be a small auxiliary app window).
enum SnapshotKeyboardWindow {
static func resolve(
pid: pid_t,
snapshot: AXTreeSnapshotEvidence?,
currentIdentity: AXTreeProcessIdentity?,
windowForID: (CGWindowID, pid_t) -> WindowGeometry.Window?
) throws -> WindowGeometry.Window {
try SnapshotProcessGuard.validate(
pid: pid, snapshot: snapshot, current: currentIdentity, expected: nil
)
guard let windowID = snapshot?.keyWindowID, windowID != kCGNullWindowID,
let window = windowForID(windowID, pid),
window.id == windowID, window.ownerPid == pid else {
throw CUError(
"stale_window",
"The snapshot's keyboard target window is no longer available. Call get_app_state before typing or pressing keys."
)
}
return window
}
}
@@ -1,357 +1,406 @@
import AppKit
import os
import CoreGraphics
import Foundation
import os
/// Tells an application it holds keyboard focus, without giving it the
/// foreground.
///
/// WHY THIS EXISTS
/// ---------------
/// Chromium/CEF apps route synthesized input by the window an event names, then
/// ignore it unless the app believes it is active. `WindowKeyFocus` buys that
/// belief by making the app genuinely frontmost — measured, and measurably
/// wrong for us: it takes the foreground away from whatever the user is doing,
/// once per click. Automating an app in the background is the entire point of
/// the feature; an implementation that yanks the user's foreground twenty times
/// during a task has not delivered it.
///
/// HOW CODEX DOES IT — and it is not what the folklore says
/// -------------------------------------------------------
/// Its service links ApplicationServices, CoreGraphics and Carbon and does NOT
/// link SkyLight; the window-manager trick of hand-building a 0xf8-byte record
/// for `SLPSPostEventRecordTo` appears nowhere. What
/// `SyntheticAppFocusEnforcer.enforceActiveState(for:)` actually does is build
/// an ordinary AppKit-defined `NSEvent` whose *subtype* is a CPS focus
/// notification, and post it to the target with `CGEventPostToPid` — the same
/// transport we already use for clicks and keystrokes.
///
/// It keeps `applicationIsActive` (reality) beside `applicationBelievesItIsActive`
/// and `applicationBelievesItHasFocus` (what the target was told), and only
/// re-sends when the two disagree. The real foreground is never touched: its
/// `setFrontProcess` calls live solely in the picture-in-picture stream, tagged
/// `causedByUser`, for when the *user* clicks the PIP window.
///
/// So this needs no private symbol at all — `NSEvent.otherEvent`, `.cgEvent`
/// and `CGEventPostToPid` are public. Only the subtype values are undocumented.
/// Synthetic activation is not the user's real foreground. Its lifetime must
/// follow observed focus changes, not just whether a notification was once sent.
enum SyntheticWindowFocus {
struct BeliefTarget: Equatable, Sendable {
let processIdentity: AXTreeProcessIdentity?
}
/// Session-scoped belief about process lifetimes that have already
/// received the synthetic focus + activation pair.
///
/// Keeping this state is load-bearing for Chromium/CEF text entry. A click
/// establishes in-app focus on a field; blindly posting another
/// `keyFocusReturned` to window 0 before `type_text` can reset that field
/// focus. Codex keeps the same distinction between real application state
/// and what the target has already been told.
struct BeliefState {
private(set) var syntheticallyActive: [pid_t: BeliefTarget] = [:]
/// Reserve the one establishment send for `pid`.
///
/// Seeing the process genuinely active clears the synthetic belief so
/// a later background transition can establish it again.
mutating func beginEnforcement(
pid: pid_t,
applicationIsActive: Bool,
target: BeliefTarget
) -> Bool {
if applicationIsActive {
observeRealActivation(pid: pid)
return false
}
guard syntheticallyActive[pid] != target else { return false }
syntheticallyActive[pid] = target
return true
}
mutating func observeRealActivation(pid: pid_t) {
syntheticallyActive.removeValue(forKey: pid)
}
mutating func cancelEnforcement(
pid: pid_t,
expectedTarget: BeliefTarget? = nil
) {
if let expectedTarget,
syntheticallyActive[pid] != expectedTarget {
return
}
syntheticallyActive.removeValue(forKey: pid)
}
mutating func drain() -> [pid_t: BeliefTarget] {
defer { syntheticallyActive.removeAll() }
return syntheticallyActive
}
}
struct EnforcementRuntime: Sendable {
let applicationIsActive: Bool
let target: BeliefTarget
let post: @Sendable (Notification, pid_t) -> Bool
}
/// CPS notifications, carried as the subtype of a synthesized event.
///
/// Values recovered from the once-initializers in Codex's service. Stored
/// as `Int32` because `keyFocusReturned` does not fit `Int16` unsigned —
/// see `subtype`.
enum Notification: Int32, Sendable {
case appActivated = 1
// Also the value CPS uses for "new front process"; the meaning comes
// from which notification the sender is making, not from the number.
case appDeactivated = 2
case lostKeyFocus = 0x1000
case keyFocusTaken = 0x4000
/// The one that makes a background app act focused.
case keyFocusReturned = 0x8000
/// `NSEvent.subtype` is a signed 16-bit field, so 0x8000 travels as the
/// negative with the same bit pattern. Truncating instead would send
/// subtype 0 — a different, meaningless notification that the target
/// accepts and ignores, with no error anywhere.
var subtype: Int16 { Int16(truncatingIfNeeded: rawValue) }
/// The event type that carries this notification. NOT the same for all
/// of them, which is the detail this file originally got wrong.
///
/// Every notification used to be posted on `.appKitDefined` (13). The
/// activation pair does belong there — Codex hardcodes type 13 with
/// subtype 1 — but the key-focus family travels on type 21, read out of
/// the lazily-initialized global its `enforceActiveState` loads the type
/// from (`mov w9, #0x15`, stored beside the 0x8000 subtype).
///
/// 21 has no name in the public `NSEventType`, yet it is a valid case:
/// `NSEvent.otherEvent` builds it and `.cgEvent` converts it. On type 13
/// the same subtype is a notification the target has no handler for —
/// accepted, ignored, no error, and the only symptom is that background
/// input never lands. That is what a whole build measured as "24
/// mutating actions, 1 effect".
var carrierEventType: NSEvent.EventType? {
switch self {
case .appActivated, .appDeactivated:
return .appKitDefined
case .appActivated, .appDeactivated: .appKitDefined
case .lostKeyFocus, .keyFocusTaken, .keyFocusReturned:
return NSEvent.EventType(rawValue: Self.keyFocusCarrierRawValue)
NSEvent.EventType(rawValue: Self.keyFocusCarrierRawValue)
}
}
/// Undocumented, so it is read back rather than assumed: a future SDK
/// that stops accepting it makes `carrierEventType` nil and `post`
/// return false, instead of trapping on a force-unwrap.
static let keyFocusCarrierRawValue: UInt = 21
}
/// Post a CPS focus notification to `pid`.
///
/// Returns false only when AppKit refuses to build the event or convert it,
/// which does not happen in practice; there is no delivery receipt to check.
struct Window: Equatable, Sendable {
let id: CGWindowID
let bounds: CGRect
// The outer optional records whether AX supplied the attribute. A
// failed query requires generic activation; a supplied but undecodable
// point still allows window activation, without inventing a click.
let activationPoint: CGPoint??
var resolvedActivationPoint: CGPoint? { activationPoint ?? nil }
}
enum Establishment: Equatable, Sendable {
case activate
case returnFocus
case none
}
struct Belief: Equatable, Sendable {
let identity: AXTreeProcessIdentity
var applicationIsActive: Bool
var applicationBelievesItIsActive: Bool
var applicationBelievesItHasFocus: Bool
var generation: UInt64 = 0
}
struct State: Sendable {
private(set) var targets: [pid_t: Belief] = [:]
mutating func prepare(
pid: pid_t, identity: AXTreeProcessIdentity,
applicationIsActive: Bool, applicationHasFocus: Bool
) -> (Establishment, UInt64) {
if targets[pid]?.identity != identity {
targets[pid] = Belief(
identity: identity, applicationIsActive: applicationIsActive,
applicationBelievesItIsActive: applicationIsActive,
applicationBelievesItHasFocus: applicationHasFocus
)
} else if targets[pid]?.applicationIsActive != applicationIsActive {
observeApplication(pid: pid, active: applicationIsActive)
}
guard let belief = targets[pid] else { return (.none, 0) }
if belief.applicationBelievesItHasFocus { return (.none, belief.generation) }
return (belief.applicationBelievesItIsActive ? .returnFocus : .activate, belief.generation)
}
mutating func observeApplication(pid: pid_t, active: Bool) {
guard var belief = targets[pid] else { return }
let wasActive = belief.applicationIsActive
belief.applicationIsActive = active
// Background -> background must preserve the synthetic field focus.
if active || wasActive {
if belief.applicationBelievesItIsActive != active
|| belief.applicationBelievesItHasFocus != active {
belief.generation &+= 1
}
belief.applicationBelievesItIsActive = active
belief.applicationBelievesItHasFocus = active
}
targets[pid] = belief
}
mutating func observeFrontmost(pid: pid_t) {
for target in Array(targets.keys) {
observeApplication(pid: target, active: target == pid)
}
}
mutating func observeDeactivation(pid: pid_t) {
guard var belief = targets[pid] else { return }
belief.applicationIsActive = false
belief.applicationBelievesItIsActive = false
belief.applicationBelievesItHasFocus = false
belief.generation &+= 1
targets[pid] = belief
}
mutating func observeFocus(pid: pid_t, hasFocus: Bool) {
guard var belief = targets[pid] else { return }
belief.applicationBelievesItHasFocus = hasFocus
// A loss during an in-flight establishment invalidates its receipt
// even when the old belief was already false. A gain can confirm it.
if !hasFocus { belief.generation &+= 1 }
targets[pid] = belief
}
mutating func confirm(pid: pid_t, identity: AXTreeProcessIdentity, generation: UInt64) -> Bool {
guard var belief = targets[pid], belief.identity == identity,
belief.generation == generation else { return false }
belief.applicationBelievesItIsActive = true
belief.applicationBelievesItHasFocus = true
targets[pid] = belief
return true
}
mutating func invalidate(pid: pid_t, identity: AXTreeProcessIdentity) {
guard targets[pid]?.identity == identity else { return }
targets[pid]?.applicationBelievesItIsActive = false
targets[pid]?.applicationBelievesItHasFocus = false
targets[pid]?.generation &+= 1
}
mutating func drain() -> [pid_t: Belief] {
defer { targets.removeAll() }
return targets
}
}
struct Runtime: Sendable {
var identity: @MainActor @Sendable (pid_t) -> AXTreeProcessIdentity?
var isActive: @MainActor @Sendable (pid_t) -> Bool
var hasFocus: @MainActor @Sendable (pid_t) -> Bool
var acceptsInput: @MainActor @Sendable (pid_t) -> Bool
var post: @MainActor @Sendable (Establishment, pid_t, Window?) -> Bool
var pause: @Sendable () async throws -> Void
var attempts: Int = 20
var validateContinuity: @MainActor @Sendable () throws -> Void = {}
}
final class Coordinator: Sendable {
private let state = OSAllocatedUnfairLock(initialState: State())
func observeFrontmost(pid: pid_t) { state.withLock { $0.observeFrontmost(pid: pid) } }
func observeDeactivation(pid: pid_t) { state.withLock { $0.observeDeactivation(pid: pid) } }
func observeFocus(pid: pid_t, hasFocus: Bool) {
state.withLock { $0.observeFocus(pid: pid, hasFocus: hasFocus) }
}
func drain() -> [pid_t: Belief] { state.withLock { $0.drain() } }
var beliefs: [pid_t: Belief] { state.withLock { $0.targets } }
@MainActor
func prepare(pid: pid_t, window: Window?, runtime: Runtime) async throws {
try Task.checkCancellation()
try runtime.validateContinuity()
guard pid > 0, let identity = runtime.identity(pid) else {
throw CUError("process_gone", "The input target is no longer running.")
}
let active = runtime.isActive(pid)
let focused = runtime.hasFocus(pid)
let (establishment, generation) = state.withLock {
$0.prepare(pid: pid, identity: identity, applicationIsActive: active, applicationHasFocus: focused)
}
try Task.checkCancellation()
try runtime.validateContinuity()
if establishment != .none, !runtime.post(establishment, pid, window) {
state.withLock { $0.invalidate(pid: pid, identity: identity) }
throw CUError("focus_event_failed", "Could not construct the target window's activation event.")
}
do {
// Yield the main actor so both the target and lifecycle observers
// can run; a fixed blocking sleep hid focus changes in the past.
if establishment != .none { try await runtime.pause() }
for attempt in 0...max(0, runtime.attempts) {
try Task.checkCancellation()
try runtime.validateContinuity()
guard runtime.identity(pid) == identity else {
throw CUError("stale_process", "The input target restarted while establishing focus.")
}
if runtime.acceptsInput(pid) {
try runtime.validateContinuity()
guard state.withLock({ $0.confirm(pid: pid, identity: identity, generation: generation) }) else {
throw CUError("focus_changed", "The target lost focus while preparing input. Read its current state before retrying.")
}
return
}
if attempt < runtime.attempts { try await runtime.pause() }
}
throw CUError("focus_not_accepted", "The target did not acknowledge activation. Input was not sent; read its current state before retrying.")
} catch {
state.withLock { $0.invalidate(pid: pid, identity: identity) }
throw error
}
}
}
/// Consume the notification itself, including a delayed activate -> leave
/// pair. Checking today's frontmost PID would silently discard that history.
final class ApplicationLifecycleObserver: @unchecked Sendable {
private let center: NotificationCenter
private var tokens: [NSObjectProtocol] = []
init(
center: NotificationCenter, coordinator: Coordinator,
onFrontmost: @escaping @Sendable (pid_t) -> Void = { _ in }
) {
self.center = center
tokens.append(center.addObserver(
forName: NSWorkspace.didActivateApplicationNotification, object: nil, queue: nil
) { notification in
guard let app = notification.userInfo?[NSWorkspace.applicationUserInfoKey]
as? NSRunningApplication else { return }
coordinator.observeFrontmost(pid: app.processIdentifier)
onFrontmost(app.processIdentifier)
})
tokens.append(center.addObserver(
forName: NSWorkspace.didDeactivateApplicationNotification, object: nil, queue: nil
) { notification in
guard let app = notification.userInfo?[NSWorkspace.applicationUserInfoKey]
as? NSRunningApplication else { return }
coordinator.observeDeactivation(pid: app.processIdentifier)
})
}
deinit { for token in tokens { center.removeObserver(token) } }
}
private static let coordinator = Coordinator()
static var beliefs: [pid_t: Belief] { coordinator.beliefs }
@MainActor private(set) static var lastPreparedWindow: (pid: pid_t, window: Window?)?
private static let applicationObserver = ApplicationLifecycleObserver(
center: NSWorkspace.shared.notificationCenter, coordinator: coordinator,
onFrontmost: { FocusEventMonitor.shared.observeRealFrontmost(pid: $0) }
)
@MainActor
@discardableResult
static func prepareInput(
pid: pid_t, window: WindowGeometry.Window? = nil,
beforeFocus: (@MainActor (FocusEventMonitor.RegistrationReceipt) async throws -> Void)? = nil
) async throws -> FocusEventMonitor.RegistrationReceipt {
try Task.checkCancellation()
_ = applicationObserver
let geometry = window ?? WindowGeometry.frontmostWindow(pid: pid)
let context = geometry.map {
Window(id: $0.id, bounds: $0.bounds, activationPoint: activationPoint(pid: pid, window: $0))
}
lastPreparedWindow = (pid, context)
let monitor = FocusEventMonitor.shared
return try await prepareInput(pid: pid, window: context, monitor: monitor, coordinator: coordinator, runtime: Runtime(
identity: { AXTree.currentProcessIdentity(pid: $0) },
isActive: { NSWorkspace.shared.frontmostApplication?.processIdentifier == $0 },
hasFocus: { monitor.isAppCurrentlyFocused(pid: $0) },
acceptsInput: { acceptsInput(pid: $0) },
post: { establishment, pid, window in
switch establishment {
case .none: return true
case .returnFocus: return post(.keyFocusReturned, to: pid)
case .activate:
guard let events = activationEvents(window: window) else { return false }
for event in events { WindowTargetedEvent.post(event, to: pid) }
return true
}
},
pause: { try await Task.sleep(for: .milliseconds(100)) }
), beforeFocus: beforeFocus)
}
/// Register protection before any preparatory pointer event, and preserve
/// that exact receipt across both the pointer delay and focus establishment.
@MainActor
static func prepareInput(
pid: pid_t, window: Window?, monitor: FocusEventMonitor,
coordinator: Coordinator, runtime: Runtime,
beforeFocus: (@MainActor (FocusEventMonitor.RegistrationReceipt) async throws -> Void)? = nil
) async throws -> FocusEventMonitor.RegistrationReceipt {
try Task.checkCancellation()
guard monitor.register(pid: pid, onFocusChanged: { hasFocus in
coordinator.observeFocus(pid: pid, hasFocus: hasFocus)
}) else {
throw CUError("focus_monitor_unavailable", "Cannot observe target focus safely. No input was sent.")
}
guard let receipt = monitor.registrationReceipt(pid: pid) else {
throw CUError("focus_monitor_interrupted", "Focus monitoring changed while preparing input.")
}
try validate(receipt, monitor: monitor)
try await beforeFocus?(receipt)
try Task.checkCancellation()
try validate(receipt, monitor: monitor)
var protectedRuntime = runtime
protectedRuntime.validateContinuity = {
try runtime.validateContinuity()
try validate(receipt, monitor: monitor)
}
try await coordinator.prepare(pid: pid, window: window, runtime: protectedRuntime)
return receipt
}
static func validate(
_ receipt: FocusEventMonitor.RegistrationReceipt?,
monitor: FocusEventMonitor = .shared
) throws {
guard let receipt, monitor.isRegistrationCurrent(receipt) else {
throw CUError("focus_monitor_interrupted", "Focus monitoring changed before input could be sent.")
}
}
/// The reference's window-bound AppKit activation protocol. The 0xc0000
/// bits here are not physical keyboard modifiers to press or hold.
static func activationEvents(window: Window?) -> [CGEvent]? {
// Codex uses app-level activation when AXActivationPoint is unsupported
// or unavailable. Keeping the window number here changes the protocol.
let activationWindow = window.flatMap { $0.activationPoint == nil ? nil : $0 }
let windowID = activationWindow?.id ?? kCGNullWindowID
guard let event = notificationEvent(
.appActivated, windowID: windowID,
flags: windowID == kCGNullWindowID ? [] : NSEvent.ModifierFlags(rawValue: 0xc0000)
) else { return nil }
var events = [event]
if let window = activationWindow, let point = window.resolvedActivationPoint,
window.id != kCGNullWindowID, point.x.isFinite, point.y.isFinite {
// Use only the window's explicit AXActivationPoint. A guessed center
// could activate an unrelated or destructive control. Custom-shell
// apps can supply an out-of-content point (NetEase: -1, screenH+1);
// the reference still delivers that activation click to this PID
// and window, never through the physical pointer or a hit-test.
let strokes: [(CGEventType, NSEvent.EventType, Int)] = [
(.leftMouseDown, .leftMouseDown, 1), (.leftMouseUp, .leftMouseUp, 2),
]
for (type, nsType, number) in strokes {
guard let mouse = WindowTargetedEvent.makeMouseEvent(
type: type, nsType: nsType, point: point, button: .left,
clickCount: 1, windowID: window.id, windowBounds: window.bounds,
eventNumber: number
) else { return nil }
events.append(mouse)
}
}
return events
}
static func notificationEvent(
_ notification: Notification,
windowID: CGWindowID = kCGNullWindowID,
flags: NSEvent.ModifierFlags = []
) -> CGEvent? {
guard let carrier = notification.carrierEventType else { return nil }
return NSEvent.otherEvent(
with: carrier, location: .zero, modifierFlags: flags,
timestamp: 0, windowNumber: Int(windowID), context: nil,
subtype: notification.subtype, data1: 0, data2: 0
)?.cgEvent
}
@discardableResult
static func post(_ notification: Notification, to pid: pid_t) -> Bool {
guard pid > 0,
let carrier = notification.carrierEventType,
let event = NSEvent.otherEvent(
with: carrier,
location: .zero,
modifierFlags: [],
timestamp: 0,
windowNumber: 0,
context: nil,
subtype: notification.subtype,
data1: 0,
data2: 0
),
let cgEvent = event.cgEvent
else { return false }
cgEvent.postToPid(pid)
guard pid > 0, let event = notificationEvent(notification) else { return false }
WindowTargetedEvent.post(event, to: pid)
return true
}
/// Make `pid` behave as a focused application for the actions that follow,
/// leaving the user's foreground exactly where it was.
///
/// Sent ONLY when the target is not already the active application. Codex
/// gates it the same way — its enforcer holds `applicationIsActive` beside
/// `applicationBelievesItIsActive` and re-sends only when the two disagree —
/// and the first version of this file described that gate in its own
/// documentation while shipping without it.
///
/// Sending it unconditionally is not a harmless extra. The notification
/// names `windowNumber: 0`, so telling an app that already owns a key
/// window that "key focus returned" to no window at all is at best noise
/// and quite possibly an instruction to let go of it. Measured on a session
/// where the target's traffic lights stayed fully coloured — the app was
/// active and its window was key throughout — and every one of nine
/// window-bound clicks was discarded anyway.
/// Both halves are required, and sending one was the other half of the bug.
///
/// Codex's enforcer tracks two separate beliefs — `applicationBelievesItIsActive`
/// and `applicationBelievesItHasFocus` — and its `enforceActiveState` posts
/// two notifications to establish them: the key-focus one, then
/// `appActivated` (hardcoded type 13, subtype 1). This only ever sent the
/// first. Telling a window that focus returned, to an application that does
/// not believe it is active, leaves the input routing exactly where it was.
///
/// Order matches the reference: focus, then activation.
@discardableResult
static func enforceActiveState(pid: pid_t) -> Bool {
// Register before reserving belief so a real activation that happens
// later is observed even when no CU request runs while the app is
// actually frontmost.
_ = applicationLifecycleObserver
let runtime = EnforcementRuntime(
applicationIsActive: isActiveApplication(pid),
target: BeliefTarget(
processIdentity: currentProcessIdentity(pid: pid)
),
post: { notification, targetPid in
post(notification, to: targetPid)
}
)
let reserved = beliefs.withLock { state in
state.beginEnforcement(
pid: pid,
applicationIsActive: runtime.applicationIsActive,
target: runtime.target
)
}
guard reserved else { return false }
guard postEnforcementPair(pid: pid, runtime: runtime) else {
beliefs.withLock {
$0.cancelEnforcement(
pid: pid,
expectedTarget: runtime.target
)
}
private static func acceptsInput(pid: pid_t) -> Bool {
let app = AXUIElementCreateApplication(pid)
AXUIElementSetMessagingTimeout(app, 0.1)
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(app, kAXFrontmostAttribute as CFString, &value) == .success else {
return false
}
return true
return (value as? NSNumber)?.boolValue == true
}
/// Testable transition used by the live wrapper above. Keeping the
/// notification sink beside the belief mutation lets tests drive the same
/// success, deduplication and rollback path production uses.
@discardableResult
static func enforceActiveState(
pid: pid_t,
state: inout BeliefState,
runtime: EnforcementRuntime
) -> Bool {
guard state.beginEnforcement(
pid: pid,
applicationIsActive: runtime.applicationIsActive,
target: runtime.target
) else { return false }
guard postEnforcementPair(pid: pid, runtime: runtime) else {
state.cancelEnforcement(pid: pid, expectedTarget: runtime.target)
return false
}
return true
@MainActor
private static func activationPoint(pid: pid_t, window: WindowGeometry.Window) -> CGPoint?? {
guard let element = try? AXTree.snapshotWindowElement(pid: pid, windowID: window.id) else { return nil }
AXUIElementSetMessagingTimeout(element, 0.1)
var raw: CFTypeRef?
let error = AXUIElementCopyAttributeValue(element, "AXActivationPoint" as CFString, &raw)
return decodeActivationPoint(error: error, raw: raw)
}
/// Post outside the belief lock. AppKit event construction and delivery
/// are external calls; keeping an unfair lock held across them risks
/// re-entrancy and makes every other focus transition wait unnecessarily.
private static func postEnforcementPair(
pid: pid_t,
runtime: EnforcementRuntime
) -> Bool {
let focused = runtime.post(.keyFocusReturned, pid)
let activated = runtime.post(.appActivated, pid)
guard focused && activated else {
if focused || activated {
// Do not leave a half-established belief behind when AppKit
// could construct only one side of the pair.
if focused { _ = runtime.post(.lostKeyFocus, pid) }
_ = runtime.post(.appDeactivated, pid)
}
return false
}
return true
static func decodeActivationPoint(error: AXError, raw: CFTypeRef?) -> CGPoint?? {
guard error == .success, let raw else { return nil }
guard CFGetTypeID(raw) == AXValueGetTypeID() else { return .some(nil) }
var point = CGPoint.zero
guard AXValueGetValue(unsafeDowncast(raw, to: AXValue.self), .cgPoint, &point),
point.x.isFinite, point.y.isFinite else { return .some(nil) }
return .some(point)
}
/// Reality, as opposed to what the target has been told.
private static func isActiveApplication(_ pid: pid_t) -> Bool {
NSWorkspace.shared.frontmostApplication?.processIdentifier == pid
}
private static func currentProcessIdentity(pid: pid_t) -> AXTreeProcessIdentity? {
guard let application = NSRunningApplication(processIdentifier: pid) else {
return nil
}
return AXTreeProcessIdentity(
bundleID: application.bundleIdentifier,
executablePath: application.executableURL?.path,
launchTime: application.launchDate?.timeIntervalSinceReferenceDate
)
}
private static func observeRealActivation(pid: pid_t) {
beliefs.withLock { $0.observeRealActivation(pid: pid) }
}
/// NSWorkspace is the observable edge the old PID-only cache lacked. If a
/// user brings a synthetic target to the real foreground and then leaves
/// it, the real deactivate invalidates what the app was told. Clearing the
/// belief on activation makes the next background request establish a new
/// pair instead of trusting stale session state.
private final class ApplicationLifecycleObserver: @unchecked Sendable {
private let activationToken: NSObjectProtocol
init() {
activationToken = NSWorkspace.shared.notificationCenter.addObserver(
forName: NSWorkspace.didActivateApplicationNotification,
object: nil,
queue: .main
) { notification in
let application = notification.userInfo?[NSWorkspace.applicationUserInfoKey]
as? NSRunningApplication
guard let pid = application?.processIdentifier,
NSWorkspace.shared.frontmostApplication?.processIdentifier == pid
else { return }
SyntheticWindowFocus.observeRealActivation(pid: pid)
}
}
deinit {
NSWorkspace.shared.notificationCenter.removeObserver(activationToken)
}
}
private static let applicationLifecycleObserver = ApplicationLifecycleObserver()
/// Tell every app we lied to that it is no longer active.
///
/// Scoped to the session, not the action: re-sending per click would cancel
/// the focus we just established before the target's run loop had used it,
/// and Codex tears its enforcer down the same way — at
/// `deactivateFocusEnforcer`, not after each event.
///
/// Without this the belief outlives its usefulness. The target goes on
/// acting focused long after we stop driving it: a caret keeps blinking in
/// an app the user is not in, and the next real click there arrives at a
/// window that never learned it had lost focus.
@MainActor
static func relinquishAll() {
let targets = beliefs.withLock { $0.drain() }
for (pid, target) in targets {
// Do not aim teardown at a recycled PID, or tell an app the user is
// genuinely using that it lost focus.
guard !isActiveApplication(pid),
currentProcessIdentity(pid: pid) == target.processIdentity
else { continue }
FocusEventMonitor.shared.unregisterAll()
let targets = coordinator.drain()
for (pid, belief) in targets {
guard NSWorkspace.shared.frontmostApplication?.processIdentifier != pid,
AXTree.currentProcessIdentity(pid: pid) == belief.identity else { continue }
post(.lostKeyFocus, to: pid)
post(.appDeactivated, to: pid)
}
}
/// Written from the daemon's request queue and read on teardown. The lock
/// also makes the reserve-before-send transition atomic if a future caller
/// reaches it off the main actor.
private static let beliefs = OSAllocatedUnfairLock(initialState: BeliefState())
}
@@ -42,6 +42,28 @@ struct WindowCaptureStreamFrame: Equatable, Sendable {
let receivedUptime: TimeInterval
}
/// Metadata only: inspecting stream health never fetches or serializes pixels.
struct WindowCaptureStreamSourceDiagnostic: Equatable, Sendable {
let hasFailed: Bool
let latestFrameSequence: UInt64?
let latestFrameReceivedUptime: TimeInterval?
let sampleCount: UInt64
let latestSampleStatus: Int?
let latestSampleReceivedUptime: TimeInterval?
}
struct WindowCaptureStreamDiagnostic: Equatable, Sendable {
let generation: UInt64
let activeKey: WindowCaptureStreamKey?
let startingKey: WindowCaptureStreamKey?
let hasFailed: Bool?
let latestFrameSequence: UInt64?
let latestFrameAgeSeconds: TimeInterval?
let sampleCount: UInt64?
let latestSampleStatus: Int?
let latestSampleAgeSeconds: TimeInterval?
}
enum WindowCaptureFrameStatusPolicy {
static func accepts(_ status: SCFrameStatus) -> Bool {
status == .complete || status == .started
@@ -71,6 +93,7 @@ protocol WindowCaptureProviding: AnyObject {
protocol WindowCaptureStreamSource: AnyObject {
var targetKey: WindowCaptureStreamKey { get }
var hasFailed: Bool { get }
func sampleDiagnostic() -> WindowCaptureStreamSourceDiagnostic
func start() async throws
func latestFrame() -> WindowCaptureStreamFrame?
func retire()
@@ -271,6 +294,23 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
var activeGenerationForTesting: UInt64? { active?.generation }
var activeKeyForTesting: WindowCaptureStreamKey? { active?.source.targetKey }
func diagnostic(
now: TimeInterval = ProcessInfo.processInfo.systemUptime
) -> WindowCaptureStreamDiagnostic {
let sample = (active ?? starting)?.source.sampleDiagnostic()
return WindowCaptureStreamDiagnostic(
generation: generation,
activeKey: active?.source.targetKey,
startingKey: starting?.source.targetKey,
hasFailed: sample?.hasFailed,
latestFrameSequence: sample?.latestFrameSequence,
latestFrameAgeSeconds: sample?.latestFrameReceivedUptime.map { max(0, now - $0) },
sampleCount: sample?.sampleCount,
latestSampleStatus: sample?.latestSampleStatus,
latestSampleAgeSeconds: sample?.latestSampleReceivedUptime.map { max(0, now - $0) }
)
}
private func source(
for target: WindowCaptureStreamTarget
) async -> (any WindowCaptureStreamSource)? {
@@ -373,6 +413,10 @@ final class ScreenCaptureKitWindowStreamSource: WindowCaptureStreamSource {
var hasFailed: Bool { output.hasFailed }
func sampleDiagnostic() -> WindowCaptureStreamSourceDiagnostic {
output.sampleDiagnostic()
}
func latestFrame() -> WindowCaptureStreamFrame? {
output.latestFrame()
}
@@ -509,12 +553,15 @@ final class ScreenCaptureKitWindowStreamSource: WindowCaptureStreamSource {
/// `.started` is the first generated frame after start and `.complete` is a
/// later generated frame. Idle, blank, suspended, and stopped notifications
/// never advance sequence or satisfy a post-mutation freshness watermark.
private final class WindowCaptureStreamMailbox: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable {
final class WindowCaptureStreamMailbox: NSObject, SCStreamOutput, SCStreamDelegate, @unchecked Sendable {
private let lock = NSLock()
private var accepting = true
private var failed = false
private var sequence: UInt64 = 0
private var frame: WindowCaptureStreamFrame?
private var sampleCount: UInt64 = 0
private var latestSampleStatus: Int?
private var latestSampleReceivedUptime: TimeInterval?
var hasFailed: Bool {
lock.lock()
@@ -528,6 +575,28 @@ private final class WindowCaptureStreamMailbox: NSObject, SCStreamOutput, SCStre
return accepting ? frame : nil
}
func sampleDiagnostic() -> WindowCaptureStreamSourceDiagnostic {
lock.lock()
defer { lock.unlock() }
return WindowCaptureStreamSourceDiagnostic(
hasFailed: failed,
latestFrameSequence: frame?.sequence,
latestFrameReceivedUptime: frame?.receivedUptime,
sampleCount: sampleCount,
latestSampleStatus: latestSampleStatus,
latestSampleReceivedUptime: latestSampleReceivedUptime
)
}
func recordSampleStatus(_ status: SCFrameStatus, receivedUptime: TimeInterval) {
lock.lock()
defer { lock.unlock() }
guard accepting else { return }
sampleCount &+= 1
latestSampleStatus = status.rawValue
latestSampleReceivedUptime = receivedUptime
}
func invalidate() {
lock.lock()
accepting = false
@@ -546,6 +615,7 @@ private final class WindowCaptureStreamMailbox: NSObject, SCStreamOutput, SCStre
let status = Self.frameStatus(sampleBuffer) else {
return
}
recordSampleStatus(status, receivedUptime: ProcessInfo.processInfo.systemUptime)
if WindowCaptureFrameStatusPolicy.marksFailure(status) {
markFailed()
return
@@ -11,12 +11,33 @@ import Foundation
/// Window *names* would require Screen Recording; these three do not, so this
/// works before any capture grant exists.
enum WindowGeometry {
struct Window: Equatable {
struct Window: Equatable, Sendable {
let id: CGWindowID
let bounds: CGRect
let ownerPid: pid_t
}
/// Revalidate the original window after an async focus/pacing boundary;
/// never replace it with whichever window is currently above the pointer.
static func window(
id: CGWindowID, pid: pid_t,
windowList: () -> [[CFString: Any]]? = systemWindowList
) -> Window? {
guard let list = windowList(),
let info = list.first(where: {
($0[kCGWindowNumber] as? Int) == Int(id)
&& ($0[kCGWindowOwnerPID] as? pid_t) == pid
&& ($0[kCGWindowLayer] as? Int) == 0
}),
let raw = info[kCGWindowBounds] as? [String: CGFloat],
let x = raw["X"], let y = raw["Y"],
let width = raw["Width"], let height = raw["Height"],
x.isFinite, y.isFinite, width.isFinite, height.isFinite,
width > 0, height > 0 else { return nil }
let bounds = CGRect(x: x, y: y, width: width, height: height)
return Window(id: id, bounds: bounds, ownerPid: pid)
}
/// Front-most ordinary window containing `point`.
///
/// - Parameter pid: when given, only that process's windows are considered.
@@ -383,6 +383,36 @@ struct ClientAttestationTests {
}
}
@Test("focus diagnostics require an authenticated daemon and do not open unknown commands")
func focusMonitorDiagnosticPolicy() {
let command = "focus_monitor_state"
#expect(HelperClientPolicy.isDaemonCommandAllowed(command))
#expect(
HelperClientPolicy.authorizeDaemon(
peer: cli, ancestors: [server, host], helper: helper
) == .allow
)
var unsignedPeer = cli
unsignedPeer.signatureValid = false
#expect(
HelperClientPolicy.authorizeDaemon(
peer: unsignedPeer, ancestors: [server, host], helper: helper
) == .deny
)
#expect(
HelperClientPolicy.authorizeOneShot(
command: command,
processChain: [helperProcess(pid: 400, parentPID: cli.pid), cli, server, host],
helper: helper
) == .deny
)
for unknown in ["", "unknown_command", "focus_monitor_state_extra", "focus_monitor_state ", "FOCUS_MONITOR_STATE"] {
#expect(!HelperClientPolicy.isDaemonCommandAllowed(unknown))
}
}
@Test("live process attestation reads a stable executable identity")
func liveSelfAttestationHook() throws {
let current = try ProcessAttestor.attest(pid: getpid())
@@ -0,0 +1,202 @@
import AppKit
import CoreGraphics
import XCTest
@testable import cc_haha_computer_use
final class ClipboardPasteReceiptTests: XCTestCase {
@MainActor
func testPasteWaitsForARealReadBeyondTheOld180MillisecondWindow() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
let lease = ClipboardLease(pasteboard: fixture.board)
var returned = false
var reader: Task<Void, Never>?
try await ClipboardPasteReceipt.perform(text: "temporary", lease: lease) { validate in
try await fixture.sendPaste(validate)
reader = Task { @MainActor in
try? await Task.sleep(for: .milliseconds(240))
XCTAssertFalse(returned, "paste cannot complete before its promised data is read")
XCTAssertTrue(lease.temporaryWriteIsCurrent())
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
}
}
returned = true
await reader?.value
let diagnostic = try XCTUnwrap(ClipboardPasteReceipt.lastDiagnostic)
XCTAssertEqual(diagnostic.status, "completed")
XCTAssertTrue(diagnostic.dataRequested)
XCTAssertTrue(diagnostic.dataSupplied)
let readElapsed = try XCTUnwrap(diagnostic.readElapsedMilliseconds)
XCTAssertGreaterThan(readElapsed, 180)
XCTAssertGreaterThanOrEqual(diagnostic.elapsedMilliseconds - readElapsed, 90)
XCTAssertTrue(diagnostic.ownedBeforeRestore)
XCTAssertTrue(diagnostic.restored)
XCTAssertEqual(fixture.board.string(forType: .string), "original")
XCTAssertEqual(fixture.events.map(\.type), [.flagsChanged, .keyDown, .flagsChanged, .keyUp])
}
@MainActor
func testNoReadThrowsInsteadOfReportingSuccessfulPaste() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
do {
try await ClipboardPasteReceipt.perform(
text: "temporary", lease: ClipboardLease(pasteboard: fixture.board),
timeout: .milliseconds(30), sendPaste: fixture.sendPaste
)
XCTFail("posting Command-V is not confirmation that its data was read")
} catch let error as CUError {
XCTAssertEqual(error.code, "clipboard_read_timeout")
}
let diagnostic = try XCTUnwrap(ClipboardPasteReceipt.lastDiagnostic)
XCTAssertEqual(diagnostic.status, "clipboard_read_timeout")
XCTAssertFalse(diagnostic.dataSupplied)
XCTAssertNil(diagnostic.readElapsedMilliseconds)
XCTAssertTrue(diagnostic.restored)
XCTAssertEqual(fixture.events.count, 4)
XCTAssertEqual(fixture.board.string(forType: .string), "original")
}
@MainActor
func testExternalCopyWhileWaitingWinsAndIsNotSuccessfulConsumption() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
do {
try await ClipboardPasteReceipt.perform(
text: "temporary", lease: ClipboardLease(pasteboard: fixture.board),
timeout: .milliseconds(30)
) { validate in
try await fixture.sendPaste(validate)
fixture.board.clearContents()
XCTAssertTrue(fixture.board.setString("new external copy", forType: .string))
}
XCTFail("a replacement pasteboard is not a read receipt")
} catch let error as CUError {
XCTAssertEqual(error.code, "clipboard_changed")
}
let diagnostic = try XCTUnwrap(ClipboardPasteReceipt.lastDiagnostic)
XCTAssertEqual(diagnostic.status, "clipboard_changed")
XCTAssertFalse(diagnostic.dataSupplied)
XCTAssertFalse(diagnostic.ownedBeforeRestore)
XCTAssertFalse(diagnostic.restored)
XCTAssertEqual(fixture.board.string(forType: .string), "new external copy")
}
@MainActor
func testIdenticalTextOnANewPasteRequiresANewReadReceipt() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
try await ClipboardPasteReceipt.perform(
text: "same temporary text", lease: ClipboardLease(pasteboard: fixture.board)
) { validate in
try await fixture.sendPaste(validate)
XCTAssertEqual(fixture.board.string(forType: .string), "same temporary text")
}
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.dataSupplied, true)
do {
try await ClipboardPasteReceipt.perform(
text: "same temporary text", lease: ClipboardLease(pasteboard: fixture.board),
timeout: .milliseconds(30), sendPaste: fixture.sendPaste
)
XCTFail("the previous operation's receipt must not satisfy a new paste")
} catch let error as CUError {
XCTAssertEqual(error.code, "clipboard_read_timeout")
}
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.dataSupplied, false)
XCTAssertEqual(fixture.events.count, 8)
XCTAssertEqual(fixture.board.string(forType: .string), "original")
}
@MainActor
func testCancellationAfterPostingStillLetsThePendingReadFinishBeforeRestore() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
let lease = ClipboardLease(pasteboard: fixture.board)
var reader: Task<Void, Never>?
let task = Task { @MainActor in
try await ClipboardPasteReceipt.perform(text: "temporary", lease: lease) { validate in
try await fixture.sendPaste(validate)
reader = Task { @MainActor in
try? await Task.sleep(for: .milliseconds(240))
XCTAssertTrue(lease.temporaryWriteIsCurrent())
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
}
withUnsafeCurrentTask { $0?.cancel() }
}
}
do {
try await task.value
XCTFail("cancellation must still reach the caller")
} catch is CancellationError {}
await reader?.value
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.status, "cancelled")
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.dataSupplied, true)
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.restored, true)
XCTAssertEqual(fixture.events.count, 4, "waiting or cancellation must not resend Command-V")
XCTAssertEqual(fixture.board.string(forType: .string), "original")
}
@MainActor
func testAlreadyCancelledPasteNeverWritesOrPosts() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
let originalCount = fixture.board.changeCount
let task = Task { @MainActor in
withUnsafeCurrentTask { $0?.cancel() }
try await ClipboardPasteReceipt.perform(
text: "temporary", lease: ClipboardLease(pasteboard: fixture.board),
sendPaste: fixture.sendPaste
)
}
do {
try await task.value
XCTFail("already cancelled")
} catch is CancellationError {}
XCTAssertTrue(fixture.events.isEmpty)
XCTAssertEqual(fixture.board.changeCount, originalCount)
XCTAssertEqual(fixture.board.string(forType: .string), "original")
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.status, "cancelled")
}
@MainActor
func testFinishedCallbackAloneNeverCountsAsRead() async throws {
let fixture = PasteReceiptFixture()
defer { fixture.close() }
let lease = ClipboardLease(pasteboard: fixture.board)
defer { lease.restoreIfUnchanged() }
let receipt = try lease.writeTemporaryStringWithReceipt("temporary")
receipt.pasteboardFinishedWithDataProvider(fixture.board)
do {
try await receipt.waitForRead(timeout: .milliseconds(20), ownsClipboard: lease.temporaryWriteIsCurrent)
XCTFail("finished can mean ownership was relinquished, not consumption")
} catch let error as CUError {
XCTAssertEqual(error.code, "clipboard_read_timeout")
}
}
}
/// The real promised-data provider, paste orchestration and keyboard factory
/// run together. Only focus preparation and actual PID event delivery are fake.
@MainActor
private final class PasteReceiptFixture {
let board = NSPasteboard.withUniqueName()
var events: [CGEvent] = []
init() {
board.clearContents()
XCTAssertTrue(board.setString("original", forType: .string))
}
func sendPaste(_ validate: @MainActor () throws -> Void) async throws {
try await KeyboardEventBurst.dispatch(
chords: KeyMapping.parse("cmd+v"), prepare: { await Task.yield() },
readFlagsState: { _ in [] }, validateBeforePosting: validate,
post: { events.append($0) }
)
}
func close() { board.releaseGlobally() }
}
@@ -111,4 +111,59 @@ final class CursorMotionStateTests: XCTestCase {
XCTFail("unexpected error: \(error)")
}
}
@MainActor
func testIndexedActionAwaitsAsyncMutationBeforeReturningItsCommittedResult() async {
var events: [String] = []
let result = await CursorIndexedActionGate.perform(
moveForAction: { events.append("moved") },
recheckStaleness: { events.append("validated") },
mutate: {
events.append("mutation-started")
let resultTask = Task { @MainActor in
events.append("async-result-ready")
return "committed"
}
let result = await resultTask.value
events.append("mutation-finished")
return result
}
)
events.append("returned")
XCTAssertEqual(result, "committed")
XCTAssertEqual(events, [
"moved", "validated", "mutation-started", "async-result-ready",
"mutation-finished", "returned",
])
}
@MainActor
func testIndexedActionPropagatesFailureFromSuspendedMutationWithoutCommitting() async {
enum ExpectedError: Error { case mutationFailed }
var events: [String] = []
do {
try await CursorIndexedActionGate.perform(
moveForAction: { events.append("moved") },
recheckStaleness: { events.append("validated") },
mutate: {
events.append("mutation-started")
let failureTask = Task { @MainActor in
events.append("async-failure")
throw ExpectedError.mutationFailed
}
try await failureTask.value
}
)
XCTFail("a failed async mutation must not commit")
} catch ExpectedError.mutationFailed {
events.append("failed")
} catch {
XCTFail("unexpected error: \(error)")
}
XCTAssertEqual(events, [
"moved", "validated", "mutation-started", "async-failure", "failed",
])
}
}
@@ -0,0 +1,599 @@
import Foundation
import os
import XCTest
@testable import cc_haha_computer_use
final class FocusEventMonitorTests: XCTestCase {
private final class FakeStream: FocusEventMonitor.Stream, @unchecked Sendable {
var starts = 0
var stops = 0
var startsSuccessfully = true
var keyboardStartsSuccessfully = true
var interruptDuringStart: String?
var addHook: (@Sendable (pid_t) -> Bool)?
private var stopped = false
var receive: (@Sendable (FocusEventMonitor.Event) -> FocusEventMonitor.Disposition)?
var interrupted: (@Sendable (String) -> Void)?
func start(
receive: @escaping @Sendable (FocusEventMonitor.Event) -> FocusEventMonitor.Disposition,
interrupted: @escaping @Sendable (String) -> Void
) -> Bool {
starts += 1
stopped = false
self.receive = receive
self.interrupted = interrupted
if let interruptDuringStart { interrupted(interruptDuringStart) }
return startsSuccessfully
}
func addProtectedPID(_ pid: pid_t) -> Bool { addHook?(pid) ?? keyboardStartsSuccessfully }
func stop() {
guard !stopped else { return }
stopped = true
stops += 1
}
}
private func event(
focus: pid_t = 42, subtype: Int64 = 0xf102,
source: pid_t = 0, target: pid_t = 901, type: UInt32 = 21
) -> FocusEventMonitor.Event {
.init(type: type, subtype: subtype, sourcePID: source,
targetPID: target, focusPID: focus, focusToken: 17)
}
private func monitor(
_ stream: FakeStream,
releaseFocus: @escaping @Sendable (UInt32) -> Bool = { _ in true },
readRealFrontmost: @escaping @Sendable () -> pid_t? = { 9 },
readProcessIdentity: @escaping @Sendable (pid_t) -> FocusEventMonitor.ProcessIdentity? = {
.init(executablePath: "/test/\($0)", launchTime: 1)
}
) -> FocusEventMonitor {
FocusEventMonitor(
helperPID: 700,
readInitialFocus: { 9 },
isFocusObserver: { $0 == 901 },
makeStream: { stream },
releaseFocus: releaseFocus,
readRealFrontmost: readRealFrontmost,
isOrdinaryApp: { _ in true },
readProcessIdentity: readProcessIdentity
)
}
func testOnlyRegistrationStartsObservationAndInitialFocusIsNotSynthetic() {
let stream = FakeStream()
let monitor = monitor(stream)
XCTAssertEqual(stream.starts, 0)
XCTAssertFalse(monitor.isAppCurrentlyFocused(pid: 9))
XCTAssertTrue(monitor.register(pid: 42) { _ in })
XCTAssertTrue(monitor.isAppCurrentlyFocused(pid: 9))
XCTAssertFalse(monitor.isAppCurrentlyFocused(pid: 42))
XCTAssertTrue(monitor.register(pid: 77) { _ in })
XCTAssertEqual(stream.starts, 1)
}
func testUnreadableViewBridgeNameRequiresTheExactSystemExecutable() {
let systemPath = "/System/Library/PrivateFrameworks/ViewBridge.framework/Versions/A/XPCServices/ViewBridgeAuxiliary.xpc/Contents/MacOS/ViewBridgeAuxiliary"
XCTAssertTrue(FocusEventMonitor.isViewBridgeProcess(name: "ViewBridgeAuxiliary", executablePath: nil))
XCTAssertTrue(FocusEventMonitor.isViewBridgeProcess(name: nil, executablePath: systemPath))
XCTAssertFalse(FocusEventMonitor.isViewBridgeProcess(name: nil, executablePath: nil))
XCTAssertFalse(FocusEventMonitor.isViewBridgeProcess(
name: nil, executablePath: "/Applications/Fake.app/Contents/MacOS/ViewBridgeAuxiliary"
))
XCTAssertFalse(FocusEventMonitor.isViewBridgeProcess(name: nil, executablePath: systemPath + "Fake"))
XCTAssertFalse(FocusEventMonitor.isViewBridgeProcess(name: "DifferentProcess", executablePath: systemPath))
XCTAssertFalse(FocusEventMonitor.isViewBridgeProcess(name: "ViewBridgeAuxili", executablePath: systemPath))
}
func testSystemTransitionNotifiesLossAndGainInBothDirections() {
let stream = FakeStream()
let monitor = monitor(stream)
let changes = OSAllocatedUnfairLock(initialState: [String]())
XCTAssertTrue(monitor.register(pid: 9) { value in changes.withLock { $0.append("9:\(value)") } })
XCTAssertTrue(monitor.register(pid: 42) { value in changes.withLock { $0.append("42:\(value)") } })
_ = stream.receive?(event())
_ = stream.receive?(event())
_ = stream.receive?(event(focus: 9))
XCTAssertEqual(changes.withLock { $0 }, ["9:false", "42:true", "42:false", "9:true"])
XCTAssertTrue(monitor.isAppCurrentlyFocused(pid: 9))
}
func testOnlyViewBridgeFocusChangesRewriteSystemFocusRegardlessOfSource() {
let stream = FakeStream()
let monitor = monitor(stream)
let changes = OSAllocatedUnfairLock(initialState: [Bool]())
XCTAssertTrue(monitor.register(pid: 42) { value in changes.withLock { $0.append(value) } })
for notification in [
event(source: 700), event(target: 42), event(type: 13),
event(subtype: 0x8000, target: 42), event(subtype: 0x4000),
event(subtype: 2), event(subtype: 0xf107), event(focus: 0),
] {
let recognized = notification.type == 21 && notification.subtype == 0xf102
&& notification.targetPID == 901 && notification.focusPID > 0
_ = stream.receive?(notification)
XCTAssertEqual(monitor.isAppCurrentlyFocused(pid: 42), recognized)
if recognized { _ = stream.receive?(event(focus: 9)) }
}
XCTAssertEqual(changes.withLock { $0 }, [true, false])
XCTAssertTrue(monitor.isAppCurrentlyFocused(pid: 9))
XCTAssertFalse(monitor.isAppCurrentlyFocused(pid: 42))
}
func testTimeoutInvalidatesBeliefAndRegistrationStartsFreshObservation() {
let stream = FakeStream()
let monitor = monitor(stream)
let changes = OSAllocatedUnfairLock(initialState: [Bool]())
let callback: FocusEventMonitor.FocusChanged = { value in changes.withLock { $0.append(value) } }
XCTAssertTrue(monitor.register(pid: 42, onFocusChanged: callback))
_ = stream.receive?(event())
let oldReceive = stream.receive
stream.interrupted?("event_tap_timeout")
XCTAssertFalse(monitor.diagnostic.available)
XCTAssertFalse(monitor.isAppCurrentlyFocused(pid: 42))
XCTAssertEqual(monitor.diagnostic.reason, "event_tap_timeout")
XCTAssertTrue(monitor.register(pid: 42, onFocusChanged: callback))
XCTAssertEqual(stream.starts, 2)
XCTAssertEqual(stream.stops, 1)
_ = oldReceive?(event())
XCTAssertTrue(monitor.isAppCurrentlyFocused(pid: 9))
XCTAssertEqual(changes.withLock { $0 }, [true, false])
_ = stream.receive?(event())
XCTAssertEqual(changes.withLock { $0 }, [true, false, true])
}
func testCreationFailureReturnsFalseAndCanRetry() {
let stream = FakeStream()
stream.startsSuccessfully = false
let monitor = monitor(stream)
XCTAssertFalse(monitor.register(pid: 42) { _ in })
XCTAssertFalse(monitor.diagnostic.available)
XCTAssertFalse(monitor.isAppCurrentlyFocused(pid: 9))
stream.startsSuccessfully = true
XCTAssertTrue(monitor.register(pid: 42) { _ in })
XCTAssertTrue(monitor.diagnostic.available)
}
func testUnregisterPreventsLateCallbacksAndCanRestart() {
let stream = FakeStream()
let monitor = monitor(stream)
let changes = OSAllocatedUnfairLock(initialState: [Bool]())
XCTAssertTrue(monitor.register(pid: 42) { value in changes.withLock { $0.append(value) } })
let oldReceive = stream.receive
let oldInterruption = stream.interrupted
monitor.unregisterAll()
_ = oldReceive?(event())
oldInterruption?("late_failure")
XCTAssertEqual(changes.withLock { $0 }, [])
XCTAssertEqual(monitor.diagnostic.reason, "stopped")
XCTAssertFalse(monitor.isAppCurrentlyFocused(pid: 9))
XCTAssertFalse(monitor.register(pid: -1) { _ in })
XCTAssertTrue(monitor.register(pid: 42) { _ in })
XCTAssertEqual(stream.starts, 2)
}
private func consume(
_ policy: inout FocusEventMonitor.ProtectionPolicy,
_ event: FocusEventMonitor.Event, front: pid_t? = 9, observer: Bool = true
) -> FocusEventMonitor.ProtectionPolicy.Effect {
policy.consume(event, helperPID: 700, protectedPIDs: [42],
realFrontmostPID: front, isSystemObserver: observer)
}
func testCompleteStealReleaseAndKeyboardReturnTransaction() {
var policy = FocusEventMonitor.ProtectionPolicy(focusedPID: 9)
XCTAssertEqual(consume(&policy, event(subtype: 0x4000, target: 9)).disposition, .suppress)
let key = event(target: 42, type: 10)
XCTAssertEqual(consume(&policy, key).disposition, .redirect(9))
let focused = consume(&policy, event())
XCTAssertEqual(focused.releaseToken, 17)
XCTAssertEqual(focused.focusChanges.map { $0.0 }, [9])
XCTAssertEqual(focused.focusChanges.map { $0.1 }, [false])
XCTAssertNil(consume(&policy, event()).releaseToken, "a focus transaction is cancelled once")
XCTAssertEqual(consume(&policy, event(subtype: 0x8000, target: 9)).disposition, .suppress)
XCTAssertEqual(consume(&policy, event(subtype: 0x8000, target: 9)).disposition, .pass)
let restored = consume(&policy, event(focus: 9))
XCTAssertEqual(restored.focusChanges.map { $0.0 }, [9])
XCTAssertEqual(restored.focusChanges.map { $0.1 }, [true])
XCTAssertEqual(policy.focusedPID, 9)
_ = consume(&policy, event(subtype: 2))
XCTAssertNil(policy.pending)
XCTAssertEqual(consume(&policy, key).disposition, .pass)
}
func testOnlyMatchedProtectedThiefAndRealVictimCanBeSuppressed() {
for unmatched in [
event(subtype: 0x4000, source: 700, target: 9),
event(focus: 77, subtype: 0x4000, target: 9),
event(subtype: 0x4000, target: 88),
event(subtype: 0x4444, target: 9),
] {
var policy = FocusEventMonitor.ProtectionPolicy(focusedPID: 9)
let matched = unmatched.subtype == 0x4000 && unmatched.focusPID == 42
&& unmatched.targetPID == 9
XCTAssertEqual(consume(&policy, unmatched).disposition, matched ? .suppress : .pass)
XCTAssertEqual(policy.pending != nil, matched)
}
var policy = FocusEventMonitor.ProtectionPolicy(focusedPID: 9)
XCTAssertEqual(consume(&policy, event(subtype: 0x4000, target: 9), front: 42).disposition, .pass)
XCTAssertNil(policy.pending)
}
func testPendingProtectionNeverRedirectsOwnOrUnrelatedInput() {
var policy = FocusEventMonitor.ProtectionPolicy(focusedPID: 9)
_ = consume(&policy, event(subtype: 0x4000, target: 9))
for keyType: UInt32 in [10, 11, 12] {
XCTAssertEqual(consume(&policy, event(source: 700, target: 42, type: keyType)).disposition, .pass)
XCTAssertEqual(consume(&policy, event(target: 77, type: keyType)).disposition, .pass)
XCTAssertEqual(consume(&policy, event(target: 42, type: keyType)).disposition, .redirect(9))
}
XCTAssertEqual(consume(&policy, event(target: 42, type: 1)).disposition, .pass)
XCTAssertEqual(consume(&policy, event(subtype: 0x8000, target: 77)).disposition, .pass)
XCTAssertNil(consume(&policy, event(), observer: false).releaseToken)
XCTAssertEqual(policy.focusedPID, 9)
}
func testRealUserActivationOrUnknownFocusClearsPendingProtection() {
for nextFront: pid_t? in [42, 77, nil] {
var policy = FocusEventMonitor.ProtectionPolicy(focusedPID: 9)
_ = consume(&policy, event(subtype: 0x4000, target: 9))
XCTAssertEqual(consume(&policy, event(target: 42, type: 10), front: nextFront).disposition, .pass)
XCTAssertNil(policy.pending)
}
var policy = FocusEventMonitor.ProtectionPolicy(focusedPID: 9)
_ = consume(&policy, event(subtype: 0x4000, target: 9))
_ = consume(&policy, event(focus: 77))
XCTAssertNil(policy.pending)
}
func testReleaseFailureStopsAutomationButRetainsOnlyTheProvenKeyboardRoute() throws {
let stream = FakeStream()
let releases = OSAllocatedUnfairLock(initialState: [UInt32]())
let monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, isFocusObserver: { $0 == 901 },
makeStream: { stream }, releaseFocus: { token in
releases.withLock { $0.append(token) }; return false
}, readRealFrontmost: { 9 }, isOrdinaryApp: { _ in true },
readProcessIdentity: { .init(executablePath: "/test/\($0)", launchTime: 1) }
)
let changes = OSAllocatedUnfairLock(initialState: [Bool]())
XCTAssertTrue(monitor.register(pid: 42) { value in changes.withLock { $0.append(value) } })
let receipt = try XCTUnwrap(monitor.registrationReceipt(pid: 42))
let continuity = monitor.diagnostic.continuityGeneration
XCTAssertEqual(stream.receive?(event(subtype: 0x4000, target: 9)), .suppress)
XCTAssertEqual(stream.receive?(event()), .pass)
XCTAssertEqual(releases.withLock { $0 }, [17])
XCTAssertEqual(changes.withLock { $0 }, [false])
XCTAssertFalse(monitor.diagnostic.available)
XCTAssertEqual(monitor.diagnostic.reason, "release_key_focus_failed_waiting_for_keyboard_recovery")
XCTAssertGreaterThan(monitor.diagnostic.continuityGeneration, continuity)
XCTAssertFalse(monitor.isRegistrationCurrent(receipt))
XCTAssertNil(monitor.registrationReceipt(pid: 42))
XCTAssertFalse(monitor.register(pid: 42) { _ in })
XCTAssertFalse(monitor.register(pid: 77) { _ in })
XCTAssertEqual(stream.starts, 1)
XCTAssertEqual(stream.stops, 0, "failure must not remove the user's only keyboard route")
for type: UInt32 in [10, 11, 12] {
XCTAssertEqual(stream.receive?(event(target: 42, type: type)), .redirect(9))
XCTAssertEqual(stream.receive?(event(source: 700, target: 42, type: type)), .pass)
XCTAssertEqual(stream.receive?(event(target: 77, type: type)), .pass)
}
XCTAssertEqual(stream.receive?(event(subtype: 0x4000, target: 9)), .pass)
XCTAssertEqual(stream.receive?(event(subtype: 0x8000, target: 9)), .pass)
XCTAssertEqual(stream.receive?(event()), .pass)
XCTAssertEqual(releases.withLock { $0 }, [17], "draining must not retry cancellation or suppress notifications")
}
func testFailedRecoveryEndsOnlyAfterTrustedKeyboardFocusRestoration() {
let stream = FakeStream()
let monitor = monitor(stream, releaseFocus: { _ in false })
XCTAssertTrue(monitor.register(pid: 42) { _ in })
_ = stream.receive?(event(subtype: 0x4000, target: 9))
_ = stream.receive?(event())
XCTAssertEqual(stream.receive?(event(focus: 9, target: 42)), .pass)
XCTAssertEqual(stream.receive?(event(focus: 9, subtype: 0xf107)), .pass)
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .redirect(9))
XCTAssertEqual(stream.receive?(event(focus: 9, source: 700)), .pass)
XCTAssertEqual(stream.stops, 1)
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .pass)
XCTAssertFalse(monitor.diagnostic.available)
XCTAssertEqual(monitor.diagnostic.reason, "keyboard_focus_recovery_observed")
XCTAssertTrue(monitor.register(pid: 42) { _ in })
XCTAssertEqual(stream.starts, 2)
}
func testActualUserSwitchEndsFailedRecoveryEvenIfFrontLaterReturns() {
let stream = FakeStream()
let front = OSAllocatedUnfairLock(initialState: pid_t(9))
let monitor = monitor(stream, releaseFocus: { _ in false }, readRealFrontmost: { front.withLock { $0 } })
XCTAssertTrue(monitor.register(pid: 42) { _ in })
_ = stream.receive?(event(subtype: 0x4000, target: 9))
_ = stream.receive?(event())
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .redirect(9))
front.withLock { $0 = 77 }
monitor.observeRealFrontmost(pid: 77)
front.withLock { $0 = 9 }
monitor.observeRealFrontmost(pid: 9)
XCTAssertEqual(stream.stops, 1)
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .pass)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
}
func testUnregisterDuringCancellationCannotDismantleFailedKeyboardRecovery() {
let stream = FakeStream()
let holder = OSAllocatedUnfairLock(initialState: Optional<FocusEventMonitor>.none)
let monitor = monitor(stream, releaseFocus: { _ in
holder.withLock { $0 }?.unregisterAll()
return false
})
holder.withLock { $0 = monitor }
defer { holder.withLock { $0 = nil } }
XCTAssertTrue(monitor.register(pid: 42) { _ in })
_ = stream.receive?(event(subtype: 0x4000, target: 9))
_ = stream.receive?(event())
XCTAssertEqual(stream.stops, 0)
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .redirect(9))
monitor.unregisterAll()
XCTAssertEqual(stream.stops, 0)
XCTAssertEqual(stream.receive?(event(target: 42, type: 11)), .redirect(9))
XCTAssertFalse(monitor.register(pid: 42) { _ in })
_ = stream.receive?(event(focus: 9))
XCTAssertEqual(stream.stops, 1)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
}
func testOSDisabledTapCannotPromiseRecoveryAndMustKeepNewAutomationBlocked() {
let stream = FakeStream()
let monitor = monitor(stream, releaseFocus: { _ in false })
XCTAssertTrue(monitor.register(pid: 42) { _ in })
_ = stream.receive?(event(subtype: 0x4000, target: 9))
_ = stream.receive?(event())
stream.interrupted?("event_tap_timeout")
XCTAssertEqual(stream.stops, 1)
XCTAssertFalse(monitor.diagnostic.available)
XCTAssertEqual(monitor.diagnostic.reason, "event_tap_timeout_keyboard_safety_forwarding_unavailable")
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .pass)
XCTAssertFalse(monitor.register(pid: 42) { _ in })
monitor.observeRealFrontmost(pid: 77)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
}
func testRecoveryNeverRedirectsToAReusedVictimPID() {
let stream = FakeStream()
let victimLaunch = OSAllocatedUnfairLock(initialState: TimeInterval(1))
let monitor = monitor(stream, releaseFocus: { _ in false }, readProcessIdentity: { pid in
.init(executablePath: "/test/\(pid)", launchTime: pid == 9 ? victimLaunch.withLock { $0 } : 1)
})
XCTAssertTrue(monitor.register(pid: 42) { _ in })
_ = stream.receive?(event(subtype: 0x4000, target: 9))
_ = stream.receive?(event())
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .redirect(9))
victimLaunch.withLock { $0 = 2 }
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .pass)
XCTAssertEqual(stream.stops, 1)
XCTAssertEqual(monitor.diagnostic.reason, "keyboard_recovery_process_identity_changed")
XCTAssertTrue(monitor.register(pid: 42) { _ in })
}
func testUnknownFrontmostDoesNotCountAsAConfirmedRecovery() {
let stream = FakeStream()
let front = OSAllocatedUnfairLock(initialState: Optional<pid_t>(9))
let monitor = monitor(stream, releaseFocus: { _ in false }, readRealFrontmost: { front.withLock { $0 } })
XCTAssertTrue(monitor.register(pid: 42) { _ in })
_ = stream.receive?(event(subtype: 0x4000, target: 9))
_ = stream.receive?(event())
front.withLock { $0 = nil }
monitor.observeRealFrontmost(pid: nil)
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .pass)
XCTAssertFalse(monitor.register(pid: 42) { _ in })
XCTAssertEqual(stream.stops, 0)
front.withLock { $0 = 9 }
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .redirect(9))
}
func testMissingCancellationAndKeyboardTapFailuresCannotEnableHalfProtection() {
let stream = FakeStream()
let unsupported = FocusEventMonitor(
makeStream: { stream }, releaseFocus: nil, isOrdinaryApp: { _ in true }
)
XCTAssertFalse(unsupported.register(pid: 42) { _ in })
XCTAssertEqual(stream.starts, 0)
stream.keyboardStartsSuccessfully = false
let monitor = monitor(stream)
XCTAssertFalse(monitor.register(pid: 42) { _ in })
XCTAssertEqual(stream.receive?(event(subtype: 0x4000, target: 9)), .pass)
stream.keyboardStartsSuccessfully = true
XCTAssertTrue(monitor.register(pid: 42) { _ in })
stream.keyboardStartsSuccessfully = false
XCTAssertFalse(monitor.register(pid: 77) { _ in })
XCTAssertFalse(monitor.diagnostic.available)
}
func testStartupInterruptionCannotBeOverwrittenBySuccessfulStartReturn() {
let stream = FakeStream()
stream.interruptDuringStart = "event_tap_timeout"
let monitor = monitor(stream)
XCTAssertFalse(monitor.register(pid: 42) { _ in })
XCTAssertFalse(monitor.diagnostic.available)
XCTAssertEqual(monitor.diagnostic.reason, "event_tap_timeout")
}
func testHelperSourcedViewBridgeFocusChangeStillReleasesAndRestoresSystemFocus() {
let stream = FakeStream()
let releases = OSAllocatedUnfairLock(initialState: [UInt32]())
let monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, isFocusObserver: { $0 == 901 },
makeStream: { stream }, releaseFocus: { token in
releases.withLock { $0.append(token) }; return true
}, readRealFrontmost: { 9 }, isOrdinaryApp: { _ in true },
readProcessIdentity: { .init(executablePath: "/test/\($0)", launchTime: 1) }
)
let changes = OSAllocatedUnfairLock(initialState: [String]())
XCTAssertTrue(monitor.register(pid: 9) { value in changes.withLock { $0.append("9:\(value)") } })
XCTAssertTrue(monitor.register(pid: 42) { value in changes.withLock { $0.append("42:\(value)") } })
XCTAssertEqual(stream.receive?(event(subtype: 0x4000, target: 9)), .suppress)
XCTAssertEqual(stream.receive?(event(subtype: 0x8000, source: 700, target: 42)), .pass)
XCTAssertTrue(monitor.isAppCurrentlyFocused(pid: 9), "our direct returned notification is not system focus")
XCTAssertEqual(stream.receive?(event(source: 700)), .pass)
XCTAssertEqual(releases.withLock { $0 }, [17])
XCTAssertTrue(monitor.isAppCurrentlyFocused(pid: 42))
XCTAssertEqual(stream.receive?(event(focus: 9, source: 700)), .pass)
XCTAssertTrue(monitor.isAppCurrentlyFocused(pid: 9))
XCTAssertEqual(changes.withLock { $0 }, ["9:false", "9:true"])
}
func testHelperKeyboardPassesThroughEveryProtectionPhase() {
let stream = FakeStream()
let monitor = monitor(stream)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
let transitions = [
event(type: 0), event(subtype: 0x4000, target: 9), event(),
event(focus: 9), event(subtype: 2),
]
for transition in transitions {
_ = stream.receive?(transition)
for keyType: UInt32 in [10, 11, 12] {
XCTAssertEqual(stream.receive?(event(source: 700, target: 42, type: keyType)), .pass)
XCTAssertEqual(stream.receive?(event(source: 700, target: 9, type: keyType)), .pass)
}
}
}
func testOldRegistrationFailureCannotInterruptAReplacementGeneration() {
let oldStream = FakeStream()
let newStream = FakeStream()
let streamIndex = OSAllocatedUnfairLock(initialState: 0)
let monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, isFocusObserver: { $0 == 901 },
makeStream: {
streamIndex.withLock { index in
defer { index += 1 }
return index == 0 ? oldStream : newStream
}
}, releaseFocus: { _ in true }, readRealFrontmost: { 9 },
isOrdinaryApp: { _ in true },
readProcessIdentity: { .init(executablePath: "/test/\($0)", launchTime: 1) }
)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
let oldReceipt = monitor.registrationReceipt(pid: 42)
oldStream.addHook = { [weak monitor] _ in
guard let monitor else { return false }
monitor.unregisterAll()
XCTAssertTrue(monitor.register(pid: 42) { _ in })
return false
}
XCTAssertFalse(monitor.register(pid: 77) { _ in })
XCTAssertTrue(monitor.diagnostic.available)
XCTAssertEqual(newStream.stops, 0)
XCTAssertFalse(oldReceipt.map { monitor.isRegistrationCurrent($0) } ?? true)
XCTAssertNotNil(monitor.registrationReceipt(pid: 42))
}
func testPIDReplacementInvalidatesReceiptAndRebuildsItsKeyboardTap() throws {
let stream = FakeStream()
let launch = OSAllocatedUnfairLock(initialState: TimeInterval(1))
let monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, isFocusObserver: { $0 == 901 },
makeStream: { stream }, releaseFocus: { _ in true }, readRealFrontmost: { 9 },
isOrdinaryApp: { _ in true }, readProcessIdentity: { pid in
.init(executablePath: "/test/\(pid)", launchTime: launch.withLock { $0 })
}
)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
let original = try XCTUnwrap(monitor.registrationReceipt(pid: 42))
let oldReceive = stream.receive
XCTAssertTrue(monitor.isRegistrationCurrent(original))
XCTAssertEqual(stream.receive?(event(subtype: 0x4000, target: 9)), .suppress)
launch.withLock { $0 = 2 }
XCTAssertFalse(monitor.isRegistrationCurrent(original))
XCTAssertEqual(stream.receive?(event(target: 42, type: 10)), .pass)
XCTAssertFalse(monitor.diagnostic.available)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
XCTAssertEqual(stream.starts, 2)
let replacement = try XCTUnwrap(monitor.registrationReceipt(pid: 42))
XCTAssertNotEqual(original, replacement)
XCTAssertTrue(monitor.isRegistrationCurrent(replacement))
XCTAssertEqual(oldReceive?(event(subtype: 0x4000, target: 9)), .pass)
XCTAssertTrue(monitor.diagnostic.available)
monitor.unregisterAll()
XCTAssertFalse(monitor.isRegistrationCurrent(replacement))
}
func testProcessReplacementDuringPIDTapInstallationCannotRegisterOldIdentity() {
let stream = FakeStream()
let launch = OSAllocatedUnfairLock(initialState: TimeInterval(1))
let monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, makeStream: { stream },
releaseFocus: { _ in true }, isOrdinaryApp: { _ in true },
readProcessIdentity: { pid in
.init(executablePath: "/test/\(pid)", launchTime: launch.withLock { $0 })
}
)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
stream.addHook = { _ in launch.withLock { $0 = 2 }; return true }
XCTAssertFalse(monitor.register(pid: 77) { _ in })
XCTAssertFalse(monitor.diagnostic.available)
XCTAssertNil(monitor.registrationReceipt(pid: 77))
}
func testWorkerReentryExecutesInlineInsteadOfWaitingForItsOwnRunLoop() {
let executed = OSAllocatedUnfairLock(initialState: false)
XCTAssertTrue(FocusNotificationStream.perform(on: CFRunLoopGetCurrent()) {
executed.withLock { $0 = true }
})
XCTAssertTrue(executed.withLock { $0 })
}
func testStopBeforeRunAndDuringEntryGapCannotLeaveAnUnboundedWorker() {
var cancelled = true
var iterations = 0
FocusNotificationStream.runWhileActive(isCancelled: { cancelled }) { _ in
iterations += 1
return .finished
}
XCTAssertEqual(iterations, 0)
cancelled = false
FocusNotificationStream.runWhileActive(isCancelled: { cancelled }) { interval in
// Models a stop arriving after the cancellation check but before
// RunInMode enters: a lost Stop wakeup still has a bounded timeout.
cancelled = true
XCTAssertLessThanOrEqual(interval, 0.1)
iterations += 1
return .timedOut
}
XCTAssertEqual(iterations, 1)
}
func testExpiredOtherPIDDoesNotPreventHealthyTargetFromRestarting() {
let stream = FakeStream()
let live = OSAllocatedUnfairLock(initialState: Set<pid_t>([42, 77]))
let monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, makeStream: { stream },
releaseFocus: { _ in true }, isOrdinaryApp: { _ in true },
readProcessIdentity: { pid in
live.withLock { $0.contains(pid) } ? .init(executablePath: "/test/\(pid)", launchTime: 1) : nil
}
)
XCTAssertTrue(monitor.register(pid: 42) { _ in })
XCTAssertTrue(monitor.register(pid: 77) { _ in })
live.withLock { _ = $0.remove(77) }
stream.interrupted?("event_tap_timeout")
stream.addHook = { $0 != 77 }
XCTAssertTrue(monitor.register(pid: 42) { _ in })
XCTAssertNotNil(monitor.registrationReceipt(pid: 42))
XCTAssertNil(monitor.registrationReceipt(pid: 77))
}
func testCallbackUnregisterInvalidatesRemainingEffectsFromThatEvent() {
let stream = FakeStream()
let monitor = monitor(stream)
let gained = OSAllocatedUnfairLock(initialState: [Bool]())
XCTAssertTrue(monitor.register(pid: 9) { [weak monitor] _ in monitor?.unregisterAll() })
XCTAssertTrue(monitor.register(pid: 42) { value in gained.withLock { $0.append(value) } })
XCTAssertEqual(stream.receive?(event()), .pass)
XCTAssertEqual(gained.withLock { $0 }, [])
XCTAssertFalse(monitor.diagnostic.available)
}
}
@@ -0,0 +1,513 @@
import AppKit
import os
import XCTest
@testable import cc_haha_computer_use
/// Only the OS observations and event transport are substituted. Each test
/// drives the production coordinator and, where relevant, the real observer.
@MainActor
final class FocusLifecycleIntegrationTests: XCTestCase {
private let targetPID: pid_t = 42
func testClickTypeAndReturnShareOneAcceptedFocusEstablishment() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
for action in ["click", "type_text", "Return"] {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.recordAction(action)
}
XCTAssertEqual(external.snapshot.posts, ["activate"])
XCTAssertEqual(external.snapshot.actions, ["click", "type_text", "Return"])
}
func testLostFocusAfterSuccessIsRestoredBeforeTheNextAction() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.recordAction("click")
external.update { $0.hasFocus = false; $0.acceptsInput = false }
coordinator.observeFocus(pid: targetPID, hasFocus: false)
for action in ["type_text", "Return"] {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.recordAction(action)
}
XCTAssertEqual(external.snapshot.posts, ["activate", "returnFocus"])
XCTAssertEqual(external.snapshot.actions, ["click", "type_text", "Return"])
}
func testAnAlreadyFocusedApplicationReceivesNoEstablishment() async throws {
for active in [false, true] {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
external.update { $0.isActive = active; $0.hasFocus = true; $0.acceptsInput = true }
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
XCTAssertTrue(external.snapshot.posts.isEmpty)
}
}
func testAnActiveApplicationWithoutFocusNeedsOnlyFocusReturned() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
external.update { $0.isActive = true }
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
XCTAssertEqual(external.snapshot.posts, ["returnFocus"])
}
func testRealActivationThenCoverWithoutAnIntermediateCURequestReestablishesInput() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
let center = NotificationCenter()
let observer = SyntheticWindowFocus.ApplicationLifecycleObserver(center: center, coordinator: coordinator)
defer { withExtendedLifetime(observer) {} }
let application = FocusNotificationApplication()
let pid = application.processIdentifier
try await coordinator.prepare(pid: pid, window: nil, runtime: external.runtime())
external.update { $0.isActive = true }
center.post(name: NSWorkspace.didActivateApplicationNotification, object: nil,
userInfo: [NSWorkspace.applicationUserInfoKey: application])
external.update { $0.isActive = false; $0.hasFocus = false; $0.acceptsInput = false }
center.post(name: NSWorkspace.didDeactivateApplicationNotification, object: nil,
userInfo: [NSWorkspace.applicationUserInfoKey: application])
try await coordinator.prepare(pid: pid, window: nil, runtime: external.runtime())
external.recordAction("search-after-cover")
XCTAssertEqual(external.snapshot.posts, ["activate", "activate"])
XCTAssertEqual(external.snapshot.actions, ["search-after-cover"])
}
func testLateActivationNotificationIsNotDroppedBecauseTheTargetIsAlreadyCovered() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
let center = NotificationCenter()
let observer = SyntheticWindowFocus.ApplicationLifecycleObserver(center: center, coordinator: coordinator)
defer { withExtendedLifetime(observer) {} }
let application = FocusNotificationApplication()
let pid = application.processIdentifier
try await coordinator.prepare(pid: pid, window: nil, runtime: external.runtime())
// Activation arrives after the target is already covered. The next
// request observes deactivation even if its notification is still
// queued; dropping this activation would lose the entire transition.
external.update { $0.isActive = false; $0.hasFocus = false; $0.acceptsInput = false }
center.post(name: NSWorkspace.didActivateApplicationNotification, object: nil,
userInfo: [NSWorkspace.applicationUserInfoKey: application])
try await coordinator.prepare(pid: pid, window: nil, runtime: external.runtime())
XCTAssertEqual(external.snapshot.posts, ["activate", "activate"])
}
func testUnrelatedAndMalformedLifecycleNotificationsDoNotResetAcceptedFocus() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
let center = NotificationCenter()
let observer = SyntheticWindowFocus.ApplicationLifecycleObserver(center: center, coordinator: coordinator)
defer { withExtendedLifetime(observer) {} }
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
center.post(name: NSWorkspace.didActivateApplicationNotification, object: nil,
userInfo: [NSWorkspace.applicationUserInfoKey: "not an application"])
let unrelatedApplication = FocusNotificationApplication(pid: 43)
center.post(name: NSWorkspace.didDeactivateApplicationNotification, object: nil,
userInfo: [NSWorkspace.applicationUserInfoKey: unrelatedApplication])
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
XCTAssertEqual(external.snapshot.posts, ["activate"])
}
func testTimeoutCannotAuthorizeAnActionAndTheNextAttemptCanRecover() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
external.update { $0.acceptOnPost = false }
do {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.recordAction("must-not-run")
XCTFail("unacknowledged activation must not authorize input")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_not_accepted")
}
XCTAssertTrue(external.snapshot.actions.isEmpty)
XCTAssertEqual(external.snapshot.posts, ["activate"])
external.update { $0.acceptOnPost = true }
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.recordAction("recovered")
XCTAssertEqual(external.snapshot.posts, ["activate", "activate"])
XCTAssertEqual(external.snapshot.actions, ["recovered"])
}
func testCachedFocusStillRequiresTheTargetToAcceptInput() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.update { $0.acceptsInput = false }
do {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.recordAction("must-not-run")
XCTFail("a cached belief is not an AXFrontmost acknowledgement")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_not_accepted")
}
XCTAssertTrue(external.snapshot.actions.isEmpty)
XCTAssertEqual(external.snapshot.posts, ["activate"])
}
func testFailedEventConstructionDoesNotCommitBeliefAndCanRetry() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
external.update { $0.postSucceeds = false }
do {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.recordAction("must-not-run")
XCTFail("a failed activation must not authorize input")
} catch {}
XCTAssertTrue(external.snapshot.actions.isEmpty)
external.update { $0.postSucceeds = true }
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
XCTAssertEqual(external.snapshot.posts, ["activate", "activate"])
}
func testProcessReplacementDuringAcceptanceWaitCannotInheritFocus() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
external.update { $0.acceptOnPost = false }
let runtime = external.runtime(onPause: {
external.update {
$0.identity = FocusExternalRuntime.identity(launchTime: 2)
$0.hasFocus = true
$0.acceptsInput = true
}
})
do {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: runtime)
external.recordAction("must-not-run")
XCTFail("the new process must not inherit an in-flight activation")
} catch let error as CUError {
XCTAssertEqual(error.code, "stale_process")
}
XCTAssertTrue(external.snapshot.actions.isEmpty)
}
func testAPIDReusedBetweenRequestsEstablishesFocusForTheNewLifetime() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.update {
$0.identity = FocusExternalRuntime.identity(launchTime: 2)
$0.hasFocus = false
$0.acceptsInput = false
}
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
XCTAssertEqual(external.snapshot.posts, ["activate", "activate"])
}
func testAConfirmationNotificationDuringTheWaitDoesNotInvalidateAcceptance() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
external.update { $0.acceptOnPost = false }
let pid = targetPID
let runtime = external.runtime(onPause: {
coordinator.observeFocus(pid: pid, hasFocus: true)
external.update { $0.hasFocus = true; $0.acceptsInput = true }
})
try await coordinator.prepare(pid: pid, window: nil, runtime: runtime)
external.recordAction("accepted")
try await coordinator.prepare(pid: pid, window: nil, runtime: external.runtime())
XCTAssertEqual(external.snapshot.posts, ["activate"])
XCTAssertEqual(external.snapshot.actions, ["accepted"])
}
func testFocusLossDuringRecoveryRejectsALateAcceptanceAndAllowsRetry() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
external.update { $0.hasFocus = false; $0.acceptsInput = false; $0.acceptOnPost = false }
coordinator.observeFocus(pid: targetPID, hasFocus: false)
let pid = targetPID
let runtime = external.runtime(onPause: {
coordinator.observeFocus(pid: pid, hasFocus: false)
external.update { $0.hasFocus = true; $0.acceptsInput = true }
})
do {
try await coordinator.prepare(pid: pid, window: nil, runtime: runtime)
external.recordAction("must-not-run")
XCTFail("a later focus loss must invalidate the activation receipt")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_changed")
}
XCTAssertTrue(external.snapshot.actions.isEmpty)
external.update { $0.hasFocus = false; $0.acceptsInput = false; $0.acceptOnPost = true }
try await coordinator.prepare(pid: pid, window: nil, runtime: external.runtime())
external.recordAction("retry")
XCTAssertEqual(external.snapshot.actions, ["retry"])
XCTAssertEqual(external.snapshot.posts.count, 3)
}
func testSessionDrainRequiresANewEstablishment() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
_ = coordinator.drain()
external.update { $0.hasFocus = false; $0.acceptsInput = false }
try await coordinator.prepare(pid: targetPID, window: nil, runtime: external.runtime())
XCTAssertEqual(external.snapshot.posts, ["activate", "activate"])
}
func testLostMonitorAfterRegistrationBeforeBeliefCreationCannotAuthorizeInput() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
let stream = FocusLifecycleStream()
let monitor = try registeredMonitor(stream: stream, coordinator: coordinator)
let runtime = protectedRuntime(external: external, monitor: monitor)
stream.interrupt()
external.update { $0.acceptsInput = true }
do {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: runtime)
external.recordAction("must-not-run")
XCTFail("lost observation cannot be replaced by a stale AXFrontmost value")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_monitor_interrupted")
}
XCTAssertTrue(external.snapshot.posts.isEmpty)
XCTAssertTrue(external.snapshot.actions.isEmpty)
}
func testMonitorLossWhileReadingInitialIdentityCannotPostActivation() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
let stream = FocusLifecycleStream()
let monitor = try registeredMonitor(stream: stream, coordinator: coordinator)
var runtime = protectedRuntime(external: external, monitor: monitor)
runtime.identity = { _ in
stream.interrupt()
return external.snapshot.identity
}
do {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: runtime)
XCTFail("the pre-post continuity check must catch loss after the entry check")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_monitor_interrupted")
}
XCTAssertTrue(external.snapshot.posts.isEmpty)
}
func testMonitorLossDuringTheWaitWithholdsInputEvenIfAXAcknowledges() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
let stream = FocusLifecycleStream()
let monitor = try registeredMonitor(stream: stream, coordinator: coordinator)
var runtime = protectedRuntime(external: external, monitor: monitor)
runtime.pause = {
stream.interrupt()
external.update { $0.acceptsInput = true }
}
do {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: runtime)
external.recordAction("must-not-run")
XCTFail("an acknowledgement after monitor loss is not safe to act on")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_monitor_interrupted")
}
XCTAssertEqual(external.snapshot.posts, ["activate"])
XCTAssertTrue(external.snapshot.actions.isEmpty)
}
func testMonitorLossDuringAcknowledgementCannotCommitTheReceipt() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
let stream = FocusLifecycleStream()
let monitor = try registeredMonitor(stream: stream, coordinator: coordinator)
var runtime = protectedRuntime(external: external, monitor: monitor)
runtime.acceptsInput = { _ in
stream.interrupt()
return true
}
do {
try await coordinator.prepare(pid: targetPID, window: nil, runtime: runtime)
external.recordAction("must-not-run")
XCTFail("the receipt needs a final continuity check after reading AX")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_monitor_interrupted")
}
XCTAssertTrue(external.snapshot.actions.isEmpty)
}
func testAlreadyCancelledPreparationNeverPostsFocusEvents() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let external = FocusExternalRuntime()
let pid = targetPID
let task = Task { @MainActor in
withUnsafeCurrentTask { $0?.cancel() }
try await coordinator.prepare(pid: pid, window: nil, runtime: external.runtime())
external.recordAction("must-not-run")
}
do {
try await task.value
XCTFail("cancelled preparation must not post activation")
} catch is CancellationError {}
XCTAssertTrue(external.snapshot.posts.isEmpty)
XCTAssertTrue(external.snapshot.actions.isEmpty)
}
func testDetachedCallerRunsAllOSObservationAndPostingCallbacksOnMainActor() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let calls = OSAllocatedUnfairLock(initialState: Set<String>())
let runtime = SyntheticWindowFocus.Runtime(
identity: { _ in
MainActor.assertIsolated()
calls.withLock { _ = $0.insert("identity") }
return FocusExternalRuntime.identity(launchTime: 1)
},
isActive: { _ in
MainActor.assertIsolated()
calls.withLock { _ = $0.insert("isActive") }
return false
},
hasFocus: { _ in
MainActor.assertIsolated()
calls.withLock { _ = $0.insert("hasFocus") }
return false
},
acceptsInput: { _ in
MainActor.assertIsolated()
calls.withLock { _ = $0.insert("acceptsInput") }
return true
},
post: { _, _, _ in
MainActor.assertIsolated()
calls.withLock { _ = $0.insert("post") }
return true
},
pause: { await Task.yield() },
validateContinuity: {
MainActor.assertIsolated()
calls.withLock { _ = $0.insert("continuity") }
}
)
let pid = targetPID
try await Task.detached {
try await coordinator.prepare(pid: pid, window: nil, runtime: runtime)
}.value
XCTAssertEqual(calls.withLock { $0 }, ["identity", "isActive", "hasFocus", "acceptsInput", "post", "continuity"])
}
private func registeredMonitor(
stream: FocusLifecycleStream,
coordinator: SyntheticWindowFocus.Coordinator
) throws -> FocusEventMonitor {
let monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, isFocusObserver: { _ in false },
makeStream: { stream }, releaseFocus: { _ in true },
readRealFrontmost: { 9 }, isOrdinaryApp: { _ in true },
readProcessIdentity: { .init(executablePath: "/test/\($0)", launchTime: 1) }
)
let pid = targetPID
XCTAssertTrue(monitor.register(pid: pid) { coordinator.observeFocus(pid: pid, hasFocus: $0) })
return monitor
}
private func protectedRuntime(
external: FocusExternalRuntime, monitor: FocusEventMonitor
) -> SyntheticWindowFocus.Runtime {
let continuity = monitor.diagnostic.continuityGeneration
var runtime = external.runtime()
runtime.validateContinuity = {
let diagnostic = monitor.diagnostic
guard diagnostic.available, diagnostic.continuityGeneration == continuity else {
throw CUError("focus_monitor_interrupted", "Focus observation was interrupted")
}
}
return runtime
}
}
private final class FocusLifecycleStream: FocusEventMonitor.Stream, @unchecked Sendable {
private let interruption = OSAllocatedUnfairLock<(@Sendable (String) -> Void)?>(initialState: nil)
func start(
receive: @escaping @Sendable (FocusEventMonitor.Event) -> FocusEventMonitor.Disposition,
interrupted: @escaping @Sendable (String) -> Void
) -> Bool {
interruption.withLock { $0 = interrupted }
return true
}
func addProtectedPID(_ pid: pid_t) -> Bool { true }
func stop() {}
func interrupt() { interruption.withLock { $0 }?("test_interruption") }
}
/// Command-line XCTest is not necessarily a LaunchServices application, so
/// NSRunningApplication.current can report -1. Keep the real payload type and
/// production observer without registering a GUI application during a test.
private final class FocusNotificationApplication: NSRunningApplication, @unchecked Sendable {
let fixturePID: pid_t
init(pid: pid_t = 42) {
fixturePID = pid
super.init()
}
override var processIdentifier: pid_t { fixturePID }
}
private final class FocusExternalRuntime: @unchecked Sendable {
struct Snapshot: Sendable {
var identity: AXTreeProcessIdentity? = FocusExternalRuntime.identity(launchTime: 1)
var isActive = false
var hasFocus = false
var acceptsInput = false
var acceptOnPost = true
var postSucceeds = true
var posts: [String] = []
var actions: [String] = []
}
private let state = OSAllocatedUnfairLock(initialState: Snapshot())
var snapshot: Snapshot { state.withLock { $0 } }
static func identity(launchTime: TimeInterval) -> AXTreeProcessIdentity {
AXTreeProcessIdentity(
bundleID: "com.example.focus-target",
executablePath: "/Applications/FocusTarget.app/Contents/MacOS/FocusTarget",
launchTime: launchTime
)
}
func update(_ mutation: @Sendable (inout Snapshot) -> Void) { state.withLock(mutation) }
func recordAction(_ action: String) { state.withLock { $0.actions.append(action) } }
func runtime(
attempts: Int = 3,
onPause: @escaping @Sendable () async throws -> Void = {}
) -> SyntheticWindowFocus.Runtime {
SyntheticWindowFocus.Runtime(
identity: { [self] _ in snapshot.identity },
isActive: { [self] _ in snapshot.isActive },
hasFocus: { [self] _ in snapshot.hasFocus },
acceptsInput: { [self] _ in snapshot.acceptsInput },
post: { [self] establishment, _, _ in
state.withLock { state in
switch establishment {
case .activate: state.posts.append("activate")
case .returnFocus: state.posts.append("returnFocus")
case .none: state.posts.append("none")
}
guard state.postSucceeds else { return false }
if state.acceptOnPost { state.hasFocus = true; state.acceptsInput = true }
return true
}
},
pause: onPause,
attempts: attempts
)
}
}
@@ -0,0 +1,487 @@
import AppKit
import Carbon.HIToolbox
import CoreGraphics
import os
import XCTest
@testable import cc_haha_computer_use
final class KeyboardEventBurstTests: XCTestCase {
@MainActor
func testProductionDispatchAllocatesAFreshHIDSourceForEachPressAndSeparatelyRestoresSessionFlags() async throws {
var sources: [CGEventSource] = []
var flagQueries: [CGEventSourceStateID] = []
var physicalFlags: CGEventFlags = []
var preparations = 0
var posted: [[CGEvent]] = []
let baselines: [CGEventFlags] = [[.maskShift, .maskAlphaShift], [.maskControl, .maskAlternate]]
for (index, key) in ["cmd+a", "Return"].enumerated() {
var burst: [CGEvent] = []
try await KeyboardEventBurst.dispatch(
chords: KeyMapping.parse(key),
prepare: {
await Task.yield()
physicalFlags = baselines[index]
preparations += 1
},
makeSource: {
let source = try KeyboardEventBurst.makeSource()
sources.append(source)
return source
},
readFlagsState: { state in
XCTAssertEqual(preparations, index + 1, "read physical modifiers after focus preparation")
flagQueries.append(state)
// A wrong read domain must not accidentally return the
// expected physical baseline and make this test pass.
return state == .combinedSessionState ? physicalFlags : .maskNumericPad
},
validateBeforePosting: {},
post: { burst.append($0) }
)
posted.append(burst)
}
XCTAssertEqual(sources.count, 2)
XCTAssertFalse(sources[0] === sources[1], "each press must own a fresh source")
XCTAssertEqual(sources.map(\.sourceStateID), [.hidSystemState, .hidSystemState])
XCTAssertEqual(flagQueries, [.combinedSessionState, .combinedSessionState])
XCTAssertEqual(posted.map { $0.map(\.type) }, [
[.flagsChanged, .keyDown, .flagsChanged, .keyUp],
[.flagsChanged, .keyDown, .flagsChanged, .keyUp],
])
XCTAssertEqual(posted[0].map(\.flags), [.maskCommand, .maskCommand, baselines[0], .maskCommand])
XCTAssertEqual(posted[1].map(\.flags), [[], [], baselines[1], []])
for event in posted.flatMap({ $0 }) {
XCTAssertEqual(event.getIntegerValueField(.eventSourceStateID), Int64(CGEventSourceStateID.hidSystemState.rawValue))
XCTAssertEqual(event.getIntegerValueField(.eventSourceUserData), HelperEventMarker.value)
}
}
func testBareReturnExplicitlyClearsModifiersBeforeKeyDown() throws {
let events = try KeyboardEventBurst.allocate(
chords: KeyMapping.parse("Return"),
source: makeSource(),
restoringFlags: []
)
XCTAssertEqual(events.map(\.type), [.flagsChanged, .keyDown, .flagsChanged, .keyUp])
XCTAssertEqual(events.map(\.flags), [[], [], [], []])
XCTAssertEqual(keyCodes(in: events), [Int64(kVK_Return), Int64(kVK_Return)])
}
func testCommandShortcutThenReturnDoesNotCarryCommandIntoReturn() throws {
let source = try makeSource()
let shortcut = try KeyMapping.parse("cmd+a")
let enter = try KeyMapping.parse("Return")
let events = try KeyboardEventBurst.allocate(
chords: shortcut,
source: source,
restoringFlags: []
) + KeyboardEventBurst.allocate(
chords: enter,
source: source,
restoringFlags: []
)
XCTAssertEqual(events.map(\.type), [
.flagsChanged, .keyDown, .flagsChanged, .keyUp,
.flagsChanged, .keyDown, .flagsChanged, .keyUp,
])
XCTAssertEqual(events.map(\.flags), [
.maskCommand, .maskCommand, [], .maskCommand,
[], [], [], [],
])
XCTAssertEqual(keyCodes(in: events), [
Int64(shortcut[0].keyCode), Int64(shortcut[0].keyCode),
Int64(enter[0].keyCode), Int64(enter[0].keyCode),
])
}
func testMultipleChordsRestoreCapturedUserModifiersWithoutAddingThemToKeys() throws {
let userFlags: CGEventFlags = [.maskShift, .maskAlternate, .maskAlphaShift]
let chords = try KeyMapping.parse("cmd+a Return")
let events = try KeyboardEventBurst.allocate(
chords: chords,
source: makeSource(),
restoringFlags: userFlags
)
XCTAssertEqual(events.map(\.type), [
.flagsChanged, .keyDown, .flagsChanged, .keyUp,
.flagsChanged, .keyDown, .flagsChanged, .keyUp,
])
XCTAssertEqual(events.map(\.flags), [
.maskCommand, .maskCommand, userFlags, .maskCommand,
[], [], userFlags, [],
])
XCTAssertEqual(keyCodes(in: events), [
Int64(chords[0].keyCode), Int64(chords[0].keyCode),
Int64(chords[1].keyCode), Int64(chords[1].keyCode),
])
}
func testEveryEventRetainsTheSuppliedSourceMarker() throws {
let source = try makeSource()
let events = try KeyboardEventBurst.allocate(
chords: KeyMapping.parse("cmd+shift+Tab Return"),
source: source,
restoringFlags: .maskControl
)
XCTAssertEqual(events.count, 8)
for event in events {
XCTAssertEqual(
event.getIntegerValueField(.eventSourceUserData),
HelperEventMarker.value
)
}
}
func testAnyAllocationFailureWithholdsTheEntireMultiChordBurst() throws {
let source = try makeSource()
let chords = try KeyMapping.parse("cmd+a Return")
for failureIndex in 0..<8 {
var allocationCount = 0
var consumed: [CGEvent] = []
XCTAssertThrowsError(try {
let events = try KeyboardEventBurst.allocate(
chords: chords,
source: source,
restoringFlags: .maskShift,
allocateEvent: { spec, source in
defer { allocationCount += 1 }
guard allocationCount != failureIndex else { return nil }
return KeyboardEventBurst.makeEvent(spec, source: source)
}
)
consumed.append(contentsOf: events)
}()) { error in
XCTAssertEqual((error as? CUError)?.code, "event_alloc")
}
XCTAssertEqual(allocationCount, failureIndex + 1)
XCTAssertTrue(consumed.isEmpty)
}
}
func testConsumerReceivesOnlyFullyAllocatedEventsInOrder() throws {
var allocationCount = 0
var consumed: [CGEvent] = []
let events = try KeyboardEventBurst.allocate(
chords: KeyMapping.parse("cmd+a Return"),
source: makeSource(),
restoringFlags: .maskAlternate,
allocateEvent: { spec, source in
XCTAssertTrue(consumed.isEmpty)
allocationCount += 1
return KeyboardEventBurst.makeEvent(spec, source: source)
}
)
for event in events {
XCTAssertEqual(allocationCount, 8)
consumed.append(event)
}
XCTAssertEqual(consumed.map(\.type), [
.flagsChanged, .keyDown, .flagsChanged, .keyUp,
.flagsChanged, .keyDown, .flagsChanged, .keyUp,
])
}
@MainActor
func testDispatchRejectsClipboardCopyDuringFocusPreparationWithoutPosting() async throws {
let pasteboard = NSPasteboard.withUniqueName()
defer { pasteboard.releaseGlobally() }
XCTAssertTrue(pasteboard.setString("original", forType: .string))
let lease = ClipboardLease(pasteboard: pasteboard)
try lease.writeTemporaryString("agent temporary text")
var posted: [CGEvent] = []
var validated = false
do {
try await KeyboardEventBurst.dispatch(
chords: KeyMapping.parse("cmd+v"), source: makeSource(),
prepare: {
await Task.yield()
pasteboard.clearContents()
XCTAssertTrue(pasteboard.setString("new user copy", forType: .string))
},
restoringFlags: { [] },
validateBeforePosting: {
validated = true
guard lease.temporaryWriteIsCurrent() else {
throw CUError("clipboard_changed", "The user copied during focus preparation")
}
},
post: { posted.append($0) }
)
XCTFail("changed clipboard must never be pasted into the automation target")
} catch let error as CUError {
XCTAssertEqual(error.code, "clipboard_changed")
}
XCTAssertTrue(validated)
XCTAssertTrue(posted.isEmpty)
XCTAssertFalse(lease.restoreIfUnchanged())
XCTAssertEqual(pasteboard.string(forType: .string), "new user copy")
}
@MainActor
func testDispatchCancelledDuringPreparationNeverValidatesOrPosts() async throws {
var posted: [CGEvent] = []
var validations = 0
let source = try makeSource()
let chords = try KeyMapping.parse("cmd+v")
let task = Task { @MainActor in
try await KeyboardEventBurst.dispatch(
chords: chords, source: source,
prepare: {
await Task.yield()
withUnsafeCurrentTask { $0?.cancel() }
},
restoringFlags: { [] },
validateBeforePosting: { validations += 1 },
post: { posted.append($0) }
)
}
do {
try await task.value
XCTFail("cancellation after focus preparation must withhold the burst")
} catch is CancellationError {}
XCTAssertEqual(validations, 0)
XCTAssertTrue(posted.isEmpty)
}
@MainActor
func testDispatchDoesNotYieldBetweenFinalClipboardValidationAndTheBurst() async throws {
let pasteboard = NSPasteboard.withUniqueName()
defer { pasteboard.releaseGlobally() }
let lease = ClipboardLease(pasteboard: pasteboard)
try lease.writeTemporaryString("agent temporary text")
var queuedCopy: Task<Void, Never>?
var posted: [CGEvent] = []
try await KeyboardEventBurst.dispatch(
chords: KeyMapping.parse("cmd+v"), source: makeSource(),
prepare: { await Task.yield() }, restoringFlags: { [] },
validateBeforePosting: {
XCTAssertTrue(lease.temporaryWriteIsCurrent())
queuedCopy = Task { @MainActor in
pasteboard.clearContents()
XCTAssertTrue(pasteboard.setString("queued user copy", forType: .string))
}
},
post: {
XCTAssertTrue(lease.temporaryWriteIsCurrent(), "validation and posting must not yield the main actor")
posted.append($0)
}
)
XCTAssertEqual(posted.count, 4)
await queuedCopy?.value
XCTAssertEqual(pasteboard.string(forType: .string), "queued user copy")
XCTAssertFalse(lease.restoreIfUnchanged())
}
@MainActor
func testDispatchCancelledBeforeEntryDoesNotPrepareOrPost() async throws {
var preparations = 0
var posted: [CGEvent] = []
let source = try makeSource()
let chords = try KeyMapping.parse("Return")
let task = Task { @MainActor in
withUnsafeCurrentTask { $0?.cancel() }
try await KeyboardEventBurst.dispatch(
chords: chords, source: source,
prepare: { preparations += 1 },
restoringFlags: { [] }, validateBeforePosting: {},
post: { posted.append($0) }
)
}
do {
try await task.value
XCTFail("already-cancelled dispatch must not establish focus")
} catch is CancellationError {}
XCTAssertEqual(preparations, 0)
XCTAssertTrue(posted.isEmpty)
}
@MainActor
func testDispatchReadsModifiersAfterPreparationAndPostsTheRealFourEventBurst() async throws {
var flags: CGEventFlags = []
var stages: [String] = []
var posted: [CGEvent] = []
let chords = try KeyMapping.parse("cmd+v")
try await KeyboardEventBurst.dispatch(
chords: chords, source: makeSource(),
prepare: {
MainActor.assertIsolated()
stages.append("prepare")
await Task.yield()
flags = [.maskShift, .maskAlphaShift]
},
restoringFlags: {
MainActor.assertIsolated()
stages.append("baseline")
return flags
},
validateBeforePosting: {
MainActor.assertIsolated()
XCTAssertTrue(posted.isEmpty)
stages.append("validate")
},
post: {
MainActor.assertIsolated()
stages.append("post")
posted.append($0)
}
)
XCTAssertEqual(stages, ["prepare", "baseline", "validate", "post", "post", "post", "post"])
XCTAssertEqual(posted.map(\.type), [.flagsChanged, .keyDown, .flagsChanged, .keyUp])
XCTAssertEqual(posted.map(\.flags), [.maskCommand, .maskCommand, flags, .maskCommand])
XCTAssertEqual(keyCodes(in: posted), [Int64(chords[0].keyCode), Int64(chords[0].keyCode)])
XCTAssertTrue(posted.allSatisfy { $0.getIntegerValueField(.eventSourceUserData) == HelperEventMarker.value })
}
@MainActor
func testDispatchRejectsThePreparedReceiptAfterMonitorLossOrReplacement() async throws {
for replaceRegistration in [false, true] {
let rig = KeyboardFocusTestRig()
var receipt: FocusEventMonitor.RegistrationReceipt?
var posted: [CGEvent] = []
var finalValidationReached = false
do {
try await KeyboardEventBurst.dispatch(
chords: KeyMapping.parse("Return"), source: makeSource(),
prepare: { receipt = try await rig.prepare() },
restoringFlags: {
XCTAssertEqual(rig.preparations, 1)
rig.stream.interrupt()
if replaceRegistration { XCTAssertTrue(rig.register()) }
return []
},
validateBeforePosting: {
finalValidationReached = true
try SyntheticWindowFocus.validate(receipt, monitor: rig.monitor)
},
post: { posted.append($0) }
)
XCTFail("a new or interrupted monitor cannot validate the preparation's original receipt")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_monitor_interrupted")
}
XCTAssertTrue(finalValidationReached)
XCTAssertTrue(posted.isEmpty)
XCTAssertEqual(rig.preparations, 1)
if replaceRegistration {
let replacement = try XCTUnwrap(rig.monitor.registrationReceipt(pid: rig.pid))
XCTAssertNotEqual(receipt, replacement)
XCTAssertNoThrow(try SyntheticWindowFocus.validate(replacement, monitor: rig.monitor))
} else {
XCTAssertNil(rig.monitor.registrationReceipt(pid: rig.pid))
}
}
}
@MainActor
func testDispatchAcceptsTheSameHealthyReceiptAndPostsTheCompleteBurst() async throws {
let rig = KeyboardFocusTestRig()
var receipt: FocusEventMonitor.RegistrationReceipt?
var posted: [CGEvent] = []
try await KeyboardEventBurst.dispatch(
chords: KeyMapping.parse("Return"), source: makeSource(),
prepare: { receipt = try await rig.prepare() },
restoringFlags: { [] },
validateBeforePosting: {
try SyntheticWindowFocus.validate(receipt, monitor: rig.monitor)
},
post: { posted.append($0) }
)
XCTAssertEqual(rig.preparations, 1)
XCTAssertEqual(receipt, rig.monitor.registrationReceipt(pid: rig.pid))
XCTAssertEqual(posted.map(\.type), [.flagsChanged, .keyDown, .flagsChanged, .keyUp])
XCTAssertEqual(posted.map(\.flags), [[], [], [], []])
XCTAssertEqual(keyCodes(in: posted), [Int64(kVK_Return), Int64(kVK_Return)])
}
private func makeSource() throws -> CGEventSource {
let source = try XCTUnwrap(CGEventSource(stateID: .privateState))
source.userData = HelperEventMarker.value
return source
}
private func keyCodes(in events: [CGEvent]) -> [Int64] {
events.filter { $0.type == .keyDown || $0.type == .keyUp }
.map { $0.getIntegerValueField(.keyboardEventKeycode) }
}
}
/// Real registration, coordinator preparation, and receipt validation; only
/// external focus acknowledgement and the underlying event stream are fake.
@MainActor
private final class KeyboardFocusTestRig {
let pid: pid_t = 42
let coordinator = SyntheticWindowFocus.Coordinator()
let stream: KeyboardFocusTestStream
let monitor: FocusEventMonitor
private(set) var preparations = 0
private var acknowledged = false
init() {
let stream = KeyboardFocusTestStream()
self.stream = stream
monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, isFocusObserver: { _ in false },
makeStream: { stream }, releaseFocus: { _ in true },
readRealFrontmost: { 9 }, isOrdinaryApp: { _ in true },
readProcessIdentity: { .init(executablePath: "/test/\($0)", launchTime: 1) }
)
}
func register() -> Bool {
let pid = pid
return monitor.register(pid: pid) { [coordinator] in
coordinator.observeFocus(pid: pid, hasFocus: $0)
}
}
func prepare() async throws -> FocusEventMonitor.RegistrationReceipt {
XCTAssertTrue(register())
let receipt = try XCTUnwrap(monitor.registrationReceipt(pid: pid))
try await coordinator.prepare(pid: pid, window: nil, runtime: .init(
identity: { _ in
AXTreeProcessIdentity(bundleID: "com.example.keyboard-target",
executablePath: "/test/42", launchTime: 1)
},
isActive: { _ in false },
hasFocus: { [monitor] in monitor.isAppCurrentlyFocused(pid: $0) },
acceptsInput: { [self] _ in acknowledged },
post: { [self] _, _, _ in acknowledged = true; return true },
pause: { await Task.yield() },
validateContinuity: { [monitor] in
try SyntheticWindowFocus.validate(receipt, monitor: monitor)
}
))
preparations += 1
return receipt
}
}
private final class KeyboardFocusTestStream: FocusEventMonitor.Stream, @unchecked Sendable {
private let interruption = OSAllocatedUnfairLock<(@Sendable (String) -> Void)?>(initialState: nil)
func start(
receive: @escaping @Sendable (FocusEventMonitor.Event) -> FocusEventMonitor.Disposition,
interrupted: @escaping @Sendable (String) -> Void
) -> Bool {
interruption.withLock { $0 = interrupted }
return true
}
func addProtectedPID(_ pid: pid_t) -> Bool { true }
func stop() {}
func interrupt() { interruption.withLock { $0 }?("keyboard_test_interruption") }
}
@@ -0,0 +1,385 @@
import AppKit
import CoreGraphics
import XCTest
@testable import cc_haha_computer_use
final class MouseEventBurstDeliveryTests: XCTestCase {
private enum ExpectedError: Error, Equatable {
case invalidTarget
case pauseFailed
case cleanupFailed
}
@MainActor
func testProductionClickAllocationAndDeliveryContainOnlyMatchingDownUpPairs() async throws {
let source = try XCTUnwrap(CGEventSource(stateID: .privateState))
let window = WindowGeometry.Window(
id: 123, bounds: CGRect(x: 200, y: 300, width: 800, height: 600), ownerPid: 456
)
let point = CGPoint(x: 410, y: 349)
for button: MouseButton in [.left, .right, .middle] {
for count in [1, 2] {
let events = try AXAction.clickEvents(
at: point, clickCount: count, button: button,
source: source, pid: window.ownerPid, window: window
)
var posted: [CGEvent] = []
var validations = 0
try await MouseEventBurstDelivery.deliver(
events: events,
validate: { validations += 1 },
post: { posted.append($0) },
release: { _, _ in XCTFail("a complete click must not need cleanup") },
pause: {}
)
XCTAssertEqual(posted.map(\.type), (0..<count).flatMap { _ in [button.down, button.up] })
XCTAssertEqual(validations, count * 2)
for (index, event) in posted.enumerated() {
XCTAssertTrue(event === events[index], "delivery must use the production allocation")
XCTAssertEqual(event.location, point)
XCTAssertEqual(event.getIntegerValueField(.mouseEventClickState), Int64(index / 2 + 1))
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 91)!), Int64(window.id))
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 92)!), Int64(window.id))
XCTAssertEqual(try XCTUnwrap(NSEvent(cgEvent: event)).windowNumber, Int(window.id))
}
let numbers = posted.map { $0.getIntegerValueField(.mouseEventNumber) }
for index in stride(from: 0, to: count * 2, by: 2) {
XCTAssertEqual(numbers[index], numbers[index + 1])
}
XCTAssertEqual(Set(stride(from: 0, to: count * 2, by: 2).map { numbers[$0] }).count, count)
}
}
}
@MainActor
func testUnpacedProductionDoubleClickDoesNotYieldBetweenPairs() async throws {
let window = WindowGeometry.Window(
id: 123, bounds: CGRect(x: 200, y: 300, width: 800, height: 600), ownerPid: 456
)
let events = try AXAction.clickEvents(
at: CGPoint(x: 410, y: 349), clickCount: 2, button: .left,
source: XCTUnwrap(CGEventSource(stateID: .privateState)), pid: window.ownerPid, window: window
)
var queuedTask: Task<Void, Never>?
var yielded = false
var posted: [CGEventType] = []
var trace: [String] = []
try await MouseEventBurstDelivery.deliver(
events: events,
validate: { trace.append("validate") },
post: {
XCTAssertFalse(yielded, "ordinary click pairs must stay in one main-actor turn")
posted.append($0.type)
trace.append("post")
if posted.count == 1 {
queuedTask = Task { @MainActor in yielded = true }
}
},
release: { _, _ in XCTFail("complete double-click already released its buttons") }
)
XCTAssertEqual(posted, [.leftMouseDown, .leftMouseUp, .leftMouseDown, .leftMouseUp])
XCTAssertEqual(trace, Array(repeating: ["validate", "post"], count: 4).flatMap { $0 })
await queuedTask?.value
XCTAssertTrue(yielded)
}
@MainActor
func testUnpacedCancellationBeforeFirstAfterDownAndAfterUpNeverStartsTheNextClick() async throws {
let events = try makeEvents([.leftMouseDown, .leftMouseUp, .leftMouseDown, .leftMouseUp])
for cancelAfterPost in 0...events.count {
var posted: [CGEventType] = []
var releases: [(CGEvent, CGPoint)] = []
let task = Task { @MainActor in
if cancelAfterPost == 0 { withUnsafeCurrentTask { $0?.cancel() } }
do {
try await MouseEventBurstDelivery.deliver(
events: events,
validate: {},
post: {
posted.append($0.type)
if posted.count == cancelAfterPost { withUnsafeCurrentTask { $0?.cancel() } }
},
release: { releases.append(($0, $1)) }
)
XCTFail("canceled unpaced delivery must throw, including after the final up")
} catch is CancellationError {
// Cancellation is observed even without a pause callback.
} catch {
XCTFail("unexpected error: \(error)")
}
}
await task.value
XCTAssertEqual(posted, events.prefix(cancelAfterPost).map(\.type))
XCTAssertEqual(releases.count, cancelAfterPost.isMultiple(of: 2) ? 0 : 1)
if let release = releases.first {
XCTAssertTrue(release.0 === events[cancelAfterPost - 1])
XCTAssertEqual(release.1, events[cancelAfterPost - 1].location)
}
}
}
@MainActor
func testUnpacedValidationFailureReleasesOnlyItsHeldDownAndPreservesOriginalError() async throws {
let events = try makeEvents([.leftMouseDown, .leftMouseUp, .leftMouseDown, .leftMouseUp])
var posted: [CGEventType] = []
var releases = 0
do {
try await MouseEventBurstDelivery.deliver(
events: events,
validate: { if !posted.isEmpty { throw ExpectedError.invalidTarget } },
post: { posted.append($0.type) },
release: { down, point in
XCTAssertTrue(down === events[0])
XCTAssertEqual(point, events[0].location)
releases += 1
throw ExpectedError.cleanupFailed
}
)
XCTFail("the invalid target must stop the burst")
} catch {
XCTAssertEqual(error as? ExpectedError, .invalidTarget)
}
XCTAssertEqual(posted, [.leftMouseDown])
XCTAssertEqual(releases, 1)
}
func testCoordinateClickUsesTheTestedFactoryAndDisablesDragPacing() throws {
let source = try String(contentsOf: URL(fileURLWithPath: #filePath)
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
.appendingPathComponent("Sources/cu-helper/AXAction.swift"), encoding: .utf8)
let start = try XCTUnwrap(source.range(of: "public static func clickPoint("))
let end = try XCTUnwrap(source.range(of: "static func clickEvents(", range: start.upperBound..<source.endIndex))
let body = String(source[start.upperBound..<end.lowerBound])
XCTAssertTrue(body.contains("let events = try clickEvents("))
XCTAssertTrue(body.contains("try await postMouseBurst("))
XCTAssertTrue(body.contains("pause: nil"), "click must not inherit drag's asynchronous pacing")
}
@MainActor
func testSuccessfulBurstValidatesEveryPostAndDoesNotReleaseTwice() async throws {
let events = try makeEvents([.mouseMoved, .leftMouseDown, .leftMouseDragged, .leftMouseUp])
var trace: [String] = []
var posted: [CGEventType] = []
try await MouseEventBurstDelivery.deliver(
events: events,
validate: { trace.append("validate") },
post: { posted.append($0.type); trace.append("post") },
release: { _, _ in XCTFail("normal mouse-up already released the button") },
pause: { trace.append("pause") }
)
XCTAssertEqual(posted, events.map(\.type))
XCTAssertEqual(trace, Array(repeating: ["validate", "post", "pause"], count: events.count).flatMap { $0 })
}
@MainActor
func testCancellationBeforeFirstEventAfterMoveAfterDownAndAfterUpStopsTheBurst() async throws {
let events = try makeEvents([
.mouseMoved, .leftMouseDown, .leftMouseUp,
.leftMouseDown, .leftMouseUp,
])
// Exercise real Task cancellation, including a pause that returns
// normally instead of throwing cancellation itself.
for cancelAfterPost in 0...events.count {
var posted: [CGEventType] = []
var releases: [(CGEvent, CGPoint)] = []
var validations = 0
let task = Task { @MainActor in
if cancelAfterPost == 0 { withUnsafeCurrentTask { $0?.cancel() } }
do {
try await MouseEventBurstDelivery.deliver(
events: events,
validate: { validations += 1 },
post: { posted.append($0.type) },
release: { releases.append(($0, $1)) },
pause: {
if posted.count == cancelAfterPost {
withUnsafeCurrentTask { $0?.cancel() }
}
}
)
XCTFail("cancelled delivery must not report success")
} catch is CancellationError {
// Expected: only a down that was actually posted needs cleanup.
} catch {
XCTFail("unexpected error: \(error)")
}
}
await task.value
XCTAssertEqual(posted, events.prefix(cancelAfterPost).map(\.type))
XCTAssertEqual(validations, cancelAfterPost)
let heldIndex: Int? = switch cancelAfterPost {
case 2: 1
case 4: 3
default: nil
}
XCTAssertEqual(releases.count, heldIndex == nil ? 0 : 1)
if let release = releases.first {
let index = try XCTUnwrap(heldIndex)
XCTAssertTrue(release.0 === events[index])
XCTAssertEqual(release.1, events[index].location)
}
}
}
@MainActor
func testDragCleanupUsesLastPostedPointNotUnsentDestination() async throws {
let events = try makeEvents([.leftMouseDown, .leftMouseDragged, .leftMouseDragged, .leftMouseUp])
var posted: [CGEventType] = []
var releases: [(CGEvent, CGPoint)] = []
do {
try await MouseEventBurstDelivery.deliver(
events: events,
validate: {},
post: { posted.append($0.type) },
release: { releases.append(($0, $1)) },
pause: { if posted.count == 2 { throw ExpectedError.pauseFailed } }
)
XCTFail("expected the pause to fail")
} catch {
XCTAssertEqual(error as? ExpectedError, .pauseFailed)
}
XCTAssertEqual(posted, [.leftMouseDown, .leftMouseDragged])
XCTAssertEqual(releases.count, 1)
let release = try XCTUnwrap(releases.first)
XCTAssertTrue(release.0 === events[0])
XCTAssertEqual(release.1, events[1].location)
XCTAssertNotEqual(release.1, events.last?.location)
}
@MainActor
func testValidationBeforeFirstPostRejectsWithoutCleanup() async throws {
var posted = false
var released = false
do {
try await MouseEventBurstDelivery.deliver(
events: makeEvents([.leftMouseDown, .leftMouseUp]),
validate: { throw ExpectedError.invalidTarget },
post: { _ in posted = true },
release: { _, _ in released = true },
pause: { XCTFail("no event was sent") }
)
XCTFail("expected validation failure")
} catch {
XCTAssertEqual(error as? ExpectedError, .invalidTarget)
}
XCTAssertFalse(posted)
XCTAssertFalse(released)
}
@MainActor
func testCancellationDuringValidationDoesNotPostTheValidatedEvent() async throws {
let events = try makeEvents([.leftMouseDown, .leftMouseUp])
var posted = false
let task = Task { @MainActor in
do {
try await MouseEventBurstDelivery.deliver(
events: events,
validate: { withUnsafeCurrentTask { $0?.cancel() } },
post: { _ in posted = true },
release: { _, _ in XCTFail("no down was posted") },
pause: { XCTFail("no event was posted") }
)
XCTFail("cancelled validation must prevent posting")
} catch is CancellationError {
// Expected.
} catch {
XCTFail("unexpected error: \(error)")
}
}
await task.value
XCTAssertFalse(posted)
}
@MainActor
func testValidationAfterYieldStopsAndReleasesOnlyPreviouslyPostedDown() async throws {
let events = try makeEvents([.rightMouseDown, .rightMouseDragged, .rightMouseUp])
var targetValid = true
var posted: [CGEventType] = []
var releases: [(CGEvent, CGPoint)] = []
do {
try await MouseEventBurstDelivery.deliver(
events: events,
validate: { if !targetValid { throw ExpectedError.invalidTarget } },
post: { posted.append($0.type) },
release: { releases.append(($0, $1)) },
pause: { targetValid = false }
)
XCTFail("expected post-yield validation failure")
} catch {
XCTAssertEqual(error as? ExpectedError, .invalidTarget)
}
XCTAssertEqual(posted, [.rightMouseDown])
XCTAssertEqual(releases.count, 1)
XCTAssertTrue(releases.first?.0 === events[0])
XCTAssertEqual(releases.first?.1, events[0].location)
}
@MainActor
func testCleanupFailureDoesNotReplaceTheOriginalErrorOrRetryRelease() async throws {
var posted: [CGEventType] = []
var releaseCount = 0
do {
try await MouseEventBurstDelivery.deliver(
events: makeEvents([.otherMouseDown, .otherMouseDragged, .otherMouseUp]),
validate: {},
post: { posted.append($0.type) },
release: { down, _ in
XCTAssertEqual(down.type, .otherMouseDown)
releaseCount += 1
throw ExpectedError.cleanupFailed
},
pause: { throw ExpectedError.pauseFailed }
)
XCTFail("expected pause failure")
} catch {
XCTAssertEqual(error as? ExpectedError, .pauseFailed)
}
XCTAssertEqual(posted, [.otherMouseDown])
XCTAssertEqual(releaseCount, 1)
}
@MainActor
func testMatchingRightAndOtherMouseUpClearHeldStateBeforePauseFailure() async throws {
for pair: [CGEventType] in [[.rightMouseDown, .rightMouseUp], [.otherMouseDown, .otherMouseUp]] {
var posted: [CGEventType] = []
do {
try await MouseEventBurstDelivery.deliver(
events: makeEvents(pair),
validate: {},
post: { posted.append($0.type) },
release: { _, _ in XCTFail("button has already been released") },
pause: { if posted.count == 2 { throw ExpectedError.pauseFailed } }
)
XCTFail("expected final pause failure")
} catch {
XCTAssertEqual(error as? ExpectedError, .pauseFailed)
}
XCTAssertEqual(posted, pair)
}
}
private func makeEvents(_ types: [CGEventType]) throws -> [CGEvent] {
let source = try XCTUnwrap(CGEventSource(stateID: .privateState))
return try types.enumerated().map { index, type in
let button: CGMouseButton
switch type {
case .rightMouseDown, .rightMouseUp, .rightMouseDragged: button = .right
case .otherMouseDown, .otherMouseUp, .otherMouseDragged: button = .center
default: button = .left
}
return try XCTUnwrap(CGEvent(
mouseEventSource: source, mouseType: type,
mouseCursorPosition: CGPoint(x: 10 + index * 20, y: 20 + index * 15),
mouseButton: button
))
}
}
}
@@ -0,0 +1,290 @@
import AppKit
import XCTest
@testable import cc_haha_computer_use
final class PreFocusPointerMoveTests: XCTestCase {
private enum ExpectedError: Error { case staleTarget }
func testProductionClickRoutesItsPointerMoveThroughProtectedPreFocusPreparation() throws {
let root = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
.appendingPathComponent("Sources/cu-helper")
let source = try String(contentsOf: root.appendingPathComponent("AXAction.swift"), encoding: .utf8)
let start = try XCTUnwrap(source.range(of: "public static func clickPoint("))
let end = try XCTUnwrap(source.range(of: "static func clickEvents(", range: start.upperBound..<source.endIndex))
let click = String(source[start.upperBound..<end.lowerBound])
XCTAssertTrue(click.contains("beforeFocus:"), "the monitor must be installed before the pointer move")
XCTAssertTrue(click.contains("try await movePointerBeforeFocus("), "visual cursor motion does not deliver the target's hover event")
let focus = try String(contentsOf: root.appendingPathComponent("SyntheticWindowFocus.swift"), encoding: .utf8)
XCTAssertTrue(focus.contains("try await beforeFocus?(receipt)"), "focus preparation must invoke the callback with its original receipt")
}
@MainActor
func testProtectedProductionMovePrecedesFocusThenUnpacedClickAndPreservesReceipt() async throws {
let fixture = Fixture()
var originalReceipt: FocusEventMonitor.RegistrationReceipt?
let receipt = try await fixture.prepare { receipt in
originalReceipt = receipt
XCTAssertTrue(fixture.stream.protectedPIDs.contains(fixture.window.ownerPid))
XCTAssertTrue(fixture.monitor.isRegistrationCurrent(receipt))
try await AXAction.movePointerBeforeFocus(
at: fixture.point, window: fixture.window,
validate: { try SyntheticWindowFocus.validate(receipt, monitor: fixture.monitor) },
post: { fixture.events.append($0); fixture.trace.append("move") },
pause: {
XCTAssertEqual($0, .milliseconds(10))
fixture.trace.append("hover-wait")
await Task.yield()
}
)
}
XCTAssertEqual(receipt, originalReceipt)
let click = try AXAction.clickEvents(
at: fixture.point, clickCount: 1, button: .left,
source: XCTUnwrap(CGEventSource(stateID: .privateState)),
pid: fixture.window.ownerPid, window: fixture.window
)
try await MouseEventBurstDelivery.deliver(
events: click,
validate: { try SyntheticWindowFocus.validate(receipt, monitor: fixture.monitor) },
post: { fixture.events.append($0) },
release: { _, _ in XCTFail("a complete click does not need cleanup") }
)
XCTAssertEqual(fixture.trace, ["move", "hover-wait", "focus", "acceptance-wait"])
XCTAssertEqual(fixture.events.map(\.type.rawValue), [5, 13, 1, 2, 1, 2])
let move = try XCTUnwrap(fixture.events.first)
XCTAssertEqual(move.location, fixture.point)
XCTAssertEqual(move.flags.rawValue, 0)
XCTAssertEqual(move.getIntegerValueField(.mouseEventClickState), 1)
XCTAssertEqual(move.getIntegerValueField(.mouseEventButtonNumber), 0)
XCTAssertEqual(move.getIntegerValueField(.mouseEventSubtype), 3)
XCTAssertEqual(move.getIntegerValueField(CGEventField(rawValue: 91)!), Int64(fixture.window.id))
XCTAssertEqual(move.getIntegerValueField(CGEventField(rawValue: 92)!), Int64(fixture.window.id))
XCTAssertEqual(try XCTUnwrap(NSEvent(cgEvent: move)).windowNumber, Int(fixture.window.id))
XCTAssertEqual(click[0].getIntegerValueField(.mouseEventNumber), move.getIntegerValueField(.mouseEventNumber) + 1)
XCTAssertEqual(click[0].getIntegerValueField(.mouseEventNumber), click[1].getIntegerValueField(.mouseEventNumber))
}
@MainActor
func testFailedMonitorInstallationCannotCallThePointerPreparation() async throws {
let fixture = Fixture()
fixture.stream.startsSuccessfully = false
do {
_ = try await fixture.prepare { _ in XCTFail("input must not precede protection") }
XCTFail("missing protection must fail closed")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_monitor_unavailable")
}
XCTAssertTrue(fixture.events.isEmpty)
}
@MainActor
func testPointerAllocationFailureCannotPostOrPrepareFocus() async throws {
let fixture = Fixture()
do {
_ = try await fixture.prepare { receipt in
try await AXAction.movePointerBeforeFocus(
at: fixture.point, window: fixture.window,
validate: { try SyntheticWindowFocus.validate(receipt, monitor: fixture.monitor) },
post: { _ in XCTFail("a failed allocation cannot post") },
pause: { _ in XCTFail("a failed allocation cannot wait") },
makeEvent: { _, _ in nil }
)
}
XCTFail("failed hover allocation must stop before activation")
} catch let error as CUError {
XCTAssertEqual(error.code, CUError.Code.eventAlloc)
}
XCTAssertTrue(fixture.events.isEmpty)
}
@MainActor
func testTargetValidationBeforeAllocationBeforePostAndAfterWaitStopsTheAction() async throws {
for failureAt in 1...3 {
let fixture = Fixture()
var validations = 0
do {
_ = try await fixture.prepare { _ in
try await AXAction.movePointerBeforeFocus(
at: fixture.point, window: fixture.window,
validate: {
validations += 1
if validations == failureAt { throw ExpectedError.staleTarget }
},
post: { fixture.events.append($0) },
pause: { _ in await Task.yield() }
)
}
XCTFail("a stale target cannot progress to focus preparation")
} catch ExpectedError.staleTarget {}
XCTAssertEqual(fixture.events.map(\.type), failureAt == 3 ? [.mouseMoved] : [])
XCTAssertTrue(fixture.trace.isEmpty)
}
}
@MainActor
func testMonitorRestartDuringPointerWaitCannotSubstituteANewReceipt() async throws {
let fixture = Fixture()
var oldReceipt: FocusEventMonitor.RegistrationReceipt?
do {
_ = try await fixture.prepare { receipt in
oldReceipt = receipt
try await AXAction.movePointerBeforeFocus(
at: fixture.point, window: fixture.window,
validate: { try SyntheticWindowFocus.validate(receipt, monitor: fixture.monitor) },
post: { fixture.events.append($0) },
pause: { _ in
await Task.yield()
fixture.stream.interrupt()
XCTAssertTrue(fixture.monitor.register(pid: fixture.window.ownerPid) { _ in })
XCTAssertNotEqual(fixture.monitor.registrationReceipt(pid: fixture.window.ownerPid), receipt)
}
)
}
XCTFail("a healthy replacement monitor cannot authorize this in-flight click")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_monitor_interrupted")
}
XCTAssertFalse(fixture.monitor.isRegistrationCurrent(try XCTUnwrap(oldReceipt)))
XCTAssertEqual(fixture.events.map(\.type), [.mouseMoved])
XCTAssertTrue(fixture.trace.isEmpty)
}
@MainActor
func testPreparationItselfRejectsAReceiptLostInsideTheCallback() async throws {
let fixture = Fixture()
do {
_ = try await fixture.prepare { _ in
fixture.stream.interrupt()
XCTAssertTrue(fixture.monitor.register(pid: fixture.window.ownerPid) { _ in })
}
XCTFail("the outer preparation must not trust a callback's success")
} catch let error as CUError {
XCTAssertEqual(error.code, "focus_monitor_interrupted")
}
XCTAssertTrue(fixture.events.isEmpty)
}
@MainActor
func testCancellationBeforeMoveAndDuringItsWaitNeverPostsActivation() async throws {
for cancelBeforeMove in [true, false] {
let fixture = Fixture()
let task = Task { @MainActor in
do {
_ = try await fixture.prepare { _ in
if cancelBeforeMove { withUnsafeCurrentTask { $0?.cancel() } }
try await AXAction.movePointerBeforeFocus(
at: fixture.point, window: fixture.window,
validate: {},
post: { fixture.events.append($0) },
pause: { _ in
withUnsafeCurrentTask { $0?.cancel() }
await Task.yield()
}
)
}
XCTFail("cancellation must stop the click")
} catch is CancellationError {} catch {
XCTFail("unexpected error: \(error)")
}
}
await task.value
XCTAssertEqual(fixture.events.map(\.type), cancelBeforeMove ? [] : [.mouseMoved])
XCTAssertTrue(fixture.trace.isEmpty)
}
}
@MainActor
func testCancellationInsideTheLastValidationCannotPostThePointerMove() async throws {
let fixture = Fixture()
var validations = 0
let task = Task { @MainActor in
do {
try await AXAction.movePointerBeforeFocus(
at: fixture.point, window: fixture.window,
validate: {
validations += 1
if validations == 2 { withUnsafeCurrentTask { $0?.cancel() } }
},
post: { _ in XCTFail("validation canceled this input") },
pause: { _ in XCTFail("an unposted move must not wait") }
)
XCTFail("cancellation must be observed after validation")
} catch is CancellationError {} catch {
XCTFail("unexpected error: \(error)")
}
}
await task.value
XCTAssertEqual(validations, 2)
}
@MainActor
func testPreparationWithoutCallbackKeepsTheExistingFocusPath() async throws {
let fixture = Fixture()
let receipt = try await fixture.prepare()
XCTAssertTrue(fixture.monitor.isRegistrationCurrent(receipt))
XCTAssertEqual(fixture.trace, ["focus", "acceptance-wait"])
XCTAssertEqual(fixture.events.map(\.type.rawValue), [13, 1, 2])
}
}
private final class PointerFocusStream: FocusEventMonitor.Stream, @unchecked Sendable {
var startsSuccessfully = true
private(set) var protectedPIDs: [pid_t] = []
private var interrupted: (@Sendable (String) -> Void)?
func start(
receive: @escaping @Sendable (FocusEventMonitor.Event) -> FocusEventMonitor.Disposition,
interrupted: @escaping @Sendable (String) -> Void
) -> Bool {
self.interrupted = interrupted
return startsSuccessfully
}
func addProtectedPID(_ pid: pid_t) -> Bool { protectedPIDs.append(pid); return true }
func stop() {}
func interrupt() { interrupted?("test_interruption") }
}
@MainActor
private final class Fixture {
let window = WindowGeometry.Window(id: 123, bounds: CGRect(x: 200, y: 300, width: 800, height: 600), ownerPid: 42)
let point = CGPoint(x: 410, y: 349)
let coordinator = SyntheticWindowFocus.Coordinator()
let stream = PointerFocusStream()
let monitor: FocusEventMonitor
var events: [CGEvent] = []
var trace: [String] = []
init() {
let stream = stream
monitor = FocusEventMonitor(
helperPID: 700, readInitialFocus: { 9 }, isFocusObserver: { $0 == 901 },
makeStream: { stream }, releaseFocus: { _ in true },
readRealFrontmost: { 9 }, isOrdinaryApp: { _ in true },
readProcessIdentity: { .init(executablePath: "/test/\($0)", launchTime: 1) }
)
}
func prepare(
beforeFocus: (@MainActor (FocusEventMonitor.RegistrationReceipt) async throws -> Void)? = nil
) async throws -> FocusEventMonitor.RegistrationReceipt {
let context = SyntheticWindowFocus.Window(id: window.id, bounds: window.bounds, activationPoint: CGPoint(x: -1, y: 1118))
return try await SyntheticWindowFocus.prepareInput(
pid: window.ownerPid, window: context, monitor: monitor, coordinator: coordinator,
runtime: .init(
identity: { _ in .init(bundleID: "com.example.pointer", executablePath: "/test/42", launchTime: 1) },
isActive: { _ in false }, hasFocus: { _ in false }, acceptsInput: { _ in true },
post: { [self] establishment, _, window in
XCTAssertEqual(establishment, .activate)
guard let activation = SyntheticWindowFocus.activationEvents(window: window) else { return false }
trace.append("focus")
events.append(contentsOf: activation)
return true
},
pause: { [self] in await MainActor.run { trace.append("acceptance-wait") } }
),
beforeFocus: beforeFocus
)
}
}
@@ -0,0 +1,205 @@
import AppKit
import CoreGraphics
import XCTest
@testable import cc_haha_computer_use
@MainActor
final class SnapshotKeyboardWindowTests: XCTestCase {
private let pid: pid_t = 66_984
private let mainID: CGWindowID = 2_389
private let auxiliaryID: CGWindowID = 2_392
private let identity = AXTreeProcessIdentity(
bundleID: "com.netease.163music",
executablePath: "/Applications/NeteaseMusic.app/Contents/MacOS/NeteaseMusic",
launchTime: 1
)
func testAuxiliaryWindowAtTheFrontDoesNotReplaceThePublishedKeyboardWindow() throws {
let windows = [auxiliaryWindow(), mainWindow()]
XCTAssertEqual(WindowGeometry.frontmostWindow(pid: pid, windowList: { windows })?.id, auxiliaryID)
let selected = try resolve(snapshotID: mainID, windows: windows)
XCTAssertEqual(selected.id, mainID)
XCTAssertEqual(selected.bounds, CGRect(x: 333, y: 199, width: 1063, height: 752))
XCTAssertEqual(selected.ownerPid, pid)
}
func testReorderingCGWindowsDoesNotChangeTheSnapshotTarget() throws {
for windows in [[mainWindow(), auxiliaryWindow()], [auxiliaryWindow(), mainWindow()]] {
XCTAssertEqual(try resolve(snapshotID: mainID, windows: windows).id, mainID)
}
}
func testASnapshotOfTheSmallWindowIsNotReplacedByTheLargestWindow() throws {
let selected = try resolve(snapshotID: auxiliaryID, windows: [mainWindow(), auxiliaryWindow()])
XCTAssertEqual(selected.id, auxiliaryID)
XCTAssertEqual(selected.bounds, CGRect(x: 343, y: 175, width: 66, height: 20))
}
func testClosingTheMainWindowRequiresANewSnapshotInsteadOfFallingBackToAuxiliary() {
assertError("stale_window") {
try resolve(snapshotID: mainID, windows: [auxiliaryWindow()])
}
}
func testExactLookupCannotReturnAWindowOwnedByAnotherPID() {
assertError("stale_window") {
try SnapshotKeyboardWindow.resolve(
pid: pid, snapshot: evidence(mainID), currentIdentity: identity,
windowForID: { id, owner in
XCTAssertEqual(id, self.mainID)
XCTAssertEqual(owner, self.pid)
return .init(id: id, bounds: CGRect(x: 333, y: 199, width: 1063, height: 752), ownerPid: owner + 1)
}
)
}
}
func testExactLookupCannotSubstituteADifferentWindowID() {
assertError("stale_window") {
try SnapshotKeyboardWindow.resolve(
pid: pid, snapshot: evidence(mainID), currentIdentity: identity,
windowForID: { _, owner in
.init(id: self.auxiliaryID, bounds: CGRect(x: 343, y: 175, width: 66, height: 20), ownerPid: owner)
}
)
}
}
func testMissingOrZeroSnapshotWindowNeverCallsTheWindowLookup() {
for windowID: CGWindowID? in [nil, 0] {
assertError("stale_window") {
try SnapshotKeyboardWindow.resolve(
pid: pid, snapshot: evidence(windowID), currentIdentity: identity,
windowForID: { _, _ in XCTFail("there is no exact window to look up"); return nil }
)
}
}
}
func testNoSnapshotDoesNotInferAWindowFromTheRunningProcess() {
assertError("stale_snapshot") {
try SnapshotKeyboardWindow.resolve(
pid: pid, snapshot: nil, currentIdentity: identity,
windowForID: { _, _ in XCTFail("no published snapshot means no lookup"); return nil }
)
}
}
func testReusedPIDCannotInheritThePreviousProcessWindowSnapshot() {
let replacement = AXTreeProcessIdentity(
bundleID: identity.bundleID, executablePath: identity.executablePath, launchTime: 2
)
assertError("stale_process") {
try SnapshotKeyboardWindow.resolve(
pid: pid, snapshot: evidence(mainID), currentIdentity: replacement,
windowForID: { _, _ in XCTFail("process evidence must be checked before window lookup"); return nil }
)
}
}
func testANewSnapshotForTheReplacementProcessCanUseItsProvenWindow() throws {
let replacement = AXTreeProcessIdentity(
bundleID: identity.bundleID, executablePath: identity.executablePath, launchTime: 2
)
let windows = [auxiliaryWindow(), mainWindow()]
let selected = try SnapshotKeyboardWindow.resolve(
pid: pid,
snapshot: .init(processIdentity: replacement, keyWindowID: mainID),
currentIdentity: replacement,
windowForID: { id, owner in WindowGeometry.window(id: id, pid: owner, windowList: { windows }) }
)
XCTAssertEqual(selected.id, mainID)
}
func testSnapshotWindowSurvivesRealActivationAndCoverInTheFocusPreparationJoin() async throws {
let coordinator = SyntheticWindowFocus.Coordinator()
let center = NotificationCenter()
let observer = SyntheticWindowFocus.ApplicationLifecycleObserver(center: center, coordinator: coordinator)
defer { withExtendedLifetime(observer) {} }
let application = SnapshotWindowNotificationApplication(pid: pid)
let observations = KeyboardWindowFocusObservations()
let identity = identity
let runtime = SyntheticWindowFocus.Runtime(
identity: { _ in identity },
isActive: { _ in observations.active },
hasFocus: { _ in observations.focused },
acceptsInput: { _ in observations.accepted },
post: { _, _, window in
observations.postedWindowIDs.append(window?.id)
observations.focused = true
observations.accepted = true
return true
},
pause: { await Task.yield() }
)
for cycle in 0..<2 {
let windows = cycle == 0 ? [mainWindow(), auxiliaryWindow()] : [auxiliaryWindow(), mainWindow()]
let target = try resolve(snapshotID: mainID, windows: windows)
let focusWindow = SyntheticWindowFocus.Window(id: target.id, bounds: target.bounds, activationPoint: nil)
try await coordinator.prepare(pid: pid, window: focusWindow, runtime: runtime)
try await coordinator.prepare(pid: pid, window: focusWindow, runtime: runtime)
if cycle == 0 {
observations.active = true
center.post(name: NSWorkspace.didActivateApplicationNotification, object: nil,
userInfo: [NSWorkspace.applicationUserInfoKey: application])
observations.active = false
observations.focused = false
observations.accepted = false
center.post(name: NSWorkspace.didDeactivateApplicationNotification, object: nil,
userInfo: [NSWorkspace.applicationUserInfoKey: application])
}
}
XCTAssertEqual(observations.postedWindowIDs, [mainID, mainID])
}
private func resolve(snapshotID: CGWindowID?, windows: [[CFString: Any]]) throws -> WindowGeometry.Window {
try SnapshotKeyboardWindow.resolve(
pid: pid, snapshot: evidence(snapshotID), currentIdentity: identity,
windowForID: { id, owner in WindowGeometry.window(id: id, pid: owner, windowList: { windows }) }
)
}
private func evidence(_ id: CGWindowID?) -> AXTreeSnapshotEvidence {
.init(processIdentity: identity, keyWindowID: id)
}
private func mainWindow() -> [CFString: Any] {
window(id: mainID, x: 333, y: 199, width: 1063, height: 752)
}
private func auxiliaryWindow() -> [CFString: Any] {
window(id: auxiliaryID, x: 343, y: 175, width: 66, height: 20)
}
private func window(id: CGWindowID, x: CGFloat, y: CGFloat, width: CGFloat, height: CGFloat) -> [CFString: Any] {
[
kCGWindowNumber: Int(id), kCGWindowOwnerPID: pid, kCGWindowLayer: 0,
kCGWindowBounds: ["X": x, "Y": y, "Width": width, "Height": height] as [String: CGFloat],
]
}
private func assertError(_ code: String, operation: () throws -> WindowGeometry.Window) {
XCTAssertThrowsError(try operation()) { error in
XCTAssertEqual((error as? CUError)?.code, code)
XCTAssertTrue((error as? CUError)?.message.contains("get_app_state") == true)
}
}
}
@MainActor
private final class KeyboardWindowFocusObservations {
var active = false
var focused = false
var accepted = false
var postedWindowIDs: [CGWindowID?] = []
}
private final class SnapshotWindowNotificationApplication: NSRunningApplication, @unchecked Sendable {
private let fixturePID: pid_t
init(pid: pid_t) { fixturePID = pid; super.init() }
override var processIdentifier: pid_t { fixturePID }
}
@@ -1,39 +1,13 @@
import AppKit
import CoreGraphics
import os
import XCTest
@testable import cc_haha_computer_use
/// The point of this type is that automating an app must not cost the user
/// their foreground. The previous implementation made the target genuinely
/// frontmost for every click — measured, `Finder → NeteaseMusic` — which is the
/// opposite of "it works in the background while I do something else".
///
/// Nothing here can prove the target *acts* on the notification; that needs a
/// real app. What is worth pinning is the one value that is undocumented, has
/// no error path, and silently means nothing if it is wrong.
/// Construct real events without posting them. Delivery and foreground
/// preservation remain real-machine acceptance requirements.
final class SyntheticWindowFocusTests: XCTestCase {
private let processA = AXTreeProcessIdentity(
bundleID: "com.example.target",
executablePath: "/Applications/Target.app/Contents/MacOS/Target",
launchTime: 1
)
private func beliefTarget(
processIdentity: AXTreeProcessIdentity? = nil
) -> SyntheticWindowFocus.BeliefTarget {
SyntheticWindowFocus.BeliefTarget(
processIdentity: processIdentity ?? processA
)
}
func testKeyFocusReturnedSurvivesTheSignedSubtypeField() {
// `NSEvent.subtype` is Int16. 0x8000 does not fit, and the obvious
// conversions either trap or clamp to 0x7FFF; truncating to the same
// bit pattern is the only one that sends the notification we mean.
// Getting this wrong sends subtype 0 — accepted, ignored, no error, and
// the only symptom is that background clicks stop landing.
XCTAssertEqual(SyntheticWindowFocus.Notification.keyFocusReturned.subtype, Int16(bitPattern: 0x8000))
XCTAssertEqual(
UInt16(bitPattern: SyntheticWindowFocus.Notification.keyFocusReturned.subtype),
@@ -42,9 +16,6 @@ final class SyntheticWindowFocusTests: XCTestCase {
}
func testTheNotificationValuesMatchTheOnesRecoveredFromCodex() {
// Recovered from the once-initializers in Codex's CU service. They are
// not derivable from any header, so a "tidy-up" that renumbers them
// would be undetectable at runtime.
XCTAssertEqual(SyntheticWindowFocus.Notification.appActivated.rawValue, 1)
XCTAssertEqual(SyntheticWindowFocus.Notification.appDeactivated.rawValue, 2)
XCTAssertEqual(SyntheticWindowFocus.Notification.lostKeyFocus.rawValue, 0x1000)
@@ -52,422 +23,230 @@ final class SyntheticWindowFocusTests: XCTestCase {
XCTAssertEqual(SyntheticWindowFocus.Notification.keyFocusReturned.rawValue, 0x8000)
}
/// The carrier event type is not the same for every notification, and this
/// test used to assert that it was.
///
/// Everything here posted on `.appKitDefined` (13), which is right for the
/// activation pair and wrong for the key-focus family — Codex's
/// `enforceActiveState` loads type 21 from the same lazily-initialized
/// global that holds the 0x8000 subtype, and hardcodes 13 only for
/// `appActivated`. On the wrong carrier the subtype names nothing the
/// target handles: accepted, ignored, no error, and background input simply
/// never lands.
func testEachNotificationTravelsOnItsOwnCarrierType() {
XCTAssertEqual(SyntheticWindowFocus.Notification.appActivated.carrierEventType, .appKitDefined)
XCTAssertEqual(SyntheticWindowFocus.Notification.appDeactivated.carrierEventType, .appKitDefined)
for keyFocus: SyntheticWindowFocus.Notification in [.keyFocusReturned, .keyFocusTaken, .lostKeyFocus] {
XCTAssertEqual(
keyFocus.carrierEventType?.rawValue,
21,
"the key-focus family does not travel on .appKitDefined"
)
for notification: SyntheticWindowFocus.Notification in [.keyFocusReturned, .keyFocusTaken, .lostKeyFocus] {
XCTAssertEqual(notification.carrierEventType?.rawValue, 21)
}
}
func testTheKeyFocusCarrierIsAcceptedByAppKit() throws {
// 21 has no name in the public NSEventType, so the thing worth pinning
// is that AppKit still builds and converts it. If an OS update ever
// rejects it, this fails here rather than silently degrading into
// clicks that go nowhere.
let carrier = try XCTUnwrap(
SyntheticWindowFocus.Notification.keyFocusReturned.carrierEventType,
"NSEvent.EventType no longer accepts the key-focus carrier"
func testTheKeyFocusCarrierSurvivesTheProductionAppKitBridge() throws {
let event = try XCTUnwrap(SyntheticWindowFocus.notificationEvent(.keyFocusReturned))
let native = try XCTUnwrap(NSEvent(cgEvent: event))
XCTAssertEqual(native.type.rawValue, 21)
XCTAssertEqual(native.subtype.rawValue, Int16(bitPattern: 0x8000))
XCTAssertEqual(native.windowNumber, 0)
}
func testNotificationRetainsTheExplicitWindowAndFlags() throws {
let event = try XCTUnwrap(SyntheticWindowFocus.notificationEvent(
.appActivated, windowID: 42, flags: NSEvent.ModifierFlags(rawValue: 0xc0000)
))
let native = try XCTUnwrap(NSEvent(cgEvent: event))
XCTAssertEqual(native.type, .appKitDefined)
XCTAssertEqual(native.subtype.rawValue, 1)
XCTAssertEqual(native.windowNumber, 42)
XCTAssertEqual(native.modifierFlags.rawValue, 0xc0000)
}
func testMissingWindowProducesOnlyTheGenericActivationNotification() throws {
let events = try XCTUnwrap(SyntheticWindowFocus.activationEvents(window: nil))
XCTAssertEqual(events.count, 1)
let native = try XCTUnwrap(events.first.flatMap { NSEvent(cgEvent: $0) })
XCTAssertEqual(native.type, .appKitDefined)
XCTAssertEqual(native.subtype.rawValue, 1)
XCTAssertEqual(native.windowNumber, 0)
XCTAssertEqual(native.modifierFlags.rawValue, 0)
}
func testMissingActivationPointFallsBackToGenericActivationWithoutAClick() throws {
let window = SyntheticWindowFocus.Window(
id: 42, bounds: CGRect(x: 100, y: 200, width: 640, height: 480), activationPoint: nil
)
let event = try XCTUnwrap(
NSEvent.otherEvent(
with: carrier,
location: .zero,
modifierFlags: [],
timestamp: 0,
windowNumber: 0,
context: nil,
subtype: SyntheticWindowFocus.Notification.keyFocusReturned.subtype,
data1: 0,
data2: 0
)
let events = try XCTUnwrap(SyntheticWindowFocus.activationEvents(window: window))
XCTAssertEqual(events.count, 1)
let native = try XCTUnwrap(events.first.flatMap { NSEvent(cgEvent: $0) })
XCTAssertEqual(native.windowNumber, 0)
XCTAssertEqual(native.modifierFlags.rawValue, 0)
}
func testExplicitAXActivationPointOutsideContentStillUsesWindowBoundActivation() throws {
let window = SyntheticWindowFocus.Window(
id: 42, bounds: CGRect(x: 100, y: 200, width: 640, height: 480),
activationPoint: CGPoint(x: 80, y: 216)
)
XCTAssertEqual(event.type.rawValue, 21)
XCTAssertEqual(event.subtype.rawValue, Int16(bitPattern: 0x8000))
// Focus is a per-process notification here, not a per-window one — the
// reference passes windowNumber 0 on both sends.
XCTAssertEqual(event.windowNumber, 0)
XCTAssertNotNil(event.cgEvent, "must survive conversion or it cannot be posted")
let events = try XCTUnwrap(SyntheticWindowFocus.activationEvents(window: window))
XCTAssertEqual(events.count, 3)
XCTAssertEqual(events.dropFirst().map(\.location), [CGPoint(x: 80, y: 216), CGPoint(x: 80, y: 216)])
}
func testFailedAXQueriesUseGenericActivationEvenWhenAWindowWasResolved() throws {
let bounds = CGRect(x: 100, y: 200, width: 640, height: 480)
var point = CGPoint(x: 138, y: 216)
let raw = try XCTUnwrap(AXValueCreate(.cgPoint, &point))
let queries: [(AXError, CFTypeRef?)] = [
(.attributeUnsupported, nil), (.noValue, nil), (.cannotComplete, raw), (.success, nil),
]
for (error, value) in queries {
let result = SyntheticWindowFocus.decodeActivationPoint(error: error, raw: value)
let events = try XCTUnwrap(SyntheticWindowFocus.activationEvents(window: .init(
id: 42, bounds: bounds, activationPoint: result
)))
XCTAssertEqual(events.count, 1)
let native = try XCTUnwrap(NSEvent(cgEvent: events[0]))
XCTAssertEqual(native.windowNumber, 0)
XCTAssertEqual(native.modifierFlags.rawValue, 0)
}
}
func testSuccessfulAXQueryDistinguishesExplicitOutsidePointsFromUnusableValues() throws {
let bounds = CGRect(x: 100, y: 200, width: 640, height: 480)
var outside = CGPoint(x: 80, y: 216)
var nonfinite = CGPoint(x: CGFloat.nan, y: 216)
let values: [CFTypeRef] = [
"not a point" as CFString,
try XCTUnwrap(AXValueCreate(.cgPoint, &outside)),
try XCTUnwrap(AXValueCreate(.cgPoint, &nonfinite)),
]
for (value, expectedCount) in zip(values, [1, 3, 1]) {
let result = SyntheticWindowFocus.decodeActivationPoint(error: .success, raw: value)
let events = try XCTUnwrap(SyntheticWindowFocus.activationEvents(window: .init(
id: 42, bounds: bounds, activationPoint: result
)))
XCTAssertEqual(events.count, expectedCount)
let native = try XCTUnwrap(NSEvent(cgEvent: events[0]))
XCTAssertEqual(native.windowNumber, 42)
XCTAssertEqual(native.modifierFlags.rawValue, 0xc0000)
}
}
func testNeteaseAXActivationPointOutsideTheScreenStillPrimesOnlyItsProvenWindow() throws {
// Captured from NetEase on macOS: its AX window supplies this point,
// despite it being outside the content rectangle and screen. Codex
// retains it in its PID/window-bound activation click, without moving
// the user's physical pointer or guessing an in-window UI control.
let bounds = CGRect(x: 332, y: 199, width: 1065, height: 752)
var point = CGPoint(x: -1, y: 1118)
let raw = try XCTUnwrap(AXValueCreate(.cgPoint, &point))
let result = SyntheticWindowFocus.decodeActivationPoint(error: .success, raw: raw)
let events = try XCTUnwrap(SyntheticWindowFocus.activationEvents(window: .init(
id: 42, bounds: bounds, activationPoint: result
)))
XCTAssertEqual(events.count, 3)
XCTAssertEqual(events.dropFirst().map(\.location), [point, point])
for event in events.dropFirst() {
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 91)!), 42)
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 92)!), 42)
XCTAssertEqual(event.getIntegerValueField(.mouseEventClickState), 1)
}
}
func testSuccessfulAXPointQueryDrivesTheWindowBoundActivationBurst() throws {
let bounds = CGRect(x: 100, y: 200, width: 640, height: 480)
var point = CGPoint(x: 138, y: 216)
let raw = try XCTUnwrap(AXValueCreate(.cgPoint, &point))
let result = SyntheticWindowFocus.decodeActivationPoint(error: .success, raw: raw)
let events = try XCTUnwrap(SyntheticWindowFocus.activationEvents(window: .init(
id: 42, bounds: bounds, activationPoint: result
)))
XCTAssertEqual(events.count, 3)
XCTAssertEqual(events.map { NSEvent(cgEvent: $0)?.windowNumber }, [42, 42, 42])
XCTAssertEqual(Array(events.dropFirst()).map(\.location), [point, point])
}
func testActivationClickUsesOnlyTheSuppliedAXPointAndWindow() throws {
let point = CGPoint(x: 138, y: 216)
let window = SyntheticWindowFocus.Window(
id: 42, bounds: CGRect(x: 100, y: 200, width: 640, height: 480), activationPoint: point
)
let events = try XCTUnwrap(SyntheticWindowFocus.activationEvents(window: window))
XCTAssertEqual(events.count, 3)
let native = try events.map { try XCTUnwrap(NSEvent(cgEvent: $0)) }
XCTAssertEqual(native.map(\.type), [.appKitDefined, .leftMouseDown, .leftMouseUp])
XCTAssertEqual(native.map(\.windowNumber), [42, 42, 42])
for event in events.dropFirst() {
XCTAssertEqual(event.location, point)
XCTAssertEqual(event.flags.rawValue, 0)
XCTAssertEqual(event.getIntegerValueField(.mouseEventButtonNumber), 0)
XCTAssertEqual(event.getIntegerValueField(.mouseEventSubtype), 3)
// The reference clears CG field 3 (left button), not field 1
// (click count). Activation remains a genuine single click.
XCTAssertEqual(event.getIntegerValueField(.mouseEventClickState), 1)
XCTAssertEqual(try XCTUnwrap(NSEvent(cgEvent: event)).clickCount, 1)
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 91)!), 42)
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 92)!), 42)
}
}
func testAnInvalidPidIsRefusedRatherThanBroadcast() {
// CGEventPostToPid with a nonsense pid is not obviously harmless, and a
// focus notification aimed at nothing is never something we meant.
XCTAssertFalse(SyntheticWindowFocus.post(.keyFocusReturned, to: 0))
XCTAssertFalse(SyntheticWindowFocus.post(.keyFocusReturned, to: -1))
}
func testEnforcementPostsOneCompletePairForTheSameTarget() {
var state = SyntheticWindowFocus.BeliefState()
let target = beliefTarget()
let sent = OSAllocatedUnfairLock(
initialState: [SyntheticWindowFocus.Notification]()
)
let runtime = SyntheticWindowFocus.EnforcementRuntime(
applicationIsActive: false,
target: target,
post: { notification, _ in
sent.withLock { $0.append(notification) }
return true
}
)
XCTAssertTrue(SyntheticWindowFocus.enforceActiveState(
pid: 42,
state: &state,
runtime: runtime
))
XCTAssertEqual(sent.withLock { $0 }, [.keyFocusReturned, .appActivated])
// click -> type_text is one focus transaction. Re-establishing focus
// between those actions can reset the CEF control the click selected.
XCTAssertFalse(SyntheticWindowFocus.enforceActiveState(
pid: 42,
state: &state,
runtime: runtime
))
XCTAssertEqual(sent.withLock { $0 }, [.keyFocusReturned, .appActivated])
XCTAssertEqual(state.syntheticallyActive, [42: target])
}
func testPartialEnforcementIsWithdrawnAndCanRetry() {
struct PostingState: Sendable {
var sent: [SyntheticWindowFocus.Notification] = []
var failActivation = true
}
var state = SyntheticWindowFocus.BeliefState()
let target = beliefTarget()
let posting = OSAllocatedUnfairLock(initialState: PostingState())
let runtime = SyntheticWindowFocus.EnforcementRuntime(
applicationIsActive: false,
target: target,
post: { notification, _ in
posting.withLock { state in
state.sent.append(notification)
if notification == .appActivated, state.failActivation {
state.failActivation = false
return false
}
return true
}
}
)
XCTAssertFalse(SyntheticWindowFocus.enforceActiveState(
pid: 42,
state: &state,
runtime: runtime
))
XCTAssertEqual(
posting.withLock { $0.sent },
[.keyFocusReturned, .appActivated, .lostKeyFocus, .appDeactivated]
)
XCTAssertTrue(state.syntheticallyActive.isEmpty)
posting.withLock { $0.sent.removeAll() }
XCTAssertTrue(SyntheticWindowFocus.enforceActiveState(
pid: 42,
state: &state,
runtime: runtime
))
XCTAssertEqual(
posting.withLock { $0.sent },
[.keyFocusReturned, .appActivated]
)
XCTAssertEqual(state.syntheticallyActive, [42: target])
}
/// A click followed by type_text is one focus transaction, not two.
/// Re-sending keyFocusReturned to window 0 between them can clear the CEF
/// field the click just focused.
func testSyntheticBeliefIsEstablishedOnlyOnceUntilReleased() {
var state = SyntheticWindowFocus.BeliefState()
let target = beliefTarget()
XCTAssertTrue(state.beginEnforcement(
pid: 42,
applicationIsActive: false,
target: target
))
XCTAssertFalse(state.beginEnforcement(
pid: 42,
applicationIsActive: false,
target: target
))
XCTAssertEqual(state.syntheticallyActive, [42: target])
XCTAssertEqual(state.drain(), [42: target])
XCTAssertTrue(state.syntheticallyActive.isEmpty)
XCTAssertTrue(state.beginEnforcement(
pid: 42,
applicationIsActive: false,
target: target
))
}
func testRealActivationSupersedesSyntheticBelief() {
var state = SyntheticWindowFocus.BeliefState()
let target = beliefTarget()
XCTAssertTrue(state.beginEnforcement(
pid: 42,
applicationIsActive: false,
target: target
))
state.observeRealActivation(pid: 42)
XCTAssertTrue(state.syntheticallyActive.isEmpty)
// Once the app is background again, it needs a fresh pair.
XCTAssertTrue(state.beginEnforcement(
pid: 42,
applicationIsActive: false,
target: target
))
XCTAssertFalse(state.beginEnforcement(
pid: 42,
applicationIsActive: true,
target: target
))
state.cancelEnforcement(pid: 42)
XCTAssertTrue(state.syntheticallyActive.isEmpty)
}
func testOnlyAProcessLifetimeChangeRequiresFreshBelief() {
var state = SyntheticWindowFocus.BeliefState()
let originalProcess = beliefTarget()
let relaunchedProcess = AXTreeProcessIdentity(
bundleID: processA.bundleID,
executablePath: processA.executablePath,
launchTime: 2
)
let relaunched = beliefTarget(
processIdentity: relaunchedProcess
)
XCTAssertTrue(state.beginEnforcement(
pid: 42,
applicationIsActive: false,
target: originalProcess
))
XCTAssertFalse(state.beginEnforcement(
pid: 42,
applicationIsActive: false,
target: originalProcess
))
XCTAssertTrue(state.beginEnforcement(
pid: 42,
applicationIsActive: false,
target: relaunched
))
XCTAssertEqual(state.syntheticallyActive[42], relaunched)
}
func testTeardownWithdrawsFocusBeforeActivationBelief() throws {
let source = try String(
contentsOfFile: URL(fileURLWithPath: #filePath)
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.appendingPathComponent("Sources/cu-helper/SyntheticWindowFocus.swift")
.path,
contentsOf: URL(fileURLWithPath: #filePath)
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
.appendingPathComponent("Sources/cu-helper/SyntheticWindowFocus.swift"),
encoding: .utf8
)
let body = try XCTUnwrap(
source.range(of: "static func relinquishAll").map {
String(source[$0.lowerBound...].prefix(1_000))
}
)
let body = try XCTUnwrap(source.range(of: "static func relinquishAll").map {
String(source[$0.lowerBound...].prefix(1_000))
})
let lostFocus = try XCTUnwrap(body.range(of: "post(.lostKeyFocus"))
let deactivated = try XCTUnwrap(body.range(of: "post(.appDeactivated"))
XCTAssertLessThan(lostFocus.lowerBound, deactivated.lowerBound)
}
func testItNeedsNoPrivateSymbols() throws {
// The whole recipe is NSEvent.otherEvent + .cgEvent + CGEventPostToPid,
// all public. That is why this survives an OS update that would break
// the SkyLight event-record trick it replaced — worth a test, because
// the tempting "improvement" is to reach for the private API again.
let event = try XCTUnwrap(
NSEvent.otherEvent(
with: .appKitDefined, location: .zero, modifierFlags: [],
timestamp: 0, windowNumber: 0, context: nil,
subtype: SyntheticWindowFocus.Notification.appDeactivated.subtype,
data1: 0, data2: 0
)
)
XCTAssertNotNil(event.cgEvent)
}
}
/// Driving an app must not cost the user their foreground.
///
/// This assertion has been made, reverted, and now made again, so the reasoning
/// is worth keeping in full.
///
/// It was first written when `WindowKeyFocus` was replaced by the synthetic
/// notification, and it went red when that change was reverted. The revert had
/// evidence: across two sessions on the notification-only build, 24 mutating
/// actions produced 1 effect, and nine window-bound clicks were discarded in
/// another.
///
/// That second session is what eventually voided the evidence. Its capture
/// showed the target's traffic lights fully coloured — the app was active and
/// its window was key, which is the entire state a foreground grant exists to
/// produce — and the clicks were dropped anyway. Focus could not have been the
/// variable.
///
/// The real defect was in the events themselves, and it was present in every
/// one of those sessions: the leading move of a click claimed `clickState 1`,
/// and the press and release carried different event numbers, so AppKit had no
/// reason to read them as one click (`MouseClickStateTests`). Single clicks
/// registered as hover. Double clicks worked, because the second press/release
/// pair got through — which is why the failure looked intermittent rather than
/// total. A foreground grant plus an 800ms settle raised the odds a malformed
/// click survived, and so read as the cure.
///
/// With the events fixed, the grant is not paying for the foreground it costs.
/// What is protected here: every input path goes through one place that makes
/// the target accept input, that place takes the foreground from nobody, and
/// the synthetic notification is not broadcast at an app that already has
/// focus.
final class InputAcceptanceContractTests: XCTestCase {
private func source(_ name: String) throws -> String {
let root = URL(fileURLWithPath: #filePath)
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent()
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
.appendingPathComponent("Sources/cu-helper")
return try String(contentsOf: root.appendingPathComponent(name), encoding: .utf8)
}
func testEveryInputPathEnsuresTheTargetWillAcceptIt() throws {
// Keyboard used to inherit whatever focus the last click had left
// behind. Once clicks stopped taking real foreground, that inheritance
// was worth nothing, and nine consecutive type_text calls went nowhere
// while every one returned "Action completed".
let axAction = try source("AXAction.swift")
let preparation = try XCTUnwrap(axAction.range(of: "private static func ensureTargetAcceptsInput").map {
String(axAction[$0.lowerBound...].prefix(400))
})
XCTAssertTrue(preparation.contains("try await SyntheticWindowFocus.prepareInput"))
for entry in ["clickPoint", "typeText", "pressKey"] {
let body = try XCTUnwrap(
axAction.range(of: "public static func \(entry)").map {
String(axAction[$0.lowerBound...].prefix(1200))
},
"\(entry) is missing"
}, "\(entry) is missing"
)
XCTAssertTrue(
body.contains("ensureTargetAcceptsInput"),
"\(entry) must ensure the target accepts input before acting"
body.contains("try await ensureTargetAcceptsInput"),
"\(entry) must await target acceptance before acting"
)
}
}
func testNoInputPathTakesTheUsersForeground() throws {
// A source guard because the effect is not observable from a unit test:
// `WindowKeyFocus` calls a private CPS symbol, and whether the
// foreground moved is a property of the running window server. What can
// be pinned is that no input path asks for it.
//
// Deliberately covers `grantIfNeeded` as well as `grant`. Gating the
// grant on "only when the target is not already frontmost" sounds
// considerate and is not: the case it fires in — the target is in the
// background — is exactly the case the feature exists for.
// Delivery cannot be proved offline; prevent reintroducing the known
// foreground-stealing escape hatch into either actual input path.
for file in ["AXAction.swift", "Injection.swift"] {
// Comments are excluded on purpose. Both files explain at length
// why the grant is gone and name it while doing so; a guard that
// cannot tell prose from a call site would forbid documenting its
// own reasoning, and the obvious way out of that is to weaken the
// guard. A real call never sits on a line that opens with `//`.
let body = try source(file)
.split(separator: "\n", omittingEmptySubsequences: false)
.filter { !$0.trimmingCharacters(in: .whitespaces).hasPrefix("//") }
.joined(separator: "\n")
XCTAssertFalse(
body.contains("WindowKeyFocus.grant"),
"\(file) must not pull the target to the foreground to deliver input. "
+ "If a real-machine regression genuinely needs this back, measure it "
+ "with a SINGLE click on a control that needs a complete click — a "
+ "text field focuses on the press alone and cannot tell the two apart."
)
XCTAssertFalse(body.contains("WindowKeyFocus.grant"), "\(file) must preserve the user's foreground")
}
}
/// `focusForClick` used to consist of nothing but the foreground grant: it
/// never sent the notification the AX path relies on. Removing the grant
/// without adding one would have left the decomposed mouse commands doing
/// no focus work at all — silently, since nothing here reports delivery.
func testTheDecomposedMousePathStillPreparesItsTarget() throws {
let injection = try source("Injection.swift")
let focus = try XCTUnwrap(
injection.range(of: "private static func focusForClick").map {
String(injection[$0.lowerBound...].prefix(400))
},
"focusForClick is missing"
)
let focus = try XCTUnwrap(injection.range(of: "private static func focusForClick").map {
String(injection[$0.lowerBound...].prefix(600))
})
XCTAssertTrue(
focus.contains("SyntheticWindowFocus.enforceActiveState"),
"the decomposed mouse path must tell its target it has focus"
)
}
func testTheSyntheticNotificationIsGatedOnRealState() throws {
// Codex holds `applicationIsActive` beside `applicationBelievesItIsActive`
// and only re-sends when they disagree. The first version of this file
// documented that gate and shipped without it — sending "key focus
// returned to window 0" to an app that already owned a key window, on
// every click.
var state = SyntheticWindowFocus.BeliefState()
let sent = OSAllocatedUnfairLock(
initialState: [SyntheticWindowFocus.Notification]()
)
let activeRuntime = SyntheticWindowFocus.EnforcementRuntime(
applicationIsActive: true,
target: SyntheticWindowFocus.BeliefTarget(processIdentity: nil),
post: { notification, _ in
sent.withLock { $0.append(notification) }
return true
}
)
XCTAssertFalse(SyntheticWindowFocus.enforceActiveState(
pid: 42,
state: &state,
runtime: activeRuntime
))
XCTAssertTrue(sent.withLock { $0 }.isEmpty)
XCTAssertTrue(state.syntheticallyActive.isEmpty)
let backgroundRuntime = SyntheticWindowFocus.EnforcementRuntime(
applicationIsActive: false,
target: activeRuntime.target,
post: activeRuntime.post
)
XCTAssertTrue(SyntheticWindowFocus.enforceActiveState(
pid: 42,
state: &state,
runtime: backgroundRuntime
))
XCTAssertFalse(SyntheticWindowFocus.enforceActiveState(
pid: 42,
state: &state,
runtime: backgroundRuntime
))
XCTAssertEqual(
sent.withLock { $0 },
[.keyFocusReturned, .appActivated]
focus.contains("SyntheticWindowFocus.prepareInput"),
"decomposed mouse commands must await target acceptance too"
)
}
}
@@ -6,6 +6,100 @@ import XCTest
@MainActor
final class WindowCaptureStreamTests: XCTestCase {
func testDiagnosticsObserveTheRealSnapshotLifecycleWithoutReadingPixelsOrStartingStreams() async throws {
let target = makeTarget(windowID: 90)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var captures = 0
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
captures += 1
return self.makeSnapshot(target, pixels: "snapshot")
})
let idle = manager.diagnostic(now: 12)
XCTAssertEqual(idle.generation, 0)
XCTAssertNil(idle.activeKey)
XCTAssertNil(idle.hasFailed)
XCTAssertNil(idle.sampleCount)
XCTAssertTrue(factory.sources.isEmpty)
_ = await manager.captureSnapshot(for: target, scale: 0.5)
let source = try XCTUnwrap(factory.sources.first)
let started = manager.diagnostic(now: 12)
XCTAssertEqual(started.activeKey, target.key)
XCTAssertNil(started.startingKey)
XCTAssertEqual(started.hasFailed, false)
XCTAssertEqual(started.sampleCount, 0)
XCTAssertNil(started.latestFrameSequence, "Successful start is not proof of a generated frame")
XCTAssertNil(started.latestFrameAgeSeconds)
source.publish(makeFrame(for: target.key, sequence: 1, uptime: 10, byte: 7))
source.publishStatus(.idle, uptime: 11)
let idleFrame = manager.diagnostic(now: 12)
XCTAssertEqual(idleFrame.generation, started.generation)
XCTAssertEqual(idleFrame.latestFrameSequence, 1)
XCTAssertEqual(idleFrame.latestFrameAgeSeconds, 2)
XCTAssertEqual(idleFrame.sampleCount, 2)
XCTAssertEqual(idleFrame.latestSampleStatus, SCFrameStatus.idle.rawValue)
XCTAssertEqual(idleFrame.latestSampleAgeSeconds, 1)
source.publish(makeFrame(for: target.key, sequence: 2, uptime: 13, byte: 7))
let refreshed = manager.diagnostic(now: 14)
XCTAssertEqual(refreshed.latestFrameSequence, 2, "Identical pixels can still be a new frame")
XCTAssertEqual(refreshed.latestFrameAgeSeconds, 1)
XCTAssertEqual(refreshed.sampleCount, 3)
XCTAssertEqual(refreshed.latestSampleStatus, SCFrameStatus.complete.rawValue)
XCTAssertEqual(source.latestReadCount, 0)
XCTAssertEqual(source.startCount, 1)
XCTAssertEqual(source.retireCount, 0)
XCTAssertEqual(captures, 1, "Inspecting metadata must not take screenshots")
}
func testDiagnosticFailureAndInvalidationDoNotRebuildOrExposeRetiredFrames() async throws {
let target = makeTarget(windowID: 91)
let factory = FakeWindowCaptureStreamFactory { source, _ in
source.startFrame = makeFrame(for: source.targetKey, sequence: 1, uptime: 10, byte: 1)
}
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
self.makeSnapshot(target, pixels: "snapshot")
})
_ = await manager.captureSnapshot(for: target, scale: 0.5)
let source = try XCTUnwrap(factory.sources.first)
let activeGeneration = manager.diagnostic(now: 12).generation
source.failed = true
XCTAssertEqual(manager.diagnostic(now: 12).hasFailed, true)
XCTAssertEqual(factory.sources.count, 1)
XCTAssertEqual(source.retireCount, 0)
manager.invalidate()
source.publish(makeFrame(for: target.key, sequence: 2, uptime: 13, byte: 2))
let retired = manager.diagnostic(now: 14)
XCTAssertGreaterThan(retired.generation, activeGeneration)
XCTAssertNil(retired.activeKey)
XCTAssertNil(retired.startingKey)
XCTAssertNil(retired.hasFailed)
XCTAssertNil(retired.latestFrameSequence)
XCTAssertNil(retired.latestFrameAgeSeconds)
XCTAssertNil(retired.latestSampleStatus)
XCTAssertEqual(source.latestReadCount, 0)
XCTAssertEqual(source.retireCount, 1)
}
func testMailboxDiagnosticRecordsNonPixelStatusesWithoutAdvancingFrameAndIgnoresRetiredCallbacks() {
let mailbox = WindowCaptureStreamMailbox()
mailbox.recordSampleStatus(.started, receivedUptime: 10)
mailbox.recordSampleStatus(.suspended, receivedUptime: 11)
let suspended = mailbox.sampleDiagnostic()
XCTAssertEqual(suspended.sampleCount, 2)
XCTAssertEqual(suspended.latestSampleStatus, SCFrameStatus.suspended.rawValue)
XCTAssertEqual(suspended.latestSampleReceivedUptime, 11)
XCTAssertNil(suspended.latestFrameSequence)
XCTAssertFalse(suspended.hasFailed)
mailbox.invalidate()
mailbox.recordSampleStatus(.complete, receivedUptime: 12)
XCTAssertEqual(mailbox.sampleDiagnostic(), suspended)
}
func testOnDemandScreenshotUsesOnlyTheTargetWindowBounds() {
let config = Capture.makeWindowShotConfiguration(width: 1061, height: 752)
XCTAssertEqual(config.width, 1061)
@@ -546,6 +640,9 @@ private final class FakeWindowCaptureStreamSource: WindowCaptureStreamSource {
private(set) var retireCount = 0
private(set) var latestReadCount = 0
private var latest: WindowCaptureStreamFrame?
private var sampleCount: UInt64 = 0
private var latestSampleStatus: Int?
private var latestSampleReceivedUptime: TimeInterval?
init(targetKey: WindowCaptureStreamKey) {
self.targetKey = targetKey
@@ -553,10 +650,21 @@ private final class FakeWindowCaptureStreamSource: WindowCaptureStreamSource {
var hasFailed: Bool { failed }
func sampleDiagnostic() -> WindowCaptureStreamSourceDiagnostic {
WindowCaptureStreamSourceDiagnostic(
hasFailed: failed,
latestFrameSequence: latest?.sequence,
latestFrameReceivedUptime: latest?.receivedUptime,
sampleCount: sampleCount,
latestSampleStatus: latestSampleStatus,
latestSampleReceivedUptime: latestSampleReceivedUptime
)
}
func start() async throws {
startCount += 1
if let startError { throw startError }
latest = startFrame
if let startFrame { publish(startFrame) }
}
func latestFrame() -> WindowCaptureStreamFrame? {
@@ -571,6 +679,13 @@ private final class FakeWindowCaptureStreamSource: WindowCaptureStreamSource {
func publish(_ frame: WindowCaptureStreamFrame) {
latest = frame
publishStatus(.complete, uptime: frame.receivedUptime)
}
func publishStatus(_ status: SCFrameStatus, uptime: TimeInterval) {
sampleCount += 1
latestSampleStatus = status.rawValue
latestSampleReceivedUptime = uptime
}
}
@@ -204,6 +204,81 @@ final class WindowGeometryTests: XCTestCase {
func testUnreadableWindowListYieldsNoWindow() {
XCTAssertNil(WindowGeometry.window(at: .zero) { nil })
}
func testExactWindowIdentityRevalidationKeepsOriginalWindowAfterMoveAndReorder() throws {
var list = [info(layer: 0, x: -500, y: -600, w: 800, h: 600, number: 3, pid: 7)]
let original = try XCTUnwrap(WindowGeometry.window(
at: CGPoint(x: -100, y: -200), pid: 7, windowList: { list }
))
// During an async action another window takes the old position while
// the original moves. Revalidation must not switch to that new window.
list = [
info(layer: 0, x: -500, y: -600, w: 800, h: 600, number: 4, pid: 7),
info(layer: 0, x: 100, y: 200, w: 900, h: 700, number: 3, pid: 7),
]
let current = try XCTUnwrap(WindowGeometry.window(
id: original.id, pid: original.ownerPid, windowList: { list }
))
XCTAssertEqual(current.id, original.id)
XCTAssertEqual(current.ownerPid, original.ownerPid)
XCTAssertEqual(current.bounds, CGRect(x: 100, y: 200, width: 900, height: 700))
XCTAssertNotEqual(current.bounds, original.bounds)
}
func testExactWindowIdentityRequiresMatchingNumberOwnerAndOrdinaryLayer() {
let candidates = [
info(layer: 0, x: 0, y: 0, w: 100, h: 100, number: 4, pid: 7),
info(layer: 0, x: 0, y: 0, w: 100, h: 100, number: 3, pid: 8),
info(layer: 25, x: 0, y: 0, w: 100, h: 100, number: 3, pid: 7),
]
for candidate in candidates {
XCTAssertNil(WindowGeometry.window(id: 3, pid: 7, windowList: { [candidate] }))
}
let valid = info(layer: 0, x: -10, y: -20, w: 100, h: 200, number: 3, pid: 7)
XCTAssertEqual(
WindowGeometry.window(id: 3, pid: 7, windowList: { candidates + [valid] }),
WindowGeometry.Window(id: 3, bounds: CGRect(x: -10, y: -20, width: 100, height: 200), ownerPid: 7)
)
}
func testExactWindowIdentityRejectsMissingEmptyAndNonfiniteBounds() {
let validBounds: [String: CGFloat] = ["X": 10, "Y": 20, "Width": 100, "Height": 200]
var cases: [(String, [String: CGFloat]?)] = [("missing bounds", nil)]
for field in ["X", "Y", "Width", "Height"] {
var missing = validBounds
missing.removeValue(forKey: field)
cases.append(("missing \(field)", missing))
for invalid: CGFloat in [.nan, .infinity, -.infinity] {
var bounds = validBounds
bounds[field] = invalid
cases.append(("\(field) = \(invalid)", bounds))
}
}
for dimension in ["Width", "Height"] {
for invalid: CGFloat in [0, -1] {
var bounds = validBounds
bounds[dimension] = invalid
cases.append(("\(dimension) = \(invalid)", bounds))
}
}
for (description, bounds) in cases {
var candidate = info(layer: 0, x: 10, y: 20, w: 100, h: 200, number: 3, pid: 7)
candidate[kCGWindowBounds] = bounds
XCTAssertNil(
WindowGeometry.window(id: 3, pid: 7, windowList: { [candidate] }),
description
)
}
}
func testExactWindowIdentityDoesNotInventWindowWhenListIsUnavailable() {
XCTAssertNil(WindowGeometry.window(id: 3, pid: 7, windowList: { nil }))
XCTAssertNil(WindowGeometry.window(id: 3, pid: 7, windowList: { [] }))
}
}
/// Guards the foreground-settle behaviour that makes background actuation