fix(computer-use): capture fresh aligned frames for covered windows

This commit is contained in:
程序员阿江(Relakkes)
2026-08-31 12:30:58 +08:00
parent 42d7cb1c93
commit 217867fff1
12 changed files with 317 additions and 46 deletions
@@ -38,12 +38,13 @@ import ImageIO
import ScreenCaptureKit
import UniformTypeIdentifiers
enum WindowShotCaptureSource: Equatable, Sendable {
enum WindowShotCaptureSource: String, Equatable, Sendable {
case stream
case streamBackedScreenshot
case screenshotManager
case screenCaptureCLI
var isLiveStream: Bool { self == .stream }
var isLiveStream: Bool { self == .stream || self == .streamBackedScreenshot }
}
struct WindowShot: Sendable {
@@ -454,7 +455,8 @@ public enum Capture {
static func windowShot(
pid: pid_t,
preferredWindowID: CGWindowID? = nil,
scale: Double = 0.5
scale: Double = 0.5,
allowCLIFallback: Bool = true
) async -> WindowShot? {
// Passive permission gate — no prompt on the hot path. A denied grant
// means SCK would hand us a black frame, so bail to `nil` early and let
@@ -495,6 +497,7 @@ public enum Capture {
return nil
}
guard allowCLIFallback else { return nil }
// ② Fallback: /usr/sbin/screencapture -l <windowID> (SCK hung or failed).
if let raw = screencaptureWindow(windowID: target.windowID) {
let scaled = (try? scaleImage(raw, scale: outputScale)) ?? raw
@@ -669,18 +672,14 @@ public enum Capture {
// lock.
let filter = SCContentFilter(desktopIndependentWindow: scWindow)
let config = SCStreamConfiguration()
let backingScale = backingScaleFactor(forWindowFrame: frame)
// Native window pixels = points × backing scale, then × requested
// downscale. Clamp to >= 1 so a tiny window never yields a 0-dim
// buffer.
config.width = max(1, Int((frame.width * backingScale * scale).rounded()))
config.height = max(1, Int((frame.height * backingScale * scale).rounded()))
config.showsCursor = false // never the REAL cursor
config.scalesToFit = true
config.pixelFormat = kCVPixelFormatType_32BGRA
config.colorSpaceName = CGColorSpace.sRGB
config.captureResolution = .best
let config = makeWindowShotConfiguration(
width: max(1, Int(ceil(frame.width * backingScale * scale))),
height: max(1, Int(ceil(frame.height * backingScale * scale)))
)
return try await SCScreenshotManager.captureImage(
contentFilter: filter,
@@ -691,6 +690,31 @@ public enum Capture {
}
}
static func makeWindowShotConfiguration(width: Int, height: Int) -> SCStreamConfiguration {
let config = SCStreamConfiguration()
config.width = max(1, width)
config.height = max(1, height)
config.showsCursor = false
config.scalesToFit = true
config.preservesAspectRatio = true
config.pixelFormat = kCVPixelFormatType_32BGRA
config.colorSpaceName = CGColorSpace.sRGB
config.captureResolution = .best
// SCScreenshotManager includes this window's shadow by default. That
// shrinks/pads the image while our click transform still names the
// shadow-free window bounds, particularly after a cold background start.
config.ignoreShadowsSingleWindow = true
config.ignoreShadowsDisplay = true
if #available(macOS 14.2, *) {
// The sharing indicator can become an attached child above the
// window. Capturing that union shrinks/offsets the main content
// without changing SCWindow.frame or filter.contentRect, breaking
// screenshot-to-click coordinates. The target is this exact window.
config.includeChildWindows = false
}
return config
}
/// CLI fallback when SCK hangs/fails: `/usr/sbin/screencapture -l <windowID>
/// -x -o <tmp.png>`. `-x` = no capture sound, `-o` = omit the window shadow.
/// Bounded at 3s with a `terminate` → `SIGKILL` escalation. Returns the
@@ -103,6 +103,7 @@ public final class CommandRouter {
AXTree.resetSessionSnapshots()
Self.lastShotTransform.removeAll()
Self.lastCaptureDigest.removeAll()
MutationClock.reset()
windowCaptureProvider?.invalidate()
// Apps we told they were focused must be told they are not, or the
// belief outlives the session that needed it.
@@ -661,17 +662,17 @@ public final class CommandRouter {
// frame. Costs nothing when no action is pending. The rendered tree
// doubles as the busy signal — a progress indicator in it means the
// app is still working, so we allow a longer window.
await MutationClock.awaitSettle(
let streamedShot = await Self.captureSettledWindowShot(
appIsBusy: result.axText.contains("progress indicator")
)
let streamedShot = await windowCaptureProvider?.windowShot(
pid: pid,
processIdentity: snapshotEvidence.processIdentity,
preferredWindowID: snapshotEvidence.keyWindowID,
scale: 0.5,
newerThanUptime: MutationClock.lastMutation()
)
) {
await windowCaptureProvider?.windowShot(
pid: pid,
processIdentity: snapshotEvidence.processIdentity,
preferredWindowID: snapshotEvidence.keyWindowID,
scale: 0.5,
newerThanUptime: MutationClock.lastMutation()
)
}
guard TargetVisibilityPolicy.captureTargetStillMatches(
snapshotWindowID: snapshotEvidence.keyWindowID,
currentWindowID: AXTree.currentKeyWindowID(pid: pid)
@@ -777,6 +778,7 @@ public final class CommandRouter {
"pointWidth": .double(shot.pointWidth),
"pointHeight": .double(shot.pointHeight),
"windowID": .int(Int(shot.windowID)),
"captureSource": .string(shot.source.rawValue),
])
// Cache the inverse transform so a later coordinate click/scroll/
// drag (which arrives in image-pixel space) can be mapped back to
@@ -800,6 +802,16 @@ public final class CommandRouter {
return .object(object)
}
/// Keep the wait and the actual capture in one production boundary so an
/// action-to-screenshot regression can exercise both without live AX/TCC.
static func captureSettledWindowShot(
appIsBusy: Bool,
capture: () async -> WindowShot?
) async -> WindowShot? {
await MutationClock.awaitSettle(appIsBusy: appIsBusy)
return await capture()
}
/// A snapshot discarded by an internal key-window retry was never delivered
/// to the model, so it cannot become the baseline for a returned diff.
static func effectiveDisableDiff(
@@ -396,6 +396,10 @@ enum ForegroundMutationRunner {
lease: ForegroundLease,
action: () async throws -> T
) async throws -> T {
// Every input path crosses this boundary, including synthetic events
// that never call AXAction.settle(). A throw can follow a partial
// delivery, so neither success nor failure may reuse pre-action pixels.
defer { MutationClock.recordMutation() }
let result: Result<T, Error>
do {
result = .success(try await action())
@@ -33,10 +33,10 @@ enum TargetVisibilityPolicy {
return """
NOTE: Another application fully covers the target window. A \
long-lived window stream remains subscribed while it is covered, \
and this screenshot comes from its latest complete frame rather \
than the visible desktop. Coverage does not block Accessibility \
actions or app- and window-targeted input; continue without \
activating or raising the target.
and this state uses a new on-demand window screenshot, not the \
stream's cached frame or the visible desktop. Keep using app- and \
window-targeted input without activating or raising the target. \
An action receipt alone does not prove the application responded.
"""
}
return """
@@ -59,13 +59,13 @@ enum TargetVisibilityPolicy {
cause = windowIsCovered
? """
The target is covered, but its long-lived window stream is \
still active; the newest complete frame contains no visible \
pixel change. Coverage does not block app- and window-targeted \
input.
still active, and a new on-demand screenshot contains no \
visible pixel change. This does not establish whether the \
app accepted the input or redrew its content.
"""
: """
The target window is not fully covered, and the newest \
complete stream frame contains no visible pixel change.
The target window is not fully covered, and the new \
on-demand screenshot contains no visible pixel change.
"""
} else {
cause = windowIsCovered
@@ -88,5 +88,7 @@ enum MutationClock {
try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
}
static func resetForTests() { lastMutationAt = nil }
static func reset() { lastMutationAt = nil }
static func resetForTests() { reset() }
}
@@ -812,8 +812,10 @@ public final class VirtualCursor {
// MARK: - Headless disk persistence
private static func runtimeDir() -> URL {
let home = FileManager.default.homeDirectoryForCurrentUser
let dir = home.appendingPathComponent(".claude/.runtime", isDirectory: true)
let config = ProcessInfo.processInfo.environment["CLAUDE_CONFIG_DIR"]
.flatMap { $0.isEmpty ? nil : ($0 as NSString).expandingTildeInPath }
?? (NSHomeDirectory() as NSString).appendingPathComponent(".claude")
let dir = URL(fileURLWithPath: config).appendingPathComponent(".runtime", isDirectory: true)
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
return dir
}
@@ -94,6 +94,7 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
private let factory: any WindowCaptureStreamSourceFactory
private let frameWaitAttempts: Int
private let frameWaitNanoseconds: UInt64
private let takeSnapshot: (WindowCaptureStreamTarget, Double) async -> WindowShot?
private var generation: UInt64 = 0
private var starting: Entry?
private var active: Entry?
@@ -105,11 +106,20 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
init(
factory: any WindowCaptureStreamSourceFactory,
frameWaitAttempts: Int = 12,
frameWaitNanoseconds: UInt64 = 50_000_000
frameWaitNanoseconds: UInt64 = 50_000_000,
takeSnapshot: @escaping (WindowCaptureStreamTarget, Double) async -> WindowShot? = { target, scale in
await Capture.windowShot(
pid: target.key.pid,
preferredWindowID: target.key.windowID,
scale: scale,
allowCLIFallback: false
)
}
) {
self.factory = factory
self.frameWaitAttempts = max(0, frameWaitAttempts)
self.frameWaitNanoseconds = frameWaitNanoseconds
self.takeSnapshot = takeSnapshot
}
func windowShot(
@@ -139,10 +149,7 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
invalidate()
return nil
}
guard let frame = await frame(
for: target,
newerThanUptime: newerThanUptime
) else {
guard let shot = await captureSnapshot(for: target, scale: scale) else {
return nil
}
if let preferredWindowID,
@@ -170,7 +177,38 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
guard current.key == target.key else {
continue
}
return Capture.windowShot(from: frame, target: current)
return WindowShot(
base64: shot.base64, width: shot.width, height: shot.height,
originX: current.originX, originY: current.originY,
pointWidth: current.pointWidth, pointHeight: current.pointHeight,
windowID: current.key.windowID, source: .streamBackedScreenshot
)
}
return nil
}
/// Match the reference's two separate lifetimes: SCStream remains a
/// consumer while covered; every state read runs an on-demand Skyshot/SCK
/// capture. An idle stream's cached frame is not evidence of the current UI.
func captureSnapshot(for target: WindowCaptureStreamTarget, scale: Double) async -> WindowShot? {
for _ in 0..<2 {
guard let source = await source(for: target) else { continue }
if source.hasFailed {
retire(source: source)
continue
}
let snapshotGeneration = generation
guard let shot = await takeSnapshot(target, scale),
snapshotGeneration == generation,
active?.source === source,
!source.hasFailed,
shot.source == .screenshotManager,
shot.windowID == target.key.windowID,
shot.width == target.key.pixelWidth,
shot.height == target.key.pixelHeight else {
return nil
}
return shot
}
return nil
}
@@ -121,9 +121,10 @@ enum WindowTargetedEvent {
}
private static let runtimeDirectory: URL? = {
let home = ProcessInfo.processInfo.environment["HOME"] ?? NSHomeDirectory()
let dir = URL(fileURLWithPath: home)
.appendingPathComponent(".claude")
let config = ProcessInfo.processInfo.environment["CLAUDE_CONFIG_DIR"]
.flatMap { $0.isEmpty ? nil : ($0 as NSString).expandingTildeInPath }
?? (NSHomeDirectory() as NSString).appendingPathComponent(".claude")
let dir = URL(fileURLWithPath: config)
.appendingPathComponent(".runtime")
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
return dir
@@ -141,6 +141,27 @@ final class CommandRouterSafetyTests: XCTestCase {
XCTAssertEqual(provider.invalidateCount, 1)
}
func testSessionResetClearsThePreviousMutationSettleDeadline() async throws {
let monitor = PhysicalInputEpochMonitor(counterReader: { _ in 0 })
let router = CommandRouter(
cursor: VirtualCursor(headless: true),
capabilities: Capabilities(headless: true),
inputMonitor: monitor
)
let process = try XCTUnwrap(ProvenProcessTarget(pid: 77, identity: processA))
let lease = try ForegroundLease.acquire(target: process, runtime: ForegroundLeaseRuntime(
inputSnapshot: { PhysicalInputEpochSnapshot(epoch: 0, available: true) },
frontmostTarget: { nil },
currentIdentity: { _ in self.processA },
activate: { _ in XCTFail("must not activate"); return false },
verifyFrontmost: { _ in XCTFail("must not activate"); return false }
))
_ = try await ForegroundMutationRunner.run(lease: lease) { true }
XCTAssertNotNil(MutationClock.lastMutation())
router.resetSessionState()
XCTAssertNil(MutationClock.lastMutation())
}
func testDiscardedWindowSnapshotForcesTheRetryToReturnAFullTree() {
XCTAssertFalse(CommandRouter.effectiveDisableDiff(
requested: false,
@@ -0,0 +1,29 @@
import Darwin
import Foundation
import XCTest
@testable import cc_haha_computer_use
@MainActor
final class RuntimeIsolationTests: XCTestCase {
func testHeadlessCursorRoundTripsOnlyThroughTheConfiguredSandbox() async throws {
let directory = FileManager.default.temporaryDirectory
.appendingPathComponent("cu-cursor-isolation-\(UUID().uuidString)")
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
let previous = getenv("CLAUDE_CONFIG_DIR").map { String(cString: $0) }
setenv("CLAUDE_CONFIG_DIR", directory.path, 1)
defer {
if let previous { setenv("CLAUDE_CONFIG_DIR", previous, 1) }
else { unsetenv("CLAUDE_CONFIG_DIR") }
try? FileManager.default.removeItem(at: directory)
}
let cursor = VirtualCursor(headless: true)
XCTAssertEqual(cursor.position, .zero)
await cursor.move(to: CGPoint(x: 13, y: 29), animated: false)
XCTAssertTrue(FileManager.default.fileExists(
atPath: directory.appendingPathComponent(".runtime/cu-helper.cursor.json").path
))
XCTAssertEqual(VirtualCursor(headless: true).position, CGPoint(x: 13, y: 29))
}
}
@@ -160,8 +160,8 @@ final class TargetVisibilityPolicyTests: XCTestCase {
liveStreamActive: true
)
XCTAssertTrue(notice.contains("long-lived window stream"))
XCTAssertTrue(notice.contains("latest complete frame"))
XCTAssertTrue(notice.contains("does not block"))
XCTAssertTrue(notice.contains("new on-demand window screenshot"))
XCTAssertTrue(notice.contains("does not prove the application responded"))
XCTAssertFalse(notice.contains("may be stale"))
XCTAssertFalse(notice.contains("paused its renderer"))
XCTAssertFalse(notice.contains("refused"))
@@ -234,8 +234,8 @@ final class TargetVisibilityPolicyTests: XCTestCase {
liveStreamActive: true
)
XCTAssertTrue(covered.contains("long-lived window stream"))
XCTAssertTrue(covered.contains("newest complete frame"))
XCTAssertTrue(covered.contains("does not block"))
XCTAssertTrue(covered.contains("new on-demand screenshot"))
XCTAssertTrue(covered.contains("does not establish whether"))
XCTAssertTrue(covered.contains("no visible pixel change"))
XCTAssertFalse(covered.contains("paused its renderer"))
}
@@ -6,6 +6,144 @@ import XCTest
@MainActor
final class WindowCaptureStreamTests: XCTestCase {
func testOnDemandScreenshotUsesOnlyTheTargetWindowBounds() {
let config = Capture.makeWindowShotConfiguration(width: 1061, height: 752)
XCTAssertEqual(config.width, 1061)
XCTAssertEqual(config.height, 752)
XCTAssertTrue(config.ignoreShadowsSingleWindow)
XCTAssertTrue(config.ignoreShadowsDisplay)
if #available(macOS 14.2, *) {
XCTAssertFalse(config.includeChildWindows, "A sharing child must not expand the screenshot beyond its click transform")
}
XCTAssertTrue(config.scalesToFit)
XCTAssertTrue(config.preservesAspectRatio)
XCTAssertFalse(config.showsCursor)
}
func testEveryStateReadTakesANewSnapshotWhileReusingTheLongLivedStream() async throws {
let target = makeTarget(windowID: 84)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var captures = 0
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
captures += 1
return self.makeSnapshot(target, pixels: "read-\(captures)")
})
let first = await manager.captureSnapshot(for: target, scale: 0.5)
let second = await manager.captureSnapshot(for: target, scale: 0.5)
XCTAssertEqual(first?.base64, "read-1")
XCTAssertEqual(second?.base64, "read-2")
XCTAssertEqual(captures, 2)
XCTAssertEqual(factory.sources.count, 1)
XCTAssertEqual(factory.sources[0].startCount, 1)
XCTAssertEqual(factory.sources[0].latestReadCount, 0, "A cached stream frame must not become the model's screenshot")
}
func testSnapshotFailureDoesNotFallBackToCachedStreamPixels() async {
let target = makeTarget(windowID: 85)
let factory = FakeWindowCaptureStreamFactory { source, _ in
source.startFrame = makeFrame(for: source.targetKey, sequence: 1, uptime: 10, byte: 7)
}
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { _, _ in nil })
let shot = await manager.captureSnapshot(for: target, scale: 0.5)
XCTAssertNil(shot)
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
}
func testSnapshotFinishingAfterSessionInvalidationIsDiscarded() async {
let target = makeTarget(windowID: 86)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var manager: WindowCaptureStreamManager!
manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
manager.invalidate()
return self.makeSnapshot(target, pixels: "late")
})
let shot = await manager.captureSnapshot(for: target, scale: 0.5)
XCTAssertNil(shot)
XCTAssertEqual(factory.sources[0].retireCount, 1)
}
private func makeSnapshot(_ target: WindowCaptureStreamTarget, pixels: String) -> WindowShot {
WindowShot(
base64: pixels, width: target.key.pixelWidth, height: target.key.pixelHeight,
originX: target.originX, originY: target.originY,
pointWidth: target.pointWidth, pointHeight: target.pointHeight,
windowID: target.key.windowID, source: .screenshotManager
)
}
func testEveryDispatchedMutationSettlesBeforeTheNextOnDemandSnapshot() async throws {
MutationClock.resetForTests()
defer { MutationClock.resetForTests() }
let target = makeTarget(windowID: 81)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
var captures = 0
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
captures += 1
self.assertMutationHasSettledBeforeCapture()
return self.makeSnapshot(target, pixels: "unchanged-pixels")
})
var previousMutation: TimeInterval?
for expectedCaptureCount in 1...2 {
// Drive the real dispatcher; do not hand-write its timestamp.
_ = try await ForegroundMutationRunner.run(lease: makeBackgroundLease(target)) { true }
XCTAssertNotNil(MutationClock.lastMutation())
XCTAssertNotEqual(MutationClock.lastMutation(), previousMutation)
previousMutation = MutationClock.lastMutation()
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
await manager.captureSnapshot(for: target, scale: 0.5)
}
XCTAssertEqual(shot?.base64, "unchanged-pixels")
XCTAssertEqual(captures, expectedCaptureCount, "Identical pixels still require a new capture")
}
XCTAssertEqual(factory.sources.count, 1)
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
}
func testPartiallyFailedDispatchAlsoSettlesBeforeTheOnDemandSnapshot() async throws {
MutationClock.resetForTests()
defer { MutationClock.resetForTests() }
let target = makeTarget(windowID: 83)
let factory = FakeWindowCaptureStreamFactory { _, _ in }
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
self.assertMutationHasSettledBeforeCapture()
return self.makeSnapshot(target, pixels: "partial-action-state")
})
do {
let _: Bool = try await ForegroundMutationRunner.run(lease: makeBackgroundLease(target)) {
throw CUError("partial_action", "An event may already have been delivered")
}
XCTFail("expected the original action failure")
} catch let error as CUError {
XCTAssertEqual(error.code, "partial_action")
}
XCTAssertNotNil(MutationClock.lastMutation())
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
await manager.captureSnapshot(for: target, scale: 0.5)
}
XCTAssertEqual(shot?.base64, "partial-action-state")
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
}
private func assertMutationHasSettledBeforeCapture() {
XCTAssertNotNil(MutationClock.lastMutation())
XCTAssertEqual(UISettlePolicy.delay(
now: ProcessInfo.processInfo.systemUptime,
lastMutationAt: MutationClock.lastMutation(),
appIsBusy: false
), 0, "The actual screenshot callback must run after the action's settle deadline")
}
private func makeBackgroundLease(_ target: WindowCaptureStreamTarget) throws -> ForegroundLease {
let process = try XCTUnwrap(ProvenProcessTarget(pid: target.key.pid, identity: target.key.processIdentity))
return try ForegroundLease.acquire(target: process, runtime: ForegroundLeaseRuntime(
inputSnapshot: { PhysicalInputEpochSnapshot(epoch: 0, available: true) },
frontmostTarget: { nil },
currentIdentity: { _ in target.key.processIdentity },
activate: { _ in XCTFail("must not take foreground"); return false },
verifyFrontmost: { _ in XCTFail("must not take foreground"); return false }
))
}
func testSameTargetReusesStreamAndReturnsNewestFrameAcrossCoveredAction() async {
let factory = FakeWindowCaptureStreamFactory { source, _ in
source.startFrame = makeFrame(for: source.targetKey, sequence: 1, uptime: 10, byte: 1)