mirror of
https://github.com/NanmiCoder/claude-code-haha.git
synced 2026-10-10 03:43:11 +08:00
fix(computer-use): capture fresh aligned frames for covered windows
This commit is contained in:
@@ -38,12 +38,13 @@ import ImageIO
|
||||
import ScreenCaptureKit
|
||||
import UniformTypeIdentifiers
|
||||
|
||||
enum WindowShotCaptureSource: Equatable, Sendable {
|
||||
enum WindowShotCaptureSource: String, Equatable, Sendable {
|
||||
case stream
|
||||
case streamBackedScreenshot
|
||||
case screenshotManager
|
||||
case screenCaptureCLI
|
||||
|
||||
var isLiveStream: Bool { self == .stream }
|
||||
var isLiveStream: Bool { self == .stream || self == .streamBackedScreenshot }
|
||||
}
|
||||
|
||||
struct WindowShot: Sendable {
|
||||
@@ -454,7 +455,8 @@ public enum Capture {
|
||||
static func windowShot(
|
||||
pid: pid_t,
|
||||
preferredWindowID: CGWindowID? = nil,
|
||||
scale: Double = 0.5
|
||||
scale: Double = 0.5,
|
||||
allowCLIFallback: Bool = true
|
||||
) async -> WindowShot? {
|
||||
// Passive permission gate — no prompt on the hot path. A denied grant
|
||||
// means SCK would hand us a black frame, so bail to `nil` early and let
|
||||
@@ -495,6 +497,7 @@ public enum Capture {
|
||||
return nil
|
||||
}
|
||||
|
||||
guard allowCLIFallback else { return nil }
|
||||
// ② Fallback: /usr/sbin/screencapture -l <windowID> (SCK hung or failed).
|
||||
if let raw = screencaptureWindow(windowID: target.windowID) {
|
||||
let scaled = (try? scaleImage(raw, scale: outputScale)) ?? raw
|
||||
@@ -669,18 +672,14 @@ public enum Capture {
|
||||
// lock.
|
||||
let filter = SCContentFilter(desktopIndependentWindow: scWindow)
|
||||
|
||||
let config = SCStreamConfiguration()
|
||||
let backingScale = backingScaleFactor(forWindowFrame: frame)
|
||||
// Native window pixels = points × backing scale, then × requested
|
||||
// downscale. Clamp to >= 1 so a tiny window never yields a 0-dim
|
||||
// buffer.
|
||||
config.width = max(1, Int((frame.width * backingScale * scale).rounded()))
|
||||
config.height = max(1, Int((frame.height * backingScale * scale).rounded()))
|
||||
config.showsCursor = false // never the REAL cursor
|
||||
config.scalesToFit = true
|
||||
config.pixelFormat = kCVPixelFormatType_32BGRA
|
||||
config.colorSpaceName = CGColorSpace.sRGB
|
||||
config.captureResolution = .best
|
||||
let config = makeWindowShotConfiguration(
|
||||
width: max(1, Int(ceil(frame.width * backingScale * scale))),
|
||||
height: max(1, Int(ceil(frame.height * backingScale * scale)))
|
||||
)
|
||||
|
||||
return try await SCScreenshotManager.captureImage(
|
||||
contentFilter: filter,
|
||||
@@ -691,6 +690,31 @@ public enum Capture {
|
||||
}
|
||||
}
|
||||
|
||||
static func makeWindowShotConfiguration(width: Int, height: Int) -> SCStreamConfiguration {
|
||||
let config = SCStreamConfiguration()
|
||||
config.width = max(1, width)
|
||||
config.height = max(1, height)
|
||||
config.showsCursor = false
|
||||
config.scalesToFit = true
|
||||
config.preservesAspectRatio = true
|
||||
config.pixelFormat = kCVPixelFormatType_32BGRA
|
||||
config.colorSpaceName = CGColorSpace.sRGB
|
||||
config.captureResolution = .best
|
||||
// SCScreenshotManager includes this window's shadow by default. That
|
||||
// shrinks/pads the image while our click transform still names the
|
||||
// shadow-free window bounds, particularly after a cold background start.
|
||||
config.ignoreShadowsSingleWindow = true
|
||||
config.ignoreShadowsDisplay = true
|
||||
if #available(macOS 14.2, *) {
|
||||
// The sharing indicator can become an attached child above the
|
||||
// window. Capturing that union shrinks/offsets the main content
|
||||
// without changing SCWindow.frame or filter.contentRect, breaking
|
||||
// screenshot-to-click coordinates. The target is this exact window.
|
||||
config.includeChildWindows = false
|
||||
}
|
||||
return config
|
||||
}
|
||||
|
||||
/// CLI fallback when SCK hangs/fails: `/usr/sbin/screencapture -l <windowID>
|
||||
/// -x -o <tmp.png>`. `-x` = no capture sound, `-o` = omit the window shadow.
|
||||
/// Bounded at 3s with a `terminate` → `SIGKILL` escalation. Returns the
|
||||
|
||||
@@ -103,6 +103,7 @@ public final class CommandRouter {
|
||||
AXTree.resetSessionSnapshots()
|
||||
Self.lastShotTransform.removeAll()
|
||||
Self.lastCaptureDigest.removeAll()
|
||||
MutationClock.reset()
|
||||
windowCaptureProvider?.invalidate()
|
||||
// Apps we told they were focused must be told they are not, or the
|
||||
// belief outlives the session that needed it.
|
||||
@@ -661,17 +662,17 @@ public final class CommandRouter {
|
||||
// frame. Costs nothing when no action is pending. The rendered tree
|
||||
// doubles as the busy signal — a progress indicator in it means the
|
||||
// app is still working, so we allow a longer window.
|
||||
await MutationClock.awaitSettle(
|
||||
let streamedShot = await Self.captureSettledWindowShot(
|
||||
appIsBusy: result.axText.contains("progress indicator")
|
||||
)
|
||||
|
||||
let streamedShot = await windowCaptureProvider?.windowShot(
|
||||
pid: pid,
|
||||
processIdentity: snapshotEvidence.processIdentity,
|
||||
preferredWindowID: snapshotEvidence.keyWindowID,
|
||||
scale: 0.5,
|
||||
newerThanUptime: MutationClock.lastMutation()
|
||||
)
|
||||
) {
|
||||
await windowCaptureProvider?.windowShot(
|
||||
pid: pid,
|
||||
processIdentity: snapshotEvidence.processIdentity,
|
||||
preferredWindowID: snapshotEvidence.keyWindowID,
|
||||
scale: 0.5,
|
||||
newerThanUptime: MutationClock.lastMutation()
|
||||
)
|
||||
}
|
||||
guard TargetVisibilityPolicy.captureTargetStillMatches(
|
||||
snapshotWindowID: snapshotEvidence.keyWindowID,
|
||||
currentWindowID: AXTree.currentKeyWindowID(pid: pid)
|
||||
@@ -777,6 +778,7 @@ public final class CommandRouter {
|
||||
"pointWidth": .double(shot.pointWidth),
|
||||
"pointHeight": .double(shot.pointHeight),
|
||||
"windowID": .int(Int(shot.windowID)),
|
||||
"captureSource": .string(shot.source.rawValue),
|
||||
])
|
||||
// Cache the inverse transform so a later coordinate click/scroll/
|
||||
// drag (which arrives in image-pixel space) can be mapped back to
|
||||
@@ -800,6 +802,16 @@ public final class CommandRouter {
|
||||
return .object(object)
|
||||
}
|
||||
|
||||
/// Keep the wait and the actual capture in one production boundary so an
|
||||
/// action-to-screenshot regression can exercise both without live AX/TCC.
|
||||
static func captureSettledWindowShot(
|
||||
appIsBusy: Bool,
|
||||
capture: () async -> WindowShot?
|
||||
) async -> WindowShot? {
|
||||
await MutationClock.awaitSettle(appIsBusy: appIsBusy)
|
||||
return await capture()
|
||||
}
|
||||
|
||||
/// A snapshot discarded by an internal key-window retry was never delivered
|
||||
/// to the model, so it cannot become the baseline for a returned diff.
|
||||
static func effectiveDisableDiff(
|
||||
|
||||
@@ -396,6 +396,10 @@ enum ForegroundMutationRunner {
|
||||
lease: ForegroundLease,
|
||||
action: () async throws -> T
|
||||
) async throws -> T {
|
||||
// Every input path crosses this boundary, including synthetic events
|
||||
// that never call AXAction.settle(). A throw can follow a partial
|
||||
// delivery, so neither success nor failure may reuse pre-action pixels.
|
||||
defer { MutationClock.recordMutation() }
|
||||
let result: Result<T, Error>
|
||||
do {
|
||||
result = .success(try await action())
|
||||
|
||||
@@ -33,10 +33,10 @@ enum TargetVisibilityPolicy {
|
||||
return """
|
||||
NOTE: Another application fully covers the target window. A \
|
||||
long-lived window stream remains subscribed while it is covered, \
|
||||
and this screenshot comes from its latest complete frame rather \
|
||||
than the visible desktop. Coverage does not block Accessibility \
|
||||
actions or app- and window-targeted input; continue without \
|
||||
activating or raising the target.
|
||||
and this state uses a new on-demand window screenshot, not the \
|
||||
stream's cached frame or the visible desktop. Keep using app- and \
|
||||
window-targeted input without activating or raising the target. \
|
||||
An action receipt alone does not prove the application responded.
|
||||
"""
|
||||
}
|
||||
return """
|
||||
@@ -59,13 +59,13 @@ enum TargetVisibilityPolicy {
|
||||
cause = windowIsCovered
|
||||
? """
|
||||
The target is covered, but its long-lived window stream is \
|
||||
still active; the newest complete frame contains no visible \
|
||||
pixel change. Coverage does not block app- and window-targeted \
|
||||
input.
|
||||
still active, and a new on-demand screenshot contains no \
|
||||
visible pixel change. This does not establish whether the \
|
||||
app accepted the input or redrew its content.
|
||||
"""
|
||||
: """
|
||||
The target window is not fully covered, and the newest \
|
||||
complete stream frame contains no visible pixel change.
|
||||
The target window is not fully covered, and the new \
|
||||
on-demand screenshot contains no visible pixel change.
|
||||
"""
|
||||
} else {
|
||||
cause = windowIsCovered
|
||||
|
||||
@@ -88,5 +88,7 @@ enum MutationClock {
|
||||
try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
|
||||
}
|
||||
|
||||
static func resetForTests() { lastMutationAt = nil }
|
||||
static func reset() { lastMutationAt = nil }
|
||||
|
||||
static func resetForTests() { reset() }
|
||||
}
|
||||
|
||||
@@ -812,8 +812,10 @@ public final class VirtualCursor {
|
||||
// MARK: - Headless disk persistence
|
||||
|
||||
private static func runtimeDir() -> URL {
|
||||
let home = FileManager.default.homeDirectoryForCurrentUser
|
||||
let dir = home.appendingPathComponent(".claude/.runtime", isDirectory: true)
|
||||
let config = ProcessInfo.processInfo.environment["CLAUDE_CONFIG_DIR"]
|
||||
.flatMap { $0.isEmpty ? nil : ($0 as NSString).expandingTildeInPath }
|
||||
?? (NSHomeDirectory() as NSString).appendingPathComponent(".claude")
|
||||
let dir = URL(fileURLWithPath: config).appendingPathComponent(".runtime", isDirectory: true)
|
||||
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
|
||||
return dir
|
||||
}
|
||||
|
||||
@@ -94,6 +94,7 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
private let factory: any WindowCaptureStreamSourceFactory
|
||||
private let frameWaitAttempts: Int
|
||||
private let frameWaitNanoseconds: UInt64
|
||||
private let takeSnapshot: (WindowCaptureStreamTarget, Double) async -> WindowShot?
|
||||
private var generation: UInt64 = 0
|
||||
private var starting: Entry?
|
||||
private var active: Entry?
|
||||
@@ -105,11 +106,20 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
init(
|
||||
factory: any WindowCaptureStreamSourceFactory,
|
||||
frameWaitAttempts: Int = 12,
|
||||
frameWaitNanoseconds: UInt64 = 50_000_000
|
||||
frameWaitNanoseconds: UInt64 = 50_000_000,
|
||||
takeSnapshot: @escaping (WindowCaptureStreamTarget, Double) async -> WindowShot? = { target, scale in
|
||||
await Capture.windowShot(
|
||||
pid: target.key.pid,
|
||||
preferredWindowID: target.key.windowID,
|
||||
scale: scale,
|
||||
allowCLIFallback: false
|
||||
)
|
||||
}
|
||||
) {
|
||||
self.factory = factory
|
||||
self.frameWaitAttempts = max(0, frameWaitAttempts)
|
||||
self.frameWaitNanoseconds = frameWaitNanoseconds
|
||||
self.takeSnapshot = takeSnapshot
|
||||
}
|
||||
|
||||
func windowShot(
|
||||
@@ -139,10 +149,7 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
invalidate()
|
||||
return nil
|
||||
}
|
||||
guard let frame = await frame(
|
||||
for: target,
|
||||
newerThanUptime: newerThanUptime
|
||||
) else {
|
||||
guard let shot = await captureSnapshot(for: target, scale: scale) else {
|
||||
return nil
|
||||
}
|
||||
if let preferredWindowID,
|
||||
@@ -170,7 +177,38 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
guard current.key == target.key else {
|
||||
continue
|
||||
}
|
||||
return Capture.windowShot(from: frame, target: current)
|
||||
return WindowShot(
|
||||
base64: shot.base64, width: shot.width, height: shot.height,
|
||||
originX: current.originX, originY: current.originY,
|
||||
pointWidth: current.pointWidth, pointHeight: current.pointHeight,
|
||||
windowID: current.key.windowID, source: .streamBackedScreenshot
|
||||
)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/// Match the reference's two separate lifetimes: SCStream remains a
|
||||
/// consumer while covered; every state read runs an on-demand Skyshot/SCK
|
||||
/// capture. An idle stream's cached frame is not evidence of the current UI.
|
||||
func captureSnapshot(for target: WindowCaptureStreamTarget, scale: Double) async -> WindowShot? {
|
||||
for _ in 0..<2 {
|
||||
guard let source = await source(for: target) else { continue }
|
||||
if source.hasFailed {
|
||||
retire(source: source)
|
||||
continue
|
||||
}
|
||||
let snapshotGeneration = generation
|
||||
guard let shot = await takeSnapshot(target, scale),
|
||||
snapshotGeneration == generation,
|
||||
active?.source === source,
|
||||
!source.hasFailed,
|
||||
shot.source == .screenshotManager,
|
||||
shot.windowID == target.key.windowID,
|
||||
shot.width == target.key.pixelWidth,
|
||||
shot.height == target.key.pixelHeight else {
|
||||
return nil
|
||||
}
|
||||
return shot
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -121,9 +121,10 @@ enum WindowTargetedEvent {
|
||||
}
|
||||
|
||||
private static let runtimeDirectory: URL? = {
|
||||
let home = ProcessInfo.processInfo.environment["HOME"] ?? NSHomeDirectory()
|
||||
let dir = URL(fileURLWithPath: home)
|
||||
.appendingPathComponent(".claude")
|
||||
let config = ProcessInfo.processInfo.environment["CLAUDE_CONFIG_DIR"]
|
||||
.flatMap { $0.isEmpty ? nil : ($0 as NSString).expandingTildeInPath }
|
||||
?? (NSHomeDirectory() as NSString).appendingPathComponent(".claude")
|
||||
let dir = URL(fileURLWithPath: config)
|
||||
.appendingPathComponent(".runtime")
|
||||
try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true)
|
||||
return dir
|
||||
|
||||
@@ -141,6 +141,27 @@ final class CommandRouterSafetyTests: XCTestCase {
|
||||
XCTAssertEqual(provider.invalidateCount, 1)
|
||||
}
|
||||
|
||||
func testSessionResetClearsThePreviousMutationSettleDeadline() async throws {
|
||||
let monitor = PhysicalInputEpochMonitor(counterReader: { _ in 0 })
|
||||
let router = CommandRouter(
|
||||
cursor: VirtualCursor(headless: true),
|
||||
capabilities: Capabilities(headless: true),
|
||||
inputMonitor: monitor
|
||||
)
|
||||
let process = try XCTUnwrap(ProvenProcessTarget(pid: 77, identity: processA))
|
||||
let lease = try ForegroundLease.acquire(target: process, runtime: ForegroundLeaseRuntime(
|
||||
inputSnapshot: { PhysicalInputEpochSnapshot(epoch: 0, available: true) },
|
||||
frontmostTarget: { nil },
|
||||
currentIdentity: { _ in self.processA },
|
||||
activate: { _ in XCTFail("must not activate"); return false },
|
||||
verifyFrontmost: { _ in XCTFail("must not activate"); return false }
|
||||
))
|
||||
_ = try await ForegroundMutationRunner.run(lease: lease) { true }
|
||||
XCTAssertNotNil(MutationClock.lastMutation())
|
||||
router.resetSessionState()
|
||||
XCTAssertNil(MutationClock.lastMutation())
|
||||
}
|
||||
|
||||
func testDiscardedWindowSnapshotForcesTheRetryToReturnAFullTree() {
|
||||
XCTAssertFalse(CommandRouter.effectiveDisableDiff(
|
||||
requested: false,
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import Darwin
|
||||
import Foundation
|
||||
import XCTest
|
||||
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
@MainActor
|
||||
final class RuntimeIsolationTests: XCTestCase {
|
||||
func testHeadlessCursorRoundTripsOnlyThroughTheConfiguredSandbox() async throws {
|
||||
let directory = FileManager.default.temporaryDirectory
|
||||
.appendingPathComponent("cu-cursor-isolation-\(UUID().uuidString)")
|
||||
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
|
||||
let previous = getenv("CLAUDE_CONFIG_DIR").map { String(cString: $0) }
|
||||
setenv("CLAUDE_CONFIG_DIR", directory.path, 1)
|
||||
defer {
|
||||
if let previous { setenv("CLAUDE_CONFIG_DIR", previous, 1) }
|
||||
else { unsetenv("CLAUDE_CONFIG_DIR") }
|
||||
try? FileManager.default.removeItem(at: directory)
|
||||
}
|
||||
|
||||
let cursor = VirtualCursor(headless: true)
|
||||
XCTAssertEqual(cursor.position, .zero)
|
||||
await cursor.move(to: CGPoint(x: 13, y: 29), animated: false)
|
||||
XCTAssertTrue(FileManager.default.fileExists(
|
||||
atPath: directory.appendingPathComponent(".runtime/cu-helper.cursor.json").path
|
||||
))
|
||||
XCTAssertEqual(VirtualCursor(headless: true).position, CGPoint(x: 13, y: 29))
|
||||
}
|
||||
}
|
||||
@@ -160,8 +160,8 @@ final class TargetVisibilityPolicyTests: XCTestCase {
|
||||
liveStreamActive: true
|
||||
)
|
||||
XCTAssertTrue(notice.contains("long-lived window stream"))
|
||||
XCTAssertTrue(notice.contains("latest complete frame"))
|
||||
XCTAssertTrue(notice.contains("does not block"))
|
||||
XCTAssertTrue(notice.contains("new on-demand window screenshot"))
|
||||
XCTAssertTrue(notice.contains("does not prove the application responded"))
|
||||
XCTAssertFalse(notice.contains("may be stale"))
|
||||
XCTAssertFalse(notice.contains("paused its renderer"))
|
||||
XCTAssertFalse(notice.contains("refused"))
|
||||
@@ -234,8 +234,8 @@ final class TargetVisibilityPolicyTests: XCTestCase {
|
||||
liveStreamActive: true
|
||||
)
|
||||
XCTAssertTrue(covered.contains("long-lived window stream"))
|
||||
XCTAssertTrue(covered.contains("newest complete frame"))
|
||||
XCTAssertTrue(covered.contains("does not block"))
|
||||
XCTAssertTrue(covered.contains("new on-demand screenshot"))
|
||||
XCTAssertTrue(covered.contains("does not establish whether"))
|
||||
XCTAssertTrue(covered.contains("no visible pixel change"))
|
||||
XCTAssertFalse(covered.contains("paused its renderer"))
|
||||
}
|
||||
|
||||
@@ -6,6 +6,144 @@ import XCTest
|
||||
|
||||
@MainActor
|
||||
final class WindowCaptureStreamTests: XCTestCase {
|
||||
func testOnDemandScreenshotUsesOnlyTheTargetWindowBounds() {
|
||||
let config = Capture.makeWindowShotConfiguration(width: 1061, height: 752)
|
||||
XCTAssertEqual(config.width, 1061)
|
||||
XCTAssertEqual(config.height, 752)
|
||||
XCTAssertTrue(config.ignoreShadowsSingleWindow)
|
||||
XCTAssertTrue(config.ignoreShadowsDisplay)
|
||||
if #available(macOS 14.2, *) {
|
||||
XCTAssertFalse(config.includeChildWindows, "A sharing child must not expand the screenshot beyond its click transform")
|
||||
}
|
||||
XCTAssertTrue(config.scalesToFit)
|
||||
XCTAssertTrue(config.preservesAspectRatio)
|
||||
XCTAssertFalse(config.showsCursor)
|
||||
}
|
||||
|
||||
func testEveryStateReadTakesANewSnapshotWhileReusingTheLongLivedStream() async throws {
|
||||
let target = makeTarget(windowID: 84)
|
||||
let factory = FakeWindowCaptureStreamFactory { _, _ in }
|
||||
var captures = 0
|
||||
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
|
||||
captures += 1
|
||||
return self.makeSnapshot(target, pixels: "read-\(captures)")
|
||||
})
|
||||
let first = await manager.captureSnapshot(for: target, scale: 0.5)
|
||||
let second = await manager.captureSnapshot(for: target, scale: 0.5)
|
||||
XCTAssertEqual(first?.base64, "read-1")
|
||||
XCTAssertEqual(second?.base64, "read-2")
|
||||
XCTAssertEqual(captures, 2)
|
||||
XCTAssertEqual(factory.sources.count, 1)
|
||||
XCTAssertEqual(factory.sources[0].startCount, 1)
|
||||
XCTAssertEqual(factory.sources[0].latestReadCount, 0, "A cached stream frame must not become the model's screenshot")
|
||||
}
|
||||
|
||||
func testSnapshotFailureDoesNotFallBackToCachedStreamPixels() async {
|
||||
let target = makeTarget(windowID: 85)
|
||||
let factory = FakeWindowCaptureStreamFactory { source, _ in
|
||||
source.startFrame = makeFrame(for: source.targetKey, sequence: 1, uptime: 10, byte: 7)
|
||||
}
|
||||
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { _, _ in nil })
|
||||
let shot = await manager.captureSnapshot(for: target, scale: 0.5)
|
||||
XCTAssertNil(shot)
|
||||
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
|
||||
}
|
||||
|
||||
func testSnapshotFinishingAfterSessionInvalidationIsDiscarded() async {
|
||||
let target = makeTarget(windowID: 86)
|
||||
let factory = FakeWindowCaptureStreamFactory { _, _ in }
|
||||
var manager: WindowCaptureStreamManager!
|
||||
manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
|
||||
manager.invalidate()
|
||||
return self.makeSnapshot(target, pixels: "late")
|
||||
})
|
||||
let shot = await manager.captureSnapshot(for: target, scale: 0.5)
|
||||
XCTAssertNil(shot)
|
||||
XCTAssertEqual(factory.sources[0].retireCount, 1)
|
||||
}
|
||||
|
||||
private func makeSnapshot(_ target: WindowCaptureStreamTarget, pixels: String) -> WindowShot {
|
||||
WindowShot(
|
||||
base64: pixels, width: target.key.pixelWidth, height: target.key.pixelHeight,
|
||||
originX: target.originX, originY: target.originY,
|
||||
pointWidth: target.pointWidth, pointHeight: target.pointHeight,
|
||||
windowID: target.key.windowID, source: .screenshotManager
|
||||
)
|
||||
}
|
||||
|
||||
func testEveryDispatchedMutationSettlesBeforeTheNextOnDemandSnapshot() async throws {
|
||||
MutationClock.resetForTests()
|
||||
defer { MutationClock.resetForTests() }
|
||||
let target = makeTarget(windowID: 81)
|
||||
let factory = FakeWindowCaptureStreamFactory { _, _ in }
|
||||
var captures = 0
|
||||
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
|
||||
captures += 1
|
||||
self.assertMutationHasSettledBeforeCapture()
|
||||
return self.makeSnapshot(target, pixels: "unchanged-pixels")
|
||||
})
|
||||
var previousMutation: TimeInterval?
|
||||
for expectedCaptureCount in 1...2 {
|
||||
// Drive the real dispatcher; do not hand-write its timestamp.
|
||||
_ = try await ForegroundMutationRunner.run(lease: makeBackgroundLease(target)) { true }
|
||||
XCTAssertNotNil(MutationClock.lastMutation())
|
||||
XCTAssertNotEqual(MutationClock.lastMutation(), previousMutation)
|
||||
previousMutation = MutationClock.lastMutation()
|
||||
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
|
||||
await manager.captureSnapshot(for: target, scale: 0.5)
|
||||
}
|
||||
XCTAssertEqual(shot?.base64, "unchanged-pixels")
|
||||
XCTAssertEqual(captures, expectedCaptureCount, "Identical pixels still require a new capture")
|
||||
}
|
||||
XCTAssertEqual(factory.sources.count, 1)
|
||||
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
|
||||
}
|
||||
|
||||
func testPartiallyFailedDispatchAlsoSettlesBeforeTheOnDemandSnapshot() async throws {
|
||||
MutationClock.resetForTests()
|
||||
defer { MutationClock.resetForTests() }
|
||||
let target = makeTarget(windowID: 83)
|
||||
let factory = FakeWindowCaptureStreamFactory { _, _ in }
|
||||
let manager = WindowCaptureStreamManager(factory: factory, takeSnapshot: { target, _ in
|
||||
self.assertMutationHasSettledBeforeCapture()
|
||||
return self.makeSnapshot(target, pixels: "partial-action-state")
|
||||
})
|
||||
do {
|
||||
let _: Bool = try await ForegroundMutationRunner.run(lease: makeBackgroundLease(target)) {
|
||||
throw CUError("partial_action", "An event may already have been delivered")
|
||||
}
|
||||
XCTFail("expected the original action failure")
|
||||
} catch let error as CUError {
|
||||
XCTAssertEqual(error.code, "partial_action")
|
||||
}
|
||||
XCTAssertNotNil(MutationClock.lastMutation())
|
||||
let shot = await CommandRouter.captureSettledWindowShot(appIsBusy: false) {
|
||||
await manager.captureSnapshot(for: target, scale: 0.5)
|
||||
}
|
||||
XCTAssertEqual(shot?.base64, "partial-action-state")
|
||||
XCTAssertEqual(factory.sources[0].latestReadCount, 0)
|
||||
}
|
||||
|
||||
private func assertMutationHasSettledBeforeCapture() {
|
||||
XCTAssertNotNil(MutationClock.lastMutation())
|
||||
XCTAssertEqual(UISettlePolicy.delay(
|
||||
now: ProcessInfo.processInfo.systemUptime,
|
||||
lastMutationAt: MutationClock.lastMutation(),
|
||||
appIsBusy: false
|
||||
), 0, "The actual screenshot callback must run after the action's settle deadline")
|
||||
}
|
||||
|
||||
private func makeBackgroundLease(_ target: WindowCaptureStreamTarget) throws -> ForegroundLease {
|
||||
let process = try XCTUnwrap(ProvenProcessTarget(pid: target.key.pid, identity: target.key.processIdentity))
|
||||
return try ForegroundLease.acquire(target: process, runtime: ForegroundLeaseRuntime(
|
||||
inputSnapshot: { PhysicalInputEpochSnapshot(epoch: 0, available: true) },
|
||||
frontmostTarget: { nil },
|
||||
currentIdentity: { _ in target.key.processIdentity },
|
||||
activate: { _ in XCTFail("must not take foreground"); return false },
|
||||
verifyFrontmost: { _ in XCTFail("must not take foreground"); return false }
|
||||
))
|
||||
}
|
||||
|
||||
func testSameTargetReusesStreamAndReturnsNewestFrameAcrossCoveredAction() async {
|
||||
let factory = FakeWindowCaptureStreamFactory { source, _ in
|
||||
source.startFrame = makeFrame(for: source.targetKey, sequence: 1, uptime: 10, byte: 1)
|
||||
|
||||
Reference in New Issue
Block a user