mirror of
https://github.com/NanmiCoder/claude-code-haha.git
synced 2026-10-10 03:43:11 +08:00
feat(computer-use): finalize native input and global app consent
Route coordinate clicks through the exact AX hit, reuse render-local attributes and keep keyboard macros within validated native boundaries. Preserve lossless capture evidence while returning bounded model images, and propagate cancellation through daemon and lock acquisition. Remove legacy per-app restrictions after feature-wide consent on both platforms while retaining global disablement and target validation. Include native helpers, regression tests and a stable receiver fixture. Validation: 224 focused TypeScript tests, 516 XCTest cases and 15 Swift Testing cases passed. Local sidecar and Electron builds, development package creation and package smoke checks passed.
This commit is contained in:
@@ -27,6 +27,7 @@ Rules closer to the code take precedence. For the directory you are changing, re
|
||||
## Implementation Rules
|
||||
|
||||
- Keep changes tied to the requested behavior. Reuse existing utilities, stores, services, and test harnesses; add dependencies or abstractions only when the task needs them.
|
||||
- Computer Use has one app-authorization boundary on both macOS and Windows: enabling it in Settings and confirming its consent dialog authorizes all apps. Do not add per-app prompts, allowlists, denylists, category-based access tiers, or host/helper exceptions after that consent. Global disablement, OS permissions, and target/process validity checks still apply.
|
||||
- Executable JS/TS production changes under `src/`, `desktop/src/`, or `adapters/` require a same-area regression test unless a maintainer explicitly approves an exception. For bugs, reproduce the failure or add a test that fails for the intended reason; report when reproduction is unavailable. Test the behavior and affected boundaries. See [test design](docs/internals/contributing.md#回归测试设计) for state transitions, replay, and coverage caveats.
|
||||
- Keep TypeScript ESM style: 2-space indentation, no semicolons, `PascalCase` components, and `camelCase` functions/hooks/stores. Use structured parsers and existing boundaries for structured data.
|
||||
- Do not commit generated output such as `artifacts/`, coverage reports, `node_modules/`, build directories, or Rust `target/` trees.
|
||||
|
||||
@@ -619,7 +619,7 @@ public enum AXAction {
|
||||
}
|
||||
}
|
||||
|
||||
/// Coordinate click that PREFERS the AX path (Codex parity). The model gives a
|
||||
/// Coordinate click that prefers the AX path. The model gives a
|
||||
/// point in the get_app_state screenshot; we map it to a GLOBAL point (caller's
|
||||
/// job) and hit-test the AX element under it, then press what's there. This is
|
||||
/// the load-bearing fix for Chromium/CEF apps (e.g. NeteaseMusic): the tree
|
||||
@@ -627,9 +627,10 @@ public enum AXAction {
|
||||
/// `AXUIElementCopyElementAtPosition` makes Chromium instantiate the a11y node
|
||||
/// for THAT point on demand — so we get a real, pressable element and call
|
||||
/// `AXPress`, with NO synthetic mouse event, without stealing the pointer, and
|
||||
/// while the app stays in the background. This is exactly what Codex does
|
||||
/// (its service imports `AXUIElementCopyElementAtPosition` + `…PerformAction`
|
||||
/// and posts ZERO `CGEvent`s). Falls back to the synthetic `postToPid` click
|
||||
/// while the app stays in the background. Only the exact hit is pressed:
|
||||
/// a canvas hit may return a window whose descendants include its close
|
||||
/// button, so index-click descendant traversal is not valid here.
|
||||
/// Falls back to the synthetic `postToPid` click
|
||||
/// (`clickPoint`) when no AX element answers or the press finds no action.
|
||||
/// Returns a tag naming the path taken, for diagnostics. Left button only takes
|
||||
/// the AX path; other buttons have no clean per-point AX analogue and go
|
||||
@@ -647,22 +648,17 @@ public enum AXAction {
|
||||
|
||||
if button == .left {
|
||||
nudgeChromiumAccessibility(pid: pid)
|
||||
if let hit = hitTest(pid: pid, at: point) {
|
||||
if let tag = pressPrimary(hit, times: reps) {
|
||||
settle()
|
||||
return "ax:point:\(tag)"
|
||||
}
|
||||
if let descendant = firstActionableDescendant(of: hit, depth: descendantScanDepth),
|
||||
let tag = pressPrimary(descendant, times: reps) {
|
||||
settle()
|
||||
return "ax:point:descendant:\(tag)"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// No AX element answered (or a non-left button): synthetic pointer click.
|
||||
try await clickPoint(pid: pid, x: x, y: y, clickCount: reps, button: button)
|
||||
return "synthetic:point"
|
||||
return try await CoordinateClickRouting.click(
|
||||
point: point,
|
||||
preferAccessibility: button == .left,
|
||||
hitTest: { hitTest(pid: pid, at: $0) },
|
||||
press: { pressPrimary($0, times: reps) },
|
||||
settle: { settle() },
|
||||
syntheticClick: { target in
|
||||
try await clickPoint(pid: pid, x: target.x, y: target.y, clickCount: reps, button: button)
|
||||
}
|
||||
)
|
||||
}
|
||||
|
||||
/// Best-effort nudge so a Chromium/Electron/CEF app exposes its accessibility
|
||||
@@ -992,7 +988,14 @@ public enum AXAction {
|
||||
pid: pid_t, _ key: String,
|
||||
validateBeforePosting: () throws -> Void = {}
|
||||
) async throws {
|
||||
let chords = try KeyMapping.parse(key)
|
||||
try await pressKey(pid: pid, chords: KeyMapping.parse(key), validateBeforePosting: validateBeforePosting)
|
||||
}
|
||||
|
||||
/// Receives already validated chords from the semantic command router.
|
||||
static func pressKey(
|
||||
pid: pid_t, chords: [KeyMapping.Chord],
|
||||
validateBeforePosting: () throws -> Void = {}
|
||||
) async throws {
|
||||
guard !chords.isEmpty else {
|
||||
throw CUError(CUError.Code.unknownKey, "Empty key sequence")
|
||||
}
|
||||
|
||||
@@ -911,13 +911,15 @@ public enum AXTree {
|
||||
pid: pid_t,
|
||||
systemWide: AXUIElement
|
||||
) -> AXUIElement? {
|
||||
// Prefer the system-wide focus only when it belongs to the target app.
|
||||
if let sysFocusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute),
|
||||
pidOf(sysFocusedApp) == pid,
|
||||
let el = copyElement(systemWide, kAXFocusedUIElementAttribute) {
|
||||
return el
|
||||
}
|
||||
return copyElement(app, kAXFocusedUIElementAttribute)
|
||||
FocusedElementRouting.select(
|
||||
targetPID: pid,
|
||||
frontmostPID: NSWorkspace.shared.frontmostApplication?.processIdentifier,
|
||||
systemFocusedPID: {
|
||||
copyElement(systemWide, kAXFocusedApplicationAttribute).map { pidOf($0) }
|
||||
},
|
||||
systemFocusedElement: { copyElement(systemWide, kAXFocusedUIElementAttribute) },
|
||||
targetFocusedElement: { copyElement(app, kAXFocusedUIElementAttribute) }
|
||||
)
|
||||
}
|
||||
|
||||
// MARK: - Renderer (format authority: blueprint §1)
|
||||
@@ -983,17 +985,29 @@ public enum AXTree {
|
||||
|
||||
let elementFingerprint = knownFingerprint ?? fingerprint(of: element)
|
||||
let role = elementFingerprint.role
|
||||
let attributes = RendererAttributeReuse(
|
||||
title: elementFingerprint.title,
|
||||
description: elementFingerprint.label,
|
||||
// AXUnknown also represents a failed role read in fingerprint.
|
||||
// Leave that case retryable instead of caching the fallback.
|
||||
role: role == "AXUnknown" ? nil : role
|
||||
)
|
||||
let subrole = elementFingerprint.subrole
|
||||
let baseRoleText = roleDescription(element, role: role, subrole: subrole)
|
||||
let label = stringValue(element, kAXDescriptionAttribute)
|
||||
let label = attributes.description { stringValue(element, kAXDescriptionAttribute) }
|
||||
let help = stringValue(element, kAXHelpAttribute)
|
||||
let value = sanitizedValue(element)
|
||||
let identifier = displayIdentifier(elementFingerprint.identifier)
|
||||
let traits = traitList(element)
|
||||
let traits = traitList(element) {
|
||||
attributes.settable { readSettable(element, kAXValueAttribute) } ?? false
|
||||
}
|
||||
let rawActions = actionNames(element)
|
||||
let prettyActions = meaningfulActions(rawActions, role: role)
|
||||
let placeholder = placeholderValue(element)
|
||||
let childElements = AXTree.walkChildren(of: element)
|
||||
let childElements = AXTree.walkChildren(
|
||||
of: element,
|
||||
role: attributes.role { stringValue(element, kAXRoleAttribute) }
|
||||
)
|
||||
let childFingerprints = childElements.map(fingerprint(of:))
|
||||
let rowTexts = role == (kAXRowRole as String) ? flattenedRowTexts(element) : []
|
||||
|
||||
@@ -1003,7 +1017,8 @@ public enum AXTree {
|
||||
label: label,
|
||||
identifier: identifier,
|
||||
explicitValue: value,
|
||||
rowTexts: rowTexts
|
||||
rowTexts: rowTexts,
|
||||
attributes: attributes
|
||||
)
|
||||
let linkText = role == axLinkRole
|
||||
? markdownLinkText(element, title: title, label: label, value: value)
|
||||
@@ -1125,7 +1140,7 @@ public enum AXTree {
|
||||
roleText: roleText,
|
||||
title: displayTitle,
|
||||
value: value,
|
||||
settable: isSettable(element, kAXValueAttribute),
|
||||
settable: attributes.settable { readSettable(element, kAXValueAttribute) } ?? false,
|
||||
frameGlobal: globalFrame(element),
|
||||
rawActions: rawActions,
|
||||
depth: depth
|
||||
@@ -1315,9 +1330,10 @@ public enum AXTree {
|
||||
label: String?,
|
||||
identifier: String?,
|
||||
explicitValue: String?,
|
||||
rowTexts: [String]
|
||||
rowTexts: [String],
|
||||
attributes: RendererAttributeReuse
|
||||
) -> String? {
|
||||
if let title = stringValue(element, kAXTitleAttribute), !title.isEmpty {
|
||||
if let title = attributes.title(read: { stringValue(element, kAXTitleAttribute) }), !title.isEmpty {
|
||||
return sanitize(title)
|
||||
}
|
||||
if role == (kAXRowRole as String) { return rowTexts.first }
|
||||
@@ -1504,7 +1520,10 @@ public enum AXTree {
|
||||
/// skipped under the menu bar. THIS is the ordering `(windowIndex, path)`
|
||||
/// indexes — `resolve` calls the SAME function so locators round-trip exactly.
|
||||
static func walkChildren(of element: AXUIElement) -> [AXUIElement] {
|
||||
let role = stringValue(element, kAXRoleAttribute)
|
||||
walkChildren(of: element, role: stringValue(element, kAXRoleAttribute))
|
||||
}
|
||||
|
||||
private static func walkChildren(of element: AXUIElement, role: String?) -> [AXUIElement] {
|
||||
let rows = copyElementArray(element, kAXRowsAttribute) ?? []
|
||||
let visibleChildren = copyElementArray(element, axVisibleChildrenAttribute) ?? []
|
||||
let attributes = childTraversalAttributes(
|
||||
@@ -1629,12 +1648,12 @@ public enum AXTree {
|
||||
|
||||
// MARK: - Traits (blueprint §1)
|
||||
|
||||
private static func traitList(_ element: AXUIElement) -> [String] {
|
||||
private static func traitList(_ element: AXUIElement, isValueSettable: () -> Bool) -> [String] {
|
||||
var values: [String] = []
|
||||
if boolValue(element, kAXSelectedAttribute) == true { values.append("selected") }
|
||||
if boolValue(element, kAXExpandedAttribute) == true { values.append("expanded") }
|
||||
if boolValue(element, kAXEnabledAttribute) == false { values.append("disabled") }
|
||||
if isSettable(element, kAXValueAttribute) {
|
||||
if isValueSettable() {
|
||||
values.append("settable")
|
||||
if let valueType = valueTypeTrait(element) { values.append(valueType) }
|
||||
}
|
||||
@@ -1843,10 +1862,10 @@ public enum AXTree {
|
||||
return trimmed.isEmpty ? nil : trimmed
|
||||
}
|
||||
|
||||
private static func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool {
|
||||
private static func readSettable(_ element: AXUIElement, _ attribute: String) -> Bool? {
|
||||
var settable = DarwinBoolean(false)
|
||||
let err = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable)
|
||||
return err == .success && settable.boolValue
|
||||
return err == .success ? settable.boolValue : nil
|
||||
}
|
||||
|
||||
private static func pidOf(_ element: AXUIElement) -> pid_t {
|
||||
|
||||
@@ -1,63 +0,0 @@
|
||||
import Foundation
|
||||
|
||||
/// Built-in macOS native policy, matched by exact resolved bundle identity.
|
||||
/// The installed official service checks terminal, Computer Use host, ChatGPT,
|
||||
/// and system-security groups. Other categories remain subject to app approval;
|
||||
/// the older generic Windows terminal/IDE/media/trading lists do not define this
|
||||
/// native boundary.
|
||||
enum AppTargetPolicy {
|
||||
enum Decision: Equatable, Sendable {
|
||||
case allow
|
||||
case deny
|
||||
}
|
||||
|
||||
/// Process identities that Computer Use must never inspect or control.
|
||||
/// Keep these independent from the mirrored generic policy set below: the
|
||||
/// host and helper remain denied even if the cross-language policy changes.
|
||||
static let intrinsicDeniedBundleIDs: Set<String> = [
|
||||
"com.claude-code-haha.desktop",
|
||||
"dev.cchaha.cu-helper",
|
||||
]
|
||||
|
||||
/// Audited against SkyComputerUseService 26.831.1000926's
|
||||
/// BundleIdentifiers.isForbiddenComputerUseTarget (0x100240a14).
|
||||
/// Keep this literal set cross-checked with nativeAppPolicy.ts.
|
||||
static let deniedBundleIDs: Set<String> = [
|
||||
// terminal
|
||||
"com.apple.Terminal",
|
||||
"com.googlecode.iterm2",
|
||||
"org.alacritty",
|
||||
"dev.warp.Warp-Stable",
|
||||
"net.kovidgoyal.kitty",
|
||||
"co.zeit.hyper",
|
||||
"com.github.wez.wezterm",
|
||||
"org.tabby",
|
||||
"com.mitchellh.ghostty",
|
||||
"com.raphaelamorim.rio",
|
||||
"dev.commandline.waveterm",
|
||||
// computerUseHost
|
||||
"com.openai.codex",
|
||||
"com.openai.codex.alpha",
|
||||
"com.openai.codex.beta",
|
||||
"com.openai.codex.dev",
|
||||
"com.openai.codex.nightly",
|
||||
// chatGPT
|
||||
"com.openai.chat",
|
||||
"com.openai.chat.alpha",
|
||||
"com.openai.chat.beta",
|
||||
"com.openai.chat.nightly",
|
||||
"com.openai.chat.mac-debug",
|
||||
// systemSecurity
|
||||
"com.apple.UserNotificationCenter",
|
||||
"com.apple.LocalAuthenticationRemoteService",
|
||||
"com.apple.SecurityAgent",
|
||||
]
|
||||
|
||||
static func decision(bundleID: String) -> Decision {
|
||||
if intrinsicDeniedBundleIDs.contains(bundleID)
|
||||
|| deniedBundleIDs.contains(bundleID) {
|
||||
return .deny
|
||||
}
|
||||
return .allow
|
||||
}
|
||||
}
|
||||
@@ -60,13 +60,74 @@ struct WindowShot: Sendable {
|
||||
/// Uniform capture fit before integer pixel-buffer rounding. Legacy shots
|
||||
/// may omit this and retain their dimension-derived transform.
|
||||
var pixelsPerPoint: Double? = nil
|
||||
var mimeType: String { NativeScreenshotPolicy.mimeType }
|
||||
var mimeType: String = "image/png"
|
||||
}
|
||||
|
||||
struct ModelWindowImage: Sendable {
|
||||
let base64: String
|
||||
let mimeType: String
|
||||
}
|
||||
|
||||
@available(macOS 14.0, *)
|
||||
@MainActor
|
||||
public enum Capture {
|
||||
|
||||
/// Apply the model image budget after capture freshness has been validated.
|
||||
/// Window geometry stays in points; returned dimensions name actual pixels.
|
||||
static func boundedModelWindowShot(_ shot: WindowShot) -> WindowShot? {
|
||||
guard shot.width > 0, shot.height > 0,
|
||||
shot.pointWidth.isFinite, shot.pointWidth > 0,
|
||||
shot.pointHeight.isFinite, shot.pointHeight > 0 else { return nil }
|
||||
let target = NativeScreenshotPolicy.pixelSize(
|
||||
pointSize: CGSize(width: shot.pointWidth, height: shot.pointHeight),
|
||||
backingScale: 1
|
||||
)
|
||||
// The production capture already applies this same policy. Preserve
|
||||
// its exact pre-rounding transform and pixels without a second resize.
|
||||
guard shot.width > Int(target.width) || shot.height > Int(target.height) else {
|
||||
return shot
|
||||
}
|
||||
guard let data = Data(base64Encoded: shot.base64),
|
||||
let source = CGImageSourceCreateWithData(data as CFData, nil),
|
||||
let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else {
|
||||
return nil
|
||||
}
|
||||
let scale = min(1, target.width / Double(image.width),
|
||||
target.height / Double(image.height))
|
||||
guard let bounded = try? scaleImage(image, scale: scale),
|
||||
let encoded = pngBase64WithSize(bounded) else {
|
||||
// Keep the existing AX-only degradation; never leak an unbounded
|
||||
// image after an allocation/encoding failure.
|
||||
return nil
|
||||
}
|
||||
return WindowShot(
|
||||
base64: encoded.base64, width: encoded.width, height: encoded.height,
|
||||
originX: shot.originX, originY: shot.originY,
|
||||
pointWidth: shot.pointWidth, pointHeight: shot.pointHeight,
|
||||
windowID: shot.windowID, source: shot.source,
|
||||
pixelsPerPoint: shot.pixelsPerPoint.map { $0 * scale }, mimeType: "image/png"
|
||||
)
|
||||
}
|
||||
|
||||
/// Presentation encoding only: keep the bounded lossless shot as the
|
||||
/// freshness/identical-capture evidence and retain its coordinate geometry.
|
||||
static func modelWindowImage(
|
||||
_ shot: WindowShot,
|
||||
quality: Double = NativeScreenshotPolicy.jpegQuality,
|
||||
encodeJPEG: @MainActor (CGImage, Double) -> String? = jpegBase64
|
||||
) -> ModelWindowImage {
|
||||
let lossless = ModelWindowImage(base64: shot.base64, mimeType: shot.mimeType)
|
||||
guard quality.isFinite, (0...1).contains(quality),
|
||||
let data = Data(base64Encoded: shot.base64),
|
||||
let source = CGImageSourceCreateWithData(data as CFData, nil),
|
||||
let image = CGImageSourceCreateImageAtIndex(source, 0, nil),
|
||||
image.width == shot.width, image.height == shot.height,
|
||||
let base64 = encodeJPEG(image, quality), !base64.isEmpty else {
|
||||
return lossless
|
||||
}
|
||||
return ModelWindowImage(base64: base64, mimeType: "image/jpeg")
|
||||
}
|
||||
|
||||
// MARK: - Permission gates
|
||||
|
||||
/// Passive Screen Recording check. Never prompts.
|
||||
@@ -421,7 +482,7 @@ public enum Capture {
|
||||
// A *window-locked* capture used by the daemon's `get_app_state`: it pins the
|
||||
// target app's single most-relevant window, captures only that window (not the
|
||||
// full display, not the desktop, not other apps), downscales by `scale`
|
||||
// (default 0.5), and returns PNG base64 in the screenshot-pixel space (top-left
|
||||
// (default native App fit), and returns lossless PNG base64 in the screenshot-pixel space (top-left
|
||||
// origin) for the server's affine map. It is the only capture path tied to a
|
||||
// *pid* rather than a *display*.
|
||||
//
|
||||
@@ -449,7 +510,7 @@ public enum Capture {
|
||||
/// - scale: an explicit factor applied to native pixels; nil uses the
|
||||
/// native App policy (point resolution, long/short side limits).
|
||||
/// - Returns: `(base64, width, height, originX, originY, pointWidth,
|
||||
/// pointHeight, windowID)` — the JPEG in screenshot-pixel space (top-left origin)
|
||||
/// pointHeight, windowID)` — lossless PNG capture evidence in screenshot-pixel space (top-left origin)
|
||||
/// PLUS the captured window's GLOBAL Quartz top-left origin and its size
|
||||
/// in POINTS. The caller uses the uniform `pixelsPerPoint` capture fit
|
||||
/// to invert image-pixel coordinates back into the
|
||||
@@ -485,7 +546,7 @@ public enum Capture {
|
||||
frame: target.frame,
|
||||
scale: outputScale
|
||||
) {
|
||||
if let encoded = appScreenshotBase64WithSize(image) {
|
||||
if let encoded = pngBase64WithSize(image) {
|
||||
return WindowShot(
|
||||
base64: encoded.base64,
|
||||
width: encoded.width,
|
||||
@@ -509,7 +570,7 @@ public enum Capture {
|
||||
if let raw = screencaptureWindow(windowID: target.windowID) {
|
||||
let scaledImage = try? scaleImage(raw, scale: outputScale)
|
||||
let scaled = scaledImage ?? raw
|
||||
if let encoded = appScreenshotBase64WithSize(scaled) {
|
||||
if let encoded = pngBase64WithSize(scaled) {
|
||||
return WindowShot(
|
||||
base64: encoded.base64,
|
||||
width: encoded.width,
|
||||
@@ -801,6 +862,20 @@ public enum Capture {
|
||||
return scaled
|
||||
}
|
||||
|
||||
/// Keep capture/freshness evidence lossless. The model attachment is JPEG
|
||||
/// encoded only after the bound window and captured frame are validated.
|
||||
static func pngBase64WithSize(
|
||||
_ image: CGImage
|
||||
) -> (base64: String, width: Int, height: Int)? {
|
||||
let data = NSMutableData()
|
||||
guard let destination = CGImageDestinationCreateWithData(
|
||||
data, UTType.png.identifier as CFString, 1, nil
|
||||
) else { return nil }
|
||||
CGImageDestinationAddImage(destination, image, nil)
|
||||
guard CGImageDestinationFinalize(destination) else { return nil }
|
||||
return ((data as Data).base64EncodedString(), image.width, image.height)
|
||||
}
|
||||
|
||||
/// Native App screenshots use the official default JPEG quality. The
|
||||
/// byte format is reported explicitly instead of relying on a fixed MIME
|
||||
/// label in a downstream wrapper. Encoding failure degrades to AX text.
|
||||
|
||||
@@ -618,14 +618,6 @@ public final class CommandRouter {
|
||||
return .object(object)
|
||||
|
||||
case .installed(let installed):
|
||||
guard AppTargetPolicy.decision(
|
||||
bundleID: installed.bundleIdentifier
|
||||
) == .allow else {
|
||||
throw CUError(
|
||||
"app_denied",
|
||||
"Computer Use is not allowed to use the app '\(installed.bundleIdentifier)' for safety reasons."
|
||||
)
|
||||
}
|
||||
return .object([
|
||||
"bundleId": .string(installed.bundleIdentifier),
|
||||
"displayName": .string(installed.displayName),
|
||||
@@ -691,14 +683,6 @@ public final class CommandRouter {
|
||||
guard case .installed(let installed) = installedOutcome else {
|
||||
throw CUError("target_not_running", "The requested target app is not running")
|
||||
}
|
||||
guard AppTargetPolicy.decision(
|
||||
bundleID: installed.bundleIdentifier
|
||||
) == .allow else {
|
||||
throw CUError(
|
||||
"app_denied",
|
||||
"Computer Use is not allowed to use the app '\(installed.bundleIdentifier)' for safety reasons."
|
||||
)
|
||||
}
|
||||
let identifier = installed.bundleURL.standardizedFileURL.path
|
||||
guard let launched = await AppTargetResolver.launch(
|
||||
identifier: identifier,
|
||||
@@ -839,7 +823,9 @@ public final class CommandRouter {
|
||||
)
|
||||
}
|
||||
|
||||
if let shot,
|
||||
// Keep stream/frame validation in its native capture dimensions.
|
||||
// Publish and map coordinates using the same final model image.
|
||||
if let shot = shot.flatMap(Capture.boundedModelWindowShot),
|
||||
AXTree.currentProcessIdentity(pid: pid) == snapshotEvidence.processIdentity {
|
||||
// An identical capture is the other half of the same problem:
|
||||
// the pixels cannot say whether the action missed or the window
|
||||
@@ -854,9 +840,12 @@ public final class CommandRouter {
|
||||
) {
|
||||
Self.appendAXNotice(notice, to: &object)
|
||||
}
|
||||
// Lossless bytes above establish identical-frame evidence;
|
||||
// presentation encoding does not rescale or change geometry.
|
||||
let modelImage = Capture.modelWindowImage(shot)
|
||||
var screenshot: [String: JSONValue] = [
|
||||
"base64": .string(shot.base64),
|
||||
"mimeType": .string(shot.mimeType),
|
||||
"base64": .string(modelImage.base64),
|
||||
"mimeType": .string(modelImage.mimeType),
|
||||
"width": .int(shot.width),
|
||||
"height": .int(shot.height),
|
||||
"originX": .double(shot.originX),
|
||||
@@ -1505,23 +1494,19 @@ public final class CommandRouter {
|
||||
let systemKeyCombos = try SystemKeyPolicy.parseGrant(
|
||||
payload["systemKeyCombos"]
|
||||
)
|
||||
try SystemKeyPolicy.enforce(
|
||||
sequence: key,
|
||||
granted: systemKeyCombos
|
||||
)
|
||||
let chords = try KeyboardCommandSequence.prepare(key, systemKeyCombos: systemKeyCombos)
|
||||
let expected = try Self.expectedProcessTarget(payload)
|
||||
let target = try resolveTargetForMutation(payload)
|
||||
setResolvedTarget(target)
|
||||
try requireSnapshotProcess(target: target, expected: expected)
|
||||
return try await withForegroundLease(
|
||||
command: "press_key",
|
||||
target: target
|
||||
) {
|
||||
_ = try Injection.validateAuthorizedTarget(target)
|
||||
try self.requireSnapshotProcess(target: target, expected: expected)
|
||||
try await AXAction.pressKey(pid: target.pid, key)
|
||||
return .bool(true)
|
||||
try await KeyboardCommandSequence.run(chords: chords) { batch in
|
||||
try await self.withForegroundLease(command: "press_key", target: target) {
|
||||
_ = try Injection.validateAuthorizedTarget(target)
|
||||
try self.requireSnapshotProcess(target: target, expected: expected)
|
||||
try await AXAction.pressKey(pid: target.pid, chords: batch)
|
||||
}
|
||||
}
|
||||
return .bool(true)
|
||||
}
|
||||
|
||||
/// `drag`: coordinate-only press→drag→release between screenshot-local points.
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import CoreGraphics
|
||||
|
||||
/// The coordinate route is separate from an index click's AX-tree traversal.
|
||||
/// Dependencies allow routing tests without posting input or contacting apps.
|
||||
enum CoordinateClickRouting {
|
||||
@MainActor
|
||||
static func click<Element>(
|
||||
point: CGPoint,
|
||||
preferAccessibility: Bool,
|
||||
hitTest: (CGPoint) -> Element?,
|
||||
press: (Element) -> String?,
|
||||
settle: () -> Void,
|
||||
syntheticClick: (CGPoint) async throws -> Void
|
||||
) async throws -> String {
|
||||
if preferAccessibility, let hit = hitTest(point) {
|
||||
if let tag = press(hit) {
|
||||
settle()
|
||||
return "ax:point:\(tag)"
|
||||
}
|
||||
}
|
||||
// A canvas can hit-test to its entire AXWindow. Its first actionable
|
||||
// descendant may be the close button, far from the requested point.
|
||||
// Only the exact hit can authorize an AX action for a coordinate click.
|
||||
try await syntheticClick(point)
|
||||
return "synthetic:point"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
import Foundation
|
||||
|
||||
/// Select focus only from the requested application. The callbacks make the
|
||||
/// background/foreground decision testable without contacting an AX server.
|
||||
enum FocusedElementRouting {
|
||||
@MainActor
|
||||
static func select<Element>(
|
||||
targetPID: pid_t,
|
||||
frontmostPID: pid_t?,
|
||||
systemFocusedPID: () -> pid_t?,
|
||||
systemFocusedElement: () -> Element?,
|
||||
targetFocusedElement: () -> Element?
|
||||
) -> Element? {
|
||||
// Reading global AX focus can wait for an unrelated foreground app's
|
||||
// AX server. A known background target needs only its own focus tree.
|
||||
// Unknown frontmost identity retains the original global query, and
|
||||
// the actual AX PID remains authoritative if focus changes meanwhile.
|
||||
if (frontmostPID == nil || frontmostPID == targetPID),
|
||||
systemFocusedPID() == targetPID,
|
||||
let element = systemFocusedElement() {
|
||||
return element
|
||||
}
|
||||
return targetFocusedElement()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
import Foundation
|
||||
|
||||
/// Preflight a complete macro before entering the per-command input boundary.
|
||||
enum KeyboardCommandSequence {
|
||||
static let maximumChordCount = 128
|
||||
|
||||
static func prepare(_ sequence: String, systemKeyCombos: Bool) throws -> [KeyMapping.Chord] {
|
||||
let chords = try KeyMapping.parse(sequence)
|
||||
guard chords.count <= maximumChordCount else {
|
||||
throw CUError("bad_payload", "press_key accepts at most \(maximumChordCount) chords; no input was sent")
|
||||
}
|
||||
try SystemKeyPolicy.enforce(sequence: sequence, granted: systemKeyCombos)
|
||||
return chords
|
||||
}
|
||||
|
||||
/// Each callback is one complete existing native command boundary, including
|
||||
/// its foreground lease, focus preparation and finalization. Awaiting only
|
||||
/// a yield between CGEvents would not reproduce that boundary. One chord's
|
||||
/// down/up group is still fully allocated before it is posted.
|
||||
@MainActor
|
||||
static func run(
|
||||
chords: [KeyMapping.Chord],
|
||||
perform: @MainActor ([KeyMapping.Chord]) async throws -> Void
|
||||
) async throws {
|
||||
var completed = 0
|
||||
var inFlight = false
|
||||
do {
|
||||
for chord in chords {
|
||||
try Task.checkCancellation()
|
||||
inFlight = true
|
||||
try await perform([chord])
|
||||
completed += 1
|
||||
inFlight = false
|
||||
}
|
||||
try Task.checkCancellation()
|
||||
} catch {
|
||||
// Keep the established single-key error contract, including paste's
|
||||
// clipboard validation. A macro can have a completed prefix even
|
||||
// when its finalization refuses, so do not invite whole-macro retry.
|
||||
guard chords.count > 1 else { throw error }
|
||||
let code = (error as? CUError)?.code
|
||||
?? (error is CancellationError ? "cancelled" : "keyboard_sequence_failed")
|
||||
let delivery = inFlight ? " The failed chord may already have been delivered." : ""
|
||||
throw CUError(
|
||||
code,
|
||||
"press_key stopped after \(completed) of \(chords.count) chords completed.\(delivery) "
|
||||
+ "Inspect the current state before continuing; do not replay the completed prefix. "
|
||||
+ "Cause: \(error.localizedDescription)"
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
/// Successful scalar observations owned by one render invocation only.
|
||||
/// Missing/failed reads deliberately remain retryable. Never store this in a
|
||||
/// published snapshot or reuse it for action-time identity/focus validation.
|
||||
@MainActor
|
||||
final class RendererAttributeReuse {
|
||||
private var titleValue: String?
|
||||
private var descriptionValue: String?
|
||||
private var roleValue: String?
|
||||
private var settableValue: Bool?
|
||||
|
||||
init(title: String?, description: String?, role: String?) {
|
||||
titleValue = title
|
||||
descriptionValue = description
|
||||
roleValue = role
|
||||
}
|
||||
|
||||
func title(read: () -> String?) -> String? {
|
||||
if let value = titleValue { return value }
|
||||
let value = read()
|
||||
if let value { titleValue = value }
|
||||
return value
|
||||
}
|
||||
func description(read: () -> String?) -> String? {
|
||||
if let value = descriptionValue { return value }
|
||||
let value = read()
|
||||
if let value { descriptionValue = value }
|
||||
return value
|
||||
}
|
||||
func role(read: () -> String?) -> String? {
|
||||
if let value = roleValue { return value }
|
||||
let value = read()
|
||||
if let value { roleValue = value }
|
||||
return value
|
||||
}
|
||||
func settable(read: () -> Bool?) -> Bool? {
|
||||
if let value = settableValue { return value }
|
||||
let value = read()
|
||||
if let value { settableValue = value }
|
||||
return value
|
||||
}
|
||||
}
|
||||
@@ -1,10 +1,10 @@
|
||||
import Darwin
|
||||
import Foundation
|
||||
|
||||
/// Authoritative native policy applied only after a resolver has produced a
|
||||
/// Native process-identity validation applied after a resolver has produced a
|
||||
/// real running process and its current process-lifetime identity. Selector
|
||||
/// form is deliberately absent, so PID/name/bundle/frontmost/launch paths have
|
||||
/// no policy bypass.
|
||||
/// no process-identity bypass. Global Computer Use consent covers every app.
|
||||
enum ResolvedTargetAuthorization {
|
||||
static func authorize(
|
||||
resolved: ResolvedAppTarget,
|
||||
@@ -43,13 +43,6 @@ enum ResolvedTargetAuthorization {
|
||||
}
|
||||
}
|
||||
|
||||
guard AppTargetPolicy.decision(bundleID: actualBundleID) == .allow else {
|
||||
throw CUError(
|
||||
"app_denied",
|
||||
"Computer Use is not allowed to use the app '\(actualBundleID)' for safety reasons."
|
||||
)
|
||||
}
|
||||
|
||||
guard let target = ProvenProcessTarget(pid: pid, identity: identity) else {
|
||||
throw CUError(
|
||||
"app_denied",
|
||||
|
||||
@@ -210,7 +210,8 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
originX: current.originX, originY: current.originY,
|
||||
pointWidth: current.pointWidth, pointHeight: current.pointHeight,
|
||||
windowID: current.key.windowID, source: .streamBackedScreenshot,
|
||||
pixelsPerPoint: shot.pixelsPerPoint
|
||||
pixelsPerPoint: shot.pixelsPerPoint,
|
||||
mimeType: shot.mimeType
|
||||
)
|
||||
}
|
||||
return nil
|
||||
@@ -840,7 +841,7 @@ extension Capture {
|
||||
guard frame.width == target.key.pixelWidth,
|
||||
frame.height == target.key.pixelHeight,
|
||||
let image = image(from: frame),
|
||||
let encoded = appScreenshotBase64WithSize(image) else {
|
||||
let encoded = pngBase64WithSize(image) else {
|
||||
return nil
|
||||
}
|
||||
return WindowShot(
|
||||
|
||||
@@ -275,13 +275,25 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
|
||||
if exerciseKeys {
|
||||
let (canvasHandle, _) = try publishedHandle(label: "Drag fixture", state: clickedState)
|
||||
_ = try await router.handle(cmd: "click", payload: .object([
|
||||
"pid": .int(Int(pid)), "index": .string(canvasHandle.rawValue),
|
||||
]))
|
||||
_ = try await router.handle(cmd: "press_key", payload: .object([
|
||||
"pid": .int(Int(pid)),
|
||||
"key": .string("Control_L+a Super_R+b A question Delete BackSpace"),
|
||||
]))
|
||||
var phase = "canvas click"
|
||||
do {
|
||||
_ = try await router.handle(cmd: "click", payload: .object([
|
||||
"pid": .int(Int(pid)), "index": .string(canvasHandle.rawValue),
|
||||
]))
|
||||
phase = "keyboard macro"
|
||||
_ = try await router.handle(cmd: "press_key", payload: .object([
|
||||
"pid": .int(Int(pid)),
|
||||
"key": .string("Control_L+a Super_R+b A question Delete BackSpace"),
|
||||
]))
|
||||
} catch {
|
||||
let windows = (CGWindowListCopyWindowInfo(.optionAll, kCGNullWindowID) as? [[String: Any]] ?? [])
|
||||
.filter { $0[kCGWindowOwnerPID as String] as? Int == Int(pid) }
|
||||
.map { ["id": $0[kCGWindowNumber as String] ?? "nil", "bounds": $0[kCGWindowBounds as String] ?? "nil", "onScreen": $0[kCGWindowIsOnscreen as String] ?? "nil", "layer": $0[kCGWindowLayer as String] ?? "nil"] }
|
||||
let frame = AXTree.record(pid: pid, index: canvasHandle.index)?.frameGlobal
|
||||
let receipt = (try? String(contentsOf: gestures, encoding: .utf8)) ?? "no receipt"
|
||||
throw CUError((error as? CUError)?.code ?? "fixture_action_failed",
|
||||
"\(error.localizedDescription) Fixture phase=\(phase), pid=\(pid), terminated=\(process.isTerminated), canvas=\(String(describing: frame)), windows=\(windows), receipt=\(receipt)")
|
||||
}
|
||||
try await waitUntil(description: "six received macro keys") {
|
||||
guard let data = try? Data(contentsOf: gestures),
|
||||
let receipt = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
@@ -442,6 +454,14 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
backing: .buffered,
|
||||
defer: false
|
||||
)
|
||||
if regularActivation {
|
||||
// This is a utility receiver, not a document in the user's active
|
||||
// app set. A regular app's window can otherwise be reduced to a
|
||||
// WindowServer preview while AX still reports its full-size frame.
|
||||
// Keep it eligible to join the current set without changing any
|
||||
// system preference, window level, or production input safeguard.
|
||||
window.collectionBehavior = [.canJoinAllApplications]
|
||||
}
|
||||
window.title = title
|
||||
if mismatchedWindowTitle {
|
||||
// Chrome exposes a decorated AX title while WindowServer uses the
|
||||
|
||||
@@ -1,105 +0,0 @@
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class AppTargetPolicyTests: XCTestCase {
|
||||
func testOfficialTerminalAndSecurityBundlesAreDenied() {
|
||||
let denied = [
|
||||
"com.apple.Terminal",
|
||||
"com.googlecode.iterm2",
|
||||
"com.raphaelamorim.rio",
|
||||
"dev.commandline.waveterm",
|
||||
"com.apple.SecurityAgent",
|
||||
"com.apple.LocalAuthenticationRemoteService",
|
||||
"com.apple.UserNotificationCenter",
|
||||
"com.openai.codex.beta",
|
||||
"com.openai.chat.mac-debug",
|
||||
]
|
||||
|
||||
for bundleID in denied {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .deny, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testNativeBrowserBundlesAreAllowed() {
|
||||
for bundleID in [
|
||||
"com.google.Chrome", "com.google.Chrome.canary", "com.apple.Safari",
|
||||
"org.mozilla.firefox", "com.microsoft.edgemac", "com.brave.Browser",
|
||||
] {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testOtherAppCategoriesAreNotImplicitlyForbidden() {
|
||||
let allowed = [
|
||||
"com.webull.desktop.v1",
|
||||
"com.binance.BinanceDesktop",
|
||||
"com.electron.exodus",
|
||||
"com.ledger.live",
|
||||
"io.trezor.TrezorSuite",
|
||||
]
|
||||
|
||||
for bundleID in allowed {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testMediaAndDevelopmentAppsAreNotImplicitlyForbidden() {
|
||||
let allowed = [
|
||||
"com.spotify.client",
|
||||
"com.apple.Music",
|
||||
"com.amazon.aiv.AIVApp",
|
||||
"tv.plex.desktop",
|
||||
"com.amazon.Kindle",
|
||||
"com.microsoft.VSCode",
|
||||
"com.apple.shortcuts",
|
||||
"com.apple.dt.Xcode",
|
||||
]
|
||||
|
||||
for bundleID in allowed {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testForbiddenPolicyUsesExactIdentityNotNameSubstringOrPrefix() {
|
||||
for id in ["com.apple.Terminal.preview", "com.openai.codex.userapp", "org.example.Terminal", "com.apple.securityagent"] {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: id), .allow, id)
|
||||
}
|
||||
}
|
||||
|
||||
func testNormalProductivityBundlesAreAllowed() {
|
||||
let allowed = [
|
||||
"com.apple.calculator",
|
||||
"com.apple.TextEdit",
|
||||
"com.apple.finder",
|
||||
]
|
||||
|
||||
for bundleID in allowed {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testIntrinsicHostAndHelperBundlesAreAlwaysDenied() {
|
||||
let expected: Set<String> = [
|
||||
"com.claude-code-haha.desktop",
|
||||
"dev.cchaha.cu-helper",
|
||||
]
|
||||
|
||||
XCTAssertEqual(AppTargetPolicy.intrinsicDeniedBundleIDs, expected)
|
||||
for bundleID in expected {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .deny, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testDeniedBundleUnionCountAndSetDuplicateHandlingAreLocked() {
|
||||
XCTAssertEqual(AppTargetPolicy.deniedBundleIDs.count, 24)
|
||||
XCTAssertTrue(
|
||||
AppTargetPolicy.deniedBundleIDs
|
||||
.isDisjoint(with: AppTargetPolicy.intrinsicDeniedBundleIDs)
|
||||
)
|
||||
|
||||
var copy = AppTargetPolicy.deniedBundleIDs
|
||||
let duplicate = copy.insert("com.apple.Terminal")
|
||||
XCTAssertFalse(duplicate.inserted)
|
||||
XCTAssertEqual(copy.count, 24)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import CoreGraphics
|
||||
import XCTest
|
||||
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class CoordinateClickRoutingTests: XCTestCase {
|
||||
private enum Element { case canvasWindow, closeButton, contentButton }
|
||||
|
||||
@MainActor
|
||||
func testCanvasWindowHitUsesExactCoordinateInsteadOfPressingItsCloseButton() async throws {
|
||||
let point = CGPoint(x: 1020, y: 710)
|
||||
var pressed: [Element] = []
|
||||
var clicked: [CGPoint] = []
|
||||
var settled = false
|
||||
let route = try await CoordinateClickRouting.click(
|
||||
point: point, preferAccessibility: true,
|
||||
hitTest: { _ in Element.canvasWindow },
|
||||
press: { element in
|
||||
pressed.append(element)
|
||||
return element == .closeButton ? "press" : nil
|
||||
},
|
||||
settle: { settled = true },
|
||||
syntheticClick: { clicked.append($0) }
|
||||
)
|
||||
XCTAssertEqual(pressed, [.canvasWindow], "a canvas click must never press the window's close control")
|
||||
XCTAssertEqual(clicked, [point])
|
||||
XCTAssertFalse(settled)
|
||||
XCTAssertEqual(route, "synthetic:point")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testExactActionableHitKeepsAccessibilityPath() async throws {
|
||||
var pressed: [Element] = []
|
||||
var settled = false
|
||||
let route = try await CoordinateClickRouting.click(
|
||||
point: CGPoint(x: 50, y: 60), preferAccessibility: true,
|
||||
hitTest: { _ in Element.contentButton },
|
||||
press: { pressed.append($0); return "press" },
|
||||
settle: { settled = true },
|
||||
syntheticClick: { _ in XCTFail("exact actionable hit needs no synthetic input") }
|
||||
)
|
||||
XCTAssertEqual(pressed, [.contentButton])
|
||||
XCTAssertTrue(settled)
|
||||
XCTAssertEqual(route, "ax:point:press")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testNonLeftClickSkipsAccessibilityAndPropagatesSyntheticFailure() async {
|
||||
enum Failure: Error { case refused }
|
||||
do {
|
||||
_ = try await CoordinateClickRouting.click(
|
||||
point: .zero, preferAccessibility: false,
|
||||
hitTest: { _ -> Element? in XCTFail("non-left button must skip AX"); return nil },
|
||||
press: { _ in XCTFail("must not press"); return nil },
|
||||
settle: { XCTFail("must not settle") },
|
||||
syntheticClick: { _ in throw Failure.refused }
|
||||
)
|
||||
XCTFail("synthetic target guards must remain authoritative")
|
||||
} catch {
|
||||
XCTAssertTrue(error is Failure)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
import XCTest
|
||||
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
@MainActor
|
||||
final class FocusedElementRoutingTests: XCTestCase {
|
||||
func testBackgroundTargetNeverContactsUnrelatedSystemFocusServer() {
|
||||
var globalQueries = 0
|
||||
let result = FocusedElementRouting.select(
|
||||
targetPID: 42, frontmostPID: 99,
|
||||
systemFocusedPID: { globalQueries += 1; return 99 },
|
||||
systemFocusedElement: { globalQueries += 1; return "other-app" },
|
||||
targetFocusedElement: { "target" }
|
||||
)
|
||||
XCTAssertEqual(result, "target")
|
||||
XCTAssertEqual(globalQueries, 0, "a background snapshot must not wait for an unrelated AX server")
|
||||
}
|
||||
|
||||
func testForegroundTargetKeepsValidatedSystemFocus() {
|
||||
let result = FocusedElementRouting.select(
|
||||
targetPID: 42, frontmostPID: 42,
|
||||
systemFocusedPID: { 42 },
|
||||
systemFocusedElement: { "target-system-focus" },
|
||||
targetFocusedElement: { XCTFail("valid system focus should retain precedence"); return "target-fallback" }
|
||||
)
|
||||
XCTAssertEqual(result, "target-system-focus")
|
||||
}
|
||||
|
||||
func testForegroundHintDoesNotAuthorizeAnotherProcessAfterFocusChanges() {
|
||||
let result = FocusedElementRouting.select(
|
||||
targetPID: 42, frontmostPID: 42,
|
||||
systemFocusedPID: { 99 },
|
||||
systemFocusedElement: { XCTFail("must retain system AX PID validation"); return "other-app" },
|
||||
targetFocusedElement: { "target" }
|
||||
)
|
||||
XCTAssertEqual(result, "target")
|
||||
}
|
||||
|
||||
func testUnknownFrontmostStillUsesSystemAXWithPIDValidation() {
|
||||
for focusedPID: Int32? in [42, 99, nil] {
|
||||
var queries = 0
|
||||
let result = FocusedElementRouting.select(
|
||||
targetPID: 42, frontmostPID: nil,
|
||||
systemFocusedPID: { queries += 1; return focusedPID },
|
||||
systemFocusedElement: { "system" },
|
||||
targetFocusedElement: { "target" }
|
||||
)
|
||||
XCTAssertEqual(queries, 1)
|
||||
XCTAssertEqual(result, focusedPID == 42 ? "system" : "target")
|
||||
}
|
||||
}
|
||||
|
||||
func testMissingSystemElementFallsBackToTargetAndMissingTargetStaysNil() {
|
||||
let result: String? = FocusedElementRouting.select(
|
||||
targetPID: 42, frontmostPID: 42,
|
||||
systemFocusedPID: { 42 },
|
||||
systemFocusedElement: { nil },
|
||||
targetFocusedElement: { "target" }
|
||||
)
|
||||
XCTAssertEqual(result, "target")
|
||||
let absent: String? = FocusedElementRouting.select(
|
||||
targetPID: 42, frontmostPID: 99,
|
||||
systemFocusedPID: { XCTFail("must not contact another app"); return 99 },
|
||||
systemFocusedElement: { "other-app" },
|
||||
targetFocusedElement: { nil }
|
||||
)
|
||||
XCTAssertNil(absent)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
import XCTest
|
||||
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class KeyboardCommandSequenceTests: XCTestCase {
|
||||
@MainActor
|
||||
func testMacroWaitsForEachCompleteSingleChordBoundaryInOrder() async throws {
|
||||
let chords = try KeyboardCommandSequence.prepare("a b c d", systemKeyCombos: false)
|
||||
var history: [String] = []
|
||||
var active = false
|
||||
try await KeyboardCommandSequence.run(chords: chords) { batch in
|
||||
XCTAssertEqual(batch.count, 1, "a macro must not become one rapid native burst")
|
||||
XCTAssertFalse(active)
|
||||
active = true
|
||||
let name = batch.map(\.semanticKey).joined(separator: ",")
|
||||
history.append("begin:\(name)")
|
||||
await Task.yield()
|
||||
history.append("end:\(name)")
|
||||
active = false
|
||||
}
|
||||
XCTAssertEqual(history, ["begin:a", "end:a", "begin:b", "end:b", "begin:c", "end:c", "begin:d", "end:d"])
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testUnknownOrForbiddenSuffixRejectsBeforeAnyInput() async {
|
||||
for (key, expectedCode) in [("a DefinitelyNotARealKey", "unknown_key"), ("a cmd+q", "grant_flag_required")] {
|
||||
var calls = 0
|
||||
do {
|
||||
let chords = try KeyboardCommandSequence.prepare(key, systemKeyCombos: false)
|
||||
try await KeyboardCommandSequence.run(chords: chords) { _ in calls += 1 }
|
||||
XCTFail("the complete macro must be preflighted")
|
||||
} catch let error as CUError {
|
||||
XCTAssertEqual(error.code, expectedCode)
|
||||
} catch { XCTFail("unexpected error: \(error)") }
|
||||
XCTAssertEqual(calls, 0)
|
||||
}
|
||||
}
|
||||
|
||||
func testChordLimitAppliesBeforeExecutionAndPreservesSpacedShortcutSyntax() throws {
|
||||
XCTAssertEqual(try KeyboardCommandSequence.prepare(String(repeating: "Return ", count: 128), systemKeyCombos: false).count, 128)
|
||||
XCTAssertThrowsError(try KeyboardCommandSequence.prepare(String(repeating: "Return ", count: 129), systemKeyCombos: false)) { error in
|
||||
XCTAssertEqual((error as? CUError)?.code, "bad_payload")
|
||||
}
|
||||
let chords = try KeyboardCommandSequence.prepare("cmd + a\n shift + Return", systemKeyCombos: false)
|
||||
XCTAssertEqual(chords.count, 2)
|
||||
XCTAssertEqual(chords.map(\.flags), [.maskCommand, .maskShift])
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testFocusFailureStopsRemainingChordsAndReportsCompletedPrefix() async throws {
|
||||
let chords = try KeyboardCommandSequence.prepare("a b c d", systemKeyCombos: false)
|
||||
var attempted: [String] = []
|
||||
do {
|
||||
try await KeyboardCommandSequence.run(chords: chords) { batch in
|
||||
attempted.append(contentsOf: batch.map(\.semanticKey))
|
||||
if attempted.count == 2 { throw CUError("focus_changed", "test focus changed") }
|
||||
}
|
||||
XCTFail("must stop after focus loss")
|
||||
} catch let error as CUError {
|
||||
XCTAssertEqual(error.code, "focus_changed")
|
||||
XCTAssertTrue(error.message.contains("1 of 4"))
|
||||
XCTAssertTrue(error.message.contains("may already have been delivered"))
|
||||
XCTAssertTrue(error.message.contains("do not replay"))
|
||||
}
|
||||
XCTAssertEqual(attempted, ["a", "b"])
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testCancellationBetweenChordsStopsBeforeNextBoundary() async throws {
|
||||
let chords = try KeyboardCommandSequence.prepare("a b c d", systemKeyCombos: false)
|
||||
var attempted: [String] = []
|
||||
let task = Task { @MainActor in
|
||||
do {
|
||||
try await KeyboardCommandSequence.run(chords: chords) { batch in
|
||||
attempted.append(contentsOf: batch.map(\.semanticKey))
|
||||
withUnsafeCurrentTask { $0?.cancel() }
|
||||
}
|
||||
XCTFail("cancellation must prevent the next chord")
|
||||
} catch let error as CUError {
|
||||
XCTAssertEqual(error.code, "cancelled")
|
||||
XCTAssertTrue(error.message.contains("1 of 4"))
|
||||
} catch { XCTFail("unexpected error: \(error)") }
|
||||
}
|
||||
await task.value
|
||||
XCTAssertEqual(attempted, ["a"])
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testSingleChordKeepsUnderlyingErrorWithoutMacroDecoration() async throws {
|
||||
let chords = try KeyboardCommandSequence.prepare("Return", systemKeyCombos: false)
|
||||
do {
|
||||
try await KeyboardCommandSequence.run(chords: chords) { _ in throw CUError("stale_process", "original error") }
|
||||
XCTFail("must propagate target refusal")
|
||||
} catch let error as CUError {
|
||||
XCTAssertEqual(error.code, "stale_process")
|
||||
XCTAssertEqual(error.message, "original error")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
import ImageIO
|
||||
import XCTest
|
||||
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
@MainActor
|
||||
final class ModelWindowShotTests: XCTestCase {
|
||||
func testLargeLandscapeAndPortraitImagesFitBudgetAndRemainPNG() throws {
|
||||
for (width, height, expectedWidth, expectedHeight) in [
|
||||
(2304, 1506, 1175, 768),
|
||||
(1506, 2304, 768, 1175),
|
||||
(2400, 600, 2048, 512),
|
||||
] {
|
||||
let input = try makeShot(width: width, height: height)
|
||||
let shot = try XCTUnwrap(Capture.boundedModelWindowShot(input))
|
||||
XCTAssertEqual(shot.width, expectedWidth)
|
||||
XCTAssertEqual(shot.height, expectedHeight)
|
||||
let data = try XCTUnwrap(Data(base64Encoded: shot.base64))
|
||||
XCTAssertEqual(Array(data.prefix(8)), [137, 80, 78, 71, 13, 10, 26, 10])
|
||||
let source = try XCTUnwrap(CGImageSourceCreateWithData(data as CFData, nil))
|
||||
let image = try XCTUnwrap(CGImageSourceCreateImageAtIndex(source, 0, nil))
|
||||
XCTAssertEqual(image.width, shot.width)
|
||||
XCTAssertEqual(image.height, shot.height)
|
||||
XCTAssertEqual(shot.originX, input.originX)
|
||||
XCTAssertEqual(shot.originY, input.originY)
|
||||
XCTAssertEqual(shot.pointWidth, input.pointWidth)
|
||||
XCTAssertEqual(shot.pointHeight, input.pointHeight)
|
||||
XCTAssertEqual(shot.windowID, input.windowID)
|
||||
XCTAssertEqual(shot.source, input.source)
|
||||
}
|
||||
}
|
||||
|
||||
func testSmallImageIsNotEnlargedOrReencoded() throws {
|
||||
let input = try makeShot(width: 640, height: 400)
|
||||
let shot = try XCTUnwrap(Capture.boundedModelWindowShot(input))
|
||||
XCTAssertEqual(shot.width, 640)
|
||||
XCTAssertEqual(shot.height, 400)
|
||||
XCTAssertEqual(shot.base64, input.base64)
|
||||
}
|
||||
|
||||
func testBoundedPixelsMapBackToOriginalWindowPointsAtDifferentBackingScales() throws {
|
||||
let identity = AXTreeProcessIdentity(bundleID: "test.window", executablePath: "/fixture/window", launchTime: 1)
|
||||
for pixelSize in [(2304, 1506), (1152, 753)] {
|
||||
let input = try makeShot(width: pixelSize.0, height: pixelSize.1, pointWidth: 1152, pointHeight: 753, pixelsPerPoint: Double(pixelSize.0) / 1152)
|
||||
let shot = try XCTUnwrap(Capture.boundedModelWindowShot(input))
|
||||
CommandRouter.clearShotTransformsForTesting()
|
||||
defer { CommandRouter.clearShotTransformsForTesting() }
|
||||
CommandRouter.recordShotTransform(
|
||||
pid: 77, originX: shot.originX, originY: shot.originY,
|
||||
pointWidth: shot.pointWidth, pointHeight: shot.pointHeight,
|
||||
imageWidth: shot.width, imageHeight: shot.height,
|
||||
processIdentity: identity, windowID: shot.windowID, pixelsPerPoint: shot.pixelsPerPoint
|
||||
)
|
||||
let point = try CommandRouter.toGlobalPoint(
|
||||
x: Double(shot.width) * 0.75, y: Double(shot.height) * 0.25,
|
||||
pid: 77, currentProcessIdentity: identity, currentWindowID: shot.windowID
|
||||
)
|
||||
XCTAssertEqual(point.x, input.originX + input.pointWidth * 0.75, accuracy: 0.000001)
|
||||
XCTAssertEqual(point.y, input.originY + input.pointHeight * 0.25, accuracy: 0.000001)
|
||||
XCTAssertThrowsError(try CommandRouter.toGlobalPoint(
|
||||
x: Double(shot.width), y: 0, pid: 77,
|
||||
currentProcessIdentity: identity, currentWindowID: shot.windowID
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
func testInvalidLargeImageCannotLeakAnUnboundedFallback() {
|
||||
let shot = WindowShot(base64: "invalid", width: 2304, height: 1506,
|
||||
originX: 0, originY: 0, pointWidth: 1152, pointHeight: 753,
|
||||
windowID: 17, source: .screenshotManager)
|
||||
XCTAssertNil(Capture.boundedModelWindowShot(shot))
|
||||
}
|
||||
|
||||
func testFinalModelJPEGReallyDecodesAtTheSameSizeAndLeavesPNGEvidenceUntouched() throws {
|
||||
let raw = try makeShot(width: 2304, height: 1506)
|
||||
let bounded = try XCTUnwrap(Capture.boundedModelWindowShot(raw))
|
||||
let evidence = bounded.base64
|
||||
let model = Capture.modelWindowImage(bounded)
|
||||
XCTAssertEqual(model.mimeType, "image/jpeg")
|
||||
let data = try XCTUnwrap(Data(base64Encoded: model.base64))
|
||||
let source = try XCTUnwrap(CGImageSourceCreateWithData(data as CFData, nil))
|
||||
XCTAssertEqual(CGImageSourceGetType(source) as String?, "public.jpeg")
|
||||
let image = try XCTUnwrap(CGImageSourceCreateImageAtIndex(source, 0, nil))
|
||||
XCTAssertEqual(image.width, bounded.width)
|
||||
XCTAssertEqual(image.height, bounded.height)
|
||||
XCTAssertEqual(bounded.base64, evidence)
|
||||
XCTAssertEqual(Array(try XCTUnwrap(Data(base64Encoded: evidence)).prefix(8)), [137, 80, 78, 71, 13, 10, 26, 10])
|
||||
}
|
||||
|
||||
func testJPEGQualityBoundsAndEncoderFailurePreserveLosslessFallback() throws {
|
||||
let shot = try makeShot(width: 96, height: 64)
|
||||
var qualities: [Double] = []
|
||||
for quality in [0.0, 0.9, 1.0] {
|
||||
let image = Capture.modelWindowImage(shot, quality: quality) { _, q in
|
||||
qualities.append(q)
|
||||
return nil
|
||||
}
|
||||
XCTAssertEqual(image.mimeType, "image/png")
|
||||
XCTAssertEqual(image.base64, shot.base64)
|
||||
}
|
||||
XCTAssertEqual(qualities, [0, 0.9, 1])
|
||||
for quality in [-0.1, 1.1, Double.nan, Double.infinity] {
|
||||
let image = Capture.modelWindowImage(shot, quality: quality) { _, _ in
|
||||
XCTFail("invalid quality must never reach the encoder")
|
||||
return nil
|
||||
}
|
||||
XCTAssertEqual(image.mimeType, "image/png")
|
||||
XCTAssertEqual(image.base64, shot.base64)
|
||||
}
|
||||
let empty = Capture.modelWindowImage(shot) { _, _ in "" }
|
||||
XCTAssertEqual(empty.mimeType, "image/png")
|
||||
let image = Capture.modelWindowImage(shot)
|
||||
let data = try XCTUnwrap(Data(base64Encoded: image.base64))
|
||||
let source = try XCTUnwrap(CGImageSourceCreateWithData(data as CFData, nil))
|
||||
let decoded = try XCTUnwrap(CGImageSourceCreateImageAtIndex(source, 0, nil))
|
||||
XCTAssertEqual(decoded.width, 96)
|
||||
XCTAssertEqual(decoded.height, 64)
|
||||
}
|
||||
|
||||
func testAlreadyFittedOfficialImagePreservesPixelsAndUniformScale() throws {
|
||||
let fit = 768.0 / 769.0
|
||||
let input = try makeShot(width: 1397, height: 768, pointWidth: 1398, pointHeight: 769, pixelsPerPoint: fit)
|
||||
let shot = try XCTUnwrap(Capture.boundedModelWindowShot(input))
|
||||
XCTAssertEqual(shot.base64, input.base64)
|
||||
XCTAssertEqual(shot.width, 1397)
|
||||
XCTAssertEqual(shot.height, 768)
|
||||
XCTAssertEqual(shot.pixelsPerPoint, fit)
|
||||
var encodings = 0
|
||||
let model = Capture.modelWindowImage(shot) { image, quality in
|
||||
encodings += 1
|
||||
XCTAssertEqual(image.width, 1397)
|
||||
XCTAssertEqual(image.height, 768)
|
||||
XCTAssertEqual(quality, NativeScreenshotPolicy.jpegQuality)
|
||||
return "jpeg-fixture"
|
||||
}
|
||||
XCTAssertEqual(encodings, 1)
|
||||
XCTAssertEqual(model.mimeType, "image/jpeg")
|
||||
XCTAssertEqual(model.base64, "jpeg-fixture")
|
||||
}
|
||||
|
||||
func testRetinaLosslessShotFitsOnceAndCarriesItsScaledUniformTransform() throws {
|
||||
let raw = try makeShot(width: 2796, height: 1538, pointWidth: 1398, pointHeight: 769, pixelsPerPoint: 2)
|
||||
let shot = try XCTUnwrap(Capture.boundedModelWindowShot(raw))
|
||||
XCTAssertEqual(shot.width, 1397)
|
||||
XCTAssertEqual(shot.height, 768)
|
||||
XCTAssertEqual(try XCTUnwrap(shot.pixelsPerPoint), 768.0 / 769.0, accuracy: 0.000000001)
|
||||
let again = try XCTUnwrap(Capture.boundedModelWindowShot(shot))
|
||||
XCTAssertEqual(again.base64, shot.base64)
|
||||
XCTAssertEqual(again.pixelsPerPoint, shot.pixelsPerPoint)
|
||||
}
|
||||
|
||||
private func makeShot(width: Int, height: Int, pointWidth: Double? = nil, pointHeight: Double? = nil, pixelsPerPoint: Double? = nil) throws -> WindowShot {
|
||||
let context = try XCTUnwrap(CGContext(
|
||||
data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0,
|
||||
space: CGColorSpaceCreateDeviceRGB(),
|
||||
bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue
|
||||
))
|
||||
context.setFillColor(CGColor(red: 0.2, green: 0.4, blue: 0.6, alpha: 1))
|
||||
context.fill(CGRect(x: 0, y: 0, width: width, height: height))
|
||||
let image = try XCTUnwrap(context.makeImage())
|
||||
let encoded = try XCTUnwrap(Capture.pngBase64WithSize(image))
|
||||
return WindowShot(
|
||||
base64: encoded.base64, width: encoded.width, height: encoded.height,
|
||||
originX: -600, originY: 200, pointWidth: pointWidth ?? Double(width), pointHeight: pointHeight ?? Double(height),
|
||||
windowID: 17, source: .streamBackedScreenshot, pixelsPerPoint: pixelsPerPoint
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
@MainActor
|
||||
final class RendererAttributeReuseTests: XCTestCase {
|
||||
func testKnownFingerprintScalarsAndSuccessfulSettableAvoidBackendRepeats() {
|
||||
var reads = 0
|
||||
func read(_ value: String) -> String { reads += 1; return value }
|
||||
let observation = RendererAttributeReuse(
|
||||
title: read("Title"), description: read("Description"), role: read("AXButton")
|
||||
)
|
||||
XCTAssertEqual(observation.title { read("Title") }, "Title")
|
||||
XCTAssertEqual(observation.description { read("Description") }, "Description")
|
||||
XCTAssertEqual(observation.role { read("AXButton") }, "AXButton")
|
||||
for _ in 0..<2 {
|
||||
XCTAssertEqual(observation.settable { reads += 1; return false }, false)
|
||||
}
|
||||
XCTAssertEqual(reads, 4, "three fingerprint reads plus one settable query, instead of eight")
|
||||
}
|
||||
|
||||
func testMissingScalarsRetryAndCacheOnlyLaterSuccess() {
|
||||
let observation = RendererAttributeReuse(title: nil, description: nil, role: nil)
|
||||
var reads = 0
|
||||
for get in [observation.title, observation.description, observation.role] {
|
||||
XCTAssertNil(get { reads += 1; return nil })
|
||||
XCTAssertEqual(get { reads += 1; return "recovered" }, "recovered")
|
||||
XCTAssertEqual(get { reads += 1; return "changed" }, "recovered")
|
||||
}
|
||||
XCTAssertEqual(reads, 6)
|
||||
}
|
||||
|
||||
func testSettableFailureRetriesButSuccessfulFalseDoesNot() {
|
||||
let observation = RendererAttributeReuse(title: nil, description: nil, role: nil)
|
||||
var reads = 0
|
||||
XCTAssertNil(observation.settable { reads += 1; return nil })
|
||||
XCTAssertEqual(observation.settable { reads += 1; return false }, false)
|
||||
XCTAssertEqual(observation.settable { reads += 1; return true }, false)
|
||||
XCTAssertEqual(reads, 2)
|
||||
}
|
||||
|
||||
func testSeparateNodesAndRenderPassesNeverShareObservations() {
|
||||
let first = RendererAttributeReuse(title: "Same", description: nil, role: "AXButton")
|
||||
let duplicate = RendererAttributeReuse(title: "Same", description: nil, role: "AXButton")
|
||||
XCTAssertEqual(first.settable { true }, true)
|
||||
XCTAssertEqual(duplicate.settable { false }, false)
|
||||
let nextPass = RendererAttributeReuse(title: "Changed", description: nil, role: "AXButton")
|
||||
XCTAssertEqual(nextPass.title { "wrong" }, "Changed")
|
||||
XCTAssertEqual(nextPass.settable { false }, false)
|
||||
}
|
||||
}
|
||||
@@ -66,15 +66,48 @@ final class ResolvedTargetAuthorizationTests: XCTestCase {
|
||||
}
|
||||
}
|
||||
|
||||
func testNumericPIDCannotBypassDeniedResolvedBundle() {
|
||||
XCTAssertThrowsError(
|
||||
try ResolvedTargetAuthorization.authorize(
|
||||
pid: 41,
|
||||
identity: terminalIdentity,
|
||||
expectedBundleID: nil
|
||||
func testNumericPIDAllowsTerminalAfterGlobalEnablement() throws {
|
||||
let target = try ResolvedTargetAuthorization.authorize(
|
||||
pid: 41,
|
||||
identity: terminalIdentity,
|
||||
expectedBundleID: nil
|
||||
)
|
||||
|
||||
XCTAssertEqual(target.pid, 41)
|
||||
XCTAssertEqual(target.identity, terminalIdentity)
|
||||
}
|
||||
|
||||
func testEveryAppCategoryAndHostAreAllowedWithProvenProcessIdentity() throws {
|
||||
let bundleIDs = [
|
||||
"com.google.Chrome",
|
||||
"com.apple.Safari",
|
||||
"com.apple.Terminal",
|
||||
"com.microsoft.VSCode",
|
||||
"com.apple.shortcuts",
|
||||
"com.webull.desktop.v1",
|
||||
"com.binance.BinanceDesktop",
|
||||
"com.ledger.live",
|
||||
"com.spotify.client",
|
||||
"com.apple.Music",
|
||||
"com.amazon.Kindle",
|
||||
"com.claude-code-haha.desktop",
|
||||
"dev.cchaha.cu-helper",
|
||||
"com.example.custom-host",
|
||||
"com.example.new-app",
|
||||
]
|
||||
for bundleID in bundleIDs {
|
||||
let identity = AXTreeProcessIdentity(
|
||||
bundleID: bundleID,
|
||||
executablePath: "/Applications/Fixture.app/Contents/MacOS/Fixture",
|
||||
launchTime: 100
|
||||
)
|
||||
) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "app_denied")
|
||||
let target = try ResolvedTargetAuthorization.authorize(
|
||||
pid: 41,
|
||||
identity: identity,
|
||||
expectedBundleID: bundleID
|
||||
)
|
||||
XCTAssertEqual(target.pid, 41, bundleID)
|
||||
XCTAssertEqual(target.identity, identity, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -96,18 +129,16 @@ final class ResolvedTargetAuthorizationTests: XCTestCase {
|
||||
let resolved = try XCTUnwrap(
|
||||
AppTargetResolver.resolve(selector: selector, candidates: [terminal])
|
||||
)
|
||||
XCTAssertThrowsError(
|
||||
try ResolvedTargetAuthorization.authorize(
|
||||
resolved: resolved,
|
||||
currentIdentity: terminalIdentity
|
||||
)
|
||||
) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "app_denied")
|
||||
}
|
||||
let target = try ResolvedTargetAuthorization.authorize(
|
||||
resolved: resolved,
|
||||
currentIdentity: terminalIdentity
|
||||
)
|
||||
XCTAssertEqual(target.pid, 41)
|
||||
XCTAssertEqual(target.identity, terminalIdentity)
|
||||
}
|
||||
}
|
||||
|
||||
func testWorktreePathResolutionStillReachesIntrinsicSelfControlDenial() throws {
|
||||
func testWorktreeHostPathResolutionAuthorizesExactProcess() throws {
|
||||
let installed = AppTargetCandidate(
|
||||
pid: 100,
|
||||
bundleIdentifier: "com.claude-code-haha.desktop",
|
||||
@@ -133,21 +164,15 @@ final class ResolvedTargetAuthorizationTests: XCTestCase {
|
||||
)
|
||||
|
||||
XCTAssertEqual(resolved.pid, worktree.pid)
|
||||
XCTAssertThrowsError(
|
||||
try ResolvedTargetAuthorization.authorize(
|
||||
resolved: resolved,
|
||||
currentIdentity: identity
|
||||
)
|
||||
) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "app_denied")
|
||||
XCTAssertEqual(
|
||||
($0 as? CUError)?.message,
|
||||
"Computer Use is not allowed to use the app 'com.claude-code-haha.desktop' for safety reasons."
|
||||
)
|
||||
}
|
||||
let target = try ResolvedTargetAuthorization.authorize(
|
||||
resolved: resolved,
|
||||
currentIdentity: identity
|
||||
)
|
||||
XCTAssertEqual(target.pid, worktree.pid)
|
||||
XCTAssertEqual(target.identity, identity)
|
||||
}
|
||||
|
||||
func testOmittedFrontmostAndLaunchedTargetsUseSameActualBundlePolicy() {
|
||||
func testOmittedFrontmostAndLaunchedTargetsUseSameActualBundlePolicy() throws {
|
||||
// Both paths ultimately produce this same resolved target shape. The
|
||||
// authorizer intentionally has no selector-specific bypass.
|
||||
let resolved = ResolvedAppTarget(
|
||||
@@ -157,14 +182,12 @@ final class ResolvedTargetAuthorizationTests: XCTestCase {
|
||||
)
|
||||
|
||||
for _ in ["omitted-frontmost", "launched-get-app-state"] {
|
||||
XCTAssertThrowsError(
|
||||
try ResolvedTargetAuthorization.authorize(
|
||||
resolved: resolved,
|
||||
currentIdentity: terminalIdentity
|
||||
)
|
||||
) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "app_denied")
|
||||
}
|
||||
let target = try ResolvedTargetAuthorization.authorize(
|
||||
resolved: resolved,
|
||||
currentIdentity: terminalIdentity
|
||||
)
|
||||
XCTAssertEqual(target.pid, 41)
|
||||
XCTAssertEqual(target.identity, terminalIdentity)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -707,7 +707,7 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
))
|
||||
}
|
||||
|
||||
func testCopiedBGRAFrameEncodesAsAStreamJPEGWithTheSameGeometry() throws {
|
||||
func testCopiedBGRAFramePreservesLosslessEvidenceAndPresentsJPEGAtTheSameGeometry() throws {
|
||||
let target = makeTarget(
|
||||
windowID: 72,
|
||||
pixelWidth: 1,
|
||||
@@ -725,9 +725,14 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
)
|
||||
|
||||
let shot = try XCTUnwrap(Capture.windowShot(from: frame, target: target))
|
||||
let jpeg = try XCTUnwrap(Data(base64Encoded: shot.base64))
|
||||
|
||||
let evidence = try XCTUnwrap(Data(base64Encoded: shot.base64))
|
||||
XCTAssertEqual(Array(evidence.prefix(8)), [137, 80, 78, 71, 13, 10, 26, 10])
|
||||
XCTAssertEqual(shot.mimeType, "image/png")
|
||||
let model = Capture.modelWindowImage(shot)
|
||||
let jpeg = try XCTUnwrap(Data(base64Encoded: model.base64))
|
||||
XCTAssertEqual(Array(jpeg.prefix(2)), [255, 216])
|
||||
XCTAssertEqual(model.mimeType, "image/jpeg")
|
||||
XCTAssertEqual(shot.base64, evidence.base64EncodedString())
|
||||
XCTAssertEqual(shot.width, 1)
|
||||
XCTAssertEqual(shot.height, 1)
|
||||
XCTAssertEqual(shot.originX, 300)
|
||||
|
||||
@@ -115,25 +115,49 @@ describe('macOS installed app enumeration', () => {
|
||||
])
|
||||
})
|
||||
|
||||
it('filters the built-in host, helper, and an additional configured host', async () => {
|
||||
it('includes the built-in host, helper, and an additional configured host like any other app', async () => {
|
||||
const metadata = new Map([
|
||||
['Desktop.app', { bundleId: 'com.claude-code-haha.desktop', displayName: 'Claude Code Haha' }],
|
||||
['Helper.app', { bundleId: 'dev.cchaha.cu-helper', displayName: 'Computer Use Helper' }],
|
||||
['Custom.app', { bundleId: 'com.example.custom-host', displayName: 'Custom Host' }],
|
||||
['Notes.app', { bundleId: 'com.example.notes', displayName: 'Notes' }],
|
||||
])
|
||||
const apps = await listInstalledMacApps({
|
||||
roots: ['/Applications'],
|
||||
hostBundleId: 'com.example.custom-host',
|
||||
readDirectory: async () => [...metadata.keys()].map(name => entry(name)),
|
||||
canonicalize: async candidate => candidate,
|
||||
readMetadata: async appPath => metadata.get(appPath.split('/').at(-1) ?? '') ?? null,
|
||||
})
|
||||
const previousHost = process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID
|
||||
process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID = 'com.example.custom-host'
|
||||
let apps: Awaited<ReturnType<typeof listInstalledMacApps>>
|
||||
try {
|
||||
apps = await listInstalledMacApps({
|
||||
roots: ['/Applications'],
|
||||
readDirectory: async () => [...metadata.keys()].map(name => entry(name)),
|
||||
canonicalize: async candidate => candidate,
|
||||
readMetadata: async appPath => metadata.get(appPath.split('/').at(-1) ?? '') ?? null,
|
||||
})
|
||||
} finally {
|
||||
if (previousHost === undefined) delete process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID
|
||||
else process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID = previousHost
|
||||
}
|
||||
|
||||
expect(apps).toEqual([{
|
||||
bundleId: 'com.example.notes',
|
||||
displayName: 'Notes',
|
||||
path: '/Applications/Notes.app',
|
||||
}])
|
||||
expect(apps).toEqual([
|
||||
{
|
||||
bundleId: 'com.claude-code-haha.desktop',
|
||||
displayName: 'Claude Code Haha',
|
||||
path: '/Applications/Desktop.app',
|
||||
},
|
||||
{
|
||||
bundleId: 'dev.cchaha.cu-helper',
|
||||
displayName: 'Computer Use Helper',
|
||||
path: '/Applications/Helper.app',
|
||||
},
|
||||
{
|
||||
bundleId: 'com.example.custom-host',
|
||||
displayName: 'Custom Host',
|
||||
path: '/Applications/Custom.app',
|
||||
},
|
||||
{
|
||||
bundleId: 'com.example.notes',
|
||||
displayName: 'Notes',
|
||||
path: '/Applications/Notes.app',
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -2,7 +2,6 @@ import type { Dirent } from 'node:fs'
|
||||
import { readdir, realpath } from 'node:fs/promises'
|
||||
import { homedir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { isIntrinsicAppDenied } from '../../vendor/computer-use-mcp/deniedApps.js'
|
||||
|
||||
export type InstalledMacApp = {
|
||||
bundleId: string
|
||||
@@ -21,7 +20,6 @@ type InstalledAppDependencies = {
|
||||
) => Promise<{ bundleId: string; displayName: string } | null>
|
||||
maxDepth?: number
|
||||
entryLimit?: number
|
||||
hostBundleId?: string
|
||||
}
|
||||
|
||||
const STANDARD_APPLICATION_ROOTS = [
|
||||
@@ -89,8 +87,6 @@ export async function listInstalledMacApps(
|
||||
const readMetadata = deps.readMetadata ?? readMetadataWithPlutil
|
||||
const maxDepth = deps.maxDepth ?? DEFAULT_MAX_DEPTH
|
||||
const entryLimit = deps.entryLimit ?? DEFAULT_ENTRY_LIMIT
|
||||
const hostBundleId = deps.hostBundleId
|
||||
?? process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID
|
||||
const byBundleId = new Map<string, InstalledMacApp>()
|
||||
|
||||
for (const configuredRoot of roots) {
|
||||
@@ -144,7 +140,6 @@ export async function listInstalledMacApps(
|
||||
|
||||
const metadata = await readMetadata(canonicalCandidate)
|
||||
if (!metadata?.bundleId || !metadata.displayName) continue
|
||||
if (isIntrinsicAppDenied(metadata.bundleId, hostBundleId)) continue
|
||||
if (byBundleId.has(metadata.bundleId)) continue
|
||||
byBundleId.set(metadata.bundleId, {
|
||||
bundleId: metadata.bundleId,
|
||||
|
||||
@@ -182,3 +182,21 @@ describe('computer-use observation batching', () => {
|
||||
expect(getComputerUsePrompt('win32')).not.toContain('cua.getApp')
|
||||
})
|
||||
})
|
||||
|
||||
test('macOS advertises bounded sequence and real key macros without changing Windows tools', () => {
|
||||
expect(getComputerUseToolAllowlist('darwin')).toContain('mcp__computer-use__js')
|
||||
expect(getComputerUseToolAllowlist('win32')).not.toContain('mcp__computer-use__sequence')
|
||||
const prompt = getComputerUsePrompt('darwin')
|
||||
expect(prompt).toContain('s x 1 period 3 5 Return')
|
||||
expect(prompt).toContain('account for actions that already ran')
|
||||
expect(prompt).toContain('never replay the whole batch')
|
||||
})
|
||||
|
||||
test('uses the sequence observation without an extra model round trip', () => {
|
||||
// Live Blender trial: individual click -> receipt -> model -> get_app_state
|
||||
// added a full provider round trip just to observe a single known action.
|
||||
const prompt = getComputerUsePrompt('darwin')
|
||||
expect(prompt).toContain('then observe at the next')
|
||||
expect(prompt).toContain('Do not force one model round trip per click')
|
||||
expect(prompt).toContain('Observe through the JS')
|
||||
})
|
||||
|
||||
@@ -48,6 +48,16 @@ Do not add a fixed sleep before an observation. The observation path waits
|
||||
for UI changes when needed. Read its result before deciding whether more context
|
||||
is necessary.
|
||||
|
||||
## Blender keyboard input
|
||||
|
||||
For Blender, ordinary \`app.typeText\` can leave text unchanged. Use
|
||||
\`await app.pressKey("s x 1 period 3 5 Return")\` for a known numeric transform,
|
||||
or \`await app.pressKey("a d d space c u b e")\` in a known search field.
|
||||
These send actual key presses. Use short macros and observe through the JS App
|
||||
after opening an unfamiliar dialog; never replay the whole batch after failure.
|
||||
Use \`space\` for a literal space, \`period\` for a decimal point, and
|
||||
\`minus\` for a negative sign.
|
||||
|
||||
## Naming the app
|
||||
|
||||
Pass the app name straight to \`cua.getApp\` — display name, bundle identifier,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test'
|
||||
import { afterEach, describe, expect, spyOn, test } from 'bun:test'
|
||||
import { EventEmitter } from 'node:events'
|
||||
import { lstatSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
@@ -173,6 +173,33 @@ describe('cu-helper daemon system commands', () => {
|
||||
})
|
||||
|
||||
describe('cu-helper daemon failure classification', () => {
|
||||
test('only multi-chord keyboard requests extend both native deadline and response timer to a bounded budget', async () => {
|
||||
const socket = new FakeSocket()
|
||||
__setDaemonSocketForTests(socket as never)
|
||||
const timerSpy = spyOn(globalThis, 'setTimeout')
|
||||
try {
|
||||
const cases = [
|
||||
['press_key', { key: 'a b c d' }, 60_000],
|
||||
['press_key', { key: 'ctrl + a' }, 20_000],
|
||||
['press_key', { key: 'Return' }, 20_000],
|
||||
['get_app_state', { key: 'a b' }, 20_000],
|
||||
] as const
|
||||
for (const [index, [command, payload, budget]] of cases.entries()) {
|
||||
const before = Date.now()
|
||||
const request = callDaemon(command, payload)
|
||||
await waitForWriteCount(socket, index + 1)
|
||||
const envelope = JSON.parse(socket.writes[index]!)
|
||||
expect(envelope.deadlineUnixMilliseconds).toBeGreaterThanOrEqual(before + budget)
|
||||
expect(envelope.deadlineUnixMilliseconds).toBeLessThanOrEqual(Date.now() + budget)
|
||||
expect(timerSpy.mock.calls.at(-1)?.[1]).toBe(budget)
|
||||
reply(socket, index, { ok: true, result: true })
|
||||
await expect(request).resolves.toBe(true)
|
||||
}
|
||||
} finally {
|
||||
timerSpy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
test('helper installation/start resolution failure is daemon infrastructure failure', async () => {
|
||||
// A bare executable can satisfy the availability probe but cannot be
|
||||
// launched as the helper .app daemon. The bridge must be allowed to use the
|
||||
|
||||
@@ -658,13 +658,20 @@ function dispatchDaemonCommand<T>(
|
||||
const isTurnScoped = !CONNECTION_SCOPED_COMMANDS.has(command)
|
||||
const turnId = activeTurnId
|
||||
?? (isTurnScoped ? (activeTurnId = randomUUID()) : `connection-${state.generation}`)
|
||||
// Native keyboard macros now complete each chord through its own input
|
||||
// boundary. Keep the native deadline and client timer aligned, without
|
||||
// extending screenshots, clicks, or single-key requests. This counts only
|
||||
// separators for budgeting; native KeyMapping remains the validating parser.
|
||||
const multipleChords = command === 'press_key' && typeof payload.key === 'string'
|
||||
&& payload.key.replace(/\s*\+\s*/g, '+').trim().split(/\s+/).length > 1
|
||||
const timeoutMs = multipleChords ? Math.min(60_000, requestTimeoutMs * 3) : requestTimeoutMs
|
||||
const request = {
|
||||
id,
|
||||
requestId: id,
|
||||
cmd: command,
|
||||
payload,
|
||||
clientApiVersion: CU_HELPER_PROTOCOL_VERSION,
|
||||
deadlineUnixMilliseconds: Date.now() + requestTimeoutMs,
|
||||
deadlineUnixMilliseconds: Date.now() + timeoutMs,
|
||||
sessionId: getSessionId(),
|
||||
turnId,
|
||||
}
|
||||
@@ -680,7 +687,7 @@ function dispatchDaemonCommand<T>(
|
||||
// Retire a daemon that missed its response deadline. Other requests that
|
||||
// were already in flight are also result-unknown, never replayable infra.
|
||||
resetState(`command ${command} timed out`, state.generation)
|
||||
}, requestTimeoutMs)
|
||||
}, timeoutMs)
|
||||
state.pending.set(id, { resolve: resolve as (v: unknown) => void, reject, timer })
|
||||
try {
|
||||
state.socket.write(`${JSON.stringify(request)}\n`)
|
||||
|
||||
@@ -116,7 +116,7 @@ describe('resolveStoredComputerUseConfig', () => {
|
||||
})
|
||||
})
|
||||
|
||||
test('derives least-privilege tiers and filters policy-denied pre-authorizations', () => {
|
||||
test('legacy pre-authorizations no longer restrict app categories after global consent', () => {
|
||||
expect(
|
||||
buildPreAuthorizedAppGrants([
|
||||
{
|
||||
@@ -141,13 +141,13 @@ describe('resolveStoredComputerUseConfig', () => {
|
||||
bundleId: 'com.google.Chrome',
|
||||
displayName: 'Google Chrome',
|
||||
grantedAt: 1234,
|
||||
tier: 'read',
|
||||
tier: 'full',
|
||||
},
|
||||
{
|
||||
bundleId: 'com.apple.Terminal',
|
||||
displayName: 'Terminal',
|
||||
grantedAt: 1234,
|
||||
tier: 'click',
|
||||
tier: 'full',
|
||||
},
|
||||
{
|
||||
bundleId: 'com.apple.Preview',
|
||||
@@ -155,6 +155,12 @@ describe('resolveStoredComputerUseConfig', () => {
|
||||
grantedAt: 1234,
|
||||
tier: 'full',
|
||||
},
|
||||
{
|
||||
bundleId: 'com.spotify.client',
|
||||
displayName: 'Spotify',
|
||||
grantedAt: 1234,
|
||||
tier: 'full',
|
||||
},
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
@@ -94,3 +94,133 @@ test('a host Escape callback aborts its turn outside any dispatch async context'
|
||||
expect(turnController.signal.aborted).toBe(true)
|
||||
expect(nextController.signal.aborted).toBe(false)
|
||||
})
|
||||
|
||||
import { withComputerUseToolContext } from './wrapper.js'
|
||||
|
||||
test('CU async contexts preserve each originating turn cancellation across concurrent awaits', async () => {
|
||||
const first = new AbortController()
|
||||
const second = new AbortController()
|
||||
const shared = buildSessionContext()
|
||||
let resume!: () => void
|
||||
let ready!: () => void
|
||||
const pending = new Promise<void>(resolve => { resume = resolve })
|
||||
const started = new Promise<void>(resolve => { ready = resolve })
|
||||
const firstRun = withComputerUseToolContext({ abortController: first } as ToolUseContext, async () => {
|
||||
expect(shared.isAborted?.()).toBe(false)
|
||||
ready()
|
||||
await pending
|
||||
expect(shared.isAborted?.()).toBe(true)
|
||||
})
|
||||
await started
|
||||
await withComputerUseToolContext({ abortController: second } as ToolUseContext, async () => {
|
||||
first.abort()
|
||||
await Promise.resolve()
|
||||
expect(shared.isAborted?.()).toBe(false)
|
||||
})
|
||||
resume()
|
||||
await firstRun
|
||||
})
|
||||
|
||||
import { spyOn } from 'bun:test'
|
||||
import * as hostAdapterModule from './hostAdapter.js'
|
||||
import * as lockModule from './computerUseLock.js'
|
||||
import * as gateModule from './gates.js'
|
||||
import * as debugModule from '../debug.js'
|
||||
import type { ComputerUseHostAdapter } from '../../vendor/computer-use-mcp/types.js'
|
||||
import type { CodexComputerEngine, ComputerExecutor } from '../../vendor/computer-use-mcp/executor.js'
|
||||
|
||||
test('real CU call wrapper delivers sequence metadata/images and releases fresh locks cancelled during acquisition', async () => {
|
||||
// Narrow, restored spies only; a separately imported wrapper keeps its cached
|
||||
// binder out of other tests. No helper command, real lock, user config or UI.
|
||||
const engineCalls: string[] = []
|
||||
let failInput = false
|
||||
const engine = {
|
||||
async resolveTarget() {
|
||||
engineCalls.push('resolve')
|
||||
return { pid: 420, bundleId: 'com.test.blender', launchTime: 100, executablePath: '/fixture/Blender' }
|
||||
},
|
||||
async pressKey() {
|
||||
engineCalls.push('pressKey')
|
||||
if (failInput) throw new Error('fixture response failure after dispatch')
|
||||
},
|
||||
async getAppState() {
|
||||
engineCalls.push('getAppState')
|
||||
return { pid: 420, appName: 'Fixture Blender', bundleId: 'com.test.blender', windowTitle: 'Untitled',
|
||||
elementCount: 0, truncated: false, durationMs: 1, axText: 'Fixture state',
|
||||
screenshot: { base64: 'Zml4dHVyZS1wbmc=', width: 1, height: 1 } }
|
||||
},
|
||||
} as unknown as CodexComputerEngine
|
||||
const adapter = {
|
||||
serverName: 'computer-use',
|
||||
logger: { silly() {}, debug() {}, info() {}, warn() {}, error() {} },
|
||||
executor: { capabilities: { platform: 'darwin', screenshotFiltering: 'native' }, engine } as ComputerExecutor,
|
||||
ensureOsPermissions: async () => ({ granted: true }),
|
||||
isDisabled: () => false,
|
||||
} as ComputerUseHostAdapter
|
||||
const adapterSpy = spyOn(hostAdapterModule, 'getComputerUseHostAdapter').mockReturnValue(adapter)
|
||||
const checkSpy = spyOn(lockModule, 'checkComputerUseLock').mockResolvedValue({ kind: 'held_by_self' })
|
||||
const acquireSpy = spyOn(lockModule, 'tryAcquireComputerUseLock').mockResolvedValue({ kind: 'acquired', fresh: true })
|
||||
const releaseSpy = spyOn(lockModule, 'releaseComputerUseLock').mockResolvedValue(true)
|
||||
const coordSpy = spyOn(gateModule, 'getChicagoCoordinateMode').mockReturnValue('pixels')
|
||||
const debugSpy = spyOn(debugModule, 'logForDebugging').mockImplementation(() => {})
|
||||
try {
|
||||
const moduleUrl = new URL('./wrapper.tsx', import.meta.url)
|
||||
moduleUrl.search = '?cu-wrapper-delivery-fixture'
|
||||
const wrapper = await import(moduleUrl.href) as typeof import('./wrapper.js')
|
||||
const makeContext = (abortController = new AbortController()) => ({
|
||||
abortController, getAppState: () => ({ computerUseMcpState: undefined }),
|
||||
setAppState() {}, sendOSNotification() {},
|
||||
}) as unknown as ToolUseContext
|
||||
const call = wrapper.getComputerUseMCPToolOverrides('sequence').call
|
||||
const args = { app: 'Fixture Blender', steps: [{ tool: 'press_key', key: 'a b c' }] }
|
||||
const delivered = await call(args, makeContext())
|
||||
const blocks = delivered.data as Array<Record<string, any>>
|
||||
expect(engineCalls).toEqual(['resolve', 'pressKey', 'getAppState'])
|
||||
expect(JSON.parse(blocks[0]!.text)).toMatchObject({ status: 'completed', completedSteps: 1, resultUnknown: false })
|
||||
expect(blocks.find(block => block.type === 'image')).toEqual({
|
||||
type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'Zml4dHVyZS1wbmc=' },
|
||||
})
|
||||
expect(blocks.some(block => block.type === 'text' && block.text.includes('Fixture state'))).toBe(true)
|
||||
|
||||
// The wrapper must deliver the actual error and structured partial outcome,
|
||||
// not replace it with a generic successful Computer Use result.
|
||||
failInput = true
|
||||
const failed = await call(args, makeContext()).then(() => '', error => String(error))
|
||||
expect(failed).toContain('fixture response failure after dispatch')
|
||||
expect(failed).toContain('"completedSteps":0')
|
||||
expect(failed).toContain('"failedStepIndex":0')
|
||||
expect(failed).toContain('"resultUnknown":true')
|
||||
expect(debugSpy).toHaveBeenCalled()
|
||||
|
||||
// Real buildSessionContext acquire callback, with only file-lock IO mocked.
|
||||
let enterAcquire!: () => void
|
||||
let finishAcquire!: () => void
|
||||
const entered = new Promise<void>(resolve => { enterAcquire = resolve })
|
||||
const holdAcquire = new Promise<void>(resolve => { finishAcquire = resolve })
|
||||
acquireSpy.mockImplementation(async () => {
|
||||
enterAcquire()
|
||||
await holdAcquire
|
||||
return { kind: 'acquired', fresh: true }
|
||||
})
|
||||
const abort = new AbortController()
|
||||
const acquiring = wrapper.withComputerUseToolContext(makeContext(abort), () => wrapper.buildSessionContext().acquireCuLock!())
|
||||
await entered
|
||||
abort.abort()
|
||||
finishAcquire()
|
||||
await expect(acquiring).rejects.toThrow('cancelled during lock acquisition')
|
||||
expect(releaseSpy).toHaveBeenCalledTimes(1)
|
||||
|
||||
const callsBefore = acquireSpy.mock.calls.length
|
||||
await expect(wrapper.withComputerUseToolContext(makeContext(abort), () => wrapper.buildSessionContext().acquireCuLock!()))
|
||||
.rejects.toThrow('cancelled before lock acquisition')
|
||||
expect(acquireSpy.mock.calls.length).toBe(callsBefore)
|
||||
|
||||
// A normal reentrant acquisition continues and does not release another
|
||||
// operation's existing session lock.
|
||||
acquireSpy.mockResolvedValue({ kind: 'acquired', fresh: false })
|
||||
await wrapper.withComputerUseToolContext(makeContext(), () => wrapper.buildSessionContext().acquireCuLock!())
|
||||
expect(releaseSpy).toHaveBeenCalledTimes(1)
|
||||
} finally {
|
||||
for (const spy of [adapterSpy, checkSpy, acquireSpy, releaseSpy, coordSpy, debugSpy]) spy.mockRestore()
|
||||
}
|
||||
})
|
||||
|
||||
@@ -20,7 +20,7 @@ import { bindSessionContext, type ComputerUseSessionContext, type CuCallToolResu
|
||||
import { getSessionId } from '../../bootstrap/state.js';
|
||||
import type { Tool, ToolUseContext } from '../../Tool.js';
|
||||
import { logForDebugging } from '../debug.js';
|
||||
import { checkComputerUseLock, tryAcquireComputerUseLock } from './computerUseLock.js';
|
||||
import { checkComputerUseLock, tryAcquireComputerUseLock, releaseComputerUseLock } from './computerUseLock.js';
|
||||
import { registerEscHotkey } from './escHotkey.js';
|
||||
import { getChicagoCoordinateMode } from './gates.js';
|
||||
import { getComputerUseHostAdapter } from './hostAdapter.js';
|
||||
@@ -42,6 +42,10 @@ type Binding = {
|
||||
*/
|
||||
let binding: Binding | undefined;
|
||||
const toolUseContexts = new AsyncLocalStorage<ToolUseContext>();
|
||||
/** Preserve the originating turn context across queued/native awaits. */
|
||||
export function withComputerUseToolContext<T>(context: ToolUseContext, run: () => T): T {
|
||||
return toolUseContexts.run(context, run);
|
||||
}
|
||||
const ENABLED_GRANT_FLAGS = {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
@@ -74,6 +78,7 @@ export function buildSessionContext(): ComputerUseSessionContext {
|
||||
// returning the legacy shapes for protocol compatibility, but do not
|
||||
// consume persisted app grants or open runtime permission prompts.
|
||||
getAllowedApps: () => [],
|
||||
isAborted: () => tuc().abortController.signal.aborted,
|
||||
getGrantFlags: () => ENABLED_GRANT_FLAGS,
|
||||
getUserDeniedBundleIds: () => [],
|
||||
getSelectedDisplayId: () => tuc().getAppState().computerUseMcpState?.selectedDisplayId,
|
||||
@@ -196,7 +201,13 @@ export function buildSessionContext(): ComputerUseSessionContext {
|
||||
// but is possible under parallel tool-use interleaving — don't spam the
|
||||
// notification in that case.
|
||||
acquireCuLock: async () => {
|
||||
const signal = tuc().abortController.signal;
|
||||
if (signal.aborted) throw new Error("Computer Use cancelled before lock acquisition");
|
||||
const r = await tryAcquireComputerUseLock();
|
||||
if (signal.aborted) {
|
||||
if (r.kind === 'acquired' && r.fresh) await releaseComputerUseLock();
|
||||
throw new Error("Computer Use cancelled during lock acquisition");
|
||||
}
|
||||
if (r.kind === 'blocked') {
|
||||
throw new Error(formatLockHeld(r.by));
|
||||
}
|
||||
|
||||
@@ -1,70 +0,0 @@
|
||||
import { expect, test } from 'bun:test'
|
||||
import { readFileSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
import { _test as deniedApps } from './deniedApps.js'
|
||||
import { NATIVE_FORBIDDEN_BUNDLE_IDS } from './nativeAppPolicy.js'
|
||||
|
||||
const SWIFT_SET_MARKER = 'static let deniedBundleIDs: Set<String> = ['
|
||||
const SWIFT_INTRINSIC_SET_MARKER = 'static let intrinsicDeniedBundleIDs: Set<String> = ['
|
||||
|
||||
function parseNativeDeniedBundleIds(source: string, marker: string): string[] {
|
||||
const markerIndex = source.indexOf(marker)
|
||||
expect(markerIndex).toBeGreaterThanOrEqual(0)
|
||||
|
||||
const bodyStart = markerIndex + marker.length
|
||||
const bodyEnd = source.indexOf('\n ]', bodyStart)
|
||||
expect(bodyEnd).toBeGreaterThan(bodyStart)
|
||||
|
||||
const body = source.slice(bodyStart, bodyEnd)
|
||||
return [...body.matchAll(/^\s*"([^"]+)",?\s*(?:\/\/.*)?$/gm)].map(match => match[1])
|
||||
}
|
||||
|
||||
test('native deny policy matches the official 24 exact forbidden identities', () => {
|
||||
const tsEntries = [...NATIVE_FORBIDDEN_BUNDLE_IDS]
|
||||
const expected = new Set(tsEntries)
|
||||
|
||||
const swiftPath = resolve(
|
||||
import.meta.dir,
|
||||
'../../../native/cu-helper/Sources/cu-helper/AppTargetPolicy.swift',
|
||||
)
|
||||
const nativeEntries = parseNativeDeniedBundleIds(
|
||||
readFileSync(swiftPath, 'utf8'),
|
||||
SWIFT_SET_MARKER,
|
||||
)
|
||||
const actual = new Set(nativeEntries)
|
||||
|
||||
const missing = [...expected].filter(bundleId => !actual.has(bundleId)).sort()
|
||||
const extra = [...actual].filter(bundleId => !expected.has(bundleId)).sort()
|
||||
|
||||
expect(tsEntries).toHaveLength(expected.size)
|
||||
expect(nativeEntries).toHaveLength(actual.size)
|
||||
expect(expected.size).toBe(24)
|
||||
expect(actual.size).toBe(24)
|
||||
expect(missing).toEqual([])
|
||||
expect(extra).toEqual([])
|
||||
expect([...actual].sort()).toEqual([...expected].sort())
|
||||
// Browser classification still exists for the Windows tool tier. Native
|
||||
// macOS control may use the same browser as the official Codex app surface.
|
||||
expect(deniedApps.BROWSER_BUNDLE_IDS.size).toBe(29)
|
||||
expect([...deniedApps.BROWSER_BUNDLE_IDS].filter(bundleId => actual.has(bundleId))).toEqual([])
|
||||
})
|
||||
|
||||
test('native intrinsic deny set stays separate and matches the TS host/helper defaults', () => {
|
||||
const swiftPath = resolve(
|
||||
import.meta.dir,
|
||||
'../../../native/cu-helper/Sources/cu-helper/AppTargetPolicy.swift',
|
||||
)
|
||||
const nativeEntries = parseNativeDeniedBundleIds(
|
||||
readFileSync(swiftPath, 'utf8'),
|
||||
SWIFT_INTRINSIC_SET_MARKER,
|
||||
)
|
||||
const expected = deniedApps.INTRINSIC_DENIED_BUNDLE_IDS
|
||||
|
||||
expect(new Set(nativeEntries)).toEqual(expected)
|
||||
expect(expected).toEqual(new Set([
|
||||
'com.claude-code-haha.desktop',
|
||||
'dev.cchaha.cu-helper',
|
||||
]))
|
||||
expect(nativeEntries).toHaveLength(2)
|
||||
})
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
import { expect, test } from 'bun:test'
|
||||
import {
|
||||
categoryToTier,
|
||||
getDefaultTierForApp,
|
||||
getDeniedCategory,
|
||||
getDeniedCategoryByDisplayName,
|
||||
getDeniedCategoryForApp,
|
||||
isPolicyDenied,
|
||||
} from './deniedApps.js'
|
||||
|
||||
// Legacy consumers still use these exports. Neither an old category value nor
|
||||
// an app lookup may reintroduce restrictions after feature-wide consent.
|
||||
test('legacy category values cannot downgrade global Computer Use consent', () => {
|
||||
for (const category of ['browser', 'terminal', 'trading', null] as const) {
|
||||
expect(categoryToTier(category)).toBe('full')
|
||||
}
|
||||
})
|
||||
|
||||
test.each([
|
||||
['com.google.Chrome', 'Google Chrome'],
|
||||
['com.apple.Terminal', 'Terminal'],
|
||||
['com.spotify.client', 'Spotify'],
|
||||
['com.webull.desktop.v1', 'Webull'],
|
||||
['com.claude-code-haha.desktop', 'Claude Code Haha'],
|
||||
['dev.cchaha.cu-helper', 'Computer Use Helper'],
|
||||
['org.example.new-app', 'New App'],
|
||||
])('legacy lookup for %s grants the same access with or without a bundle ID', (bundleId, displayName) => {
|
||||
expect(getDeniedCategory(bundleId)).toBeNull()
|
||||
expect(getDeniedCategoryByDisplayName(displayName)).toBeNull()
|
||||
for (const id of [bundleId, undefined]) {
|
||||
expect(getDeniedCategoryForApp(id, displayName)).toBeNull()
|
||||
expect(isPolicyDenied(id, displayName)).toBe(false)
|
||||
expect(getDefaultTierForApp(id, displayName)).toBe('full')
|
||||
}
|
||||
})
|
||||
+27
-572
@@ -1,589 +1,44 @@
|
||||
/**
|
||||
* App category lookup for tiered CU permissions. Three categories land at a
|
||||
* restricted tier instead of `"full"`:
|
||||
* Computer Use is authorized once through the global enable dialog. All
|
||||
* apps have full access regardless of category, bundle ID, display name, or host/helper identity.
|
||||
*
|
||||
* - **browser** → `"read"` tier — visible in screenshots, NO interaction.
|
||||
* The model can read an already-open page but must use the Claude-in-Chrome
|
||||
* MCP for navigation/clicking/typing.
|
||||
* - **terminal** → `"click"` tier — visible + clickable, NO typing. The
|
||||
* model can click a Run button or scroll test output in an IDE, but can't
|
||||
* type into the integrated terminal. Use the Bash tool for shell work.
|
||||
* - **trading** → `"read"` tier — same restrictions as browsers, but no
|
||||
* CiC-MCP alternative exists. For platforms where a stray click can
|
||||
* execute a trade or send a message to a counterparty.
|
||||
*
|
||||
* Uncategorized apps default to `"full"`. See `getDefaultTierForApp`.
|
||||
*
|
||||
* Identification is two-layered:
|
||||
* 1. Bundle ID match (macOS-only; `InstalledApp.bundleId` is a
|
||||
* CFBundleIdentifier and meaningless on Windows). Fast, exact, the
|
||||
* primary mechanism while CU is darwin-gated.
|
||||
* 2. Display-name substring match (cross-platform fallback). Catches
|
||||
* unresolved requests ("Chrome" when Chrome isn't installed) AND will
|
||||
* be the primary mechanism on Windows/Linux where there's no bundle ID.
|
||||
* Windows-relevant names (PowerShell, cmd, Windows Terminal) are
|
||||
* included now so they activate the moment the darwin gate lifts.
|
||||
*
|
||||
* Keep this file **import-free** (like sentinelApps.ts) — the renderer may
|
||||
* import it via a package.json subpath export, and pulling in
|
||||
* `@modelcontextprotocol/sdk` (a devDep) through the index → mcpServer chain
|
||||
* would fail module resolution in Next.js. The `CuAppPermTier` type is
|
||||
* duplicated as a string literal below rather than imported.
|
||||
* Keep the legacy lookup exports for the Windows dispatcher and stored-grant
|
||||
* compatibility code. They no longer impose per-app policy or access tiers.
|
||||
* This module stays import-free because the desktop also consumes it.
|
||||
*/
|
||||
export type DeniedCategory = 'browser' | 'terminal' | 'trading'
|
||||
|
||||
export type DeniedCategory = "browser" | "terminal" | "trading";
|
||||
|
||||
/**
|
||||
* Map a category to its hardcoded tier. Return-type is the string-literal
|
||||
* union inline (this file is import-free; see header comment). The
|
||||
* authoritative type is `CuAppPermTier` in types.ts — keep in sync.
|
||||
*
|
||||
* Not bijective — both `"browser"` and `"trading"` map to `"read"`. Copy
|
||||
* that differs by category (the "use CiC" hint is browser-only) must check
|
||||
* the category, not just the tier.
|
||||
*/
|
||||
export function categoryToTier(
|
||||
category: DeniedCategory | null,
|
||||
): "read" | "click" | "full" {
|
||||
if (category === "browser" || category === "trading") return "read";
|
||||
if (category === "terminal") return "click";
|
||||
return "full";
|
||||
_category: DeniedCategory | null,
|
||||
): 'read' | 'click' | 'full' {
|
||||
return 'full'
|
||||
}
|
||||
|
||||
// ─── Bundle-ID deny sets (macOS) ─────────────────────────────────────────
|
||||
|
||||
const BROWSER_BUNDLE_IDS: ReadonlySet<string> = new Set([
|
||||
// Apple
|
||||
"com.apple.Safari",
|
||||
"com.apple.SafariTechnologyPreview",
|
||||
// Google
|
||||
"com.google.Chrome",
|
||||
"com.google.Chrome.beta",
|
||||
"com.google.Chrome.dev",
|
||||
"com.google.Chrome.canary",
|
||||
// Microsoft
|
||||
"com.microsoft.edgemac",
|
||||
"com.microsoft.edgemac.Beta",
|
||||
"com.microsoft.edgemac.Dev",
|
||||
"com.microsoft.edgemac.Canary",
|
||||
// Mozilla
|
||||
"org.mozilla.firefox",
|
||||
"org.mozilla.firefoxdeveloperedition",
|
||||
"org.mozilla.nightly",
|
||||
// Chromium-based
|
||||
"org.chromium.Chromium",
|
||||
"com.brave.Browser",
|
||||
"com.brave.Browser.beta",
|
||||
"com.brave.Browser.nightly",
|
||||
"com.operasoftware.Opera",
|
||||
"com.operasoftware.OperaGX",
|
||||
"com.operasoftware.OperaDeveloper",
|
||||
"com.vivaldi.Vivaldi",
|
||||
// The Browser Company
|
||||
"company.thebrowser.Browser", // Arc
|
||||
"company.thebrowser.dia", // Dia (agentic)
|
||||
// Privacy-focused
|
||||
"org.torproject.torbrowser",
|
||||
"com.duckduckgo.macos.browser",
|
||||
"ru.yandex.desktop.yandex-browser",
|
||||
// Agentic / AI browsers — newer entrants with LLM integrations
|
||||
"ai.perplexity.comet",
|
||||
"com.sigmaos.sigmaos.macos", // SigmaOS
|
||||
// Webkit-based misc
|
||||
"com.kagi.kagimacOS", // Orion
|
||||
]);
|
||||
|
||||
/**
|
||||
* Terminals + IDEs with integrated terminals. Supersets
|
||||
* `SHELL_ACCESS_BUNDLE_IDS` from sentinelApps.ts — terminals proceed to the
|
||||
* approval dialog at tier "click", and the sentinel warning renders
|
||||
* alongside the tier badge.
|
||||
*/
|
||||
const TERMINAL_BUNDLE_IDS: ReadonlySet<string> = new Set([
|
||||
// Dedicated terminals
|
||||
"com.apple.Terminal",
|
||||
"com.googlecode.iterm2",
|
||||
"dev.warp.Warp-Stable",
|
||||
"dev.warp.Warp-Beta",
|
||||
"com.github.wez.wezterm",
|
||||
"org.alacritty",
|
||||
"io.alacritty", // pre-v0.11.0 (renamed 2022-07) — kept for legacy installs
|
||||
"net.kovidgoyal.kitty",
|
||||
"co.zeit.hyper",
|
||||
"com.mitchellh.ghostty",
|
||||
"org.tabby",
|
||||
"com.termius-dmg.mac", // Termius
|
||||
// IDEs with integrated terminals — we can't distinguish "type in the
|
||||
// editor" from "type in the integrated terminal" via screenshot+click.
|
||||
// VS Code family
|
||||
"com.microsoft.VSCode",
|
||||
"com.microsoft.VSCodeInsiders",
|
||||
"com.vscodium", // VSCodium
|
||||
"com.todesktop.230313mzl4w4u92", // Cursor
|
||||
"com.exafunction.windsurf", // Windsurf / Codeium
|
||||
"dev.zed.Zed",
|
||||
"dev.zed.Zed-Preview",
|
||||
// JetBrains family (all have integrated terminals)
|
||||
"com.jetbrains.intellij",
|
||||
"com.jetbrains.intellij.ce",
|
||||
"com.jetbrains.pycharm",
|
||||
"com.jetbrains.pycharm.ce",
|
||||
"com.jetbrains.WebStorm",
|
||||
"com.jetbrains.CLion",
|
||||
"com.jetbrains.goland",
|
||||
"com.jetbrains.rubymine",
|
||||
"com.jetbrains.PhpStorm",
|
||||
"com.jetbrains.datagrip",
|
||||
"com.jetbrains.rider",
|
||||
"com.jetbrains.AppCode",
|
||||
"com.jetbrains.rustrover",
|
||||
"com.jetbrains.fleet",
|
||||
"com.google.android.studio", // Android Studio (JetBrains-based)
|
||||
// Other IDEs
|
||||
"com.axosoft.gitkraken", // GitKraken has an integrated terminal panel. Also keeps the "kraken" trading-substring from miscategorizing it — bundle-ID wins.
|
||||
"com.sublimetext.4",
|
||||
"com.sublimetext.3",
|
||||
"org.vim.MacVim",
|
||||
"com.neovim.neovim",
|
||||
"org.gnu.Emacs",
|
||||
// Xcode's previous carve-out (full tier for Interface Builder / simulator)
|
||||
// was reversed — at tier "click" IB and simulator taps still work (both are
|
||||
// plain clicks) while the integrated terminal is blocked from keyboard input.
|
||||
"com.apple.dt.Xcode",
|
||||
"org.eclipse.platform.ide",
|
||||
"org.netbeans.ide",
|
||||
"com.microsoft.visual-studio", // Visual Studio for Mac
|
||||
// AppleScript/automation execution surfaces — same threat as terminals:
|
||||
// type(script) → key("cmd+r") runs arbitrary code. Added after #28011
|
||||
// removed the osascript MCP server, making CU the only tool-call route
|
||||
// to AppleScript.
|
||||
"com.apple.ScriptEditor2",
|
||||
"com.apple.Automator",
|
||||
"com.apple.shortcuts",
|
||||
]);
|
||||
|
||||
/**
|
||||
* Trading / crypto platforms — granted at tier `"read"` so the agent can see
|
||||
* balances and prices but can't click into an order, transfer, or IB chat.
|
||||
* Bundle IDs populated from Homebrew cask `uninstall.quit` stanzas as they're
|
||||
* verified; the name-substring fallback below is the primary check. Bloomberg
|
||||
* Terminal has no native macOS build per their FAQ (web/Citrix only).
|
||||
*
|
||||
* Budgeting/accounting apps (Quicken, YNAB, QuickBooks, etc.) are NOT listed
|
||||
* here — they default to tier `"full"`. The risk model for brokerage/crypto
|
||||
* (a stray click can execute a trade) doesn't apply to budgeting apps; the
|
||||
* Cowork system prompt carries the soft instruction to never execute trades
|
||||
* or transfer money on the user's behalf.
|
||||
*/
|
||||
const TRADING_BUNDLE_IDS: ReadonlySet<string> = new Set([
|
||||
// Verified via Homebrew quit/zap stanzas + mdls + electron-builder source.
|
||||
// Trading
|
||||
"com.webull.desktop.v1", // Webull (direct download, Qt)
|
||||
"com.webull.trade.mac.v1", // Webull (Mac App Store)
|
||||
"com.tastytrade.desktop",
|
||||
"com.tradingview.tradingviewapp.desktop",
|
||||
"com.fidelity.activetrader", // Fidelity Trader+ (new)
|
||||
"com.fmr.activetrader", // Fidelity Active Trader Pro (legacy)
|
||||
// Interactive Brokers TWS — install4j wrapper; Homebrew quit stanza is
|
||||
// authoritative for this exact value but install4j IDs can drift across
|
||||
// major versions — name-substring "trader workstation" is the fallback.
|
||||
"com.install4j.5889-6375-8446-2021",
|
||||
// Crypto
|
||||
"com.binance.BinanceDesktop",
|
||||
"com.electron.exodus",
|
||||
// Electrum uses PyInstaller with bundle_identifier=None → defaults to
|
||||
// org.pythonmac.unspecified.<AppName>. Confirmed in spesmilo/electrum
|
||||
// source + Homebrew zap. IntuneBrew's "org.electrum.electrum" is a fork.
|
||||
"org.pythonmac.unspecified.Electrum",
|
||||
"com.ledger.live",
|
||||
"io.trezor.TrezorSuite",
|
||||
// No native macOS app (name-substring only): Schwab, E*TRADE, TradeStation,
|
||||
// Robinhood, NinjaTrader, Coinbase, Kraken, Bloomberg. thinkorswim
|
||||
// install4j ID drifts per-install — substring safer.
|
||||
]);
|
||||
|
||||
// ─── Policy-deny (not a tier — cannot be granted at all) ─────────────────
|
||||
//
|
||||
// Streaming / ebook / music apps and a handful of publisher apps. These
|
||||
// are auto-denied before the approval dialog — no tier can be granted.
|
||||
// Rationale is copyright / content-control (the agent has no legitimate
|
||||
// need to screenshot Netflix or click Play on Spotify).
|
||||
//
|
||||
// Sourced from the ACP CU-apps blocklist xlsx ("Full block" tab). See
|
||||
// /tmp/extract_cu_blocklist.py for the extraction script.
|
||||
|
||||
const POLICY_DENIED_BUNDLE_IDS: ReadonlySet<string> = new Set([
|
||||
// Verified via Homebrew quit/zap + mdls /System/Applications + IntuneBrew.
|
||||
// Apple built-ins
|
||||
"com.apple.TV",
|
||||
"com.apple.Music",
|
||||
"com.apple.iBooksX",
|
||||
"com.apple.podcasts",
|
||||
// Music
|
||||
"com.spotify.client",
|
||||
"com.amazon.music",
|
||||
"com.tidal.desktop",
|
||||
"com.deezer.deezer-desktop",
|
||||
"com.pandora.desktop",
|
||||
"com.electron.pocket-casts", // direct-download Electron wrapper
|
||||
"au.com.shiftyjelly.PocketCasts", // Mac App Store
|
||||
// Video
|
||||
"tv.plex.desktop",
|
||||
"tv.plex.htpc",
|
||||
"tv.plex.plexamp",
|
||||
"com.amazon.aiv.AIVApp", // Prime Video (iOS-on-Apple-Silicon)
|
||||
// Ebooks
|
||||
"net.kovidgoyal.calibre",
|
||||
"com.amazon.Kindle", // legacy desktop, discontinued
|
||||
"com.amazon.Lassen", // current Mac App Store (iOS-on-Mac)
|
||||
"com.kobo.desktop.Kobo",
|
||||
// No native macOS app (name-substring only): Netflix, Disney+, Hulu,
|
||||
// HBO Max, Peacock, Paramount+, YouTube, Crunchyroll, Tubi, Vudu,
|
||||
// Audible, Reddit, NYTimes. Their iOS apps don't opt into iPad-on-Mac.
|
||||
]);
|
||||
|
||||
const POLICY_DENIED_NAME_SUBSTRINGS: readonly string[] = [
|
||||
// Video streaming
|
||||
"netflix",
|
||||
"disney+",
|
||||
"hulu",
|
||||
"prime video",
|
||||
"apple tv",
|
||||
"peacock",
|
||||
"paramount+",
|
||||
// "plex" is too generic — would match "Perplexity". Covered by
|
||||
// tv.plex.* bundle IDs on macOS.
|
||||
"tubi",
|
||||
"crunchyroll",
|
||||
"vudu",
|
||||
// E-readers / audiobooks
|
||||
"kindle",
|
||||
"apple books",
|
||||
"kobo",
|
||||
"play books",
|
||||
"calibre",
|
||||
"libby",
|
||||
"readium",
|
||||
"audible",
|
||||
"libro.fm",
|
||||
"speechify",
|
||||
// Music
|
||||
"spotify",
|
||||
"apple music",
|
||||
"amazon music",
|
||||
"youtube music",
|
||||
"tidal",
|
||||
"deezer",
|
||||
"pandora",
|
||||
"pocket casts",
|
||||
// Publisher / social apps (from the same blocklist tab)
|
||||
"naver",
|
||||
"reddit",
|
||||
"sony music",
|
||||
"vegas pro",
|
||||
"pitchfork",
|
||||
"economist",
|
||||
"nytimes",
|
||||
// Skipped (too generic for substring matching — need bundle ID):
|
||||
// HBO Max / Max, YouTube (non-Music), Nook, Sony Catalyst, Wired
|
||||
];
|
||||
|
||||
/**
|
||||
* Policy-level auto-deny. Unlike `userDeniedBundleIds` (per-user Settings
|
||||
* page), this is baked into the build. `buildAccessRequest` strips these
|
||||
* before the approval dialog with "blocked by policy" guidance; the agent
|
||||
* is told to not retry.
|
||||
*/
|
||||
export function isPolicyDenied(
|
||||
bundleId: string | undefined,
|
||||
displayName: string,
|
||||
): boolean {
|
||||
if (bundleId && POLICY_DENIED_BUNDLE_IDS.has(bundleId)) return true;
|
||||
const lower = displayName.toLowerCase();
|
||||
for (const sub of POLICY_DENIED_NAME_SUBSTRINGS) {
|
||||
if (lower.includes(sub)) return true;
|
||||
}
|
||||
return false;
|
||||
export function getDeniedCategory(_bundleId: string): DeniedCategory | null {
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* Apps the agent can never drive, no matter what the user grants: our own
|
||||
* desktop shell and the Computer Use helper that executes the actions.
|
||||
*
|
||||
* Kept as its own set rather than folded into `POLICY_DENIED_BUNDLE_IDS`
|
||||
* because the two mean different things. The policy sets are *product* policy
|
||||
* (streaming, trading) — debatable, revisable, and surfaced to the user as
|
||||
* "blocked by policy". This one is a structural invariant: driving the host
|
||||
* lets the agent click its own approval dialogs, and driving the helper lets
|
||||
* it reach around the very process enforcing the grants. The native
|
||||
* `AppTargetPolicy.swift` mirrors both sets, separately, for the same reason.
|
||||
*/
|
||||
const INTRINSIC_DENIED_BUNDLE_IDS: ReadonlySet<string> = new Set([
|
||||
"com.claude-code-haha.desktop",
|
||||
"dev.cchaha.cu-helper",
|
||||
]);
|
||||
|
||||
/**
|
||||
* True when the target is the host app or its helper — a permanent denial that
|
||||
* no grant can lift.
|
||||
*
|
||||
* `hostBundleId` is checked *in addition to* the baked-in set, never instead
|
||||
* of it: it catches builds whose bundle id differs from the shipped default
|
||||
* (dev and beta channels), while the constants still hold if a caller has no
|
||||
* host id to pass.
|
||||
*/
|
||||
export function isIntrinsicAppDenied(
|
||||
bundleId: string | undefined,
|
||||
hostBundleId?: string,
|
||||
): boolean {
|
||||
if (!bundleId) return false;
|
||||
if (INTRINSIC_DENIED_BUNDLE_IDS.has(bundleId)) return true;
|
||||
return hostBundleId !== undefined && bundleId === hostBundleId;
|
||||
export function getDeniedCategoryByDisplayName(_name: string): DeniedCategory | null {
|
||||
return null
|
||||
}
|
||||
|
||||
export function getDeniedCategory(bundleId: string): DeniedCategory | null {
|
||||
if (BROWSER_BUNDLE_IDS.has(bundleId)) return "browser";
|
||||
if (TERMINAL_BUNDLE_IDS.has(bundleId)) return "terminal";
|
||||
if (TRADING_BUNDLE_IDS.has(bundleId)) return "trading";
|
||||
return null;
|
||||
}
|
||||
|
||||
// ─── Display-name fallback (cross-platform) ──────────────────────────────
|
||||
|
||||
/**
|
||||
* Lowercase substrings checked against the requested display name. Catches:
|
||||
* - Unresolved requests (app not installed, Spotlight miss)
|
||||
* - Future Windows/Linux support where bundleId is meaningless
|
||||
*
|
||||
* Matched via `.includes()` on `name.toLowerCase()`. Entries are ordered
|
||||
* by specificity (more-specific first is irrelevant since we return on
|
||||
* first match, but groupings are by category for readability).
|
||||
*/
|
||||
const BROWSER_NAME_SUBSTRINGS: readonly string[] = [
|
||||
"safari",
|
||||
"chrome",
|
||||
"firefox",
|
||||
"microsoft edge",
|
||||
"brave",
|
||||
"opera",
|
||||
"vivaldi",
|
||||
"chromium",
|
||||
// Arc/Dia: the canonical display name is just "Arc"/"Dia" — too short for
|
||||
// substring matching (false-positives: "Arcade", "Diagram"). Covered by
|
||||
// bundle ID on macOS. The "... browser" entries below catch natural-language
|
||||
// phrasings ("the arc browser") but NOT the canonical short name.
|
||||
"arc browser",
|
||||
"tor browser",
|
||||
"duckduckgo",
|
||||
"yandex",
|
||||
"orion browser",
|
||||
// Agentic / AI browsers
|
||||
"comet", // Perplexity's browser — "Comet" substring risks false positives
|
||||
// but leaving for now; "comet" in an app name is rare
|
||||
"sigmaos",
|
||||
"dia browser",
|
||||
];
|
||||
|
||||
const TERMINAL_NAME_SUBSTRINGS: readonly string[] = [
|
||||
// macOS / cross-platform terminals
|
||||
"terminal", // catches Terminal, Windows Terminal (NOT iTerm — separate entry)
|
||||
"iterm",
|
||||
"wezterm",
|
||||
"alacritty",
|
||||
"kitty",
|
||||
"ghostty",
|
||||
"tabby",
|
||||
"termius",
|
||||
// AppleScript runners — see bundle-ID comment above. "shortcuts" is too
|
||||
// generic for substring matching (many apps have "shortcuts" in the name);
|
||||
// covered by bundle ID only, like warp/hyper.
|
||||
"script editor",
|
||||
"automator",
|
||||
// NOTE: "warp" and "hyper" are too generic for substring matching —
|
||||
// they'd false-positive on "Warpaint" or "Hyperion". Covered by bundle ID
|
||||
// (dev.warp.Warp-Stable, co.zeit.hyper) for macOS; Windows exe-name
|
||||
// matching can be added when Windows CU ships.
|
||||
// Windows shells (activate when the darwin gate lifts)
|
||||
"powershell",
|
||||
"cmd.exe",
|
||||
"command prompt",
|
||||
"git bash",
|
||||
"conemu",
|
||||
"cmder",
|
||||
// IDEs (VS Code family)
|
||||
"visual studio code",
|
||||
"visual studio", // catches VS for Mac + Windows
|
||||
"vscode",
|
||||
"vs code",
|
||||
"vscodium",
|
||||
"cursor", // Cursor IDE — "cursor" is generic but IDE is the only common app
|
||||
"windsurf",
|
||||
// Zed: display name is just "Zed" — too short for substring matching
|
||||
// (false-positives). Covered by bundle ID (dev.zed.Zed) on macOS.
|
||||
// IDEs (JetBrains family)
|
||||
"intellij",
|
||||
"pycharm",
|
||||
"webstorm",
|
||||
"clion",
|
||||
"goland",
|
||||
"rubymine",
|
||||
"phpstorm",
|
||||
"datagrip",
|
||||
"rider",
|
||||
"appcode",
|
||||
"rustrover",
|
||||
"fleet",
|
||||
"android studio",
|
||||
// Other IDEs
|
||||
"sublime text",
|
||||
"macvim",
|
||||
"neovim",
|
||||
"emacs",
|
||||
"xcode",
|
||||
"eclipse",
|
||||
"netbeans",
|
||||
];
|
||||
|
||||
const TRADING_NAME_SUBSTRINGS: readonly string[] = [
|
||||
// Trading — brokerage apps. Sourced from the ACP CU-apps blocklist xlsx
|
||||
// ("Read Only" tab). Name-substring safe for proper nouns below; generic
|
||||
// names (IG, Delta, HTX) are skipped and need bundle-ID matching once
|
||||
// verified.
|
||||
"bloomberg",
|
||||
"ameritrade",
|
||||
"thinkorswim",
|
||||
"schwab",
|
||||
"fidelity",
|
||||
"e*trade",
|
||||
"interactive brokers",
|
||||
"trader workstation", // Interactive Brokers TWS
|
||||
"tradestation",
|
||||
"webull",
|
||||
"robinhood",
|
||||
"tastytrade",
|
||||
"ninjatrader",
|
||||
"tradingview",
|
||||
"moomoo",
|
||||
"tradezero",
|
||||
"prorealtime",
|
||||
"plus500",
|
||||
"saxotrader",
|
||||
"oanda",
|
||||
"metatrader",
|
||||
"forex.com",
|
||||
"avaoptions",
|
||||
"ctrader",
|
||||
"jforex",
|
||||
"iq option",
|
||||
"olymp trade",
|
||||
"binomo",
|
||||
"pocket option",
|
||||
"raceoption",
|
||||
"expertoption",
|
||||
"quotex",
|
||||
"naga",
|
||||
"morgan stanley",
|
||||
"ubs neo",
|
||||
"eikon", // Thomson Reuters / LSEG Workspace
|
||||
// Crypto — exchanges, wallets, portfolio trackers
|
||||
"coinbase",
|
||||
"kraken",
|
||||
"binance",
|
||||
"okx",
|
||||
"bybit",
|
||||
// "gate.io" is too generic — the ".io" TLD suffix is common in app names
|
||||
// (e.g., "Draw.io"). Needs bundle-ID matching once verified.
|
||||
"phemex",
|
||||
"stormgain",
|
||||
"crypto.com",
|
||||
// "exodus" is too generic — it's a common noun and would match unrelated
|
||||
// apps/games. Needs bundle-ID matching once verified.
|
||||
"electrum",
|
||||
"ledger live",
|
||||
"trezor",
|
||||
"guarda",
|
||||
"atomic wallet",
|
||||
"bitpay",
|
||||
"bisq",
|
||||
"koinly",
|
||||
"cointracker",
|
||||
"blockfi",
|
||||
"stripe cli",
|
||||
// Crypto games / metaverse (same trade-execution risk model)
|
||||
"decentraland",
|
||||
"axie infinity",
|
||||
"gods unchained",
|
||||
];
|
||||
|
||||
/**
|
||||
* Display-name substring match. Called when bundle-ID resolution returned
|
||||
* nothing (`resolved === undefined`) or when no bundle-ID deny-list entry
|
||||
* matched. Returns the category for the first matching substring, or null.
|
||||
*
|
||||
* Case-insensitive, substring — so `"Google Chrome"`, `"chrome"`, and
|
||||
* `"Chrome Canary"` all match the `"chrome"` entry.
|
||||
*/
|
||||
export function getDeniedCategoryByDisplayName(
|
||||
name: string,
|
||||
): DeniedCategory | null {
|
||||
const lower = name.toLowerCase();
|
||||
// Trading first — proper-noun-only set, most specific. "Bloomberg Terminal"
|
||||
// contains "terminal" and would miscategorize if TERMINAL_NAME_SUBSTRINGS
|
||||
// ran first.
|
||||
for (const sub of TRADING_NAME_SUBSTRINGS) {
|
||||
if (lower.includes(sub)) return "trading";
|
||||
}
|
||||
for (const sub of BROWSER_NAME_SUBSTRINGS) {
|
||||
if (lower.includes(sub)) return "browser";
|
||||
}
|
||||
for (const sub of TERMINAL_NAME_SUBSTRINGS) {
|
||||
if (lower.includes(sub)) return "terminal";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Combined check — bundle ID first (exact, fast), then display-name
|
||||
* fallback. This is the function tool-call handlers should use.
|
||||
*
|
||||
* `bundleId` may be undefined (unresolved request — model asked for an app
|
||||
* that isn't installed or Spotlight didn't find). In that case only the
|
||||
* display-name check runs.
|
||||
*/
|
||||
export function getDeniedCategoryForApp(
|
||||
bundleId: string | undefined,
|
||||
displayName: string,
|
||||
_bundleId: string | undefined,
|
||||
_displayName: string,
|
||||
): DeniedCategory | null {
|
||||
if (bundleId) {
|
||||
const byId = getDeniedCategory(bundleId);
|
||||
if (byId) return byId;
|
||||
}
|
||||
return getDeniedCategoryByDisplayName(displayName);
|
||||
return null
|
||||
}
|
||||
|
||||
export function isPolicyDenied(
|
||||
_bundleId: string | undefined,
|
||||
_displayName: string,
|
||||
): boolean {
|
||||
return false
|
||||
}
|
||||
|
||||
/**
|
||||
* Default tier for an app at grant time. Wraps `getDeniedCategoryForApp` +
|
||||
* `categoryToTier`. Browsers → `"read"`, terminals/IDEs → `"click"`,
|
||||
* everything else → `"full"`.
|
||||
*
|
||||
* Called by `buildAccessRequest` to populate `ResolvedAppRequest.proposedTier`
|
||||
* before the approval dialog shows.
|
||||
*/
|
||||
export function getDefaultTierForApp(
|
||||
bundleId: string | undefined,
|
||||
displayName: string,
|
||||
): "read" | "click" | "full" {
|
||||
return categoryToTier(getDeniedCategoryForApp(bundleId, displayName));
|
||||
_bundleId: string | undefined,
|
||||
_displayName: string,
|
||||
): 'read' | 'click' | 'full' {
|
||||
return 'full'
|
||||
}
|
||||
|
||||
export const _test = {
|
||||
BROWSER_BUNDLE_IDS,
|
||||
TERMINAL_BUNDLE_IDS,
|
||||
TRADING_BUNDLE_IDS,
|
||||
POLICY_DENIED_BUNDLE_IDS,
|
||||
INTRINSIC_DENIED_BUNDLE_IDS,
|
||||
BROWSER_NAME_SUBSTRINGS,
|
||||
TERMINAL_NAME_SUBSTRINGS,
|
||||
TRADING_NAME_SUBSTRINGS,
|
||||
POLICY_DENIED_NAME_SUBSTRINGS,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { COMPUTER_USE_INSTRUCTIONS } from './instructions.js'
|
||||
|
||||
describe('macOS action and observation guidance', () => {
|
||||
test('uses persistent JS for known actions without demanding a model round trip per click', () => {
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('Do not force one model round trip per click')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('then observe at a decision point')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('Do not add a fixed sleep before observing')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('Use copied `gN:id` handles')
|
||||
})
|
||||
|
||||
test('keeps standalone receipts and unknown paste results subject to observation', () => {
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('receiving a standalone dispatch receipt')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('Inspect a fresh observation')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('treat the result as unknown and call')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('stop and re-observe')
|
||||
})
|
||||
})
|
||||
-36
@@ -1,36 +0,0 @@
|
||||
import { isIntrinsicAppDenied } from './deniedApps.js'
|
||||
|
||||
// Official macOS SkyComputerUseService 26.831.1000926:
|
||||
// isForbiddenComputerUseTarget checks these four sets with exact equality.
|
||||
// Legacy cross-platform tiers and display-name matches do not apply here.
|
||||
export const NATIVE_FORBIDDEN_BUNDLE_IDS: ReadonlySet<string> = new Set([
|
||||
'com.apple.Terminal',
|
||||
'com.googlecode.iterm2',
|
||||
'org.alacritty',
|
||||
'dev.warp.Warp-Stable',
|
||||
'net.kovidgoyal.kitty',
|
||||
'co.zeit.hyper',
|
||||
'com.github.wez.wezterm',
|
||||
'org.tabby',
|
||||
'com.mitchellh.ghostty',
|
||||
'com.raphaelamorim.rio',
|
||||
'dev.commandline.waveterm',
|
||||
'com.openai.codex',
|
||||
'com.openai.codex.alpha',
|
||||
'com.openai.codex.beta',
|
||||
'com.openai.codex.dev',
|
||||
'com.openai.codex.nightly',
|
||||
'com.openai.chat',
|
||||
'com.openai.chat.alpha',
|
||||
'com.openai.chat.beta',
|
||||
'com.openai.chat.nightly',
|
||||
'com.openai.chat.mac-debug',
|
||||
'com.apple.UserNotificationCenter',
|
||||
'com.apple.LocalAuthenticationRemoteService',
|
||||
'com.apple.SecurityAgent',
|
||||
])
|
||||
|
||||
export function isNativeAppDenied(bundleId: string | undefined, hostBundleId?: string): boolean {
|
||||
return isIntrinsicAppDenied(bundleId, hostBundleId) ||
|
||||
(bundleId !== undefined && NATIVE_FORBIDDEN_BUNDLE_IDS.has(bundleId))
|
||||
}
|
||||
+106
@@ -736,4 +736,110 @@ describe('Computer Use platform routing', () => {
|
||||
await connection.close()
|
||||
}
|
||||
})
|
||||
|
||||
test('win32 retries a legacy clipboard stash restore without clearing clipboard by app', async () => {
|
||||
const calls: string[] = []
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
let stash: string | undefined = 'preserved clipboard'
|
||||
let clipboard = 'current clipboard'
|
||||
let writeAttempts = 0
|
||||
adapter.executor.getFrontmostApp = async () => ({
|
||||
bundleId: 'powershell.exe', displayName: 'PowerShell',
|
||||
})
|
||||
adapter.executor.readClipboard = async () => clipboard
|
||||
adapter.executor.writeClipboard = async text => {
|
||||
writeAttempts += 1
|
||||
if (writeAttempts === 1) throw new Error('clipboard temporarily busy')
|
||||
clipboard = text
|
||||
}
|
||||
const getSubGates = adapter.getSubGates
|
||||
adapter.getSubGates = () => ({ ...getSubGates(), clipboardGuard: true })
|
||||
const connection = await connect(adapter, makeSessionContext({
|
||||
getClipboardStash: () => stash,
|
||||
onClipboardStashChanged: value => { stash = value },
|
||||
getGrantFlags: () => ({ clipboardRead: true, clipboardWrite: true, systemKeyCombos: true }),
|
||||
}))
|
||||
try {
|
||||
expect((await connection.client.callTool({ name: 'read_clipboard' })).isError).toBeFalsy()
|
||||
expect(stash).toBe('preserved clipboard')
|
||||
expect(clipboard).toBe('current clipboard')
|
||||
expect((await connection.client.callTool({ name: 'read_clipboard' })).isError).toBeFalsy()
|
||||
expect(stash).toBeUndefined()
|
||||
expect(clipboard).toBe('preserved clipboard')
|
||||
expect(writeAttempts).toBe(2)
|
||||
|
||||
expect((await connection.client.callTool({
|
||||
name: 'write_clipboard', arguments: { text: 'new clipboard' },
|
||||
})).isError).toBeFalsy()
|
||||
expect((await connection.client.callTool({
|
||||
name: 'key', arguments: { text: 'ctrl+v' },
|
||||
})).isError).toBeFalsy()
|
||||
expect(clipboard).toBe('new clipboard')
|
||||
expect(stash).toBeUndefined()
|
||||
expect(writeAttempts).toBe(3)
|
||||
expect(calls).toContain('key:ctrl+v')
|
||||
} finally {
|
||||
await connection.close()
|
||||
}
|
||||
})
|
||||
|
||||
test.each([
|
||||
['chrome.exe', 'Google Chrome'],
|
||||
['powershell.exe', 'PowerShell'],
|
||||
['spotify.exe', 'Spotify'],
|
||||
['tradingview.exe', 'TradingView'],
|
||||
['com.example.host', 'Claude Code Haha'],
|
||||
['dev.cchaha.cu-helper', 'Computer Use Helper'],
|
||||
])('win32 global consent permits input, launch, and clipboard for %s', async (bundleId, displayName) => {
|
||||
const calls: string[] = []
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
const app = { bundleId, displayName }
|
||||
let frontmost = app
|
||||
let clipboard = 'existing clipboard'
|
||||
adapter.executor.getFrontmostApp = async () => frontmost
|
||||
adapter.executor.appUnderPoint = async () => app
|
||||
adapter.executor.listInstalledApps = async () => [{ ...app, path: `C:\\Apps\\${bundleId}` }]
|
||||
adapter.executor.readClipboard = async () => clipboard
|
||||
adapter.executor.writeClipboard = async text => { clipboard = text }
|
||||
const getSubGates = adapter.getSubGates
|
||||
adapter.getSubGates = () => ({ ...getSubGates(), clipboardGuard: true })
|
||||
const connection = await connect(adapter, makeSessionContext({
|
||||
// Cached state from the old app-specific model cannot narrow the
|
||||
// feature-wide consent given when Computer Use was enabled.
|
||||
getAllowedApps: () => [{ ...app, grantedAt: 1, tier: 'read' }],
|
||||
getUserDeniedBundleIds: () => [bundleId],
|
||||
getGrantFlags: () => ({ clipboardRead: true, clipboardWrite: true, systemKeyCombos: true }),
|
||||
}))
|
||||
try {
|
||||
for (const request of [
|
||||
{ name: 'screenshot' },
|
||||
{ name: 'open_application', arguments: { app: displayName } },
|
||||
{ name: 'key', arguments: { text: 'ctrl+a' } },
|
||||
{ name: 'type', arguments: { text: 'ok' } },
|
||||
{ name: 'read_clipboard' },
|
||||
]) {
|
||||
const result = await connection.client.callTool(request)
|
||||
expect(result.isError, `${request.name}: ${JSON.stringify(result.content)}`).toBeFalsy()
|
||||
}
|
||||
expect(clipboard).toBe('existing clipboard')
|
||||
expect((await connection.client.callTool({
|
||||
name: 'write_clipboard', arguments: { text: 'new clipboard' },
|
||||
})).isError).toBeFalsy()
|
||||
expect(clipboard).toBe('new clipboard')
|
||||
|
||||
// Coordinate clicks remain authorized regardless of which application
|
||||
// is foreground or owns the window receiving the click.
|
||||
frontmost = { bundleId: 'notepad.exe', displayName: 'Notepad' }
|
||||
expect((await connection.client.callTool({
|
||||
name: 'right_click', arguments: { coordinate: [10, 20] },
|
||||
})).isError).toBeFalsy()
|
||||
expect(calls).toContain(`openApp:${bundleId}`)
|
||||
expect(calls).toContain('key:ctrl+a')
|
||||
expect(calls).toContain('type:ok')
|
||||
expect(calls).toContain('click:10,20,right,1')
|
||||
expect(clipboard).toBe('new clipboard')
|
||||
} finally {
|
||||
await connection.close()
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
+104
-64
@@ -17,6 +17,8 @@ import {
|
||||
handleToolCall,
|
||||
resetMouseButtonHeld,
|
||||
} from './toolCalls.js'
|
||||
import { bindSessionContext } from './mcpServer.js'
|
||||
import type { ComputerUseSessionContext } from './types.js'
|
||||
import { buildComputerUseTools } from './tools.js'
|
||||
import { bindSessionContext } from './mcpServer.js'
|
||||
import { COMPUTER_USE_INSTRUCTIONS } from './instructions.js'
|
||||
@@ -224,7 +226,6 @@ describe('buildComputerUseTools — current Codex tool face', () => {
|
||||
'click',
|
||||
'drag',
|
||||
'get_app_state',
|
||||
'sequence',
|
||||
'list_apps',
|
||||
'perform_secondary_action',
|
||||
'paste',
|
||||
@@ -233,6 +234,7 @@ describe('buildComputerUseTools — current Codex tool face', () => {
|
||||
'select_text',
|
||||
'set_value',
|
||||
'type_text',
|
||||
'sequence',
|
||||
].sort(),
|
||||
)
|
||||
})
|
||||
@@ -515,6 +517,29 @@ describe('frameAppStateEnvelope', () => {
|
||||
])
|
||||
})
|
||||
|
||||
test('preserves declared JPEG and legacy PNG bytes with real decoded dimensions', async () => {
|
||||
const { default: sharp } = await import('sharp')
|
||||
for (const format of ['jpeg', 'png'] as const) {
|
||||
const bytes = await sharp({ create: { width: 96, height: 64, channels: 3, background: '#4070a0' } })
|
||||
.toFormat(format).toBuffer()
|
||||
const out = frameAppStateEnvelope({
|
||||
pid: 1, elementCount: 0, truncated: false, durationMs: 1, axText: 'App=x (pid 1)',
|
||||
screenshot: { base64: bytes.toString('base64'), width: 96, height: 64,
|
||||
...(format === 'jpeg' ? { mimeType: 'image/jpeg' as const } : {}) },
|
||||
})
|
||||
const [image] = imageBlocks(out)
|
||||
expect(image.mimeType).toBe(`image/${format}`)
|
||||
const returned = Buffer.from(image.data, 'base64')
|
||||
expect(returned.equals(bytes)).toBe(true)
|
||||
const metadata = await sharp(returned).metadata()
|
||||
expect(metadata.format).toBe(format)
|
||||
expect(metadata.width).toBe(96)
|
||||
expect(metadata.height).toBe(64)
|
||||
// Force pixel decoding too, not just signature/metadata recognition.
|
||||
expect((await sharp(returned).raw().toBuffer({ resolveWithObject: true })).info.width).toBe(96)
|
||||
}
|
||||
})
|
||||
|
||||
test('no image block when screenshot is absent (capture failed)', () => {
|
||||
const out = frameAppStateEnvelope({
|
||||
pid: 1, elementCount: 0, truncated: false, durationMs: 1, axText: 'App=x (pid 1)',
|
||||
@@ -592,47 +617,7 @@ describe('arg parsing helpers', () => {
|
||||
expect(() => _test.parsePoint({}, 'from', 'from_x', 'from_y')).toThrow()
|
||||
})
|
||||
|
||||
test('policyDenyMessage: terminals refused with Codex text, normal apps pass', () => {
|
||||
const msg = _test.policyDenyMessage({
|
||||
bundleId: 'com.googlecode.iterm2',
|
||||
displayName: 'iTerm2',
|
||||
}, 'com.googlecode.iterm2')
|
||||
expect(msg).toBe("Computer Use is not allowed to use the app 'com.googlecode.iterm2' for safety reasons.")
|
||||
expect(_test.policyDenyMessage({
|
||||
bundleId: 'com.apple.finder',
|
||||
displayName: 'Finder',
|
||||
})).toBeUndefined()
|
||||
})
|
||||
|
||||
test('native policy uses the official exact identities instead of legacy app categories or names', () => {
|
||||
for (const bundleId of ['com.microsoft.VSCode', 'com.apple.Music', 'com.spotify.client', 'com.tradingview.tradingviewapp.desktop', 'dev.test.Terminal']) {
|
||||
expect(_test.policyDenyMessage({ bundleId, displayName: 'Terminal Music Editor' })).toBeUndefined()
|
||||
}
|
||||
for (const bundleId of ['com.raphaelamorim.rio', 'dev.commandline.waveterm', 'com.openai.codex.beta', 'com.openai.chat.mac-debug', 'com.apple.SecurityAgent']) {
|
||||
expect(_test.policyDenyMessage({ bundleId, displayName: 'Fixture' })).toContain('not allowed')
|
||||
}
|
||||
})
|
||||
|
||||
test('policyDenyMessage permanently denies the host and helper while a host override only adds', () => {
|
||||
const customHostBundleId = 'com.example.custom-host'
|
||||
for (const bundleId of [
|
||||
'com.claude-code-haha.desktop',
|
||||
'dev.cchaha.cu-helper',
|
||||
customHostBundleId,
|
||||
]) {
|
||||
expect(_test.policyDenyMessage({
|
||||
bundleId,
|
||||
displayName: bundleId,
|
||||
}, bundleId, customHostBundleId)).toBe(
|
||||
`Computer Use is not allowed to use the app '${bundleId}' for safety reasons.`,
|
||||
)
|
||||
}
|
||||
|
||||
expect(_test.policyDenyMessage({
|
||||
bundleId: 'com.apple.TextEdit',
|
||||
displayName: 'TextEdit',
|
||||
}, 'TextEdit', customHostBundleId)).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -703,12 +688,11 @@ describe('handleToolCall — gates', () => {
|
||||
expect(acquired).toBe(0)
|
||||
})
|
||||
|
||||
test('safety denylist refuses a terminal before the engine', async () => {
|
||||
test('global enablement allows terminal state reads without per-app grants', async () => {
|
||||
const { engine, calls } = makeEngine()
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'get_app_state', { app: 'com.googlecode.iterm2' }, baseOverrides())
|
||||
expect(r.isError).toBe(true)
|
||||
expect(textOf(r)).toContain('for safety reasons')
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
expect(r.isError).toBeFalsy()
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'getAppState'])
|
||||
})
|
||||
|
||||
test('native browser actions follow the observed official Chrome App path with normal identity checks', async () => {
|
||||
@@ -729,7 +713,7 @@ describe('handleToolCall — gates', () => {
|
||||
}
|
||||
})
|
||||
|
||||
test('configured host bundle is refused before permission or engine dispatch', async () => {
|
||||
test('configured host bundle uses the same global consent as every app', async () => {
|
||||
const customHostBundleId = 'com.example.custom-host'
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async () => ({
|
||||
@@ -767,10 +751,9 @@ describe('handleToolCall — gates', () => {
|
||||
}),
|
||||
)
|
||||
|
||||
expect(r.isError).toBe(true)
|
||||
expect(r.telemetry?.error_kind).toBe('app_denied')
|
||||
expect(r.isError).toBeFalsy()
|
||||
expect(permissionRequests).toBe(0)
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'getAppState'])
|
||||
})
|
||||
|
||||
test('missing engine → feature_unavailable', async () => {
|
||||
@@ -1020,24 +1003,51 @@ describe('handleToolCall — gates', () => {
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'typeText'])
|
||||
})
|
||||
|
||||
test('the product denylist still blocks a resolved app before mutation', async () => {
|
||||
test.each([
|
||||
['com.google.Chrome', 'Google Chrome'],
|
||||
['com.claude-code-haha.desktop', 'Claude Code Haha'],
|
||||
['dev.cchaha.cu-helper', 'Computer Use Helper'],
|
||||
['com.test.host', 'Custom Host'],
|
||||
['com.googlecode.iterm2', 'iTerm2'],
|
||||
['com.microsoft.VSCode', 'Visual Studio Code'],
|
||||
['com.tradingview.tradingviewapp.desktop', 'TradingView'],
|
||||
['com.spotify.client', 'Spotify'],
|
||||
[undefined, 'Netflix'],
|
||||
[undefined, 'Windows Terminal'],
|
||||
])('global enablement permits reading and operating %s (%s) without app approval', async (bundleId, displayName) => {
|
||||
let permissionCalls = 0
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async () => ({
|
||||
pid: 900,
|
||||
bundleId: 'com.googlecode.iterm2',
|
||||
displayName: 'iTerm2',
|
||||
bundleId,
|
||||
displayName,
|
||||
launchTime: 1900,
|
||||
}),
|
||||
})
|
||||
const r = await handleToolCall(
|
||||
makeAdapter({ engine }),
|
||||
'type_text',
|
||||
{ app: 'iTerm2', text: 'blocked' },
|
||||
baseOverrides({ allowedApps: [] }),
|
||||
)
|
||||
const overrides = baseOverrides({
|
||||
allowedApps: [],
|
||||
userDeniedBundleIds: bundleId ? [bundleId] : [],
|
||||
onPermissionRequest: async () => {
|
||||
permissionCalls++
|
||||
throw new Error('global consent must not ask for per-app approval')
|
||||
},
|
||||
})
|
||||
for (const app of [displayName, ...(bundleId ? [bundleId] : []), `/Applications/${displayName}.app`, '900']) {
|
||||
for (const [name, args, method] of [
|
||||
['get_app_state', {}, 'getAppState'],
|
||||
['click', { x: 10, y: 20 }, 'click'],
|
||||
['type_text', { text: 'hello' }, 'typeText'],
|
||||
] as const) {
|
||||
calls.length = 0
|
||||
const r = await handleToolCall(makeAdapter({ engine }), name, { app, ...args }, overrides)
|
||||
|
||||
expect(r.isError).toBe(true)
|
||||
expect(r.telemetry?.error_kind).toBe('app_denied')
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
expect(r.isError).toBeFalsy()
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', method])
|
||||
const target = name === 'get_app_state' ? calls[1].args : (calls[1].args as { target: AppTarget }).target
|
||||
expect(target).toMatchObject({ pid: 900, expectedProcessIdentity: { pid: 900, launchTime: 1900 } })
|
||||
}
|
||||
}
|
||||
expect(permissionCalls).toBe(0)
|
||||
})
|
||||
|
||||
test('invalid app values return bad_args without resolving a target', async () => {
|
||||
@@ -1791,6 +1801,35 @@ describe('resetMouseButtonHeld', () => {
|
||||
// A sequence is a bounded set of existing native operations, not a script.
|
||||
describe('same-app sequence', () => {
|
||||
const steps = [{ tool: 'press_key', key: 's x 1 period 3 5 Return' }, { tool: 'click', x: 12, y: 24 }]
|
||||
test.each([
|
||||
'com.google.Chrome',
|
||||
'com.claude-code-haha.desktop',
|
||||
'dev.cchaha.cu-helper',
|
||||
'com.test.host',
|
||||
])('global consent also covers sequences targeting %s', async bundleId => {
|
||||
const { engine, calls } = makeEngine({
|
||||
getAppState: async () => ({
|
||||
pid: 1234, appName: bundleId, bundleId, elementCount: 0,
|
||||
truncated: false, durationMs: 1, axText: '',
|
||||
screenshot: { base64: 'PNG', width: 10, height: 10 },
|
||||
}),
|
||||
})
|
||||
const result = await handleToolCall(
|
||||
makeAdapter({ engine }), 'sequence', { app: bundleId, steps },
|
||||
baseOverrides({
|
||||
allowedApps: [], userDeniedBundleIds: [bundleId],
|
||||
onPermissionRequest: async () => { throw new Error('must not ask for app approval') },
|
||||
}),
|
||||
)
|
||||
expect(result.isError).toBeFalsy()
|
||||
expect(result.structuredContent).toMatchObject({ status: 'completed', completedSteps: 2 })
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'pressKey', 'click', 'getAppState'])
|
||||
const target = (calls[1].args as { target: AppTarget }).target
|
||||
expect(target.expectedProcessIdentity?.bundleId).toBe(bundleId)
|
||||
expect((calls[2].args as { target: AppTarget }).target).toEqual(target)
|
||||
expect(calls[3].args).toEqual(target)
|
||||
expect(imageBlocks(result)).toHaveLength(1)
|
||||
})
|
||||
test('runs in order against one proven identity and returns one final screenshot', async () => {
|
||||
const { engine, calls } = makeEngine({ getAppState: () => ({ pid: 1234, appName: 'Finder', bundleId: 'com.apple.finder', windowTitle: 'Docs', elementCount: 0, truncated: false, durationMs: 1, axText: '', screenshot: { base64: 'PNG', width: 10, height: 10 } }) })
|
||||
const result = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps }, baseOverrides())
|
||||
@@ -2005,12 +2044,13 @@ describe('canvas action batches', () => {
|
||||
expect(calls[3]!.args).toMatchObject({ from: { x: 100, y: 200 }, to: { x: 100, y: 200 } })
|
||||
})
|
||||
|
||||
test('a batch uses the existing resolved-target policy before any mutation', async () => {
|
||||
const { engine, calls } = makeEngine()
|
||||
test('a batch requires a proven process lifetime before any mutation', async () => {
|
||||
const { engine, calls } = makeEngine({ resolveTarget: async () => ({ pid: 1234, bundleId: 'com.test.host' }) })
|
||||
const result = await handleToolCall(makeAdapter({ engine }), 'sequence', {
|
||||
app: 'com.test.host', steps: [{ tool: 'click', x: 1, y: 2 }],
|
||||
}, baseOverrides())
|
||||
expect(result.telemetry?.error_kind).toBe('app_denied')
|
||||
expect(result.telemetry?.error_kind).toBe('executor_threw')
|
||||
expect(result.isError).toBe(true)
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
})
|
||||
|
||||
|
||||
+8
-48
@@ -21,12 +21,11 @@
|
||||
* before any mutation, because a bare pid can be recycled between the moment
|
||||
* we resolved it and the moment we act on it.
|
||||
*
|
||||
* **2. Safety checks happen after resolution, on the resolved identity.**
|
||||
* The model names an app loosely ("Notes", a path, a pid). `resolveTarget` is
|
||||
* the single seam that turns that into one real running process; every policy
|
||||
* check then runs against *that* process's bundle id, not against the string
|
||||
* the model typed. Checking the string instead would let "friendly alias"
|
||||
* walk past a denylist that names the bundle id.
|
||||
* **2. One global consent covers every app.** Once Computer Use is enabled
|
||||
* in Settings and its consent dialog is confirmed, no app category, bundle
|
||||
* id, display name, host identity, or legacy app grant may deny access.
|
||||
* `resolveTarget` identifies the actual process for dispatch and lifetime
|
||||
* checks; it is not a second app-authorization step.
|
||||
*
|
||||
* **3. Standalone mutations never take an implicit snapshot.** They return a fixed
|
||||
* receipt and the model calls `get_app_state` when it wants to see the result.
|
||||
@@ -44,9 +43,8 @@
|
||||
* 4. Global CU lock (`overrides.checkCuLock`).
|
||||
* 5. Engine presence.
|
||||
* 6. `resolveTarget` → one running process + its lifetime identity.
|
||||
* 7. Product/intrinsic denylists against the RESOLVED identity.
|
||||
* 8. Proven process lifetime — mutating tools only.
|
||||
* 9. Engine dispatch.
|
||||
* 7. Proven process lifetime — mutating tools only.
|
||||
* 8. Engine dispatch.
|
||||
*
|
||||
* The `<app_state>` envelope is framed here in TS. Swift renders the inner tree;
|
||||
* we wrap it in the version banner + optional <app_specific_instructions> +
|
||||
@@ -56,7 +54,6 @@
|
||||
|
||||
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
|
||||
|
||||
import { isNativeAppDenied } from './nativeAppPolicy.js'
|
||||
import { NATIVE_ERROR, NATIVE_SERVER_ERROR_CODES, toNativeErrorMetadata, type NativeErrorMetadata } from './nativeError.js'
|
||||
import type {
|
||||
AppStateResult,
|
||||
@@ -489,34 +486,6 @@ export function resetMouseButtonHeld(): void {
|
||||
/* no cross-call mouse state in the semantic engine */
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Native app policy
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Refusal check against a RESOLVED target — the real bundle id and display
|
||||
* name of the process we are about to drive, not the string the model typed.
|
||||
*
|
||||
* Official native forbidden identities plus our own app and helper. The
|
||||
* resolved bundle ID is authoritative; display-name substrings are not policy.
|
||||
*
|
||||
* `requestedApp` only shapes the message — the model should see the name it
|
||||
* used. `hostBundleId` extends the intrinsic set for builds whose bundle id
|
||||
* differs from the shipped default.
|
||||
*/
|
||||
function policyDenyMessage(
|
||||
resolved: { bundleId?: string; displayName?: string },
|
||||
requestedApp?: string,
|
||||
hostBundleId?: string,
|
||||
): string | undefined {
|
||||
const bundleId = resolved.bundleId;
|
||||
const displayName = resolved.displayName ?? bundleId ?? requestedApp ?? "";
|
||||
if (!isNativeAppDenied(bundleId, hostBundleId)) return undefined;
|
||||
const shown = requestedApp ?? displayName ?? bundleId ?? "";
|
||||
// Preserve the existing product refusal message.
|
||||
return `Computer Use is not allowed to use the app '${shown}' for safety reasons.`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Apps whose `<app_specific_instructions>` block has already been delivered.
|
||||
*
|
||||
@@ -1086,15 +1055,7 @@ export async function handleToolCall(
|
||||
const resolved = await engine.resolveTarget(request.target);
|
||||
const requestedApp = request.requestedApp ?? "";
|
||||
|
||||
// ─── Gate 7: denylists, against the RESOLVED identity ──────────────
|
||||
const denied = policyDenyMessage(
|
||||
resolved,
|
||||
requestedApp,
|
||||
adapter.executor.capabilities.hostBundleId,
|
||||
);
|
||||
if (denied) return errorResult(denied, "app_denied");
|
||||
|
||||
// ─── Gate 8: proven process lifetime (mutations only) ──────────────
|
||||
// ─── Gate 7: proven process lifetime (mutations only) ──────────────
|
||||
if (request.mutating && !provenIdentity(resolved)) {
|
||||
return errorResult(
|
||||
`Resolving '${requestedApp}' did not prove the running process ` +
|
||||
@@ -1278,7 +1239,6 @@ export const _test = {
|
||||
parseDirection,
|
||||
parseTarget,
|
||||
parsePoint,
|
||||
policyDenyMessage,
|
||||
provenIdentity,
|
||||
dispatchTarget,
|
||||
CUA_APP_VERSION,
|
||||
|
||||
+27
-459
@@ -12,7 +12,7 @@
|
||||
* For input actions (click/type/key/scroll/drag/move_mouse) the tool-specific
|
||||
* gates are, in order:
|
||||
* a. `prepareForAction` — platform preparation and host defocus.
|
||||
* b. Resolve the frontmost app and apply product/intrinsic safety tiers.
|
||||
* b. Resolve the foreground application before dispatching input.
|
||||
*
|
||||
* For click variants only, AFTER the above gates but BEFORE the executor call:
|
||||
* c. Pixel-validation staleness check (sub-gated).
|
||||
@@ -50,7 +50,6 @@ import type {
|
||||
* Finder is never hidden by the hide loop (hiding Finder kills the Desktop),
|
||||
* so it's always a valid frontmost.
|
||||
*/
|
||||
const FINDER_BUNDLE_ID = "com.apple.finder";
|
||||
|
||||
/**
|
||||
* Categorical error classes for the cu_tool_call telemetry event. Never
|
||||
@@ -318,82 +317,19 @@ function tierSatisfies(
|
||||
return tier === "click" || tier === "full";
|
||||
}
|
||||
|
||||
function automaticTier(bundleId: string | undefined, displayName: string): CuAppPermTier {
|
||||
return getDefaultTierForApp(bundleId, displayName);
|
||||
}
|
||||
|
||||
// Appended to every tier_insufficient error. The model may try to route
|
||||
// around the gate (osascript, System Events, cliclick via Bash) — this
|
||||
// closes that door explicitly. Leading space so it concatenates cleanly.
|
||||
const TIER_ANTI_SUBVERSION =
|
||||
" Do not attempt to work around this restriction — never use AppleScript, " +
|
||||
"System Events, shell commands, or any other method to send clicks or " +
|
||||
"keystrokes to this app.";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Clipboard guard — stash+clear while a click-tier app is frontmost
|
||||
// ---------------------------------------------------------------------------
|
||||
//
|
||||
// Threat: tier "click" blocks type/key/right-click-Paste, but a click-tier
|
||||
// terminal/IDE may have a UI Paste button that's plain-left-clickable. If the
|
||||
// clipboard holds `rm -rf /` — from the user, from a prior full-tier paste,
|
||||
// OR from the agent's own write_clipboard call (which doesn't route through
|
||||
// runInputActionGates) — a left_click on that button injects it.
|
||||
//
|
||||
// Mitigation: stash the user's clipboard on first entry to click-tier, then
|
||||
// RE-CLEAR before every input action while click-tier stays frontmost. The
|
||||
// re-clear is the load-bearing part — a stash-on-transition-only design
|
||||
// leaves a gap between an agent write_clipboard and the next left_click.
|
||||
// When frontmost becomes anything else, restore. Turn-end restore is inlined
|
||||
// in the host's result-handler + leavingRunning (same dual-location as
|
||||
// cuHiddenDuringTurn unhide) — reads `session.cuClipboardStash` directly and
|
||||
// writes via Electron's `clipboard.writeText`, so no nest-only import.
|
||||
//
|
||||
// State lives on the session (via `overrides.getClipboardStash` /
|
||||
// `onClipboardStashChanged`), not module-level. The CU lock still guarantees
|
||||
// one session at a time, but session-scoped state means the host's turn-end
|
||||
// restore doesn't need to reach back into this package.
|
||||
|
||||
async function syncClipboardStash(
|
||||
/** Restore a clipboard stash from the former app-tier guard without creating
|
||||
* new restrictions based on the foreground application's identity. */
|
||||
async function restoreLegacyClipboardStash(
|
||||
adapter: ComputerUseHostAdapter,
|
||||
overrides: ComputerUseOverrides,
|
||||
frontmostIsClickTier: boolean,
|
||||
): Promise<void> {
|
||||
const current = overrides.getClipboardStash?.();
|
||||
if (!frontmostIsClickTier) {
|
||||
// Restore + clear. Idempotent — if nothing is stashed, no-op.
|
||||
if (current === undefined) return;
|
||||
try {
|
||||
await adapter.executor.writeClipboard(current);
|
||||
// Clear only after a successful write — a transient pasteboard
|
||||
// failure must not irrecoverably drop the stash.
|
||||
overrides.onClipboardStashChanged?.(undefined);
|
||||
} catch {
|
||||
// Best effort — stash held, next non-click action retries.
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Stash the user's clipboard on FIRST entry to click-tier only.
|
||||
if (current === undefined) {
|
||||
try {
|
||||
const read = await adapter.executor.readClipboard();
|
||||
overrides.onClipboardStashChanged?.(read);
|
||||
} catch {
|
||||
// readClipboard failed — use empty sentinel so we don't retry the stash
|
||||
// on the next action; restore becomes a harmless writeClipboard("").
|
||||
overrides.onClipboardStashChanged?.("");
|
||||
}
|
||||
}
|
||||
// Re-clear on EVERY click-tier action, not just the first. Defeats the
|
||||
// bypass where the agent calls write_clipboard (which doesn't route
|
||||
// through runInputActionGates) between stash and a left_click on a UI
|
||||
// Paste button — the next action's clear clobbers the agent's write
|
||||
// before the click lands.
|
||||
const current = overrides.getClipboardStash?.()
|
||||
if (current === undefined) return
|
||||
try {
|
||||
await adapter.executor.writeClipboard("");
|
||||
await adapter.executor.writeClipboard(current)
|
||||
overrides.onClipboardStashChanged?.(undefined)
|
||||
} catch {
|
||||
// Transient pasteboard failure. The tier-"click" right-click/modifier
|
||||
// block still holds; this is a net, not a promise.
|
||||
// Preserve the stash so a transient clipboard failure can be retried.
|
||||
}
|
||||
}
|
||||
|
||||
@@ -404,7 +340,6 @@ async function runInputActionGates(
|
||||
adapter: ComputerUseHostAdapter,
|
||||
overrides: ComputerUseOverrides,
|
||||
subGates: CuSubGates,
|
||||
actionKind: CuActionKind,
|
||||
): Promise<CuCallToolResult | null> {
|
||||
// Windows shows the full desktop. Preparation gets an empty filter so it may
|
||||
// perform platform bookkeeping without hiding apps based on an allowlist.
|
||||
@@ -432,147 +367,11 @@ async function runInputActionGates(
|
||||
);
|
||||
}
|
||||
|
||||
const { hostBundleId } = adapter.executor.capabilities;
|
||||
|
||||
if (frontmost.bundleId === hostBundleId) {
|
||||
if (actionKind !== "keyboard") return null;
|
||||
return errorResult(
|
||||
"Claude's own window still has keyboard focus. Click on the target " +
|
||||
"application first so typing cannot land in the chat box.",
|
||||
"state_conflict",
|
||||
);
|
||||
}
|
||||
|
||||
if (isPolicyDenied(frontmost.bundleId, frontmost.displayName)) {
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is not supported by Computer Use for product-safety reasons.`,
|
||||
"app_denied",
|
||||
);
|
||||
}
|
||||
|
||||
const frontmostTier = automaticTier(frontmost.bundleId, frontmost.displayName);
|
||||
|
||||
// Feature-wide consent covers every identified app, including our host.
|
||||
if (subGates.clipboardGuard) {
|
||||
await syncClipboardStash(adapter, overrides, frontmostTier === "click");
|
||||
await restoreLegacyClipboardStash(adapter, overrides)
|
||||
}
|
||||
|
||||
if (!tierSatisfies(frontmostTier, actionKind)) {
|
||||
if (frontmostTier === "read") {
|
||||
// tier "read" is not category-unique (browser AND trading map to it) —
|
||||
// re-look-up so the CiC hint only shows for actual browsers.
|
||||
const isBrowser =
|
||||
getDeniedCategoryForApp(frontmost.bundleId, frontmost.displayName) ===
|
||||
"browser";
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is restricted to tier "read" — ` +
|
||||
`visible in screenshots only, no clicks or typing.` +
|
||||
(isBrowser
|
||||
? " Use the Claude-in-Chrome MCP for browser interaction (tools " +
|
||||
"named `mcp__Claude_in_Chrome__*`; load via ToolSearch if " +
|
||||
"deferred)."
|
||||
: " No interaction is permitted; ask the user to take any " +
|
||||
"actions in this app themselves.") +
|
||||
TIER_ANTI_SUBVERSION,
|
||||
"tier_insufficient",
|
||||
);
|
||||
}
|
||||
if (actionKind === "keyboard") {
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is restricted to tier "click" — ` +
|
||||
`typing, key presses, and paste require tier "full". The keys ` +
|
||||
`would go to this app's text fields or integrated terminal. To ` +
|
||||
`type into a different app, click it first to bring it forward. ` +
|
||||
`For shell commands, use the Bash tool.` + TIER_ANTI_SUBVERSION,
|
||||
"tier_insufficient",
|
||||
);
|
||||
}
|
||||
// actionKind === "mouse_full" ("mouse" and "mouse_position" pass at "click")
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is restricted to tier "click" — ` +
|
||||
`right-click, middle-click, and clicks with modifier keys require ` +
|
||||
`tier "full". Right-click opens a context menu with Paste/Cut, and ` +
|
||||
`modifier chords fire as keystrokes before the click. Plain ` +
|
||||
`left_click is allowed here.` + TIER_ANTI_SUBVERSION,
|
||||
"tier_insufficient",
|
||||
);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hit-test gate: reject a mouse action if the window under (x, y) belongs
|
||||
* to an app whose tier doesn't cover mouse input. Closes the gap where a
|
||||
* tier-"full" app is frontmost but the click lands on a tier-"read" window
|
||||
* overlapping it — `runInputActionGates` passes (frontmost is fine), but the
|
||||
* click actually goes to the read-tier app.
|
||||
*
|
||||
* Runs AFTER `scaleCoord` (needs global coords) and BEFORE the executor call.
|
||||
* Returns null on pass (target is tier-"click"/"full", or desktop/Finder/us),
|
||||
* error-result on block.
|
||||
*
|
||||
* When `appUnderPoint` returns null (desktop, or platform without hit-test),
|
||||
* falls through — the frontmost check in `runInputActionGates` already ran.
|
||||
*/
|
||||
async function runHitTestGate(
|
||||
adapter: ComputerUseHostAdapter,
|
||||
overrides: ComputerUseOverrides,
|
||||
subGates: CuSubGates,
|
||||
x: number,
|
||||
y: number,
|
||||
actionKind: CuActionKind,
|
||||
): Promise<CuCallToolResult | null> {
|
||||
const target = await adapter.executor.appUnderPoint(x, y);
|
||||
if (!target) return null; // desktop / nothing under point / platform no-op
|
||||
|
||||
// Finder (desktop, file dialogs) and our click-through overlay are valid.
|
||||
if (target.bundleId === FINDER_BUNDLE_ID) return null;
|
||||
if (target.bundleId === adapter.executor.capabilities.hostBundleId) return null;
|
||||
if (isPolicyDenied(target.bundleId, target.displayName)) {
|
||||
return errorResult(
|
||||
`Click at these coordinates would land on "${target.displayName}", ` +
|
||||
"which Computer Use does not support for product-safety reasons.",
|
||||
"app_denied",
|
||||
);
|
||||
}
|
||||
|
||||
const targetTier = automaticTier(target.bundleId, target.displayName);
|
||||
|
||||
// Frontmost-based sync (runInputActionGates) misses the case where
|
||||
// the click lands on a NON-FRONTMOST click-tier window. Re-sync by
|
||||
// the hit-test target's tier — if target is click-tier, stash+clear
|
||||
// before the click lands, regardless of what's frontmost.
|
||||
if (subGates.clipboardGuard && targetTier === "click") {
|
||||
await syncClipboardStash(adapter, overrides, true);
|
||||
}
|
||||
|
||||
if (tierSatisfies(targetTier, actionKind)) return null;
|
||||
|
||||
// Target is in the allowlist but tier doesn't cover this action.
|
||||
// runHitTestGate is only called with mouse/mouse_full (keyboard routes to
|
||||
// frontmost, not window-under-cursor). The branch above catches
|
||||
// mouse_full ∧ click; the only remaining fall-through is tier "read".
|
||||
if (actionKind === "mouse_full" && targetTier === "click") {
|
||||
return errorResult(
|
||||
`Click at these coordinates would land on "${target.displayName}", ` +
|
||||
`which is restricted to tier "click" — right-click, middle-click, and ` +
|
||||
`clicks with modifier keys require tier "full" (they can Paste via ` +
|
||||
`the context menu or fire modifier-chord keystrokes). Plain ` +
|
||||
`left_click is allowed here.` + TIER_ANTI_SUBVERSION,
|
||||
"tier_insufficient",
|
||||
);
|
||||
}
|
||||
const isBrowser =
|
||||
getDeniedCategoryForApp(target.bundleId, target.displayName) === "browser";
|
||||
return errorResult(
|
||||
`Click at these coordinates would land on "${target.displayName}", ` +
|
||||
`which is restricted to tier "read" (screenshots only, no interaction). ` +
|
||||
(isBrowser
|
||||
? "Use the Claude-in-Chrome MCP for browser interaction."
|
||||
: "Ask the user to take any actions in this app themselves.") +
|
||||
TIER_ANTI_SUBVERSION,
|
||||
"tier_insufficient",
|
||||
);
|
||||
return null
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -691,17 +490,12 @@ export async function releaseHeldMouseForSession(
|
||||
): Promise<boolean> {
|
||||
return releaseHeldMouse(adapter, owner)
|
||||
}
|
||||
/** Whether mouse_move occurred between left_mouse_down and left_mouse_up.
|
||||
* When false at mouseUp, the decomposed sequence is a click-release (not a
|
||||
* drop) — hit-test at "mouse", not "mouse_full". */
|
||||
let mouseMoved = false;
|
||||
|
||||
/** Clears the cross-call drag flags. Called from Gate-3 on lock-acquire and
|
||||
* from `bindSessionContext` in mcpServer.ts — a fresh lock holder must not
|
||||
* inherit a prior session's mid-drag state. */
|
||||
export function resetMouseButtonHeld(): void {
|
||||
mouseButtonHeld = false;
|
||||
mouseMoved = false;
|
||||
mouseButtonOwner = undefined
|
||||
mouseHoldGeneration += 1
|
||||
}
|
||||
@@ -1265,9 +1059,7 @@ function buildTierGuidanceMessage(tiered: TieredApp[]): string {
|
||||
}
|
||||
|
||||
if (parts.length === 0) return "";
|
||||
// Same anti-subversion clause the gate errors carry — said upfront so the
|
||||
// model doesn't reach for osascript/cliclick after seeing "no clicks/typing".
|
||||
return parts.join("\n\n") + TIER_ANTI_SUBVERSION;
|
||||
return parts.join("\n\n");
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -2196,14 +1988,7 @@ async function handleClickVariant(
|
||||
button: "left" | "right" | "middle",
|
||||
count: 1 | 2 | 3,
|
||||
): Promise<CuCallToolResult> {
|
||||
// A prior left_mouse_down may have set mouseButtonHeld without a matching
|
||||
// left_mouse_up (e.g. drag rejected by a tier gate, model falls back to
|
||||
// left_click). executor.click() does its own mouseDown+mouseUp, releasing
|
||||
// the OS button — but without this, the JS flag stays true and all
|
||||
// subsequent mouse_move calls take the held-button path ("mouse"/
|
||||
// "mouse_full" actionKind + hit-test), causing spurious rejections on
|
||||
// click-tier and read-tier windows. Release first so click() gets a clean
|
||||
// slate.
|
||||
// Release a previous unmatched mouseDown before issuing an atomic click.
|
||||
if (mouseButtonHeld) {
|
||||
await releaseHeldMouse(adapter, mouseOwner(overrides));
|
||||
}
|
||||
@@ -2236,19 +2021,10 @@ async function handleClickVariant(
|
||||
modifiers = parseKeyChord(args.text);
|
||||
}
|
||||
|
||||
// Right/middle-click and any click with a modifier chord escalate to
|
||||
// keyboard-equivalent input at tier "click" (context-menu Paste, chord
|
||||
// keystrokes). Compute once, pass to both gates.
|
||||
const clickActionKind: CuActionKind =
|
||||
button !== "left" || (modifiers !== undefined && modifiers.length > 0)
|
||||
? "mouse_full"
|
||||
: "mouse";
|
||||
|
||||
const gate = await runInputActionGates(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
clickActionKind,
|
||||
);
|
||||
if (gate) return gate;
|
||||
|
||||
@@ -2302,16 +2078,6 @@ async function handleClickVariant(
|
||||
adapter.logger,
|
||||
);
|
||||
|
||||
const hitGate = await runHitTestGate(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
x,
|
||||
y,
|
||||
clickActionKind,
|
||||
);
|
||||
if (hitGate) return hitGate;
|
||||
|
||||
await adapter.executor.click(x, y, button, count, modifiers);
|
||||
return okText("Clicked.");
|
||||
}
|
||||
@@ -2329,7 +2095,6 @@ async function handleType(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
"keyboard",
|
||||
);
|
||||
if (gate) return gate;
|
||||
|
||||
@@ -2452,7 +2217,6 @@ async function handleKey(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
"keyboard",
|
||||
);
|
||||
if (gate) return gate;
|
||||
|
||||
@@ -2490,7 +2254,7 @@ async function handleScroll(
|
||||
const dx = dir === "left" ? -amount : dir === "right" ? amount : 0;
|
||||
const dy = dir === "up" ? -amount : dir === "down" ? amount : 0;
|
||||
|
||||
const gate = await runInputActionGates(adapter, overrides, subGates, "mouse");
|
||||
const gate = await runInputActionGates(adapter, overrides, subGates);
|
||||
if (gate) return gate;
|
||||
|
||||
const display = await adapter.executor.getDisplaySize(
|
||||
@@ -2505,23 +2269,6 @@ async function handleScroll(
|
||||
adapter.logger,
|
||||
);
|
||||
|
||||
// When the button is held, executor.scroll's internal moveMouse generates
|
||||
// a leftMouseDragged event (enigo reads NSEvent.pressedMouseButtons) —
|
||||
// same mechanism as handleMoveMouse's held-button path. Upgrade the
|
||||
// hit-test to "mouse_full" so scroll can't be used to drag-drop text onto
|
||||
// a click-tier terminal, and mark mouseMoved so the subsequent
|
||||
// left_mouse_up hit-tests as a drop not a click-release.
|
||||
const hitGate = await runHitTestGate(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
x,
|
||||
y,
|
||||
mouseButtonHeld ? "mouse_full" : "mouse",
|
||||
);
|
||||
if (hitGate) return hitGate;
|
||||
if (mouseButtonHeld) mouseMoved = true;
|
||||
|
||||
await adapter.executor.scroll(x, y, dx, dy);
|
||||
return okText("Scrolled.");
|
||||
}
|
||||
@@ -2558,7 +2305,7 @@ async function handleDrag(
|
||||
}
|
||||
// else: rawFrom stays undefined → executor drags from current cursor.
|
||||
|
||||
const gate = await runInputActionGates(adapter, overrides, subGates, "mouse");
|
||||
const gate = await runInputActionGates(adapter, overrides, subGates);
|
||||
if (gate) return gate;
|
||||
|
||||
const display = await adapter.executor.getDisplaySize(
|
||||
@@ -2584,35 +2331,6 @@ async function handleDrag(
|
||||
adapter.logger,
|
||||
);
|
||||
|
||||
// Check both drag endpoints. `from` is where the mouseDown happens (picks
|
||||
// up), `to` is where mouseUp happens (drops). When start_coordinate is
|
||||
// omitted the drag begins at the cursor — same bypass as mouse_move →
|
||||
// left_mouse_down, so read the cursor and hit-test it (mirrors
|
||||
// handleLeftMouseDown).
|
||||
//
|
||||
// The `to` endpoint uses "mouse_full" (not "mouse"): dropping text onto a
|
||||
// terminal inserts it as if typed (macOS text drag-drop). Same threat as
|
||||
// right-click→Paste. `from` stays "mouse" — picking up is a read.
|
||||
const fromPoint = from ?? (await adapter.executor.getCursorPosition());
|
||||
const fromGate = await runHitTestGate(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
fromPoint.x,
|
||||
fromPoint.y,
|
||||
"mouse",
|
||||
);
|
||||
if (fromGate) return fromGate;
|
||||
const toGate = await runHitTestGate(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
to.x,
|
||||
to.y,
|
||||
"mouse_full",
|
||||
);
|
||||
if (toGate) return toGate;
|
||||
|
||||
await adapter.executor.drag(from, to);
|
||||
return okText("Dragged.");
|
||||
}
|
||||
@@ -2627,17 +2345,10 @@ async function handleMoveMouse(
|
||||
if (coord instanceof Error) return errorResult(coord.message, "bad_args");
|
||||
const [rawX, rawY] = coord;
|
||||
|
||||
// When the button is held, moveMouse generates leftMouseDragged events on
|
||||
// the window under the cursor — that's interaction, not positioning.
|
||||
// Upgrade to "mouse" and hit-test the destination. When the button is NOT
|
||||
// held: pure positioning, passes at any tier, no hit-test (mouseDown/Up
|
||||
// hit-test the cursor to close the mouse_move→left_mouse_down decomposition).
|
||||
const actionKind: CuActionKind = mouseButtonHeld ? "mouse" : "mouse_position";
|
||||
const gate = await runInputActionGates(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
actionKind,
|
||||
);
|
||||
if (gate) return gate;
|
||||
|
||||
@@ -2653,23 +2364,7 @@ async function handleMoveMouse(
|
||||
adapter.logger,
|
||||
);
|
||||
|
||||
if (mouseButtonHeld) {
|
||||
// "mouse_full" — same as left_click_drag's to-endpoint. Dragging onto a
|
||||
// click-tier terminal is text injection regardless of which primitive
|
||||
// (atomic drag vs. decomposed down/move/up) delivers the events.
|
||||
const hitGate = await runHitTestGate(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
x,
|
||||
y,
|
||||
"mouse_full",
|
||||
);
|
||||
if (hitGate) return hitGate;
|
||||
}
|
||||
|
||||
await adapter.executor.moveMouse(x, y);
|
||||
if (mouseButtonHeld) mouseMoved = true;
|
||||
return okText("Moved.");
|
||||
}
|
||||
|
||||
@@ -2698,17 +2393,6 @@ async function handleOpenApplication(
|
||||
);
|
||||
}
|
||||
|
||||
if (isPolicyDenied(match.bundleId, match.displayName)) {
|
||||
return errorResult(
|
||||
`"${match.displayName}" is not supported by Computer Use for product-safety reasons.`,
|
||||
"app_denied",
|
||||
);
|
||||
}
|
||||
|
||||
// open_application works at any tier — bringing an app forward is exactly
|
||||
// what tier "read" enables (you need it on screen to screenshot it). The
|
||||
// tier gates on click/type catch any follow-up interaction.
|
||||
|
||||
await adapter.executor.openApp(match.bundleId);
|
||||
|
||||
// On multi-monitor setups, macOS may place the opened window on a monitor
|
||||
@@ -2817,19 +2501,10 @@ async function handleReadClipboard(
|
||||
);
|
||||
}
|
||||
|
||||
// read_clipboard doesn't route through runInputActionGates — sync here so
|
||||
// reading after clicking into a click-tier app sees the cleared clipboard
|
||||
// (same as what the app's own Paste would see).
|
||||
if (subGates.clipboardGuard) {
|
||||
const frontmost = await adapter.executor.getFrontmostApp();
|
||||
const frontmostTier = frontmost
|
||||
? automaticTier(frontmost.bundleId, frontmost.displayName)
|
||||
: undefined;
|
||||
await syncClipboardStash(adapter, overrides, frontmostTier === "click");
|
||||
await restoreLegacyClipboardStash(adapter, overrides)
|
||||
}
|
||||
|
||||
// clipboardGuard may have stashed+cleared — read the actual (possibly
|
||||
// empty) clipboard. The agent sees what the app would see.
|
||||
const text = await adapter.executor.readClipboard();
|
||||
return okJson({ text });
|
||||
}
|
||||
@@ -2850,32 +2525,7 @@ async function handleWriteClipboard(
|
||||
if (text instanceof Error) return errorResult(text.message, "bad_args");
|
||||
|
||||
if (subGates.clipboardGuard) {
|
||||
const frontmost = await adapter.executor.getFrontmostApp();
|
||||
const frontmostTier = frontmost
|
||||
? automaticTier(frontmost.bundleId, frontmost.displayName)
|
||||
: undefined;
|
||||
|
||||
// Defense-in-depth for the clipboardGuard bypass: write_clipboard +
|
||||
// left_click on a click-tier app's UI Paste button. The re-clear in
|
||||
// syncClipboardStash already defeats it (the next action clobbers the
|
||||
// write), but rejecting here gives the agent a clear signal instead of
|
||||
// silently voiding its write.
|
||||
if (frontmost && frontmostTier === "click") {
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is a tier-"click" app and currently ` +
|
||||
`frontmost. write_clipboard is blocked because the next action ` +
|
||||
`would clear the clipboard anyway — a UI Paste button in this ` +
|
||||
`app cannot be used to inject text. Bring a tier-"full" app ` +
|
||||
`forward before writing to the clipboard.` +
|
||||
TIER_ANTI_SUBVERSION,
|
||||
"tier_insufficient",
|
||||
);
|
||||
}
|
||||
|
||||
// write_clipboard doesn't route through runInputActionGates — sync here
|
||||
// so clicking away from a click-tier app then writing restores the user's
|
||||
// stash before the agent's text lands.
|
||||
await syncClipboardStash(adapter, overrides, frontmostTier === "click");
|
||||
await restoreLegacyClipboardStash(adapter, overrides)
|
||||
}
|
||||
|
||||
await adapter.executor.writeClipboard(text);
|
||||
@@ -3002,7 +2652,6 @@ async function handleHoldKey(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
"keyboard",
|
||||
);
|
||||
if (gate) return gate;
|
||||
|
||||
@@ -3027,28 +2676,11 @@ async function handleLeftMouseDown(
|
||||
);
|
||||
}
|
||||
|
||||
const gate = await runInputActionGates(adapter, overrides, subGates, "mouse");
|
||||
const gate = await runInputActionGates(adapter, overrides, subGates);
|
||||
if (gate) return gate;
|
||||
|
||||
// macOS routes mouseDown to the window under the cursor, not the frontmost
|
||||
// app. Without this hit-test, mouse_move (positioning, passes at any tier)
|
||||
// + left_mouse_down decomposes a click that lands on a tier-"read" window
|
||||
// overlapping a tier-"full" frontmost app — bypassing runHitTestGate's
|
||||
// whole purpose. All three are batchable, so the bypass is atomic.
|
||||
const cursor = await adapter.executor.getCursorPosition();
|
||||
const hitGate = await runHitTestGate(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
cursor.x,
|
||||
cursor.y,
|
||||
"mouse",
|
||||
);
|
||||
if (hitGate) return hitGate;
|
||||
|
||||
await adapter.executor.mouseDown();
|
||||
mouseButtonHeld = true;
|
||||
mouseMoved = false;
|
||||
mouseButtonOwner = mouseOwner(overrides)
|
||||
mouseHoldGeneration += 1
|
||||
return okText("Mouse button pressed.");
|
||||
@@ -3063,16 +2695,8 @@ async function handleLeftMouseUp(
|
||||
overrides: ComputerUseOverrides,
|
||||
subGates: CuSubGates,
|
||||
): Promise<CuCallToolResult> {
|
||||
// Any gate rejection here must release the button FIRST — otherwise the
|
||||
// OS button stays pressed and mouseButtonHeld stays true. Recovery
|
||||
// attempts (mouse_move back to a safe app) would generate leftMouseDragged
|
||||
// events into whatever window is under the cursor, including the very
|
||||
// read-tier window the gate was protecting. A single mouseUp on a
|
||||
// restricted window is one event; a stuck button is cascading damage.
|
||||
//
|
||||
// This includes the frontmost gate: focus can change between mouseDown and
|
||||
// mouseUp (something else grabbed focus), in which case runInputActionGates
|
||||
// rejects here even though it passed at mouseDown.
|
||||
// Always release a held button when the foreground target becomes unknown,
|
||||
// so a failed action cannot leave the user's mouse stuck down.
|
||||
const releaseFirst = async (
|
||||
err: CuCallToolResult,
|
||||
): Promise<CuCallToolResult> => {
|
||||
@@ -3081,27 +2705,9 @@ async function handleLeftMouseUp(
|
||||
return err;
|
||||
};
|
||||
|
||||
const gate = await runInputActionGates(adapter, overrides, subGates, "mouse");
|
||||
const gate = await runInputActionGates(adapter, overrides, subGates);
|
||||
if (gate) return releaseFirst(gate);
|
||||
|
||||
// When the cursor moved since mouseDown, this is a drop (text-injection
|
||||
// vector) — hit-test at "mouse_full" same as left_click_drag's `to`. When
|
||||
// NO move happened, this is a click-release — same semantics as the atomic
|
||||
// left_click, hit-test at "mouse". Without this distinction, a decomposed
|
||||
// click on a click-tier app fails here while the atomic left_click works,
|
||||
// and releaseFirst fires mouseUp anyway so the OS sees a complete click
|
||||
// while the model gets a misleading error.
|
||||
const cursor = await adapter.executor.getCursorPosition();
|
||||
const hitGate = await runHitTestGate(
|
||||
adapter,
|
||||
overrides,
|
||||
subGates,
|
||||
cursor.x,
|
||||
cursor.y,
|
||||
mouseMoved ? "mouse_full" : "mouse",
|
||||
);
|
||||
if (hitGate) return releaseFirst(hitGate);
|
||||
|
||||
await adapter.executor.mouseUp();
|
||||
resetMouseButtonHeld();
|
||||
return okText("Mouse button released.");
|
||||
@@ -3150,10 +2756,9 @@ interface BatchActionResult {
|
||||
* - Kill-switch + TCC: checked ONCE by handleToolCall before reaching here.
|
||||
* - prepareForAction: run ONCE at the top. The user approved "do this
|
||||
* sequence"; hiding apps per-action is wasted work and fast-pathed anyway.
|
||||
* - Frontmost gate: checked PER ACTION. State can change mid-batch — a
|
||||
* click might open a non-allowed app. This is the safety net: if action
|
||||
* 3 of 5 opened Safari (not allowed), action 4's frontmost check fires
|
||||
* and stops the batch there.
|
||||
* - Foreground identification: checked PER ACTION. State can change
|
||||
* mid-batch; if the foreground application can no longer be identified,
|
||||
* the next input action stops the batch.
|
||||
* - PixelCompare: SKIPPED inside batch. The model committed to the full
|
||||
* sequence without intermediate screenshots; validating mid-batch clicks
|
||||
* against a pre-batch screenshot would false-positive constantly.
|
||||
@@ -3376,45 +2981,8 @@ export async function handleToolCall(
|
||||
): Promise<CuCallToolResult> {
|
||||
const { logger, serverName } = adapter;
|
||||
|
||||
// Normalize the allowlist before any gate runs:
|
||||
//
|
||||
// (a) Strip user-denied. A grant from a previous session (before the user
|
||||
// added the app to Settings → Desktop app → Computer Use → Denied apps)
|
||||
// must not survive. Without
|
||||
// this, a stale grant bypasses the auto-deny. Stripped silently — the
|
||||
// agent already saw the userDenied guidance at request_access time, and
|
||||
// a live frontmost-gate rejection cites "not in allowed applications".
|
||||
//
|
||||
// (b) Strip policy-denied. Same story as (a) for a grant that predates a
|
||||
// blocklist addition. buildAccessRequest denies these up front for new
|
||||
// requests; this catches stale persisted grants.
|
||||
//
|
||||
// (c) Backfill tier. A grant persisted before the tier field existed has
|
||||
// `tier: undefined`, which `tierSatisfies` treats as `"full"` — wrong
|
||||
// for a legacy Chrome grant. Assign the hardcoded tier based on
|
||||
// bundle-ID category. Modern grants already have a tier.
|
||||
//
|
||||
// `.some()` guard keeps the hot path (empty deny list, no legacy grants)
|
||||
// zero-alloc.
|
||||
const userDeniedSet = new Set(rawOverrides.userDeniedBundleIds);
|
||||
const overrides: ComputerUseOverrides = rawOverrides.allowedApps.some(
|
||||
(a) =>
|
||||
a.tier === undefined ||
|
||||
userDeniedSet.has(a.bundleId) ||
|
||||
isPolicyDenied(a.bundleId, a.displayName),
|
||||
)
|
||||
? {
|
||||
...rawOverrides,
|
||||
allowedApps: rawOverrides.allowedApps
|
||||
.filter((a) => !userDeniedSet.has(a.bundleId))
|
||||
.filter((a) => !isPolicyDenied(a.bundleId, a.displayName))
|
||||
.map((a) =>
|
||||
a.tier !== undefined
|
||||
? a
|
||||
: { ...a, tier: getDefaultTierForApp(a.bundleId, a.displayName) },
|
||||
),
|
||||
}
|
||||
: rawOverrides;
|
||||
// Legacy grants and deny lists do not narrow feature-wide consent.
|
||||
const overrides = rawOverrides
|
||||
|
||||
// ─── Gate 1: kill switch ─────────────────────────────────────────────
|
||||
if (adapter.isDisabled()) {
|
||||
|
||||
Reference in New Issue
Block a user