From 8336cd8cc92f7c82e4891a49fba77f27d6aaa9ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Thu, 10 Sep 2026 20:58:39 +0800 Subject: [PATCH] feat(computer-use): finalize native input and global app consent Route coordinate clicks through the exact AX hit, reuse render-local attributes and keep keyboard macros within validated native boundaries. Preserve lossless capture evidence while returning bounded model images, and propagate cancellation through daemon and lock acquisition. Remove legacy per-app restrictions after feature-wide consent on both platforms while retaining global disablement and target validation. Include native helpers, regression tests and a stable receiver fixture. Validation: 224 focused TypeScript tests, 516 XCTest cases and 15 Swift Testing cases passed. Local sidecar and Electron builds, development package creation and package smoke checks passed. --- AGENTS.md | 1 + .../Sources/cu-helper/AXAction.swift | 43 +- .../cu-helper/Sources/cu-helper/AXTree.swift | 57 +- .../Sources/cu-helper/AppTargetPolicy.swift | 63 -- .../cu-helper/Sources/cu-helper/Capture.swift | 85 ++- .../Sources/cu-helper/CommandRouter.swift | 47 +- .../cu-helper/CoordinateClickRouting.swift | 27 + .../cu-helper/FocusedElementRouting.swift | 25 + .../cu-helper/KeyboardCommandSequence.swift | 52 ++ .../cu-helper/RendererAttributeReuse.swift | 41 ++ .../ResolvedTargetAuthorization.swift | 11 +- .../cu-helper/WindowCaptureStream.swift | 5 +- .../AXTreePublicationIntegrationTests.swift | 34 +- .../CuHelperTests/AppTargetPolicyTests.swift | 105 --- .../CoordinateClickRoutingTests.swift | 63 ++ .../FocusedElementRoutingTests.swift | 69 ++ .../KeyboardCommandSequenceTests.swift | 99 +++ .../CuHelperTests/ModelWindowShotTests.swift | 170 +++++ .../RendererAttributeReuseTests.swift | 50 ++ .../ResolvedTargetAuthorizationTests.swift | 99 +-- .../WindowCaptureStreamTests.swift | 11 +- .../__tests__/mac-installed-apps.test.ts | 50 +- src/server/api/macInstalledApps.ts | 5 - src/skills/bundled/computerUse.test.ts | 18 + src/skills/bundled/computerUse.ts | 10 + src/utils/computerUse/cuHelperDaemon.test.ts | 29 +- src/utils/computerUse/cuHelperDaemon.ts | 11 +- .../computerUse/preauthorizedConfig.test.ts | 12 +- src/utils/computerUse/wrapper.test.ts | 130 ++++ src/utils/computerUse/wrapper.tsx | 13 +- .../appTargetPolicyParity.test.ts | 70 -- .../computer-use-mcp/deniedApps.test.ts | 35 + src/vendor/computer-use-mcp/deniedApps.ts | 599 +----------------- .../computer-use-mcp/instructions.test.ts | 18 + .../computer-use-mcp/nativeAppPolicy.ts | 36 -- .../computer-use-mcp/platformRouting.test.ts | 106 ++++ src/vendor/computer-use-mcp/toolCalls.test.ts | 168 +++-- src/vendor/computer-use-mcp/toolCalls.ts | 56 +- .../windowsLegacyToolCalls.ts | 486 +------------- 39 files changed, 1433 insertions(+), 1576 deletions(-) delete mode 100644 native/cu-helper/Sources/cu-helper/AppTargetPolicy.swift create mode 100644 native/cu-helper/Sources/cu-helper/CoordinateClickRouting.swift create mode 100644 native/cu-helper/Sources/cu-helper/FocusedElementRouting.swift create mode 100644 native/cu-helper/Sources/cu-helper/KeyboardCommandSequence.swift create mode 100644 native/cu-helper/Sources/cu-helper/RendererAttributeReuse.swift delete mode 100644 native/cu-helper/Tests/CuHelperTests/AppTargetPolicyTests.swift create mode 100644 native/cu-helper/Tests/CuHelperTests/CoordinateClickRoutingTests.swift create mode 100644 native/cu-helper/Tests/CuHelperTests/FocusedElementRoutingTests.swift create mode 100644 native/cu-helper/Tests/CuHelperTests/KeyboardCommandSequenceTests.swift create mode 100644 native/cu-helper/Tests/CuHelperTests/ModelWindowShotTests.swift create mode 100644 native/cu-helper/Tests/CuHelperTests/RendererAttributeReuseTests.swift delete mode 100644 src/vendor/computer-use-mcp/appTargetPolicyParity.test.ts create mode 100644 src/vendor/computer-use-mcp/deniedApps.test.ts create mode 100644 src/vendor/computer-use-mcp/instructions.test.ts delete mode 100644 src/vendor/computer-use-mcp/nativeAppPolicy.ts diff --git a/AGENTS.md b/AGENTS.md index 2def1d57..a89ebc29 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -27,6 +27,7 @@ Rules closer to the code take precedence. For the directory you are changing, re ## Implementation Rules - Keep changes tied to the requested behavior. Reuse existing utilities, stores, services, and test harnesses; add dependencies or abstractions only when the task needs them. +- Computer Use has one app-authorization boundary on both macOS and Windows: enabling it in Settings and confirming its consent dialog authorizes all apps. Do not add per-app prompts, allowlists, denylists, category-based access tiers, or host/helper exceptions after that consent. Global disablement, OS permissions, and target/process validity checks still apply. - Executable JS/TS production changes under `src/`, `desktop/src/`, or `adapters/` require a same-area regression test unless a maintainer explicitly approves an exception. For bugs, reproduce the failure or add a test that fails for the intended reason; report when reproduction is unavailable. Test the behavior and affected boundaries. See [test design](docs/internals/contributing.md#回归测试设计) for state transitions, replay, and coverage caveats. - Keep TypeScript ESM style: 2-space indentation, no semicolons, `PascalCase` components, and `camelCase` functions/hooks/stores. Use structured parsers and existing boundaries for structured data. - Do not commit generated output such as `artifacts/`, coverage reports, `node_modules/`, build directories, or Rust `target/` trees. diff --git a/native/cu-helper/Sources/cu-helper/AXAction.swift b/native/cu-helper/Sources/cu-helper/AXAction.swift index 7f2bc461..b2005dcb 100644 --- a/native/cu-helper/Sources/cu-helper/AXAction.swift +++ b/native/cu-helper/Sources/cu-helper/AXAction.swift @@ -619,7 +619,7 @@ public enum AXAction { } } - /// Coordinate click that PREFERS the AX path (Codex parity). The model gives a + /// Coordinate click that prefers the AX path. The model gives a /// point in the get_app_state screenshot; we map it to a GLOBAL point (caller's /// job) and hit-test the AX element under it, then press what's there. This is /// the load-bearing fix for Chromium/CEF apps (e.g. NeteaseMusic): the tree @@ -627,9 +627,10 @@ public enum AXAction { /// `AXUIElementCopyElementAtPosition` makes Chromium instantiate the a11y node /// for THAT point on demand — so we get a real, pressable element and call /// `AXPress`, with NO synthetic mouse event, without stealing the pointer, and - /// while the app stays in the background. This is exactly what Codex does - /// (its service imports `AXUIElementCopyElementAtPosition` + `…PerformAction` - /// and posts ZERO `CGEvent`s). Falls back to the synthetic `postToPid` click + /// while the app stays in the background. Only the exact hit is pressed: + /// a canvas hit may return a window whose descendants include its close + /// button, so index-click descendant traversal is not valid here. + /// Falls back to the synthetic `postToPid` click /// (`clickPoint`) when no AX element answers or the press finds no action. /// Returns a tag naming the path taken, for diagnostics. Left button only takes /// the AX path; other buttons have no clean per-point AX analogue and go @@ -647,22 +648,17 @@ public enum AXAction { if button == .left { nudgeChromiumAccessibility(pid: pid) - if let hit = hitTest(pid: pid, at: point) { - if let tag = pressPrimary(hit, times: reps) { - settle() - return "ax:point:\(tag)" - } - if let descendant = firstActionableDescendant(of: hit, depth: descendantScanDepth), - let tag = pressPrimary(descendant, times: reps) { - settle() - return "ax:point:descendant:\(tag)" - } - } } - - // No AX element answered (or a non-left button): synthetic pointer click. - try await clickPoint(pid: pid, x: x, y: y, clickCount: reps, button: button) - return "synthetic:point" + return try await CoordinateClickRouting.click( + point: point, + preferAccessibility: button == .left, + hitTest: { hitTest(pid: pid, at: $0) }, + press: { pressPrimary($0, times: reps) }, + settle: { settle() }, + syntheticClick: { target in + try await clickPoint(pid: pid, x: target.x, y: target.y, clickCount: reps, button: button) + } + ) } /// Best-effort nudge so a Chromium/Electron/CEF app exposes its accessibility @@ -992,7 +988,14 @@ public enum AXAction { pid: pid_t, _ key: String, validateBeforePosting: () throws -> Void = {} ) async throws { - let chords = try KeyMapping.parse(key) + try await pressKey(pid: pid, chords: KeyMapping.parse(key), validateBeforePosting: validateBeforePosting) + } + + /// Receives already validated chords from the semantic command router. + static func pressKey( + pid: pid_t, chords: [KeyMapping.Chord], + validateBeforePosting: () throws -> Void = {} + ) async throws { guard !chords.isEmpty else { throw CUError(CUError.Code.unknownKey, "Empty key sequence") } diff --git a/native/cu-helper/Sources/cu-helper/AXTree.swift b/native/cu-helper/Sources/cu-helper/AXTree.swift index 27faeff2..ec23852e 100644 --- a/native/cu-helper/Sources/cu-helper/AXTree.swift +++ b/native/cu-helper/Sources/cu-helper/AXTree.swift @@ -911,13 +911,15 @@ public enum AXTree { pid: pid_t, systemWide: AXUIElement ) -> AXUIElement? { - // Prefer the system-wide focus only when it belongs to the target app. - if let sysFocusedApp = copyElement(systemWide, kAXFocusedApplicationAttribute), - pidOf(sysFocusedApp) == pid, - let el = copyElement(systemWide, kAXFocusedUIElementAttribute) { - return el - } - return copyElement(app, kAXFocusedUIElementAttribute) + FocusedElementRouting.select( + targetPID: pid, + frontmostPID: NSWorkspace.shared.frontmostApplication?.processIdentifier, + systemFocusedPID: { + copyElement(systemWide, kAXFocusedApplicationAttribute).map { pidOf($0) } + }, + systemFocusedElement: { copyElement(systemWide, kAXFocusedUIElementAttribute) }, + targetFocusedElement: { copyElement(app, kAXFocusedUIElementAttribute) } + ) } // MARK: - Renderer (format authority: blueprint §1) @@ -983,17 +985,29 @@ public enum AXTree { let elementFingerprint = knownFingerprint ?? fingerprint(of: element) let role = elementFingerprint.role + let attributes = RendererAttributeReuse( + title: elementFingerprint.title, + description: elementFingerprint.label, + // AXUnknown also represents a failed role read in fingerprint. + // Leave that case retryable instead of caching the fallback. + role: role == "AXUnknown" ? nil : role + ) let subrole = elementFingerprint.subrole let baseRoleText = roleDescription(element, role: role, subrole: subrole) - let label = stringValue(element, kAXDescriptionAttribute) + let label = attributes.description { stringValue(element, kAXDescriptionAttribute) } let help = stringValue(element, kAXHelpAttribute) let value = sanitizedValue(element) let identifier = displayIdentifier(elementFingerprint.identifier) - let traits = traitList(element) + let traits = traitList(element) { + attributes.settable { readSettable(element, kAXValueAttribute) } ?? false + } let rawActions = actionNames(element) let prettyActions = meaningfulActions(rawActions, role: role) let placeholder = placeholderValue(element) - let childElements = AXTree.walkChildren(of: element) + let childElements = AXTree.walkChildren( + of: element, + role: attributes.role { stringValue(element, kAXRoleAttribute) } + ) let childFingerprints = childElements.map(fingerprint(of:)) let rowTexts = role == (kAXRowRole as String) ? flattenedRowTexts(element) : [] @@ -1003,7 +1017,8 @@ public enum AXTree { label: label, identifier: identifier, explicitValue: value, - rowTexts: rowTexts + rowTexts: rowTexts, + attributes: attributes ) let linkText = role == axLinkRole ? markdownLinkText(element, title: title, label: label, value: value) @@ -1125,7 +1140,7 @@ public enum AXTree { roleText: roleText, title: displayTitle, value: value, - settable: isSettable(element, kAXValueAttribute), + settable: attributes.settable { readSettable(element, kAXValueAttribute) } ?? false, frameGlobal: globalFrame(element), rawActions: rawActions, depth: depth @@ -1315,9 +1330,10 @@ public enum AXTree { label: String?, identifier: String?, explicitValue: String?, - rowTexts: [String] + rowTexts: [String], + attributes: RendererAttributeReuse ) -> String? { - if let title = stringValue(element, kAXTitleAttribute), !title.isEmpty { + if let title = attributes.title(read: { stringValue(element, kAXTitleAttribute) }), !title.isEmpty { return sanitize(title) } if role == (kAXRowRole as String) { return rowTexts.first } @@ -1504,7 +1520,10 @@ public enum AXTree { /// skipped under the menu bar. THIS is the ordering `(windowIndex, path)` /// indexes — `resolve` calls the SAME function so locators round-trip exactly. static func walkChildren(of element: AXUIElement) -> [AXUIElement] { - let role = stringValue(element, kAXRoleAttribute) + walkChildren(of: element, role: stringValue(element, kAXRoleAttribute)) + } + + private static func walkChildren(of element: AXUIElement, role: String?) -> [AXUIElement] { let rows = copyElementArray(element, kAXRowsAttribute) ?? [] let visibleChildren = copyElementArray(element, axVisibleChildrenAttribute) ?? [] let attributes = childTraversalAttributes( @@ -1629,12 +1648,12 @@ public enum AXTree { // MARK: - Traits (blueprint §1) - private static func traitList(_ element: AXUIElement) -> [String] { + private static func traitList(_ element: AXUIElement, isValueSettable: () -> Bool) -> [String] { var values: [String] = [] if boolValue(element, kAXSelectedAttribute) == true { values.append("selected") } if boolValue(element, kAXExpandedAttribute) == true { values.append("expanded") } if boolValue(element, kAXEnabledAttribute) == false { values.append("disabled") } - if isSettable(element, kAXValueAttribute) { + if isValueSettable() { values.append("settable") if let valueType = valueTypeTrait(element) { values.append(valueType) } } @@ -1843,10 +1862,10 @@ public enum AXTree { return trimmed.isEmpty ? nil : trimmed } - private static func isSettable(_ element: AXUIElement, _ attribute: String) -> Bool { + private static func readSettable(_ element: AXUIElement, _ attribute: String) -> Bool? { var settable = DarwinBoolean(false) let err = AXUIElementIsAttributeSettable(element, attribute as CFString, &settable) - return err == .success && settable.boolValue + return err == .success ? settable.boolValue : nil } private static func pidOf(_ element: AXUIElement) -> pid_t { diff --git a/native/cu-helper/Sources/cu-helper/AppTargetPolicy.swift b/native/cu-helper/Sources/cu-helper/AppTargetPolicy.swift deleted file mode 100644 index 3031255c..00000000 --- a/native/cu-helper/Sources/cu-helper/AppTargetPolicy.swift +++ /dev/null @@ -1,63 +0,0 @@ -import Foundation - -/// Built-in macOS native policy, matched by exact resolved bundle identity. -/// The installed official service checks terminal, Computer Use host, ChatGPT, -/// and system-security groups. Other categories remain subject to app approval; -/// the older generic Windows terminal/IDE/media/trading lists do not define this -/// native boundary. -enum AppTargetPolicy { - enum Decision: Equatable, Sendable { - case allow - case deny - } - - /// Process identities that Computer Use must never inspect or control. - /// Keep these independent from the mirrored generic policy set below: the - /// host and helper remain denied even if the cross-language policy changes. - static let intrinsicDeniedBundleIDs: Set = [ - "com.claude-code-haha.desktop", - "dev.cchaha.cu-helper", - ] - - /// Audited against SkyComputerUseService 26.831.1000926's - /// BundleIdentifiers.isForbiddenComputerUseTarget (0x100240a14). - /// Keep this literal set cross-checked with nativeAppPolicy.ts. - static let deniedBundleIDs: Set = [ - // terminal - "com.apple.Terminal", - "com.googlecode.iterm2", - "org.alacritty", - "dev.warp.Warp-Stable", - "net.kovidgoyal.kitty", - "co.zeit.hyper", - "com.github.wez.wezterm", - "org.tabby", - "com.mitchellh.ghostty", - "com.raphaelamorim.rio", - "dev.commandline.waveterm", - // computerUseHost - "com.openai.codex", - "com.openai.codex.alpha", - "com.openai.codex.beta", - "com.openai.codex.dev", - "com.openai.codex.nightly", - // chatGPT - "com.openai.chat", - "com.openai.chat.alpha", - "com.openai.chat.beta", - "com.openai.chat.nightly", - "com.openai.chat.mac-debug", - // systemSecurity - "com.apple.UserNotificationCenter", - "com.apple.LocalAuthenticationRemoteService", - "com.apple.SecurityAgent", - ] - - static func decision(bundleID: String) -> Decision { - if intrinsicDeniedBundleIDs.contains(bundleID) - || deniedBundleIDs.contains(bundleID) { - return .deny - } - return .allow - } -} diff --git a/native/cu-helper/Sources/cu-helper/Capture.swift b/native/cu-helper/Sources/cu-helper/Capture.swift index 2d0e5fcb..6822f286 100644 --- a/native/cu-helper/Sources/cu-helper/Capture.swift +++ b/native/cu-helper/Sources/cu-helper/Capture.swift @@ -60,13 +60,74 @@ struct WindowShot: Sendable { /// Uniform capture fit before integer pixel-buffer rounding. Legacy shots /// may omit this and retain their dimension-derived transform. var pixelsPerPoint: Double? = nil - var mimeType: String { NativeScreenshotPolicy.mimeType } + var mimeType: String = "image/png" +} + +struct ModelWindowImage: Sendable { + let base64: String + let mimeType: String } @available(macOS 14.0, *) @MainActor public enum Capture { + /// Apply the model image budget after capture freshness has been validated. + /// Window geometry stays in points; returned dimensions name actual pixels. + static func boundedModelWindowShot(_ shot: WindowShot) -> WindowShot? { + guard shot.width > 0, shot.height > 0, + shot.pointWidth.isFinite, shot.pointWidth > 0, + shot.pointHeight.isFinite, shot.pointHeight > 0 else { return nil } + let target = NativeScreenshotPolicy.pixelSize( + pointSize: CGSize(width: shot.pointWidth, height: shot.pointHeight), + backingScale: 1 + ) + // The production capture already applies this same policy. Preserve + // its exact pre-rounding transform and pixels without a second resize. + guard shot.width > Int(target.width) || shot.height > Int(target.height) else { + return shot + } + guard let data = Data(base64Encoded: shot.base64), + let source = CGImageSourceCreateWithData(data as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else { + return nil + } + let scale = min(1, target.width / Double(image.width), + target.height / Double(image.height)) + guard let bounded = try? scaleImage(image, scale: scale), + let encoded = pngBase64WithSize(bounded) else { + // Keep the existing AX-only degradation; never leak an unbounded + // image after an allocation/encoding failure. + return nil + } + return WindowShot( + base64: encoded.base64, width: encoded.width, height: encoded.height, + originX: shot.originX, originY: shot.originY, + pointWidth: shot.pointWidth, pointHeight: shot.pointHeight, + windowID: shot.windowID, source: shot.source, + pixelsPerPoint: shot.pixelsPerPoint.map { $0 * scale }, mimeType: "image/png" + ) + } + + /// Presentation encoding only: keep the bounded lossless shot as the + /// freshness/identical-capture evidence and retain its coordinate geometry. + static func modelWindowImage( + _ shot: WindowShot, + quality: Double = NativeScreenshotPolicy.jpegQuality, + encodeJPEG: @MainActor (CGImage, Double) -> String? = jpegBase64 + ) -> ModelWindowImage { + let lossless = ModelWindowImage(base64: shot.base64, mimeType: shot.mimeType) + guard quality.isFinite, (0...1).contains(quality), + let data = Data(base64Encoded: shot.base64), + let source = CGImageSourceCreateWithData(data as CFData, nil), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil), + image.width == shot.width, image.height == shot.height, + let base64 = encodeJPEG(image, quality), !base64.isEmpty else { + return lossless + } + return ModelWindowImage(base64: base64, mimeType: "image/jpeg") + } + // MARK: - Permission gates /// Passive Screen Recording check. Never prompts. @@ -421,7 +482,7 @@ public enum Capture { // A *window-locked* capture used by the daemon's `get_app_state`: it pins the // target app's single most-relevant window, captures only that window (not the // full display, not the desktop, not other apps), downscales by `scale` - // (default 0.5), and returns PNG base64 in the screenshot-pixel space (top-left + // (default native App fit), and returns lossless PNG base64 in the screenshot-pixel space (top-left // origin) for the server's affine map. It is the only capture path tied to a // *pid* rather than a *display*. // @@ -449,7 +510,7 @@ public enum Capture { /// - scale: an explicit factor applied to native pixels; nil uses the /// native App policy (point resolution, long/short side limits). /// - Returns: `(base64, width, height, originX, originY, pointWidth, - /// pointHeight, windowID)` — the JPEG in screenshot-pixel space (top-left origin) + /// pointHeight, windowID)` — lossless PNG capture evidence in screenshot-pixel space (top-left origin) /// PLUS the captured window's GLOBAL Quartz top-left origin and its size /// in POINTS. The caller uses the uniform `pixelsPerPoint` capture fit /// to invert image-pixel coordinates back into the @@ -485,7 +546,7 @@ public enum Capture { frame: target.frame, scale: outputScale ) { - if let encoded = appScreenshotBase64WithSize(image) { + if let encoded = pngBase64WithSize(image) { return WindowShot( base64: encoded.base64, width: encoded.width, @@ -509,7 +570,7 @@ public enum Capture { if let raw = screencaptureWindow(windowID: target.windowID) { let scaledImage = try? scaleImage(raw, scale: outputScale) let scaled = scaledImage ?? raw - if let encoded = appScreenshotBase64WithSize(scaled) { + if let encoded = pngBase64WithSize(scaled) { return WindowShot( base64: encoded.base64, width: encoded.width, @@ -801,6 +862,20 @@ public enum Capture { return scaled } + /// Keep capture/freshness evidence lossless. The model attachment is JPEG + /// encoded only after the bound window and captured frame are validated. + static func pngBase64WithSize( + _ image: CGImage + ) -> (base64: String, width: Int, height: Int)? { + let data = NSMutableData() + guard let destination = CGImageDestinationCreateWithData( + data, UTType.png.identifier as CFString, 1, nil + ) else { return nil } + CGImageDestinationAddImage(destination, image, nil) + guard CGImageDestinationFinalize(destination) else { return nil } + return ((data as Data).base64EncodedString(), image.width, image.height) + } + /// Native App screenshots use the official default JPEG quality. The /// byte format is reported explicitly instead of relying on a fixed MIME /// label in a downstream wrapper. Encoding failure degrades to AX text. diff --git a/native/cu-helper/Sources/cu-helper/CommandRouter.swift b/native/cu-helper/Sources/cu-helper/CommandRouter.swift index 01c198ae..80c2111e 100644 --- a/native/cu-helper/Sources/cu-helper/CommandRouter.swift +++ b/native/cu-helper/Sources/cu-helper/CommandRouter.swift @@ -618,14 +618,6 @@ public final class CommandRouter { return .object(object) case .installed(let installed): - guard AppTargetPolicy.decision( - bundleID: installed.bundleIdentifier - ) == .allow else { - throw CUError( - "app_denied", - "Computer Use is not allowed to use the app '\(installed.bundleIdentifier)' for safety reasons." - ) - } return .object([ "bundleId": .string(installed.bundleIdentifier), "displayName": .string(installed.displayName), @@ -691,14 +683,6 @@ public final class CommandRouter { guard case .installed(let installed) = installedOutcome else { throw CUError("target_not_running", "The requested target app is not running") } - guard AppTargetPolicy.decision( - bundleID: installed.bundleIdentifier - ) == .allow else { - throw CUError( - "app_denied", - "Computer Use is not allowed to use the app '\(installed.bundleIdentifier)' for safety reasons." - ) - } let identifier = installed.bundleURL.standardizedFileURL.path guard let launched = await AppTargetResolver.launch( identifier: identifier, @@ -839,7 +823,9 @@ public final class CommandRouter { ) } - if let shot, + // Keep stream/frame validation in its native capture dimensions. + // Publish and map coordinates using the same final model image. + if let shot = shot.flatMap(Capture.boundedModelWindowShot), AXTree.currentProcessIdentity(pid: pid) == snapshotEvidence.processIdentity { // An identical capture is the other half of the same problem: // the pixels cannot say whether the action missed or the window @@ -854,9 +840,12 @@ public final class CommandRouter { ) { Self.appendAXNotice(notice, to: &object) } + // Lossless bytes above establish identical-frame evidence; + // presentation encoding does not rescale or change geometry. + let modelImage = Capture.modelWindowImage(shot) var screenshot: [String: JSONValue] = [ - "base64": .string(shot.base64), - "mimeType": .string(shot.mimeType), + "base64": .string(modelImage.base64), + "mimeType": .string(modelImage.mimeType), "width": .int(shot.width), "height": .int(shot.height), "originX": .double(shot.originX), @@ -1505,23 +1494,19 @@ public final class CommandRouter { let systemKeyCombos = try SystemKeyPolicy.parseGrant( payload["systemKeyCombos"] ) - try SystemKeyPolicy.enforce( - sequence: key, - granted: systemKeyCombos - ) + let chords = try KeyboardCommandSequence.prepare(key, systemKeyCombos: systemKeyCombos) let expected = try Self.expectedProcessTarget(payload) let target = try resolveTargetForMutation(payload) setResolvedTarget(target) try requireSnapshotProcess(target: target, expected: expected) - return try await withForegroundLease( - command: "press_key", - target: target - ) { - _ = try Injection.validateAuthorizedTarget(target) - try self.requireSnapshotProcess(target: target, expected: expected) - try await AXAction.pressKey(pid: target.pid, key) - return .bool(true) + try await KeyboardCommandSequence.run(chords: chords) { batch in + try await self.withForegroundLease(command: "press_key", target: target) { + _ = try Injection.validateAuthorizedTarget(target) + try self.requireSnapshotProcess(target: target, expected: expected) + try await AXAction.pressKey(pid: target.pid, chords: batch) + } } + return .bool(true) } /// `drag`: coordinate-only press→drag→release between screenshot-local points. diff --git a/native/cu-helper/Sources/cu-helper/CoordinateClickRouting.swift b/native/cu-helper/Sources/cu-helper/CoordinateClickRouting.swift new file mode 100644 index 00000000..a4394c42 --- /dev/null +++ b/native/cu-helper/Sources/cu-helper/CoordinateClickRouting.swift @@ -0,0 +1,27 @@ +import CoreGraphics + +/// The coordinate route is separate from an index click's AX-tree traversal. +/// Dependencies allow routing tests without posting input or contacting apps. +enum CoordinateClickRouting { + @MainActor + static func click( + point: CGPoint, + preferAccessibility: Bool, + hitTest: (CGPoint) -> Element?, + press: (Element) -> String?, + settle: () -> Void, + syntheticClick: (CGPoint) async throws -> Void + ) async throws -> String { + if preferAccessibility, let hit = hitTest(point) { + if let tag = press(hit) { + settle() + return "ax:point:\(tag)" + } + } + // A canvas can hit-test to its entire AXWindow. Its first actionable + // descendant may be the close button, far from the requested point. + // Only the exact hit can authorize an AX action for a coordinate click. + try await syntheticClick(point) + return "synthetic:point" + } +} diff --git a/native/cu-helper/Sources/cu-helper/FocusedElementRouting.swift b/native/cu-helper/Sources/cu-helper/FocusedElementRouting.swift new file mode 100644 index 00000000..905abd9c --- /dev/null +++ b/native/cu-helper/Sources/cu-helper/FocusedElementRouting.swift @@ -0,0 +1,25 @@ +import Foundation + +/// Select focus only from the requested application. The callbacks make the +/// background/foreground decision testable without contacting an AX server. +enum FocusedElementRouting { + @MainActor + static func select( + targetPID: pid_t, + frontmostPID: pid_t?, + systemFocusedPID: () -> pid_t?, + systemFocusedElement: () -> Element?, + targetFocusedElement: () -> Element? + ) -> Element? { + // Reading global AX focus can wait for an unrelated foreground app's + // AX server. A known background target needs only its own focus tree. + // Unknown frontmost identity retains the original global query, and + // the actual AX PID remains authoritative if focus changes meanwhile. + if (frontmostPID == nil || frontmostPID == targetPID), + systemFocusedPID() == targetPID, + let element = systemFocusedElement() { + return element + } + return targetFocusedElement() + } +} diff --git a/native/cu-helper/Sources/cu-helper/KeyboardCommandSequence.swift b/native/cu-helper/Sources/cu-helper/KeyboardCommandSequence.swift new file mode 100644 index 00000000..81c30922 --- /dev/null +++ b/native/cu-helper/Sources/cu-helper/KeyboardCommandSequence.swift @@ -0,0 +1,52 @@ +import Foundation + +/// Preflight a complete macro before entering the per-command input boundary. +enum KeyboardCommandSequence { + static let maximumChordCount = 128 + + static func prepare(_ sequence: String, systemKeyCombos: Bool) throws -> [KeyMapping.Chord] { + let chords = try KeyMapping.parse(sequence) + guard chords.count <= maximumChordCount else { + throw CUError("bad_payload", "press_key accepts at most \(maximumChordCount) chords; no input was sent") + } + try SystemKeyPolicy.enforce(sequence: sequence, granted: systemKeyCombos) + return chords + } + + /// Each callback is one complete existing native command boundary, including + /// its foreground lease, focus preparation and finalization. Awaiting only + /// a yield between CGEvents would not reproduce that boundary. One chord's + /// down/up group is still fully allocated before it is posted. + @MainActor + static func run( + chords: [KeyMapping.Chord], + perform: @MainActor ([KeyMapping.Chord]) async throws -> Void + ) async throws { + var completed = 0 + var inFlight = false + do { + for chord in chords { + try Task.checkCancellation() + inFlight = true + try await perform([chord]) + completed += 1 + inFlight = false + } + try Task.checkCancellation() + } catch { + // Keep the established single-key error contract, including paste's + // clipboard validation. A macro can have a completed prefix even + // when its finalization refuses, so do not invite whole-macro retry. + guard chords.count > 1 else { throw error } + let code = (error as? CUError)?.code + ?? (error is CancellationError ? "cancelled" : "keyboard_sequence_failed") + let delivery = inFlight ? " The failed chord may already have been delivered." : "" + throw CUError( + code, + "press_key stopped after \(completed) of \(chords.count) chords completed.\(delivery) " + + "Inspect the current state before continuing; do not replay the completed prefix. " + + "Cause: \(error.localizedDescription)" + ) + } + } +} diff --git a/native/cu-helper/Sources/cu-helper/RendererAttributeReuse.swift b/native/cu-helper/Sources/cu-helper/RendererAttributeReuse.swift new file mode 100644 index 00000000..5cffbc97 --- /dev/null +++ b/native/cu-helper/Sources/cu-helper/RendererAttributeReuse.swift @@ -0,0 +1,41 @@ +/// Successful scalar observations owned by one render invocation only. +/// Missing/failed reads deliberately remain retryable. Never store this in a +/// published snapshot or reuse it for action-time identity/focus validation. +@MainActor +final class RendererAttributeReuse { + private var titleValue: String? + private var descriptionValue: String? + private var roleValue: String? + private var settableValue: Bool? + + init(title: String?, description: String?, role: String?) { + titleValue = title + descriptionValue = description + roleValue = role + } + + func title(read: () -> String?) -> String? { + if let value = titleValue { return value } + let value = read() + if let value { titleValue = value } + return value + } + func description(read: () -> String?) -> String? { + if let value = descriptionValue { return value } + let value = read() + if let value { descriptionValue = value } + return value + } + func role(read: () -> String?) -> String? { + if let value = roleValue { return value } + let value = read() + if let value { roleValue = value } + return value + } + func settable(read: () -> Bool?) -> Bool? { + if let value = settableValue { return value } + let value = read() + if let value { settableValue = value } + return value + } +} diff --git a/native/cu-helper/Sources/cu-helper/ResolvedTargetAuthorization.swift b/native/cu-helper/Sources/cu-helper/ResolvedTargetAuthorization.swift index 45de7cf7..8b7e14ea 100644 --- a/native/cu-helper/Sources/cu-helper/ResolvedTargetAuthorization.swift +++ b/native/cu-helper/Sources/cu-helper/ResolvedTargetAuthorization.swift @@ -1,10 +1,10 @@ import Darwin import Foundation -/// Authoritative native policy applied only after a resolver has produced a +/// Native process-identity validation applied after a resolver has produced a /// real running process and its current process-lifetime identity. Selector /// form is deliberately absent, so PID/name/bundle/frontmost/launch paths have -/// no policy bypass. +/// no process-identity bypass. Global Computer Use consent covers every app. enum ResolvedTargetAuthorization { static func authorize( resolved: ResolvedAppTarget, @@ -43,13 +43,6 @@ enum ResolvedTargetAuthorization { } } - guard AppTargetPolicy.decision(bundleID: actualBundleID) == .allow else { - throw CUError( - "app_denied", - "Computer Use is not allowed to use the app '\(actualBundleID)' for safety reasons." - ) - } - guard let target = ProvenProcessTarget(pid: pid, identity: identity) else { throw CUError( "app_denied", diff --git a/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift b/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift index be30c688..12906d45 100644 --- a/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift +++ b/native/cu-helper/Sources/cu-helper/WindowCaptureStream.swift @@ -210,7 +210,8 @@ final class WindowCaptureStreamManager: WindowCaptureProviding { originX: current.originX, originY: current.originY, pointWidth: current.pointWidth, pointHeight: current.pointHeight, windowID: current.key.windowID, source: .streamBackedScreenshot, - pixelsPerPoint: shot.pixelsPerPoint + pixelsPerPoint: shot.pixelsPerPoint, + mimeType: shot.mimeType ) } return nil @@ -840,7 +841,7 @@ extension Capture { guard frame.width == target.key.pixelWidth, frame.height == target.key.pixelHeight, let image = image(from: frame), - let encoded = appScreenshotBase64WithSize(image) else { + let encoded = pngBase64WithSize(image) else { return nil } return WindowShot( diff --git a/native/cu-helper/Tests/CuHelperTests/AXTreePublicationIntegrationTests.swift b/native/cu-helper/Tests/CuHelperTests/AXTreePublicationIntegrationTests.swift index e1012b6f..58e0092b 100644 --- a/native/cu-helper/Tests/CuHelperTests/AXTreePublicationIntegrationTests.swift +++ b/native/cu-helper/Tests/CuHelperTests/AXTreePublicationIntegrationTests.swift @@ -275,13 +275,25 @@ final class AXTreePublicationIntegrationTests: XCTestCase { if exerciseKeys { let (canvasHandle, _) = try publishedHandle(label: "Drag fixture", state: clickedState) - _ = try await router.handle(cmd: "click", payload: .object([ - "pid": .int(Int(pid)), "index": .string(canvasHandle.rawValue), - ])) - _ = try await router.handle(cmd: "press_key", payload: .object([ - "pid": .int(Int(pid)), - "key": .string("Control_L+a Super_R+b A question Delete BackSpace"), - ])) + var phase = "canvas click" + do { + _ = try await router.handle(cmd: "click", payload: .object([ + "pid": .int(Int(pid)), "index": .string(canvasHandle.rawValue), + ])) + phase = "keyboard macro" + _ = try await router.handle(cmd: "press_key", payload: .object([ + "pid": .int(Int(pid)), + "key": .string("Control_L+a Super_R+b A question Delete BackSpace"), + ])) + } catch { + let windows = (CGWindowListCopyWindowInfo(.optionAll, kCGNullWindowID) as? [[String: Any]] ?? []) + .filter { $0[kCGWindowOwnerPID as String] as? Int == Int(pid) } + .map { ["id": $0[kCGWindowNumber as String] ?? "nil", "bounds": $0[kCGWindowBounds as String] ?? "nil", "onScreen": $0[kCGWindowIsOnscreen as String] ?? "nil", "layer": $0[kCGWindowLayer as String] ?? "nil"] } + let frame = AXTree.record(pid: pid, index: canvasHandle.index)?.frameGlobal + let receipt = (try? String(contentsOf: gestures, encoding: .utf8)) ?? "no receipt" + throw CUError((error as? CUError)?.code ?? "fixture_action_failed", + "\(error.localizedDescription) Fixture phase=\(phase), pid=\(pid), terminated=\(process.isTerminated), canvas=\(String(describing: frame)), windows=\(windows), receipt=\(receipt)") + } try await waitUntil(description: "six received macro keys") { guard let data = try? Data(contentsOf: gestures), let receipt = try? JSONSerialization.jsonObject(with: data) as? [String: Any], @@ -442,6 +454,14 @@ final class AXTreePublicationIntegrationTests: XCTestCase { backing: .buffered, defer: false ) + if regularActivation { + // This is a utility receiver, not a document in the user's active + // app set. A regular app's window can otherwise be reduced to a + // WindowServer preview while AX still reports its full-size frame. + // Keep it eligible to join the current set without changing any + // system preference, window level, or production input safeguard. + window.collectionBehavior = [.canJoinAllApplications] + } window.title = title if mismatchedWindowTitle { // Chrome exposes a decorated AX title while WindowServer uses the diff --git a/native/cu-helper/Tests/CuHelperTests/AppTargetPolicyTests.swift b/native/cu-helper/Tests/CuHelperTests/AppTargetPolicyTests.swift deleted file mode 100644 index 8ce377c6..00000000 --- a/native/cu-helper/Tests/CuHelperTests/AppTargetPolicyTests.swift +++ /dev/null @@ -1,105 +0,0 @@ -import XCTest -@testable import cc_haha_computer_use - -final class AppTargetPolicyTests: XCTestCase { - func testOfficialTerminalAndSecurityBundlesAreDenied() { - let denied = [ - "com.apple.Terminal", - "com.googlecode.iterm2", - "com.raphaelamorim.rio", - "dev.commandline.waveterm", - "com.apple.SecurityAgent", - "com.apple.LocalAuthenticationRemoteService", - "com.apple.UserNotificationCenter", - "com.openai.codex.beta", - "com.openai.chat.mac-debug", - ] - - for bundleID in denied { - XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .deny, bundleID) - } - } - - func testNativeBrowserBundlesAreAllowed() { - for bundleID in [ - "com.google.Chrome", "com.google.Chrome.canary", "com.apple.Safari", - "org.mozilla.firefox", "com.microsoft.edgemac", "com.brave.Browser", - ] { - XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID) - } - } - - func testOtherAppCategoriesAreNotImplicitlyForbidden() { - let allowed = [ - "com.webull.desktop.v1", - "com.binance.BinanceDesktop", - "com.electron.exodus", - "com.ledger.live", - "io.trezor.TrezorSuite", - ] - - for bundleID in allowed { - XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID) - } - } - - func testMediaAndDevelopmentAppsAreNotImplicitlyForbidden() { - let allowed = [ - "com.spotify.client", - "com.apple.Music", - "com.amazon.aiv.AIVApp", - "tv.plex.desktop", - "com.amazon.Kindle", - "com.microsoft.VSCode", - "com.apple.shortcuts", - "com.apple.dt.Xcode", - ] - - for bundleID in allowed { - XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID) - } - } - - func testForbiddenPolicyUsesExactIdentityNotNameSubstringOrPrefix() { - for id in ["com.apple.Terminal.preview", "com.openai.codex.userapp", "org.example.Terminal", "com.apple.securityagent"] { - XCTAssertEqual(AppTargetPolicy.decision(bundleID: id), .allow, id) - } - } - - func testNormalProductivityBundlesAreAllowed() { - let allowed = [ - "com.apple.calculator", - "com.apple.TextEdit", - "com.apple.finder", - ] - - for bundleID in allowed { - XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID) - } - } - - func testIntrinsicHostAndHelperBundlesAreAlwaysDenied() { - let expected: Set = [ - "com.claude-code-haha.desktop", - "dev.cchaha.cu-helper", - ] - - XCTAssertEqual(AppTargetPolicy.intrinsicDeniedBundleIDs, expected) - for bundleID in expected { - XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .deny, bundleID) - } - } - - func testDeniedBundleUnionCountAndSetDuplicateHandlingAreLocked() { - XCTAssertEqual(AppTargetPolicy.deniedBundleIDs.count, 24) - XCTAssertTrue( - AppTargetPolicy.deniedBundleIDs - .isDisjoint(with: AppTargetPolicy.intrinsicDeniedBundleIDs) - ) - - var copy = AppTargetPolicy.deniedBundleIDs - let duplicate = copy.insert("com.apple.Terminal") - XCTAssertFalse(duplicate.inserted) - XCTAssertEqual(copy.count, 24) - } -} diff --git a/native/cu-helper/Tests/CuHelperTests/CoordinateClickRoutingTests.swift b/native/cu-helper/Tests/CuHelperTests/CoordinateClickRoutingTests.swift new file mode 100644 index 00000000..32ec7332 --- /dev/null +++ b/native/cu-helper/Tests/CuHelperTests/CoordinateClickRoutingTests.swift @@ -0,0 +1,63 @@ +import CoreGraphics +import XCTest + +@testable import cc_haha_computer_use + +final class CoordinateClickRoutingTests: XCTestCase { + private enum Element { case canvasWindow, closeButton, contentButton } + + @MainActor + func testCanvasWindowHitUsesExactCoordinateInsteadOfPressingItsCloseButton() async throws { + let point = CGPoint(x: 1020, y: 710) + var pressed: [Element] = [] + var clicked: [CGPoint] = [] + var settled = false + let route = try await CoordinateClickRouting.click( + point: point, preferAccessibility: true, + hitTest: { _ in Element.canvasWindow }, + press: { element in + pressed.append(element) + return element == .closeButton ? "press" : nil + }, + settle: { settled = true }, + syntheticClick: { clicked.append($0) } + ) + XCTAssertEqual(pressed, [.canvasWindow], "a canvas click must never press the window's close control") + XCTAssertEqual(clicked, [point]) + XCTAssertFalse(settled) + XCTAssertEqual(route, "synthetic:point") + } + + @MainActor + func testExactActionableHitKeepsAccessibilityPath() async throws { + var pressed: [Element] = [] + var settled = false + let route = try await CoordinateClickRouting.click( + point: CGPoint(x: 50, y: 60), preferAccessibility: true, + hitTest: { _ in Element.contentButton }, + press: { pressed.append($0); return "press" }, + settle: { settled = true }, + syntheticClick: { _ in XCTFail("exact actionable hit needs no synthetic input") } + ) + XCTAssertEqual(pressed, [.contentButton]) + XCTAssertTrue(settled) + XCTAssertEqual(route, "ax:point:press") + } + + @MainActor + func testNonLeftClickSkipsAccessibilityAndPropagatesSyntheticFailure() async { + enum Failure: Error { case refused } + do { + _ = try await CoordinateClickRouting.click( + point: .zero, preferAccessibility: false, + hitTest: { _ -> Element? in XCTFail("non-left button must skip AX"); return nil }, + press: { _ in XCTFail("must not press"); return nil }, + settle: { XCTFail("must not settle") }, + syntheticClick: { _ in throw Failure.refused } + ) + XCTFail("synthetic target guards must remain authoritative") + } catch { + XCTAssertTrue(error is Failure) + } + } +} diff --git a/native/cu-helper/Tests/CuHelperTests/FocusedElementRoutingTests.swift b/native/cu-helper/Tests/CuHelperTests/FocusedElementRoutingTests.swift new file mode 100644 index 00000000..ded598fd --- /dev/null +++ b/native/cu-helper/Tests/CuHelperTests/FocusedElementRoutingTests.swift @@ -0,0 +1,69 @@ +import XCTest + +@testable import cc_haha_computer_use + +@MainActor +final class FocusedElementRoutingTests: XCTestCase { + func testBackgroundTargetNeverContactsUnrelatedSystemFocusServer() { + var globalQueries = 0 + let result = FocusedElementRouting.select( + targetPID: 42, frontmostPID: 99, + systemFocusedPID: { globalQueries += 1; return 99 }, + systemFocusedElement: { globalQueries += 1; return "other-app" }, + targetFocusedElement: { "target" } + ) + XCTAssertEqual(result, "target") + XCTAssertEqual(globalQueries, 0, "a background snapshot must not wait for an unrelated AX server") + } + + func testForegroundTargetKeepsValidatedSystemFocus() { + let result = FocusedElementRouting.select( + targetPID: 42, frontmostPID: 42, + systemFocusedPID: { 42 }, + systemFocusedElement: { "target-system-focus" }, + targetFocusedElement: { XCTFail("valid system focus should retain precedence"); return "target-fallback" } + ) + XCTAssertEqual(result, "target-system-focus") + } + + func testForegroundHintDoesNotAuthorizeAnotherProcessAfterFocusChanges() { + let result = FocusedElementRouting.select( + targetPID: 42, frontmostPID: 42, + systemFocusedPID: { 99 }, + systemFocusedElement: { XCTFail("must retain system AX PID validation"); return "other-app" }, + targetFocusedElement: { "target" } + ) + XCTAssertEqual(result, "target") + } + + func testUnknownFrontmostStillUsesSystemAXWithPIDValidation() { + for focusedPID: Int32? in [42, 99, nil] { + var queries = 0 + let result = FocusedElementRouting.select( + targetPID: 42, frontmostPID: nil, + systemFocusedPID: { queries += 1; return focusedPID }, + systemFocusedElement: { "system" }, + targetFocusedElement: { "target" } + ) + XCTAssertEqual(queries, 1) + XCTAssertEqual(result, focusedPID == 42 ? "system" : "target") + } + } + + func testMissingSystemElementFallsBackToTargetAndMissingTargetStaysNil() { + let result: String? = FocusedElementRouting.select( + targetPID: 42, frontmostPID: 42, + systemFocusedPID: { 42 }, + systemFocusedElement: { nil }, + targetFocusedElement: { "target" } + ) + XCTAssertEqual(result, "target") + let absent: String? = FocusedElementRouting.select( + targetPID: 42, frontmostPID: 99, + systemFocusedPID: { XCTFail("must not contact another app"); return 99 }, + systemFocusedElement: { "other-app" }, + targetFocusedElement: { nil } + ) + XCTAssertNil(absent) + } +} diff --git a/native/cu-helper/Tests/CuHelperTests/KeyboardCommandSequenceTests.swift b/native/cu-helper/Tests/CuHelperTests/KeyboardCommandSequenceTests.swift new file mode 100644 index 00000000..5a8bc639 --- /dev/null +++ b/native/cu-helper/Tests/CuHelperTests/KeyboardCommandSequenceTests.swift @@ -0,0 +1,99 @@ +import XCTest + +@testable import cc_haha_computer_use + +final class KeyboardCommandSequenceTests: XCTestCase { + @MainActor + func testMacroWaitsForEachCompleteSingleChordBoundaryInOrder() async throws { + let chords = try KeyboardCommandSequence.prepare("a b c d", systemKeyCombos: false) + var history: [String] = [] + var active = false + try await KeyboardCommandSequence.run(chords: chords) { batch in + XCTAssertEqual(batch.count, 1, "a macro must not become one rapid native burst") + XCTAssertFalse(active) + active = true + let name = batch.map(\.semanticKey).joined(separator: ",") + history.append("begin:\(name)") + await Task.yield() + history.append("end:\(name)") + active = false + } + XCTAssertEqual(history, ["begin:a", "end:a", "begin:b", "end:b", "begin:c", "end:c", "begin:d", "end:d"]) + } + + @MainActor + func testUnknownOrForbiddenSuffixRejectsBeforeAnyInput() async { + for (key, expectedCode) in [("a DefinitelyNotARealKey", "unknown_key"), ("a cmd+q", "grant_flag_required")] { + var calls = 0 + do { + let chords = try KeyboardCommandSequence.prepare(key, systemKeyCombos: false) + try await KeyboardCommandSequence.run(chords: chords) { _ in calls += 1 } + XCTFail("the complete macro must be preflighted") + } catch let error as CUError { + XCTAssertEqual(error.code, expectedCode) + } catch { XCTFail("unexpected error: \(error)") } + XCTAssertEqual(calls, 0) + } + } + + func testChordLimitAppliesBeforeExecutionAndPreservesSpacedShortcutSyntax() throws { + XCTAssertEqual(try KeyboardCommandSequence.prepare(String(repeating: "Return ", count: 128), systemKeyCombos: false).count, 128) + XCTAssertThrowsError(try KeyboardCommandSequence.prepare(String(repeating: "Return ", count: 129), systemKeyCombos: false)) { error in + XCTAssertEqual((error as? CUError)?.code, "bad_payload") + } + let chords = try KeyboardCommandSequence.prepare("cmd + a\n shift + Return", systemKeyCombos: false) + XCTAssertEqual(chords.count, 2) + XCTAssertEqual(chords.map(\.flags), [.maskCommand, .maskShift]) + } + + @MainActor + func testFocusFailureStopsRemainingChordsAndReportsCompletedPrefix() async throws { + let chords = try KeyboardCommandSequence.prepare("a b c d", systemKeyCombos: false) + var attempted: [String] = [] + do { + try await KeyboardCommandSequence.run(chords: chords) { batch in + attempted.append(contentsOf: batch.map(\.semanticKey)) + if attempted.count == 2 { throw CUError("focus_changed", "test focus changed") } + } + XCTFail("must stop after focus loss") + } catch let error as CUError { + XCTAssertEqual(error.code, "focus_changed") + XCTAssertTrue(error.message.contains("1 of 4")) + XCTAssertTrue(error.message.contains("may already have been delivered")) + XCTAssertTrue(error.message.contains("do not replay")) + } + XCTAssertEqual(attempted, ["a", "b"]) + } + + @MainActor + func testCancellationBetweenChordsStopsBeforeNextBoundary() async throws { + let chords = try KeyboardCommandSequence.prepare("a b c d", systemKeyCombos: false) + var attempted: [String] = [] + let task = Task { @MainActor in + do { + try await KeyboardCommandSequence.run(chords: chords) { batch in + attempted.append(contentsOf: batch.map(\.semanticKey)) + withUnsafeCurrentTask { $0?.cancel() } + } + XCTFail("cancellation must prevent the next chord") + } catch let error as CUError { + XCTAssertEqual(error.code, "cancelled") + XCTAssertTrue(error.message.contains("1 of 4")) + } catch { XCTFail("unexpected error: \(error)") } + } + await task.value + XCTAssertEqual(attempted, ["a"]) + } + + @MainActor + func testSingleChordKeepsUnderlyingErrorWithoutMacroDecoration() async throws { + let chords = try KeyboardCommandSequence.prepare("Return", systemKeyCombos: false) + do { + try await KeyboardCommandSequence.run(chords: chords) { _ in throw CUError("stale_process", "original error") } + XCTFail("must propagate target refusal") + } catch let error as CUError { + XCTAssertEqual(error.code, "stale_process") + XCTAssertEqual(error.message, "original error") + } + } +} diff --git a/native/cu-helper/Tests/CuHelperTests/ModelWindowShotTests.swift b/native/cu-helper/Tests/CuHelperTests/ModelWindowShotTests.swift new file mode 100644 index 00000000..275d1e74 --- /dev/null +++ b/native/cu-helper/Tests/CuHelperTests/ModelWindowShotTests.swift @@ -0,0 +1,170 @@ +import CoreGraphics +import Foundation +import ImageIO +import XCTest + +@testable import cc_haha_computer_use + +@MainActor +final class ModelWindowShotTests: XCTestCase { + func testLargeLandscapeAndPortraitImagesFitBudgetAndRemainPNG() throws { + for (width, height, expectedWidth, expectedHeight) in [ + (2304, 1506, 1175, 768), + (1506, 2304, 768, 1175), + (2400, 600, 2048, 512), + ] { + let input = try makeShot(width: width, height: height) + let shot = try XCTUnwrap(Capture.boundedModelWindowShot(input)) + XCTAssertEqual(shot.width, expectedWidth) + XCTAssertEqual(shot.height, expectedHeight) + let data = try XCTUnwrap(Data(base64Encoded: shot.base64)) + XCTAssertEqual(Array(data.prefix(8)), [137, 80, 78, 71, 13, 10, 26, 10]) + let source = try XCTUnwrap(CGImageSourceCreateWithData(data as CFData, nil)) + let image = try XCTUnwrap(CGImageSourceCreateImageAtIndex(source, 0, nil)) + XCTAssertEqual(image.width, shot.width) + XCTAssertEqual(image.height, shot.height) + XCTAssertEqual(shot.originX, input.originX) + XCTAssertEqual(shot.originY, input.originY) + XCTAssertEqual(shot.pointWidth, input.pointWidth) + XCTAssertEqual(shot.pointHeight, input.pointHeight) + XCTAssertEqual(shot.windowID, input.windowID) + XCTAssertEqual(shot.source, input.source) + } + } + + func testSmallImageIsNotEnlargedOrReencoded() throws { + let input = try makeShot(width: 640, height: 400) + let shot = try XCTUnwrap(Capture.boundedModelWindowShot(input)) + XCTAssertEqual(shot.width, 640) + XCTAssertEqual(shot.height, 400) + XCTAssertEqual(shot.base64, input.base64) + } + + func testBoundedPixelsMapBackToOriginalWindowPointsAtDifferentBackingScales() throws { + let identity = AXTreeProcessIdentity(bundleID: "test.window", executablePath: "/fixture/window", launchTime: 1) + for pixelSize in [(2304, 1506), (1152, 753)] { + let input = try makeShot(width: pixelSize.0, height: pixelSize.1, pointWidth: 1152, pointHeight: 753, pixelsPerPoint: Double(pixelSize.0) / 1152) + let shot = try XCTUnwrap(Capture.boundedModelWindowShot(input)) + CommandRouter.clearShotTransformsForTesting() + defer { CommandRouter.clearShotTransformsForTesting() } + CommandRouter.recordShotTransform( + pid: 77, originX: shot.originX, originY: shot.originY, + pointWidth: shot.pointWidth, pointHeight: shot.pointHeight, + imageWidth: shot.width, imageHeight: shot.height, + processIdentity: identity, windowID: shot.windowID, pixelsPerPoint: shot.pixelsPerPoint + ) + let point = try CommandRouter.toGlobalPoint( + x: Double(shot.width) * 0.75, y: Double(shot.height) * 0.25, + pid: 77, currentProcessIdentity: identity, currentWindowID: shot.windowID + ) + XCTAssertEqual(point.x, input.originX + input.pointWidth * 0.75, accuracy: 0.000001) + XCTAssertEqual(point.y, input.originY + input.pointHeight * 0.25, accuracy: 0.000001) + XCTAssertThrowsError(try CommandRouter.toGlobalPoint( + x: Double(shot.width), y: 0, pid: 77, + currentProcessIdentity: identity, currentWindowID: shot.windowID + )) + } + } + + func testInvalidLargeImageCannotLeakAnUnboundedFallback() { + let shot = WindowShot(base64: "invalid", width: 2304, height: 1506, + originX: 0, originY: 0, pointWidth: 1152, pointHeight: 753, + windowID: 17, source: .screenshotManager) + XCTAssertNil(Capture.boundedModelWindowShot(shot)) + } + + func testFinalModelJPEGReallyDecodesAtTheSameSizeAndLeavesPNGEvidenceUntouched() throws { + let raw = try makeShot(width: 2304, height: 1506) + let bounded = try XCTUnwrap(Capture.boundedModelWindowShot(raw)) + let evidence = bounded.base64 + let model = Capture.modelWindowImage(bounded) + XCTAssertEqual(model.mimeType, "image/jpeg") + let data = try XCTUnwrap(Data(base64Encoded: model.base64)) + let source = try XCTUnwrap(CGImageSourceCreateWithData(data as CFData, nil)) + XCTAssertEqual(CGImageSourceGetType(source) as String?, "public.jpeg") + let image = try XCTUnwrap(CGImageSourceCreateImageAtIndex(source, 0, nil)) + XCTAssertEqual(image.width, bounded.width) + XCTAssertEqual(image.height, bounded.height) + XCTAssertEqual(bounded.base64, evidence) + XCTAssertEqual(Array(try XCTUnwrap(Data(base64Encoded: evidence)).prefix(8)), [137, 80, 78, 71, 13, 10, 26, 10]) + } + + func testJPEGQualityBoundsAndEncoderFailurePreserveLosslessFallback() throws { + let shot = try makeShot(width: 96, height: 64) + var qualities: [Double] = [] + for quality in [0.0, 0.9, 1.0] { + let image = Capture.modelWindowImage(shot, quality: quality) { _, q in + qualities.append(q) + return nil + } + XCTAssertEqual(image.mimeType, "image/png") + XCTAssertEqual(image.base64, shot.base64) + } + XCTAssertEqual(qualities, [0, 0.9, 1]) + for quality in [-0.1, 1.1, Double.nan, Double.infinity] { + let image = Capture.modelWindowImage(shot, quality: quality) { _, _ in + XCTFail("invalid quality must never reach the encoder") + return nil + } + XCTAssertEqual(image.mimeType, "image/png") + XCTAssertEqual(image.base64, shot.base64) + } + let empty = Capture.modelWindowImage(shot) { _, _ in "" } + XCTAssertEqual(empty.mimeType, "image/png") + let image = Capture.modelWindowImage(shot) + let data = try XCTUnwrap(Data(base64Encoded: image.base64)) + let source = try XCTUnwrap(CGImageSourceCreateWithData(data as CFData, nil)) + let decoded = try XCTUnwrap(CGImageSourceCreateImageAtIndex(source, 0, nil)) + XCTAssertEqual(decoded.width, 96) + XCTAssertEqual(decoded.height, 64) + } + + func testAlreadyFittedOfficialImagePreservesPixelsAndUniformScale() throws { + let fit = 768.0 / 769.0 + let input = try makeShot(width: 1397, height: 768, pointWidth: 1398, pointHeight: 769, pixelsPerPoint: fit) + let shot = try XCTUnwrap(Capture.boundedModelWindowShot(input)) + XCTAssertEqual(shot.base64, input.base64) + XCTAssertEqual(shot.width, 1397) + XCTAssertEqual(shot.height, 768) + XCTAssertEqual(shot.pixelsPerPoint, fit) + var encodings = 0 + let model = Capture.modelWindowImage(shot) { image, quality in + encodings += 1 + XCTAssertEqual(image.width, 1397) + XCTAssertEqual(image.height, 768) + XCTAssertEqual(quality, NativeScreenshotPolicy.jpegQuality) + return "jpeg-fixture" + } + XCTAssertEqual(encodings, 1) + XCTAssertEqual(model.mimeType, "image/jpeg") + XCTAssertEqual(model.base64, "jpeg-fixture") + } + + func testRetinaLosslessShotFitsOnceAndCarriesItsScaledUniformTransform() throws { + let raw = try makeShot(width: 2796, height: 1538, pointWidth: 1398, pointHeight: 769, pixelsPerPoint: 2) + let shot = try XCTUnwrap(Capture.boundedModelWindowShot(raw)) + XCTAssertEqual(shot.width, 1397) + XCTAssertEqual(shot.height, 768) + XCTAssertEqual(try XCTUnwrap(shot.pixelsPerPoint), 768.0 / 769.0, accuracy: 0.000000001) + let again = try XCTUnwrap(Capture.boundedModelWindowShot(shot)) + XCTAssertEqual(again.base64, shot.base64) + XCTAssertEqual(again.pixelsPerPoint, shot.pixelsPerPoint) + } + + private func makeShot(width: Int, height: Int, pointWidth: Double? = nil, pointHeight: Double? = nil, pixelsPerPoint: Double? = nil) throws -> WindowShot { + let context = try XCTUnwrap(CGContext( + data: nil, width: width, height: height, bitsPerComponent: 8, bytesPerRow: 0, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue + )) + context.setFillColor(CGColor(red: 0.2, green: 0.4, blue: 0.6, alpha: 1)) + context.fill(CGRect(x: 0, y: 0, width: width, height: height)) + let image = try XCTUnwrap(context.makeImage()) + let encoded = try XCTUnwrap(Capture.pngBase64WithSize(image)) + return WindowShot( + base64: encoded.base64, width: encoded.width, height: encoded.height, + originX: -600, originY: 200, pointWidth: pointWidth ?? Double(width), pointHeight: pointHeight ?? Double(height), + windowID: 17, source: .streamBackedScreenshot, pixelsPerPoint: pixelsPerPoint + ) + } +} diff --git a/native/cu-helper/Tests/CuHelperTests/RendererAttributeReuseTests.swift b/native/cu-helper/Tests/CuHelperTests/RendererAttributeReuseTests.swift new file mode 100644 index 00000000..d68318c5 --- /dev/null +++ b/native/cu-helper/Tests/CuHelperTests/RendererAttributeReuseTests.swift @@ -0,0 +1,50 @@ +import XCTest +@testable import cc_haha_computer_use + +@MainActor +final class RendererAttributeReuseTests: XCTestCase { + func testKnownFingerprintScalarsAndSuccessfulSettableAvoidBackendRepeats() { + var reads = 0 + func read(_ value: String) -> String { reads += 1; return value } + let observation = RendererAttributeReuse( + title: read("Title"), description: read("Description"), role: read("AXButton") + ) + XCTAssertEqual(observation.title { read("Title") }, "Title") + XCTAssertEqual(observation.description { read("Description") }, "Description") + XCTAssertEqual(observation.role { read("AXButton") }, "AXButton") + for _ in 0..<2 { + XCTAssertEqual(observation.settable { reads += 1; return false }, false) + } + XCTAssertEqual(reads, 4, "three fingerprint reads plus one settable query, instead of eight") + } + + func testMissingScalarsRetryAndCacheOnlyLaterSuccess() { + let observation = RendererAttributeReuse(title: nil, description: nil, role: nil) + var reads = 0 + for get in [observation.title, observation.description, observation.role] { + XCTAssertNil(get { reads += 1; return nil }) + XCTAssertEqual(get { reads += 1; return "recovered" }, "recovered") + XCTAssertEqual(get { reads += 1; return "changed" }, "recovered") + } + XCTAssertEqual(reads, 6) + } + + func testSettableFailureRetriesButSuccessfulFalseDoesNot() { + let observation = RendererAttributeReuse(title: nil, description: nil, role: nil) + var reads = 0 + XCTAssertNil(observation.settable { reads += 1; return nil }) + XCTAssertEqual(observation.settable { reads += 1; return false }, false) + XCTAssertEqual(observation.settable { reads += 1; return true }, false) + XCTAssertEqual(reads, 2) + } + + func testSeparateNodesAndRenderPassesNeverShareObservations() { + let first = RendererAttributeReuse(title: "Same", description: nil, role: "AXButton") + let duplicate = RendererAttributeReuse(title: "Same", description: nil, role: "AXButton") + XCTAssertEqual(first.settable { true }, true) + XCTAssertEqual(duplicate.settable { false }, false) + let nextPass = RendererAttributeReuse(title: "Changed", description: nil, role: "AXButton") + XCTAssertEqual(nextPass.title { "wrong" }, "Changed") + XCTAssertEqual(nextPass.settable { false }, false) + } +} diff --git a/native/cu-helper/Tests/CuHelperTests/ResolvedTargetAuthorizationTests.swift b/native/cu-helper/Tests/CuHelperTests/ResolvedTargetAuthorizationTests.swift index ab783cf9..29670666 100644 --- a/native/cu-helper/Tests/CuHelperTests/ResolvedTargetAuthorizationTests.swift +++ b/native/cu-helper/Tests/CuHelperTests/ResolvedTargetAuthorizationTests.swift @@ -66,15 +66,48 @@ final class ResolvedTargetAuthorizationTests: XCTestCase { } } - func testNumericPIDCannotBypassDeniedResolvedBundle() { - XCTAssertThrowsError( - try ResolvedTargetAuthorization.authorize( - pid: 41, - identity: terminalIdentity, - expectedBundleID: nil + func testNumericPIDAllowsTerminalAfterGlobalEnablement() throws { + let target = try ResolvedTargetAuthorization.authorize( + pid: 41, + identity: terminalIdentity, + expectedBundleID: nil + ) + + XCTAssertEqual(target.pid, 41) + XCTAssertEqual(target.identity, terminalIdentity) + } + + func testEveryAppCategoryAndHostAreAllowedWithProvenProcessIdentity() throws { + let bundleIDs = [ + "com.google.Chrome", + "com.apple.Safari", + "com.apple.Terminal", + "com.microsoft.VSCode", + "com.apple.shortcuts", + "com.webull.desktop.v1", + "com.binance.BinanceDesktop", + "com.ledger.live", + "com.spotify.client", + "com.apple.Music", + "com.amazon.Kindle", + "com.claude-code-haha.desktop", + "dev.cchaha.cu-helper", + "com.example.custom-host", + "com.example.new-app", + ] + for bundleID in bundleIDs { + let identity = AXTreeProcessIdentity( + bundleID: bundleID, + executablePath: "/Applications/Fixture.app/Contents/MacOS/Fixture", + launchTime: 100 ) - ) { - XCTAssertEqual(($0 as? CUError)?.code, "app_denied") + let target = try ResolvedTargetAuthorization.authorize( + pid: 41, + identity: identity, + expectedBundleID: bundleID + ) + XCTAssertEqual(target.pid, 41, bundleID) + XCTAssertEqual(target.identity, identity, bundleID) } } @@ -96,18 +129,16 @@ final class ResolvedTargetAuthorizationTests: XCTestCase { let resolved = try XCTUnwrap( AppTargetResolver.resolve(selector: selector, candidates: [terminal]) ) - XCTAssertThrowsError( - try ResolvedTargetAuthorization.authorize( - resolved: resolved, - currentIdentity: terminalIdentity - ) - ) { - XCTAssertEqual(($0 as? CUError)?.code, "app_denied") - } + let target = try ResolvedTargetAuthorization.authorize( + resolved: resolved, + currentIdentity: terminalIdentity + ) + XCTAssertEqual(target.pid, 41) + XCTAssertEqual(target.identity, terminalIdentity) } } - func testWorktreePathResolutionStillReachesIntrinsicSelfControlDenial() throws { + func testWorktreeHostPathResolutionAuthorizesExactProcess() throws { let installed = AppTargetCandidate( pid: 100, bundleIdentifier: "com.claude-code-haha.desktop", @@ -133,21 +164,15 @@ final class ResolvedTargetAuthorizationTests: XCTestCase { ) XCTAssertEqual(resolved.pid, worktree.pid) - XCTAssertThrowsError( - try ResolvedTargetAuthorization.authorize( - resolved: resolved, - currentIdentity: identity - ) - ) { - XCTAssertEqual(($0 as? CUError)?.code, "app_denied") - XCTAssertEqual( - ($0 as? CUError)?.message, - "Computer Use is not allowed to use the app 'com.claude-code-haha.desktop' for safety reasons." - ) - } + let target = try ResolvedTargetAuthorization.authorize( + resolved: resolved, + currentIdentity: identity + ) + XCTAssertEqual(target.pid, worktree.pid) + XCTAssertEqual(target.identity, identity) } - func testOmittedFrontmostAndLaunchedTargetsUseSameActualBundlePolicy() { + func testOmittedFrontmostAndLaunchedTargetsUseSameActualBundlePolicy() throws { // Both paths ultimately produce this same resolved target shape. The // authorizer intentionally has no selector-specific bypass. let resolved = ResolvedAppTarget( @@ -157,14 +182,12 @@ final class ResolvedTargetAuthorizationTests: XCTestCase { ) for _ in ["omitted-frontmost", "launched-get-app-state"] { - XCTAssertThrowsError( - try ResolvedTargetAuthorization.authorize( - resolved: resolved, - currentIdentity: terminalIdentity - ) - ) { - XCTAssertEqual(($0 as? CUError)?.code, "app_denied") - } + let target = try ResolvedTargetAuthorization.authorize( + resolved: resolved, + currentIdentity: terminalIdentity + ) + XCTAssertEqual(target.pid, 41) + XCTAssertEqual(target.identity, terminalIdentity) } } diff --git a/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift b/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift index a603932e..1b8b3bbe 100644 --- a/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift +++ b/native/cu-helper/Tests/CuHelperTests/WindowCaptureStreamTests.swift @@ -707,7 +707,7 @@ final class WindowCaptureStreamTests: XCTestCase { )) } - func testCopiedBGRAFrameEncodesAsAStreamJPEGWithTheSameGeometry() throws { + func testCopiedBGRAFramePreservesLosslessEvidenceAndPresentsJPEGAtTheSameGeometry() throws { let target = makeTarget( windowID: 72, pixelWidth: 1, @@ -725,9 +725,14 @@ final class WindowCaptureStreamTests: XCTestCase { ) let shot = try XCTUnwrap(Capture.windowShot(from: frame, target: target)) - let jpeg = try XCTUnwrap(Data(base64Encoded: shot.base64)) - + let evidence = try XCTUnwrap(Data(base64Encoded: shot.base64)) + XCTAssertEqual(Array(evidence.prefix(8)), [137, 80, 78, 71, 13, 10, 26, 10]) + XCTAssertEqual(shot.mimeType, "image/png") + let model = Capture.modelWindowImage(shot) + let jpeg = try XCTUnwrap(Data(base64Encoded: model.base64)) XCTAssertEqual(Array(jpeg.prefix(2)), [255, 216]) + XCTAssertEqual(model.mimeType, "image/jpeg") + XCTAssertEqual(shot.base64, evidence.base64EncodedString()) XCTAssertEqual(shot.width, 1) XCTAssertEqual(shot.height, 1) XCTAssertEqual(shot.originX, 300) diff --git a/src/server/__tests__/mac-installed-apps.test.ts b/src/server/__tests__/mac-installed-apps.test.ts index 4452ab65..c1c3e54a 100644 --- a/src/server/__tests__/mac-installed-apps.test.ts +++ b/src/server/__tests__/mac-installed-apps.test.ts @@ -115,25 +115,49 @@ describe('macOS installed app enumeration', () => { ]) }) - it('filters the built-in host, helper, and an additional configured host', async () => { + it('includes the built-in host, helper, and an additional configured host like any other app', async () => { const metadata = new Map([ ['Desktop.app', { bundleId: 'com.claude-code-haha.desktop', displayName: 'Claude Code Haha' }], ['Helper.app', { bundleId: 'dev.cchaha.cu-helper', displayName: 'Computer Use Helper' }], ['Custom.app', { bundleId: 'com.example.custom-host', displayName: 'Custom Host' }], ['Notes.app', { bundleId: 'com.example.notes', displayName: 'Notes' }], ]) - const apps = await listInstalledMacApps({ - roots: ['/Applications'], - hostBundleId: 'com.example.custom-host', - readDirectory: async () => [...metadata.keys()].map(name => entry(name)), - canonicalize: async candidate => candidate, - readMetadata: async appPath => metadata.get(appPath.split('/').at(-1) ?? '') ?? null, - }) + const previousHost = process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID + process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID = 'com.example.custom-host' + let apps: Awaited> + try { + apps = await listInstalledMacApps({ + roots: ['/Applications'], + readDirectory: async () => [...metadata.keys()].map(name => entry(name)), + canonicalize: async candidate => candidate, + readMetadata: async appPath => metadata.get(appPath.split('/').at(-1) ?? '') ?? null, + }) + } finally { + if (previousHost === undefined) delete process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID + else process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID = previousHost + } - expect(apps).toEqual([{ - bundleId: 'com.example.notes', - displayName: 'Notes', - path: '/Applications/Notes.app', - }]) + expect(apps).toEqual([ + { + bundleId: 'com.claude-code-haha.desktop', + displayName: 'Claude Code Haha', + path: '/Applications/Desktop.app', + }, + { + bundleId: 'dev.cchaha.cu-helper', + displayName: 'Computer Use Helper', + path: '/Applications/Helper.app', + }, + { + bundleId: 'com.example.custom-host', + displayName: 'Custom Host', + path: '/Applications/Custom.app', + }, + { + bundleId: 'com.example.notes', + displayName: 'Notes', + path: '/Applications/Notes.app', + }, + ]) }) }) diff --git a/src/server/api/macInstalledApps.ts b/src/server/api/macInstalledApps.ts index 380f794c..4ef57178 100644 --- a/src/server/api/macInstalledApps.ts +++ b/src/server/api/macInstalledApps.ts @@ -2,7 +2,6 @@ import type { Dirent } from 'node:fs' import { readdir, realpath } from 'node:fs/promises' import { homedir } from 'node:os' import path from 'node:path' -import { isIntrinsicAppDenied } from '../../vendor/computer-use-mcp/deniedApps.js' export type InstalledMacApp = { bundleId: string @@ -21,7 +20,6 @@ type InstalledAppDependencies = { ) => Promise<{ bundleId: string; displayName: string } | null> maxDepth?: number entryLimit?: number - hostBundleId?: string } const STANDARD_APPLICATION_ROOTS = [ @@ -89,8 +87,6 @@ export async function listInstalledMacApps( const readMetadata = deps.readMetadata ?? readMetadataWithPlutil const maxDepth = deps.maxDepth ?? DEFAULT_MAX_DEPTH const entryLimit = deps.entryLimit ?? DEFAULT_ENTRY_LIMIT - const hostBundleId = deps.hostBundleId - ?? process.env.CC_HAHA_COMPUTER_USE_HOST_BUNDLE_ID const byBundleId = new Map() for (const configuredRoot of roots) { @@ -144,7 +140,6 @@ export async function listInstalledMacApps( const metadata = await readMetadata(canonicalCandidate) if (!metadata?.bundleId || !metadata.displayName) continue - if (isIntrinsicAppDenied(metadata.bundleId, hostBundleId)) continue if (byBundleId.has(metadata.bundleId)) continue byBundleId.set(metadata.bundleId, { bundleId: metadata.bundleId, diff --git a/src/skills/bundled/computerUse.test.ts b/src/skills/bundled/computerUse.test.ts index 65f7600c..775bef0f 100644 --- a/src/skills/bundled/computerUse.test.ts +++ b/src/skills/bundled/computerUse.test.ts @@ -182,3 +182,21 @@ describe('computer-use observation batching', () => { expect(getComputerUsePrompt('win32')).not.toContain('cua.getApp') }) }) + +test('macOS advertises bounded sequence and real key macros without changing Windows tools', () => { + expect(getComputerUseToolAllowlist('darwin')).toContain('mcp__computer-use__js') + expect(getComputerUseToolAllowlist('win32')).not.toContain('mcp__computer-use__sequence') + const prompt = getComputerUsePrompt('darwin') + expect(prompt).toContain('s x 1 period 3 5 Return') + expect(prompt).toContain('account for actions that already ran') + expect(prompt).toContain('never replay the whole batch') +}) + +test('uses the sequence observation without an extra model round trip', () => { + // Live Blender trial: individual click -> receipt -> model -> get_app_state + // added a full provider round trip just to observe a single known action. + const prompt = getComputerUsePrompt('darwin') + expect(prompt).toContain('then observe at the next') + expect(prompt).toContain('Do not force one model round trip per click') + expect(prompt).toContain('Observe through the JS') +}) diff --git a/src/skills/bundled/computerUse.ts b/src/skills/bundled/computerUse.ts index e195c3f4..69e0315f 100644 --- a/src/skills/bundled/computerUse.ts +++ b/src/skills/bundled/computerUse.ts @@ -48,6 +48,16 @@ Do not add a fixed sleep before an observation. The observation path waits for UI changes when needed. Read its result before deciding whether more context is necessary. +## Blender keyboard input + +For Blender, ordinary \`app.typeText\` can leave text unchanged. Use +\`await app.pressKey("s x 1 period 3 5 Return")\` for a known numeric transform, +or \`await app.pressKey("a d d space c u b e")\` in a known search field. +These send actual key presses. Use short macros and observe through the JS App +after opening an unfamiliar dialog; never replay the whole batch after failure. +Use \`space\` for a literal space, \`period\` for a decimal point, and +\`minus\` for a negative sign. + ## Naming the app Pass the app name straight to \`cua.getApp\` — display name, bundle identifier, diff --git a/src/utils/computerUse/cuHelperDaemon.test.ts b/src/utils/computerUse/cuHelperDaemon.test.ts index f3fbb939..75bcb6ab 100644 --- a/src/utils/computerUse/cuHelperDaemon.test.ts +++ b/src/utils/computerUse/cuHelperDaemon.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, test } from 'bun:test' +import { afterEach, describe, expect, spyOn, test } from 'bun:test' import { EventEmitter } from 'node:events' import { lstatSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from 'node:fs' import { tmpdir } from 'node:os' @@ -173,6 +173,33 @@ describe('cu-helper daemon system commands', () => { }) describe('cu-helper daemon failure classification', () => { + test('only multi-chord keyboard requests extend both native deadline and response timer to a bounded budget', async () => { + const socket = new FakeSocket() + __setDaemonSocketForTests(socket as never) + const timerSpy = spyOn(globalThis, 'setTimeout') + try { + const cases = [ + ['press_key', { key: 'a b c d' }, 60_000], + ['press_key', { key: 'ctrl + a' }, 20_000], + ['press_key', { key: 'Return' }, 20_000], + ['get_app_state', { key: 'a b' }, 20_000], + ] as const + for (const [index, [command, payload, budget]] of cases.entries()) { + const before = Date.now() + const request = callDaemon(command, payload) + await waitForWriteCount(socket, index + 1) + const envelope = JSON.parse(socket.writes[index]!) + expect(envelope.deadlineUnixMilliseconds).toBeGreaterThanOrEqual(before + budget) + expect(envelope.deadlineUnixMilliseconds).toBeLessThanOrEqual(Date.now() + budget) + expect(timerSpy.mock.calls.at(-1)?.[1]).toBe(budget) + reply(socket, index, { ok: true, result: true }) + await expect(request).resolves.toBe(true) + } + } finally { + timerSpy.mockRestore() + } + }) + test('helper installation/start resolution failure is daemon infrastructure failure', async () => { // A bare executable can satisfy the availability probe but cannot be // launched as the helper .app daemon. The bridge must be allowed to use the diff --git a/src/utils/computerUse/cuHelperDaemon.ts b/src/utils/computerUse/cuHelperDaemon.ts index 54f7bfb3..64e897c7 100644 --- a/src/utils/computerUse/cuHelperDaemon.ts +++ b/src/utils/computerUse/cuHelperDaemon.ts @@ -658,13 +658,20 @@ function dispatchDaemonCommand( const isTurnScoped = !CONNECTION_SCOPED_COMMANDS.has(command) const turnId = activeTurnId ?? (isTurnScoped ? (activeTurnId = randomUUID()) : `connection-${state.generation}`) + // Native keyboard macros now complete each chord through its own input + // boundary. Keep the native deadline and client timer aligned, without + // extending screenshots, clicks, or single-key requests. This counts only + // separators for budgeting; native KeyMapping remains the validating parser. + const multipleChords = command === 'press_key' && typeof payload.key === 'string' + && payload.key.replace(/\s*\+\s*/g, '+').trim().split(/\s+/).length > 1 + const timeoutMs = multipleChords ? Math.min(60_000, requestTimeoutMs * 3) : requestTimeoutMs const request = { id, requestId: id, cmd: command, payload, clientApiVersion: CU_HELPER_PROTOCOL_VERSION, - deadlineUnixMilliseconds: Date.now() + requestTimeoutMs, + deadlineUnixMilliseconds: Date.now() + timeoutMs, sessionId: getSessionId(), turnId, } @@ -680,7 +687,7 @@ function dispatchDaemonCommand( // Retire a daemon that missed its response deadline. Other requests that // were already in flight are also result-unknown, never replayable infra. resetState(`command ${command} timed out`, state.generation) - }, requestTimeoutMs) + }, timeoutMs) state.pending.set(id, { resolve: resolve as (v: unknown) => void, reject, timer }) try { state.socket.write(`${JSON.stringify(request)}\n`) diff --git a/src/utils/computerUse/preauthorizedConfig.test.ts b/src/utils/computerUse/preauthorizedConfig.test.ts index c941820d..00660e57 100644 --- a/src/utils/computerUse/preauthorizedConfig.test.ts +++ b/src/utils/computerUse/preauthorizedConfig.test.ts @@ -116,7 +116,7 @@ describe('resolveStoredComputerUseConfig', () => { }) }) - test('derives least-privilege tiers and filters policy-denied pre-authorizations', () => { + test('legacy pre-authorizations no longer restrict app categories after global consent', () => { expect( buildPreAuthorizedAppGrants([ { @@ -141,13 +141,13 @@ describe('resolveStoredComputerUseConfig', () => { bundleId: 'com.google.Chrome', displayName: 'Google Chrome', grantedAt: 1234, - tier: 'read', + tier: 'full', }, { bundleId: 'com.apple.Terminal', displayName: 'Terminal', grantedAt: 1234, - tier: 'click', + tier: 'full', }, { bundleId: 'com.apple.Preview', @@ -155,6 +155,12 @@ describe('resolveStoredComputerUseConfig', () => { grantedAt: 1234, tier: 'full', }, + { + bundleId: 'com.spotify.client', + displayName: 'Spotify', + grantedAt: 1234, + tier: 'full', + }, ]) }) }) diff --git a/src/utils/computerUse/wrapper.test.ts b/src/utils/computerUse/wrapper.test.ts index 511d863c..01dcc4ee 100644 --- a/src/utils/computerUse/wrapper.test.ts +++ b/src/utils/computerUse/wrapper.test.ts @@ -94,3 +94,133 @@ test('a host Escape callback aborts its turn outside any dispatch async context' expect(turnController.signal.aborted).toBe(true) expect(nextController.signal.aborted).toBe(false) }) + +import { withComputerUseToolContext } from './wrapper.js' + +test('CU async contexts preserve each originating turn cancellation across concurrent awaits', async () => { + const first = new AbortController() + const second = new AbortController() + const shared = buildSessionContext() + let resume!: () => void + let ready!: () => void + const pending = new Promise(resolve => { resume = resolve }) + const started = new Promise(resolve => { ready = resolve }) + const firstRun = withComputerUseToolContext({ abortController: first } as ToolUseContext, async () => { + expect(shared.isAborted?.()).toBe(false) + ready() + await pending + expect(shared.isAborted?.()).toBe(true) + }) + await started + await withComputerUseToolContext({ abortController: second } as ToolUseContext, async () => { + first.abort() + await Promise.resolve() + expect(shared.isAborted?.()).toBe(false) + }) + resume() + await firstRun +}) + +import { spyOn } from 'bun:test' +import * as hostAdapterModule from './hostAdapter.js' +import * as lockModule from './computerUseLock.js' +import * as gateModule from './gates.js' +import * as debugModule from '../debug.js' +import type { ComputerUseHostAdapter } from '../../vendor/computer-use-mcp/types.js' +import type { CodexComputerEngine, ComputerExecutor } from '../../vendor/computer-use-mcp/executor.js' + +test('real CU call wrapper delivers sequence metadata/images and releases fresh locks cancelled during acquisition', async () => { + // Narrow, restored spies only; a separately imported wrapper keeps its cached + // binder out of other tests. No helper command, real lock, user config or UI. + const engineCalls: string[] = [] + let failInput = false + const engine = { + async resolveTarget() { + engineCalls.push('resolve') + return { pid: 420, bundleId: 'com.test.blender', launchTime: 100, executablePath: '/fixture/Blender' } + }, + async pressKey() { + engineCalls.push('pressKey') + if (failInput) throw new Error('fixture response failure after dispatch') + }, + async getAppState() { + engineCalls.push('getAppState') + return { pid: 420, appName: 'Fixture Blender', bundleId: 'com.test.blender', windowTitle: 'Untitled', + elementCount: 0, truncated: false, durationMs: 1, axText: 'Fixture state', + screenshot: { base64: 'Zml4dHVyZS1wbmc=', width: 1, height: 1 } } + }, + } as unknown as CodexComputerEngine + const adapter = { + serverName: 'computer-use', + logger: { silly() {}, debug() {}, info() {}, warn() {}, error() {} }, + executor: { capabilities: { platform: 'darwin', screenshotFiltering: 'native' }, engine } as ComputerExecutor, + ensureOsPermissions: async () => ({ granted: true }), + isDisabled: () => false, + } as ComputerUseHostAdapter + const adapterSpy = spyOn(hostAdapterModule, 'getComputerUseHostAdapter').mockReturnValue(adapter) + const checkSpy = spyOn(lockModule, 'checkComputerUseLock').mockResolvedValue({ kind: 'held_by_self' }) + const acquireSpy = spyOn(lockModule, 'tryAcquireComputerUseLock').mockResolvedValue({ kind: 'acquired', fresh: true }) + const releaseSpy = spyOn(lockModule, 'releaseComputerUseLock').mockResolvedValue(true) + const coordSpy = spyOn(gateModule, 'getChicagoCoordinateMode').mockReturnValue('pixels') + const debugSpy = spyOn(debugModule, 'logForDebugging').mockImplementation(() => {}) + try { + const moduleUrl = new URL('./wrapper.tsx', import.meta.url) + moduleUrl.search = '?cu-wrapper-delivery-fixture' + const wrapper = await import(moduleUrl.href) as typeof import('./wrapper.js') + const makeContext = (abortController = new AbortController()) => ({ + abortController, getAppState: () => ({ computerUseMcpState: undefined }), + setAppState() {}, sendOSNotification() {}, + }) as unknown as ToolUseContext + const call = wrapper.getComputerUseMCPToolOverrides('sequence').call + const args = { app: 'Fixture Blender', steps: [{ tool: 'press_key', key: 'a b c' }] } + const delivered = await call(args, makeContext()) + const blocks = delivered.data as Array> + expect(engineCalls).toEqual(['resolve', 'pressKey', 'getAppState']) + expect(JSON.parse(blocks[0]!.text)).toMatchObject({ status: 'completed', completedSteps: 1, resultUnknown: false }) + expect(blocks.find(block => block.type === 'image')).toEqual({ + type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'Zml4dHVyZS1wbmc=' }, + }) + expect(blocks.some(block => block.type === 'text' && block.text.includes('Fixture state'))).toBe(true) + + // The wrapper must deliver the actual error and structured partial outcome, + // not replace it with a generic successful Computer Use result. + failInput = true + const failed = await call(args, makeContext()).then(() => '', error => String(error)) + expect(failed).toContain('fixture response failure after dispatch') + expect(failed).toContain('"completedSteps":0') + expect(failed).toContain('"failedStepIndex":0') + expect(failed).toContain('"resultUnknown":true') + expect(debugSpy).toHaveBeenCalled() + + // Real buildSessionContext acquire callback, with only file-lock IO mocked. + let enterAcquire!: () => void + let finishAcquire!: () => void + const entered = new Promise(resolve => { enterAcquire = resolve }) + const holdAcquire = new Promise(resolve => { finishAcquire = resolve }) + acquireSpy.mockImplementation(async () => { + enterAcquire() + await holdAcquire + return { kind: 'acquired', fresh: true } + }) + const abort = new AbortController() + const acquiring = wrapper.withComputerUseToolContext(makeContext(abort), () => wrapper.buildSessionContext().acquireCuLock!()) + await entered + abort.abort() + finishAcquire() + await expect(acquiring).rejects.toThrow('cancelled during lock acquisition') + expect(releaseSpy).toHaveBeenCalledTimes(1) + + const callsBefore = acquireSpy.mock.calls.length + await expect(wrapper.withComputerUseToolContext(makeContext(abort), () => wrapper.buildSessionContext().acquireCuLock!())) + .rejects.toThrow('cancelled before lock acquisition') + expect(acquireSpy.mock.calls.length).toBe(callsBefore) + + // A normal reentrant acquisition continues and does not release another + // operation's existing session lock. + acquireSpy.mockResolvedValue({ kind: 'acquired', fresh: false }) + await wrapper.withComputerUseToolContext(makeContext(), () => wrapper.buildSessionContext().acquireCuLock!()) + expect(releaseSpy).toHaveBeenCalledTimes(1) + } finally { + for (const spy of [adapterSpy, checkSpy, acquireSpy, releaseSpy, coordSpy, debugSpy]) spy.mockRestore() + } +}) diff --git a/src/utils/computerUse/wrapper.tsx b/src/utils/computerUse/wrapper.tsx index 02c7e907..41df9297 100644 --- a/src/utils/computerUse/wrapper.tsx +++ b/src/utils/computerUse/wrapper.tsx @@ -20,7 +20,7 @@ import { bindSessionContext, type ComputerUseSessionContext, type CuCallToolResu import { getSessionId } from '../../bootstrap/state.js'; import type { Tool, ToolUseContext } from '../../Tool.js'; import { logForDebugging } from '../debug.js'; -import { checkComputerUseLock, tryAcquireComputerUseLock } from './computerUseLock.js'; +import { checkComputerUseLock, tryAcquireComputerUseLock, releaseComputerUseLock } from './computerUseLock.js'; import { registerEscHotkey } from './escHotkey.js'; import { getChicagoCoordinateMode } from './gates.js'; import { getComputerUseHostAdapter } from './hostAdapter.js'; @@ -42,6 +42,10 @@ type Binding = { */ let binding: Binding | undefined; const toolUseContexts = new AsyncLocalStorage(); +/** Preserve the originating turn context across queued/native awaits. */ +export function withComputerUseToolContext(context: ToolUseContext, run: () => T): T { + return toolUseContexts.run(context, run); +} const ENABLED_GRANT_FLAGS = { clipboardRead: true, clipboardWrite: true, @@ -74,6 +78,7 @@ export function buildSessionContext(): ComputerUseSessionContext { // returning the legacy shapes for protocol compatibility, but do not // consume persisted app grants or open runtime permission prompts. getAllowedApps: () => [], + isAborted: () => tuc().abortController.signal.aborted, getGrantFlags: () => ENABLED_GRANT_FLAGS, getUserDeniedBundleIds: () => [], getSelectedDisplayId: () => tuc().getAppState().computerUseMcpState?.selectedDisplayId, @@ -196,7 +201,13 @@ export function buildSessionContext(): ComputerUseSessionContext { // but is possible under parallel tool-use interleaving — don't spam the // notification in that case. acquireCuLock: async () => { + const signal = tuc().abortController.signal; + if (signal.aborted) throw new Error("Computer Use cancelled before lock acquisition"); const r = await tryAcquireComputerUseLock(); + if (signal.aborted) { + if (r.kind === 'acquired' && r.fresh) await releaseComputerUseLock(); + throw new Error("Computer Use cancelled during lock acquisition"); + } if (r.kind === 'blocked') { throw new Error(formatLockHeld(r.by)); } diff --git a/src/vendor/computer-use-mcp/appTargetPolicyParity.test.ts b/src/vendor/computer-use-mcp/appTargetPolicyParity.test.ts deleted file mode 100644 index 5f1afba8..00000000 --- a/src/vendor/computer-use-mcp/appTargetPolicyParity.test.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { expect, test } from 'bun:test' -import { readFileSync } from 'node:fs' -import { resolve } from 'node:path' - -import { _test as deniedApps } from './deniedApps.js' -import { NATIVE_FORBIDDEN_BUNDLE_IDS } from './nativeAppPolicy.js' - -const SWIFT_SET_MARKER = 'static let deniedBundleIDs: Set = [' -const SWIFT_INTRINSIC_SET_MARKER = 'static let intrinsicDeniedBundleIDs: Set = [' - -function parseNativeDeniedBundleIds(source: string, marker: string): string[] { - const markerIndex = source.indexOf(marker) - expect(markerIndex).toBeGreaterThanOrEqual(0) - - const bodyStart = markerIndex + marker.length - const bodyEnd = source.indexOf('\n ]', bodyStart) - expect(bodyEnd).toBeGreaterThan(bodyStart) - - const body = source.slice(bodyStart, bodyEnd) - return [...body.matchAll(/^\s*"([^"]+)",?\s*(?:\/\/.*)?$/gm)].map(match => match[1]) -} - -test('native deny policy matches the official 24 exact forbidden identities', () => { - const tsEntries = [...NATIVE_FORBIDDEN_BUNDLE_IDS] - const expected = new Set(tsEntries) - - const swiftPath = resolve( - import.meta.dir, - '../../../native/cu-helper/Sources/cu-helper/AppTargetPolicy.swift', - ) - const nativeEntries = parseNativeDeniedBundleIds( - readFileSync(swiftPath, 'utf8'), - SWIFT_SET_MARKER, - ) - const actual = new Set(nativeEntries) - - const missing = [...expected].filter(bundleId => !actual.has(bundleId)).sort() - const extra = [...actual].filter(bundleId => !expected.has(bundleId)).sort() - - expect(tsEntries).toHaveLength(expected.size) - expect(nativeEntries).toHaveLength(actual.size) - expect(expected.size).toBe(24) - expect(actual.size).toBe(24) - expect(missing).toEqual([]) - expect(extra).toEqual([]) - expect([...actual].sort()).toEqual([...expected].sort()) - // Browser classification still exists for the Windows tool tier. Native - // macOS control may use the same browser as the official Codex app surface. - expect(deniedApps.BROWSER_BUNDLE_IDS.size).toBe(29) - expect([...deniedApps.BROWSER_BUNDLE_IDS].filter(bundleId => actual.has(bundleId))).toEqual([]) -}) - -test('native intrinsic deny set stays separate and matches the TS host/helper defaults', () => { - const swiftPath = resolve( - import.meta.dir, - '../../../native/cu-helper/Sources/cu-helper/AppTargetPolicy.swift', - ) - const nativeEntries = parseNativeDeniedBundleIds( - readFileSync(swiftPath, 'utf8'), - SWIFT_INTRINSIC_SET_MARKER, - ) - const expected = deniedApps.INTRINSIC_DENIED_BUNDLE_IDS - - expect(new Set(nativeEntries)).toEqual(expected) - expect(expected).toEqual(new Set([ - 'com.claude-code-haha.desktop', - 'dev.cchaha.cu-helper', - ])) - expect(nativeEntries).toHaveLength(2) -}) diff --git a/src/vendor/computer-use-mcp/deniedApps.test.ts b/src/vendor/computer-use-mcp/deniedApps.test.ts new file mode 100644 index 00000000..3ba97d84 --- /dev/null +++ b/src/vendor/computer-use-mcp/deniedApps.test.ts @@ -0,0 +1,35 @@ +import { expect, test } from 'bun:test' +import { + categoryToTier, + getDefaultTierForApp, + getDeniedCategory, + getDeniedCategoryByDisplayName, + getDeniedCategoryForApp, + isPolicyDenied, +} from './deniedApps.js' + +// Legacy consumers still use these exports. Neither an old category value nor +// an app lookup may reintroduce restrictions after feature-wide consent. +test('legacy category values cannot downgrade global Computer Use consent', () => { + for (const category of ['browser', 'terminal', 'trading', null] as const) { + expect(categoryToTier(category)).toBe('full') + } +}) + +test.each([ + ['com.google.Chrome', 'Google Chrome'], + ['com.apple.Terminal', 'Terminal'], + ['com.spotify.client', 'Spotify'], + ['com.webull.desktop.v1', 'Webull'], + ['com.claude-code-haha.desktop', 'Claude Code Haha'], + ['dev.cchaha.cu-helper', 'Computer Use Helper'], + ['org.example.new-app', 'New App'], +])('legacy lookup for %s grants the same access with or without a bundle ID', (bundleId, displayName) => { + expect(getDeniedCategory(bundleId)).toBeNull() + expect(getDeniedCategoryByDisplayName(displayName)).toBeNull() + for (const id of [bundleId, undefined]) { + expect(getDeniedCategoryForApp(id, displayName)).toBeNull() + expect(isPolicyDenied(id, displayName)).toBe(false) + expect(getDefaultTierForApp(id, displayName)).toBe('full') + } +}) diff --git a/src/vendor/computer-use-mcp/deniedApps.ts b/src/vendor/computer-use-mcp/deniedApps.ts index 8e872698..400c390f 100644 --- a/src/vendor/computer-use-mcp/deniedApps.ts +++ b/src/vendor/computer-use-mcp/deniedApps.ts @@ -1,589 +1,44 @@ /** - * App category lookup for tiered CU permissions. Three categories land at a - * restricted tier instead of `"full"`: + * Computer Use is authorized once through the global enable dialog. All + * apps have full access regardless of category, bundle ID, display name, or host/helper identity. * - * - **browser** → `"read"` tier — visible in screenshots, NO interaction. - * The model can read an already-open page but must use the Claude-in-Chrome - * MCP for navigation/clicking/typing. - * - **terminal** → `"click"` tier — visible + clickable, NO typing. The - * model can click a Run button or scroll test output in an IDE, but can't - * type into the integrated terminal. Use the Bash tool for shell work. - * - **trading** → `"read"` tier — same restrictions as browsers, but no - * CiC-MCP alternative exists. For platforms where a stray click can - * execute a trade or send a message to a counterparty. - * - * Uncategorized apps default to `"full"`. See `getDefaultTierForApp`. - * - * Identification is two-layered: - * 1. Bundle ID match (macOS-only; `InstalledApp.bundleId` is a - * CFBundleIdentifier and meaningless on Windows). Fast, exact, the - * primary mechanism while CU is darwin-gated. - * 2. Display-name substring match (cross-platform fallback). Catches - * unresolved requests ("Chrome" when Chrome isn't installed) AND will - * be the primary mechanism on Windows/Linux where there's no bundle ID. - * Windows-relevant names (PowerShell, cmd, Windows Terminal) are - * included now so they activate the moment the darwin gate lifts. - * - * Keep this file **import-free** (like sentinelApps.ts) — the renderer may - * import it via a package.json subpath export, and pulling in - * `@modelcontextprotocol/sdk` (a devDep) through the index → mcpServer chain - * would fail module resolution in Next.js. The `CuAppPermTier` type is - * duplicated as a string literal below rather than imported. + * Keep the legacy lookup exports for the Windows dispatcher and stored-grant + * compatibility code. They no longer impose per-app policy or access tiers. + * This module stays import-free because the desktop also consumes it. */ +export type DeniedCategory = 'browser' | 'terminal' | 'trading' -export type DeniedCategory = "browser" | "terminal" | "trading"; - -/** - * Map a category to its hardcoded tier. Return-type is the string-literal - * union inline (this file is import-free; see header comment). The - * authoritative type is `CuAppPermTier` in types.ts — keep in sync. - * - * Not bijective — both `"browser"` and `"trading"` map to `"read"`. Copy - * that differs by category (the "use CiC" hint is browser-only) must check - * the category, not just the tier. - */ export function categoryToTier( - category: DeniedCategory | null, -): "read" | "click" | "full" { - if (category === "browser" || category === "trading") return "read"; - if (category === "terminal") return "click"; - return "full"; + _category: DeniedCategory | null, +): 'read' | 'click' | 'full' { + return 'full' } -// ─── Bundle-ID deny sets (macOS) ───────────────────────────────────────── - -const BROWSER_BUNDLE_IDS: ReadonlySet = new Set([ - // Apple - "com.apple.Safari", - "com.apple.SafariTechnologyPreview", - // Google - "com.google.Chrome", - "com.google.Chrome.beta", - "com.google.Chrome.dev", - "com.google.Chrome.canary", - // Microsoft - "com.microsoft.edgemac", - "com.microsoft.edgemac.Beta", - "com.microsoft.edgemac.Dev", - "com.microsoft.edgemac.Canary", - // Mozilla - "org.mozilla.firefox", - "org.mozilla.firefoxdeveloperedition", - "org.mozilla.nightly", - // Chromium-based - "org.chromium.Chromium", - "com.brave.Browser", - "com.brave.Browser.beta", - "com.brave.Browser.nightly", - "com.operasoftware.Opera", - "com.operasoftware.OperaGX", - "com.operasoftware.OperaDeveloper", - "com.vivaldi.Vivaldi", - // The Browser Company - "company.thebrowser.Browser", // Arc - "company.thebrowser.dia", // Dia (agentic) - // Privacy-focused - "org.torproject.torbrowser", - "com.duckduckgo.macos.browser", - "ru.yandex.desktop.yandex-browser", - // Agentic / AI browsers — newer entrants with LLM integrations - "ai.perplexity.comet", - "com.sigmaos.sigmaos.macos", // SigmaOS - // Webkit-based misc - "com.kagi.kagimacOS", // Orion -]); - -/** - * Terminals + IDEs with integrated terminals. Supersets - * `SHELL_ACCESS_BUNDLE_IDS` from sentinelApps.ts — terminals proceed to the - * approval dialog at tier "click", and the sentinel warning renders - * alongside the tier badge. - */ -const TERMINAL_BUNDLE_IDS: ReadonlySet = new Set([ - // Dedicated terminals - "com.apple.Terminal", - "com.googlecode.iterm2", - "dev.warp.Warp-Stable", - "dev.warp.Warp-Beta", - "com.github.wez.wezterm", - "org.alacritty", - "io.alacritty", // pre-v0.11.0 (renamed 2022-07) — kept for legacy installs - "net.kovidgoyal.kitty", - "co.zeit.hyper", - "com.mitchellh.ghostty", - "org.tabby", - "com.termius-dmg.mac", // Termius - // IDEs with integrated terminals — we can't distinguish "type in the - // editor" from "type in the integrated terminal" via screenshot+click. - // VS Code family - "com.microsoft.VSCode", - "com.microsoft.VSCodeInsiders", - "com.vscodium", // VSCodium - "com.todesktop.230313mzl4w4u92", // Cursor - "com.exafunction.windsurf", // Windsurf / Codeium - "dev.zed.Zed", - "dev.zed.Zed-Preview", - // JetBrains family (all have integrated terminals) - "com.jetbrains.intellij", - "com.jetbrains.intellij.ce", - "com.jetbrains.pycharm", - "com.jetbrains.pycharm.ce", - "com.jetbrains.WebStorm", - "com.jetbrains.CLion", - "com.jetbrains.goland", - "com.jetbrains.rubymine", - "com.jetbrains.PhpStorm", - "com.jetbrains.datagrip", - "com.jetbrains.rider", - "com.jetbrains.AppCode", - "com.jetbrains.rustrover", - "com.jetbrains.fleet", - "com.google.android.studio", // Android Studio (JetBrains-based) - // Other IDEs - "com.axosoft.gitkraken", // GitKraken has an integrated terminal panel. Also keeps the "kraken" trading-substring from miscategorizing it — bundle-ID wins. - "com.sublimetext.4", - "com.sublimetext.3", - "org.vim.MacVim", - "com.neovim.neovim", - "org.gnu.Emacs", - // Xcode's previous carve-out (full tier for Interface Builder / simulator) - // was reversed — at tier "click" IB and simulator taps still work (both are - // plain clicks) while the integrated terminal is blocked from keyboard input. - "com.apple.dt.Xcode", - "org.eclipse.platform.ide", - "org.netbeans.ide", - "com.microsoft.visual-studio", // Visual Studio for Mac - // AppleScript/automation execution surfaces — same threat as terminals: - // type(script) → key("cmd+r") runs arbitrary code. Added after #28011 - // removed the osascript MCP server, making CU the only tool-call route - // to AppleScript. - "com.apple.ScriptEditor2", - "com.apple.Automator", - "com.apple.shortcuts", -]); - -/** - * Trading / crypto platforms — granted at tier `"read"` so the agent can see - * balances and prices but can't click into an order, transfer, or IB chat. - * Bundle IDs populated from Homebrew cask `uninstall.quit` stanzas as they're - * verified; the name-substring fallback below is the primary check. Bloomberg - * Terminal has no native macOS build per their FAQ (web/Citrix only). - * - * Budgeting/accounting apps (Quicken, YNAB, QuickBooks, etc.) are NOT listed - * here — they default to tier `"full"`. The risk model for brokerage/crypto - * (a stray click can execute a trade) doesn't apply to budgeting apps; the - * Cowork system prompt carries the soft instruction to never execute trades - * or transfer money on the user's behalf. - */ -const TRADING_BUNDLE_IDS: ReadonlySet = new Set([ - // Verified via Homebrew quit/zap stanzas + mdls + electron-builder source. - // Trading - "com.webull.desktop.v1", // Webull (direct download, Qt) - "com.webull.trade.mac.v1", // Webull (Mac App Store) - "com.tastytrade.desktop", - "com.tradingview.tradingviewapp.desktop", - "com.fidelity.activetrader", // Fidelity Trader+ (new) - "com.fmr.activetrader", // Fidelity Active Trader Pro (legacy) - // Interactive Brokers TWS — install4j wrapper; Homebrew quit stanza is - // authoritative for this exact value but install4j IDs can drift across - // major versions — name-substring "trader workstation" is the fallback. - "com.install4j.5889-6375-8446-2021", - // Crypto - "com.binance.BinanceDesktop", - "com.electron.exodus", - // Electrum uses PyInstaller with bundle_identifier=None → defaults to - // org.pythonmac.unspecified.. Confirmed in spesmilo/electrum - // source + Homebrew zap. IntuneBrew's "org.electrum.electrum" is a fork. - "org.pythonmac.unspecified.Electrum", - "com.ledger.live", - "io.trezor.TrezorSuite", - // No native macOS app (name-substring only): Schwab, E*TRADE, TradeStation, - // Robinhood, NinjaTrader, Coinbase, Kraken, Bloomberg. thinkorswim - // install4j ID drifts per-install — substring safer. -]); - -// ─── Policy-deny (not a tier — cannot be granted at all) ───────────────── -// -// Streaming / ebook / music apps and a handful of publisher apps. These -// are auto-denied before the approval dialog — no tier can be granted. -// Rationale is copyright / content-control (the agent has no legitimate -// need to screenshot Netflix or click Play on Spotify). -// -// Sourced from the ACP CU-apps blocklist xlsx ("Full block" tab). See -// /tmp/extract_cu_blocklist.py for the extraction script. - -const POLICY_DENIED_BUNDLE_IDS: ReadonlySet = new Set([ - // Verified via Homebrew quit/zap + mdls /System/Applications + IntuneBrew. - // Apple built-ins - "com.apple.TV", - "com.apple.Music", - "com.apple.iBooksX", - "com.apple.podcasts", - // Music - "com.spotify.client", - "com.amazon.music", - "com.tidal.desktop", - "com.deezer.deezer-desktop", - "com.pandora.desktop", - "com.electron.pocket-casts", // direct-download Electron wrapper - "au.com.shiftyjelly.PocketCasts", // Mac App Store - // Video - "tv.plex.desktop", - "tv.plex.htpc", - "tv.plex.plexamp", - "com.amazon.aiv.AIVApp", // Prime Video (iOS-on-Apple-Silicon) - // Ebooks - "net.kovidgoyal.calibre", - "com.amazon.Kindle", // legacy desktop, discontinued - "com.amazon.Lassen", // current Mac App Store (iOS-on-Mac) - "com.kobo.desktop.Kobo", - // No native macOS app (name-substring only): Netflix, Disney+, Hulu, - // HBO Max, Peacock, Paramount+, YouTube, Crunchyroll, Tubi, Vudu, - // Audible, Reddit, NYTimes. Their iOS apps don't opt into iPad-on-Mac. -]); - -const POLICY_DENIED_NAME_SUBSTRINGS: readonly string[] = [ - // Video streaming - "netflix", - "disney+", - "hulu", - "prime video", - "apple tv", - "peacock", - "paramount+", - // "plex" is too generic — would match "Perplexity". Covered by - // tv.plex.* bundle IDs on macOS. - "tubi", - "crunchyroll", - "vudu", - // E-readers / audiobooks - "kindle", - "apple books", - "kobo", - "play books", - "calibre", - "libby", - "readium", - "audible", - "libro.fm", - "speechify", - // Music - "spotify", - "apple music", - "amazon music", - "youtube music", - "tidal", - "deezer", - "pandora", - "pocket casts", - // Publisher / social apps (from the same blocklist tab) - "naver", - "reddit", - "sony music", - "vegas pro", - "pitchfork", - "economist", - "nytimes", - // Skipped (too generic for substring matching — need bundle ID): - // HBO Max / Max, YouTube (non-Music), Nook, Sony Catalyst, Wired -]; - -/** - * Policy-level auto-deny. Unlike `userDeniedBundleIds` (per-user Settings - * page), this is baked into the build. `buildAccessRequest` strips these - * before the approval dialog with "blocked by policy" guidance; the agent - * is told to not retry. - */ -export function isPolicyDenied( - bundleId: string | undefined, - displayName: string, -): boolean { - if (bundleId && POLICY_DENIED_BUNDLE_IDS.has(bundleId)) return true; - const lower = displayName.toLowerCase(); - for (const sub of POLICY_DENIED_NAME_SUBSTRINGS) { - if (lower.includes(sub)) return true; - } - return false; +export function getDeniedCategory(_bundleId: string): DeniedCategory | null { + return null } -/** - * Apps the agent can never drive, no matter what the user grants: our own - * desktop shell and the Computer Use helper that executes the actions. - * - * Kept as its own set rather than folded into `POLICY_DENIED_BUNDLE_IDS` - * because the two mean different things. The policy sets are *product* policy - * (streaming, trading) — debatable, revisable, and surfaced to the user as - * "blocked by policy". This one is a structural invariant: driving the host - * lets the agent click its own approval dialogs, and driving the helper lets - * it reach around the very process enforcing the grants. The native - * `AppTargetPolicy.swift` mirrors both sets, separately, for the same reason. - */ -const INTRINSIC_DENIED_BUNDLE_IDS: ReadonlySet = new Set([ - "com.claude-code-haha.desktop", - "dev.cchaha.cu-helper", -]); - -/** - * True when the target is the host app or its helper — a permanent denial that - * no grant can lift. - * - * `hostBundleId` is checked *in addition to* the baked-in set, never instead - * of it: it catches builds whose bundle id differs from the shipped default - * (dev and beta channels), while the constants still hold if a caller has no - * host id to pass. - */ -export function isIntrinsicAppDenied( - bundleId: string | undefined, - hostBundleId?: string, -): boolean { - if (!bundleId) return false; - if (INTRINSIC_DENIED_BUNDLE_IDS.has(bundleId)) return true; - return hostBundleId !== undefined && bundleId === hostBundleId; +export function getDeniedCategoryByDisplayName(_name: string): DeniedCategory | null { + return null } -export function getDeniedCategory(bundleId: string): DeniedCategory | null { - if (BROWSER_BUNDLE_IDS.has(bundleId)) return "browser"; - if (TERMINAL_BUNDLE_IDS.has(bundleId)) return "terminal"; - if (TRADING_BUNDLE_IDS.has(bundleId)) return "trading"; - return null; -} - -// ─── Display-name fallback (cross-platform) ────────────────────────────── - -/** - * Lowercase substrings checked against the requested display name. Catches: - * - Unresolved requests (app not installed, Spotlight miss) - * - Future Windows/Linux support where bundleId is meaningless - * - * Matched via `.includes()` on `name.toLowerCase()`. Entries are ordered - * by specificity (more-specific first is irrelevant since we return on - * first match, but groupings are by category for readability). - */ -const BROWSER_NAME_SUBSTRINGS: readonly string[] = [ - "safari", - "chrome", - "firefox", - "microsoft edge", - "brave", - "opera", - "vivaldi", - "chromium", - // Arc/Dia: the canonical display name is just "Arc"/"Dia" — too short for - // substring matching (false-positives: "Arcade", "Diagram"). Covered by - // bundle ID on macOS. The "... browser" entries below catch natural-language - // phrasings ("the arc browser") but NOT the canonical short name. - "arc browser", - "tor browser", - "duckduckgo", - "yandex", - "orion browser", - // Agentic / AI browsers - "comet", // Perplexity's browser — "Comet" substring risks false positives - // but leaving for now; "comet" in an app name is rare - "sigmaos", - "dia browser", -]; - -const TERMINAL_NAME_SUBSTRINGS: readonly string[] = [ - // macOS / cross-platform terminals - "terminal", // catches Terminal, Windows Terminal (NOT iTerm — separate entry) - "iterm", - "wezterm", - "alacritty", - "kitty", - "ghostty", - "tabby", - "termius", - // AppleScript runners — see bundle-ID comment above. "shortcuts" is too - // generic for substring matching (many apps have "shortcuts" in the name); - // covered by bundle ID only, like warp/hyper. - "script editor", - "automator", - // NOTE: "warp" and "hyper" are too generic for substring matching — - // they'd false-positive on "Warpaint" or "Hyperion". Covered by bundle ID - // (dev.warp.Warp-Stable, co.zeit.hyper) for macOS; Windows exe-name - // matching can be added when Windows CU ships. - // Windows shells (activate when the darwin gate lifts) - "powershell", - "cmd.exe", - "command prompt", - "git bash", - "conemu", - "cmder", - // IDEs (VS Code family) - "visual studio code", - "visual studio", // catches VS for Mac + Windows - "vscode", - "vs code", - "vscodium", - "cursor", // Cursor IDE — "cursor" is generic but IDE is the only common app - "windsurf", - // Zed: display name is just "Zed" — too short for substring matching - // (false-positives). Covered by bundle ID (dev.zed.Zed) on macOS. - // IDEs (JetBrains family) - "intellij", - "pycharm", - "webstorm", - "clion", - "goland", - "rubymine", - "phpstorm", - "datagrip", - "rider", - "appcode", - "rustrover", - "fleet", - "android studio", - // Other IDEs - "sublime text", - "macvim", - "neovim", - "emacs", - "xcode", - "eclipse", - "netbeans", -]; - -const TRADING_NAME_SUBSTRINGS: readonly string[] = [ - // Trading — brokerage apps. Sourced from the ACP CU-apps blocklist xlsx - // ("Read Only" tab). Name-substring safe for proper nouns below; generic - // names (IG, Delta, HTX) are skipped and need bundle-ID matching once - // verified. - "bloomberg", - "ameritrade", - "thinkorswim", - "schwab", - "fidelity", - "e*trade", - "interactive brokers", - "trader workstation", // Interactive Brokers TWS - "tradestation", - "webull", - "robinhood", - "tastytrade", - "ninjatrader", - "tradingview", - "moomoo", - "tradezero", - "prorealtime", - "plus500", - "saxotrader", - "oanda", - "metatrader", - "forex.com", - "avaoptions", - "ctrader", - "jforex", - "iq option", - "olymp trade", - "binomo", - "pocket option", - "raceoption", - "expertoption", - "quotex", - "naga", - "morgan stanley", - "ubs neo", - "eikon", // Thomson Reuters / LSEG Workspace - // Crypto — exchanges, wallets, portfolio trackers - "coinbase", - "kraken", - "binance", - "okx", - "bybit", - // "gate.io" is too generic — the ".io" TLD suffix is common in app names - // (e.g., "Draw.io"). Needs bundle-ID matching once verified. - "phemex", - "stormgain", - "crypto.com", - // "exodus" is too generic — it's a common noun and would match unrelated - // apps/games. Needs bundle-ID matching once verified. - "electrum", - "ledger live", - "trezor", - "guarda", - "atomic wallet", - "bitpay", - "bisq", - "koinly", - "cointracker", - "blockfi", - "stripe cli", - // Crypto games / metaverse (same trade-execution risk model) - "decentraland", - "axie infinity", - "gods unchained", -]; - -/** - * Display-name substring match. Called when bundle-ID resolution returned - * nothing (`resolved === undefined`) or when no bundle-ID deny-list entry - * matched. Returns the category for the first matching substring, or null. - * - * Case-insensitive, substring — so `"Google Chrome"`, `"chrome"`, and - * `"Chrome Canary"` all match the `"chrome"` entry. - */ -export function getDeniedCategoryByDisplayName( - name: string, -): DeniedCategory | null { - const lower = name.toLowerCase(); - // Trading first — proper-noun-only set, most specific. "Bloomberg Terminal" - // contains "terminal" and would miscategorize if TERMINAL_NAME_SUBSTRINGS - // ran first. - for (const sub of TRADING_NAME_SUBSTRINGS) { - if (lower.includes(sub)) return "trading"; - } - for (const sub of BROWSER_NAME_SUBSTRINGS) { - if (lower.includes(sub)) return "browser"; - } - for (const sub of TERMINAL_NAME_SUBSTRINGS) { - if (lower.includes(sub)) return "terminal"; - } - return null; -} - -/** - * Combined check — bundle ID first (exact, fast), then display-name - * fallback. This is the function tool-call handlers should use. - * - * `bundleId` may be undefined (unresolved request — model asked for an app - * that isn't installed or Spotlight didn't find). In that case only the - * display-name check runs. - */ export function getDeniedCategoryForApp( - bundleId: string | undefined, - displayName: string, + _bundleId: string | undefined, + _displayName: string, ): DeniedCategory | null { - if (bundleId) { - const byId = getDeniedCategory(bundleId); - if (byId) return byId; - } - return getDeniedCategoryByDisplayName(displayName); + return null +} + +export function isPolicyDenied( + _bundleId: string | undefined, + _displayName: string, +): boolean { + return false } -/** - * Default tier for an app at grant time. Wraps `getDeniedCategoryForApp` + - * `categoryToTier`. Browsers → `"read"`, terminals/IDEs → `"click"`, - * everything else → `"full"`. - * - * Called by `buildAccessRequest` to populate `ResolvedAppRequest.proposedTier` - * before the approval dialog shows. - */ export function getDefaultTierForApp( - bundleId: string | undefined, - displayName: string, -): "read" | "click" | "full" { - return categoryToTier(getDeniedCategoryForApp(bundleId, displayName)); + _bundleId: string | undefined, + _displayName: string, +): 'read' | 'click' | 'full' { + return 'full' } - -export const _test = { - BROWSER_BUNDLE_IDS, - TERMINAL_BUNDLE_IDS, - TRADING_BUNDLE_IDS, - POLICY_DENIED_BUNDLE_IDS, - INTRINSIC_DENIED_BUNDLE_IDS, - BROWSER_NAME_SUBSTRINGS, - TERMINAL_NAME_SUBSTRINGS, - TRADING_NAME_SUBSTRINGS, - POLICY_DENIED_NAME_SUBSTRINGS, -}; diff --git a/src/vendor/computer-use-mcp/instructions.test.ts b/src/vendor/computer-use-mcp/instructions.test.ts new file mode 100644 index 00000000..c1c40f48 --- /dev/null +++ b/src/vendor/computer-use-mcp/instructions.test.ts @@ -0,0 +1,18 @@ +import { describe, expect, test } from 'bun:test' +import { COMPUTER_USE_INSTRUCTIONS } from './instructions.js' + +describe('macOS action and observation guidance', () => { + test('uses persistent JS for known actions without demanding a model round trip per click', () => { + expect(COMPUTER_USE_INSTRUCTIONS).toContain('Do not force one model round trip per click') + expect(COMPUTER_USE_INSTRUCTIONS).toContain('then observe at a decision point') + expect(COMPUTER_USE_INSTRUCTIONS).toContain('Do not add a fixed sleep before observing') + expect(COMPUTER_USE_INSTRUCTIONS).toContain('Use copied `gN:id` handles') + }) + + test('keeps standalone receipts and unknown paste results subject to observation', () => { + expect(COMPUTER_USE_INSTRUCTIONS).toContain('receiving a standalone dispatch receipt') + expect(COMPUTER_USE_INSTRUCTIONS).toContain('Inspect a fresh observation') + expect(COMPUTER_USE_INSTRUCTIONS).toContain('treat the result as unknown and call') + expect(COMPUTER_USE_INSTRUCTIONS).toContain('stop and re-observe') + }) +}) diff --git a/src/vendor/computer-use-mcp/nativeAppPolicy.ts b/src/vendor/computer-use-mcp/nativeAppPolicy.ts deleted file mode 100644 index 1648c7f8..00000000 --- a/src/vendor/computer-use-mcp/nativeAppPolicy.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { isIntrinsicAppDenied } from './deniedApps.js' - -// Official macOS SkyComputerUseService 26.831.1000926: -// isForbiddenComputerUseTarget checks these four sets with exact equality. -// Legacy cross-platform tiers and display-name matches do not apply here. -export const NATIVE_FORBIDDEN_BUNDLE_IDS: ReadonlySet = new Set([ - 'com.apple.Terminal', - 'com.googlecode.iterm2', - 'org.alacritty', - 'dev.warp.Warp-Stable', - 'net.kovidgoyal.kitty', - 'co.zeit.hyper', - 'com.github.wez.wezterm', - 'org.tabby', - 'com.mitchellh.ghostty', - 'com.raphaelamorim.rio', - 'dev.commandline.waveterm', - 'com.openai.codex', - 'com.openai.codex.alpha', - 'com.openai.codex.beta', - 'com.openai.codex.dev', - 'com.openai.codex.nightly', - 'com.openai.chat', - 'com.openai.chat.alpha', - 'com.openai.chat.beta', - 'com.openai.chat.nightly', - 'com.openai.chat.mac-debug', - 'com.apple.UserNotificationCenter', - 'com.apple.LocalAuthenticationRemoteService', - 'com.apple.SecurityAgent', -]) - -export function isNativeAppDenied(bundleId: string | undefined, hostBundleId?: string): boolean { - return isIntrinsicAppDenied(bundleId, hostBundleId) || - (bundleId !== undefined && NATIVE_FORBIDDEN_BUNDLE_IDS.has(bundleId)) -} diff --git a/src/vendor/computer-use-mcp/platformRouting.test.ts b/src/vendor/computer-use-mcp/platformRouting.test.ts index e4f338ad..f7ccab1a 100644 --- a/src/vendor/computer-use-mcp/platformRouting.test.ts +++ b/src/vendor/computer-use-mcp/platformRouting.test.ts @@ -736,4 +736,110 @@ describe('Computer Use platform routing', () => { await connection.close() } }) + + test('win32 retries a legacy clipboard stash restore without clearing clipboard by app', async () => { + const calls: string[] = [] + const adapter = makeWindowsAdapter(calls) + let stash: string | undefined = 'preserved clipboard' + let clipboard = 'current clipboard' + let writeAttempts = 0 + adapter.executor.getFrontmostApp = async () => ({ + bundleId: 'powershell.exe', displayName: 'PowerShell', + }) + adapter.executor.readClipboard = async () => clipboard + adapter.executor.writeClipboard = async text => { + writeAttempts += 1 + if (writeAttempts === 1) throw new Error('clipboard temporarily busy') + clipboard = text + } + const getSubGates = adapter.getSubGates + adapter.getSubGates = () => ({ ...getSubGates(), clipboardGuard: true }) + const connection = await connect(adapter, makeSessionContext({ + getClipboardStash: () => stash, + onClipboardStashChanged: value => { stash = value }, + getGrantFlags: () => ({ clipboardRead: true, clipboardWrite: true, systemKeyCombos: true }), + })) + try { + expect((await connection.client.callTool({ name: 'read_clipboard' })).isError).toBeFalsy() + expect(stash).toBe('preserved clipboard') + expect(clipboard).toBe('current clipboard') + expect((await connection.client.callTool({ name: 'read_clipboard' })).isError).toBeFalsy() + expect(stash).toBeUndefined() + expect(clipboard).toBe('preserved clipboard') + expect(writeAttempts).toBe(2) + + expect((await connection.client.callTool({ + name: 'write_clipboard', arguments: { text: 'new clipboard' }, + })).isError).toBeFalsy() + expect((await connection.client.callTool({ + name: 'key', arguments: { text: 'ctrl+v' }, + })).isError).toBeFalsy() + expect(clipboard).toBe('new clipboard') + expect(stash).toBeUndefined() + expect(writeAttempts).toBe(3) + expect(calls).toContain('key:ctrl+v') + } finally { + await connection.close() + } + }) + + test.each([ + ['chrome.exe', 'Google Chrome'], + ['powershell.exe', 'PowerShell'], + ['spotify.exe', 'Spotify'], + ['tradingview.exe', 'TradingView'], + ['com.example.host', 'Claude Code Haha'], + ['dev.cchaha.cu-helper', 'Computer Use Helper'], + ])('win32 global consent permits input, launch, and clipboard for %s', async (bundleId, displayName) => { + const calls: string[] = [] + const adapter = makeWindowsAdapter(calls) + const app = { bundleId, displayName } + let frontmost = app + let clipboard = 'existing clipboard' + adapter.executor.getFrontmostApp = async () => frontmost + adapter.executor.appUnderPoint = async () => app + adapter.executor.listInstalledApps = async () => [{ ...app, path: `C:\\Apps\\${bundleId}` }] + adapter.executor.readClipboard = async () => clipboard + adapter.executor.writeClipboard = async text => { clipboard = text } + const getSubGates = adapter.getSubGates + adapter.getSubGates = () => ({ ...getSubGates(), clipboardGuard: true }) + const connection = await connect(adapter, makeSessionContext({ + // Cached state from the old app-specific model cannot narrow the + // feature-wide consent given when Computer Use was enabled. + getAllowedApps: () => [{ ...app, grantedAt: 1, tier: 'read' }], + getUserDeniedBundleIds: () => [bundleId], + getGrantFlags: () => ({ clipboardRead: true, clipboardWrite: true, systemKeyCombos: true }), + })) + try { + for (const request of [ + { name: 'screenshot' }, + { name: 'open_application', arguments: { app: displayName } }, + { name: 'key', arguments: { text: 'ctrl+a' } }, + { name: 'type', arguments: { text: 'ok' } }, + { name: 'read_clipboard' }, + ]) { + const result = await connection.client.callTool(request) + expect(result.isError, `${request.name}: ${JSON.stringify(result.content)}`).toBeFalsy() + } + expect(clipboard).toBe('existing clipboard') + expect((await connection.client.callTool({ + name: 'write_clipboard', arguments: { text: 'new clipboard' }, + })).isError).toBeFalsy() + expect(clipboard).toBe('new clipboard') + + // Coordinate clicks remain authorized regardless of which application + // is foreground or owns the window receiving the click. + frontmost = { bundleId: 'notepad.exe', displayName: 'Notepad' } + expect((await connection.client.callTool({ + name: 'right_click', arguments: { coordinate: [10, 20] }, + })).isError).toBeFalsy() + expect(calls).toContain(`openApp:${bundleId}`) + expect(calls).toContain('key:ctrl+a') + expect(calls).toContain('type:ok') + expect(calls).toContain('click:10,20,right,1') + expect(clipboard).toBe('new clipboard') + } finally { + await connection.close() + } + }) }) diff --git a/src/vendor/computer-use-mcp/toolCalls.test.ts b/src/vendor/computer-use-mcp/toolCalls.test.ts index 5af4deee..8b6d584f 100644 --- a/src/vendor/computer-use-mcp/toolCalls.test.ts +++ b/src/vendor/computer-use-mcp/toolCalls.test.ts @@ -17,6 +17,8 @@ import { handleToolCall, resetMouseButtonHeld, } from './toolCalls.js' +import { bindSessionContext } from './mcpServer.js' +import type { ComputerUseSessionContext } from './types.js' import { buildComputerUseTools } from './tools.js' import { bindSessionContext } from './mcpServer.js' import { COMPUTER_USE_INSTRUCTIONS } from './instructions.js' @@ -224,7 +226,6 @@ describe('buildComputerUseTools — current Codex tool face', () => { 'click', 'drag', 'get_app_state', - 'sequence', 'list_apps', 'perform_secondary_action', 'paste', @@ -233,6 +234,7 @@ describe('buildComputerUseTools — current Codex tool face', () => { 'select_text', 'set_value', 'type_text', + 'sequence', ].sort(), ) }) @@ -515,6 +517,29 @@ describe('frameAppStateEnvelope', () => { ]) }) + test('preserves declared JPEG and legacy PNG bytes with real decoded dimensions', async () => { + const { default: sharp } = await import('sharp') + for (const format of ['jpeg', 'png'] as const) { + const bytes = await sharp({ create: { width: 96, height: 64, channels: 3, background: '#4070a0' } }) + .toFormat(format).toBuffer() + const out = frameAppStateEnvelope({ + pid: 1, elementCount: 0, truncated: false, durationMs: 1, axText: 'App=x (pid 1)', + screenshot: { base64: bytes.toString('base64'), width: 96, height: 64, + ...(format === 'jpeg' ? { mimeType: 'image/jpeg' as const } : {}) }, + }) + const [image] = imageBlocks(out) + expect(image.mimeType).toBe(`image/${format}`) + const returned = Buffer.from(image.data, 'base64') + expect(returned.equals(bytes)).toBe(true) + const metadata = await sharp(returned).metadata() + expect(metadata.format).toBe(format) + expect(metadata.width).toBe(96) + expect(metadata.height).toBe(64) + // Force pixel decoding too, not just signature/metadata recognition. + expect((await sharp(returned).raw().toBuffer({ resolveWithObject: true })).info.width).toBe(96) + } + }) + test('no image block when screenshot is absent (capture failed)', () => { const out = frameAppStateEnvelope({ pid: 1, elementCount: 0, truncated: false, durationMs: 1, axText: 'App=x (pid 1)', @@ -592,47 +617,7 @@ describe('arg parsing helpers', () => { expect(() => _test.parsePoint({}, 'from', 'from_x', 'from_y')).toThrow() }) - test('policyDenyMessage: terminals refused with Codex text, normal apps pass', () => { - const msg = _test.policyDenyMessage({ - bundleId: 'com.googlecode.iterm2', - displayName: 'iTerm2', - }, 'com.googlecode.iterm2') - expect(msg).toBe("Computer Use is not allowed to use the app 'com.googlecode.iterm2' for safety reasons.") - expect(_test.policyDenyMessage({ - bundleId: 'com.apple.finder', - displayName: 'Finder', - })).toBeUndefined() - }) - test('native policy uses the official exact identities instead of legacy app categories or names', () => { - for (const bundleId of ['com.microsoft.VSCode', 'com.apple.Music', 'com.spotify.client', 'com.tradingview.tradingviewapp.desktop', 'dev.test.Terminal']) { - expect(_test.policyDenyMessage({ bundleId, displayName: 'Terminal Music Editor' })).toBeUndefined() - } - for (const bundleId of ['com.raphaelamorim.rio', 'dev.commandline.waveterm', 'com.openai.codex.beta', 'com.openai.chat.mac-debug', 'com.apple.SecurityAgent']) { - expect(_test.policyDenyMessage({ bundleId, displayName: 'Fixture' })).toContain('not allowed') - } - }) - - test('policyDenyMessage permanently denies the host and helper while a host override only adds', () => { - const customHostBundleId = 'com.example.custom-host' - for (const bundleId of [ - 'com.claude-code-haha.desktop', - 'dev.cchaha.cu-helper', - customHostBundleId, - ]) { - expect(_test.policyDenyMessage({ - bundleId, - displayName: bundleId, - }, bundleId, customHostBundleId)).toBe( - `Computer Use is not allowed to use the app '${bundleId}' for safety reasons.`, - ) - } - - expect(_test.policyDenyMessage({ - bundleId: 'com.apple.TextEdit', - displayName: 'TextEdit', - }, 'TextEdit', customHostBundleId)).toBeUndefined() - }) }) // --------------------------------------------------------------------------- @@ -703,12 +688,11 @@ describe('handleToolCall — gates', () => { expect(acquired).toBe(0) }) - test('safety denylist refuses a terminal before the engine', async () => { + test('global enablement allows terminal state reads without per-app grants', async () => { const { engine, calls } = makeEngine() const r = await handleToolCall(makeAdapter({ engine }), 'get_app_state', { app: 'com.googlecode.iterm2' }, baseOverrides()) - expect(r.isError).toBe(true) - expect(textOf(r)).toContain('for safety reasons') - expect(calls.map(call => call.method)).toEqual(['resolveTarget']) + expect(r.isError).toBeFalsy() + expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'getAppState']) }) test('native browser actions follow the observed official Chrome App path with normal identity checks', async () => { @@ -729,7 +713,7 @@ describe('handleToolCall — gates', () => { } }) - test('configured host bundle is refused before permission or engine dispatch', async () => { + test('configured host bundle uses the same global consent as every app', async () => { const customHostBundleId = 'com.example.custom-host' const { engine, calls } = makeEngine({ resolveTarget: async () => ({ @@ -767,10 +751,9 @@ describe('handleToolCall — gates', () => { }), ) - expect(r.isError).toBe(true) - expect(r.telemetry?.error_kind).toBe('app_denied') + expect(r.isError).toBeFalsy() expect(permissionRequests).toBe(0) - expect(calls.map(call => call.method)).toEqual(['resolveTarget']) + expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'getAppState']) }) test('missing engine → feature_unavailable', async () => { @@ -1020,24 +1003,51 @@ describe('handleToolCall — gates', () => { expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'typeText']) }) - test('the product denylist still blocks a resolved app before mutation', async () => { + test.each([ + ['com.google.Chrome', 'Google Chrome'], + ['com.claude-code-haha.desktop', 'Claude Code Haha'], + ['dev.cchaha.cu-helper', 'Computer Use Helper'], + ['com.test.host', 'Custom Host'], + ['com.googlecode.iterm2', 'iTerm2'], + ['com.microsoft.VSCode', 'Visual Studio Code'], + ['com.tradingview.tradingviewapp.desktop', 'TradingView'], + ['com.spotify.client', 'Spotify'], + [undefined, 'Netflix'], + [undefined, 'Windows Terminal'], + ])('global enablement permits reading and operating %s (%s) without app approval', async (bundleId, displayName) => { + let permissionCalls = 0 const { engine, calls } = makeEngine({ resolveTarget: async () => ({ pid: 900, - bundleId: 'com.googlecode.iterm2', - displayName: 'iTerm2', + bundleId, + displayName, + launchTime: 1900, }), }) - const r = await handleToolCall( - makeAdapter({ engine }), - 'type_text', - { app: 'iTerm2', text: 'blocked' }, - baseOverrides({ allowedApps: [] }), - ) + const overrides = baseOverrides({ + allowedApps: [], + userDeniedBundleIds: bundleId ? [bundleId] : [], + onPermissionRequest: async () => { + permissionCalls++ + throw new Error('global consent must not ask for per-app approval') + }, + }) + for (const app of [displayName, ...(bundleId ? [bundleId] : []), `/Applications/${displayName}.app`, '900']) { + for (const [name, args, method] of [ + ['get_app_state', {}, 'getAppState'], + ['click', { x: 10, y: 20 }, 'click'], + ['type_text', { text: 'hello' }, 'typeText'], + ] as const) { + calls.length = 0 + const r = await handleToolCall(makeAdapter({ engine }), name, { app, ...args }, overrides) - expect(r.isError).toBe(true) - expect(r.telemetry?.error_kind).toBe('app_denied') - expect(calls.map(call => call.method)).toEqual(['resolveTarget']) + expect(r.isError).toBeFalsy() + expect(calls.map(call => call.method)).toEqual(['resolveTarget', method]) + const target = name === 'get_app_state' ? calls[1].args : (calls[1].args as { target: AppTarget }).target + expect(target).toMatchObject({ pid: 900, expectedProcessIdentity: { pid: 900, launchTime: 1900 } }) + } + } + expect(permissionCalls).toBe(0) }) test('invalid app values return bad_args without resolving a target', async () => { @@ -1791,6 +1801,35 @@ describe('resetMouseButtonHeld', () => { // A sequence is a bounded set of existing native operations, not a script. describe('same-app sequence', () => { const steps = [{ tool: 'press_key', key: 's x 1 period 3 5 Return' }, { tool: 'click', x: 12, y: 24 }] + test.each([ + 'com.google.Chrome', + 'com.claude-code-haha.desktop', + 'dev.cchaha.cu-helper', + 'com.test.host', + ])('global consent also covers sequences targeting %s', async bundleId => { + const { engine, calls } = makeEngine({ + getAppState: async () => ({ + pid: 1234, appName: bundleId, bundleId, elementCount: 0, + truncated: false, durationMs: 1, axText: '', + screenshot: { base64: 'PNG', width: 10, height: 10 }, + }), + }) + const result = await handleToolCall( + makeAdapter({ engine }), 'sequence', { app: bundleId, steps }, + baseOverrides({ + allowedApps: [], userDeniedBundleIds: [bundleId], + onPermissionRequest: async () => { throw new Error('must not ask for app approval') }, + }), + ) + expect(result.isError).toBeFalsy() + expect(result.structuredContent).toMatchObject({ status: 'completed', completedSteps: 2 }) + expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'pressKey', 'click', 'getAppState']) + const target = (calls[1].args as { target: AppTarget }).target + expect(target.expectedProcessIdentity?.bundleId).toBe(bundleId) + expect((calls[2].args as { target: AppTarget }).target).toEqual(target) + expect(calls[3].args).toEqual(target) + expect(imageBlocks(result)).toHaveLength(1) + }) test('runs in order against one proven identity and returns one final screenshot', async () => { const { engine, calls } = makeEngine({ getAppState: () => ({ pid: 1234, appName: 'Finder', bundleId: 'com.apple.finder', windowTitle: 'Docs', elementCount: 0, truncated: false, durationMs: 1, axText: '', screenshot: { base64: 'PNG', width: 10, height: 10 } }) }) const result = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps }, baseOverrides()) @@ -2005,12 +2044,13 @@ describe('canvas action batches', () => { expect(calls[3]!.args).toMatchObject({ from: { x: 100, y: 200 }, to: { x: 100, y: 200 } }) }) - test('a batch uses the existing resolved-target policy before any mutation', async () => { - const { engine, calls } = makeEngine() + test('a batch requires a proven process lifetime before any mutation', async () => { + const { engine, calls } = makeEngine({ resolveTarget: async () => ({ pid: 1234, bundleId: 'com.test.host' }) }) const result = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'com.test.host', steps: [{ tool: 'click', x: 1, y: 2 }], }, baseOverrides()) - expect(result.telemetry?.error_kind).toBe('app_denied') + expect(result.telemetry?.error_kind).toBe('executor_threw') + expect(result.isError).toBe(true) expect(calls.map(call => call.method)).toEqual(['resolveTarget']) }) diff --git a/src/vendor/computer-use-mcp/toolCalls.ts b/src/vendor/computer-use-mcp/toolCalls.ts index d7bc68b3..e90e9734 100644 --- a/src/vendor/computer-use-mcp/toolCalls.ts +++ b/src/vendor/computer-use-mcp/toolCalls.ts @@ -21,12 +21,11 @@ * before any mutation, because a bare pid can be recycled between the moment * we resolved it and the moment we act on it. * - * **2. Safety checks happen after resolution, on the resolved identity.** - * The model names an app loosely ("Notes", a path, a pid). `resolveTarget` is - * the single seam that turns that into one real running process; every policy - * check then runs against *that* process's bundle id, not against the string - * the model typed. Checking the string instead would let "friendly alias" - * walk past a denylist that names the bundle id. + * **2. One global consent covers every app.** Once Computer Use is enabled + * in Settings and its consent dialog is confirmed, no app category, bundle + * id, display name, host identity, or legacy app grant may deny access. + * `resolveTarget` identifies the actual process for dispatch and lifetime + * checks; it is not a second app-authorization step. * * **3. Standalone mutations never take an implicit snapshot.** They return a fixed * receipt and the model calls `get_app_state` when it wants to see the result. @@ -44,9 +43,8 @@ * 4. Global CU lock (`overrides.checkCuLock`). * 5. Engine presence. * 6. `resolveTarget` → one running process + its lifetime identity. - * 7. Product/intrinsic denylists against the RESOLVED identity. - * 8. Proven process lifetime — mutating tools only. - * 9. Engine dispatch. + * 7. Proven process lifetime — mutating tools only. + * 8. Engine dispatch. * * The `` envelope is framed here in TS. Swift renders the inner tree; * we wrap it in the version banner + optional + @@ -56,7 +54,6 @@ import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js"; -import { isNativeAppDenied } from './nativeAppPolicy.js' import { NATIVE_ERROR, NATIVE_SERVER_ERROR_CODES, toNativeErrorMetadata, type NativeErrorMetadata } from './nativeError.js' import type { AppStateResult, @@ -489,34 +486,6 @@ export function resetMouseButtonHeld(): void { /* no cross-call mouse state in the semantic engine */ } -// --------------------------------------------------------------------------- -// Native app policy -// --------------------------------------------------------------------------- - -/** - * Refusal check against a RESOLVED target — the real bundle id and display - * name of the process we are about to drive, not the string the model typed. - * - * Official native forbidden identities plus our own app and helper. The - * resolved bundle ID is authoritative; display-name substrings are not policy. - * - * `requestedApp` only shapes the message — the model should see the name it - * used. `hostBundleId` extends the intrinsic set for builds whose bundle id - * differs from the shipped default. - */ -function policyDenyMessage( - resolved: { bundleId?: string; displayName?: string }, - requestedApp?: string, - hostBundleId?: string, -): string | undefined { - const bundleId = resolved.bundleId; - const displayName = resolved.displayName ?? bundleId ?? requestedApp ?? ""; - if (!isNativeAppDenied(bundleId, hostBundleId)) return undefined; - const shown = requestedApp ?? displayName ?? bundleId ?? ""; - // Preserve the existing product refusal message. - return `Computer Use is not allowed to use the app '${shown}' for safety reasons.`; -} - /** * Apps whose `` block has already been delivered. * @@ -1086,15 +1055,7 @@ export async function handleToolCall( const resolved = await engine.resolveTarget(request.target); const requestedApp = request.requestedApp ?? ""; - // ─── Gate 7: denylists, against the RESOLVED identity ────────────── - const denied = policyDenyMessage( - resolved, - requestedApp, - adapter.executor.capabilities.hostBundleId, - ); - if (denied) return errorResult(denied, "app_denied"); - - // ─── Gate 8: proven process lifetime (mutations only) ────────────── + // ─── Gate 7: proven process lifetime (mutations only) ────────────── if (request.mutating && !provenIdentity(resolved)) { return errorResult( `Resolving '${requestedApp}' did not prove the running process ` + @@ -1278,7 +1239,6 @@ export const _test = { parseDirection, parseTarget, parsePoint, - policyDenyMessage, provenIdentity, dispatchTarget, CUA_APP_VERSION, diff --git a/src/vendor/computer-use-mcp/windowsLegacyToolCalls.ts b/src/vendor/computer-use-mcp/windowsLegacyToolCalls.ts index eaaa9661..71e50cef 100644 --- a/src/vendor/computer-use-mcp/windowsLegacyToolCalls.ts +++ b/src/vendor/computer-use-mcp/windowsLegacyToolCalls.ts @@ -12,7 +12,7 @@ * For input actions (click/type/key/scroll/drag/move_mouse) the tool-specific * gates are, in order: * a. `prepareForAction` — platform preparation and host defocus. - * b. Resolve the frontmost app and apply product/intrinsic safety tiers. + * b. Resolve the foreground application before dispatching input. * * For click variants only, AFTER the above gates but BEFORE the executor call: * c. Pixel-validation staleness check (sub-gated). @@ -50,7 +50,6 @@ import type { * Finder is never hidden by the hide loop (hiding Finder kills the Desktop), * so it's always a valid frontmost. */ -const FINDER_BUNDLE_ID = "com.apple.finder"; /** * Categorical error classes for the cu_tool_call telemetry event. Never @@ -318,82 +317,19 @@ function tierSatisfies( return tier === "click" || tier === "full"; } -function automaticTier(bundleId: string | undefined, displayName: string): CuAppPermTier { - return getDefaultTierForApp(bundleId, displayName); -} - -// Appended to every tier_insufficient error. The model may try to route -// around the gate (osascript, System Events, cliclick via Bash) — this -// closes that door explicitly. Leading space so it concatenates cleanly. -const TIER_ANTI_SUBVERSION = - " Do not attempt to work around this restriction — never use AppleScript, " + - "System Events, shell commands, or any other method to send clicks or " + - "keystrokes to this app."; - -// --------------------------------------------------------------------------- -// Clipboard guard — stash+clear while a click-tier app is frontmost -// --------------------------------------------------------------------------- -// -// Threat: tier "click" blocks type/key/right-click-Paste, but a click-tier -// terminal/IDE may have a UI Paste button that's plain-left-clickable. If the -// clipboard holds `rm -rf /` — from the user, from a prior full-tier paste, -// OR from the agent's own write_clipboard call (which doesn't route through -// runInputActionGates) — a left_click on that button injects it. -// -// Mitigation: stash the user's clipboard on first entry to click-tier, then -// RE-CLEAR before every input action while click-tier stays frontmost. The -// re-clear is the load-bearing part — a stash-on-transition-only design -// leaves a gap between an agent write_clipboard and the next left_click. -// When frontmost becomes anything else, restore. Turn-end restore is inlined -// in the host's result-handler + leavingRunning (same dual-location as -// cuHiddenDuringTurn unhide) — reads `session.cuClipboardStash` directly and -// writes via Electron's `clipboard.writeText`, so no nest-only import. -// -// State lives on the session (via `overrides.getClipboardStash` / -// `onClipboardStashChanged`), not module-level. The CU lock still guarantees -// one session at a time, but session-scoped state means the host's turn-end -// restore doesn't need to reach back into this package. - -async function syncClipboardStash( +/** Restore a clipboard stash from the former app-tier guard without creating + * new restrictions based on the foreground application's identity. */ +async function restoreLegacyClipboardStash( adapter: ComputerUseHostAdapter, overrides: ComputerUseOverrides, - frontmostIsClickTier: boolean, ): Promise { - const current = overrides.getClipboardStash?.(); - if (!frontmostIsClickTier) { - // Restore + clear. Idempotent — if nothing is stashed, no-op. - if (current === undefined) return; - try { - await adapter.executor.writeClipboard(current); - // Clear only after a successful write — a transient pasteboard - // failure must not irrecoverably drop the stash. - overrides.onClipboardStashChanged?.(undefined); - } catch { - // Best effort — stash held, next non-click action retries. - } - return; - } - // Stash the user's clipboard on FIRST entry to click-tier only. - if (current === undefined) { - try { - const read = await adapter.executor.readClipboard(); - overrides.onClipboardStashChanged?.(read); - } catch { - // readClipboard failed — use empty sentinel so we don't retry the stash - // on the next action; restore becomes a harmless writeClipboard(""). - overrides.onClipboardStashChanged?.(""); - } - } - // Re-clear on EVERY click-tier action, not just the first. Defeats the - // bypass where the agent calls write_clipboard (which doesn't route - // through runInputActionGates) between stash and a left_click on a UI - // Paste button — the next action's clear clobbers the agent's write - // before the click lands. + const current = overrides.getClipboardStash?.() + if (current === undefined) return try { - await adapter.executor.writeClipboard(""); + await adapter.executor.writeClipboard(current) + overrides.onClipboardStashChanged?.(undefined) } catch { - // Transient pasteboard failure. The tier-"click" right-click/modifier - // block still holds; this is a net, not a promise. + // Preserve the stash so a transient clipboard failure can be retried. } } @@ -404,7 +340,6 @@ async function runInputActionGates( adapter: ComputerUseHostAdapter, overrides: ComputerUseOverrides, subGates: CuSubGates, - actionKind: CuActionKind, ): Promise { // Windows shows the full desktop. Preparation gets an empty filter so it may // perform platform bookkeeping without hiding apps based on an allowlist. @@ -432,147 +367,11 @@ async function runInputActionGates( ); } - const { hostBundleId } = adapter.executor.capabilities; - - if (frontmost.bundleId === hostBundleId) { - if (actionKind !== "keyboard") return null; - return errorResult( - "Claude's own window still has keyboard focus. Click on the target " + - "application first so typing cannot land in the chat box.", - "state_conflict", - ); - } - - if (isPolicyDenied(frontmost.bundleId, frontmost.displayName)) { - return errorResult( - `"${frontmost.displayName}" is not supported by Computer Use for product-safety reasons.`, - "app_denied", - ); - } - - const frontmostTier = automaticTier(frontmost.bundleId, frontmost.displayName); - + // Feature-wide consent covers every identified app, including our host. if (subGates.clipboardGuard) { - await syncClipboardStash(adapter, overrides, frontmostTier === "click"); + await restoreLegacyClipboardStash(adapter, overrides) } - - if (!tierSatisfies(frontmostTier, actionKind)) { - if (frontmostTier === "read") { - // tier "read" is not category-unique (browser AND trading map to it) — - // re-look-up so the CiC hint only shows for actual browsers. - const isBrowser = - getDeniedCategoryForApp(frontmost.bundleId, frontmost.displayName) === - "browser"; - return errorResult( - `"${frontmost.displayName}" is restricted to tier "read" — ` + - `visible in screenshots only, no clicks or typing.` + - (isBrowser - ? " Use the Claude-in-Chrome MCP for browser interaction (tools " + - "named `mcp__Claude_in_Chrome__*`; load via ToolSearch if " + - "deferred)." - : " No interaction is permitted; ask the user to take any " + - "actions in this app themselves.") + - TIER_ANTI_SUBVERSION, - "tier_insufficient", - ); - } - if (actionKind === "keyboard") { - return errorResult( - `"${frontmost.displayName}" is restricted to tier "click" — ` + - `typing, key presses, and paste require tier "full". The keys ` + - `would go to this app's text fields or integrated terminal. To ` + - `type into a different app, click it first to bring it forward. ` + - `For shell commands, use the Bash tool.` + TIER_ANTI_SUBVERSION, - "tier_insufficient", - ); - } - // actionKind === "mouse_full" ("mouse" and "mouse_position" pass at "click") - return errorResult( - `"${frontmost.displayName}" is restricted to tier "click" — ` + - `right-click, middle-click, and clicks with modifier keys require ` + - `tier "full". Right-click opens a context menu with Paste/Cut, and ` + - `modifier chords fire as keystrokes before the click. Plain ` + - `left_click is allowed here.` + TIER_ANTI_SUBVERSION, - "tier_insufficient", - ); - } - - return null; -} - -/** - * Hit-test gate: reject a mouse action if the window under (x, y) belongs - * to an app whose tier doesn't cover mouse input. Closes the gap where a - * tier-"full" app is frontmost but the click lands on a tier-"read" window - * overlapping it — `runInputActionGates` passes (frontmost is fine), but the - * click actually goes to the read-tier app. - * - * Runs AFTER `scaleCoord` (needs global coords) and BEFORE the executor call. - * Returns null on pass (target is tier-"click"/"full", or desktop/Finder/us), - * error-result on block. - * - * When `appUnderPoint` returns null (desktop, or platform without hit-test), - * falls through — the frontmost check in `runInputActionGates` already ran. - */ -async function runHitTestGate( - adapter: ComputerUseHostAdapter, - overrides: ComputerUseOverrides, - subGates: CuSubGates, - x: number, - y: number, - actionKind: CuActionKind, -): Promise { - const target = await adapter.executor.appUnderPoint(x, y); - if (!target) return null; // desktop / nothing under point / platform no-op - - // Finder (desktop, file dialogs) and our click-through overlay are valid. - if (target.bundleId === FINDER_BUNDLE_ID) return null; - if (target.bundleId === adapter.executor.capabilities.hostBundleId) return null; - if (isPolicyDenied(target.bundleId, target.displayName)) { - return errorResult( - `Click at these coordinates would land on "${target.displayName}", ` + - "which Computer Use does not support for product-safety reasons.", - "app_denied", - ); - } - - const targetTier = automaticTier(target.bundleId, target.displayName); - - // Frontmost-based sync (runInputActionGates) misses the case where - // the click lands on a NON-FRONTMOST click-tier window. Re-sync by - // the hit-test target's tier — if target is click-tier, stash+clear - // before the click lands, regardless of what's frontmost. - if (subGates.clipboardGuard && targetTier === "click") { - await syncClipboardStash(adapter, overrides, true); - } - - if (tierSatisfies(targetTier, actionKind)) return null; - - // Target is in the allowlist but tier doesn't cover this action. - // runHitTestGate is only called with mouse/mouse_full (keyboard routes to - // frontmost, not window-under-cursor). The branch above catches - // mouse_full ∧ click; the only remaining fall-through is tier "read". - if (actionKind === "mouse_full" && targetTier === "click") { - return errorResult( - `Click at these coordinates would land on "${target.displayName}", ` + - `which is restricted to tier "click" — right-click, middle-click, and ` + - `clicks with modifier keys require tier "full" (they can Paste via ` + - `the context menu or fire modifier-chord keystrokes). Plain ` + - `left_click is allowed here.` + TIER_ANTI_SUBVERSION, - "tier_insufficient", - ); - } - const isBrowser = - getDeniedCategoryForApp(target.bundleId, target.displayName) === "browser"; - return errorResult( - `Click at these coordinates would land on "${target.displayName}", ` + - `which is restricted to tier "read" (screenshots only, no interaction). ` + - (isBrowser - ? "Use the Claude-in-Chrome MCP for browser interaction." - : "Ask the user to take any actions in this app themselves.") + - TIER_ANTI_SUBVERSION, - "tier_insufficient", - ); + return null } // --------------------------------------------------------------------------- @@ -691,17 +490,12 @@ export async function releaseHeldMouseForSession( ): Promise { return releaseHeldMouse(adapter, owner) } -/** Whether mouse_move occurred between left_mouse_down and left_mouse_up. - * When false at mouseUp, the decomposed sequence is a click-release (not a - * drop) — hit-test at "mouse", not "mouse_full". */ -let mouseMoved = false; /** Clears the cross-call drag flags. Called from Gate-3 on lock-acquire and * from `bindSessionContext` in mcpServer.ts — a fresh lock holder must not * inherit a prior session's mid-drag state. */ export function resetMouseButtonHeld(): void { mouseButtonHeld = false; - mouseMoved = false; mouseButtonOwner = undefined mouseHoldGeneration += 1 } @@ -1265,9 +1059,7 @@ function buildTierGuidanceMessage(tiered: TieredApp[]): string { } if (parts.length === 0) return ""; - // Same anti-subversion clause the gate errors carry — said upfront so the - // model doesn't reach for osascript/cliclick after seeing "no clicks/typing". - return parts.join("\n\n") + TIER_ANTI_SUBVERSION; + return parts.join("\n\n"); } /** @@ -2196,14 +1988,7 @@ async function handleClickVariant( button: "left" | "right" | "middle", count: 1 | 2 | 3, ): Promise { - // A prior left_mouse_down may have set mouseButtonHeld without a matching - // left_mouse_up (e.g. drag rejected by a tier gate, model falls back to - // left_click). executor.click() does its own mouseDown+mouseUp, releasing - // the OS button — but without this, the JS flag stays true and all - // subsequent mouse_move calls take the held-button path ("mouse"/ - // "mouse_full" actionKind + hit-test), causing spurious rejections on - // click-tier and read-tier windows. Release first so click() gets a clean - // slate. + // Release a previous unmatched mouseDown before issuing an atomic click. if (mouseButtonHeld) { await releaseHeldMouse(adapter, mouseOwner(overrides)); } @@ -2236,19 +2021,10 @@ async function handleClickVariant( modifiers = parseKeyChord(args.text); } - // Right/middle-click and any click with a modifier chord escalate to - // keyboard-equivalent input at tier "click" (context-menu Paste, chord - // keystrokes). Compute once, pass to both gates. - const clickActionKind: CuActionKind = - button !== "left" || (modifiers !== undefined && modifiers.length > 0) - ? "mouse_full" - : "mouse"; - const gate = await runInputActionGates( adapter, overrides, subGates, - clickActionKind, ); if (gate) return gate; @@ -2302,16 +2078,6 @@ async function handleClickVariant( adapter.logger, ); - const hitGate = await runHitTestGate( - adapter, - overrides, - subGates, - x, - y, - clickActionKind, - ); - if (hitGate) return hitGate; - await adapter.executor.click(x, y, button, count, modifiers); return okText("Clicked."); } @@ -2329,7 +2095,6 @@ async function handleType( adapter, overrides, subGates, - "keyboard", ); if (gate) return gate; @@ -2452,7 +2217,6 @@ async function handleKey( adapter, overrides, subGates, - "keyboard", ); if (gate) return gate; @@ -2490,7 +2254,7 @@ async function handleScroll( const dx = dir === "left" ? -amount : dir === "right" ? amount : 0; const dy = dir === "up" ? -amount : dir === "down" ? amount : 0; - const gate = await runInputActionGates(adapter, overrides, subGates, "mouse"); + const gate = await runInputActionGates(adapter, overrides, subGates); if (gate) return gate; const display = await adapter.executor.getDisplaySize( @@ -2505,23 +2269,6 @@ async function handleScroll( adapter.logger, ); - // When the button is held, executor.scroll's internal moveMouse generates - // a leftMouseDragged event (enigo reads NSEvent.pressedMouseButtons) — - // same mechanism as handleMoveMouse's held-button path. Upgrade the - // hit-test to "mouse_full" so scroll can't be used to drag-drop text onto - // a click-tier terminal, and mark mouseMoved so the subsequent - // left_mouse_up hit-tests as a drop not a click-release. - const hitGate = await runHitTestGate( - adapter, - overrides, - subGates, - x, - y, - mouseButtonHeld ? "mouse_full" : "mouse", - ); - if (hitGate) return hitGate; - if (mouseButtonHeld) mouseMoved = true; - await adapter.executor.scroll(x, y, dx, dy); return okText("Scrolled."); } @@ -2558,7 +2305,7 @@ async function handleDrag( } // else: rawFrom stays undefined → executor drags from current cursor. - const gate = await runInputActionGates(adapter, overrides, subGates, "mouse"); + const gate = await runInputActionGates(adapter, overrides, subGates); if (gate) return gate; const display = await adapter.executor.getDisplaySize( @@ -2584,35 +2331,6 @@ async function handleDrag( adapter.logger, ); - // Check both drag endpoints. `from` is where the mouseDown happens (picks - // up), `to` is where mouseUp happens (drops). When start_coordinate is - // omitted the drag begins at the cursor — same bypass as mouse_move → - // left_mouse_down, so read the cursor and hit-test it (mirrors - // handleLeftMouseDown). - // - // The `to` endpoint uses "mouse_full" (not "mouse"): dropping text onto a - // terminal inserts it as if typed (macOS text drag-drop). Same threat as - // right-click→Paste. `from` stays "mouse" — picking up is a read. - const fromPoint = from ?? (await adapter.executor.getCursorPosition()); - const fromGate = await runHitTestGate( - adapter, - overrides, - subGates, - fromPoint.x, - fromPoint.y, - "mouse", - ); - if (fromGate) return fromGate; - const toGate = await runHitTestGate( - adapter, - overrides, - subGates, - to.x, - to.y, - "mouse_full", - ); - if (toGate) return toGate; - await adapter.executor.drag(from, to); return okText("Dragged."); } @@ -2627,17 +2345,10 @@ async function handleMoveMouse( if (coord instanceof Error) return errorResult(coord.message, "bad_args"); const [rawX, rawY] = coord; - // When the button is held, moveMouse generates leftMouseDragged events on - // the window under the cursor — that's interaction, not positioning. - // Upgrade to "mouse" and hit-test the destination. When the button is NOT - // held: pure positioning, passes at any tier, no hit-test (mouseDown/Up - // hit-test the cursor to close the mouse_move→left_mouse_down decomposition). - const actionKind: CuActionKind = mouseButtonHeld ? "mouse" : "mouse_position"; const gate = await runInputActionGates( adapter, overrides, subGates, - actionKind, ); if (gate) return gate; @@ -2653,23 +2364,7 @@ async function handleMoveMouse( adapter.logger, ); - if (mouseButtonHeld) { - // "mouse_full" — same as left_click_drag's to-endpoint. Dragging onto a - // click-tier terminal is text injection regardless of which primitive - // (atomic drag vs. decomposed down/move/up) delivers the events. - const hitGate = await runHitTestGate( - adapter, - overrides, - subGates, - x, - y, - "mouse_full", - ); - if (hitGate) return hitGate; - } - await adapter.executor.moveMouse(x, y); - if (mouseButtonHeld) mouseMoved = true; return okText("Moved."); } @@ -2698,17 +2393,6 @@ async function handleOpenApplication( ); } - if (isPolicyDenied(match.bundleId, match.displayName)) { - return errorResult( - `"${match.displayName}" is not supported by Computer Use for product-safety reasons.`, - "app_denied", - ); - } - - // open_application works at any tier — bringing an app forward is exactly - // what tier "read" enables (you need it on screen to screenshot it). The - // tier gates on click/type catch any follow-up interaction. - await adapter.executor.openApp(match.bundleId); // On multi-monitor setups, macOS may place the opened window on a monitor @@ -2817,19 +2501,10 @@ async function handleReadClipboard( ); } - // read_clipboard doesn't route through runInputActionGates — sync here so - // reading after clicking into a click-tier app sees the cleared clipboard - // (same as what the app's own Paste would see). if (subGates.clipboardGuard) { - const frontmost = await adapter.executor.getFrontmostApp(); - const frontmostTier = frontmost - ? automaticTier(frontmost.bundleId, frontmost.displayName) - : undefined; - await syncClipboardStash(adapter, overrides, frontmostTier === "click"); + await restoreLegacyClipboardStash(adapter, overrides) } - // clipboardGuard may have stashed+cleared — read the actual (possibly - // empty) clipboard. The agent sees what the app would see. const text = await adapter.executor.readClipboard(); return okJson({ text }); } @@ -2850,32 +2525,7 @@ async function handleWriteClipboard( if (text instanceof Error) return errorResult(text.message, "bad_args"); if (subGates.clipboardGuard) { - const frontmost = await adapter.executor.getFrontmostApp(); - const frontmostTier = frontmost - ? automaticTier(frontmost.bundleId, frontmost.displayName) - : undefined; - - // Defense-in-depth for the clipboardGuard bypass: write_clipboard + - // left_click on a click-tier app's UI Paste button. The re-clear in - // syncClipboardStash already defeats it (the next action clobbers the - // write), but rejecting here gives the agent a clear signal instead of - // silently voiding its write. - if (frontmost && frontmostTier === "click") { - return errorResult( - `"${frontmost.displayName}" is a tier-"click" app and currently ` + - `frontmost. write_clipboard is blocked because the next action ` + - `would clear the clipboard anyway — a UI Paste button in this ` + - `app cannot be used to inject text. Bring a tier-"full" app ` + - `forward before writing to the clipboard.` + - TIER_ANTI_SUBVERSION, - "tier_insufficient", - ); - } - - // write_clipboard doesn't route through runInputActionGates — sync here - // so clicking away from a click-tier app then writing restores the user's - // stash before the agent's text lands. - await syncClipboardStash(adapter, overrides, frontmostTier === "click"); + await restoreLegacyClipboardStash(adapter, overrides) } await adapter.executor.writeClipboard(text); @@ -3002,7 +2652,6 @@ async function handleHoldKey( adapter, overrides, subGates, - "keyboard", ); if (gate) return gate; @@ -3027,28 +2676,11 @@ async function handleLeftMouseDown( ); } - const gate = await runInputActionGates(adapter, overrides, subGates, "mouse"); + const gate = await runInputActionGates(adapter, overrides, subGates); if (gate) return gate; - // macOS routes mouseDown to the window under the cursor, not the frontmost - // app. Without this hit-test, mouse_move (positioning, passes at any tier) - // + left_mouse_down decomposes a click that lands on a tier-"read" window - // overlapping a tier-"full" frontmost app — bypassing runHitTestGate's - // whole purpose. All three are batchable, so the bypass is atomic. - const cursor = await adapter.executor.getCursorPosition(); - const hitGate = await runHitTestGate( - adapter, - overrides, - subGates, - cursor.x, - cursor.y, - "mouse", - ); - if (hitGate) return hitGate; - await adapter.executor.mouseDown(); mouseButtonHeld = true; - mouseMoved = false; mouseButtonOwner = mouseOwner(overrides) mouseHoldGeneration += 1 return okText("Mouse button pressed."); @@ -3063,16 +2695,8 @@ async function handleLeftMouseUp( overrides: ComputerUseOverrides, subGates: CuSubGates, ): Promise { - // Any gate rejection here must release the button FIRST — otherwise the - // OS button stays pressed and mouseButtonHeld stays true. Recovery - // attempts (mouse_move back to a safe app) would generate leftMouseDragged - // events into whatever window is under the cursor, including the very - // read-tier window the gate was protecting. A single mouseUp on a - // restricted window is one event; a stuck button is cascading damage. - // - // This includes the frontmost gate: focus can change between mouseDown and - // mouseUp (something else grabbed focus), in which case runInputActionGates - // rejects here even though it passed at mouseDown. + // Always release a held button when the foreground target becomes unknown, + // so a failed action cannot leave the user's mouse stuck down. const releaseFirst = async ( err: CuCallToolResult, ): Promise => { @@ -3081,27 +2705,9 @@ async function handleLeftMouseUp( return err; }; - const gate = await runInputActionGates(adapter, overrides, subGates, "mouse"); + const gate = await runInputActionGates(adapter, overrides, subGates); if (gate) return releaseFirst(gate); - // When the cursor moved since mouseDown, this is a drop (text-injection - // vector) — hit-test at "mouse_full" same as left_click_drag's `to`. When - // NO move happened, this is a click-release — same semantics as the atomic - // left_click, hit-test at "mouse". Without this distinction, a decomposed - // click on a click-tier app fails here while the atomic left_click works, - // and releaseFirst fires mouseUp anyway so the OS sees a complete click - // while the model gets a misleading error. - const cursor = await adapter.executor.getCursorPosition(); - const hitGate = await runHitTestGate( - adapter, - overrides, - subGates, - cursor.x, - cursor.y, - mouseMoved ? "mouse_full" : "mouse", - ); - if (hitGate) return releaseFirst(hitGate); - await adapter.executor.mouseUp(); resetMouseButtonHeld(); return okText("Mouse button released."); @@ -3150,10 +2756,9 @@ interface BatchActionResult { * - Kill-switch + TCC: checked ONCE by handleToolCall before reaching here. * - prepareForAction: run ONCE at the top. The user approved "do this * sequence"; hiding apps per-action is wasted work and fast-pathed anyway. - * - Frontmost gate: checked PER ACTION. State can change mid-batch — a - * click might open a non-allowed app. This is the safety net: if action - * 3 of 5 opened Safari (not allowed), action 4's frontmost check fires - * and stops the batch there. + * - Foreground identification: checked PER ACTION. State can change + * mid-batch; if the foreground application can no longer be identified, + * the next input action stops the batch. * - PixelCompare: SKIPPED inside batch. The model committed to the full * sequence without intermediate screenshots; validating mid-batch clicks * against a pre-batch screenshot would false-positive constantly. @@ -3376,45 +2981,8 @@ export async function handleToolCall( ): Promise { const { logger, serverName } = adapter; - // Normalize the allowlist before any gate runs: - // - // (a) Strip user-denied. A grant from a previous session (before the user - // added the app to Settings → Desktop app → Computer Use → Denied apps) - // must not survive. Without - // this, a stale grant bypasses the auto-deny. Stripped silently — the - // agent already saw the userDenied guidance at request_access time, and - // a live frontmost-gate rejection cites "not in allowed applications". - // - // (b) Strip policy-denied. Same story as (a) for a grant that predates a - // blocklist addition. buildAccessRequest denies these up front for new - // requests; this catches stale persisted grants. - // - // (c) Backfill tier. A grant persisted before the tier field existed has - // `tier: undefined`, which `tierSatisfies` treats as `"full"` — wrong - // for a legacy Chrome grant. Assign the hardcoded tier based on - // bundle-ID category. Modern grants already have a tier. - // - // `.some()` guard keeps the hot path (empty deny list, no legacy grants) - // zero-alloc. - const userDeniedSet = new Set(rawOverrides.userDeniedBundleIds); - const overrides: ComputerUseOverrides = rawOverrides.allowedApps.some( - (a) => - a.tier === undefined || - userDeniedSet.has(a.bundleId) || - isPolicyDenied(a.bundleId, a.displayName), - ) - ? { - ...rawOverrides, - allowedApps: rawOverrides.allowedApps - .filter((a) => !userDeniedSet.has(a.bundleId)) - .filter((a) => !isPolicyDenied(a.bundleId, a.displayName)) - .map((a) => - a.tier !== undefined - ? a - : { ...a, tier: getDefaultTierForApp(a.bundleId, a.displayName) }, - ), - } - : rawOverrides; + // Legacy grants and deny lists do not narrow feature-wide consent. + const overrides = rawOverrides // ─── Gate 1: kill switch ───────────────────────────────────────────── if (adapter.isDisabled()) {