mirror of
https://github.com/NanmiCoder/claude-code-haha.git
synced 2026-10-10 03:43:11 +08:00
feat(computer-use): align native app automation with Codex
Add a persistent isolated JavaScript worker for native app actions and batch known operations without a model round trip between each input. Preserve per-cell context, native errors, screenshot coordinates, and image types. Align macOS gesture, key, inventory, capture, scroll, and clipboard behavior; include a signed native receiver fixture and compiled sidecar regression tests. Keep the Windows pixel route and fix cancellation with session-owned mouse cleanup, lock revalidation, and portable signing-fixture tests.
This commit is contained in:
程序员阿江(Relakkes)
@@ -20,6 +20,15 @@
|
||||
|
||||
import { parseLauncherArgs, resolveSidecarInvocation } from './launcherRouting'
|
||||
|
||||
// The compiled Computer Use runtime relaunches this executable directly. Its
|
||||
// isolated worker has no app-root and must not load preload, CLI configuration,
|
||||
// providers, or any desktop mode before serving the JSON-only kernel protocol.
|
||||
if (process.argv[2] === '--computer-use-repl-worker') {
|
||||
const { runComputerUseReplWorker } = await import('../../src/utils/computerUse/replWorker')
|
||||
await runComputerUseReplWorker()
|
||||
process.exit(0)
|
||||
}
|
||||
|
||||
type AdapterConfigShape = Awaited<
|
||||
ReturnType<typeof import('../../adapters/common/config.ts')['loadConfig']>
|
||||
>
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
// @vitest-environment node
|
||||
|
||||
import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import { mkdtemp, readdir, realpath, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import { createInterface } from 'node:readline'
|
||||
import { describe, expect, it } from 'vitest'
|
||||
import { createComputerUseReplSandboxCommand } from '../../src/utils/computerUse/replRuntime'
|
||||
import { REPL_BOOTSTRAP_SOURCE } from '../../src/vendor/computer-use-mcp/replApi'
|
||||
import type { ReplInput, ReplOutput } from '../../src/vendor/computer-use-mcp/replProtocol'
|
||||
|
||||
const repoRoot = path.resolve(import.meta.dirname, '../..')
|
||||
|
||||
async function run(command: string, args: string[], cwd: string, env: NodeJS.ProcessEnv) {
|
||||
const child = spawn(command, args, { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], timeout: 90_000 })
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
child.stdout.on('data', chunk => { stdout += String(chunk) })
|
||||
child.stderr.on('data', chunk => { stderr += String(chunk) })
|
||||
const code = await new Promise<number | null>((resolve, reject) => {
|
||||
child.once('error', reject)
|
||||
child.once('exit', resolve)
|
||||
})
|
||||
if (code !== 0) throw new Error(`${path.basename(command)} exited ${code}: ${stderr}\n${stdout}`)
|
||||
return stdout
|
||||
}
|
||||
|
||||
describe.skipIf(process.platform !== 'darwin')('compiled desktop Computer Use worker routing', () => {
|
||||
it('boots the real merged entrypoint inside the production sandbox without preload and retains native App bindings', async () => {
|
||||
const directory = await realpath(await mkdtemp(path.join(tmpdir(), 'cc-haha-sidecar-cu-worker-')))
|
||||
const executable = path.join(directory, 'claude-sidecar-aarch64-apple-darwin')
|
||||
const env: NodeJS.ProcessEnv = {
|
||||
PATH: process.env.PATH,
|
||||
HOME: directory,
|
||||
TMPDIR: directory,
|
||||
TMP: directory,
|
||||
TEMP: directory,
|
||||
CLAUDE_CONFIG_DIR: path.join(directory, '.claude'),
|
||||
BUN_OPTIONS: '--no-env-file',
|
||||
// preload.ts would chdir here. The internal worker must never load it.
|
||||
CALLER_DIR: path.join(directory, 'must-not-enter-preload'),
|
||||
}
|
||||
let child: ChildProcessWithoutNullStreams | undefined
|
||||
let closed: Promise<{ code: number | null; signal: NodeJS.Signals | null }> | undefined
|
||||
try {
|
||||
// Compile the real production entrypoint, not a handwritten worker shim.
|
||||
// These are the optional externals used by build-sidecars.ts; Acorn and
|
||||
// sandbox-runtime remain bundled. No signing identity/keychain discovery.
|
||||
const build = {
|
||||
entrypoints: [path.join(repoRoot, 'desktop/sidecars/claude-sidecar.ts')],
|
||||
features: ['TRANSCRIPT_CLASSIFIER'],
|
||||
minify: { whitespace: true, identifiers: true, syntax: true },
|
||||
sourcemap: 'none',
|
||||
target: 'bun',
|
||||
external: [
|
||||
'@opentelemetry/exporter-trace-otlp-grpc', '@opentelemetry/exporter-trace-otlp-http',
|
||||
'@opentelemetry/exporter-trace-otlp-proto', '@opentelemetry/exporter-logs-otlp-grpc',
|
||||
'@opentelemetry/exporter-logs-otlp-http', '@opentelemetry/exporter-logs-otlp-proto',
|
||||
'@opentelemetry/exporter-metrics-otlp-grpc', '@opentelemetry/exporter-metrics-otlp-http',
|
||||
'@opentelemetry/exporter-metrics-otlp-proto', '@opentelemetry/exporter-prometheus',
|
||||
'@aws-sdk/client-bedrock', '@aws-sdk/client-sts', '@anthropic-ai/bedrock-sdk',
|
||||
'@anthropic-ai/foundry-sdk', '@anthropic-ai/vertex-sdk', '@azure/identity',
|
||||
'@anthropic-ai/mcpb', 'fflate', 'sharp', 'react-devtools-core',
|
||||
],
|
||||
compile: { outfile: executable, autoloadTsconfig: true, autoloadPackageJson: true },
|
||||
}
|
||||
await run('bun', ['--no-env-file', '-e', `const r=await Bun.build(${JSON.stringify(build)});if(!r.success){console.error(r.logs);process.exit(1)}`], repoRoot, { ...env, CALLER_DIR: undefined })
|
||||
await run('/usr/bin/codesign', ['--remove-signature', executable], directory, env)
|
||||
await run('/usr/bin/codesign', ['--sign', '-', '--force', '--timestamp=none', executable], directory, env)
|
||||
|
||||
const command = createComputerUseReplSandboxCommand({
|
||||
args: [executable, '--computer-use-repl-worker'],
|
||||
readable: [executable, directory],
|
||||
directory,
|
||||
})
|
||||
child = spawn('/bin/sh', ['-c', `exec ${command}`], { cwd: directory, env, stdio: 'pipe' })
|
||||
closed = new Promise(resolve => child!.once('close', (code, signal) => resolve({ code, signal })))
|
||||
const messages: ReplOutput[] = []
|
||||
const invocations: Array<{ name: string; args: unknown }> = []
|
||||
let stderr = ''
|
||||
let failure: Error | undefined
|
||||
child.stderr.on('data', chunk => { stderr += String(chunk) })
|
||||
child.on('error', error => { failure = error })
|
||||
child.stdin.on('error', error => { failure = error })
|
||||
const lines = createInterface({ input: child.stdout })
|
||||
const send = (message: ReplInput) => child!.stdin.write(`${JSON.stringify(message)}\n`)
|
||||
lines.on('line', line => {
|
||||
try {
|
||||
const message = JSON.parse(line) as ReplOutput
|
||||
messages.push(message)
|
||||
if (message.type === 'invoke') {
|
||||
invocations.push({ name: message.name, args: message.args })
|
||||
send({
|
||||
type: 'response', cellId: message.cellId, requestId: message.requestId,
|
||||
result: {
|
||||
app: (message.args as { app?: string }).app === 'Replacement'
|
||||
? '/Applications/Replacement.app' : '/Applications/Fixture.app',
|
||||
content: [{ type: 'text', text: '<app_state>\ng1:0 window\n\tg1:1 button Test\n</app_state>' }],
|
||||
},
|
||||
})
|
||||
}
|
||||
} catch (error) {
|
||||
failure = error instanceof Error ? error : new Error(String(error))
|
||||
}
|
||||
})
|
||||
async function until(predicate: () => boolean) {
|
||||
const deadline = Date.now() + 10_000
|
||||
while (!predicate()) {
|
||||
if (failure) throw failure
|
||||
if (child!.exitCode !== null || child!.signalCode !== null) {
|
||||
throw new Error(`Worker exited before readiness/completion: ${stderr}`)
|
||||
}
|
||||
if (Date.now() > deadline) throw new Error(`Worker timed out: ${stderr}`)
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
}
|
||||
}
|
||||
send({ type: 'init', bootstrap: REPL_BOOTSTRAP_SOURCE })
|
||||
await until(() => messages.some(message => message.type === 'ready'))
|
||||
send({ type: 'run', cellId: 1, code: 'let app = await cua.getApp("Fixture"); let count = 0; async function press(){await app.click([10,10]);count++}; function current(){return count}' })
|
||||
await until(() => messages.some(message => message.type === 'done' && message.cellId === 1))
|
||||
send({ type: 'run', cellId: 2, code: 'for (let i=0;i<3;i++) await press(); nodeRepl.write(count)' })
|
||||
await until(() => messages.some(message => message.type === 'done' && message.cellId === 2))
|
||||
send({ type: 'run', cellId: 3, code: 'app = await cua.getApp("Replacement"); count=10; await press(); nodeRepl.write({count,current:current()})' })
|
||||
await until(() => messages.some(message => message.type === 'done' && message.cellId === 3))
|
||||
expect(messages.filter(message => message.type === 'done' && message.error)).toEqual([])
|
||||
expect(invocations.map(call => call.name)).toEqual(['get_app_state', 'click', 'click', 'click', 'get_app_state', 'click'])
|
||||
expect(invocations[1]?.args).toEqual({ app: '/Applications/Fixture.app', x: 10, y: 10 })
|
||||
expect(invocations[5]?.args).toEqual({ app: '/Applications/Replacement.app', x: 10, y: 10 })
|
||||
expect(messages).toContainEqual({ type: 'emit', cellId: 2, content: { type: 'text', text: '3' } })
|
||||
const closureResult = messages.find(message => message.type === 'emit' && message.cellId === 3 && message.content.type === 'text' && message.content.text.includes('current'))
|
||||
expect(closureResult?.type === 'emit' && closureResult.content.type === 'text' && JSON.parse(closureResult.content.text)).toEqual({ count: 11, current: 11 })
|
||||
child.stdin.end()
|
||||
expect(await closed).toEqual({ code: 0, signal: null })
|
||||
expect(stderr).toBe('')
|
||||
expect((await readdir(directory)).filter(name => name !== path.basename(executable))).toEqual([])
|
||||
} finally {
|
||||
child?.kill('SIGKILL')
|
||||
if (closed) await closed
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
}, 120_000)
|
||||
})
|
||||
@@ -0,0 +1,146 @@
|
||||
---
|
||||
title: Codex Native Computer Use Compatibility Contracts
|
||||
nav_title: Codex Compatibility
|
||||
description: Verified contracts for the official plugin, native events, action timing, and batched observations, with current compatibility limits.
|
||||
order: 15
|
||||
---
|
||||
|
||||
# Codex Native Computer Use Compatibility Contracts
|
||||
|
||||
The macOS native Computer Use implementation uses the plugin, JavaScript client, and native service shipped with official Codex as its compatibility reference. This page describes verified call and event contracts so source readers can distinguish input delivery, observation, and model scheduling. Third-party replicas no longer define official behavior; similar interfaces or success on one task do not establish complete compatibility.
|
||||
|
||||
## Reference version and layers
|
||||
|
||||
These contracts come from the official build inspected on September 9, 2026:
|
||||
|
||||
| Component | Version or location |
|
||||
| --- | --- |
|
||||
| Computer Use plugin | `openai-bundled/unified-computer-use/26.901.51231` |
|
||||
| Native service | `SkyComputerUseService`, version `26.831.1000926` |
|
||||
| Native service SHA-256 | `25e9141499b94c396f39afbdb7b19ed8f49e45dc8c61be61028ceab8f3807ce6` |
|
||||
| Plugin cache | `${CODEX_HOME}/plugins/cache/openai-bundled/unified-computer-use/<version>/` |
|
||||
| JavaScript packages inside the App | `Contents/Resources/cua_node/lib/node_modules/@oai/cua` and `@oai/sky` |
|
||||
| Local native service | `${CODEX_HOME}/computer-use/Codex Computer Use.app/Contents/MacOS/SkyComputerUseService` |
|
||||
|
||||
These versions define the scope of the contracts. Recheck both caller and implementation after upgrades. Native function addresses apply only to this binary; detailed evidence lives in `native/cu-helper/README.md` in the source tree.
|
||||
|
||||
Native App operations follow this path:
|
||||
|
||||
```text
|
||||
A cua App object in a persistent JavaScript session
|
||||
→ @oai/cua / @oai/sky client
|
||||
→ persistent local IPC
|
||||
→ SkyComputerUseService
|
||||
→ AX queries or events targeting a process and window
|
||||
```
|
||||
|
||||
The browser Tab provider is a separate path. Having a browser extension installed does not mean every Chrome action uses the DOM: a native App object obtained through `cua.getApp(...)` can send coordinate clicks and drags directly. Identify the execution mechanism from the actual object, method, and transport used.
|
||||
|
||||
The macOS native App path permits control of browsers such as Chrome while retaining other application restrictions, authorization, signature, and process-identity checks. Previously, both TypeScript dispatch and native `AppTargetPolicy` rejected targets solely because they were browsers, blocking this valid native route. Both layers now apply the same native-browser policy. Windows keeps its existing browser category and permission tiers. Allowing a browser as a native App does not supply Tab bindings, DOM nodes, or Playwright methods.
|
||||
|
||||
The designated successful Townscaper trace illustrates this distinction. Its 21 JavaScript calls begin with browser inventory and a timed-out `getTab`, then bind the native Chrome App. All 57 input actions use that App: 47 drags, 3 clicks, 3 scrolls, 3 key presses, and 1 paste, with 18 screenshots returned. Six cells containing loops execute 41 drags: 39 inside loop bodies and 2 palette actions outside them. There are no successful Tab, DOM, or Playwright construction calls. This case's consecutive construction actions use the native App path; the independent browser provider is outside this compatibility scope.
|
||||
|
||||
The public Codex CLI and App Server source includes MCP integration, tool dispatch, and message handling, but not this native service's mouse-event implementation. The official installation provides readable JavaScript clients; native contracts also require checking exported symbols, actual call arguments, and machine code. Missing CGEvent symbols in a static import table do not prove an AX-only implementation: the service also sends events through dynamically resolved function pointers.
|
||||
|
||||
## App discovery and native APIs
|
||||
|
||||
Native application restrictions match the official 24 forbidden bundle IDs exactly, in addition to this product's host and helper identities. Legacy IDE, music, trading, and display-name substring categories no longer determine macOS native access. Discovery can include forbidden targets; binding and actions still enforce authorization.
|
||||
|
||||
`cua.listApps()` combines regular running Apps with Spotlight Apps used in the last 14 days, preserving `id`, `displayName`, `isRunning`, and optional `lastUsedDate` and `useCount`. The bundle-based `id` is distinct from a resolved App path. Structured inventory crosses the helper, dispatcher, and worker intact, with compatibility for older helpers that only report running Apps. The 11 snake_case native window methods are also available through `cua.computer`, using the same authorization and process checks.
|
||||
|
||||
## The five-event coordinate drag contract
|
||||
|
||||
Official `app.drag()` passes its origin and optional drag destination through the native coordinate controller to `ApplicationUIElement.sendClick`. The underlying `SynthesizedEvent.click` creates these events, with one window binding throughout the gesture:
|
||||
|
||||
| Order | Event | Location | clickCount | eventNumber |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | mouseDown | Origin | 1 | Gesture number |
|
||||
| 2 | mouseDragged | Origin | 0 | Motion number |
|
||||
| 3 | mouseDragged | Midpoint | 0 | Same motion number |
|
||||
| 4 | mouseDragged | Destination | 0 | Same motion number |
|
||||
| 5 | mouseUp | Destination | 1 | Same gesture number |
|
||||
|
||||
Zero-distance drags retain all five events and must not collapse into ordinary clicks. The initial dragged event at the origin must also remain. Ordinary click pairs and drag motion events have different semantics; an arbitrary interpolation count is not a substitute for this contract.
|
||||
|
||||
The native service constructs mouse events through AppKit, obtains their CGEvents, assigns target-window information and global/window-local positions, then posts to the target PID. This is neither AXPress alone nor foreground clicking after moving the user's real pointer. Coordinate conversion, window identity, and process lifetime must agree throughout the path; similar window titles cannot establish a screenshot/input binding by themselves.
|
||||
|
||||
The corresponding implementation is concentrated in `native/cu-helper/Sources/cu-helper/AXAction.swift`, `WindowTargetedEvent.swift`, and `WindowGeometry.swift`. `MouseDragEventTests` uses the production event factory to verify order, coordinates, event numbers, and window binding.
|
||||
|
||||
## Keyboard input
|
||||
|
||||
A native receiving window verified the actual modifier events for `Control_L/R`, `Super_L/R`, `Meta_L/R`, `Shift_L/R`, and `Alt_L/R`. `Delete` maps to forward-delete keyCode 117 and `BackSpace` to 51. Uppercase letters and named symbols such as `question` retain Shift. Both the parser and system-key grant checks recognize these aliases.
|
||||
|
||||
## Scroll pages and target regions
|
||||
|
||||
Official native `scroll` sends precise pixel wheel events and preserves fractional pages. Indexed vertical scrolling multiplies pages by the target element's outer frame height. Coordinate scrolling uses the current window height, not the scroll area under that point. In the same 210-point-high scroll area, indexed requests for 0.5 and 1.5 pages produced deltas of -105 and -315. In its 552-point-high window, a coordinate request for 0.5 pages produced -276, or +276 when scrolling up. These requests must not become rounded whole-page AX actions or a fixed 12-line wheel conversion. Explicit `performSecondaryAction` calls such as Scroll Down retain the control's exposed AX page-action semantics.
|
||||
|
||||
The inspected official version creates a single-axis wheel event for horizontal requests, and receiver measurements confirmed that horizontal deltas were ignored. This implementation retains working two-axis horizontal scrolling. That is a documented behavioral difference rather than a reproduction of the ineffective action.
|
||||
|
||||
Wheel events also carry window fields 51, 91, and 92 and window-local coordinates. Fields 91 and 92 alone do not establish AppKit's `windowNumber`, allowing event construction to succeed while the receiver gets nothing. Actual receiver tests verified precise scrolling after adding field 51. Noninteger distances round to the nearest integer; for example, 210 × 0.123 points produces a 26-pixel delta.
|
||||
|
||||
For its four page directions, `performSecondaryAction` first reads `AXVerticalScrollBar` or `AXHorizontalScrollBar`. It searches the scrollbar's `AXChildren` in order for the first `AXButton` with subrole `AXDecrementPage` (up/left) or `AXIncrementPage` (down/right), then performs `AXPress`. Only a missing scrollbar or button falls back to the target's raw `AXScroll*ByPage` action. A failed button press propagates without replay. `AXIncrementPage` and `AXDecrementPage` are subrole values, not queryable attribute names. The control determines the page distance; no pixel distance is hardcoded.
|
||||
|
||||
## Paste consumption and timing
|
||||
|
||||
The native paste operation's two seconds are a read timeout, not a mandatory delay. Before writing temporary content, it captures supported `AXSelectedTextRange` and `AXNumberOfCharacters` values from the target process's focused element and observes that element's selection and value changes. Notifications become armed after the write. A successful promised-data supply ends the first stage early.
|
||||
|
||||
The second stage checks the target every 25 milliseconds for at most two seconds. A target notification after data supply or a valid attribute change permits early completion. Another clipboard observer's read alone does not establish target consumption, and a failed AX read is not a changed value. With no observable signals, the operation retains a 100-millisecond window after the read. With signals but no observed change, it reports a target-confirmation timeout and never replays the paste. Cancellation still drains the bounded consumption window before restoration. External copies always win; the previous clipboard is restored only while the temporary content remains owned.
|
||||
|
||||
The deterministic receiver regression uses a separate named pasteboard while retaining the real Router, Command-V, menu, data provider, target AX confirmation, and restoration. It isolates the shared data source and does not establish general-clipboard behavior in every environment.
|
||||
|
||||
## Screenshot size and format
|
||||
|
||||
Official native state capture normalizes to logical point dimensions, then caps the long edge at 2048 and the short edge at 768 without upscaling. Output pixel dimensions are rounded up, with JPEG quality 0.8. A 1398 × 769 point window therefore produces 1397 × 768 pixels. This difference is scaling, not edge cropping. Coordinate conversion retains the complete window frame and the uniform pre-rounding `pixelsPerPoint` scale rather than deriving separate axis scales from rounded dimensions. Older snapshots without the uniform scale keep the previous conversion.
|
||||
|
||||
The native result's `mimeType` follows image data across dispatch and the worker. Older helpers without this field remain PNG-compatible. The official wrapper has labeled actual JPEG bytes as PNG; this implementation keeps the correct JPEG MIME label so downstream model requests can interpret the content correctly.
|
||||
|
||||
## Action timing and the visible cursor
|
||||
|
||||
The inspected coordinate click and drag calls explicitly pass `delay: nil`. Both direct and virtual-cursor senders skip their optional sleep. There is therefore no fixed 30 ms delay per event or fixed 100 ms button hold on this path. The service's `humanClickInterval` constant belongs to other explicit click or press paths and must not be applied to coordinate drags.
|
||||
|
||||
The virtual-cursor branch updates pressed state synchronously and posts events. The coordinate click/drag path does not wait for a cursor movement animation to complete before sending input. Dedicated `moveMouse` operations have their own animation and next-interaction timing on a different path. Cursor feedback should remain visible without adding an unconditional visual delay to every coordinate gesture.
|
||||
|
||||
The machine code contains Swift executor transitions, which do not guarantee a suspension or a specific interval between events. Adding a guessed sleep or `Task.yield` would not reproduce a verified contract. This implementation removes confirmed artificial waits while retaining target validation before delivery, cancellation checks, and button-release cleanup after errors.
|
||||
|
||||
Focus preparation must also remain separate from visual timing. `SyntheticWindowFocus` preserves process-lifetime, focus-change, and input-acknowledgement checks. Established focus can be reused; acknowledgement waits apply when focus actually needs to be established or restored. Removing all focus checks is not a compatibility optimization.
|
||||
|
||||
## Batched actions and observations
|
||||
|
||||
The official native App API can retain App bindings, compute coordinates, and loop over actions in a persistent JavaScript session. One tool call can perform several known actions before reading AX state or taking a screenshot. `await app.drag()` inside that loop waits for a local action; it does not require a new model response between gestures.
|
||||
|
||||
The native coordinate controller enters UI settling and capture only when `returnSkyshot: true`; false marks state for refresh and returns. The verified settle call includes a 250 ms notification-delay parameter. This is neither an unconditional 250 ms sleep after every action nor a promise about total capture time. Automatically taking a new screenshot after every mutation is not a fixed official contract.
|
||||
|
||||
On macOS, only `js` and `js_reset` are advertised to the model, reducing duplicated per-action schemas. `js` supports persistent variables and App bindings, top-level `await`, loops, calculations, and observations within a call. The worker's native `cua` App methods send JSON messages to the host and enter the existing semantic tools' permission, process-identity, and window checks. A script's App object cannot bypass those checks. `cua.getApp()` displays initial AX text; the first visible App selection or inventory also includes brief API guidance. App bindings use the resolved path returned by the host, and subsequent actions still validate the target again.
|
||||
|
||||
`app.getAXState()`, `app.getScreenshot()`, and `app.getAXStateAndScreenshot()` return and display text, an image, or both. `emit: false` preserves the return value while suppressing display. The inspected official JavaScript implementation also maps all three methods to `get_app_state`, so output selection does not imply skipping AX traversal or capture. Actions do not automatically observe; scripts explicitly request state at the next decision point.
|
||||
|
||||
Underlying element handles remain `gN:id`. The facade maps only AX rows actually returned to the caller to integer indices, applies additions, changes, and removals from diffs, and clears old mappings when the generation changes. Image-only requests can refresh native state and therefore clear integer mappings. Call `app.getAXState({disableDiffing: true})` before using integers again. Copied opaque handles still pass through native validation.
|
||||
|
||||
| Limit per `js` call | Value |
|
||||
| --- | --- |
|
||||
| Native calls | At most 256, including observations |
|
||||
| Code | At most 256 KiB |
|
||||
| Displayed output | At most 128 blocks and 16 MiB in total |
|
||||
| Timeout | 30 seconds by default, at most 60 seconds |
|
||||
|
||||
Shared lexical accessors connect variables across cells, so older functions and newer scripts read the same binding. Previously defined functions use the new App after rebinding. A function may also refer to an App declared in a later cell. Local parameters, block scopes, destructuring, and class-local bindings retain their own semantics; copying variable values into each new cell cannot provide this persistence.
|
||||
|
||||
Ordinary script errors retain recoverable existing bindings and declarations or direct writes that executed. Unreached `var/function` declarations in a failed cell do not leave extra bindings merely because they are hoisted. When an App replacement initializer fails, the previous App binding remains available. `js_reset`, cancellation, or timeout discards the worker and its bindings. Completed actions are not rolled back: bind again and observe before deciding how to continue partially completed work. Imports, Node, filesystem, and networking APIs are currently unavailable. Execution and isolation boundaries live in `src/utils/computerUse/replRuntime.ts`, `replWorker.ts`, and `replCompiler.ts`; native method and observation-output adaptation lives in `src/vendor/computer-use-mcp/replApi.ts`.
|
||||
|
||||
Native errors with proven equivalents retain the `SkyComputerUseError` name, code, errorName, and request fields. Unmapped helper errors retain their original `nativeCode`; messages are not used to guess official codes. Permission or argument refusals before dispatch increment `nativeCallsRejectedBeforeDispatch` and are not reported as completed actions or unknown execution results. Timeouts and errors with possible partial effects remain result-unknown and are never automatically replayed.
|
||||
|
||||
The desktop uses a merged sidecar executable. It must recognize the internal worker argument before parsing ordinary modes or `app-root`, or loading `preload`, and start the isolated kernel directly. Handling the argument only in the CLI sub-entrypoint makes the actual desktop executable fail earlier; source-worker or handwritten compiled-entrypoint tests cannot cover this boundary. The worker's HOME and temporary paths point to disposable directories. Temporary-directory variables are set again inside the sandbox so the wrapping library cannot replace them.
|
||||
|
||||
The existing `sequence` remains a compatible JSON batch entry point for one App. It executes serially, observes once at the end, and reports completed steps after failure or cancellation. It accepts at most 256 steps and uses a cooperative 60-second deadline; an in-flight native command must finish before the runner returns. Standalone semantic tools also retain direct-call compatibility, but these interfaces are no longer advertised to the model by default. Windows continues using its existing pixel tools without these JavaScript interfaces.
|
||||
|
||||
Whether batching is appropriate depends on interface stability. Known canvas actions can run consecutively; opening menus, changing windows, or otherwise changing subsequent targets creates a new decision point that needs observation. Measure action counts, observation counts, model round trips, tool duration, and receiver-visible results separately.
|
||||
|
||||
## Current compatibility limits
|
||||
|
||||
Event-factory tests establish the five-event construction. After one observation, `AXTreePublicationIntegrationTests` sends twelve consecutive zero-distance or short drags to a disposable native App and uses a receiver-side counter to verify complete gesture delivery. These tests validate native input; they do not establish equivalent success rates for Townscaper, Blender, or all real tasks.
|
||||
|
||||
Targeted process-identity regression runs passed all 15 rounds: 45 disposable Apps, 180 complete drags, and 1,095 samples from actual validation calls. Identity fields and launch-time floating-point bits remained unchanged within each process. Two earlier anomalies—a `stale_process` rejection and a timeout without phase records—did not recur and remain unexplained. Identity comparisons were not relaxed, and no tolerance or automatic retry was added. `ProcessValidationObservationTests` and the native integration tests preserve these diagnostic boundaries.
|
||||
|
||||
The compatibility scope is native Computer Use. Independent browser Tab and DOM providers and general Node capabilities are outside this scope. Tests of the real worker child process use temporary directories and simulated native tools to establish isolation, persistence, and output boundaries; native receiver tests establish event delivery. Neither replaces success-rate evaluation on real model tasks or establishes behavioral parity across all real Apps.
|
||||
|
||||
AX rendering, element relocation, focus, keyboard input, window coordinates, capture, and cursor animation each have their own contracts. This page does not prescribe speculative algorithms or third-party constants for unverified areas. Further compatibility work should establish evidence along the actual call path, then encode confirmed behavior in the corresponding source and regression tests.
|
||||
@@ -1,141 +1,146 @@
|
||||
<!-- LOCAL build spec. Distilled from 3 subagents deep-reading open-source Codex CU
|
||||
reverse-implementations (iFurySt/open-codex-computer-use [no license → reference
|
||||
only], OpenCodexLabs/open-codex-computer-use [MIT], vtomnet/codex-cua-tea [docs +
|
||||
OpenAI's verbatim AppInstructions → reference only]). 2026-06-09.
|
||||
IP rule: implement our OWN code to this spec; do NOT copy their Swift verbatim. -->
|
||||
|
||||
# Computer Use — Codex 实现蓝图(对照 3 个开源逆向实现)
|
||||
|
||||
3 个开源逆向实现 **一致印证了我们的方向**(AX 树感知 + AX 优先注入 + index 一等公民 + Electron 强开 AX),并给出了精确格式 + 算法。我刚写的 v1(AXTree.swift / AXAction.swift)**方向对、但有具体缺口**:axText 格式不对、遍历不全(Electron 会变噪声)、注入不安全。本文是把 v1 改对的权威规格。
|
||||
|
||||
参考源(只读参考,不抄码):
|
||||
- `/tmp/iFurySt-cu/.../OpenComputerUseKit/AccessibilitySnapshot.swift`(渲染器=格式权威)、`ComputerUseService.swift`(注入梯度)、`KeyMapping.swift`、`ToolDefinitions.swift`;`docs/references/codex-computer-use-reverse-engineering/{baseline-architecture,state-rendering-1.0.770,tool-call-samples-2026-04-17}.md`(Codex 真实输出样本)。
|
||||
- `/tmp/opencodexlabs-cu/Sources/ClaudexComputerUseCore/{AppState,UIElementService,WindowCapture,CodexCompat,AppGuidance}.swift` + `.../ClaudexComputerUseMCP/main.swift`(MCP 框装 + 变更后自动重拍)。
|
||||
- `/tmp/codex-cua-tea/{SkyComputerUseService.md, AppInstructions/*.md}`。
|
||||
|
||||
---
|
||||
title: Codex 原生 Computer Use 兼容契约
|
||||
nav_title: Codex 兼容契约
|
||||
description: 官方插件、原生事件、动作时序与批量观察的已验证契约，以及当前实现的兼容边界。
|
||||
order: 15
|
||||
---
|
||||
|
||||
## 1. 【最高优先】skyshot/get_app_state 文本格式(v1 render 必须重写)
|
||||
# Codex 原生 Computer Use 兼容契约
|
||||
|
||||
Codex 真实格式(iFurySt 抓的样本,高置信):
|
||||
macOS 原生 Computer Use 以官方 Codex 安装包中的插件、JavaScript 客户端和原生服务为兼容基准。本页说明已经验证的调用与事件契约，帮助源码读者区分底层输入、观察和模型调度。第三方复刻不再作为官方行为的规格来源；接口相似或单个任务成功，也不能证明完整兼容。
|
||||
|
||||
## 参考版本与分层
|
||||
|
||||
当前契约来自 2026 年 9 月 9 日检查的官方构建：
|
||||
|
||||
| 组件 | 版本或位置 |
|
||||
| --- | --- |
|
||||
| Computer Use 插件 | `openai-bundled/unified-computer-use/26.901.51231` |
|
||||
| 原生服务 | `SkyComputerUseService`，版本 `26.831.1000926` |
|
||||
| 原生服务 SHA-256 | `25e9141499b94c396f39afbdb7b19ed8f49e45dc8c61be61028ceab8f3807ce6` |
|
||||
| 插件缓存 | `${CODEX_HOME}/plugins/cache/openai-bundled/unified-computer-use/<version>/` |
|
||||
| App 内 JavaScript 包 | `Contents/Resources/cua_node/lib/node_modules/@oai/cua` 和 `@oai/sky` |
|
||||
| 本地原生服务 | `${CODEX_HOME}/computer-use/Codex Computer Use.app/Contents/MacOS/SkyComputerUseService` |
|
||||
|
||||
这些版本是契约的适用范围，升级后需要重新核对调用方与实现方。原生函数地址只适用于该二进制，细节记录在源码目录的 `native/cu-helper/README.md`。
|
||||
|
||||
原生 App 操作经过以下链路：
|
||||
|
||||
```text
|
||||
持久 JavaScript 会话中的 cua App 对象
|
||||
→ @oai/cua / @oai/sky 客户端
|
||||
→ 本地持久 IPC
|
||||
→ SkyComputerUseService
|
||||
→ AX 查询或面向目标进程、窗口的合成事件
|
||||
```
|
||||
App=com.apple.finder (pid 1106)
|
||||
Window: "open-codex-computer-use", App: Finder.
|
||||
0 standard window open-codex-computer-use, ID: FinderWindow, Secondary Actions: Raise
|
||||
1 split group
|
||||
2 scroll area
|
||||
3 outline sidebar
|
||||
4 row (selectable, expanded) Value: Favorites, Secondary Actions: Collapse
|
||||
|
||||
The focused UI element is 2 outline.
|
||||
```
|
||||
规则(每条都是我 v1 的偏差):
|
||||
- **头两行**:`App=<bundleId 否则 name> (pid N)` + `Window: "<title 空则 appName>", App: <appName>.`(带句点)。**删掉**我的 `Elements: N` 行和空行。
|
||||
- **index 无方括号**:`0 standard window`,不是 `[0] AXWindow`。格式 `"\(index) \(roleText)"`。
|
||||
- **role 用人性化 `kAXRoleDescription`**(`standard window`/`split group`/`scroll area`/`outline`/`row`/`button`/`radio button`/`pop up button`/`text`…),**不是**原始 `AXRole`。特例:`AXRow`→`row`、`AXGroup`→`container`、`AXLink`→`link`、`AXWebArea`→其 roleDescription、menu-bar-item→`""`、static text→`text`。**这是最大单点偏差。**
|
||||
- **缩进 = 每层一个 `\t`**,根窗口在 depth 0(零缩进)。我用的两空格要换成 `\t`。
|
||||
- **traits 括号逗号列表**,紧跟 role:`(selectable, expanded)`、`(selected)`、`(settable, string)`、`(settable, float)`、`(disabled)`。词表:selected/expanded/disabled/settable + 值类型(string/float/boolean,仅当 kAXValue settable 时附加)。**删掉我的独立 `(focused)`**。
|
||||
- **字段顺序**(role/traits/title 之后):`title`(裸,不加引号)→ ` Description: <kAXDescription>` → ` Help: <kAXHelp>`(我没读,要加)→ `, URL: <kAXURL>`(WebArea)→ ` ID: <kAXIdentifier>`(**`_NS:` 开头的要滤掉**)→ value(通常 ` Value: <v>`,但 static text/scroll bar/value indicator/text area/search field 用裸 ` <v>`)→ ` Placeholder: <v>`(我没读)→ ` Secondary Actions: <pretty,逗号>`。多个段之间用 `, ` 连。
|
||||
- **不在文本里打 frame**(`{x,y wxh}` 删掉)——frame 只内部存给注入用,窗口相关坐标。
|
||||
- **Secondary Actions 用 pretty 名 + 过滤**:去 `AX` 前缀 + 拆驼峰,`AXRaise`→`Raise`、`AXScrollUpByPage`→`Scroll Up`;**denylist 隐藏**:AXPress/AXShowDefaultUI/AXShowAlternateUI/AXShowMenu/AXConfirm/AXScrollToVisible(菜单再加 AXCancel/AXPick)。原始 action 列表内部保留给注入用。
|
||||
- **value 数值/布尔**:checkbox/radio/tab 的 0/1 渲染成 `on`/`off`;slider/scrollbar 渲染原始 float。
|
||||
- **focus 作为尾行**(不是每元素):树后空行 + `The focused UI element is <该节点的 index role (traits) title>.`;若有选中文本则 `Selected text: [<text>]`。focus 元素从 `kAXFocusedUIElement` 读。
|
||||
- **sanitize**:换行→字面 `\n`,trim,**截断 160 字符 + `...`**(我现在 200 + `…`,改)。
|
||||
浏览器 Tab provider 是另一条链路。安装了浏览器扩展，不代表每次操作 Chrome 都在使用 DOM：`cua.getApp(...)` 获得的原生 App 对象可以直接发送坐标点击和拖拽。判断执行方式应看实际调用的对象、方法与传输路径。
|
||||
|
||||
> 实现策略:**自己实现** TreeRenderer(对照 iFurySt:595-1788 的字段/变换),不抄文件。手按上面列表写会漂,务必拿 `tool-call-samples-2026-04-17.md` 的真实样本做断言测试。
|
||||
macOS 的原生 App 路径允许控制 Chrome 等浏览器，同时保留其他应用限制、授权、签名和进程身份校验。此前 TypeScript 分发与原生 `AppTargetPolicy` 都会因浏览器类别拒绝目标，阻断这条合法的原生路径；两层现在采用一致的浏览器原生控制策略。Windows 原有浏览器类别与权限等级不变。允许浏览器作为原生 App，不会提供 Tab 绑定、DOM 节点或 Playwright 方法。
|
||||
|
||||
---
|
||||
Townscaper 的指定成功轨迹说明了这个区别：21 次 JavaScript 调用中，先有一次浏览器清单读取和一次超时的 `getTab`，随后绑定 Chrome 原生 App。全部 57 次输入都来自该 App：47 次拖拽、3 次点击、3 次滚动、3 次按键和 1 次粘贴，返回 18 张截图。6 个含循环的调用共执行 41 次拖拽，其中循环体执行 39 次，另外 2 次是循环外的调色板操作；没有成功的 Tab、DOM 或 Playwright 建造调用。这个案例的连续建造能力来自原生 App 路径；独立浏览器 provider 不属于本次兼容范围。
|
||||
|
||||
## 2. 遍历(v1 walk 缺口大,Electron 会变噪声)
|
||||
公开的 Codex CLI 和 App Server 源码包含 MCP 集成、工具调度及消息处理，但不包含这份原生服务的鼠标事件实现。官方安装包提供可读的 JavaScript 客户端；原生契约还需要结合导出符号、实际调用参数和机器码核验。不能因为静态导入表缺少某个 CGEvent 符号，就断言它只使用 AX：服务会通过动态解析后的函数指针发送事件。
|
||||
|
||||
- **子节点不止 kAXChildren**:并 `kAXChildren + kAXRows + AXContents + AXVisibleChildren`,按角色选主源(outline/list/table/AXBrowser 用 AXRows),CFEqual 去重,跳过菜单栏下的 Apple 菜单。**否则 Finder/系统设置/活动监视器的行全丢。**
|
||||
- **环路守卫**:传 ancestors 集合,CFEqual 命中祖先则跳过(Electron 树有环,否则重复子树)。
|
||||
- **【关键】泛容器消除 + 扁平化**:剪掉无描述的 AXGroup/AXUnknown 包装(同深递归进子)、单子无意义组折叠、纯文本兄弟合并成一个 ` text …`、链接渲染成 Markdown 的文本加 URL 形式并吞子。**没有这步,Electron 的 AXWebArea 是几千个空 wrapper,在到达有用控件前就撑爆 cap——这正是"Electron 看起来读不到树"的真因。**
|
||||
- **菜单栏第二趟**:走完窗口后 `walk(copyElement(app, kAXMenuBar))` 追加(否则不能按 index 点菜单)。
|
||||
- **行可见性窗口化**:outline/list 只 emit 可见行(与父框相交),cap 20,容器加 ` (showing 0-N of M items)` 摘要。
|
||||
- **window-relative frame**:`localFrame = elementFrame - windowBounds.origin`,内部存。
|
||||
- **caps**:深度降到 ~16-20(我现在 80 危险),emitted ~1200-1500;但 cap 只有在加了消除后才行为正确。
|
||||
- **窗口解析更稳**:focused→main→first 之上加 not-minimized + kAXWindowRole 校验 + 隐藏 Electron 窗的 unhide/raise/un-minimize 恢复。
|
||||
## 应用发现与原生接口
|
||||
|
||||
---
|
||||
原生应用限制按已解析的 bundle ID 精确匹配官方 24 项禁止目标，另保留本产品宿主和 helper 的固有限制。此前第三方表中的 IDE、音乐、交易分类及显示名子串不再参与 macOS 原生判断。发现列表仍可包含禁止目标，真正绑定或动作时才执行授权检查。
|
||||
|
||||
## 3. index 的可重解析定位(opencodexlabs 的最大稳健性优势)
|
||||
`cua.listApps()` 合并正在运行的普通 App 与 Spotlight 中最近 14 天使用过的 App，保留 `id`、`displayName`、`isRunning` 及可选的 `lastUsedDate`、`useCount`。`id` 来自 bundle ID，不等于绑定后的规范路径；结构化数据跨 helper、分发器及 worker 原样传递。旧 helper 仅返回运行列表时仍兼容。低层原生窗口 API 也可通过 `cua.computer` 的 11 个 snake_case 方法访问，同样经过授权和进程校验。
|
||||
|
||||
每个元素除 flat `index` 外,存 `(windowIndex, path)`——从窗口根到该节点的子索引链。动作时 `resolveElement` **从 app 元素现场重走**:`windows[windowIndex]` → 逐级 `children[path[i]]`。这样 index 在一次"新 AX 查询"后仍有效(只要窗口拓扑没变),不依赖缓存的 AXUIElement 指针存活。menu item 用 directRef。
|
||||
> 改 AXTree.Element 加 `windowIndex/path`,walk 累积 childPath;AXAction 先 `cachedElement(index)` 快路,失败再 `resolve(index)` 重走。
|
||||
## 坐标拖拽的五事件契约
|
||||
|
||||
---
|
||||
官方 `app.drag()` 经原生坐标控制器，将起点和可选拖拽终点传给 `ApplicationUIElement.sendClick`。底层 `SynthesizedEvent.click` 生成以下事件，窗口绑定在整个手势内保持一致：
|
||||
|
||||
## 4. 注入梯度(AXAction v1 太薄 + 有危险)
|
||||
| 顺序 | 事件 | 位置 | clickCount | eventNumber |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 1 | mouseDown | 起点 | 1 | 手势编号 |
|
||||
| 2 | mouseDragged | 起点 | 0 | 移动编号 |
|
||||
| 3 | mouseDragged | 起终点的中点 | 0 | 同一移动编号 |
|
||||
| 4 | mouseDragged | 终点 | 0 | 同一移动编号 |
|
||||
| 5 | mouseUp | 终点 | 1 | 同一手势编号 |
|
||||
|
||||
- **click 不是单 AXPress**,顺序梯度:① 原生 list 行→对父 AXList 设 `AXSelectedChildren`(**选中,不是 press**——Finder 侧栏/活动监视器/系统设置的行这样"点");② AXPress→AXConfirm→AXOpen(**我的 press() 选"第一个 action"是危险的——可能把 AXShowMenu 当点击;限制到 {AXPress,AXConfirm,AXOpen}**);③ 向下 3 层找暴露 Press/Confirm/Open/ShowMenu 的后代;④ 元素中心 AX hit-test;⑤ 仅 window-role 元素的 activation-only(AXRaise/kAXMain/kAXFocused);⑥ 才回退合成 `CGEvent.postToPid`。**右键=AXShowMenu;clickCount>1=重复 N 次。**
|
||||
- **set_value**:先 `AXUIElementIsAttributeSettable(kAXValue)` 门控,**不强行 focus**,返回 before/after 值;不可设则报 `"Cannot set a value for an element that is not settable"`。(改我 v1 的强 focus + 盲写。)
|
||||
- **type_text(新增)**:先试把 focused 元素的 kAXValue 设成 当前值+text(**追加**);失败且 focused 是 text field/area 才回退 Unicode 键盘(keyboardSetUnicodeString,≤64 UTF16 分块)。
|
||||
- **press_key(新增)**:实现 xdotool 词表(`super+c`/`Return`/`Tab`/`KP_0`/`Prior`/`Next`/`F1-12`),super/cmd/command/meta→Command,postToPid 带修饰键 down/up 括号。
|
||||
- **scroll(新增)**:元素域,优先 `AXScroll{Up,Down,Left,Right}ByPage` 重复 floor(pages) 次,否则元素中心 postToPid 滚轮(12 行/页)。
|
||||
- **perform_secondary_action**:**模型看到的是 pretty 名**(Raise/Scroll Up),要 pretty→raw 翻译再 AXUIElementPerformAction(我 v1 收原始名,匹配不上=bug)。无效报 `"<action> is not a valid secondary action for <index>"`。
|
||||
- **drag**:坐标-only,postToPid down→10 段 dragged→up。
|
||||
- **所有合成事件用 `CGEvent.postToPid(pid)` + source `.combinedSessionState`,绝不用 `.cghidEventTap`/`.hidSystemState`**——这才是"不抢真鼠标"。
|
||||
起点等于终点时仍保留五个事件，不能折叠成普通点击。按下后的起点 dragged 事件也不能省略。普通点击的事件对与拖拽的移动事件具有不同语义，不能用任意插值步数替代这个契约。
|
||||
|
||||
---
|
||||
原生服务通过 AppKit 构造鼠标事件，取得 CGEvent 后写入目标窗口信息、全局和窗口内位置，再发送到目标 PID。这既不是单纯调用 AXPress，也不是移动用户的真实鼠标后在前台点击。坐标转换、窗口身份和进程生命周期必须在同一条链路上成立，不能只用相似窗口标题绑定截图与输入。
|
||||
|
||||
## 5. 截图(给 get_app_state 用)
|
||||
对应实现集中在 `native/cu-helper/Sources/cu-helper/AXAction.swift`、`WindowTargetedEvent.swift` 和 `WindowGeometry.swift`。`MouseDragEventTests` 使用生产事件工厂验证事件顺序、坐标、编号和窗口绑定。
|
||||
|
||||
- 锁**目标 App 的单个关键窗口**(rank 选:on-screen+1M/active+2M/else area),`SCContentFilter(desktopIndependentWindow:)`,**scale 0.5**,showsCursor=false。坐标 = 截图像素空间(左上原点),server 端做 scale/offset→全局仿射。
|
||||
- **SCK 卡死兜底**:detached Task + `DispatchSemaphore.wait(2.5s)`,超时取消→回退 `/usr/sbin/screencapture -l <windowID> -x -o`(3s + terminate→SIGKILL 升级)。
|
||||
- **get_app_state 的截图包 `try?`**——截图失败也要返回 AX 文本。(单 main-actor daemon 里 SCK 卡死会 wedge 整个 run loop。)
|
||||
## 键盘输入
|
||||
|
||||
---
|
||||
原生接收窗口验证了 `Control_L/R`、`Super_L/R`、`Meta_L/R`、`Shift_L/R` 和 `Alt_L/R` 的实际修饰键行为。键名 `Delete` 对应向前删除的 keyCode 117，`BackSpace` 对应 51；大写字母和 `question` 等具名符号必须保留 Shift。解析器与系统组合键授权检查同时识别这些别名。
|
||||
|
||||
## 6. staleness(3 守卫,无 hash)+ 变更后自动重拍
|
||||
## 滚动页数与目标区域
|
||||
|
||||
- ① 该 pid 没拍过 → 文本 `"The user changed '<app>'. Re-query the latest state with get_app_state before sending more actions."`;② index 越界 → `isError` `"Element index N not found in snapshot (has M elements)."`;③ 动作后 NSRunningApplication(pid)==nil → 作废 session。
|
||||
- **每个变更工具的返回 = 重跑 get_app_state(树+窗口截图)**(codexMutationResponse,main.swift:1517-1580)——模型几乎不用手动在动作间 get_app_state,index 几乎永远只老一步。这是让粗粒度 staleness 可用 + agent 循环高效的关键行为。
|
||||
- daemon 持有 **per-pid AppStateSession**(快照+元素 refs+paths),跨 socket 请求存活。
|
||||
官方原生 `scroll` 发送精确像素滚轮事件，保留小数页数。通过元素索引滚动时，垂直页数乘以该元素的外框高度；通过坐标滚动时，乘以当前窗口高度，不改用坐标下的滚动区。实测同一 210 点高滚动区，元素方式滚动 0.5、1.5 页分别收到 -105、-315；在 552 点高窗口中，坐标方式滚动 0.5 页收到 -276，向上则为 +276。不能将这些请求舍入为整页 AX 动作，或固定换成 12 行滚轮事件。显式 `performSecondaryAction` 的 Scroll Down 等动作仍采用该控件公开的 AX 页操作语义。
|
||||
|
||||
---
|
||||
当前官方版本的水平调用传入单轴滚轮事件，实测横向增量被忽略。本实现保留双轴水平滚动能力；这是明确记录的行为差异，不复制该无效操作。
|
||||
|
||||
## 7. MCP 工具面(逐字 Codex 名)+ get_app_state 框装
|
||||
滚轮事件同时设置窗口字段 51、91、92 和窗口内坐标。只有 91、92 不能建立 AppKit 的 `windowNumber`，会造成事件构造成功但目标收不到。实际接收端验证了字段 51 修复后的精确像素滚动。非整数距离按最近整数取整，例如 210 × 0.123 点产生 26 像素增量。
|
||||
|
||||
工具(9 个原生名):`list_apps, get_app_state, click, perform_secondary_action, set_value, scroll, drag, press_key, type_text`(`select_text` 不是独立原生工具)。
|
||||
- `click` 接 `element_index?`(**string 或 int**,Codex 发 string)**或** `x,y`;`click_count?`、`mouse_button?`(1=left/2=middle/3=right/4=back/5=forward)。
|
||||
- **get_app_state 响应在 MCP server 框装**(不在 Swift):content = `[{type:text, text:envelope}, {type:image, data:base64 PNG, mimeType:image/png}]`(截图成功才附图)。envelope:
|
||||
```
|
||||
Computer Use state (CUA App Version: <v>)
|
||||
<app_specific_instructions>…</app_specific_instructions> // 该 app 首次 session 才投,非空才投
|
||||
<app_state>
|
||||
App=… (pid …)
|
||||
Window: …
|
||||
\t0 …
|
||||
</app_state>
|
||||
The focused UI element is …
|
||||
```
|
||||
另带 structuredContent(完整类型化结果)给非-Codex harness。
|
||||
- 动作回执:`Action completed. Call get_app_state to fetch the updated UI state.`
|
||||
- 遥测:`time_to_first_get_app_state`/`time_from_first_get_app_state_to_first_write`/`time_to_first_write`。
|
||||
`performSecondaryAction` 的四个翻页方向优先读取 `AXVerticalScrollBar` 或 `AXHorizontalScrollBar`，从滚条的 `AXChildren` 中找到首个角色为 `AXButton`、子角色为 `AXDecrementPage`（上/左）或 `AXIncrementPage`（下/右）的节点，并执行 `AXPress`。缺少滚条或按钮时才尝试目标的原始 `AXScroll*ByPage` 动作；按钮执行失败则传播错误，不重发。`AXIncrementPage` 和 `AXDecrementPage` 是子角色值，不是可查询的属性名。页距由控件决定，不写死像素数。
|
||||
|
||||
---
|
||||
## 粘贴消费确认与时序
|
||||
|
||||
## 8. 每-App 指令 + frameReliability(高价值低成本)
|
||||
原生粘贴的两秒是读取超时上限，不是每次必须等待的时长。写入临时内容之前，先捕获目标进程当前输入框支持的 `AXSelectedTextRange`、`AXNumberOfCharacters`，并监听该元素的选区和值变化。内容写入后才启用通知计数；数据提供回调成功供给字节后，第一阶段即可结束。
|
||||
|
||||
- `<app_specific_instructions>` = 按 bundleId 加载 `AppInstructions/<App>.md` 注入。Codex 自带 Clock/Spotify/Notion/Numbers/AppleMusic 等(OpenAI 版权内容,**只参考格式、自己写**)。格式:H2 标题 + H3 主题 + `<Key>` 尖括号键名 + 友好元素 id + 鼓励并行调用。典型内容=app 特定怪癖(选择模型、异步状态、占位符文本、哪些按钮、别做什么)。
|
||||
- **frameReliability 分类器**(bundleId→browser/finder/jetbrains/electron/native→high/medium/low):low(Electron/微信/飞书)→ 不信 frame 坐标、优先 pasteboard 输入 + delivery:direct、虚拟光标锚窗口中心。兼容矩阵:原生 AppKit=可靠,Electron=需变通,自绘(微信~13 元素)=尽力。
|
||||
第二阶段最多等两秒，每 25 毫秒检查目标。只有读取后发生的目标通知或有效属性变化才支持提前返回，不能把其他剪贴板观察者的读取当成目标消费确认，也不能把 AX 读取失败当成值变化。完全没有可观察信号时，读取后保留 100 毫秒窗口。有信号但一直没有变化时会报目标确认超时；已执行的粘贴不重放。取消后仍完成当前有界消费窗口再恢复，外部复制始终优先，只有仍持有临时内容时才恢复旧剪贴板。
|
||||
|
||||
---
|
||||
接收端确定性回归使用独立 named pasteboard，保留真实 Router、Command-V、菜单、数据提供、目标 AX 确认和恢复。它隔离共享数据源，不等于正式 general 剪贴板链的全环境验收。
|
||||
|
||||
## 9. 外围(真全复刻,后置)
|
||||
## 截图尺寸与格式
|
||||
|
||||
- **per-app 授权 elicitation**(session vs persistent,prompt-injection 警告,JSON 存),turn-scoped session 按 thread id,turn 末作废。
|
||||
- 浏览器走独立 Browsing MCP(对齐我们既有 claude-in-chrome),不在原生 CU 里。
|
||||
- Skysight 被动记忆 / Record&Replay / LockScreenGuardian:独立子系统,阶段2/后置。
|
||||
官方原生状态截图默认归一到逻辑点尺寸，再限制长边为 2048、短边为 768，不放大小图，最终像素尺寸向上取整，JPEG 编码质量为 0.8。1398 × 769 点的窗口因此返回 1397 × 768 像素；这是缩放结果，不是裁掉窗口边缘。坐标转换保留完整窗口 frame，并使用取整前的统一 `pixelsPerPoint` 比例；不能按取整后的横纵尺寸各算一个比例，也不能修改窗口边界。旧快照未提供统一比例时保留原有转换方式。
|
||||
|
||||
---
|
||||
原生结果的 `mimeType` 随图片跨分发与 worker 传递。旧 helper 未提供该字段时仍按 PNG 处理。官方包装器曾把真实 JPEG 字节标记为 PNG；本实现保留正确的 JPEG MIME 标签，避免模型请求端误解内容。
|
||||
|
||||
## 我 v1 要改的清单(按优先级)
|
||||
1. **AXTree.render 重写成 §1 格式**(最高杠杆)。
|
||||
2. **§2 遍历**:rows/contents/visibleChildren + 环路守卫 + 泛容器消除 + 菜单栏 + 行窗口化(Electron 能用的前提)。
|
||||
3. **§3 (windowIndex,path) 定位** + §6 staleness/session。
|
||||
4. **§4 注入梯度**:限制 press 到 {Press,Confirm,Open}、list 选中、set_value 门控、perform pretty→raw、新增 type_text/press_key/scroll。
|
||||
5. **§5 窗口锁定截图 + 卡死兜底**(P1-C)。
|
||||
6. **§7 工具面 + envelope 框装**(P1-D)。
|
||||
7. **§8 per-app 指令 + frameReliability**、**§9 外围**(后置)。
|
||||
## 动作时序与可见光标
|
||||
|
||||
已检查的坐标点击、拖拽调用明确传入 `delay: nil`。直接发送和虚拟光标发送两条分支都跳过可选 sleep，因此没有每事件固定 30 ms 的等待，也没有固定 100 ms 的按住时间。服务中的 `humanClickInterval` 常量用于其他明确的点击或 press 路径，不能套用到坐标拖拽。
|
||||
|
||||
虚拟光标分支同步更新按下状态并发送事件。坐标点击、拖拽路径没有在发送前等待光标移动动画完成。专用 `moveMouse` 操作具有自己的动画和后续交互时机，属于不同调用路径。光标需要让用户看见操作，但不应额外给每次坐标手势附加固定的可见等待。
|
||||
|
||||
机器码中存在 Swift executor 切换，这不保证每次都会暂停或产生确定的事件间隔。因此不能为了模拟它，另加猜测的 sleep 或 `Task.yield`。本项目取消的是已确定多余的固定等待，仍保留事件发送前的目标校验、取消检查和异常后的抬键清理。
|
||||
|
||||
焦点准备和纯视觉等待也要分开。`SyntheticWindowFocus` 保留进程生命周期、焦点变化和输入确认检查；已建立的焦点状态可以复用，真正需要建立或恢复焦点时才等待确认。删除所有焦点检查并不构成兼容性优化。
|
||||
|
||||
## 批量动作与观察
|
||||
|
||||
官方原生 App API 可以在持久 JavaScript 会话里保留 App 绑定、计算坐标并循环调用动作。一个工具调用可以连续执行多个已确定动作，最后才读取 AX 状态或截图。循环中的 `await app.drag()` 等待本地动作完成，不要求模型在每个动作之间重新生成一次回答。
|
||||
|
||||
原生坐标控制器仅在 `returnSkyshot: true` 时进入 UI settle 和截图分支；false 标记状态需要刷新后直接返回。已核验的 settle 调用包含 250 ms 的通知延迟参数，但这不是每次动作无条件 sleep 250 ms，更不是完整截图耗时的保证。不能把“每次变更后自动重截图”当成官方的固定行为。
|
||||
|
||||
macOS 向模型公布的工具只有 `js` 和 `js_reset`，减少重复的逐动作 schema。`js` 提供持久变量、App 绑定、顶层 `await`、循环和同次调用内的计算与观察。worker 内的 `cua` 原生 App 方法通过 JSON 消息回到主进程，再进入已有语义工具的权限、进程身份和窗口校验；脚本中的 App 对象不能绕过这些检查。`cua.getApp()` 初次观察显示 AX 文本，首次可见的 App 选择或状态清单同时显示简短 API 说明。App 绑定使用主进程返回的已解析路径，后续动作仍重新校验目标。
|
||||
|
||||
`app.getAXState()`、`app.getScreenshot()` 和 `app.getAXStateAndScreenshot()` 分别返回、展示文本、图片或两者；`emit: false` 保留返回值并抑制展示。已核验的官方 JavaScript 实现也将三个方法都转为 `get_app_state`，因此这种输出分离不代表跳过 AX 遍历或截图。动作本身不自动观察，脚本在需要决定下一步时显式请求状态。
|
||||
|
||||
元素的底层句柄仍是 `gN:id`。兼容层只把实际返回过的 AX 行映射为整数索引，处理差异中的新增、变更和删除，并在世代改变时清空旧映射。只取图片的请求也可能刷新原生状态，所以会清空整数映射；再次使用整数前应调用 `app.getAXState({disableDiffing: true})`。直接使用复制的句柄仍须通过原生校验。
|
||||
|
||||
| 每个 `js` 调用的限制 | 数值 |
|
||||
| --- | --- |
|
||||
| 原生调用 | 最多 256 次，包含观察 |
|
||||
| 代码 | 最多 256 KiB |
|
||||
| 展示输出 | 最多 128 块、合计 16 MiB |
|
||||
| 超时 | 默认 30 秒，最多 60 秒 |
|
||||
|
||||
跨 cell 的变量通过共享词法访问器连接，旧函数与新脚本读取同一绑定。重新选择 App 后，此前定义的函数会使用新 App；先定义函数、下一 cell 再声明其引用的 App 也可用。局部参数、块作用域、解构及类内部绑定保持自身语义，不能为了持久化而复制每次调用的变量值。
|
||||
|
||||
普通脚本错误保留可恢复的既有绑定和已经执行的声明或直接写入；失败脚本中尚未执行的 `var/function` 声明不会仅因变量提升而留下额外绑定。重绑 App 的初始化失败时，旧 App 绑定仍可用。`js_reset`、取消或超时会丢弃 worker 及其绑定。已经执行的动作不回滚，重新绑定后必须先观察，再判断部分完成的工作是否需要继续。当前 worker 不开放 imports、Node、文件系统或网络接口。执行和隔离边界位于 `src/utils/computerUse/replRuntime.ts`、`replWorker.ts` 与 `replCompiler.ts`；原生方法及观察输出适配位于 `src/vendor/computer-use-mcp/replApi.ts`。
|
||||
|
||||
有确切对应关系的原生错误保留 `SkyComputerUseError` 的 `name`、`code`、`errorName` 和请求字段；不能确定官方分类的 helper 错误仅保留原始 `nativeCode`，不从消息猜测错误码。权限或参数检查在派发前拒绝时，单独计入 `nativeCallsRejectedBeforeDispatch`，不会报成已经完成的动作或未知执行结果。超时及可能发生部分副作用的错误仍保留结果未知语义，不自动重放。
|
||||
|
||||
桌面使用合并 sidecar 可执行文件。它必须在解析普通运行模式、`app-root` 或加载 `preload` 之前识别内部 worker 参数，直接启动隔离 kernel。只在 CLI 子入口处理该参数会让实际桌面程序提前报错；源码 worker 或手写编译入口测试无法覆盖这个边界。worker 的 HOME 和临时目录均指向可丢弃目录，沙箱内部重新指定临时目录变量，避免包装库覆盖它们。
|
||||
|
||||
旧的 `sequence` 保留为同一 App 的 JSON 批量兼容入口，顺序执行后观察一次，并在失败或取消时报告已完成步骤。它最多接受 256 步，使用协作式 60 秒截止时间；正在执行的原生命令必须结束后才能返回。独立语义工具也保留直接调用兼容，但这些接口不再默认展示给模型。Windows 继续使用原有像素工具，没有新增这组 JavaScript 接口。
|
||||
|
||||
是否适合批量执行取决于界面是否稳定。已经验证有效的画布动作可以连续提交；打开菜单、切换窗口或其他会改变后续目标的动作，应在新的决策点重新观察。衡量速度时应分别记录动作数、观察次数、模型往返、工具耗时和目标 App 实际完成的结果。
|
||||
|
||||
## 当前兼容边界
|
||||
|
||||
五事件工厂测试证明事件构造；`AXTreePublicationIntegrationTests` 在一次观察之后，向临时原生 App 连续发送十二次零距离或短距离拖拽，通过接收端计数验证完整手势交付。这类测试验证底层输入，不等于 Townscaper、Blender 或所有真实任务已经达到同样成功率。
|
||||
|
||||
针对进程身份连续性的定向回归完成了 15 轮：45 个临时 App、180 次完整拖拽和 1,095 次真实校验取样均通过。同一进程的身份字段及启动时间浮点位保持一致。曾出现的两次异常——`stale_process` 和缺少阶段记录的超时——没有重现，原因仍未定位；没有因此放宽身份比较、加入容差或自动重试。`ProcessValidationObservationTests` 与原生集成测试保留这类诊断边界。
|
||||
|
||||
本兼容范围仅包含原生 Computer Use。独立浏览器 Tab、DOM provider 与通用 Node 能力不在范围内。worker 的真实子进程测试使用临时目录和模拟原生工具，证明隔离、持久化及输出边界；原生接收端测试证明事件交付。两者都不能替代真实模型任务的成功率评估，也不能推出所有真实 App 的行为已经一致。
|
||||
|
||||
AX 树格式、元素重定位、焦点、键盘输入、窗口坐标、截图和光标动画各有独立契约。本页没有为尚未核验的部分指定推测算法或第三方常量。后续兼容工作应沿真实调用链补齐证据，再将确定行为落实到相应源码与回归测试。
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# Native Computer Use compatibility
|
||||
|
||||
The compatibility reference is the installed **official** Codex Computer Use
|
||||
plugin, its shipped JavaScript client, and its native service. Third-party
|
||||
replicas are historical references, not specifications for current Codex.
|
||||
`docs/internals/computer-use-codex-impl-blueprint.md` describes the verified
|
||||
contracts and the boundaries of our implementation.
|
||||
|
||||
## Reference inspected on 2026-09-09
|
||||
|
||||
- Plugin: `openai-bundled/unified-computer-use/26.901.51231`.
|
||||
- Native service: `SkyComputerUseService` **26.831.1000926**, SHA-256
|
||||
`25e9141499b94c396f39afbdb7b19ed8f49e45dc8c61be61028ceab8f3807ce6`.
|
||||
- The app ships `@oai/cua` and `@oai/sky` under
|
||||
`Contents/Resources/cua_node/lib/node_modules/`. Its native client sends
|
||||
length-prefixed JSON-RPC over a persistent local pipe. The browser provider is
|
||||
separate from the native app provider.
|
||||
- The public Codex CLI/App Server repository supplies MCP integration, not the
|
||||
native service's mouse-event implementation. Missing static CGEvent imports
|
||||
do not prove an AX-only implementation: the service also resolves functions
|
||||
indirectly.
|
||||
|
||||
These observations apply to this build. Recheck the actual caller and callee
|
||||
when changing versions; a nearby symbol or a third-party constant is insufficient
|
||||
evidence for a timing or event contract.
|
||||
|
||||
## Native browser Apps and the separate browser provider
|
||||
|
||||
The native App route permits Chrome and other browsers on macOS. The earlier
|
||||
TypeScript and `AppTargetPolicy` browser-category rejection blocked this route
|
||||
despite it being the route used by the official native App client. Both dispatch
|
||||
layers now permit native browser targets while preserving other denials,
|
||||
authorization, signatures, and process/window-identity checks. Windows browser
|
||||
categories and tiers remain unchanged. This permission is for native App
|
||||
control; it does not implement the browser service's Tab, DOM, or Playwright API.
|
||||
|
||||
In the user-designated Townscaper success trace, 21 JS calls include one browser
|
||||
inventory and a `getTab` attempt that times out after 30.0355 seconds. The next
|
||||
binding is `cua.getApp("com.google.Chrome")`. All 57 later input actions are
|
||||
native: 47 drags, 3 clicks, 3 scrolls, 3 key presses, and 1 paste. The trace
|
||||
returns 18 screenshot blocks. Six cells containing loops perform 41 drags;
|
||||
39 occur inside loop bodies and two are palette selections outside them.
|
||||
Those six cells take 19.9703 seconds including their final observations. There
|
||||
are no successful Tab/DOM/Playwright construction calls. Browser provider setup
|
||||
is outside this native compatibility scope.
|
||||
|
||||
## Coordinate drag contract
|
||||
|
||||
The native coordinate click controller calls `ApplicationUIElement.sendClick`
|
||||
with an optional drag destination and `delay: nil`. The specialized
|
||||
`SynthesizedEvent.click` implementation emits:
|
||||
|
||||
| Event | Location | Click count | Event number |
|
||||
| --- | --- | --- | --- |
|
||||
| Mouse down | origin | 1 | gesture number |
|
||||
| Mouse dragged | origin | 0 | motion number |
|
||||
| Mouse dragged | midpoint | 0 | motion number |
|
||||
| Mouse dragged | destination | 0 | motion number |
|
||||
| Mouse up | destination | 1 | gesture number |
|
||||
|
||||
Zero-distance drags retain all five events. Do not replace them with clicks or
|
||||
drop the initial dragged event. The three motion events share a number distinct
|
||||
from the down/up pair. The origin window stays bound for the complete gesture.
|
||||
|
||||
In the inspected arm64 service, the specialized function is at `0x100723c60`,
|
||||
the midpoint calculation is at `0x100723d34`, and the five NSEvent constructions
|
||||
are at `0x100723dd8`, `0x100723fbc`, `0x100724160`, `0x100724300`, and
|
||||
`0x1007244a8`. The coordinate caller at `0x10007c8dc` supplies a nil delay to
|
||||
`sendClick` at `0x1006e22d8`. Both the direct and virtual-cursor sender branches
|
||||
skip their optional sleep for this call. Actor transitions are not evidence for
|
||||
a guaranteed yield or a substitute fixed delay.
|
||||
|
||||
`AXAction.dragEvents` and `MouseDragEventTests` encode this event contract using
|
||||
the production event builder. `MouseEventBurstDelivery` retains target/window
|
||||
validation, cancellation checks, and release of a held button at the last
|
||||
posted point. Removing artificial waits must not remove those checks.
|
||||
|
||||
## Batches and observations
|
||||
|
||||
The official native app API supports persistent JavaScript bindings and loops.
|
||||
It recommends batching known actions and observing the resulting state in the
|
||||
same call. In the inspected coordinate controller, only `returnSkyshot: true`
|
||||
enters the UI-settle/capture branch (notification delay 250ms); false invalidates
|
||||
state and returns. The full observation time is not a fixed 250ms promise.
|
||||
|
||||
Only `js`/`js_reset` are advertised to models on macOS, avoiding duplicate
|
||||
per-action schemas. These entry points provide an isolated persistent worker.
|
||||
Variables, App bindings, top-level await, loops, calculations, and intermediate
|
||||
observations survive across ordinary cells. Native App methods pass JSON messages
|
||||
through the existing host semantic-tool dispatch and target checks. Binding an
|
||||
App uses the approved path returned by initial observation; it does not cache a
|
||||
PID as authority. The first visible App selection or inventory includes concise
|
||||
API guidance; selecting an App initially displays AX text only.
|
||||
|
||||
The official facade's `getAXState`, `getScreenshot`, and
|
||||
`getAXStateAndScreenshot` all call `get_app_state`. Our facade mirrors that
|
||||
boundary: output selection and `emit:false` do not optimize away native AX
|
||||
traversal or screenshot work. Actions do not add implicit observations. Copied
|
||||
`gN:id` handles remain opaque; integer aliases are mapped only from returned AX
|
||||
rows and track native diffs/generations. Image-only observations clear integer
|
||||
aliases; a full AX observation is required to rebuild them.
|
||||
|
||||
Each JS cell permits 256 native calls, 256 KiB of source, 128 emitted content
|
||||
blocks and 16 MiB of emitted data. The wall timeout defaults to 30 seconds and
|
||||
is capped at 60 seconds. Ordinary script errors preserve bindings; timeout,
|
||||
cancellation, or explicit reset discards them. Already-dispatched actions may
|
||||
have run and are never automatically replayed. Imports, Node, filesystem and
|
||||
networking are not exposed. Runtime, compiler, and worker boundaries live under
|
||||
`src/utils/computerUse/`; `src/vendor/computer-use-mcp/replApi.ts` owns the native
|
||||
method facade and observation output.
|
||||
|
||||
The existing `sequence` still executes validated JSON actions serially against
|
||||
one proven process and observes once. It stops after failure/cancellation and
|
||||
reports completed steps. Its limits remain 256 steps and a cooperative 60-second
|
||||
deadline; an in-flight native command must settle before the runner returns.
|
||||
Standalone semantic tools retain direct-call compatibility, but they and
|
||||
`sequence` are not advertised by default. Windows retains its pixel tool face.
|
||||
|
||||
`AXTreePublicationIntegrationTests` exercises twelve consecutive zero/one-pixel
|
||||
drags against a disposable native app with a receiver-side gesture counter,
|
||||
without observations between gestures. Its AX/Screen Recording prerequisites
|
||||
are explicit skips when unavailable. Factory tests prove the exact event list;
|
||||
the receiving-app test proves delivery of complete gestures even if AppKit
|
||||
coalesces intermediate motion.
|
||||
|
||||
The identity diagnostic added to `Injection.validateAuthorizedTarget` is
|
||||
DEBUG-only and records the exact target/current values used by strict comparison.
|
||||
`ProcessValidationObservationTests` verifies that this observer does not alter
|
||||
the comparison or allow missing identity evidence. Controlled repeated runs
|
||||
passed 15 rounds, comprising 45 disposable Apps, 180 complete drags, and 1,095
|
||||
actual validation samples (1,080 during drag delivery). Each process retained
|
||||
one complete identity, including identical launch-time Double bits. Twelve
|
||||
gestures and a final observation took 1.178–1.405 seconds in those fixture runs.
|
||||
This is fixture throughput, not model task performance.
|
||||
|
||||
Two earlier anomalies, `stale_process` and a timeout lacking phase records,
|
||||
were not reproduced. Their causes remain unknown. The investigation added
|
||||
diagnostics, not launch-time tolerances, retries, a relaxed comparison, or a
|
||||
different fixture bundle ID.
|
||||
|
||||
## Compatibility limits
|
||||
|
||||
Matching these contracts does not establish complete Codex compatibility or
|
||||
Townscaper/Blender task success. The persistent JS native App facade is present,
|
||||
but browser-tab/DOM providers and the official runtime's general Node facilities
|
||||
are not. Tests that run the actual sandboxed worker with simulated native tools
|
||||
establish persistence, isolation and output behavior. The native receiver test
|
||||
establishes gesture delivery. Neither proves parity across real model tasks.
|
||||
|
||||
Focus acquisition, AX fallbacks, key synthesis, coordinate transforms, capture,
|
||||
and gesture delivery must each be verified at their actual boundary. Do not
|
||||
remove focus or interference checks merely because they cost time. Compare
|
||||
action counts, observations, model round trips, per-action tool time and
|
||||
receiver-visible outcomes separately; tool-time improvements alone do not
|
||||
predict end-to-end model task performance.
|
||||
@@ -351,9 +351,6 @@ public enum AXAction {
|
||||
/// Max UTF-16 units per `keyboardSetUnicodeString` chunk (blueprint §4 ≤64).
|
||||
private static let unicodeChunk = 64
|
||||
|
||||
/// Lines scrolled per "page" for the synthetic-wheel fallback (blueprint §4).
|
||||
private static let linesPerPage = 12
|
||||
|
||||
/// How far down to scan for an actionable descendant (blueprint §4: "向下3层").
|
||||
private static let descendantScanDepth = 3
|
||||
|
||||
@@ -767,6 +764,27 @@ public enum AXAction {
|
||||
throw CUError("invalid_action", "\(action) is not a valid secondary action for \(index)")
|
||||
}
|
||||
|
||||
// Official UIElementProtocol.perform prioritizes native scroll-bar
|
||||
// page buttons. AppKit may advertise AXScroll…ByPage on the area while
|
||||
// rejecting a direct AXPerformAction; the page button still works.
|
||||
if let navigation = NativeScroll.pageNavigation(action: raw),
|
||||
let bar = copyElement(element, navigation.axisAttribute),
|
||||
let button = children(of: bar).first(where: {
|
||||
stringAttribute($0, kAXRoleAttribute) == (kAXButtonRole as String)
|
||||
&& stringAttribute($0, kAXSubroleAttribute) == navigation.buttonSubrole
|
||||
}) {
|
||||
var buttonPID: pid_t = 0
|
||||
guard AXUIElementGetPid(button, &buttonPID) == .success, buttonPID == pid else {
|
||||
throw CUError("stale_element", "The scroll page button no longer belongs to the target process")
|
||||
}
|
||||
let error = AXUIElementPerformAction(button, kAXPressAction as CFString)
|
||||
guard error == .success else {
|
||||
throw CUError("ax_failed", "The scroll page button rejected AXPress (AXError \(error.rawValue))")
|
||||
}
|
||||
settle()
|
||||
return
|
||||
}
|
||||
|
||||
let err = AXUIElementPerformAction(element, raw as CFString)
|
||||
guard err == .success else {
|
||||
throw CUError("ax_failed", "perform_secondary_action(\(raw)) failed (AXError \(err.rawValue)) on element \(index)")
|
||||
@@ -778,9 +796,9 @@ public enum AXAction {
|
||||
|
||||
/// Scroll the element at `index` (preferred) — or, if `index` is nil, the
|
||||
/// element under the given GLOBAL point — `pages` pages in `direction`
|
||||
/// (up/down/left/right). Element-domain first: repeat `AXScroll{Up,Down,Left,
|
||||
/// Right}ByPage` floor(pages) times if the element exposes it; else synthesize a
|
||||
/// wheel event at the element center (12 lines/page) via `postToPid`.
|
||||
/// (up/down/left/right). Fractional pages become precise pixel deltas using
|
||||
/// the addressed element's frame, or the whole window for coordinates.
|
||||
/// AX Scroll…ByPage remains available as a distinct secondary action.
|
||||
public static func scroll(
|
||||
pid: pid_t,
|
||||
index: Int?,
|
||||
@@ -798,30 +816,17 @@ public enum AXAction {
|
||||
}
|
||||
|
||||
// Resolve the target element (by index) and its center, or a bare point.
|
||||
var element: AXUIElement?
|
||||
var elementFrame: CGRect?
|
||||
var center: CGPoint?
|
||||
if let index {
|
||||
element = try resolveElement(pid: pid, index: index)
|
||||
center = centerGlobal(AXTree.record(pid: pid, index: index))
|
||||
let element = try resolveElement(pid: pid, index: index)
|
||||
elementFrame = AXTree.frameRect(element)
|
||||
center = elementFrame.map { CGPoint(x: $0.midX, y: $0.midY) }
|
||||
}
|
||||
if center == nil, let x, let y {
|
||||
center = CGPoint(x: x, y: y)
|
||||
}
|
||||
|
||||
// ── Element domain: AXScroll…ByPage, floor(pages) repeats. ────────────
|
||||
let wholePages = Int(pages.rounded(.down))
|
||||
if wholePages >= 1, let element {
|
||||
let pageAction = "AXScroll\(dir.capitalized)ByPage" // AXScrollUpByPage, …
|
||||
if actionNames(element).contains(pageAction) {
|
||||
for _ in 0..<wholePages {
|
||||
_ = AXUIElementPerformAction(element, pageAction as CFString)
|
||||
try await Task.sleep(for: .milliseconds(50))
|
||||
}
|
||||
settle()
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// ── Synthetic wheel at the element center (or point). ─────────────────
|
||||
guard let point = center else {
|
||||
throw CUError("no_target", "scroll: element \(index.map(String.init) ?? "?") has no frame and no x/y point was given")
|
||||
@@ -834,7 +839,13 @@ public enum AXAction {
|
||||
guard WindowGeometry.window(id: window.id, pid: pid) == window else {
|
||||
throw CUError("stale_window", "The scroll target moved or closed. Read its current state before retrying.")
|
||||
}
|
||||
try scrollWheel(at: point, direction: dir, pages: pages, pid: pid)
|
||||
let delta = try NativeScroll.delta(direction: dir, pages: pages, frameSize: (elementFrame ?? window.bounds).size)
|
||||
guard let eventSource,
|
||||
let event = WindowTargetedEvent.makeScrollEvent(
|
||||
source: eventSource, point: point, deltaX: delta.x, deltaY: delta.y, window: window
|
||||
) else { throw CUError(CUError.Code.eventAlloc, "Failed to allocate a marked pixel scroll event") }
|
||||
try Task.checkCancellation()
|
||||
WindowTargetedEvent.post(event, to: pid)
|
||||
settle()
|
||||
}
|
||||
|
||||
@@ -940,11 +951,12 @@ public enum AXAction {
|
||||
static func pasteText(
|
||||
pid: pid_t,
|
||||
_ text: String,
|
||||
format: ClipboardPasteFormat
|
||||
format: ClipboardPasteFormat,
|
||||
lease: ClipboardLease? = nil
|
||||
) async throws {
|
||||
guard !text.isEmpty else { return }
|
||||
let target = try Injection.authorizeResolvedTarget(pid: pid)
|
||||
try await typeViaClipboard(target: target, text, format: format)
|
||||
try await typeViaClipboard(target: target, text, format: format, lease: lease)
|
||||
}
|
||||
|
||||
/// Paste `text` into whatever holds in-app keyboard focus in `pid`, via the
|
||||
@@ -954,13 +966,15 @@ public enum AXAction {
|
||||
private static func typeViaClipboard(
|
||||
target: ProvenProcessTarget,
|
||||
_ text: String,
|
||||
format: ClipboardPasteFormat = .text
|
||||
format: ClipboardPasteFormat = .text,
|
||||
lease: ClipboardLease? = nil
|
||||
) async throws {
|
||||
let pid = target.pid
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: text,
|
||||
format: format,
|
||||
lease: ClipboardLease()
|
||||
lease: lease ?? ClipboardLease(),
|
||||
targetPID: pid
|
||||
) { validate in
|
||||
try await pressKey(pid: pid, "super+v", validateBeforePosting: {
|
||||
_ = try Injection.validateAuthorizedTarget(target)
|
||||
@@ -1016,7 +1030,7 @@ public enum AXAction {
|
||||
// MARK: - Drag
|
||||
|
||||
/// Synthetic left-button (or `button`) drag between two GLOBAL points: down at
|
||||
/// `from`, ten interpolated dragged steps, up at `to`. Posted to the window
|
||||
/// `from`, dragged at the origin/midpoint/destination, up at `to`. Posted to the window
|
||||
/// explicitly resolved target via `postToPid`. Coordinate-only by contract —
|
||||
/// the model drives drags by pixel, not by element index.
|
||||
public static func drag(pid: pid_t, from: CGPoint, to: CGPoint, button: MouseButton = .left) async throws {
|
||||
@@ -1032,20 +1046,29 @@ public enum AXAction {
|
||||
let target = try Injection.authorizeResolvedTarget(pid: pid)
|
||||
let focusReceipt = try await ensureTargetAcceptsInput(pid: pid, window: dragWindow)
|
||||
|
||||
// Movement carries clickState 0 (both the leading move and every
|
||||
// dragged step); only the press and the release belong to the click.
|
||||
// The press and release bracket one gesture and share a number; every
|
||||
// movement in between shares a second one.
|
||||
let events = try dragEvents(
|
||||
from: from, to: to, button: button,
|
||||
source: src, pid: pid, window: dragWindow
|
||||
)
|
||||
try await postMouseBurst(
|
||||
events, target: target, window: dragWindow, button: button,
|
||||
focusReceipt: focusReceipt, pause: nil
|
||||
)
|
||||
}
|
||||
|
||||
static func dragEvents(
|
||||
from: CGPoint, to: CGPoint, button: MouseButton,
|
||||
source: CGEventSource, pid: pid_t, window: WindowGeometry.Window
|
||||
) throws -> [CGEvent] {
|
||||
// Match the installed official SkyComputerUseService 26.831.1000926
|
||||
// SynthesizedEvent.click implementation: down, dragged at origin,
|
||||
// dragged at midpoint, dragged at destination, up. A zero-distance
|
||||
// drag retains all five events. This is a drag gesture, not a click.
|
||||
// Motion carries clickState 0 and a shared event number; down/up carry
|
||||
// clickState 1 and a different shared number.
|
||||
let gestureNumber = WindowTargetedEvent.nextEventNumber()
|
||||
let motionNumber = WindowTargetedEvent.nextEventNumber()
|
||||
var specs = [
|
||||
MouseBurstSpec(
|
||||
type: .mouseMoved,
|
||||
point: from,
|
||||
button: button,
|
||||
clickState: mouseClickState(for: .mouseMoved, click: 1),
|
||||
eventNumber: motionNumber
|
||||
),
|
||||
MouseBurstSpec(
|
||||
type: button.down,
|
||||
point: from,
|
||||
@@ -1054,12 +1077,11 @@ public enum AXAction {
|
||||
eventNumber: gestureNumber
|
||||
),
|
||||
]
|
||||
for step in 1...10 {
|
||||
let t = CGFloat(step) / 10
|
||||
let p = CGPoint(x: from.x + (to.x - from.x) * t, y: from.y + (to.y - from.y) * t)
|
||||
let midpoint = CGPoint(x: (from.x + to.x) / 2, y: (from.y + to.y) / 2)
|
||||
for point in [from, midpoint, to] {
|
||||
specs.append(MouseBurstSpec(
|
||||
type: button.dragged,
|
||||
point: p,
|
||||
point: point,
|
||||
button: button,
|
||||
clickState: mouseClickState(for: button.dragged, click: 1),
|
||||
eventNumber: motionNumber
|
||||
@@ -1072,23 +1094,21 @@ public enum AXAction {
|
||||
clickState: mouseClickState(for: button.up, click: 1),
|
||||
eventNumber: gestureNumber
|
||||
))
|
||||
let events: [CGEvent] = try EventBurst.allocateAll(
|
||||
return try EventBurst.allocateAll(
|
||||
specs: specs
|
||||
) { spec in
|
||||
makeMouse(spec, source: src, targetPid: pid, window: dragWindow)
|
||||
makeMouse(spec, source: source, targetPid: pid, window: window)
|
||||
}
|
||||
try await postMouseBurst(
|
||||
events, target: target, window: dragWindow, button: button, focusReceipt: focusReceipt
|
||||
)
|
||||
}
|
||||
|
||||
private static func postMouseBurst(
|
||||
_ events: [CGEvent], target: ProvenProcessTarget,
|
||||
window: WindowGeometry.Window, button: MouseButton,
|
||||
focusReceipt: FocusEventMonitor.RegistrationReceipt,
|
||||
pause: (@MainActor () async throws -> Void)? = {
|
||||
try await Task.sleep(for: .milliseconds(30))
|
||||
}
|
||||
// The official coordinate click/drag controller supplies no per-event
|
||||
// delay to sendClick. Preserve an injectable seam for paced callers,
|
||||
// without imposing the old 30 ms sleep on every coordinate event.
|
||||
pause: (@MainActor () async throws -> Void)? = nil
|
||||
) async throws {
|
||||
try await MouseEventBurstDelivery.deliver(
|
||||
events: events,
|
||||
@@ -1312,44 +1332,6 @@ public enum AXAction {
|
||||
return out
|
||||
}
|
||||
|
||||
// MARK: Scroll synthesis
|
||||
|
||||
/// Synthetic scroll-wheel event at a GLOBAL point, `pages` pages in `direction`
|
||||
/// (12 lines/page). Vertical on wheel1, horizontal on wheel2; posted to the
|
||||
/// window owner via `postToPid`.
|
||||
private static func scrollWheel(at point: CGPoint, direction: String, pages: Double, pid: pid_t) throws {
|
||||
let magnitude = Int32(clamping: Int((Double(linesPerPage) * pages).rounded(.toNearestOrAwayFromZero)))
|
||||
let amount = max(1, magnitude)
|
||||
// Quartz wheel sign: +up / -down (vertical), +left / -right (horizontal).
|
||||
let wheel1: Int32 = direction == "up" ? amount : (direction == "down" ? -amount : 0)
|
||||
let wheel2: Int32 = direction == "left" ? amount : (direction == "right" ? -amount : 0)
|
||||
guard let eventSource else {
|
||||
throw CUError(
|
||||
CUError.Code.eventAlloc,
|
||||
"Failed to allocate a marked scroll-wheel event source"
|
||||
)
|
||||
}
|
||||
guard let event = CGEvent(
|
||||
scrollWheelEvent2Source: eventSource,
|
||||
units: .line,
|
||||
wheelCount: 2,
|
||||
wheel1: wheel1,
|
||||
wheel2: wheel2,
|
||||
wheel3: 0
|
||||
) else {
|
||||
throw CUError(CUError.Code.eventAlloc, "Failed to allocate a scroll-wheel event")
|
||||
}
|
||||
event.location = point
|
||||
// Same window binding the click path needs: Chromium routes scroll by
|
||||
// the window the event claims, so an unbound wheel event is discarded.
|
||||
if let window = WindowGeometry.window(at: point, pid: pid) {
|
||||
event.setIntegerValueField(CGEventField(rawValue: 91)!, value: Int64(window.id))
|
||||
event.setIntegerValueField(CGEventField(rawValue: 92)!, value: Int64(window.id))
|
||||
}
|
||||
WindowTargetedEvent.post(event, to: pid)
|
||||
Thread.sleep(forTimeInterval: 0.1)
|
||||
}
|
||||
|
||||
// MARK: Keyboard helpers
|
||||
|
||||
/// UTF-16 chunks of `text`, each ≤ `unicodeChunk` units, never splitting a
|
||||
|
||||
@@ -0,0 +1,96 @@
|
||||
import AppKit
|
||||
import Foundation
|
||||
|
||||
/// Public native AppInfo shape. Discovery does not grant access to an app.
|
||||
struct AppInventoryEntry: Encodable, Sendable, Equatable {
|
||||
let id: String
|
||||
let displayName: String
|
||||
var isRunning: Bool
|
||||
var lastUsedDate: Date? = nil
|
||||
var useCount: Int? = nil
|
||||
|
||||
private enum CodingKeys: String, CodingKey { case id, displayName, isRunning, lastUsedDate, useCount }
|
||||
|
||||
func encode(to encoder: any Encoder) throws {
|
||||
var values = encoder.container(keyedBy: CodingKeys.self)
|
||||
try values.encode(id, forKey: .id)
|
||||
try values.encode(displayName, forKey: .displayName)
|
||||
try values.encode(isRunning, forKey: .isRunning)
|
||||
if let lastUsedDate {
|
||||
try values.encode(ISO8601DateFormatter().string(from: lastUsedDate), forKey: .lastUsedDate)
|
||||
}
|
||||
try values.encodeIfPresent(useCount, forKey: .useCount)
|
||||
}
|
||||
}
|
||||
|
||||
enum AppInventory {
|
||||
static func fromMetadata(_ values: [String: Any]) -> AppInventoryEntry? {
|
||||
guard let id = values["kMDItemCFBundleIdentifier"] as? String, !id.isEmpty else { return nil }
|
||||
let useCount = (values["kMDItemUseCount"] as? NSNumber)?.intValue
|
||||
return AppInventoryEntry(
|
||||
id: id,
|
||||
displayName: values["kMDItemDisplayName"] as? String ?? id,
|
||||
isRunning: false,
|
||||
lastUsedDate: values["kMDItemLastUsedDate_Ranking"] as? Date,
|
||||
useCount: useCount.flatMap { $0 >= 0 ? $0 : nil }
|
||||
)
|
||||
}
|
||||
|
||||
static func merge(running: [AppRef], recent: [AppInventoryEntry]) -> [AppInventoryEntry] {
|
||||
var metadata: [String: AppInventoryEntry] = [:]
|
||||
for app in recent where metadata[app.id] == nil { metadata[app.id] = app }
|
||||
var seen = Set<String>()
|
||||
var result: [AppInventoryEntry] = []
|
||||
for app in running where seen.insert(app.bundleId).inserted {
|
||||
result.append(AppInventoryEntry(
|
||||
id: app.bundleId, displayName: app.displayName, isRunning: true,
|
||||
lastUsedDate: metadata[app.bundleId]?.lastUsedDate,
|
||||
useCount: metadata[app.bundleId]?.useCount
|
||||
))
|
||||
}
|
||||
for var app in recent where seen.insert(app.id).inserted {
|
||||
app.isRunning = false
|
||||
result.append(app)
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
/// The official native catalog combines regular running applications with
|
||||
/// Spotlight application bundles used in the last fourteen days. Keep a live
|
||||
/// metadata query so later observations do not rescan the filesystem or render
|
||||
/// icons. A bounded initial gather also works when Spotlight is unavailable.
|
||||
@MainActor
|
||||
final class RecentAppCatalog {
|
||||
static let shared = RecentAppCatalog()
|
||||
private var query: NSMetadataQuery?
|
||||
private let keys = ["kMDItemCFBundleIdentifier", "kMDItemDisplayName", "kMDItemLastUsedDate_Ranking", "kMDItemUseCount"]
|
||||
|
||||
func entries() async throws -> [AppInventoryEntry] {
|
||||
if query == nil {
|
||||
let metadata = NSMetadataQuery()
|
||||
metadata.searchScopes = [NSMetadataQueryLocalComputerScope]
|
||||
metadata.predicate = NSPredicate(
|
||||
format: "kMDItemContentType == %@ AND kMDItemFSName LIKE %@ AND kMDItemLastUsedDate_Ranking >= %@",
|
||||
"com.apple.application-bundle", "*.app",
|
||||
Calendar.current.startOfDay(for: Date()).addingTimeInterval(-14 * 24 * 60 * 60) as NSDate
|
||||
)
|
||||
guard metadata.start() else { return [] }
|
||||
query = metadata
|
||||
}
|
||||
guard let query else { return [] }
|
||||
let deadline = ContinuousClock.now.advanced(by: .seconds(2))
|
||||
while query.isGathering && ContinuousClock.now < deadline {
|
||||
try await Task.sleep(for: .milliseconds(20))
|
||||
}
|
||||
try Task.checkCancellation()
|
||||
query.disableUpdates()
|
||||
defer { query.enableUpdates() }
|
||||
return query.results.compactMap { result in
|
||||
guard let item = result as? NSMetadataItem else { return nil }
|
||||
var values: [String: Any] = [:]
|
||||
for key in keys { values[key] = item.value(forAttribute: key) }
|
||||
return AppInventory.fromMetadata(values)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,9 +1,10 @@
|
||||
import Foundation
|
||||
|
||||
/// Native fail-closed app policy used after target resolution has produced a
|
||||
/// real bundle identifier. The list mirrors the complete union of the four
|
||||
/// exact bundle-ID sets exported from `deniedApps.ts`; display-name substring
|
||||
/// policy deliberately remains outside this native resolver seam.
|
||||
/// Built-in macOS native policy, matched by exact resolved bundle identity.
|
||||
/// The installed official service checks terminal, Computer Use host, ChatGPT,
|
||||
/// and system-security groups. Other categories remain subject to app approval;
|
||||
/// the older generic Windows terminal/IDE/media/trading lists do not define this
|
||||
/// native boundary.
|
||||
enum AppTargetPolicy {
|
||||
enum Decision: Equatable, Sendable {
|
||||
case allow
|
||||
@@ -18,123 +19,38 @@ enum AppTargetPolicy {
|
||||
"dev.cchaha.cu-helper",
|
||||
]
|
||||
|
||||
/// Keep this as one string-literal set so the TypeScript parity test can
|
||||
/// parse and compare it directly with the authoritative four-set union.
|
||||
/// Audited against SkyComputerUseService 26.831.1000926's
|
||||
/// BundleIdentifiers.isForbiddenComputerUseTarget (0x100240a14).
|
||||
/// Keep this literal set cross-checked with nativeAppPolicy.ts.
|
||||
static let deniedBundleIDs: Set<String> = [
|
||||
// BROWSER_BUNDLE_IDS
|
||||
"com.apple.Safari",
|
||||
"com.apple.SafariTechnologyPreview",
|
||||
"com.google.Chrome",
|
||||
"com.google.Chrome.beta",
|
||||
"com.google.Chrome.dev",
|
||||
"com.google.Chrome.canary",
|
||||
"com.microsoft.edgemac",
|
||||
"com.microsoft.edgemac.Beta",
|
||||
"com.microsoft.edgemac.Dev",
|
||||
"com.microsoft.edgemac.Canary",
|
||||
"org.mozilla.firefox",
|
||||
"org.mozilla.firefoxdeveloperedition",
|
||||
"org.mozilla.nightly",
|
||||
"org.chromium.Chromium",
|
||||
"com.brave.Browser",
|
||||
"com.brave.Browser.beta",
|
||||
"com.brave.Browser.nightly",
|
||||
"com.operasoftware.Opera",
|
||||
"com.operasoftware.OperaGX",
|
||||
"com.operasoftware.OperaDeveloper",
|
||||
"com.vivaldi.Vivaldi",
|
||||
"company.thebrowser.Browser",
|
||||
"company.thebrowser.dia",
|
||||
"org.torproject.torbrowser",
|
||||
"com.duckduckgo.macos.browser",
|
||||
"ru.yandex.desktop.yandex-browser",
|
||||
"ai.perplexity.comet",
|
||||
"com.sigmaos.sigmaos.macos",
|
||||
"com.kagi.kagimacOS",
|
||||
|
||||
// TERMINAL_BUNDLE_IDS
|
||||
// terminal
|
||||
"com.apple.Terminal",
|
||||
"com.googlecode.iterm2",
|
||||
"dev.warp.Warp-Stable",
|
||||
"dev.warp.Warp-Beta",
|
||||
"com.github.wez.wezterm",
|
||||
"org.alacritty",
|
||||
"io.alacritty",
|
||||
"dev.warp.Warp-Stable",
|
||||
"net.kovidgoyal.kitty",
|
||||
"co.zeit.hyper",
|
||||
"com.mitchellh.ghostty",
|
||||
"com.github.wez.wezterm",
|
||||
"org.tabby",
|
||||
"com.termius-dmg.mac",
|
||||
"com.microsoft.VSCode",
|
||||
"com.microsoft.VSCodeInsiders",
|
||||
"com.vscodium",
|
||||
"com.todesktop.230313mzl4w4u92",
|
||||
"com.exafunction.windsurf",
|
||||
"dev.zed.Zed",
|
||||
"dev.zed.Zed-Preview",
|
||||
"com.jetbrains.intellij",
|
||||
"com.jetbrains.intellij.ce",
|
||||
"com.jetbrains.pycharm",
|
||||
"com.jetbrains.pycharm.ce",
|
||||
"com.jetbrains.WebStorm",
|
||||
"com.jetbrains.CLion",
|
||||
"com.jetbrains.goland",
|
||||
"com.jetbrains.rubymine",
|
||||
"com.jetbrains.PhpStorm",
|
||||
"com.jetbrains.datagrip",
|
||||
"com.jetbrains.rider",
|
||||
"com.jetbrains.AppCode",
|
||||
"com.jetbrains.rustrover",
|
||||
"com.jetbrains.fleet",
|
||||
"com.google.android.studio",
|
||||
"com.axosoft.gitkraken",
|
||||
"com.sublimetext.4",
|
||||
"com.sublimetext.3",
|
||||
"org.vim.MacVim",
|
||||
"com.neovim.neovim",
|
||||
"org.gnu.Emacs",
|
||||
"com.apple.dt.Xcode",
|
||||
"org.eclipse.platform.ide",
|
||||
"org.netbeans.ide",
|
||||
"com.microsoft.visual-studio",
|
||||
"com.apple.ScriptEditor2",
|
||||
"com.apple.Automator",
|
||||
"com.apple.shortcuts",
|
||||
|
||||
// TRADING_BUNDLE_IDS
|
||||
"com.webull.desktop.v1",
|
||||
"com.webull.trade.mac.v1",
|
||||
"com.tastytrade.desktop",
|
||||
"com.tradingview.tradingviewapp.desktop",
|
||||
"com.fidelity.activetrader",
|
||||
"com.fmr.activetrader",
|
||||
"com.install4j.5889-6375-8446-2021",
|
||||
"com.binance.BinanceDesktop",
|
||||
"com.electron.exodus",
|
||||
"org.pythonmac.unspecified.Electrum",
|
||||
"com.ledger.live",
|
||||
"io.trezor.TrezorSuite",
|
||||
|
||||
// POLICY_DENIED_BUNDLE_IDS
|
||||
"com.apple.TV",
|
||||
"com.apple.Music",
|
||||
"com.apple.iBooksX",
|
||||
"com.apple.podcasts",
|
||||
"com.spotify.client",
|
||||
"com.amazon.music",
|
||||
"com.tidal.desktop",
|
||||
"com.deezer.deezer-desktop",
|
||||
"com.pandora.desktop",
|
||||
"com.electron.pocket-casts",
|
||||
"au.com.shiftyjelly.PocketCasts",
|
||||
"tv.plex.desktop",
|
||||
"tv.plex.htpc",
|
||||
"tv.plex.plexamp",
|
||||
"com.amazon.aiv.AIVApp",
|
||||
"net.kovidgoyal.calibre",
|
||||
"com.amazon.Kindle",
|
||||
"com.amazon.Lassen",
|
||||
"com.kobo.desktop.Kobo",
|
||||
"com.mitchellh.ghostty",
|
||||
"com.raphaelamorim.rio",
|
||||
"dev.commandline.waveterm",
|
||||
// computerUseHost
|
||||
"com.openai.codex",
|
||||
"com.openai.codex.alpha",
|
||||
"com.openai.codex.beta",
|
||||
"com.openai.codex.dev",
|
||||
"com.openai.codex.nightly",
|
||||
// chatGPT
|
||||
"com.openai.chat",
|
||||
"com.openai.chat.alpha",
|
||||
"com.openai.chat.beta",
|
||||
"com.openai.chat.nightly",
|
||||
"com.openai.chat.mac-debug",
|
||||
// systemSecurity
|
||||
"com.apple.UserNotificationCenter",
|
||||
"com.apple.LocalAuthenticationRemoteService",
|
||||
"com.apple.SecurityAgent",
|
||||
]
|
||||
|
||||
static func decision(bundleID: String) -> Decision {
|
||||
|
||||
@@ -201,6 +201,19 @@ public enum Apps {
|
||||
|
||||
// MARK: list_running_apps
|
||||
|
||||
/// Native discovery includes recently used apps even when they are closed.
|
||||
/// Approval and exact process authorization are enforced when selecting or
|
||||
/// controlling a target, not by making forbidden apps disappear here.
|
||||
static func listApps() async throws -> [AppInventoryEntry] {
|
||||
let recent = try await RecentAppCatalog.shared.entries()
|
||||
let running = NSWorkspace.shared.runningApplications.compactMap { app -> AppRef? in
|
||||
guard app.activationPolicy == .regular,
|
||||
let bundleId = app.bundleIdentifier, !bundleId.isEmpty else { return nil }
|
||||
return AppRef(bundleId: bundleId, displayName: app.localizedName ?? bundleId)
|
||||
}
|
||||
return AppInventory.merge(running: running, recent: recent)
|
||||
}
|
||||
|
||||
/// Running applications that carry a bundle identifier, deduped and sorted
|
||||
/// case-insensitively by display name (matches mac_helper.running_apps).
|
||||
public static func listRunning() -> [AppRef] {
|
||||
|
||||
@@ -57,6 +57,10 @@ struct WindowShot: Sendable {
|
||||
let pointHeight: Double
|
||||
let windowID: CGWindowID
|
||||
let source: WindowShotCaptureSource
|
||||
/// Uniform capture fit before integer pixel-buffer rounding. Legacy shots
|
||||
/// may omit this and retain their dimension-derived transform.
|
||||
var pixelsPerPoint: Double? = nil
|
||||
var mimeType: String { NativeScreenshotPolicy.mimeType }
|
||||
}
|
||||
|
||||
@available(macOS 14.0, *)
|
||||
@@ -442,20 +446,19 @@ public enum Capture {
|
||||
/// - Parameters:
|
||||
/// - pid: the target application's process id (the app whose AX tree
|
||||
/// `get_app_state` just rendered).
|
||||
/// - scale: output downscale factor applied to the window's native pixel
|
||||
/// size. `0.5` (the blueprint default) halves each axis. Values `<= 0`
|
||||
/// are coerced to `0.5`.
|
||||
/// - scale: an explicit factor applied to native pixels; nil uses the
|
||||
/// native App policy (point resolution, long/short side limits).
|
||||
/// - Returns: `(base64, width, height, originX, originY, pointWidth,
|
||||
/// pointHeight, windowID)` — the PNG in screenshot-pixel space (top-left origin)
|
||||
/// pointHeight, windowID)` — the JPEG in screenshot-pixel space (top-left origin)
|
||||
/// PLUS the captured window's GLOBAL Quartz top-left origin and its size
|
||||
/// in POINTS. The caller uses `(originX, originY)` + `imgWidth/pointWidth`
|
||||
/// (pixels-per-point) to invert image-pixel coordinates back into the
|
||||
/// in POINTS. The caller uses the uniform `pixelsPerPoint` capture fit
|
||||
/// to invert image-pixel coordinates back into the
|
||||
/// global-point space that clicks/cursor/glow all live in. `nil` on any
|
||||
/// failure. Never throws; never prompts.
|
||||
static func windowShot(
|
||||
pid: pid_t,
|
||||
preferredWindowID: CGWindowID? = nil,
|
||||
scale: Double = 0.5,
|
||||
scale: Double? = nil,
|
||||
allowCLIFallback: Bool = true
|
||||
) async -> WindowShot? {
|
||||
// Passive permission gate — no prompt on the hot path. A denied grant
|
||||
@@ -468,7 +471,10 @@ public enum Capture {
|
||||
preferredWindowID: preferredWindowID
|
||||
) else { return nil }
|
||||
|
||||
let outputScale = scale > 0 ? scale : 0.5
|
||||
let backingScale = backingScaleFactor(forWindowFrame: target.frame)
|
||||
let outputScale = scale.flatMap { $0 > 0 ? $0 : nil } ?? NativeScreenshotPolicy.scale(
|
||||
pointSize: target.frame.size, backingScale: backingScale
|
||||
)
|
||||
// The captured window's global Quartz top-left frame (points). Threaded
|
||||
// to every return point so the caller can build the inverse transform.
|
||||
let f = target.frame
|
||||
@@ -479,7 +485,7 @@ public enum Capture {
|
||||
frame: target.frame,
|
||||
scale: outputScale
|
||||
) {
|
||||
if let encoded = pngBase64WithSize(image) {
|
||||
if let encoded = appScreenshotBase64WithSize(image) {
|
||||
return WindowShot(
|
||||
base64: encoded.base64,
|
||||
width: encoded.width,
|
||||
@@ -489,10 +495,11 @@ public enum Capture {
|
||||
pointWidth: Double(f.width),
|
||||
pointHeight: Double(f.height),
|
||||
windowID: target.windowID,
|
||||
source: .screenshotManager
|
||||
source: .screenshotManager,
|
||||
pixelsPerPoint: backingScale * outputScale
|
||||
)
|
||||
}
|
||||
// SCK produced pixels but PNG/base64 failed — degrade to `nil`
|
||||
// SCK produced pixels but image encoding failed — degrade to `nil`
|
||||
// rather than re-capturing; the caller still gets AX text.
|
||||
return nil
|
||||
}
|
||||
@@ -500,8 +507,9 @@ public enum Capture {
|
||||
guard allowCLIFallback else { return nil }
|
||||
// ② Fallback: /usr/sbin/screencapture -l <windowID> (SCK hung or failed).
|
||||
if let raw = screencaptureWindow(windowID: target.windowID) {
|
||||
let scaled = (try? scaleImage(raw, scale: outputScale)) ?? raw
|
||||
if let encoded = pngBase64WithSize(scaled) {
|
||||
let scaledImage = try? scaleImage(raw, scale: outputScale)
|
||||
let scaled = scaledImage ?? raw
|
||||
if let encoded = appScreenshotBase64WithSize(scaled) {
|
||||
return WindowShot(
|
||||
base64: encoded.base64,
|
||||
width: encoded.width,
|
||||
@@ -511,7 +519,8 @@ public enum Capture {
|
||||
pointWidth: Double(f.width),
|
||||
pointHeight: Double(f.height),
|
||||
windowID: target.windowID,
|
||||
source: .screenCaptureCLI
|
||||
source: .screenCaptureCLI,
|
||||
pixelsPerPoint: scaledImage == nil ? nil : backingScale * outputScale
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -766,8 +775,8 @@ public enum Capture {
|
||||
/// the caller can fall back to the unscaled image.
|
||||
private static func scaleImage(_ image: CGImage, scale: Double) throws -> CGImage {
|
||||
guard scale > 0, scale != 1.0 else { return image }
|
||||
let outW = max(1, Int((Double(image.width) * scale).rounded()))
|
||||
let outH = max(1, Int((Double(image.height) * scale).rounded()))
|
||||
let outW = max(1, Int(ceil(Double(image.width) * scale)))
|
||||
let outH = max(1, Int(ceil(Double(image.height) * scale)))
|
||||
if outW == image.width && outH == image.height { return image }
|
||||
|
||||
guard let colorSpace = image.colorSpace ?? CGColorSpace(name: CGColorSpace.sRGB) else {
|
||||
@@ -792,19 +801,18 @@ public enum Capture {
|
||||
return scaled
|
||||
}
|
||||
|
||||
/// Encode a `CGImage` to base64 **PNG** plus its actual pixel dimensions.
|
||||
/// `get_app_state`'s MCP envelope uses `mimeType: "image/png"`, so unlike the
|
||||
/// JPEG `screenshot` path this is lossless PNG. Returns `nil` on any encode
|
||||
/// failure (never throws) so `windowShot` degrades to AX-text-only.
|
||||
static func pngBase64WithSize(
|
||||
/// Native App screenshots use the official default JPEG quality. The
|
||||
/// byte format is reported explicitly instead of relying on a fixed MIME
|
||||
/// label in a downstream wrapper. Encoding failure degrades to AX text.
|
||||
static func appScreenshotBase64WithSize(
|
||||
_ image: CGImage
|
||||
) -> (base64: String, width: Int, height: Int)? {
|
||||
let data = NSMutableData()
|
||||
let type = UTType.png.identifier as CFString
|
||||
let type = UTType.jpeg.identifier as CFString
|
||||
guard let dest = CGImageDestinationCreateWithData(data, type, 1, nil) else {
|
||||
return nil
|
||||
}
|
||||
CGImageDestinationAddImage(dest, image, nil)
|
||||
CGImageDestinationAddImage(dest, image, [kCGImageDestinationLossyCompressionQuality: NativeScreenshotPolicy.jpegQuality] as CFDictionary)
|
||||
guard CGImageDestinationFinalize(dest) else { return nil }
|
||||
return (
|
||||
(data as Data).base64EncodedString(),
|
||||
|
||||
@@ -54,6 +54,8 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
private let promisedData: [NSPasteboard.PasteboardType: Data]
|
||||
private let state = OSAllocatedUnfairLock(initialState: State())
|
||||
|
||||
var hasSuppliedData: Bool { state.withLock { $0.suppliedAt != nil } }
|
||||
|
||||
init(text: String) {
|
||||
promisedData = ClipboardPasteFormat.text.promisedData(for: text)
|
||||
super.init()
|
||||
@@ -89,14 +91,18 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
format: ClipboardPasteFormat = .text,
|
||||
lease: ClipboardLease,
|
||||
timeout: Duration = .seconds(2),
|
||||
targetPID: pid_t? = nil,
|
||||
targetObservation: ClipboardPasteObservation? = nil,
|
||||
sendPaste: @MainActor (_ validateBeforePosting: @MainActor () throws -> Void) async throws -> Void
|
||||
) async throws {
|
||||
lastDiagnostic = nil
|
||||
let started = ContinuousClock.now
|
||||
var receipt: ClipboardPasteReceipt?
|
||||
var observation: ClipboardPasteObservation?
|
||||
var posted = false
|
||||
var status = "failed"
|
||||
defer {
|
||||
observation?.close()
|
||||
let owned = lease.temporaryWriteIsCurrent()
|
||||
let observed = receipt?.state.withLock { $0 }
|
||||
let restored = lease.restoreIfUnchanged()
|
||||
@@ -116,8 +122,11 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
}
|
||||
do {
|
||||
try Task.checkCancellation()
|
||||
let target = targetObservation ?? ClipboardPasteObservation.capture(pid: targetPID)
|
||||
observation = target
|
||||
let written = try lease.writeTemporaryContentWithReceipt(text, format: format)
|
||||
receipt = written
|
||||
target.arm(written)
|
||||
try await Task.sleep(for: .milliseconds(40))
|
||||
let validate: @MainActor () throws -> Void = {
|
||||
guard lease.temporaryWriteIsCurrent() else {
|
||||
@@ -128,6 +137,9 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
try await sendPaste(validate)
|
||||
posted = true
|
||||
try await written.waitForRead(timeout: timeout, ownsClipboard: lease.temporaryWriteIsCurrent)
|
||||
try await written.waitForTarget(
|
||||
target, timeout: timeout, ownsClipboard: lease.temporaryWriteIsCurrent
|
||||
)
|
||||
guard lease.temporaryWriteIsCurrent() else {
|
||||
throw CUError("clipboard_changed", "The clipboard changed after paste data was supplied")
|
||||
}
|
||||
@@ -139,11 +151,10 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
}
|
||||
}
|
||||
|
||||
/// Once Command-V was sent, cancellation must not restore the previous
|
||||
/// clipboard while the target can still be reading this one. A clipboard
|
||||
/// observer may request the bytes before the target does, so an unidentified
|
||||
/// read cannot shorten the bounded consumption window. The window starts
|
||||
/// when sendPaste returns; after it ends, surface cancellation to the caller.
|
||||
/// A promised-data read ends the first stage, but does not identify the
|
||||
/// reader. Target observation below guards early clipboard restoration.
|
||||
/// After posting, cancellation is surfaced only after this bounded read and
|
||||
/// target-consumption interval so that a pending paste keeps its bytes.
|
||||
@MainActor
|
||||
func waitForRead(timeout: Duration, ownsClipboard: @MainActor () -> Bool) async throws {
|
||||
let deadline = ContinuousClock.now.advanced(by: timeout)
|
||||
@@ -151,16 +162,38 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
guard ownsClipboard() else {
|
||||
throw CUError("clipboard_changed", "The clipboard changed while waiting for paste consumption")
|
||||
}
|
||||
if hasSuppliedData { return }
|
||||
let remaining = ContinuousClock.now.duration(to: deadline)
|
||||
guard remaining > .zero else {
|
||||
try Task.checkCancellation()
|
||||
if state.withLock({ $0.suppliedAt != nil }) { return }
|
||||
throw CUError("clipboard_read_timeout", "No pasteboard data read was observed within the paste deadline; inspect the target before retrying")
|
||||
}
|
||||
await Self.pause(for: min(.milliseconds(10), remaining))
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private func waitForTarget(
|
||||
_ observation: ClipboardPasteObservation,
|
||||
timeout: Duration,
|
||||
ownsClipboard: @MainActor () -> Bool
|
||||
) async throws {
|
||||
let deadline = ContinuousClock.now.advanced(by: observation.hasSignals ? timeout : .milliseconds(100))
|
||||
while true {
|
||||
guard ownsClipboard() else {
|
||||
throw CUError("clipboard_changed", "The clipboard changed while waiting for the paste target")
|
||||
}
|
||||
if observation.hasSignals && observation.hasChanged() { return }
|
||||
let remaining = ContinuousClock.now.duration(to: deadline)
|
||||
guard remaining > .zero else {
|
||||
try Task.checkCancellation()
|
||||
guard observation.hasSignals else { return }
|
||||
throw CUError("clipboard_target_timeout", "Pasteboard bytes were read, but no change in the target field was observed; inspect the target before retrying")
|
||||
}
|
||||
await Self.pause(for: min(.milliseconds(25), remaining))
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private static func pause(for duration: Duration) async {
|
||||
let seconds = milliseconds(duration) / 1_000
|
||||
@@ -177,3 +210,144 @@ final class ClipboardPasteReceipt: NSObject, NSPasteboardItemDataProvider, @unch
|
||||
return Double(parts.seconds) * 1_000 + Double(parts.attoseconds) / 1e15
|
||||
}
|
||||
}
|
||||
|
||||
/// The target's AX signals, captured before writing the temporary pasteboard.
|
||||
/// Tests can supply deterministic signals while keeping real named pasteboards.
|
||||
@MainActor
|
||||
struct ClipboardPasteObservation {
|
||||
let hasSignals: Bool
|
||||
var arm: (ClipboardPasteReceipt) -> Void = { _ in }
|
||||
var hasChanged: () -> Bool = { false }
|
||||
var close: () -> Void = {}
|
||||
|
||||
static func capture(pid: pid_t?) -> ClipboardPasteObservation {
|
||||
guard let pid, pid > 0 else { return ClipboardPasteObservation(hasSignals: false) }
|
||||
let target = NativeClipboardPasteObservation(pid: pid)
|
||||
return ClipboardPasteObservation(
|
||||
hasSignals: target.hasSignals,
|
||||
arm: { target.notifications.arm($0) },
|
||||
hasChanged: { target.hasChanged() },
|
||||
close: { target.close() }
|
||||
)
|
||||
}
|
||||
|
||||
static func attributesChanged(baseline: [String: CFTypeRef], current: [String: CFTypeRef]) -> Bool {
|
||||
guard !baseline.isEmpty, current.count == baseline.count,
|
||||
baseline.keys.allSatisfy({ current[$0] != nil }) else { return false }
|
||||
return baseline.contains { name, value in
|
||||
guard let latest = current[name] else { return false }
|
||||
return !CFEqual(value, latest)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// AXObserver's callback only acknowledges this target after the pasteboard is
|
||||
/// armed and its promised bytes have been supplied. A clipboard observer alone
|
||||
/// cannot set this flag, and an earlier AX notification is not replayed later.
|
||||
final class ClipboardPasteNotifications: @unchecked Sendable {
|
||||
private struct State {
|
||||
var receipt: ClipboardPasteReceipt?
|
||||
var changed = false
|
||||
}
|
||||
private let state = OSAllocatedUnfairLock(initialState: State())
|
||||
|
||||
func arm(_ receipt: ClipboardPasteReceipt) {
|
||||
state.withLock { $0.receipt = receipt }
|
||||
}
|
||||
|
||||
func recordTargetChange() {
|
||||
state.withLock { value in
|
||||
if value.receipt?.hasSuppliedData == true { value.changed = true }
|
||||
}
|
||||
}
|
||||
|
||||
var hasChanged: Bool { state.withLock { $0.changed } }
|
||||
|
||||
func close() {
|
||||
state.withLock { $0.receipt = nil }
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private final class NativeClipboardPasteObservation {
|
||||
let notifications = ClipboardPasteNotifications()
|
||||
private let focused: AXUIElement?
|
||||
private let baseline: [String: CFTypeRef]
|
||||
private var observer: AXObserver?
|
||||
private var registered: [CFString] = []
|
||||
|
||||
var hasSignals: Bool { !baseline.isEmpty || !registered.isEmpty }
|
||||
|
||||
init(pid: pid_t) {
|
||||
let app = AXUIElementCreateApplication(pid)
|
||||
var focusedValue: CFTypeRef?
|
||||
if AXUIElementCopyAttributeValue(app, kAXFocusedUIElementAttribute as CFString, &focusedValue) == .success,
|
||||
let focusedValue, CFGetTypeID(focusedValue) == AXUIElementGetTypeID() {
|
||||
let element = unsafeBitCast(focusedValue, to: AXUIElement.self)
|
||||
var elementPID: pid_t = 0
|
||||
focused = AXUIElementGetPid(element, &elementPID) == .success && elementPID == pid ? element : nil
|
||||
} else {
|
||||
focused = nil
|
||||
}
|
||||
guard let focused else {
|
||||
baseline = [:]
|
||||
return
|
||||
}
|
||||
var attributeNames: CFArray?
|
||||
let names: [String]
|
||||
if AXUIElementCopyAttributeNames(focused, &attributeNames) == .success {
|
||||
names = attributeNames as? [String] ?? []
|
||||
} else {
|
||||
names = []
|
||||
}
|
||||
let attributes = [kAXSelectedTextRangeAttribute, kAXNumberOfCharactersAttribute].filter { names.contains($0) }
|
||||
baseline = Self.read(focused, attributes: attributes)
|
||||
|
||||
var created: AXObserver?
|
||||
guard AXObserverCreate(pid, { _, _, _, context in
|
||||
guard let context else { return }
|
||||
Unmanaged<ClipboardPasteNotifications>.fromOpaque(context).takeUnretainedValue().recordTargetChange()
|
||||
}, &created) == .success, let created else { return }
|
||||
let context = Unmanaged.passUnretained(notifications).toOpaque()
|
||||
for name in [kAXSelectedTextChangedNotification, kAXValueChangedNotification] {
|
||||
if AXObserverAddNotification(created, focused, name as CFString, context) == .success {
|
||||
registered.append(name as CFString)
|
||||
}
|
||||
}
|
||||
if !registered.isEmpty {
|
||||
observer = created
|
||||
CFRunLoopAddSource(CFRunLoopGetMain(), AXObserverGetRunLoopSource(created), .commonModes)
|
||||
}
|
||||
}
|
||||
|
||||
func hasChanged() -> Bool {
|
||||
if notifications.hasChanged { return true }
|
||||
guard let focused, !baseline.isEmpty else { return false }
|
||||
let current = Self.read(focused, attributes: Array(baseline.keys))
|
||||
// Failed AX reads do not confirm consumption.
|
||||
return ClipboardPasteObservation.attributesChanged(baseline: baseline, current: current)
|
||||
}
|
||||
|
||||
func close() {
|
||||
if let observer {
|
||||
CFRunLoopRemoveSource(CFRunLoopGetMain(), AXObserverGetRunLoopSource(observer), .commonModes)
|
||||
if let focused {
|
||||
for name in registered { AXObserverRemoveNotification(observer, focused, name) }
|
||||
}
|
||||
}
|
||||
observer = nil
|
||||
registered = []
|
||||
notifications.close()
|
||||
}
|
||||
|
||||
private static func read(_ element: AXUIElement, attributes: [String]) -> [String: CFTypeRef] {
|
||||
var result: [String: CFTypeRef] = [:]
|
||||
for name in attributes {
|
||||
var value: CFTypeRef?
|
||||
if AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success, let value {
|
||||
result[name] = value
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
@@ -67,11 +67,13 @@ enum SnapshotProcessGuard {
|
||||
/// serialises to `{ ok: false, error: { message, code } }`.
|
||||
@MainActor
|
||||
public final class CommandRouter {
|
||||
typealias PasteExecutor = @MainActor (pid_t, String, ClipboardPasteFormat) async throws -> Void
|
||||
private let cursor: VirtualCursor
|
||||
private let capabilities: Capabilities
|
||||
private let inputMonitor: PhysicalInputEpochMonitor
|
||||
private let foregroundRuntime: ForegroundLeaseRuntime
|
||||
private let windowCaptureProvider: (any WindowCaptureProviding)?
|
||||
private let pasteExecutor: PasteExecutor
|
||||
|
||||
/// After a left `mouse_down` (decomposed drag) the held point is parked
|
||||
/// here so a following `mouse_up` releases at the same logical location.
|
||||
@@ -84,13 +86,17 @@ public final class CommandRouter {
|
||||
cursor: VirtualCursor,
|
||||
capabilities: Capabilities,
|
||||
inputMonitor: PhysicalInputEpochMonitor,
|
||||
windowCaptureProvider: (any WindowCaptureProviding)? = nil
|
||||
windowCaptureProvider: (any WindowCaptureProviding)? = nil,
|
||||
pasteExecutor: @escaping PasteExecutor = { pid, text, format in
|
||||
try await AXAction.pasteText(pid: pid, text, format: format)
|
||||
}
|
||||
) {
|
||||
self.cursor = cursor
|
||||
self.capabilities = capabilities
|
||||
self.inputMonitor = inputMonitor
|
||||
self.foregroundRuntime = .live(monitor: inputMonitor)
|
||||
self.windowCaptureProvider = windowCaptureProvider
|
||||
self.pasteExecutor = pasteExecutor
|
||||
}
|
||||
|
||||
func resetHeldSessionState() {
|
||||
@@ -229,9 +235,10 @@ public final class CommandRouter {
|
||||
case "list_installed_apps":
|
||||
return try encode(Apps.listInstalled())
|
||||
|
||||
case "list_running_apps", "list_apps":
|
||||
// `list_apps` is the Codex-parity contract name; `list_running_apps`
|
||||
// is the legacy bridge name. Both enumerate targetable running apps.
|
||||
case "list_apps":
|
||||
return try encode(await Apps.listApps())
|
||||
|
||||
case "list_running_apps":
|
||||
return try encode(Apps.listRunning())
|
||||
|
||||
case "resolve_app_target":
|
||||
@@ -747,7 +754,7 @@ public final class CommandRouter {
|
||||
pid: pid,
|
||||
processIdentity: snapshotEvidence.processIdentity,
|
||||
preferredWindowID: snapshotEvidence.keyWindowID,
|
||||
scale: 0.5,
|
||||
scale: nil,
|
||||
newerThanUptime: pendingMutation
|
||||
)
|
||||
}
|
||||
@@ -782,7 +789,7 @@ public final class CommandRouter {
|
||||
shot = await Capture.windowShot(
|
||||
pid: pid,
|
||||
preferredWindowID: snapshotEvidence.keyWindowID,
|
||||
scale: 0.5
|
||||
scale: nil
|
||||
)
|
||||
} else {
|
||||
// A one-shot capture can repeat compositor-cached pixels for a
|
||||
@@ -847,8 +854,9 @@ public final class CommandRouter {
|
||||
) {
|
||||
Self.appendAXNotice(notice, to: &object)
|
||||
}
|
||||
object["screenshot"] = .object([
|
||||
var screenshot: [String: JSONValue] = [
|
||||
"base64": .string(shot.base64),
|
||||
"mimeType": .string(shot.mimeType),
|
||||
"width": .int(shot.width),
|
||||
"height": .int(shot.height),
|
||||
"originX": .double(shot.originX),
|
||||
@@ -857,7 +865,11 @@ public final class CommandRouter {
|
||||
"pointHeight": .double(shot.pointHeight),
|
||||
"windowID": .int(Int(shot.windowID)),
|
||||
"captureSource": .string(shot.source.rawValue),
|
||||
])
|
||||
]
|
||||
if let pixelsPerPoint = shot.pixelsPerPoint {
|
||||
screenshot["pixelsPerPoint"] = .double(pixelsPerPoint)
|
||||
}
|
||||
object["screenshot"] = .object(screenshot)
|
||||
// Cache the inverse transform so a later coordinate click/scroll/
|
||||
// drag (which arrives in image-pixel space) can be mapped back to
|
||||
// global points. `ppp` = image pixels per window point.
|
||||
@@ -871,7 +883,8 @@ public final class CommandRouter {
|
||||
imageWidth: shot.width,
|
||||
imageHeight: shot.height,
|
||||
processIdentity: snapshotEvidence.processIdentity,
|
||||
windowID: shot.windowID
|
||||
windowID: shot.windowID,
|
||||
pixelsPerPoint: shot.pixelsPerPoint
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -932,7 +945,8 @@ public final class CommandRouter {
|
||||
imageWidth: Int,
|
||||
imageHeight: Int,
|
||||
processIdentity: AXTreeProcessIdentity,
|
||||
windowID: CGWindowID
|
||||
windowID: CGWindowID,
|
||||
pixelsPerPoint: Double? = nil
|
||||
) {
|
||||
guard pid > 0,
|
||||
processIdentity.isProven,
|
||||
@@ -943,6 +957,7 @@ public final class CommandRouter {
|
||||
pointHeight.isFinite,
|
||||
pointWidth > 0,
|
||||
pointHeight > 0,
|
||||
pixelsPerPoint.map({ $0.isFinite && $0 > 0 }) ?? true,
|
||||
imageWidth > 0,
|
||||
imageHeight > 0 else {
|
||||
lastShotTransform.removeValue(forKey: pid)
|
||||
@@ -951,8 +966,8 @@ public final class CommandRouter {
|
||||
let transform = ShotTransform(
|
||||
originX: originX,
|
||||
originY: originY,
|
||||
pixelsPerPointX: Double(imageWidth) / pointWidth,
|
||||
pixelsPerPointY: Double(imageHeight) / pointHeight,
|
||||
pixelsPerPointX: pixelsPerPoint ?? Double(imageWidth) / pointWidth,
|
||||
pixelsPerPointY: pixelsPerPoint ?? Double(imageHeight) / pointHeight,
|
||||
imageWidth: imageWidth,
|
||||
imageHeight: imageHeight,
|
||||
processIdentity: processIdentity,
|
||||
@@ -1181,7 +1196,11 @@ public final class CommandRouter {
|
||||
y: y,
|
||||
pid: target.pid
|
||||
)
|
||||
await cursor.moveForAction(to: g, targetPid: target.pid)
|
||||
// Official coordinate click/drag dispatch does not await a
|
||||
// cursor animation. Start feedback without delaying input.
|
||||
await cursor.moveForAction(
|
||||
to: g, targetPid: target.pid, waitForVisualFeedback: false
|
||||
)
|
||||
_ = try Self.validatedGlobalPoint(
|
||||
x: x,
|
||||
y: y,
|
||||
@@ -1471,7 +1490,7 @@ public final class CommandRouter {
|
||||
return try await withForegroundLease(command: "paste", target: target) {
|
||||
_ = try Injection.validateAuthorizedTarget(target)
|
||||
try self.requireSnapshotProcess(target: target, expected: expected)
|
||||
try await AXAction.pasteText(pid: target.pid, text, format: format)
|
||||
try await pasteExecutor(target.pid, text, format)
|
||||
return .bool(true)
|
||||
}
|
||||
}
|
||||
@@ -1545,7 +1564,9 @@ public final class CommandRouter {
|
||||
pid: target.pid
|
||||
)
|
||||
await cursor.move(to: from, animated: false)
|
||||
await cursor.moveForAction(to: to, targetPid: target.pid)
|
||||
await cursor.moveForAction(
|
||||
to: to, targetPid: target.pid, waitForVisualFeedback: false
|
||||
)
|
||||
|
||||
_ = try Self.validatedGlobalPoint(
|
||||
x: rawFrom.x,
|
||||
|
||||
@@ -56,10 +56,12 @@ struct CursorMotionState: Equatable, Sendable {
|
||||
|
||||
@MainActor
|
||||
enum CursorActionTiming {
|
||||
/// Start a foreground glide, then wait only the bounded action delay. A
|
||||
/// background/unrequested action snaps and does not call `sleep` at all.
|
||||
/// Start visible feedback immediately. Coordinate gestures opt out of the
|
||||
/// bounded readability delay; indexed actions retain their current policy.
|
||||
/// Hidden/unrequested feedback snaps without calling `sleep`.
|
||||
static func perform(
|
||||
decision: OverlayPolicy.Decision,
|
||||
waitForVisualFeedback: Bool = true,
|
||||
startGlide: () -> Void,
|
||||
snap: () -> Void,
|
||||
sleep: (TimeInterval) async -> Void
|
||||
@@ -69,7 +71,7 @@ enum CursorActionTiming {
|
||||
return
|
||||
}
|
||||
startGlide()
|
||||
if decision.actionDelay > 0 {
|
||||
if waitForVisualFeedback, decision.actionDelay > 0 {
|
||||
await sleep(decision.actionDelay)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -169,6 +169,12 @@ private enum HeldState {
|
||||
@MainActor
|
||||
public enum Injection {
|
||||
|
||||
#if DEBUG
|
||||
/// Test-only observation of the exact immutable values used by the safety
|
||||
/// check. It cannot replace either identity or change validation policy.
|
||||
static var targetValidationObserver: ((ProvenProcessTarget, AXTreeProcessIdentity?) -> Void)?
|
||||
#endif
|
||||
|
||||
/// One shared event source per process, in the HID system state so the
|
||||
/// synthesized events inherit the real keyboard/modifier baseline. May be
|
||||
/// nil in extremely locked-down sandboxes; call sites throw `event_alloc`.
|
||||
@@ -354,6 +360,9 @@ public enum Injection {
|
||||
_ target: ProvenProcessTarget
|
||||
) throws -> ProvenProcessTarget {
|
||||
let currentIdentity = AXTree.currentProcessIdentity(pid: target.pid)
|
||||
#if DEBUG
|
||||
targetValidationObserver?(target, currentIdentity)
|
||||
#endif
|
||||
guard target.validatedPid(currentIdentity: currentIdentity) != nil else {
|
||||
throw CUError(
|
||||
"stale_process",
|
||||
|
||||
@@ -147,6 +147,15 @@ public enum KeyMapping {
|
||||
}
|
||||
|
||||
let keyCode = try keyCode(for: keyToken)
|
||||
// XKeysym names denote the resulting key, including the modifier
|
||||
// required to produce it. The official macOS receiver gets Shift for
|
||||
// both `A` and `question`; dropping it changes the requested input.
|
||||
if shiftedNamedKeys.contains(keyToken.lowercased()) {
|
||||
flags.insert(.maskShift)
|
||||
} else if keyToken.count == 1, let character = keyToken.first,
|
||||
let mapping = characterMapping(character) {
|
||||
flags.formUnion(mapping.flags)
|
||||
}
|
||||
return Chord(
|
||||
keyCode: keyCode,
|
||||
flags: flags,
|
||||
@@ -195,13 +204,13 @@ public enum KeyMapping {
|
||||
/// not a modifier. Matches the xdotool + mac_helper alias vocabulary.
|
||||
public static func modifierFlag(for token: String) -> CGEventFlags? {
|
||||
switch token.lowercased() {
|
||||
case "super", "cmd", "command", "meta", "win":
|
||||
case "super", "super_l", "super_r", "cmd", "command", "meta", "meta_l", "meta_r", "win":
|
||||
return .maskCommand
|
||||
case "ctrl", "control":
|
||||
case "ctrl", "control", "control_l", "control_r":
|
||||
return .maskControl
|
||||
case "shift":
|
||||
case "shift", "shift_l", "shift_r":
|
||||
return .maskShift
|
||||
case "alt", "option", "opt":
|
||||
case "alt", "alt_l", "alt_r", "option", "opt":
|
||||
return .maskAlternate
|
||||
case "fn":
|
||||
return .maskSecondaryFn
|
||||
@@ -237,8 +246,8 @@ public enum KeyMapping {
|
||||
put(kVK_Space, "space", "spacebar")
|
||||
put(kVK_Escape, "escape", "esc")
|
||||
// kVK_Delete is the Backspace (delete-left) key in Carbon naming.
|
||||
put(kVK_Delete, "backspace", "back_space", "delete")
|
||||
put(kVK_ForwardDelete, "forwarddelete", "forward_delete", "del", "deletef")
|
||||
put(kVK_Delete, "backspace", "back_space")
|
||||
put(kVK_ForwardDelete, "delete", "forwarddelete", "forward_delete", "del", "deletef")
|
||||
|
||||
// ── Navigation ────────────────────────────────────────────────────
|
||||
put(kVK_UpArrow, "up", "uparrow", "up_arrow")
|
||||
@@ -298,10 +307,27 @@ public enum KeyMapping {
|
||||
put(kVK_ANSI_Comma, "comma", "less")
|
||||
put(kVK_ANSI_Period, "period", "greater")
|
||||
put(kVK_ANSI_Slash, "slash", "question")
|
||||
put(kVK_ANSI_1, "exclam")
|
||||
put(kVK_ANSI_2, "at")
|
||||
put(kVK_ANSI_3, "numbersign")
|
||||
put(kVK_ANSI_4, "dollar")
|
||||
put(kVK_ANSI_5, "percent")
|
||||
put(kVK_ANSI_6, "asciicircum")
|
||||
put(kVK_ANSI_7, "ampersand")
|
||||
put(kVK_ANSI_8, "asterisk")
|
||||
put(kVK_ANSI_9, "parenleft")
|
||||
put(kVK_ANSI_0, "parenright")
|
||||
|
||||
return t
|
||||
}()
|
||||
|
||||
private static let shiftedNamedKeys: Set<String> = [
|
||||
"asciitilde", "underscore", "plus", "braceleft", "braceright",
|
||||
"bar", "colon", "quotedbl", "less", "greater", "question",
|
||||
"exclam", "at", "numbersign", "dollar", "percent", "asciicircum",
|
||||
"ampersand", "asterisk", "parenleft", "parenright",
|
||||
]
|
||||
|
||||
// MARK: - Layout reverse-map
|
||||
|
||||
/// Reverse-map a single character to the physical keycode that produces it
|
||||
@@ -311,13 +337,13 @@ public enum KeyMapping {
|
||||
/// output equals `ch`. Returns `nil` for characters not reachable on the
|
||||
/// active layout (the caller then throws `unknown_key`).
|
||||
///
|
||||
/// Note: only the bare keycode is returned — we do NOT fold in the shift/
|
||||
/// option modifier that was needed to *produce* the glyph during the scan.
|
||||
/// The injection layer types literal text via `keyboardSetUnicodeString`
|
||||
/// (see Injection.type); `press_key` is for addressing physical keys and
|
||||
/// chords, so returning the unshifted physical key is the correct behavior
|
||||
/// for tokens like "/" or ";".
|
||||
/// Callers needing a physical code use this compatibility wrapper. Chord
|
||||
/// parsing also keeps the layout modifiers from `characterMapping`.
|
||||
static func keyCodeForCharacter(_ ch: Character) -> CGKeyCode? {
|
||||
characterMapping(ch)?.keyCode
|
||||
}
|
||||
|
||||
private static func characterMapping(_ ch: Character) -> (keyCode: CGKeyCode, flags: CGEventFlags)? {
|
||||
guard
|
||||
let inputSource = TISCopyCurrentKeyboardLayoutInputSource()?.takeRetainedValue(),
|
||||
let layoutPtr = TISGetInputSourceProperty(inputSource, kTISPropertyUnicodeKeyLayoutData)
|
||||
@@ -361,7 +387,10 @@ public enum KeyMapping {
|
||||
if status == noErr, length > 0 {
|
||||
let produced = String(utf16CodeUnits: chars, count: length)
|
||||
if produced == target {
|
||||
return CGKeyCode(vk)
|
||||
var flags: CGEventFlags = []
|
||||
if mod & UInt32(shiftKey >> 8) != 0 { flags.insert(.maskShift) }
|
||||
if mod & UInt32(optionKey >> 8) != 0 { flags.insert(.maskAlternate) }
|
||||
return (CGKeyCode(vk), flags)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import CoreGraphics
|
||||
|
||||
/// Native App screenshot defaults from the official macOS controller:
|
||||
/// normalize Retina pixels to points, then fit long side <= 2048 and short
|
||||
/// side <= 768 without enlarging. Pixel buffers round dimensions upward.
|
||||
/// The image transform still covers the entire original window frame.
|
||||
enum NativeScreenshotPolicy {
|
||||
static let jpegQuality = 0.8
|
||||
static let mimeType = "image/jpeg"
|
||||
|
||||
static func scale(pointSize: CGSize, backingScale: Double) -> Double {
|
||||
let width = pointSize.width
|
||||
let height = pointSize.height
|
||||
guard width.isFinite, height.isFinite, width > 0, height > 0,
|
||||
backingScale.isFinite, backingScale > 0 else { return 1 }
|
||||
let fit = min(1, 2048 / max(width, height), 768 / min(width, height))
|
||||
return fit / backingScale
|
||||
}
|
||||
|
||||
static func pixelSize(pointSize: CGSize, backingScale: Double) -> CGSize {
|
||||
let factor = scale(pointSize: pointSize, backingScale: backingScale) * backingScale
|
||||
return CGSize(width: max(1, ceil(pointSize.width * factor)), height: max(1, ceil(pointSize.height * factor)))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import CoreGraphics
|
||||
|
||||
enum NativeScroll {
|
||||
static func pageNavigation(action: String) -> (axisAttribute: String, buttonSubrole: String)? {
|
||||
switch action {
|
||||
case "AXScrollUpByPage": return ("AXVerticalScrollBar", "AXDecrementPage")
|
||||
case "AXScrollDownByPage": return ("AXVerticalScrollBar", "AXIncrementPage")
|
||||
case "AXScrollLeftByPage": return ("AXHorizontalScrollBar", "AXDecrementPage")
|
||||
case "AXScrollRightByPage": return ("AXHorizontalScrollBar", "AXIncrementPage")
|
||||
default: return nil
|
||||
}
|
||||
}
|
||||
|
||||
/// Native App.scroll pages use the addressed element's frame; a coordinate
|
||||
/// target uses the whole window frame even when it hits a nested scroll view.
|
||||
static func delta(direction: String, pages: Double, frameSize: CGSize) throws -> (x: Int32, y: Int32) {
|
||||
let horizontal = direction == "left" || direction == "right"
|
||||
guard horizontal || direction == "up" || direction == "down",
|
||||
pages.isFinite, pages > 0 else {
|
||||
throw CUError("bad_payload", "Scroll direction and pages must be valid")
|
||||
}
|
||||
let extent = horizontal ? frameSize.width : frameSize.height
|
||||
let pixels = extent * pages
|
||||
guard extent.isFinite, extent > 0, pixels.isFinite else {
|
||||
throw CUError("bad_payload", "Scroll frame and distance must be finite and positive")
|
||||
}
|
||||
let magnitude = Int32(min(Double(Int32.max), pixels.rounded(.toNearestOrAwayFromZero)))
|
||||
let signed = direction == "down" || direction == "right" ? -magnitude : magnitude
|
||||
return horizontal ? (signed, 0) : (0, signed)
|
||||
}
|
||||
}
|
||||
@@ -180,10 +180,12 @@ public final class VirtualCursor {
|
||||
await runGlide(to: target, duration: duration)
|
||||
}
|
||||
|
||||
/// Action movement starts the owner-managed spring immediately but waits
|
||||
/// only the foreground policy delay. Background/unrequested actions snap and
|
||||
/// return without delay. The real OS cursor is never moved.
|
||||
public func moveForAction(to p: CGPoint, targetPid: pid_t) async {
|
||||
/// Start the owner-managed spring immediately. Coordinate gestures skip the
|
||||
/// visual readability wait; indexed actions retain the visible policy delay.
|
||||
/// Hidden/unrequested actions snap. The real OS cursor is never moved.
|
||||
public func moveForAction(
|
||||
to p: CGPoint, targetPid: pid_t, waitForVisualFeedback: Bool = true
|
||||
) async {
|
||||
let target = sanitized(p)
|
||||
if headless {
|
||||
motionState.snap(to: target)
|
||||
@@ -203,6 +205,7 @@ public final class VirtualCursor {
|
||||
let decision = currentVisualDecision()
|
||||
await CursorActionTiming.perform(
|
||||
decision: decision,
|
||||
waitForVisualFeedback: waitForVisualFeedback,
|
||||
startGlide: { [self] in
|
||||
if !startGlide(to: target) {
|
||||
snap(to: target)
|
||||
|
||||
@@ -27,6 +27,7 @@ struct WindowCaptureStreamTarget: Equatable, Sendable {
|
||||
let originY: Double
|
||||
let pointWidth: Double
|
||||
let pointHeight: Double
|
||||
var pixelsPerPoint: Double? = nil
|
||||
}
|
||||
|
||||
/// An immutable copy of the newest complete BGRA frame. The ScreenCaptureKit
|
||||
@@ -79,7 +80,7 @@ protocol WindowCaptureProviding: AnyObject {
|
||||
pid: pid_t,
|
||||
processIdentity: AXTreeProcessIdentity,
|
||||
preferredWindowID: CGWindowID?,
|
||||
scale: Double,
|
||||
scale: Double?,
|
||||
newerThanUptime: TimeInterval?
|
||||
) async -> WindowShot?
|
||||
|
||||
@@ -117,7 +118,7 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
private let factory: any WindowCaptureStreamSourceFactory
|
||||
private let frameWaitAttempts: Int
|
||||
private let frameWaitNanoseconds: UInt64
|
||||
private let takeSnapshot: (WindowCaptureStreamTarget, Double) async -> WindowShot?
|
||||
private let takeSnapshot: (WindowCaptureStreamTarget, Double?) async -> WindowShot?
|
||||
private var generation: UInt64 = 0
|
||||
private var starting: Entry?
|
||||
private var active: Entry?
|
||||
@@ -130,7 +131,7 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
factory: any WindowCaptureStreamSourceFactory,
|
||||
frameWaitAttempts: Int = 12,
|
||||
frameWaitNanoseconds: UInt64 = 50_000_000,
|
||||
takeSnapshot: @escaping (WindowCaptureStreamTarget, Double) async -> WindowShot? = { target, scale in
|
||||
takeSnapshot: @escaping (WindowCaptureStreamTarget, Double?) async -> WindowShot? = { target, scale in
|
||||
await Capture.windowShot(
|
||||
pid: target.key.pid,
|
||||
preferredWindowID: target.key.windowID,
|
||||
@@ -149,7 +150,7 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
pid: pid_t,
|
||||
processIdentity: AXTreeProcessIdentity,
|
||||
preferredWindowID: CGWindowID?,
|
||||
scale: Double,
|
||||
scale: Double?,
|
||||
newerThanUptime: TimeInterval?
|
||||
) async -> WindowShot? {
|
||||
guard Capture.hasScreenRecordingPermission(),
|
||||
@@ -208,7 +209,8 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
base64: shot.base64, width: shot.width, height: shot.height,
|
||||
originX: current.originX, originY: current.originY,
|
||||
pointWidth: current.pointWidth, pointHeight: current.pointHeight,
|
||||
windowID: current.key.windowID, source: .streamBackedScreenshot
|
||||
windowID: current.key.windowID, source: .streamBackedScreenshot,
|
||||
pixelsPerPoint: shot.pixelsPerPoint
|
||||
)
|
||||
}
|
||||
return nil
|
||||
@@ -220,7 +222,7 @@ final class WindowCaptureStreamManager: WindowCaptureProviding {
|
||||
/// pixel frame before its on-demand screenshot may be treated as live.
|
||||
func captureSnapshot(
|
||||
for target: WindowCaptureStreamTarget,
|
||||
scale: Double,
|
||||
scale: Double?,
|
||||
newerThanUptime: TimeInterval? = nil
|
||||
) async -> WindowShot? {
|
||||
// The stream is a long-lived render/freshness consumer, not the model
|
||||
@@ -797,7 +799,7 @@ extension Capture {
|
||||
pid: pid_t,
|
||||
processIdentity: AXTreeProcessIdentity,
|
||||
preferredWindowID: CGWindowID?,
|
||||
scale: Double
|
||||
scale: Double?
|
||||
) -> WindowCaptureStreamTarget? {
|
||||
guard processIdentity.isProven,
|
||||
let candidate = bestWindow(
|
||||
@@ -809,8 +811,10 @@ extension Capture {
|
||||
let frame = candidate.frame
|
||||
guard frame.width > 1, frame.height > 1 else { return nil }
|
||||
|
||||
let outputScale = scale > 0 ? scale : 0.5
|
||||
let backingScale = backingScaleFactor(forWindowFrame: frame)
|
||||
let outputScale = scale.flatMap { $0 > 0 ? $0 : nil } ?? NativeScreenshotPolicy.scale(
|
||||
pointSize: frame.size, backingScale: backingScale
|
||||
)
|
||||
let width = max(1, Int(ceil(frame.width * backingScale * outputScale)))
|
||||
let height = max(1, Int(ceil(frame.height * backingScale * outputScale)))
|
||||
return WindowCaptureStreamTarget(
|
||||
@@ -824,7 +828,8 @@ extension Capture {
|
||||
originX: Double(frame.origin.x),
|
||||
originY: Double(frame.origin.y),
|
||||
pointWidth: Double(frame.width),
|
||||
pointHeight: Double(frame.height)
|
||||
pointHeight: Double(frame.height),
|
||||
pixelsPerPoint: backingScale * outputScale
|
||||
)
|
||||
}
|
||||
|
||||
@@ -835,7 +840,7 @@ extension Capture {
|
||||
guard frame.width == target.key.pixelWidth,
|
||||
frame.height == target.key.pixelHeight,
|
||||
let image = image(from: frame),
|
||||
let encoded = pngBase64WithSize(image) else {
|
||||
let encoded = appScreenshotBase64WithSize(image) else {
|
||||
return nil
|
||||
}
|
||||
return WindowShot(
|
||||
@@ -847,7 +852,8 @@ extension Capture {
|
||||
pointWidth: target.pointWidth,
|
||||
pointHeight: target.pointHeight,
|
||||
windowID: target.key.windowID,
|
||||
source: .stream
|
||||
source: .stream,
|
||||
pixelsPerPoint: target.pixelsPerPoint
|
||||
)
|
||||
}
|
||||
|
||||
|
||||
@@ -17,9 +17,9 @@ import Foundation
|
||||
///
|
||||
/// The fix is to stamp the window identity onto the event before posting:
|
||||
///
|
||||
/// 1. `+[NSEvent mouseEventWithType:…windowNumber:…]` — the ONLY way to set a
|
||||
/// window number, since CGEvent exposes no public API for it. Take the
|
||||
/// resulting event's `.cgEvent`.
|
||||
/// 1. Mouse events use `+[NSEvent mouseEventWithType:…windowNumber:…]` and its
|
||||
/// `.cgEvent`. Scroll events use the official private CGEvent field 51
|
||||
/// (windowNumber), confirmed by its initializer at 0x100714188.
|
||||
/// 2. `kCGMouseEventSubtype = 3` (NX_SUBTYPE_MOUSE_TOUCH) plus fields 91/92
|
||||
/// (`WindowUnderMousePointer` / `…ThatCanHandleThisEvent`) = the window ID.
|
||||
/// 3. `CGEventSetWindowLocation` with the point expressed in WINDOW-LOCAL
|
||||
@@ -267,6 +267,28 @@ enum WindowTargetedEvent {
|
||||
event.postToPid(pid)
|
||||
}
|
||||
|
||||
static func makeScrollEvent(
|
||||
source: CGEventSource,
|
||||
point: CGPoint,
|
||||
deltaX: Int32,
|
||||
deltaY: Int32,
|
||||
window: WindowGeometry.Window
|
||||
) -> CGEvent? {
|
||||
// Keep two usable axes. The inspected official version passes one and
|
||||
// consequently loses horizontal deltas; reproducing that bug is not
|
||||
// required for its pixel/page contract.
|
||||
guard let event = CGEvent(scrollWheelEvent2Source: source, units: .pixel,
|
||||
wheelCount: 2, wheel1: deltaY, wheel2: deltaX, wheel3: 0) else { return nil }
|
||||
event.location = point
|
||||
event.setIntegerValueField(CGEventField(rawValue: 51)!, value: Int64(window.id))
|
||||
event.setIntegerValueField(windowUnderPointerField, value: Int64(window.id))
|
||||
event.setIntegerValueField(windowThatCanHandleField, value: Int64(window.id))
|
||||
if let stamp = setWindowLocation {
|
||||
stamp(event, windowLocalPoint(globalPoint: point, windowBounds: window.bounds))
|
||||
}
|
||||
return event
|
||||
}
|
||||
|
||||
// CGEventField has no public constants for these three.
|
||||
private static let mouseSubtypeField = CGEventField(rawValue: 7)!
|
||||
private static let windowUnderPointerField = CGEventField(rawValue: 91)!
|
||||
|
||||
@@ -13,6 +13,10 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
private static let fixtureStopPath = "CC_HAHA_AX_PUBLICATION_STOP"
|
||||
private static let fixtureTitle = "CC_HAHA_AX_PUBLICATION_TITLE"
|
||||
private static let fixtureMismatchedTitle = "CC_HAHA_AX_PUBLICATION_MISMATCHED_TITLE"
|
||||
private static let fixtureGesturePath = "CC_HAHA_AX_PUBLICATION_GESTURES"
|
||||
private static let fixtureRegularActivation = "CC_HAHA_AX_PUBLICATION_REGULAR_ACTIVATION"
|
||||
private static let fixtureWideWindow = "CC_HAHA_AX_PUBLICATION_WIDE_WINDOW"
|
||||
private static let fixtureMethods = "CC_HAHA_AX_PUBLICATION_METHOD_CONTRACT"
|
||||
|
||||
func testPublishedControlBelowDuplicateAncestorClicksImmediatelyAndRejectsOldGeneration() async throws {
|
||||
try await verifyPublishedControl(mismatchedWindowTitle: false)
|
||||
@@ -22,10 +26,160 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
try await verifyPublishedControl(mismatchedWindowTitle: true)
|
||||
}
|
||||
|
||||
private func verifyPublishedControl(mismatchedWindowTitle: Bool) async throws {
|
||||
func testConsecutiveZeroAndOnePixelDragsReachTheAppWithoutIntermediateObservations() async throws {
|
||||
try await verifyPublishedControl(mismatchedWindowTitle: false, exerciseDrags: true)
|
||||
}
|
||||
|
||||
func testOfficialKeyboardAliasesAndMacroReachTheNativeReceiver() async throws {
|
||||
try await verifyPublishedControl(mismatchedWindowTitle: false, exerciseKeys: true)
|
||||
}
|
||||
|
||||
func testWideWindowUsesOfficialScreenshotSizeAndKeepsDragCoordinatesAligned() async throws {
|
||||
try await verifyPublishedControl(mismatchedWindowTitle: false, exerciseDrags: true, wideWindow: true)
|
||||
}
|
||||
|
||||
func testTextScrollAndSecondaryMethodsReachTheSameOfficialReceiver() async throws {
|
||||
if ProcessInfo.processInfo.environment[Self.fixtureFlag] == "1" {
|
||||
try await runFixtureProcess(mismatchedWindowTitle: false, regularActivation: true, wideWindow: false)
|
||||
exit(0)
|
||||
}
|
||||
try XCTSkipUnless(AXIsProcessTrusted(), "Native contract receiver requires Accessibility permission")
|
||||
try XCTSkipUnless(Capture.hasScreenRecordingPermission(), "Coordinate scroll requires Screen Recording permission")
|
||||
let root = FileManager.default.temporaryDirectory.appendingPathComponent("cc-haha-method-contract-\(UUID().uuidString)")
|
||||
try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
|
||||
defer { try? FileManager.default.removeItem(at: root) }
|
||||
let ready = root.appendingPathComponent("ready")
|
||||
let stop = root.appendingPathComponent("stop")
|
||||
let receiptPath = root.appendingPathComponent("contract.json")
|
||||
let pasteboardName = "cc-haha-method-contract-\(UUID().uuidString)"
|
||||
let pasteboard = NSPasteboard(name: NSPasteboard.Name(pasteboardName))
|
||||
pasteboard.clearContents()
|
||||
pasteboard.setString("disposable original clipboard", forType: .string)
|
||||
defer { pasteboard.releaseGlobally() }
|
||||
let app = try makeFixtureApp(in: root)
|
||||
let configuration = NSWorkspace.OpenConfiguration()
|
||||
configuration.arguments = Array(CommandLine.arguments.dropFirst())
|
||||
configuration.environment = ProcessInfo.processInfo.environment.merging([
|
||||
Self.fixtureFlag: "1", Self.fixtureMethods: "1",
|
||||
Self.fixtureReadyPath: ready.path, Self.fixtureStopPath: stop.path,
|
||||
Self.fixtureTitle: "Native method contract fixture",
|
||||
"CC_HAHA_METHOD_FIXTURE_PASTEBOARD": pasteboardName,
|
||||
]) { _, value in value }
|
||||
configuration.activates = false
|
||||
configuration.createsNewApplicationInstance = true
|
||||
let process = try await NSWorkspace.shared.openApplication(at: app, configuration: configuration)
|
||||
defer {
|
||||
FileManager.default.createFile(atPath: stop.path, contents: Data())
|
||||
if !process.isTerminated { process.terminate() }
|
||||
}
|
||||
try await waitUntil(description: "method fixture ready") { FileManager.default.fileExists(atPath: ready.path) && FileManager.default.fileExists(atPath: receiptPath.path) }
|
||||
let pid = process.processIdentifier
|
||||
defer { AXTree.invalidate(pid: pid) }
|
||||
let monitor = PhysicalInputEpochMonitor(counterReader: { _ in 0 })
|
||||
_ = monitor.startAndWait()
|
||||
defer { monitor.stop() }
|
||||
let cursor = VirtualCursor(headless: false)
|
||||
defer { cursor.hide() }
|
||||
let router = CommandRouter(cursor: cursor, capabilities: Capabilities(headless: false), inputMonitor: monitor,
|
||||
pasteExecutor: { pid, text, format in
|
||||
try await AXAction.pasteText(pid: pid, text, format: format, lease: ClipboardLease(pasteboard: pasteboard))
|
||||
})
|
||||
func state() async throws -> AXTree.Result { try await AXTree.appState(pid: pid, disableDiff: true) }
|
||||
func receipt() -> [String: Any] {
|
||||
guard let data = try? Data(contentsOf: receiptPath),
|
||||
let value = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { return [:] }
|
||||
return value
|
||||
}
|
||||
var coordinateCaptureDiagnostic = "not captured"
|
||||
func diagnostic() -> String {
|
||||
let windows = (CGWindowListCopyWindowInfo(.optionAll, kCGNullWindowID) as? [[String: Any]] ?? [])
|
||||
.filter { $0[kCGWindowOwnerPID as String] as? Int == Int(pid) }
|
||||
.map { ["id": $0[kCGWindowNumber as String] ?? "nil", "bounds": $0[kCGWindowBounds as String] ?? "nil", "onScreen": $0[kCGWindowIsOnscreen as String] ?? "nil", "layer": $0[kCGWindowLayer as String] ?? "nil"] }
|
||||
return "snapshot=\(String(describing: AXTree.snapshotEvidence(pid: pid))) capture=\(coordinateCaptureDiagnostic) windows=\(windows) receipt=\(receipt())"
|
||||
}
|
||||
var observed = try await state()
|
||||
var editor = try publishedHandle(label: "Contract editor", state: observed).0
|
||||
func action(_ command: String, _ args: [String: JSONValue]) async throws {
|
||||
do {
|
||||
_ = try await router.handle(cmd: command, payload: .object(args.merging(["pid": .int(Int(pid))]) { current, _ in current }))
|
||||
} catch {
|
||||
print("[native-six-method-failure] command=\(command) \(diagnostic())")
|
||||
throw error
|
||||
}
|
||||
}
|
||||
try await action("set_value", ["index": .string(editor.rawValue), "value": .string("alpha beta gamma")])
|
||||
observed = try await state()
|
||||
editor = try publishedHandle(label: "Contract editor", state: observed).0
|
||||
try await action("select_text", ["index": .string(editor.rawValue), "text": .string("beta")])
|
||||
try await waitUntil(description: "beta selected") { receipt()["selectionLocation"] as? Int == 6 && receipt()["selectionLength"] as? Int == 4 }
|
||||
try await action("type_text", ["text": .string("typed")])
|
||||
try await waitUntil(description: "selection replaced by typeText", diagnostic: { "\(receipt())" }) { receipt()["text"] as? String == "alpha typed gamma" }
|
||||
observed = try await state()
|
||||
editor = try publishedHandle(label: "Contract editor", state: observed).0
|
||||
try await action("select_text", ["index": .string(editor.rawValue), "text": .string("typed")])
|
||||
try await action("paste", ["text": .string("pasted"), "format": .string("text")])
|
||||
try await waitUntil(description: "selection replaced by paste") { receipt()["text"] as? String == "alpha pasted gamma" }
|
||||
XCTAssertEqual(pasteboard.string(forType: .string), "disposable original clipboard")
|
||||
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.status, "completed")
|
||||
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.dataSupplied, true)
|
||||
XCTAssertEqual(ClipboardPasteReceipt.lastDiagnostic?.restored, true)
|
||||
XCTAssertEqual(receipt()["selectionLocation"] as? Int, 12)
|
||||
XCTAssertEqual(receipt()["selectionLength"] as? Int, 0)
|
||||
observed = try await state()
|
||||
let scroll = try publishedHandle(label: "Contract scroll", state: observed).0
|
||||
var wheelCount = 0
|
||||
func scrollAndReceive(_ args: [String: JSONValue], x: Double? = nil, y: Double? = nil) async throws {
|
||||
try await action("scroll", args)
|
||||
wheelCount += 1
|
||||
try await waitUntil(description: "scroll event \(wheelCount)", diagnostic: diagnostic) {
|
||||
let current = receipt()
|
||||
guard (current["wheel"] as? [[String: Any]])?.count == wheelCount else { return false }
|
||||
if let x, abs((current["scrollX"] as? Double ?? -.infinity) - x) > 0.01 { return false }
|
||||
if let y, abs((current["scrollY"] as? Double ?? -.infinity) - y) > 0.01 { return false }
|
||||
return true
|
||||
}
|
||||
}
|
||||
try await scrollAndReceive(["index": .string(scroll.rawValue), "direction": .string("down"), "pages": .double(0.5)], y: 105)
|
||||
XCTAssertEqual(try XCTUnwrap(receipt()["scrollY"] as? Double), 105, accuracy: 0.01)
|
||||
try await scrollAndReceive(["index": .string(scroll.rawValue), "direction": .string("down"), "pages": .double(1.5)], y: 420)
|
||||
XCTAssertEqual(try XCTUnwrap(receipt()["scrollY"] as? Double), 420, accuracy: 0.01)
|
||||
let coordinateState = try await router.handle(cmd: "get_app_state", payload: .object(["pid": .int(Int(pid)), "disableDiff": .bool(true)]))
|
||||
if case .object(let stateObject) = coordinateState,
|
||||
case .object(let screenshot) = stateObject["screenshot"] {
|
||||
coordinateCaptureDiagnostic = "\(screenshot.filter { $0.key != "base64" })"
|
||||
}
|
||||
try await scrollAndReceive(["x": .int(380), "y": .int(377), "direction": .string("down"), "pages": .double(0.5)], y: 696)
|
||||
XCTAssertEqual(try XCTUnwrap(receipt()["scrollY"] as? Double), 696, accuracy: 0.01)
|
||||
try await scrollAndReceive(["x": .int(380), "y": .int(377), "direction": .string("up"), "pages": .double(0.5)], y: 420)
|
||||
XCTAssertEqual(try XCTUnwrap(receipt()["scrollY"] as? Double), 420, accuracy: 0.01)
|
||||
try await scrollAndReceive(["index": .string(scroll.rawValue), "direction": .string("up"), "pages": .double(0.5)], y: 315)
|
||||
try await scrollAndReceive(["index": .string(scroll.rawValue), "direction": .string("down"), "pages": .double(0.123)], y: 341)
|
||||
XCTAssertEqual(try XCTUnwrap(receipt()["scrollY"] as? Double), 341, accuracy: 0.01)
|
||||
// Intentional improvement over the inspected official wheelCount=1:
|
||||
// keep the second wheel axis so horizontal scrolling remains usable.
|
||||
try await scrollAndReceive(["index": .string(scroll.rawValue), "direction": .string("right"), "pages": .double(0.5)], x: 360)
|
||||
XCTAssertEqual(try XCTUnwrap(receipt()["scrollX"] as? Double), 360, accuracy: 0.01)
|
||||
try await scrollAndReceive(["index": .string(scroll.rawValue), "direction": .string("left"), "pages": .double(0.5)], x: 0)
|
||||
XCTAssertEqual(try XCTUnwrap(receipt()["scrollX"] as? Double), 0, accuracy: 0.01)
|
||||
try await action("perform_secondary_action", ["index": .string(scroll.rawValue), "action": .string("Scroll Down")])
|
||||
try await waitUntil(description: "secondary AX page scroll", diagnostic: diagnostic) {
|
||||
receipt()["scrollY"] as? Double == 524
|
||||
}
|
||||
let wheel = try XCTUnwrap(receipt()["wheel"] as? [[String: Any]])
|
||||
XCTAssertEqual(wheel.compactMap { $0["y"] as? Double }, [-105, -315, -276, 276, 105, -26, 0, 0])
|
||||
XCTAssertEqual(wheel.compactMap { $0["x"] as? Double }, [0, 0, 0, 0, 0, 0, -360, 360])
|
||||
XCTAssertTrue(wheel.allSatisfy { $0["precise"] as? Bool == true })
|
||||
print("[native-six-method-smoke] \(receipt())")
|
||||
FileManager.default.createFile(atPath: stop.path, contents: Data())
|
||||
try await waitUntil(description: "method fixture exit") { process.isTerminated }
|
||||
}
|
||||
|
||||
private func verifyPublishedControl(mismatchedWindowTitle: Bool, exerciseDrags: Bool = false, exerciseKeys: Bool = false, wideWindow: Bool = false) async throws {
|
||||
if ProcessInfo.processInfo.environment[Self.fixtureFlag] == "1" {
|
||||
try await runFixtureProcess(
|
||||
mismatchedWindowTitle: ProcessInfo.processInfo.environment[Self.fixtureMismatchedTitle] == "1"
|
||||
mismatchedWindowTitle: ProcessInfo.processInfo.environment[Self.fixtureMismatchedTitle] == "1",
|
||||
regularActivation: ProcessInfo.processInfo.environment[Self.fixtureRegularActivation] == "1",
|
||||
wideWindow: ProcessInfo.processInfo.environment[Self.fixtureWideWindow] == "1"
|
||||
)
|
||||
// This process is a disposable UI fixture, not another suite run.
|
||||
exit(0)
|
||||
@@ -34,7 +188,7 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
AXIsProcessTrusted(),
|
||||
"Live AX publication requires Accessibility permission for the test runner"
|
||||
)
|
||||
if mismatchedWindowTitle {
|
||||
if mismatchedWindowTitle || exerciseDrags {
|
||||
try XCTSkipUnless(
|
||||
Capture.hasScreenRecordingPermission(),
|
||||
"Coordinate publication requires Screen Recording permission for the test runner"
|
||||
@@ -45,6 +199,7 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
.appendingPathComponent("cc-haha-ax-publication-\(UUID().uuidString)")
|
||||
let ready = root.appendingPathComponent("ready")
|
||||
let stop = root.appendingPathComponent("stop")
|
||||
let gestures = root.appendingPathComponent("gestures.json")
|
||||
let title = "Computer Use snapshot publication \(UUID().uuidString)"
|
||||
try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true)
|
||||
defer { try? FileManager.default.removeItem(at: root) }
|
||||
@@ -62,6 +217,11 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
Self.fixtureStopPath: stop.path,
|
||||
Self.fixtureTitle: title,
|
||||
Self.fixtureMismatchedTitle: mismatchedWindowTitle ? "1" : "0",
|
||||
Self.fixtureGesturePath: gestures.path,
|
||||
// A full-suite child enters the first test, so fixture modes must
|
||||
// travel with this launch rather than that test method's defaults.
|
||||
Self.fixtureRegularActivation: exerciseKeys ? "1" : "0",
|
||||
Self.fixtureWideWindow: wideWindow ? "1" : "0",
|
||||
]) { _, fixture in fixture }
|
||||
configuration.activates = false
|
||||
configuration.createsNewApplicationInstance = true
|
||||
@@ -74,9 +234,11 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
if !process.isTerminated { process.terminate() }
|
||||
}
|
||||
|
||||
try await waitUntil { FileManager.default.fileExists(atPath: ready.path) }
|
||||
try await waitUntil(description: "fixture launch") { FileManager.default.fileExists(atPath: ready.path) }
|
||||
let pid = process.processIdentifier
|
||||
defer { AXTree.invalidate(pid: pid) }
|
||||
let identities = FixtureIdentityDiagnostics(pid: pid)
|
||||
defer { identities.finish() }
|
||||
|
||||
let state = try await AXTree.appState(pid: pid, disableDiff: true)
|
||||
let windowID = try XCTUnwrap(AXTree.snapshotEvidence(pid: pid)?.keyWindowID)
|
||||
@@ -97,6 +259,7 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
capabilities: Capabilities(headless: false),
|
||||
inputMonitor: inputMonitor
|
||||
)
|
||||
identities.stage = "initial indexed click"
|
||||
let click = try await router.handle(
|
||||
cmd: "click",
|
||||
payload: .object([
|
||||
@@ -110,6 +273,104 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
let (_, clickedLine) = try publishedHandle(label: "Bold", state: clickedState)
|
||||
XCTAssertTrue(clickedLine.contains("Value: 1"), clickedLine)
|
||||
|
||||
if exerciseKeys {
|
||||
let (canvasHandle, _) = try publishedHandle(label: "Drag fixture", state: clickedState)
|
||||
_ = try await router.handle(cmd: "click", payload: .object([
|
||||
"pid": .int(Int(pid)), "index": .string(canvasHandle.rawValue),
|
||||
]))
|
||||
_ = try await router.handle(cmd: "press_key", payload: .object([
|
||||
"pid": .int(Int(pid)),
|
||||
"key": .string("Control_L+a Super_R+b A question Delete BackSpace"),
|
||||
]))
|
||||
try await waitUntil(description: "six received macro keys") {
|
||||
guard let data = try? Data(contentsOf: gestures),
|
||||
let receipt = try? JSONSerialization.jsonObject(with: data) as? [String: Any],
|
||||
let keys = receipt["keys"] as? [[String: Any]] else { return false }
|
||||
return keys.count == 6
|
||||
}
|
||||
let receipt = try XCTUnwrap(try JSONSerialization.jsonObject(with: Data(contentsOf: gestures)) as? [String: Any])
|
||||
let keys = try XCTUnwrap(receipt["keys"] as? [[String: Any]])
|
||||
XCTAssertEqual(keys.compactMap { $0["keyCode"] as? Int }, [0, 11, 0, 44, 117, 51])
|
||||
XCTAssertEqual(keys.compactMap { $0["modifiers"] as? UInt }, [262144, 1048576, 131072, 131072, 0, 0])
|
||||
let discovered = try await Apps.listApps()
|
||||
XCTAssertEqual(discovered.first { $0.id == process.bundleIdentifier }?.isRunning, true)
|
||||
print("[native-key-smoke] six-key macro received, inventory entries=\(discovered.count), recent=\(discovered.filter { !$0.isRunning }.count)")
|
||||
}
|
||||
|
||||
if exerciseDrags {
|
||||
let captured = try await router.handle(cmd: "get_app_state", payload: .object([
|
||||
"pid": .int(Int(pid)), "disableDiff": .bool(true),
|
||||
]))
|
||||
let shot = try XCTUnwrap(captured["screenshot"])
|
||||
if wideWindow {
|
||||
XCTAssertEqual(shot["width"]?.asInt, 1397)
|
||||
XCTAssertEqual(shot["height"]?.asInt, 768)
|
||||
XCTAssertEqual(shot["pointWidth"]?.asDouble, 1398)
|
||||
XCTAssertEqual(shot["pointHeight"]?.asDouble, 769)
|
||||
XCTAssertEqual(shot["mimeType"]?.asString, "image/jpeg")
|
||||
XCTAssertEqual(try XCTUnwrap(shot["pixelsPerPoint"]?.asDouble), 768.0 / 769.0, accuracy: 0.000001)
|
||||
let bytes = try XCTUnwrap(Data(base64Encoded: try XCTUnwrap(shot["base64"]?.asString)))
|
||||
XCTAssertEqual(Array(bytes.prefix(2)), [255, 216])
|
||||
}
|
||||
let canvasState = try await AXTree.appState(pid: pid, disableDiff: true)
|
||||
let (canvasHandle, _) = try publishedHandle(label: "Drag fixture", state: canvasState)
|
||||
let frame = try XCTUnwrap(AXTree.record(pid: pid, index: canvasHandle.index)?.frameGlobal)
|
||||
let x = wideWindow ? 300 : (frame.x + frame.w / 2 - (try XCTUnwrap(shot["originX"]?.asDouble)))
|
||||
* Double(try XCTUnwrap(shot["width"]?.asInt))
|
||||
/ (try XCTUnwrap(shot["pointWidth"]?.asDouble))
|
||||
let y = wideWindow ? 180 : (frame.y + frame.h / 2 - (try XCTUnwrap(shot["originY"]?.asDouble)))
|
||||
* Double(try XCTUnwrap(shot["height"]?.asInt))
|
||||
/ (try XCTUnwrap(shot["pointHeight"]?.asDouble))
|
||||
let started = ContinuousClock.now
|
||||
for index in 0..<12 {
|
||||
// Same observed canvas and viewport throughout. Alternating zero
|
||||
// and one pixel exercises the Townscaper-style gesture without
|
||||
// relying on a browser, a real document, or model decisions.
|
||||
identities.stage = "drag \(index)"
|
||||
do {
|
||||
_ = try await router.handle(cmd: "drag", payload: .object([
|
||||
"pid": .int(Int(pid)),
|
||||
"from": .object(["x": .double(x), "y": .double(y)]),
|
||||
"to": .object(["x": .double(x + Double(index % 2)), "y": .double(y)]),
|
||||
]))
|
||||
} catch {
|
||||
print("[native-drag-smoke] failed step \(index), fixture terminated=\(process.isTerminated), stop=\(FileManager.default.fileExists(atPath: stop.path)), snapshot=\(String(describing: AXTree.snapshotEvidence(pid: pid)?.processIdentity)), live=\(String(describing: AXTree.currentProcessIdentity(pid: pid))), receiver=\((try? String(contentsOf: gestures, encoding: .utf8)) ?? "no receipt")")
|
||||
throw error
|
||||
}
|
||||
}
|
||||
identities.stage = "final observation"
|
||||
let final = try await router.handle(cmd: "get_app_state", payload: .object([
|
||||
"pid": .int(Int(pid)), "disableDiff": .bool(true),
|
||||
]))
|
||||
XCTAssertNotNil(final["screenshot"])
|
||||
try await waitUntil(description: "12 received gestures", diagnostic: {
|
||||
"terminated=\(process.isTerminated) stop=\(FileManager.default.fileExists(atPath: stop.path)) receiver=\((try? String(contentsOf: gestures, encoding: .utf8)) ?? "no gesture receipt")"
|
||||
}) {
|
||||
guard let data = try? Data(contentsOf: gestures),
|
||||
let receipt = try? JSONSerialization.jsonObject(with: data) as? [String: Any]
|
||||
else { return false }
|
||||
return receipt["completed"] as? Int == 12
|
||||
}
|
||||
let receipt = try XCTUnwrap(
|
||||
try JSONSerialization.jsonObject(with: Data(contentsOf: gestures)) as? [String: Any]
|
||||
)
|
||||
XCTAssertEqual(receipt["completed"] as? Int, 12)
|
||||
XCTAssertEqual(receipt["drags"] as? Int, 12, "zero-distance gestures must still contain a dragged event")
|
||||
XCTAssertEqual(receipt["unpaired"] as? Int, 0)
|
||||
if wideWindow {
|
||||
let events = try XCTUnwrap(receipt["events"] as? [[String: Any]])
|
||||
let down = try XCTUnwrap(events.first { $0["type"] as? UInt == NSEvent.EventType.leftMouseDown.rawValue })
|
||||
// Measured from the official App.drag on the same window size:
|
||||
// x300/y180 uses a uniform 769/768 inverse fit on both axes.
|
||||
XCTAssertEqual(try XCTUnwrap(down["x"] as? Double), 300.390625, accuracy: 0.000001)
|
||||
XCTAssertEqual(try XCTUnwrap(down["y"] as? Double), 588.765625, accuracy: 0.000001)
|
||||
}
|
||||
#if DEBUG
|
||||
XCTAssertGreaterThan(identities.samples, 0, "the identity experiment must observe real production validation")
|
||||
#endif
|
||||
print("[native-drag-smoke] 12 received gestures + final observation: \(started.duration(to: .now))")
|
||||
}
|
||||
|
||||
if mismatchedWindowTitle {
|
||||
// Exercise the actual state → screenshot → coordinate action path,
|
||||
// using only this disposable fixture window. Previously the image
|
||||
@@ -159,20 +420,24 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
XCTAssertTrue(unchangedLine.contains("Value: 1"), unchangedLine)
|
||||
|
||||
FileManager.default.createFile(atPath: stop.path, contents: Data())
|
||||
try await waitUntil { process.isTerminated }
|
||||
try await waitUntil(description: "fixture exit") { process.isTerminated }
|
||||
}
|
||||
|
||||
private func runFixtureProcess(mismatchedWindowTitle: Bool) async throws {
|
||||
private func runFixtureProcess(mismatchedWindowTitle: Bool, regularActivation: Bool, wideWindow: Bool) async throws {
|
||||
let environment = ProcessInfo.processInfo.environment
|
||||
let readyPath = try XCTUnwrap(environment[Self.fixtureReadyPath])
|
||||
if environment[Self.fixtureMethods] == "1" {
|
||||
try NativeMethodContractFixture.run(root: URL(fileURLWithPath: readyPath).deletingLastPathComponent())
|
||||
return
|
||||
}
|
||||
let stopPath = try XCTUnwrap(environment[Self.fixtureStopPath])
|
||||
let title = try XCTUnwrap(environment[Self.fixtureTitle])
|
||||
|
||||
let app = NSApplication.shared
|
||||
app.setActivationPolicy(.accessory)
|
||||
app.setActivationPolicy(regularActivation ? .regular : .accessory)
|
||||
app.finishLaunching()
|
||||
let window = NSWindow(
|
||||
contentRect: NSRect(x: 200, y: 200, width: 420, height: 180),
|
||||
contentRect: NSRect(x: 200, y: 200, width: wideWindow ? 1398 : 420, height: wideWindow ? 737 : 180),
|
||||
styleMask: [.titled],
|
||||
backing: .buffered,
|
||||
defer: false
|
||||
@@ -213,6 +478,14 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
let content = NSView(frame: NSRect(x: 0, y: 0, width: 420, height: 180))
|
||||
content.addSubview(formattingGroup)
|
||||
content.addSubview(alignmentGroup)
|
||||
let canvas = DragReceiptView(frame: wideWindow
|
||||
? NSRect(x: 20, y: 130, width: 1358, height: 580)
|
||||
: NSRect(x: 20, y: 15, width: 380, height: 30))
|
||||
canvas.receiptPath = try XCTUnwrap(environment[Self.fixtureGesturePath])
|
||||
canvas.setAccessibilityElement(true)
|
||||
canvas.setAccessibilityRole(.group)
|
||||
canvas.setAccessibilityLabel("Drag fixture")
|
||||
content.addSubview(canvas)
|
||||
window.contentView = content
|
||||
window.makeKeyAndOrderFront(nil)
|
||||
app.activate()
|
||||
@@ -223,14 +496,21 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
try await Task.sleep(for: .milliseconds(100))
|
||||
|
||||
FileManager.default.createFile(atPath: readyPath, contents: Data())
|
||||
while !FileManager.default.fileExists(atPath: stopPath) {
|
||||
try await Task.sleep(for: .milliseconds(20))
|
||||
// XCTest's async wait services AX requests, but does not run AppKit's
|
||||
// native event dispatcher. A receiving-app fixture needs NSApp.run to
|
||||
// consume synthetic activation and mouse events just like a real app.
|
||||
let stopTimer = Timer.scheduledTimer(withTimeInterval: 0.02, repeats: true) { _ in
|
||||
if FileManager.default.fileExists(atPath: stopPath) { exit(0) }
|
||||
}
|
||||
defer { stopTimer.invalidate() }
|
||||
app.run()
|
||||
}
|
||||
|
||||
private func waitUntil(
|
||||
_ predicate: () -> Bool,
|
||||
timeout: Duration = .seconds(5)
|
||||
description: String,
|
||||
timeout: Duration = .seconds(5),
|
||||
diagnostic: () -> String = { "" },
|
||||
_ predicate: () -> Bool
|
||||
) async throws {
|
||||
let clock = ContinuousClock()
|
||||
let deadline = clock.now.advanced(by: timeout)
|
||||
@@ -238,7 +518,7 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
if predicate() { return }
|
||||
try await Task.sleep(for: .milliseconds(20))
|
||||
}
|
||||
XCTFail("Timed out waiting for AX fixture")
|
||||
XCTFail("Timed out waiting for \(description): \(diagnostic())")
|
||||
throw FixtureError.timedOut
|
||||
}
|
||||
|
||||
@@ -279,7 +559,7 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
) throws -> (SnapshotElementHandle, String) {
|
||||
let line = try XCTUnwrap(
|
||||
state.axText.split(separator: "\n").first {
|
||||
$0.contains("Description: \(label)")
|
||||
$0.contains("Description: \(label)") || $0.hasSuffix("container \(label)")
|
||||
}.map(String.init),
|
||||
state.axText
|
||||
)
|
||||
@@ -289,3 +569,111 @@ final class AXTreePublicationIntegrationTests: XCTestCase {
|
||||
return (try XCTUnwrap(SnapshotElementHandle(rawValue: rawHandle)), line)
|
||||
}
|
||||
}
|
||||
|
||||
/// Records what the receiving process actually consumed, independently of the
|
||||
/// sender's event builder. AppKit may coalesce motion, so assert complete drag
|
||||
/// gestures rather than requiring every intermediate event at this boundary.
|
||||
@MainActor
|
||||
private final class DragReceiptView: NSView {
|
||||
var receiptPath = ""
|
||||
private var held = false
|
||||
private var dragged = false
|
||||
private var completed = 0
|
||||
private var drags = 0
|
||||
private var unpaired = 0
|
||||
private var events: [[String: Any]] = []
|
||||
private var keys: [[String: Any]] = []
|
||||
|
||||
override var acceptsFirstResponder: Bool { true }
|
||||
override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
|
||||
|
||||
override func mouseDown(with event: NSEvent) {
|
||||
window?.makeFirstResponder(self)
|
||||
if held { unpaired += 1 }
|
||||
held = true
|
||||
dragged = false
|
||||
record(event)
|
||||
}
|
||||
|
||||
override func keyDown(with event: NSEvent) {
|
||||
keys.append([
|
||||
"keyCode": Int(event.keyCode),
|
||||
"modifiers": event.modifierFlags.intersection([.command, .control, .shift, .option]).rawValue,
|
||||
])
|
||||
writeReceipt()
|
||||
}
|
||||
|
||||
override func mouseDragged(with event: NSEvent) {
|
||||
if !held { unpaired += 1 }
|
||||
dragged = true
|
||||
record(event)
|
||||
}
|
||||
|
||||
override func mouseUp(with event: NSEvent) {
|
||||
if !held { unpaired += 1 }
|
||||
completed += 1
|
||||
if dragged { drags += 1 }
|
||||
held = false
|
||||
record(event)
|
||||
}
|
||||
|
||||
private func record(_ event: NSEvent) {
|
||||
events.append([
|
||||
"type": event.type.rawValue,
|
||||
"number": event.eventNumber,
|
||||
"clicks": event.clickCount,
|
||||
"x": event.locationInWindow.x,
|
||||
"y": event.locationInWindow.y,
|
||||
"timestamp": event.timestamp,
|
||||
])
|
||||
writeReceipt()
|
||||
}
|
||||
|
||||
private func writeReceipt() {
|
||||
let receipt: [String: Any] = [
|
||||
"completed": completed, "drags": drags, "unpaired": unpaired,
|
||||
"held": held, "events": events, "keys": keys,
|
||||
]
|
||||
if let data = try? JSONSerialization.data(withJSONObject: receipt) {
|
||||
try? data.write(to: URL(fileURLWithPath: receiptPath), options: .atomic)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private final class FixtureIdentityDiagnostics {
|
||||
let pid: pid_t
|
||||
var stage = "state"
|
||||
private(set) var samples = 0
|
||||
private var observed = Set<String>()
|
||||
#if DEBUG
|
||||
private let previous: ((ProvenProcessTarget, AXTreeProcessIdentity?) -> Void)?
|
||||
#endif
|
||||
|
||||
init(pid: pid_t) {
|
||||
self.pid = pid
|
||||
#if DEBUG
|
||||
previous = Injection.targetValidationObserver
|
||||
Injection.targetValidationObserver = { [weak self] expected, current in
|
||||
guard let self, expected.pid == self.pid else { return }
|
||||
self.samples += 1
|
||||
self.observed.insert(Self.describe(current))
|
||||
if expected.validatedPid(currentIdentity: current) == nil {
|
||||
print("[identity-validation] failed pid=\(pid) stage=\(self.stage) sample=\(self.samples) expected=\(Self.describe(expected.identity)) actual=\(Self.describe(current))")
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
func finish() {
|
||||
#if DEBUG
|
||||
Injection.targetValidationObserver = previous
|
||||
print("[identity-validation] pid=\(pid) samples=\(samples) identities=\(observed.sorted())")
|
||||
#endif
|
||||
}
|
||||
|
||||
private static func describe(_ identity: AXTreeProcessIdentity?) -> String {
|
||||
guard let identity else { return "nil" }
|
||||
return "bundle=\(identity.bundleID ?? "nil") path=\(identity.executablePath ?? "nil") launch=\(identity.launchTime.map(String.init(describing:)) ?? "nil") bits=\(identity.launchTime.map { String($0.bitPattern, radix: 16) } ?? "nil")"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import Foundation
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class AppInventoryTests: XCTestCase {
|
||||
func testRunningAndRecentMetadataMergeWithoutDroppingUsageOrAddingUnseenApps() throws {
|
||||
let date = Date(timeIntervalSince1970: 1_783_200_000)
|
||||
let recent = [
|
||||
AppInventoryEntry(id: "dev.fixture.running", displayName: "Indexed", isRunning: false, lastUsedDate: date, useCount: 12),
|
||||
AppInventoryEntry(id: "dev.fixture.recent", displayName: "Recent", isRunning: false, lastUsedDate: date, useCount: 3),
|
||||
]
|
||||
let running = [AppRef(bundleId: "dev.fixture.running", displayName: "Live"), AppRef(bundleId: "dev.fixture.new", displayName: "New")]
|
||||
let result = AppInventory.merge(running: running, recent: recent)
|
||||
XCTAssertEqual(result.map(\.id), ["dev.fixture.running", "dev.fixture.new", "dev.fixture.recent"])
|
||||
XCTAssertEqual(result.map(\.isRunning), [true, true, false])
|
||||
XCTAssertEqual(result[0].displayName, "Live")
|
||||
XCTAssertEqual(result[0].useCount, 12)
|
||||
XCTAssertEqual(result[0].lastUsedDate, date)
|
||||
XCTAssertNil(result[1].useCount)
|
||||
XCTAssertNil(result[1].lastUsedDate)
|
||||
let data = try JSONEncoder().encode(result[1])
|
||||
let json = try XCTUnwrap(JSONSerialization.jsonObject(with: data) as? [String: Any])
|
||||
XCTAssertEqual(json["id"] as? String, "dev.fixture.new")
|
||||
XCTAssertNil(json["useCount"])
|
||||
XCTAssertNil(json["lastUsedDate"])
|
||||
}
|
||||
|
||||
func testMetadataSchemaPreservesDatesAndIgnoresMalformedUsage() throws {
|
||||
let date = Date(timeIntervalSince1970: 1_783_200_000)
|
||||
let fields: [String: Any] = [
|
||||
"kMDItemCFBundleIdentifier": "dev.fixture", "kMDItemDisplayName": "Fixture",
|
||||
"kMDItemLastUsedDate_Ranking": date, "kMDItemUseCount": 7,
|
||||
]
|
||||
let entry = try XCTUnwrap(AppInventory.fromMetadata(fields))
|
||||
XCTAssertEqual(entry.lastUsedDate, date)
|
||||
XCTAssertEqual(entry.useCount, 7)
|
||||
let json = try XCTUnwrap(JSONSerialization.jsonObject(with: JSONEncoder().encode(entry)) as? [String: Any])
|
||||
XCTAssertEqual(json["lastUsedDate"] as? String, ISO8601DateFormatter().string(from: date))
|
||||
XCTAssertNil(AppInventory.fromMetadata(["kMDItemDisplayName": "missing identity"]))
|
||||
XCTAssertNil(AppInventory.fromMetadata(fields.merging(["kMDItemUseCount": -1]) { _, new in new })?.useCount)
|
||||
}
|
||||
|
||||
func testDiscoveryDoesNotConflateForbiddenTargetsWithMissingApplications() {
|
||||
let recent = [
|
||||
AppInventoryEntry(id: "dev.fixture", displayName: "Duplicate", isRunning: false),
|
||||
AppInventoryEntry(id: "com.apple.Terminal", displayName: "Terminal", isRunning: false),
|
||||
AppInventoryEntry(id: "com.apple.Music", displayName: "Music", isRunning: false),
|
||||
]
|
||||
let result = AppInventory.merge(running: [AppRef(bundleId: "dev.fixture", displayName: "Fixture")], recent: recent)
|
||||
XCTAssertEqual(result.map(\.id), ["dev.fixture", "com.apple.Terminal", "com.apple.Music"])
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,17 @@ import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class AppTargetPolicyTests: XCTestCase {
|
||||
func testTerminalAndAutomationBundlesAreDenied() {
|
||||
func testOfficialTerminalAndSecurityBundlesAreDenied() {
|
||||
let denied = [
|
||||
"com.apple.Terminal",
|
||||
"com.googlecode.iterm2",
|
||||
"com.microsoft.VSCode",
|
||||
"com.apple.shortcuts",
|
||||
"com.raphaelamorim.rio",
|
||||
"dev.commandline.waveterm",
|
||||
"com.apple.SecurityAgent",
|
||||
"com.apple.LocalAuthenticationRemoteService",
|
||||
"com.apple.UserNotificationCenter",
|
||||
"com.openai.codex.beta",
|
||||
"com.openai.chat.mac-debug",
|
||||
]
|
||||
|
||||
for bundleID in denied {
|
||||
@@ -15,14 +20,17 @@ final class AppTargetPolicyTests: XCTestCase {
|
||||
}
|
||||
}
|
||||
|
||||
func testBrowserBundlesAreDenied() {
|
||||
for bundleID in ["com.google.Chrome", "com.apple.Safari"] {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .deny, bundleID)
|
||||
func testNativeBrowserBundlesAreAllowed() {
|
||||
for bundleID in [
|
||||
"com.google.Chrome", "com.google.Chrome.canary", "com.apple.Safari",
|
||||
"org.mozilla.firefox", "com.microsoft.edgemac", "com.brave.Browser",
|
||||
] {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testTradingAndWalletBundlesAreDenied() {
|
||||
let denied = [
|
||||
func testOtherAppCategoriesAreNotImplicitlyForbidden() {
|
||||
let allowed = [
|
||||
"com.webull.desktop.v1",
|
||||
"com.binance.BinanceDesktop",
|
||||
"com.electron.exodus",
|
||||
@@ -30,22 +38,31 @@ final class AppTargetPolicyTests: XCTestCase {
|
||||
"io.trezor.TrezorSuite",
|
||||
]
|
||||
|
||||
for bundleID in denied {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .deny, bundleID)
|
||||
for bundleID in allowed {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testStreamingMusicAndPublisherPolicyBundlesAreDenied() {
|
||||
let denied = [
|
||||
func testMediaAndDevelopmentAppsAreNotImplicitlyForbidden() {
|
||||
let allowed = [
|
||||
"com.spotify.client",
|
||||
"com.apple.Music",
|
||||
"com.amazon.aiv.AIVApp",
|
||||
"tv.plex.desktop",
|
||||
"com.amazon.Kindle",
|
||||
"com.microsoft.VSCode",
|
||||
"com.apple.shortcuts",
|
||||
"com.apple.dt.Xcode",
|
||||
]
|
||||
|
||||
for bundleID in denied {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .deny, bundleID)
|
||||
for bundleID in allowed {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: bundleID), .allow, bundleID)
|
||||
}
|
||||
}
|
||||
|
||||
func testForbiddenPolicyUsesExactIdentityNotNameSubstringOrPrefix() {
|
||||
for id in ["com.apple.Terminal.preview", "com.openai.codex.userapp", "org.example.Terminal", "com.apple.securityagent"] {
|
||||
XCTAssertEqual(AppTargetPolicy.decision(bundleID: id), .allow, id)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,15 +91,15 @@ final class AppTargetPolicyTests: XCTestCase {
|
||||
}
|
||||
|
||||
func testDeniedBundleUnionCountAndSetDuplicateHandlingAreLocked() {
|
||||
XCTAssertEqual(AppTargetPolicy.deniedBundleIDs.count, 107)
|
||||
XCTAssertEqual(AppTargetPolicy.deniedBundleIDs.count, 24)
|
||||
XCTAssertTrue(
|
||||
AppTargetPolicy.deniedBundleIDs
|
||||
.isDisjoint(with: AppTargetPolicy.intrinsicDeniedBundleIDs)
|
||||
)
|
||||
|
||||
var copy = AppTargetPolicy.deniedBundleIDs
|
||||
let duplicate = copy.insert("com.apple.Safari")
|
||||
let duplicate = copy.insert("com.apple.Terminal")
|
||||
XCTAssertFalse(duplicate.inserted)
|
||||
XCTAssertEqual(copy.count, 107)
|
||||
XCTAssertEqual(copy.count, 24)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,24 @@ import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class ClipboardPasteReceiptTests: XCTestCase {
|
||||
@MainActor
|
||||
func testReadWithoutTargetSignalsUsesOnlyTheShortFallbackWindow() async throws {
|
||||
let fixture = PasteReceiptFixture()
|
||||
defer { fixture.close() }
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: "temporary", lease: ClipboardLease(pasteboard: fixture.board)
|
||||
) { validate in
|
||||
try await fixture.sendPaste(validate)
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
|
||||
}
|
||||
let diagnostic = try XCTUnwrap(ClipboardPasteReceipt.lastDiagnostic)
|
||||
let readElapsed = try XCTUnwrap(diagnostic.readElapsedMilliseconds)
|
||||
XCTAssertGreaterThanOrEqual(diagnostic.elapsedMilliseconds - readElapsed, 90)
|
||||
XCTAssertLessThan(diagnostic.elapsedMilliseconds - readElapsed, 600,
|
||||
"a successful read without AX signals has a 100 ms fallback, not a fixed two-second hold")
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "original")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testHtmlPastePromisesRichAndPlainRepresentationsThenRestoresClipboard() async throws {
|
||||
let fixture = PasteReceiptFixture()
|
||||
@@ -66,10 +84,12 @@ final class ClipboardPasteReceiptTests: XCTestCase {
|
||||
defer { fixture.close() }
|
||||
let lease = ClipboardLease(pasteboard: fixture.board)
|
||||
var returned = false
|
||||
var targetChanged = false
|
||||
var targetReader: Task<Void, Never>?
|
||||
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: "temporary", lease: lease, timeout: .milliseconds(450)
|
||||
text: "temporary", lease: lease, timeout: .milliseconds(450),
|
||||
targetObservation: ClipboardPasteObservation(hasSignals: true, hasChanged: { targetChanged })
|
||||
) { validate in
|
||||
try await fixture.sendPaste(validate)
|
||||
// A clipboard observer can request the promised bytes first.
|
||||
@@ -79,6 +99,7 @@ final class ClipboardPasteReceiptTests: XCTestCase {
|
||||
XCTAssertFalse(returned, "an unidentified reader cannot end the target's consumption window")
|
||||
XCTAssertTrue(lease.temporaryWriteIsCurrent())
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
|
||||
targetChanged = true
|
||||
}
|
||||
}
|
||||
returned = true
|
||||
@@ -98,9 +119,11 @@ final class ClipboardPasteReceiptTests: XCTestCase {
|
||||
defer { fixture.close() }
|
||||
let lease = ClipboardLease(pasteboard: fixture.board)
|
||||
var targetReader: Task<Void, Never>?
|
||||
var targetChanged = false
|
||||
let task = Task { @MainActor in
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: "temporary", lease: lease, timeout: .milliseconds(450)
|
||||
text: "temporary", lease: lease, timeout: .milliseconds(450),
|
||||
targetObservation: ClipboardPasteObservation(hasSignals: true, hasChanged: { targetChanged })
|
||||
) { validate in
|
||||
try await fixture.sendPaste(validate)
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
|
||||
@@ -108,6 +131,7 @@ final class ClipboardPasteReceiptTests: XCTestCase {
|
||||
try? await Task.sleep(for: .milliseconds(300))
|
||||
XCTAssertTrue(lease.temporaryWriteIsCurrent())
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
|
||||
targetChanged = true
|
||||
}
|
||||
withUnsafeCurrentTask { $0?.cancel() }
|
||||
}
|
||||
@@ -133,7 +157,8 @@ final class ClipboardPasteReceiptTests: XCTestCase {
|
||||
do {
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: "temporary", lease: ClipboardLease(pasteboard: fixture.board),
|
||||
timeout: .milliseconds(450)
|
||||
timeout: .milliseconds(450),
|
||||
targetObservation: ClipboardPasteObservation(hasSignals: true)
|
||||
) { validate in
|
||||
try await fixture.sendPaste(validate)
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
|
||||
@@ -295,6 +320,104 @@ final class ClipboardPasteReceiptTests: XCTestCase {
|
||||
XCTAssertEqual(error.code, "clipboard_read_timeout")
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testObservedTargetChangeCompletesBeforeTheTwoSecondDeadline() async throws {
|
||||
let fixture = PasteReceiptFixture()
|
||||
defer { fixture.close() }
|
||||
let notifications = ClipboardPasteNotifications()
|
||||
var closed = false
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: "temporary", lease: ClipboardLease(pasteboard: fixture.board),
|
||||
targetObservation: ClipboardPasteObservation(
|
||||
hasSignals: true,
|
||||
arm: { notifications.arm($0) },
|
||||
hasChanged: { notifications.hasChanged },
|
||||
close: {
|
||||
notifications.close()
|
||||
closed = true
|
||||
}
|
||||
)
|
||||
) { validate in
|
||||
try await fixture.sendPaste(validate)
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
|
||||
notifications.recordTargetChange()
|
||||
}
|
||||
let diagnostic = try XCTUnwrap(ClipboardPasteReceipt.lastDiagnostic)
|
||||
XCTAssertEqual(diagnostic.status, "completed")
|
||||
XCTAssertLessThan(diagnostic.elapsedMilliseconds, 600)
|
||||
XCTAssertTrue(closed)
|
||||
XCTAssertEqual(fixture.events.count, 4)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testReadWithoutAChangeInObservableTargetTimesOutAndRestores() async throws {
|
||||
let fixture = PasteReceiptFixture()
|
||||
defer { fixture.close() }
|
||||
var closed = false
|
||||
do {
|
||||
try await ClipboardPasteReceipt.perform(
|
||||
text: "temporary", lease: ClipboardLease(pasteboard: fixture.board),
|
||||
timeout: .milliseconds(60),
|
||||
targetObservation: ClipboardPasteObservation(hasSignals: true, close: { closed = true })
|
||||
) { validate in
|
||||
try await fixture.sendPaste(validate)
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
|
||||
}
|
||||
XCTFail("a read alone cannot acknowledge an observable field that did not change")
|
||||
} catch let error as CUError {
|
||||
XCTAssertEqual(error.code, "clipboard_target_timeout")
|
||||
}
|
||||
let diagnostic = try XCTUnwrap(ClipboardPasteReceipt.lastDiagnostic)
|
||||
XCTAssertEqual(diagnostic.status, "clipboard_target_timeout")
|
||||
XCTAssertTrue(diagnostic.dataSupplied)
|
||||
XCTAssertGreaterThanOrEqual(diagnostic.elapsedMilliseconds - (diagnostic.readElapsedMilliseconds ?? 0), 55)
|
||||
XCTAssertTrue(closed)
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "original")
|
||||
XCTAssertEqual(fixture.events.count, 4, "timeout must not resend the paste")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testOnlyAnArmedNotificationAfterDataReadConfirmsTheTarget() throws {
|
||||
let fixture = PasteReceiptFixture()
|
||||
defer { fixture.close() }
|
||||
let lease = ClipboardLease(pasteboard: fixture.board)
|
||||
defer { lease.restoreIfUnchanged() }
|
||||
let receipt = try lease.writeTemporaryStringWithReceipt("temporary")
|
||||
let notifications = ClipboardPasteNotifications()
|
||||
notifications.recordTargetChange()
|
||||
XCTAssertFalse(notifications.hasChanged, "an unarmed callback cannot confirm this paste")
|
||||
notifications.arm(receipt)
|
||||
notifications.recordTargetChange()
|
||||
XCTAssertFalse(notifications.hasChanged, "a callback before the data read cannot confirm consumption")
|
||||
XCTAssertEqual(fixture.board.string(forType: .string), "temporary")
|
||||
XCTAssertFalse(notifications.hasChanged, "an unrelated clipboard reader cannot confirm the target")
|
||||
notifications.recordTargetChange()
|
||||
XCTAssertTrue(notifications.hasChanged)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testFailedOrUnchangedAXReadsCannotConfirmTargetConsumption() throws {
|
||||
var initialRange = CFRange(location: 2, length: 3)
|
||||
var sameRange = initialRange
|
||||
var changedRange = CFRange(location: 5, length: 0)
|
||||
let initial = try XCTUnwrap(AXValueCreate(.cfRange, &initialRange))
|
||||
let same = try XCTUnwrap(AXValueCreate(.cfRange, &sameRange))
|
||||
let changed = try XCTUnwrap(AXValueCreate(.cfRange, &changedRange))
|
||||
let baseline: [String: CFTypeRef] = ["range": initial, "count": NSNumber(value: 12)]
|
||||
XCTAssertFalse(ClipboardPasteObservation.attributesChanged(baseline: baseline, current: [:]))
|
||||
XCTAssertFalse(ClipboardPasteObservation.attributesChanged(baseline: baseline, current: ["range": changed]),
|
||||
"a partial AX read must not claim target success")
|
||||
XCTAssertFalse(ClipboardPasteObservation.attributesChanged(
|
||||
baseline: baseline, current: ["range": same, "count": NSNumber(value: 12)]
|
||||
), "AX value equality must compare values rather than object identity")
|
||||
XCTAssertTrue(ClipboardPasteObservation.attributesChanged(
|
||||
baseline: baseline, current: ["range": changed, "count": NSNumber(value: 12)]
|
||||
))
|
||||
XCTAssertTrue(ClipboardPasteObservation.attributesChanged(
|
||||
baseline: baseline, current: ["range": same, "count": NSNumber(value: 15)]
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// The real promised-data provider, paste orchestration and keyboard factory
|
||||
|
||||
@@ -15,6 +15,42 @@ final class CommandRouterSafetyTests: XCTestCase {
|
||||
launchTime: 200
|
||||
)
|
||||
|
||||
func testRoundedScreenshotDimensionsPreserveOfficialUniformCoordinateScale() throws {
|
||||
defer { CommandRouter.clearShotTransformsForTesting() }
|
||||
CommandRouter.recordShotTransform(
|
||||
pid: 77, originX: 100, originY: 200,
|
||||
pointWidth: 1398, pointHeight: 769,
|
||||
imageWidth: 1397, imageHeight: 768,
|
||||
processIdentity: processA, windowID: 17,
|
||||
pixelsPerPoint: 768.0 / 769.0
|
||||
)
|
||||
let point = try CommandRouter.toGlobalPoint(
|
||||
x: 300, y: 180, pid: 77,
|
||||
currentProcessIdentity: processA, currentWindowID: 17
|
||||
)
|
||||
// Actual official native receiver, 1398×769 points → 1397×768 JPEG.
|
||||
// Both axes use the original fit, before the pixel buffer's ceil.
|
||||
XCTAssertEqual(point.x, 400.390625, accuracy: 0.000001)
|
||||
XCTAssertEqual(point.y, 380.234375, accuracy: 0.000001)
|
||||
}
|
||||
|
||||
func testInvalidUniformCaptureScaleCannotPublishACoordinateTransform() throws {
|
||||
defer { CommandRouter.clearShotTransformsForTesting() }
|
||||
for scale in [0, -1, Double.nan, Double.infinity] {
|
||||
CommandRouter.recordShotTransform(
|
||||
pid: 77, originX: 100, originY: 200,
|
||||
pointWidth: 1398, pointHeight: 769,
|
||||
imageWidth: 1397, imageHeight: 768,
|
||||
processIdentity: processA, windowID: 17,
|
||||
pixelsPerPoint: scale
|
||||
)
|
||||
XCTAssertThrowsError(try CommandRouter.toGlobalPoint(
|
||||
x: 300, y: 180, pid: 77,
|
||||
currentProcessIdentity: processA, currentWindowID: 17
|
||||
)) { XCTAssertEqual(($0 as? CUError)?.code, "stale_snapshot") }
|
||||
}
|
||||
}
|
||||
|
||||
func testHeadlessMouseDownAndMouseUpRequireDaemon() async throws {
|
||||
let monitor = PhysicalInputEpochMonitor(counterReader: { _ in 0 })
|
||||
_ = monitor.startAndWait()
|
||||
@@ -438,7 +474,7 @@ private final class WindowCaptureProviderSpy: WindowCaptureProviding {
|
||||
pid: pid_t,
|
||||
processIdentity: AXTreeProcessIdentity,
|
||||
preferredWindowID: CGWindowID?,
|
||||
scale: Double,
|
||||
scale: Double?,
|
||||
newerThanUptime: TimeInterval?
|
||||
) async -> WindowShot? {
|
||||
nil
|
||||
|
||||
@@ -80,6 +80,88 @@ final class CursorMotionStateTests: XCTestCase {
|
||||
XCTAssertEqual(sleptFor, decision.actionDelay)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testCoordinateActionsStartVisibleFeedbackWithoutSleepingAtAnyDistance() async {
|
||||
for frontmostPid: pid_t in [41, 99] {
|
||||
let decision = OverlayPolicy.decision(
|
||||
targetPid: 41, frontmostPid: frontmostPid,
|
||||
overlayRequested: true, targetWindowExposed: true
|
||||
)
|
||||
XCTAssertTrue(decision.visible)
|
||||
XCTAssertGreaterThan(decision.actionDelay, 0, "the element-action policy remains available")
|
||||
let origin = CGPoint(x: 410, y: 349)
|
||||
for destination in [origin, CGPoint(x: 411, y: 349), CGPoint(x: -1000, y: 1400)] {
|
||||
var motion = CursorMotionState(position: origin)
|
||||
var events: [String] = []
|
||||
await CursorActionTiming.perform(
|
||||
decision: decision,
|
||||
waitForVisualFeedback: false,
|
||||
startGlide: {
|
||||
motion.startGlide(to: destination)
|
||||
events.append("glide-started")
|
||||
},
|
||||
snap: { events.append("snap") },
|
||||
sleep: { _ in events.append("sleep") }
|
||||
)
|
||||
events.append("returned")
|
||||
XCTAssertEqual(events, ["glide-started", "returned"])
|
||||
XCTAssertEqual(motion.destination, destination, "visual feedback must still be started")
|
||||
XCTAssertEqual(motion.position, origin, "input must not wait for the displayed cursor to arrive")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testCoveredCoordinateActionStillSnapsWithoutStartingHiddenAnimation() async {
|
||||
let decision = OverlayPolicy.decision(
|
||||
targetPid: 41, frontmostPid: 99,
|
||||
overlayRequested: true, targetWindowExposed: false
|
||||
)
|
||||
var events: [String] = []
|
||||
await CursorActionTiming.perform(
|
||||
decision: decision,
|
||||
waitForVisualFeedback: false,
|
||||
startGlide: { events.append("glide") },
|
||||
snap: { events.append("snap") },
|
||||
sleep: { _ in events.append("sleep") }
|
||||
)
|
||||
XCTAssertEqual(events, ["snap"])
|
||||
}
|
||||
|
||||
func testRouterSkipsOnlyCoordinateVisualWaitsAndPreservesPostMoveValidation() throws {
|
||||
// These wiring checks complement the executable timing seam tests;
|
||||
// resolving real app windows would make a unit test mutate user UI.
|
||||
let root = URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
|
||||
.appendingPathComponent("Sources/cu-helper")
|
||||
let router = try String(contentsOf: root.appendingPathComponent("CommandRouter.swift"), encoding: .utf8)
|
||||
let clickStart = try XCTUnwrap(router.range(of: "private func handleClick("))
|
||||
let indexedStart = try XCTUnwrap(router.range(of: "// Index click", range: clickStart.upperBound..<router.endIndex))
|
||||
let clickEnd = try XCTUnwrap(router.range(of: "private func handleSetValue(", range: indexedStart.upperBound..<router.endIndex))
|
||||
let coordinateClick = String(router[clickStart.upperBound..<indexedStart.lowerBound])
|
||||
let indexedClick = String(router[indexedStart.upperBound..<clickEnd.lowerBound])
|
||||
let dragStart = try XCTUnwrap(router.range(of: "private func handleDrag("))
|
||||
let dragEnd = try XCTUnwrap(router.range(of: "// MARK: - Target resolution", range: dragStart.upperBound..<router.endIndex))
|
||||
let drag = String(router[dragStart.upperBound..<dragEnd.lowerBound])
|
||||
|
||||
for action in [coordinateClick, drag] {
|
||||
let move = try XCTUnwrap(action.range(of: "waitForVisualFeedback: false"))
|
||||
let afterMove = action[move.upperBound...]
|
||||
XCTAssertTrue(afterMove.contains("Self.validatedGlobalPoint("))
|
||||
XCTAssertTrue(afterMove.contains("Injection.validateAuthorizedTarget(target)"))
|
||||
}
|
||||
XCTAssertTrue(coordinateClick.contains("cursor.showClick("), "the click ripple remains active")
|
||||
XCTAssertTrue(drag.contains("cursor.move(to: from, animated: false)"))
|
||||
XCTAssertTrue(indexedClick.contains("cursor.moveForAction("))
|
||||
XCTAssertFalse(indexedClick.contains("waitForVisualFeedback: false"), "element actions retain their current policy")
|
||||
XCTAssertTrue(indexedClick.contains("CursorIndexedActionGate.perform("))
|
||||
XCTAssertTrue(indexedClick.contains("guardStaleness(pid:"))
|
||||
|
||||
let cursor = try String(contentsOf: root.appendingPathComponent("VirtualCursor.swift"), encoding: .utf8)
|
||||
XCTAssertTrue(cursor.contains("waitForVisualFeedback: Bool = true"))
|
||||
XCTAssertTrue(cursor.contains("waitForVisualFeedback: waitForVisualFeedback"), "the live cursor must use the tested timing seam")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testIndexedActionRechecksStalenessAfterCursorDelayBeforeMutation() async {
|
||||
var events: [String] = []
|
||||
|
||||
@@ -0,0 +1,61 @@
|
||||
import Carbon.HIToolbox
|
||||
import CoreGraphics
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
/// Expectations come from events received by a disposable AppKit app using
|
||||
/// the installed official macOS App.pressKey implementation, not X11 docs.
|
||||
final class KeyMappingParityTests: XCTestCase {
|
||||
func testNamedNumberRowSymbolsMatchActualMacOSReceiverKeys() throws {
|
||||
let names = "exclam at numbersign dollar percent asciicircum ampersand asterisk parenleft parenright"
|
||||
let chords = try KeyMapping.parse(names)
|
||||
XCTAssertEqual(chords.map(\.keyCode), [18, 19, 20, 21, 23, 22, 26, 28, 25, 29])
|
||||
XCTAssertEqual(chords.map(\.flags), Array(repeating: .maskShift, count: 10))
|
||||
}
|
||||
|
||||
func testMacOSXKeysymModifierAliasesProduceTheReceivedFlags() throws {
|
||||
let cases: [(String, CGEventFlags)] = [
|
||||
("Control_L", .maskControl), ("Control_R", .maskControl),
|
||||
("Super_L", .maskCommand), ("Super_R", .maskCommand),
|
||||
("Meta_L", .maskCommand), ("Meta_R", .maskCommand),
|
||||
("Shift_L", .maskShift), ("Shift_R", .maskShift),
|
||||
("Alt_L", .maskAlternate), ("Alt_R", .maskAlternate),
|
||||
]
|
||||
for (modifier, flags) in cases {
|
||||
let chord = try XCTUnwrap(KeyMapping.parse("\(modifier)+a").first)
|
||||
XCTAssertEqual(chord.keyCode, try KeyMapping.keyCode(for: "a"), modifier)
|
||||
XCTAssertEqual(chord.flags, flags, modifier)
|
||||
}
|
||||
XCTAssertEqual(try KeyMapping.parse("Control_L+a Super_R+b").map(\.flags), [.maskControl, .maskCommand])
|
||||
}
|
||||
|
||||
func testDeleteAndBackSpaceRemainDifferentPhysicalKeys() throws {
|
||||
XCTAssertEqual(try KeyMapping.parse("Delete").first?.keyCode, CGKeyCode(kVK_ForwardDelete))
|
||||
XCTAssertEqual(try KeyMapping.parse("BackSpace").first?.keyCode, CGKeyCode(kVK_Delete))
|
||||
}
|
||||
|
||||
func testUppercaseAndNamedShiftedGlyphPreserveRequiredModifier() throws {
|
||||
let uppercase = try XCTUnwrap(KeyMapping.parse("A").first)
|
||||
XCTAssertEqual(uppercase.keyCode, try KeyMapping.keyCode(for: "a"))
|
||||
XCTAssertEqual(uppercase.flags, .maskShift)
|
||||
let punctuation = try XCTUnwrap(KeyMapping.parse("question").first)
|
||||
XCTAssertEqual(punctuation.keyCode, CGKeyCode(kVK_ANSI_Slash))
|
||||
XCTAssertEqual(punctuation.flags, .maskShift)
|
||||
XCTAssertEqual(try KeyMapping.parse("Control_R+question").first?.flags, [.maskControl, .maskShift])
|
||||
}
|
||||
|
||||
func testNewCommandAliasesCannotBypassSystemShortcutGrant() throws {
|
||||
for sequence in ["Super_L+q", "Super_R+Tab", "Meta_L+space", "Control_R+Super_R+q"] {
|
||||
XCTAssertTrue(try SystemKeyPolicy.requiresGrant(sequence), sequence)
|
||||
XCTAssertThrowsError(try SystemKeyPolicy.enforce(sequence: sequence, granted: false)) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "grant_flag_required")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func testUnsupportedModifierOnlyAndMultipleKeyChordsStillReject() {
|
||||
for sequence in ["Control_L", "Hyper_L+k", "Control_L+a+b", "a+Control_L"] {
|
||||
XCTAssertThrowsError(try KeyMapping.parse(sequence), sequence)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,164 @@
|
||||
import AppKit
|
||||
import CoreGraphics
|
||||
import XCTest
|
||||
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
/// Exercises the events allocated by the production drag path without posting
|
||||
/// them to any application. The sequence comes from the installed official
|
||||
/// SkyComputerUseService 26.831.1000926 SynthesizedEvent.click specialization:
|
||||
/// down(origin), dragged(origin), dragged(midpoint), dragged(destination), up.
|
||||
final class MouseDragEventTests: XCTestCase {
|
||||
@MainActor
|
||||
func testStationaryDragRetainsTheCompleteGesture() throws {
|
||||
try assertDrag(from: CGPoint(x: 410, y: 349), to: CGPoint(x: 410, y: 349))
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testOnePixelDragKeepsItsFractionalMidpoint() throws {
|
||||
try assertDrag(from: CGPoint(x: 410, y: 349), to: CGPoint(x: 411, y: 350))
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testLongDragUsesTheSameFiveEventsAndOriginWindow() throws {
|
||||
// The destination can leave the origin window. It must not be clamped
|
||||
// or rebound to a different window during allocation.
|
||||
try assertDrag(from: CGPoint(x: 410, y: 349), to: CGPoint(x: 2410, y: 1749))
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testDragAcrossNegativeDisplayCoordinatesPreservesGlobalPositions() throws {
|
||||
try assertDrag(from: CGPoint(x: -2201, y: -1001), to: CGPoint(x: -21, y: 499))
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testProductionDragFieldsPairTransitionsSeparatelyFromMotionForEveryButton() throws {
|
||||
let window = fixtureWindow
|
||||
for button: MouseButton in [.left, .right, .middle] {
|
||||
let events = try makeDrag(from: CGPoint(x: 410, y: 349), to: CGPoint(x: 411, y: 349), button: button)
|
||||
XCTAssertEqual(events.map(\.type), [button.down, button.dragged, button.dragged, button.dragged, button.up])
|
||||
XCTAssertEqual(events.map { $0.getIntegerValueField(.mouseEventClickState) }, [1, 0, 0, 0, 1])
|
||||
guard events.count == 5 else { continue }
|
||||
let numbers = events.map { $0.getIntegerValueField(.mouseEventNumber) }
|
||||
XCTAssertEqual(numbers[0], numbers[4], "down and up must identify the same gesture")
|
||||
XCTAssertEqual(numbers[1], numbers[2])
|
||||
XCTAssertEqual(numbers[2], numbers[3])
|
||||
XCTAssertNotEqual(numbers[0], numbers[1], "motion has a separate event number")
|
||||
for event in events {
|
||||
XCTAssertEqual(event.getIntegerValueField(.mouseEventButtonNumber), Int64(button.cg.rawValue))
|
||||
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 7)!), 3)
|
||||
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 91)!), Int64(window.id))
|
||||
XCTAssertEqual(event.getIntegerValueField(CGEventField(rawValue: 92)!), Int64(window.id))
|
||||
let bridged = try XCTUnwrap(NSEvent(cgEvent: event))
|
||||
XCTAssertEqual(bridged.windowNumber, Int(window.id))
|
||||
XCTAssertEqual(bridged.clickCount, Int(event.getIntegerValueField(.mouseEventClickState)))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testUnpacedProductionDragPostsItsWholeGestureWithoutWaitingBetweenEvents() async throws {
|
||||
let events = try makeDrag(from: CGPoint(x: 410, y: 349), to: CGPoint(x: 1410, y: 849), button: .left)
|
||||
var queuedTask: Task<Void, Never>?
|
||||
var yielded = false
|
||||
var posted: [CGEvent] = []
|
||||
var validations = 0
|
||||
|
||||
try await MouseEventBurstDelivery.deliver(
|
||||
events: events,
|
||||
validate: { validations += 1 },
|
||||
post: {
|
||||
XCTAssertFalse(yielded, "coordinate gestures must not insert per-event sleeps")
|
||||
posted.append($0)
|
||||
if posted.count == 1 { queuedTask = Task { @MainActor in yielded = true } }
|
||||
},
|
||||
release: { _, _ in XCTFail("a completed drag has already released its button") }
|
||||
)
|
||||
XCTAssertEqual(posted.map(\.type), [.leftMouseDown, .leftMouseDragged, .leftMouseDragged, .leftMouseDragged, .leftMouseUp])
|
||||
XCTAssertEqual(validations, events.count)
|
||||
for (index, event) in posted.enumerated() {
|
||||
XCTAssertTrue(event === events[index], "delivery must preserve the production event objects")
|
||||
}
|
||||
await queuedTask?.value
|
||||
XCTAssertTrue(yielded)
|
||||
}
|
||||
|
||||
func testCoordinateDragUsesTheTestedFactoryAndExplicitlyUnpacedDelivery() throws {
|
||||
// The executable delivery tests below the actuation boundary do not
|
||||
// resolve live windows. Guard the production caller's wiring as well,
|
||||
// so they cannot pass while drag still uses a different event builder
|
||||
// or accidentally inherits a future paced default.
|
||||
let source = try String(contentsOf: URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
|
||||
.appendingPathComponent("Sources/cu-helper/AXAction.swift"), encoding: .utf8)
|
||||
let start = try XCTUnwrap(source.range(of: "public static func drag("))
|
||||
let end = try XCTUnwrap(source.range(of: "static func dragEvents(", range: start.upperBound..<source.endIndex))
|
||||
let body = String(source[start.upperBound..<end.lowerBound])
|
||||
XCTAssertTrue(body.contains("let events = try dragEvents("))
|
||||
XCTAssertTrue(body.contains("try await postMouseBurst("))
|
||||
XCTAssertTrue(body.contains("pause: nil"), "coordinate drag must not insert an artificial per-event delay")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testCancellationAtEachProductionDragBoundaryReleasesOnlyTheLastPostedPoint() async throws {
|
||||
for button: MouseButton in [.left, .right, .middle] {
|
||||
let events = try makeDrag(from: CGPoint(x: 410, y: 349), to: CGPoint(x: 1410, y: 849), button: button)
|
||||
for cancelAfterPost in 0...events.count {
|
||||
var posted: [CGEvent] = []
|
||||
var releases: [(CGEvent, CGPoint)] = []
|
||||
let task = Task { @MainActor in
|
||||
if cancelAfterPost == 0 { withUnsafeCurrentTask { $0?.cancel() } }
|
||||
do {
|
||||
try await MouseEventBurstDelivery.deliver(
|
||||
events: events,
|
||||
validate: {},
|
||||
post: {
|
||||
posted.append($0)
|
||||
if posted.count == cancelAfterPost { withUnsafeCurrentTask { $0?.cancel() } }
|
||||
},
|
||||
release: { releases.append(($0, $1)) }
|
||||
)
|
||||
XCTFail("cancellation must stop the gesture, even after its final mouse-up")
|
||||
} catch is CancellationError {
|
||||
// The remaining tail must never be posted.
|
||||
} catch {
|
||||
XCTFail("unexpected error: \(error)")
|
||||
}
|
||||
}
|
||||
await task.value
|
||||
XCTAssertEqual(posted.map(\.type), events.prefix(cancelAfterPost).map(\.type))
|
||||
let hasOutstandingDown = posted.contains { $0.type == button.down }
|
||||
&& !posted.contains { $0.type == button.up }
|
||||
XCTAssertEqual(releases.count, hasOutstandingDown ? 1 : 0)
|
||||
if let release = releases.first {
|
||||
let down = try XCTUnwrap(posted.first { $0.type == button.down })
|
||||
XCTAssertTrue(release.0 === down, "cleanup must retain the original press and its event number")
|
||||
XCTAssertEqual(release.1, posted.last?.location, "cleanup must not jump to an unsent destination")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private func assertDrag(from: CGPoint, to: CGPoint, file: StaticString = #filePath, line: UInt = #line) throws {
|
||||
let midpoint = CGPoint(x: (from.x + to.x) / 2, y: (from.y + to.y) / 2)
|
||||
for button: MouseButton in [.left, .right, .middle] {
|
||||
let events = try makeDrag(from: from, to: to, button: button)
|
||||
XCTAssertEqual(events.map(\.type), [button.down, button.dragged, button.dragged, button.dragged, button.up], file: file, line: line)
|
||||
XCTAssertEqual(events.map(\.location), [from, from, midpoint, to, to], file: file, line: line)
|
||||
}
|
||||
}
|
||||
|
||||
private var fixtureWindow: WindowGeometry.Window {
|
||||
WindowGeometry.Window(id: 123, bounds: CGRect(x: 200, y: 300, width: 800, height: 600), ownerPid: 456)
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private func makeDrag(from: CGPoint, to: CGPoint, button: MouseButton) throws -> [CGEvent] {
|
||||
let window = fixtureWindow
|
||||
return try AXAction.dragEvents(
|
||||
from: from, to: to, button: button,
|
||||
source: XCTUnwrap(CGEventSource(stateID: .privateState)), pid: window.ownerPid, window: window
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -145,7 +145,7 @@ final class MouseEventBurstDeliveryTests: XCTestCase {
|
||||
XCTAssertEqual(releases, 1)
|
||||
}
|
||||
|
||||
func testCoordinateClickUsesTheTestedFactoryAndDisablesDragPacing() throws {
|
||||
func testCoordinateClickUsesTheTestedFactoryAndExplicitlyUnpacedDelivery() throws {
|
||||
let source = try String(contentsOf: URL(fileURLWithPath: #filePath)
|
||||
.deletingLastPathComponent().deletingLastPathComponent().deletingLastPathComponent()
|
||||
.appendingPathComponent("Sources/cu-helper/AXAction.swift"), encoding: .utf8)
|
||||
@@ -154,7 +154,7 @@ final class MouseEventBurstDeliveryTests: XCTestCase {
|
||||
let body = String(source[start.upperBound..<end.lowerBound])
|
||||
XCTAssertTrue(body.contains("let events = try clickEvents("))
|
||||
XCTAssertTrue(body.contains("try await postMouseBurst("))
|
||||
XCTAssertTrue(body.contains("pause: nil"), "click must not inherit drag's asynchronous pacing")
|
||||
XCTAssertTrue(body.contains("pause: nil"), "coordinate click must not insert an artificial per-event delay")
|
||||
}
|
||||
|
||||
@MainActor
|
||||
|
||||
@@ -0,0 +1,209 @@
|
||||
import AppKit
|
||||
#if !NATIVE_CONTRACT_STANDALONE
|
||||
@testable import cc_haha_computer_use
|
||||
#endif
|
||||
|
||||
/// The same disposable receiver is compiled standalone for official CUA and
|
||||
/// hosted by the XCTest fixture child for the local native router.
|
||||
@MainActor
|
||||
enum NativeMethodContractFixture {
|
||||
static func run(root: URL) throws {
|
||||
let app = NSApplication.shared
|
||||
app.setActivationPolicy(.accessory)
|
||||
// Native paste is delivered through the normal Command-V responder
|
||||
// chain. A standalone NSApplication has no nib-provided Edit menu.
|
||||
let mainMenu = NSMenu()
|
||||
let editItem = NSMenuItem(title: "Edit", action: nil, keyEquivalent: "")
|
||||
let editMenu = NSMenu(title: "Edit")
|
||||
editMenu.autoenablesItems = false
|
||||
for (title, selector, key) in [
|
||||
("Cut", #selector(NSText.cut(_:)), "x"),
|
||||
("Copy", #selector(NSText.copy(_:)), "c"),
|
||||
("Paste", #selector(NSText.paste(_:)), "v"),
|
||||
("Select All", #selector(NSText.selectAll(_:)), "a"),
|
||||
] { editMenu.addItem(withTitle: title, action: selector, keyEquivalent: key) }
|
||||
editItem.submenu = editMenu
|
||||
mainMenu.addItem(editItem)
|
||||
app.mainMenu = mainMenu
|
||||
app.finishLaunching()
|
||||
let window = ContractWindow(contentRect: NSRect(x: 120, y: 160, width: 760, height: 520), styleMask: [.titled, .closable], backing: .buffered, defer: false)
|
||||
window.delegate = window
|
||||
window.collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary, .canJoinAllApplications]
|
||||
window.title = "Native method contract fixture"
|
||||
let content = NSView(frame: window.contentView!.bounds)
|
||||
let editor = ContractEditor(frame: NSRect(x: 20, y: 310, width: 720, height: 170))
|
||||
editor.isRichText = true
|
||||
editor.font = NSFont.systemFont(ofSize: 18)
|
||||
editor.string = "alpha beta gamma"
|
||||
editor.setAccessibilityLabel("Contract editor")
|
||||
content.addSubview(editor)
|
||||
let scroll = ContractScrollView(frame: NSRect(x: 20, y: 70, width: 720, height: 210))
|
||||
// Native AX page increments depend on the visible clip size. Pin the
|
||||
// receiver geometry instead of inheriting dynamic system scrollbar style.
|
||||
scroll.scrollerStyle = .legacy
|
||||
scroll.hasVerticalScroller = true
|
||||
scroll.hasHorizontalScroller = true
|
||||
scroll.setAccessibilityLabel("Contract scroll")
|
||||
let document = NSTextView(frame: NSRect(x: 0, y: 0, width: 1400, height: 1800))
|
||||
document.isEditable = false
|
||||
document.font = NSFont.systemFont(ofSize: 18)
|
||||
document.string = (1...70).map { "Disposable row \($0)" }.joined(separator: "\n")
|
||||
document.setAccessibilityLabel("Contract rows")
|
||||
scroll.documentView = document
|
||||
content.addSubview(scroll)
|
||||
let action = ContractSecondaryView(frame: NSRect(x: 20, y: 15, width: 400, height: 40))
|
||||
action.setAccessibilityElement(true)
|
||||
action.setAccessibilityRole(.button)
|
||||
action.setAccessibilityLabel("Contract secondary action")
|
||||
content.addSubview(action)
|
||||
window.contentView = content
|
||||
window.orderFrontRegardless()
|
||||
window.makeMain()
|
||||
window.makeFirstResponder(editor)
|
||||
let ready = root.appendingPathComponent("ready")
|
||||
let receipt = root.appendingPathComponent("contract.json")
|
||||
var previous: Data?
|
||||
let timer = Timer.scheduledTimer(withTimeInterval: 0.02, repeats: true) { _ in
|
||||
MainActor.assumeIsolated {
|
||||
let selection = editor.selectedRange()
|
||||
let value: [String: Any] = [
|
||||
"text": editor.string,
|
||||
"selectionLocation": selection.location,
|
||||
"selectionLength": selection.length,
|
||||
"scrollY": scroll.contentView.bounds.origin.y,
|
||||
"scrollX": scroll.contentView.bounds.origin.x,
|
||||
"wheel": scroll.events,
|
||||
"secondaryCount": action.count,
|
||||
"visible": window.isVisible,
|
||||
"hidden": app.isHidden,
|
||||
"windowNumber": window.windowNumber,
|
||||
"windowFrame": NSStringFromRect(window.frame),
|
||||
"onActiveSpace": window.isOnActiveSpace,
|
||||
"miniaturized": window.isMiniaturized,
|
||||
"occlusionState": window.occlusionState.rawValue,
|
||||
"windowEvents": window.events,
|
||||
"clipSize": NSStringFromSize(scroll.contentView.bounds.size),
|
||||
"scrollerStyle": scroll.scrollerStyle.rawValue,
|
||||
"editorEvents": editor.events,
|
||||
]
|
||||
if let data = try? JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]), data != previous {
|
||||
try? data.write(to: receipt, options: .atomic)
|
||||
previous = data
|
||||
}
|
||||
if FileManager.default.fileExists(atPath: root.appendingPathComponent("stop").path) { exit(0) }
|
||||
}
|
||||
}
|
||||
defer { timer.invalidate() }
|
||||
// NSWorkspace/open -g finishes its background-launch policy after
|
||||
// main() starts. Publish readiness only after that launch boundary.
|
||||
Timer.scheduledTimer(withTimeInterval: 0.2, repeats: false) { _ in
|
||||
MainActor.assumeIsolated {
|
||||
window.orderFrontRegardless()
|
||||
try? Data(String(ProcessInfo.processInfo.processIdentifier).utf8).write(to: ready)
|
||||
}
|
||||
}
|
||||
// A bounded receiver process never survives a failed parent test.
|
||||
Timer.scheduledTimer(withTimeInterval: 600, repeats: false) { _ in exit(0) }
|
||||
app.run()
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private final class ContractWindow: NSWindow, NSWindowDelegate {
|
||||
var events: [[String: Any]] = []
|
||||
private func record(_ kind: String, includeStack: Bool = false) {
|
||||
var event: [String: Any] = [
|
||||
"kind": kind, "uptime": ProcessInfo.processInfo.systemUptime,
|
||||
"visible": isVisible, "miniaturized": isMiniaturized,
|
||||
"onActiveSpace": isOnActiveSpace,
|
||||
]
|
||||
if let current = NSApplication.shared.currentEvent {
|
||||
event["eventType"] = current.type.rawValue
|
||||
if [.keyDown, .keyUp, .flagsChanged].contains(current.type) {
|
||||
event["keyCode"] = current.keyCode
|
||||
}
|
||||
event["eventMarker"] = current.cgEvent?.getIntegerValueField(.eventSourceUserData)
|
||||
}
|
||||
if includeStack { event["stack"] = Array(Thread.callStackSymbols.prefix(12)) }
|
||||
events.append(event)
|
||||
if events.count > 40 { events.removeFirst(events.count - 40) }
|
||||
}
|
||||
override func orderOut(_ sender: Any?) {
|
||||
record("orderOut", includeStack: true)
|
||||
super.orderOut(sender)
|
||||
}
|
||||
override func close() {
|
||||
record("close", includeStack: true)
|
||||
super.close()
|
||||
}
|
||||
override func miniaturize(_ sender: Any?) {
|
||||
record("miniaturize", includeStack: true)
|
||||
super.miniaturize(sender)
|
||||
}
|
||||
func windowWillClose(_ notification: Notification) { record("willClose") }
|
||||
func windowDidMiniaturize(_ notification: Notification) { record("didMiniaturize") }
|
||||
func windowDidDeminiaturize(_ notification: Notification) { record("didDeminiaturize") }
|
||||
func windowDidChangeOcclusionState(_ notification: Notification) { record("occlusionChanged") }
|
||||
func windowDidBecomeKey(_ notification: Notification) { record("becameKey") }
|
||||
func windowDidResignKey(_ notification: Notification) { record("resignedKey") }
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private final class ContractScrollView: NSScrollView {
|
||||
var events: [[String: Any]] = []
|
||||
override func scrollWheel(with event: NSEvent) {
|
||||
events.append(["x": event.scrollingDeltaX, "y": event.scrollingDeltaY, "precise": event.hasPreciseScrollingDeltas])
|
||||
super.scrollWheel(with: event)
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private final class ContractEditor: NSTextView {
|
||||
var events: [[String: Any]] = []
|
||||
override func paste(_ sender: Any?) {
|
||||
if let name = ProcessInfo.processInfo.environment["CC_HAHA_METHOD_FIXTURE_PASTEBOARD"] {
|
||||
// The real menu/responder chain consumes promised data from the
|
||||
// test-owned pasteboard, without consulting the user's clipboard.
|
||||
_ = readSelection(from: NSPasteboard(name: NSPasteboard.Name(name)))
|
||||
} else {
|
||||
super.paste(sender)
|
||||
}
|
||||
}
|
||||
override func keyDown(with event: NSEvent) {
|
||||
#if !NATIVE_CONTRACT_STANDALONE
|
||||
// The test app is a receiver, never a destination for physical typing.
|
||||
// Native generated keys are marked; AX writes need no keyboard event.
|
||||
guard event.cgEvent?.getIntegerValueField(.eventSourceUserData) == HelperEventMarker.value else { return }
|
||||
#endif
|
||||
events.append(["kind": "keyDown", "keyCode": Int(event.keyCode), "flags": event.modifierFlags.rawValue])
|
||||
super.keyDown(with: event)
|
||||
}
|
||||
override func didChangeText() {
|
||||
super.didChangeText()
|
||||
events.append(["kind": "changed", "text": string])
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
private final class ContractSecondaryView: NSView {
|
||||
var count = 0
|
||||
override func accessibilityPerformShowMenu() -> Bool {
|
||||
count += 1
|
||||
needsDisplay = true
|
||||
return true
|
||||
}
|
||||
override func draw(_ dirtyRect: NSRect) {
|
||||
NSColor.windowBackgroundColor.setFill()
|
||||
dirtyRect.fill()
|
||||
("Secondary actions received: \(count)" as NSString).draw(at: NSPoint(x: 5, y: 8), withAttributes: [.font: NSFont.systemFont(ofSize: 16)])
|
||||
}
|
||||
}
|
||||
|
||||
#if NATIVE_CONTRACT_STANDALONE
|
||||
@main
|
||||
struct NativeContractMain {
|
||||
@MainActor static func main() throws {
|
||||
try NativeMethodContractFixture.run(root: URL(fileURLWithPath: CommandLine.arguments[1]))
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -0,0 +1,48 @@
|
||||
import AppKit
|
||||
import ImageIO
|
||||
import UniformTypeIdentifiers
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class NativeScreenshotPolicyTests: XCTestCase {
|
||||
func testPointResolutionAndOfficialLongAndShortSideLimits() {
|
||||
for backing in [1.0, 2.0] {
|
||||
XCTAssertEqual(NativeScreenshotPolicy.pixelSize(pointSize: CGSize(width: 620, height: 392), backingScale: backing), CGSize(width: 620, height: 392))
|
||||
XCTAssertEqual(NativeScreenshotPolicy.pixelSize(pointSize: CGSize(width: 1398, height: 769), backingScale: backing), CGSize(width: 1397, height: 768))
|
||||
XCTAssertEqual(NativeScreenshotPolicy.pixelSize(pointSize: CGSize(width: 4000, height: 1000), backingScale: backing), CGSize(width: 2048, height: 512))
|
||||
XCTAssertEqual(NativeScreenshotPolicy.pixelSize(pointSize: CGSize(width: 1000, height: 4000), backingScale: backing), CGSize(width: 512, height: 2048))
|
||||
XCTAssertEqual(NativeScreenshotPolicy.pixelSize(pointSize: CGSize(width: 1000, height: 1000), backingScale: backing), CGSize(width: 768, height: 768))
|
||||
}
|
||||
}
|
||||
|
||||
@MainActor
|
||||
func testJPEGEncodingMatchesOfficialQuantizationAndPreservesImageSize() throws {
|
||||
let context = try XCTUnwrap(CGContext(data: nil, width: 16, height: 16, bitsPerComponent: 8, bytesPerRow: 0, space: CGColorSpaceCreateDeviceRGB(), bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue))
|
||||
context.setFillColor(CGColor(red: 0.7, green: 0.2, blue: 0.1, alpha: 1))
|
||||
context.fill(CGRect(x: 0, y: 0, width: 16, height: 16))
|
||||
let image = try XCTUnwrap(context.makeImage())
|
||||
let encoded = try XCTUnwrap(Capture.appScreenshotBase64WithSize(image))
|
||||
let data = try XCTUnwrap(Data(base64Encoded: encoded.base64))
|
||||
let source = try XCTUnwrap(CGImageSourceCreateWithData(data as CFData, nil))
|
||||
XCTAssertEqual(CGImageSourceGetType(source) as String?, UTType.jpeg.identifier)
|
||||
XCTAssertEqual(encoded.width, 16)
|
||||
XCTAssertEqual(encoded.height, 16)
|
||||
let decoded = try XCTUnwrap(CGImageSourceCreateImageAtIndex(source, 0, nil))
|
||||
XCTAssertEqual(decoded.width, 16)
|
||||
// DQT is independent of fixture pixels. These tables were read from
|
||||
// the actual official getScreenshot JPEG and match ImageIO quality .8.
|
||||
let expectedLuminance: [UInt8] = [0,2,2,2,2,2,2,3,2,2,3,4,3,3,3,4,5,4,4,4,4,5,7,5,5,5,5,5,7,8,7,7,7,7,7,7,8,8,8,8,8,8,8,8,10,10,10,10,10,10,11,11,11,11,11,13,13,13,13,13,13,13,13,13,13]
|
||||
let bytes = Array(data)
|
||||
var offset = 2
|
||||
var quantization: [[UInt8]] = []
|
||||
while offset + 4 < bytes.count {
|
||||
let marker = bytes[offset + 1]
|
||||
if marker == 0xda { break }
|
||||
let length = Int(bytes[offset + 2]) * 256 + Int(bytes[offset + 3])
|
||||
guard length >= 2, offset + 2 + length <= bytes.count else { break }
|
||||
if marker == 0xdb { quantization.append(Array(bytes[(offset + 4)..<(offset + 2 + length)])) }
|
||||
offset += length + 2
|
||||
}
|
||||
XCTAssertEqual(quantization.first, expectedLuminance)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
import AppKit
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
final class NativeScrollTests: XCTestCase {
|
||||
func testSecondaryPageActionsChooseOfficialAxisAndButtonSubrole() throws {
|
||||
for (action, axis, subrole) in [
|
||||
("AXScrollUpByPage", "AXVerticalScrollBar", "AXDecrementPage"),
|
||||
("AXScrollDownByPage", "AXVerticalScrollBar", "AXIncrementPage"),
|
||||
("AXScrollLeftByPage", "AXHorizontalScrollBar", "AXDecrementPage"),
|
||||
("AXScrollRightByPage", "AXHorizontalScrollBar", "AXIncrementPage"),
|
||||
] {
|
||||
let result = try XCTUnwrap(NativeScroll.pageNavigation(action: action))
|
||||
XCTAssertEqual(result.axisAttribute, axis)
|
||||
XCTAssertEqual(result.buttonSubrole, subrole)
|
||||
}
|
||||
XCTAssertNil(NativeScroll.pageNavigation(action: "AXPress"))
|
||||
}
|
||||
|
||||
func testFractionalPagesUseAddressedFrameAndRoundPixelDeltas() throws {
|
||||
let element = CGSize(width: 720, height: 210)
|
||||
XCTAssertEqual(try NativeScroll.delta(direction: "down", pages: 0.5, frameSize: element).y, -105)
|
||||
XCTAssertEqual(try NativeScroll.delta(direction: "down", pages: 1.5, frameSize: element).y, -315)
|
||||
XCTAssertEqual(try NativeScroll.delta(direction: "down", pages: 0.123, frameSize: element).y, -26)
|
||||
XCTAssertEqual(try NativeScroll.delta(direction: "up", pages: 0.5, frameSize: element).y, 105)
|
||||
XCTAssertEqual(try NativeScroll.delta(direction: "right", pages: 0.5, frameSize: element).x, -360)
|
||||
XCTAssertEqual(try NativeScroll.delta(direction: "left", pages: 1.5, frameSize: element).x, 1080)
|
||||
XCTAssertEqual(try NativeScroll.delta(direction: "down", pages: 0.5, frameSize: CGSize(width: 760, height: 552)).y, -276)
|
||||
for pages in [0, -1, Double.infinity, Double.nan] {
|
||||
XCTAssertThrowsError(try NativeScroll.delta(direction: "down", pages: pages, frameSize: element))
|
||||
}
|
||||
}
|
||||
|
||||
func testPreciseScrollEventCarriesBothAxesAndTargetWindow() throws {
|
||||
let source = try XCTUnwrap(CGEventSource(stateID: .privateState))
|
||||
let window = WindowGeometry.Window(id: 71, bounds: CGRect(x: 100, y: 200, width: 760, height: 552), ownerPid: 123)
|
||||
let event = try XCTUnwrap(WindowTargetedEvent.makeScrollEvent(source: source, point: CGPoint(x: 300, y: 400), deltaX: -360, deltaY: -105, window: window))
|
||||
let native = try XCTUnwrap(NSEvent(cgEvent: event))
|
||||
XCTAssertTrue(native.hasPreciseScrollingDeltas)
|
||||
XCTAssertEqual(native.scrollingDeltaX, -360)
|
||||
XCTAssertEqual(native.scrollingDeltaY, -105)
|
||||
XCTAssertEqual(native.windowNumber, 71)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import XCTest
|
||||
@testable import cc_haha_computer_use
|
||||
|
||||
#if DEBUG
|
||||
final class ProcessValidationObservationTests: XCTestCase {
|
||||
@MainActor
|
||||
func testObserverSeesTheActualFailedComparisonWithoutAllowingTheTarget() throws {
|
||||
let expected = try XCTUnwrap(ProvenProcessTarget(
|
||||
pid: .max,
|
||||
identity: AXTreeProcessIdentity(
|
||||
bundleID: "dev.cchaha.tests.missing",
|
||||
executablePath: "/missing/process",
|
||||
launchTime: 100
|
||||
)
|
||||
))
|
||||
let previous = Injection.targetValidationObserver
|
||||
defer { Injection.targetValidationObserver = previous }
|
||||
var observed: [(ProvenProcessTarget, AXTreeProcessIdentity?)] = []
|
||||
Injection.targetValidationObserver = { observed.append(($0, $1)) }
|
||||
XCTAssertThrowsError(try Injection.validateAuthorizedTarget(expected)) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "stale_process")
|
||||
}
|
||||
XCTAssertEqual(observed.count, 1)
|
||||
XCTAssertEqual(observed.first?.0, expected)
|
||||
XCTAssertNil(observed.first?.1)
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -12,6 +12,59 @@ final class ResolvedTargetAuthorizationTests: XCTestCase {
|
||||
executablePath: "/System/Applications/Calculator.app/Contents/MacOS/Calculator",
|
||||
launchTime: 200
|
||||
)
|
||||
private let chromeIdentity = AXTreeProcessIdentity(
|
||||
bundleID: "com.google.Chrome",
|
||||
executablePath: "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
launchTime: 300
|
||||
)
|
||||
|
||||
func testChromePIDBundleNameAndPathSelectorsPreserveExactProvenIdentity() throws {
|
||||
let chrome = AppTargetCandidate(
|
||||
pid: 43,
|
||||
bundleIdentifier: "com.google.Chrome",
|
||||
bundleURL: URL(fileURLWithPath: "/Applications/Google Chrome.app"),
|
||||
localizedName: "Google Chrome",
|
||||
executableName: "Google Chrome"
|
||||
)
|
||||
let selectors: [AppTargetSelector] = [
|
||||
.pid(43),
|
||||
.bundleIdentifier("com.google.Chrome"),
|
||||
.app("com.google.Chrome"),
|
||||
.app("Google Chrome"),
|
||||
.app("Google Chrome.app"),
|
||||
.app("/Applications/Google Chrome.app"),
|
||||
]
|
||||
|
||||
for selector in selectors {
|
||||
let resolved = try XCTUnwrap(
|
||||
AppTargetResolver.resolve(selector: selector, candidates: [chrome])
|
||||
)
|
||||
let target = try ResolvedTargetAuthorization.authorize(
|
||||
resolved: resolved,
|
||||
currentIdentity: chromeIdentity
|
||||
)
|
||||
XCTAssertEqual(target.pid, chrome.pid)
|
||||
XCTAssertEqual(target.identity, chromeIdentity)
|
||||
}
|
||||
}
|
||||
|
||||
func testAllowingChromeStillRejectsUnprovenOrMismatchedIdentity() {
|
||||
let unproven = AXTreeProcessIdentity(
|
||||
bundleID: chromeIdentity.bundleID,
|
||||
executablePath: chromeIdentity.executablePath,
|
||||
launchTime: nil
|
||||
)
|
||||
XCTAssertThrowsError(try ResolvedTargetAuthorization.authorize(
|
||||
pid: 43, identity: unproven, expectedBundleID: "com.google.Chrome"
|
||||
)) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "app_denied")
|
||||
}
|
||||
XCTAssertThrowsError(try ResolvedTargetAuthorization.authorize(
|
||||
pid: 43, identity: terminalIdentity, expectedBundleID: "com.google.Chrome"
|
||||
)) {
|
||||
XCTAssertEqual(($0 as? CUError)?.code, "stale_process")
|
||||
}
|
||||
}
|
||||
|
||||
func testNumericPIDCannotBypassDeniedResolvedBundle() {
|
||||
XCTAssertThrowsError(
|
||||
|
||||
@@ -707,7 +707,7 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
))
|
||||
}
|
||||
|
||||
func testCopiedBGRAFrameEncodesAsAStreamPNGWithTheSameGeometry() throws {
|
||||
func testCopiedBGRAFrameEncodesAsAStreamJPEGWithTheSameGeometry() throws {
|
||||
let target = makeTarget(
|
||||
windowID: 72,
|
||||
pixelWidth: 1,
|
||||
@@ -725,9 +725,9 @@ final class WindowCaptureStreamTests: XCTestCase {
|
||||
)
|
||||
|
||||
let shot = try XCTUnwrap(Capture.windowShot(from: frame, target: target))
|
||||
let png = try XCTUnwrap(Data(base64Encoded: shot.base64))
|
||||
let jpeg = try XCTUnwrap(Data(base64Encoded: shot.base64))
|
||||
|
||||
XCTAssertEqual(Array(png.prefix(8)), [137, 80, 78, 71, 13, 10, 26, 10])
|
||||
XCTAssertEqual(Array(jpeg.prefix(2)), [255, 216])
|
||||
XCTAssertEqual(shot.width, 1)
|
||||
XCTAssertEqual(shot.height, 1)
|
||||
XCTAssertEqual(shot.originX, 300)
|
||||
|
||||
+3
-2
@@ -14,7 +14,7 @@
|
||||
"perf:local-index:10k": "bun run scripts/perf/local-index-benchmark.ts --sessions 10000 --runs 20",
|
||||
"check:pr": "bun run scripts/pr/check-pr.ts",
|
||||
"check:impact": "bun run scripts/pr/impact-report.ts",
|
||||
"check:policy": "bun test ./scripts/pr/bun-test-filter.test.ts ./scripts/pr/change-policy.test.ts ./scripts/pr/changed-files.test.ts ./scripts/pr/dead-imports.test.ts ./scripts/pr/module-graph.test.ts ./scripts/pr/pr-triage-workflow.test.ts ./scripts/pr/pr-quality-workflow.test.ts ./scripts/pr/release-workflow.test.ts ./scripts/pr/quality-contract.test.ts ./scripts/pr/test-environment.test.ts ./scripts/pr/run-swift-checks.test.ts ./scripts/release-update-metadata.test.ts ./scripts/git-hooks/install.test.ts ./scripts/quality-gate/quarantine.test.ts ./scripts/quality-gate/coverage.test.ts ./scripts/quality-gate/provider-smoke/execute.test.ts ./scripts/quality-gate/desktop-smoke/execute.test.ts ./scripts/quality-gate/providerTargets.test.ts ./scripts/quality-gate/runner.test.ts ./scripts/quality-gate/sandbox.test.ts ./scripts/quality-gate/agent-flow/scenarios.test.ts ./scripts/quality-gate/agent-flow/live.test.ts ./scripts/quality-gate/desktop-smoke/deterministic.test.ts ./scripts/quality-gate/computer-use-live-smoke.test.ts",
|
||||
"check:policy": "bun test ./scripts/pr/bun-test-filter.test.ts ./scripts/pr/change-policy.test.ts ./scripts/pr/changed-files.test.ts ./scripts/pr/dead-imports.test.ts ./scripts/pr/module-graph.test.ts ./scripts/pr/pr-triage-workflow.test.ts ./scripts/pr/pr-quality-workflow.test.ts ./scripts/pr/release-workflow.test.ts ./scripts/pr/quality-contract.test.ts ./scripts/pr/test-environment.test.ts ./scripts/pr/run-swift-checks.test.ts ./scripts/release-update-metadata.test.ts ./scripts/git-hooks/install.test.ts ./scripts/quality-gate/quarantine.test.ts ./scripts/quality-gate/coverage.test.ts ./scripts/quality-gate/provider-smoke/execute.test.ts ./scripts/quality-gate/desktop-smoke/execute.test.ts ./scripts/quality-gate/providerTargets.test.ts ./scripts/quality-gate/runner.test.ts ./scripts/quality-gate/sandbox.test.ts ./scripts/quality-gate/agent-flow/scenarios.test.ts ./scripts/quality-gate/agent-flow/live.test.ts ./scripts/quality-gate/desktop-smoke/deterministic.test.ts ./scripts/quality-gate/computer-use-live-smoke.test.ts ./scripts/quality-gate/computer-use-signed-chain.test.ts",
|
||||
"check:server": "bun run scripts/pr/run-server-tests.ts",
|
||||
"check:provider-contract": "bun run scripts/pr/run-provider-contract-tests.ts",
|
||||
"check:chat-contract": "bun run scripts/pr/run-chat-contract-tests.ts",
|
||||
@@ -45,7 +45,8 @@
|
||||
"docs:preview": "npm --prefix site run preview",
|
||||
"check:agent-flow:live": "bun run scripts/quality-gate/agent-flow/live-cli.ts",
|
||||
"check:swift": "bun run scripts/pr/run-swift-checks.ts",
|
||||
"check:computer-use-live-smoke": "bun run scripts/quality-gate/computer-use-live-smoke.ts"
|
||||
"check:computer-use-live-smoke": "bun run scripts/quality-gate/computer-use-live-smoke.ts",
|
||||
"check:computer-use-signed-chain": "bun run scripts/quality-gate/computer-use-signed-chain.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@anthropic-ai/sandbox-runtime": "^0.0.44",
|
||||
|
||||
@@ -137,6 +137,11 @@ describe('this repository', () => {
|
||||
// desktop/tsconfig.json, so `check:desktop` alone cannot prove it still builds.
|
||||
expect(dependentsOf(['desktop/src/lib/browserSafePort.ts'], graph))
|
||||
.toContain('desktop/electron/services/sidecarManager.ts')
|
||||
// The desktop CU worker enters the merged sidecar before normal routing.
|
||||
expect(dependentsOf(['src/utils/computerUse/replWorker.ts'], graph))
|
||||
.toContain('desktop/sidecars/claude-sidecar.ts')
|
||||
expect(dependentsOf(['preload.ts'], graph))
|
||||
.toContain('desktop/sidecars/claude-sidecar.ts')
|
||||
})
|
||||
|
||||
test('resolves every repo-local module specifier so selection stays trustworthy', () => {
|
||||
|
||||
@@ -22,7 +22,7 @@ const RESOLUTION_EXTENSIONS = ['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.m
|
||||
const INDEX_BASENAMES = ['index.ts', 'index.tsx', 'index.js', 'index.jsx', 'index.mjs', 'index.cjs'] as const
|
||||
const SKIPPED_DIRECTORIES = new Set(['node_modules', '.git', 'dist', 'build', 'coverage', 'artifacts', 'electron-dist', 'build-artifacts', 'target'])
|
||||
|
||||
export const GRAPH_SOURCE_ROOTS = ['src', 'desktop/src', 'desktop/electron', 'adapters', 'scripts'] as const
|
||||
export const GRAPH_SOURCE_ROOTS = ['src', 'desktop/src', 'desktop/electron', 'desktop/sidecars', 'adapters', 'scripts', 'preload.ts'] as const
|
||||
|
||||
/**
|
||||
* Matches `import ... from 'x'`, `export ... from 'x'`, bare `import 'x'`,
|
||||
@@ -80,7 +80,9 @@ export function listGraphSourceFiles(rootDir: string, roots: readonly string[] =
|
||||
|
||||
for (const root of roots) {
|
||||
const absolute = join(rootDir, root)
|
||||
if (existsSync(absolute)) walk(absolute)
|
||||
if (!existsSync(absolute)) continue
|
||||
if (statSync(absolute).isFile() && isSourceFile(root)) files.push(normalizeGraphPath(root))
|
||||
else walk(absolute)
|
||||
}
|
||||
|
||||
return files.sort()
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
import { expect, test } from 'bun:test'
|
||||
import path from 'node:path'
|
||||
import { evaluateSignedChainReport, fixtureSigningEnvironment, parseSignedChainArgs, planFixtureSigning, privateKeychainCommands, verifySignedChainReport } from './computer-use-signed-chain'
|
||||
|
||||
test('default signing plans use only the explicitly scoped private keychain', () => {
|
||||
const directory = path.resolve('/tmp/fixture')
|
||||
const keychain = path.join(directory, 'fixture.keychain-db')
|
||||
const plan = planFixtureSigning({ directory })
|
||||
expect(plan.mode).toBe('temporary')
|
||||
expect(plan.requiresPackagedInstall).toBe(false)
|
||||
expect(plan.setupCommands('password').every(command => command.args.includes(keychain))).toBe(true)
|
||||
expect(plan.signCommand('/tmp/fixture/helper.app', 'dev.cchaha.cu-helper')).toEqual({
|
||||
command: path.join(directory, 'private-signer'),
|
||||
args: [keychain, '/tmp/fixture/helper.app', 'dev.cchaha.cu-helper'],
|
||||
})
|
||||
expect(() => planFixtureSigning({ directory: 'relative' })).toThrow()
|
||||
})
|
||||
|
||||
test('real identity signing must be explicit, nonempty, and cannot fall back to ad-hoc signing', () => {
|
||||
const identity = 'Developer ID Application: Disposable Test (TEAM123456)'
|
||||
const plan = planFixtureSigning({ directory: '/tmp/fixture', signingIdentity: identity })
|
||||
expect(plan.mode).toBe('explicit')
|
||||
expect(plan.requiresPackagedInstall).toBe(true)
|
||||
expect(plan.setupCommands('unused')).toEqual([])
|
||||
expect(plan.signCommand('/tmp/fixture/sidecar', 'sidecar', '/tmp/fixture/entitlements.plist')).toEqual({
|
||||
command: '/usr/bin/codesign',
|
||||
args: ['--force', '--sign', identity, '--identifier', 'sidecar', '--options', 'runtime', '--timestamp=none', '--entitlements', '/tmp/fixture/entitlements.plist', '/tmp/fixture/sidecar'],
|
||||
})
|
||||
for (const signingIdentity of ['', ' ', '-', '--deep']) {
|
||||
expect(() => planFixtureSigning({ directory: '/tmp/fixture', signingIdentity })).toThrow()
|
||||
}
|
||||
})
|
||||
|
||||
test('the CLI never discovers a signing identity from defaults or a missing flag value', () => {
|
||||
expect(parseSignedChainArgs([])).toEqual({})
|
||||
expect(parseSignedChainArgs(['report.json'])).toEqual({ output: 'report.json' })
|
||||
expect(parseSignedChainArgs(['--signing-identity', 'Explicit certificate', 'report.json'])).toEqual({ signingIdentity: 'Explicit certificate', output: 'report.json' })
|
||||
for (const args of [['--signing-identity'], ['--signing-identity', ' '], ['--signing-identity', '-'], ['--signing-identity', 'A', '--signing-identity', 'B'], ['--unknown']]) {
|
||||
expect(() => parseSignedChainArgs(args)).toThrow()
|
||||
}
|
||||
})
|
||||
|
||||
test('an authorized signing keychain is explicit even when the test HOME is isolated', () => {
|
||||
const signingIdentity = 'Developer ID Application: Disposable Test (TEAM123456)'
|
||||
const signingKeychain = '/authorized/keychains/signing.keychain-db'
|
||||
expect(parseSignedChainArgs(['--signing-identity', signingIdentity, '--signing-keychain', signingKeychain, 'report.json']))
|
||||
.toEqual({ signingIdentity, signingKeychain, output: 'report.json' })
|
||||
const plan = planFixtureSigning({ directory: '/tmp/fixture', signingIdentity, signingKeychain })
|
||||
const command = plan.signCommand('/tmp/fixture/helper.app', 'dev.cchaha.cu-helper')
|
||||
expect(command.args).toContain('--keychain')
|
||||
expect(command.args[command.args.indexOf('--keychain') + 1]).toBe(signingKeychain)
|
||||
expect(plan.setupCommands('unused')).toEqual([])
|
||||
})
|
||||
|
||||
test('a signing keychain cannot implicitly enable real-key signing or use an ambiguous path', () => {
|
||||
for (const signingKeychain of ['', ' ', 'relative.keychain-db', '--deep']) {
|
||||
expect(() => planFixtureSigning({ directory: '/tmp/fixture', signingIdentity: 'Explicit', signingKeychain })).toThrow()
|
||||
}
|
||||
expect(() => planFixtureSigning({ directory: '/tmp/fixture', signingKeychain: '/authorized/keychain' })).toThrow()
|
||||
for (const args of [
|
||||
['--signing-keychain', '/authorized/keychain'],
|
||||
['--signing-identity', 'Explicit', '--signing-keychain'],
|
||||
['--signing-identity', 'Explicit', '--signing-keychain', '/first', '--signing-keychain', '/second'],
|
||||
]) expect(() => parseSignedChainArgs(args)).toThrow()
|
||||
})
|
||||
|
||||
test('only explicit signing gets the authorized keychain HOME while app configuration stays isolated', () => {
|
||||
const isolated = { HOME: '/tmp/fixture/home', CFFIXED_USER_HOME: '/tmp/fixture/home', CLAUDE_CONFIG_DIR: '/tmp/fixture/config', TMPDIR: '/tmp/fixture/tmp' }
|
||||
expect(fixtureSigningEnvironment('temporary', isolated, '/authorized/home')).toBe(isolated)
|
||||
expect(fixtureSigningEnvironment('explicit', isolated, '/authorized/home')).toEqual({ ...isolated, HOME: '/authorized/home' })
|
||||
expect(isolated.HOME).toBe('/tmp/fixture/home')
|
||||
for (const userHome of [undefined, '', 'relative']) {
|
||||
expect(() => fixtureSigningEnvironment('explicit', isolated, userHome)).toThrow()
|
||||
}
|
||||
})
|
||||
|
||||
test('private signing fixtures always address their explicit keychain and never global trust or search lists', () => {
|
||||
const keychain = '/tmp/fixture/private.keychain-db'
|
||||
const commands = privateKeychainCommands(keychain, '/tmp/fixture/identity.p12', 'disposable')
|
||||
expect(commands).toHaveLength(4)
|
||||
for (const args of commands) expect(args).toContain(keychain)
|
||||
expect(commands.map(args => args[0])).toEqual(['create-keychain', 'unlock-keychain', 'import', 'set-key-partition-list'])
|
||||
expect(() => privateKeychainCommands('relative.keychain', '/tmp/id', 'test')).toThrow()
|
||||
expect(() => privateKeychainCommands('/tmp/login.keychain-db', '/tmp/id', 'test')).toThrow()
|
||||
})
|
||||
|
||||
test('an authenticated permission denial cannot be reported as a native GUI replay pass', () => {
|
||||
const report = {
|
||||
result: {
|
||||
ping: { protocolVersion: 'CCHahaComputerUseIPC-2' },
|
||||
permissions: { accessibility: false, screenRecording: false },
|
||||
first: { content: [{ text: '1' }] }, second: { content: [{ text: '2' }] },
|
||||
nativeObservation: { isError: true },
|
||||
},
|
||||
receiver: { completed: 0, held: false, unpaired: 0 },
|
||||
unauthorizedDirect: { ok: false, error: { code: 'unauthorized_client' } },
|
||||
}
|
||||
expect(verifySignedChainReport(report)).toBe('blocked_os_permissions')
|
||||
expect(() => verifySignedChainReport({ ...report, receiver: { ...report.receiver, completed: 1 } })).toThrow()
|
||||
expect(() => verifySignedChainReport({ ...report, result: { ...report.result, permissions: { accessibility: true, screenRecording: true } } })).toThrow()
|
||||
const failedReplay = { ...report, result: { ...report.result, permissions: { accessibility: true, screenRecording: true }, nativeObservation: {}, replay: { isError: true } }, receiver: { completed: 3, held: true, unpaired: 0 } }
|
||||
const captured = evaluateSignedChainReport(failedReplay)
|
||||
expect(captured.outcome).toBe('failed_validation')
|
||||
expect(captured.validationError).toContain('Native gesture replay failed')
|
||||
expect(captured.receiver).toEqual(failedReplay.receiver)
|
||||
expect(captured.result).toEqual(failedReplay.result)
|
||||
expect(evaluateSignedChainReport(report).outcome).toBe('blocked_os_permissions')
|
||||
expect(evaluateSignedChainReport({ ...failedReplay, result: { ...failedReplay.result, nativeObservation: { isError: true } } }).validationError).toContain('Native app observation failed')
|
||||
expect(evaluateSignedChainReport({ ...failedReplay, result: { ...failedReplay.result, nativeObservation: undefined, replay: {} }, receiver: { completed: 12, held: false, unpaired: 0 } }).outcome).toBe('failed_validation')
|
||||
expect(verifySignedChainReport({ ...report, result: { ...report.result, permissions: { accessibility: true, screenRecording: true }, nativeObservation: {}, replay: {} }, receiver: { ...report.receiver, completed: 12 } })).toBe('passed_native_gui')
|
||||
const signed = { ...report, signing: { mode: 'explicit' as const, teamIdentifier: 'TEAM123456' }, result: { ...report.result, packagedInstall: true } }
|
||||
expect(verifySignedChainReport(signed)).toBe('blocked_os_permissions')
|
||||
expect(() => verifySignedChainReport({ ...signed, result: { ...signed.result, packagedInstall: false } })).toThrow()
|
||||
for (const teamIdentifier of [undefined, '', ' ', 'not set']) {
|
||||
expect(() => verifySignedChainReport({ ...signed, signing: { mode: 'explicit', teamIdentifier } })).toThrow()
|
||||
}
|
||||
})
|
||||
@@ -0,0 +1,374 @@
|
||||
import { spawn } from 'node:child_process'
|
||||
import { copyFile, cp, mkdir, mkdtemp, readFile, realpath, rm, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { createSandboxedTestEnvironment } from '../pr/test-environment'
|
||||
|
||||
const repoRoot = path.resolve(import.meta.dirname, '../..')
|
||||
|
||||
type SignedChainOptions = { signingIdentity?: string, signingKeychain?: string }
|
||||
type PlannedCommand = { command: string, args: string[] }
|
||||
|
||||
function explicitSigningIdentity(identity: string): string {
|
||||
const value = identity.trim()
|
||||
if (!value || value.startsWith('-')) throw new Error('--signing-identity requires a nonempty certificate name; ad-hoc signing is not accepted')
|
||||
return value
|
||||
}
|
||||
|
||||
function explicitSigningKeychain(options: SignedChainOptions): string | undefined {
|
||||
if (options.signingKeychain === undefined) return undefined
|
||||
if (options.signingIdentity === undefined || !path.isAbsolute(options.signingKeychain)) {
|
||||
throw new Error('--signing-keychain requires an absolute path and an explicit --signing-identity')
|
||||
}
|
||||
return options.signingKeychain
|
||||
}
|
||||
|
||||
export function parseSignedChainArgs(args: string[]): SignedChainOptions & { output?: string } {
|
||||
const result: SignedChainOptions & { output?: string } = {}
|
||||
for (let index = 0; index < args.length; index++) {
|
||||
const arg = args[index]!
|
||||
if (arg === '--signing-identity') {
|
||||
if (result.signingIdentity !== undefined) throw new Error('--signing-identity may only be supplied once')
|
||||
result.signingIdentity = explicitSigningIdentity(args[++index] ?? '')
|
||||
} else if (arg === '--signing-keychain') {
|
||||
if (result.signingKeychain !== undefined) throw new Error('--signing-keychain may only be supplied once')
|
||||
result.signingKeychain = args[++index] ?? ''
|
||||
} else if (arg.startsWith('-')) {
|
||||
throw new Error(`Unknown fixture option: ${arg}`)
|
||||
} else {
|
||||
if (result.output !== undefined) throw new Error('Only one output report path may be supplied')
|
||||
result.output = arg
|
||||
}
|
||||
}
|
||||
explicitSigningKeychain(result)
|
||||
return result
|
||||
}
|
||||
|
||||
/** Pure command planning. Explicit identity mode is opt-in and never discovers
|
||||
* or imports identities. Running that mode requires separate user authorization
|
||||
* to use the named signing key; the default only uses our private keychain. */
|
||||
export function planFixtureSigning(options: SignedChainOptions & { directory: string }) {
|
||||
if (!path.isAbsolute(options.directory)) throw new Error('The signing fixture directory must be absolute')
|
||||
const identity = options.signingIdentity === undefined ? undefined : explicitSigningIdentity(options.signingIdentity)
|
||||
const signingKeychain = explicitSigningKeychain(options)
|
||||
const keychain = path.join(options.directory, 'fixture.keychain-db')
|
||||
const privateSigner = path.join(options.directory, 'private-signer')
|
||||
return {
|
||||
mode: identity === undefined ? 'temporary' as const : 'explicit' as const,
|
||||
requiresPackagedInstall: identity !== undefined,
|
||||
setupCommands(password: string): PlannedCommand[] {
|
||||
return identity === undefined
|
||||
? privateKeychainCommands(keychain, path.join(options.directory, 'identity.p12'), password)
|
||||
.map(args => ({ command: '/usr/bin/security', args }))
|
||||
: []
|
||||
},
|
||||
signCommand(file: string, identifier: string, entitlements?: string): PlannedCommand {
|
||||
return identity === undefined
|
||||
? { command: privateSigner, args: [keychain, file, identifier, ...(entitlements ? [entitlements] : [])] }
|
||||
: {
|
||||
command: '/usr/bin/codesign',
|
||||
args: ['--force', '--sign', identity, '--identifier', identifier, '--options', 'runtime', '--timestamp=none',
|
||||
...(signingKeychain ? ['--keychain', signingKeychain] : []),
|
||||
...(entitlements ? ['--entitlements', entitlements] : []), file],
|
||||
}
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
/** Security's identity lookup still requires the keychain owner's HOME even
|
||||
* with --keychain. Only the explicitly authorized signing process receives it;
|
||||
* the helper, sidecar, compiler, CF preferences and config remain isolated. */
|
||||
export function fixtureSigningEnvironment(
|
||||
mode: 'temporary' | 'explicit',
|
||||
isolated: Record<string, string>,
|
||||
userHome: string | undefined,
|
||||
): Record<string, string> {
|
||||
if (mode === 'temporary') return isolated
|
||||
if (!userHome || !path.isAbsolute(userHome)) throw new Error('Explicit signing requires the authorized keychain owner HOME')
|
||||
return { ...isolated, HOME: userHome }
|
||||
}
|
||||
|
||||
/** Every keychain operation must name our private file. Never enumerate or
|
||||
* update the user's default keychains/search list/trust settings. Apple’s
|
||||
* private-keychain creation policy leaves that search list unchanged. */
|
||||
export function privateKeychainCommands(keychain: string, archive: string, password: string) {
|
||||
if (!path.isAbsolute(keychain) || path.basename(keychain).includes('login.keychain')) {
|
||||
throw new Error('A private absolute fixture keychain path is required')
|
||||
}
|
||||
return [
|
||||
['create-keychain', '-p', password, keychain],
|
||||
['unlock-keychain', '-p', password, keychain],
|
||||
['import', archive, '-k', keychain, '-P', password, '-A'],
|
||||
['set-key-partition-list', '-S', 'apple-tool:,apple:,codesign:', '-s', '-k', password, keychain],
|
||||
]
|
||||
}
|
||||
|
||||
export function verifySignedChainReport(report: {
|
||||
signing?: { mode: 'temporary' | 'explicit', teamIdentifier?: string }
|
||||
result: {
|
||||
error?: string
|
||||
packagedInstall?: boolean
|
||||
ping?: { protocolVersion?: string }
|
||||
permissions?: { accessibility?: boolean, screenRecording?: boolean }
|
||||
first?: { isError?: boolean, content?: Array<{ text?: string }> }
|
||||
second?: { isError?: boolean, content?: Array<{ text?: string }> }
|
||||
nativeObservation?: { isError?: boolean }
|
||||
replay?: { isError?: boolean }
|
||||
}
|
||||
receiver: { completed?: number, held?: boolean, unpaired?: number }
|
||||
unauthorizedDirect: { ok?: boolean, error?: { code?: string } }
|
||||
}) {
|
||||
const { result, receiver } = report
|
||||
if (result.error) throw new Error(result.error)
|
||||
if (report.signing?.mode === 'explicit'
|
||||
&& (!report.signing.teamIdentifier?.trim() || report.signing.teamIdentifier === 'not set' || result.packagedInstall !== true)) {
|
||||
throw new Error('Explicit identity mode requires a Team ID and successful production packaged-helper installation')
|
||||
}
|
||||
if (result.ping?.protocolVersion !== 'CCHahaComputerUseIPC-2') throw new Error('Production daemon handshake was not proven')
|
||||
if (result.first?.isError || result.first?.content?.[0]?.text !== '1'
|
||||
|| result.second?.isError || result.second?.content?.[0]?.text !== '2') {
|
||||
throw new Error('Compiled desktop worker did not preserve bindings across cells')
|
||||
}
|
||||
if (report.unauthorizedDirect.ok !== false || report.unauthorizedDirect.error?.code !== 'unauthorized_client') {
|
||||
throw new Error('The direct unsigned caller was not rejected')
|
||||
}
|
||||
const granted = result.permissions?.accessibility === true && result.permissions?.screenRecording === true
|
||||
if (granted) {
|
||||
if (!result.nativeObservation || result.nativeObservation.isError) throw new Error('Native app observation failed or did not run')
|
||||
if (!result.replay || result.replay.isError) throw new Error('Native gesture replay failed or did not run')
|
||||
if (receiver.completed !== 12 || receiver.held !== false || receiver.unpaired !== 0) {
|
||||
throw new Error('The receiving app did not consume exactly twelve complete gestures')
|
||||
}
|
||||
return 'passed_native_gui' as const
|
||||
}
|
||||
if (result.nativeObservation?.isError !== true || receiver.completed !== 0 || receiver.held !== false || receiver.unpaired !== 0) {
|
||||
throw new Error('The ungranted helper did not fail before injecting input')
|
||||
}
|
||||
return 'blocked_os_permissions' as const
|
||||
}
|
||||
|
||||
/** Keep receiver/native failure evidence even when acceptance fails. */
|
||||
export function evaluateSignedChainReport<T extends Parameters<typeof verifySignedChainReport>[0]>(report: T) {
|
||||
try {
|
||||
return { ...report, outcome: verifySignedChainReport(report), validationError: undefined }
|
||||
} catch (error) {
|
||||
return { ...report, outcome: 'failed_validation' as const, validationError: error instanceof Error ? error.message : String(error) }
|
||||
}
|
||||
}
|
||||
|
||||
export async function runSignedComputerUseChain(options: SignedChainOptions = {}) {
|
||||
// Reject missing/empty explicit identities before any filesystem or process
|
||||
// side effects. No environment variable supplies a signing identity.
|
||||
if (options.signingIdentity !== undefined) explicitSigningIdentity(options.signingIdentity)
|
||||
explicitSigningKeychain(options)
|
||||
if (process.platform !== 'darwin') throw new Error('This integration fixture requires macOS')
|
||||
// Darwin sockaddr_un has a short fixed path limit. A normal per-user TMPDIR
|
||||
// plus the production .runtime socket suffix can exceed it before bind().
|
||||
const directory = await realpath(await mkdtemp('/tmp/cu-chain-'))
|
||||
const env = createSandboxedTestEnvironment(path.join(directory, 'home'))
|
||||
env.CFFIXED_USER_HOME = env.HOME!
|
||||
const commandResults: Array<{ command: string, code: number | null, stdout: string, stderr: string }> = []
|
||||
let keychainCreated = false
|
||||
let target: ReturnType<typeof spawn> | undefined
|
||||
let targetClosed: Promise<unknown> | undefined
|
||||
let targetBinary: string | undefined
|
||||
let targetPID: number | undefined
|
||||
const targetLifecycle: { pid?: number, launcherExited?: boolean, forcedTermination?: boolean } = {}
|
||||
const stopPath = path.join(directory, 'receiver-stop')
|
||||
const keychain = path.join(directory, 'fixture.keychain-db')
|
||||
const signingPlan = planFixtureSigning({ directory, ...options })
|
||||
let signingTeam: string | undefined
|
||||
async function run(command: string, args: string[], timeout = 120_000, allowFailure = false, commandEnv = env) {
|
||||
const child = spawn(command, args, { cwd: directory, env: commandEnv, stdio: ['ignore', 'pipe', 'pipe'], timeout })
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
let signal: NodeJS.Signals | null = null
|
||||
child.stdout.on('data', chunk => { stdout += String(chunk) })
|
||||
child.stderr.on('data', chunk => { stderr += String(chunk) })
|
||||
const code = await new Promise<number | null>((resolve, reject) => {
|
||||
child.once('error', reject)
|
||||
child.once('close', (code, closedSignal) => { signal = closedSignal; resolve(code) })
|
||||
})
|
||||
// Command arguments include disposable private-key passwords. They are
|
||||
// deliberately never retained in artifacts or printed on failure.
|
||||
commandResults.push({ command: path.basename(command), code, stdout, stderr })
|
||||
if (code !== 0 && !allowFailure) throw new Error(`${path.basename(command)} failed (${code}, ${signal}): ${stderr}`)
|
||||
return { code, stdout, stderr }
|
||||
}
|
||||
try {
|
||||
const signingEnv = fixtureSigningEnvironment(signingPlan.mode, env, process.env.HOME)
|
||||
if (signingPlan.mode === 'temporary') {
|
||||
const certificateName = `CC Haha disposable CU ${randomUUID()}`
|
||||
const password = randomUUID()
|
||||
const config = path.join(directory, 'certificate.cnf')
|
||||
await writeFile(config, `[req]\nprompt=no\ndistinguished_name=dn\nx509_extensions=ext\n[dn]\nCN=${certificateName}\n[ext]\nbasicConstraints=critical,CA:false\nkeyUsage=critical,digitalSignature\nextendedKeyUsage=critical,codeSigning\n`)
|
||||
await run('/usr/bin/openssl', ['req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', '-config', config, '-keyout', 'key.pem', '-out', 'cert.pem'])
|
||||
await run('/usr/bin/openssl', ['pkcs12', '-export', '-inkey', 'key.pem', '-in', 'cert.pem', '-out', 'identity.p12', '-passout', `pass:${password}`])
|
||||
for (const { command, args } of signingPlan.setupCommands(password)) {
|
||||
await run(command, args)
|
||||
if (args[0] === 'create-keychain') keychainCreated = true
|
||||
}
|
||||
await run('/usr/bin/security', ['find-identity', '-p', 'codesigning', keychain])
|
||||
await run('/usr/bin/clang', ['-fobjc-arc', '-framework', 'Foundation', '-framework', 'Security',
|
||||
path.join(repoRoot, 'scripts/quality-gate/fixtures/computer-use-private-signer.m'), '-o', path.join(directory, 'private-signer')])
|
||||
}
|
||||
const sign = async (file: string, identifier: string, entitlements?: string) => {
|
||||
await run('/usr/bin/codesign', ['--remove-signature', file], 30_000, true)
|
||||
const command = signingPlan.signCommand(file, identifier, entitlements)
|
||||
await run(command.command, command.args, 120_000, false, signingEnv)
|
||||
await run('/usr/bin/codesign', ['--verify', '--strict', '--verbose=2', file])
|
||||
if (signingPlan.requiresPackagedInstall) {
|
||||
const details = await run('/usr/bin/codesign', ['-dv', '--verbose=4', file])
|
||||
const team = `${details.stdout}\n${details.stderr}`.match(/^TeamIdentifier=(.+)$/m)?.[1]?.trim()
|
||||
if (!team || team === 'not set' || (signingTeam !== undefined && signingTeam !== team)) {
|
||||
throw new Error('Explicit identity signatures must have the same nonempty Team ID')
|
||||
}
|
||||
signingTeam = team
|
||||
}
|
||||
}
|
||||
const hostApp = path.join(directory, 'Fixture Host.app')
|
||||
const macos = path.join(hostApp, 'Contents/MacOS')
|
||||
const binaries = path.join(hostApp, 'Contents/Resources/app.asar.unpacked/src-tauri/binaries')
|
||||
await mkdir(macos, { recursive: true })
|
||||
await mkdir(binaries, { recursive: true })
|
||||
const host = path.join(macos, 'FixtureHost')
|
||||
await writeFile(path.join(hostApp, 'Contents/Info.plist'), `<?xml version="1.0"?><plist version="1.0"><dict><key>CFBundleExecutable</key><string>FixtureHost</string><key>CFBundleIdentifier</key><string>com.claude-code-haha.desktop</string><key>CFBundlePackageType</key><string>APPL</string></dict></plist>`)
|
||||
await run('/usr/bin/clang', [path.join(repoRoot, 'scripts/quality-gate/fixtures/computer-use-signed-chain-host.c'), '-o', host])
|
||||
const executable = path.join(binaries, `claude-sidecar-${process.arch === 'arm64' ? 'aarch64' : 'x86_64'}-apple-darwin`)
|
||||
const build = await Bun.build({
|
||||
entrypoints: [path.join(repoRoot, 'scripts/quality-gate/fixtures/computer-use-signed-chain-broker.ts')],
|
||||
features: ['TRANSCRIPT_CLASSIFIER'], target: 'bun',
|
||||
minify: { whitespace: true, identifiers: true, syntax: true },
|
||||
external: [
|
||||
'@opentelemetry/exporter-trace-otlp-grpc', '@opentelemetry/exporter-trace-otlp-http', '@opentelemetry/exporter-trace-otlp-proto',
|
||||
'@opentelemetry/exporter-logs-otlp-grpc', '@opentelemetry/exporter-logs-otlp-http', '@opentelemetry/exporter-logs-otlp-proto',
|
||||
'@opentelemetry/exporter-metrics-otlp-grpc', '@opentelemetry/exporter-metrics-otlp-http', '@opentelemetry/exporter-metrics-otlp-proto',
|
||||
'@opentelemetry/exporter-prometheus', '@aws-sdk/client-bedrock', '@aws-sdk/client-sts', '@anthropic-ai/bedrock-sdk',
|
||||
'@anthropic-ai/foundry-sdk', '@anthropic-ai/vertex-sdk', '@azure/identity', '@anthropic-ai/mcpb', 'fflate', 'sharp', 'react-devtools-core',
|
||||
],
|
||||
compile: { outfile: executable, autoloadTsconfig: true, autoloadPackageJson: true },
|
||||
})
|
||||
if (!build.success) throw new Error(build.logs.join('\n'))
|
||||
const entitlements = path.join(directory, 'sidecar-entitlements.plist')
|
||||
await writeFile(entitlements, '<?xml version="1.0"?><plist version="1.0"><dict><key>com.apple.security.cs.allow-jit</key><true/><key>com.apple.security.cs.allow-unsigned-executable-memory</key><true/></dict></plist>')
|
||||
await sign(executable, 'com.claude-code-haha.desktop.sidecar', entitlements)
|
||||
|
||||
// Build current production Swift sources, with the production embedded
|
||||
// Info.plist. Avoid build.sh's automatic user-keychain identity discovery.
|
||||
const swiftBuild = path.join(directory, 'swift-build')
|
||||
const packagePath = path.join(repoRoot, 'native/cu-helper')
|
||||
const buildArgs = ['build', '-c', 'release', '--package-path', packagePath, '--scratch-path', swiftBuild]
|
||||
await run('/usr/bin/swift', [...buildArgs, '-Xlinker', '-sectcreate', '-Xlinker', '__TEXT', '-Xlinker', '__info_plist', '-Xlinker', path.join(packagePath, 'Info.plist')], 300_000)
|
||||
const binDir = (await run('/usr/bin/swift', [...buildArgs, '--show-bin-path'])).stdout.trim()
|
||||
const helperApp = path.join(directory, 'cc-haha-computer-use.app')
|
||||
await mkdir(path.join(helperApp, 'Contents/MacOS'), { recursive: true })
|
||||
await mkdir(path.join(helperApp, 'Contents/Resources'), { recursive: true })
|
||||
await copyFile(path.join(packagePath, 'Info.plist'), path.join(helperApp, 'Contents/Info.plist'))
|
||||
// LaunchServices does not inherit the caller's HOME. Keep AppKit's own
|
||||
// potential preference/cache writes inside the disposable home as well.
|
||||
await run('/usr/bin/plutil', ['-insert', 'LSEnvironment', '-json', JSON.stringify({
|
||||
HOME: env.HOME, CFFIXED_USER_HOME: env.HOME, TMPDIR: env.TMPDIR, TMP: env.TMP, TEMP: env.TEMP,
|
||||
CLAUDE_CONFIG_DIR: env.CLAUDE_CONFIG_DIR,
|
||||
}), path.join(helperApp, 'Contents/Info.plist')])
|
||||
await copyFile(path.join(binDir, 'cc-haha-computer-use'), path.join(helperApp, 'Contents/MacOS/cc-haha-computer-use'))
|
||||
await cp(path.join(binDir, 'cu-helper_cc-haha-computer-use.bundle'), path.join(helperApp, 'Contents/Resources/cu-helper_cc-haha-computer-use.bundle'), { recursive: true })
|
||||
await sign(helperApp, 'dev.cchaha.cu-helper')
|
||||
const nested = path.join(binaries, 'cc-haha-computer-use.app')
|
||||
await cp(helperApp, nested, { recursive: true })
|
||||
await sign(hostApp, 'com.claude-code-haha.desktop')
|
||||
const helperBinary = path.join(helperApp, 'Contents/MacOS/cc-haha-computer-use')
|
||||
if (signingPlan.requiresPackagedInstall) {
|
||||
env.CLAUDE_APP_ROOT = path.join(hostApp, 'Contents/Resources/app.asar')
|
||||
env.CU_FIXTURE_REQUIRE_PACKAGED_INSTALL = '1'
|
||||
} else {
|
||||
env.CC_HAHA_CU_HELPER_PATH = helperBinary
|
||||
}
|
||||
env.CU_FIXTURE_NESTED_HELPER = nested
|
||||
const targetApp = path.join(directory, 'Drag Receiver.app')
|
||||
await mkdir(path.join(targetApp, 'Contents/MacOS'), { recursive: true })
|
||||
targetBinary = path.join(targetApp, 'Contents/MacOS/DragReceiver')
|
||||
await writeFile(path.join(targetApp, 'Contents/Info.plist'), `<?xml version="1.0"?><plist version="1.0"><dict><key>CFBundleExecutable</key><string>DragReceiver</string><key>CFBundleIdentifier</key><string>dev.cchaha.fixture.${randomUUID()}</string><key>CFBundlePackageType</key><string>APPL</string><key>LSUIElement</key><true/></dict></plist>`)
|
||||
await run('/usr/bin/plutil', ['-insert', 'LSEnvironment', '-json', JSON.stringify({
|
||||
HOME: env.HOME, CFFIXED_USER_HOME: env.HOME, TMPDIR: env.TMPDIR, TMP: env.TMP, TEMP: env.TEMP,
|
||||
CLAUDE_CONFIG_DIR: env.CLAUDE_CONFIG_DIR,
|
||||
}), path.join(targetApp, 'Contents/Info.plist')])
|
||||
await run('/usr/bin/swiftc', ['-framework', 'AppKit', path.join(repoRoot, 'scripts/quality-gate/fixtures/computer-use-signed-chain-target.swift'), '-o', targetBinary])
|
||||
const receipt = path.join(directory, 'receiver.json')
|
||||
const helperPIDFile = path.join(directory, 'helper-pid')
|
||||
env.CU_FIXTURE_HELPER_PID_FILE = helperPIDFile
|
||||
// A direct spawn has no NSRunningApplication.launchDate, so production
|
||||
// process-lifetime validation correctly refuses it. Launch as a real App.
|
||||
target = spawn('/usr/bin/open', ['-g', '-n', '-W', targetApp, '--args', receipt, helperPIDFile, stopPath], { env, stdio: 'ignore' })
|
||||
targetClosed = new Promise(resolve => target!.once('close', resolve))
|
||||
target.on('error', () => {})
|
||||
const deadline = Date.now() + 10_000
|
||||
while (true) {
|
||||
try {
|
||||
const ready = JSON.parse(await readFile(receipt, 'utf8'))
|
||||
if (ready.ready === true && Number.isInteger(ready.pid) && ready.pid > 0) {
|
||||
targetPID = ready.pid
|
||||
targetLifecycle.pid = ready.pid
|
||||
break
|
||||
}
|
||||
} catch {}
|
||||
if (target.exitCode !== null || target.signalCode !== null || Date.now() > deadline) throw new Error('Temporary receiving app did not start')
|
||||
await new Promise(resolve => setTimeout(resolve, 20))
|
||||
}
|
||||
env.CU_FIXTURE_TARGET_APP = targetApp
|
||||
// The test broker runs as the exact signed sidecar child; runtime relaunch
|
||||
// uses its real compiled desktop worker branch inside the OS sandbox.
|
||||
const execution = await run(host, [executable], 60_000)
|
||||
const result = JSON.parse(execution.stdout.trim().split('\n').at(-1)!)
|
||||
const direct = await run(helperBinary, ['check_permissions', '--payload', '{}'], 10_000, true)
|
||||
let receiver = JSON.parse(await readFile(receipt, 'utf8'))
|
||||
// A successful input call is not a receiving-app acknowledgement. Wait for
|
||||
// actual consumption; never repost gestures to make the count pass.
|
||||
if (result.replay && !result.replay.isError) {
|
||||
const receiveDeadline = Date.now() + 5_000
|
||||
while (receiver.completed !== 12 || receiver.held !== false || receiver.unpaired !== 0) {
|
||||
if (Date.now() >= receiveDeadline || target.exitCode !== null || target.signalCode !== null
|
||||
|| receiver.completed > 12 || receiver.unpaired > 0) break
|
||||
await new Promise(resolve => setTimeout(resolve, 20))
|
||||
receiver = JSON.parse(await readFile(receipt, 'utf8'))
|
||||
}
|
||||
}
|
||||
return { signing: { mode: signingPlan.mode, teamIdentifier: signingTeam }, result, receiver, targetLifecycle, unauthorizedDirect: JSON.parse(direct.stdout.trim()), commands: commandResults }
|
||||
} finally {
|
||||
if (target && targetClosed) {
|
||||
await writeFile(stopPath, '')
|
||||
let stopTimer: ReturnType<typeof setTimeout> | undefined
|
||||
const exited = await Promise.race([
|
||||
targetClosed.then(() => true),
|
||||
new Promise<false>(resolve => { stopTimer = setTimeout(() => resolve(false), 5_000) }),
|
||||
])
|
||||
clearTimeout(stopTimer)
|
||||
targetLifecycle.forcedTermination = !exited
|
||||
if (!exited) {
|
||||
// open -W's PID is not the App PID. Only terminate a receiver still
|
||||
// running our exact disposable binary; never signal a recycled PID.
|
||||
if (targetPID && targetBinary) {
|
||||
const current = await run('/bin/ps', ['-p', String(targetPID), '-o', 'comm='], 5_000, true)
|
||||
if (current.code === 0 && current.stdout.trim() === targetBinary) {
|
||||
try { process.kill(targetPID, 'SIGTERM') } catch {}
|
||||
}
|
||||
}
|
||||
target.kill('SIGTERM')
|
||||
await targetClosed
|
||||
}
|
||||
targetLifecycle.launcherExited = true
|
||||
}
|
||||
// Lock only our private keychain, then remove its disposable directory.
|
||||
// security delete-keychain may touch search-list preferences; avoid it.
|
||||
if (keychainCreated) await run('/usr/bin/security', ['lock-keychain', keychain], 10_000, true)
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
const { output, ...options } = parseSignedChainArgs(process.argv.slice(2))
|
||||
const report = await runSignedComputerUseChain(options)
|
||||
const completed = evaluateSignedChainReport(report)
|
||||
if (output) await writeFile(output, `${JSON.stringify(completed, null, 2)}\n`)
|
||||
console.log(JSON.stringify({ ...completed, commands: undefined }, null, 2))
|
||||
if (completed.outcome === 'failed_validation') process.exitCode = 1
|
||||
}
|
||||
@@ -0,0 +1,62 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
#import <Security/Security.h>
|
||||
#include <arpa/inet.h>
|
||||
|
||||
// Apple's codesign command filters identity discovery through system trust.
|
||||
// This fixture passes its single private-keychain identity directly to the
|
||||
// same signing API. It never changes trust settings. Resulting signatures must
|
||||
// still pass the unchanged production strict checks and peer attestation.
|
||||
typedef struct __SecCodeSigner *SecCodeSignerRef;
|
||||
extern const CFStringRef kSecCodeSignerIdentity;
|
||||
extern const CFStringRef kSecCodeSignerIdentifier;
|
||||
extern const CFStringRef kSecCodeSignerFlags;
|
||||
extern const CFStringRef kSecCodeSignerRequireTimestamp;
|
||||
extern const CFStringRef kSecCodeSignerEntitlements;
|
||||
extern OSStatus SecCodeSignerCreate(CFDictionaryRef, SecCSFlags, SecCodeSignerRef *);
|
||||
extern OSStatus SecCodeSignerAddSignature(SecCodeSignerRef, SecStaticCodeRef, SecCSFlags);
|
||||
|
||||
int main(int argc, char **argv) {
|
||||
@autoreleasepool {
|
||||
if (argc < 4) return 2;
|
||||
SecKeychainSetUserInteractionAllowed(false);
|
||||
SecKeychainRef keychain = NULL;
|
||||
OSStatus status = SecKeychainOpen(argv[1], &keychain);
|
||||
if (status) { fprintf(stderr, "private keychain open: %d\n", (int)status); return 1; }
|
||||
NSDictionary *query = @{
|
||||
(__bridge id)kSecClass: (__bridge id)kSecClassIdentity,
|
||||
(__bridge id)kSecMatchSearchList: @[(__bridge id)keychain],
|
||||
(__bridge id)kSecReturnRef: @YES,
|
||||
(__bridge id)kSecMatchLimit: (__bridge id)kSecMatchLimitOne,
|
||||
};
|
||||
CFTypeRef identity = NULL;
|
||||
status = SecItemCopyMatching((__bridge CFDictionaryRef)query, &identity);
|
||||
if (status) { fprintf(stderr, "private identity lookup: %d\n", (int)status); return 1; }
|
||||
NSMutableDictionary *parameters = [@{
|
||||
(__bridge id)kSecCodeSignerIdentity: (__bridge id)identity,
|
||||
(__bridge id)kSecCodeSignerIdentifier: @(argv[3]),
|
||||
(__bridge id)kSecCodeSignerFlags: @(kSecCodeSignatureRuntime),
|
||||
(__bridge id)kSecCodeSignerRequireTimestamp: @NO,
|
||||
} mutableCopy];
|
||||
if (argc > 4) {
|
||||
NSData *plist = [NSData dataWithContentsOfFile:@(argv[4])];
|
||||
// SecCodeSigner consumes the code-signing entitlement blob, not the
|
||||
// bare XML accepted by the codesign command-line frontend.
|
||||
uint32_t header[] = { htonl(0xfade7171), htonl((uint32_t)plist.length + 8) };
|
||||
NSMutableData *blob = [NSMutableData dataWithBytes:header length:8];
|
||||
[blob appendData:plist];
|
||||
parameters[(__bridge id)kSecCodeSignerEntitlements] = blob;
|
||||
}
|
||||
SecCodeSignerRef signer = NULL;
|
||||
status = SecCodeSignerCreate((__bridge CFDictionaryRef)parameters, 0, &signer);
|
||||
if (status) { fprintf(stderr, "signer create: %d\n", (int)status); return 1; }
|
||||
SecStaticCodeRef code = NULL;
|
||||
status = SecStaticCodeCreateWithPath((__bridge CFURLRef)[NSURL fileURLWithPath:@(argv[2])], 0, &code);
|
||||
if (!status) status = SecCodeSignerAddSignature(signer, code, 0);
|
||||
if (status) fprintf(stderr, "private signing: %d\n", (int)status);
|
||||
if (code) CFRelease(code);
|
||||
CFRelease(signer);
|
||||
CFRelease(identity);
|
||||
CFRelease(keychain);
|
||||
return status == 0 ? 0 : 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
// Test-only launch shell. The worker branch enters the real merged desktop
|
||||
// sidecar; native calls use the production binder, executor and daemon client.
|
||||
if (process.argv[2] === '--computer-use-repl-worker') {
|
||||
await import('../../../desktop/sidecars/claude-sidecar.ts')
|
||||
} else {
|
||||
const { bindSessionContext } = await import('../../../src/vendor/computer-use-mcp/mcpServer.ts')
|
||||
const { createCliExecutor } = await import('../../../src/utils/computerUse/executor.ts')
|
||||
const { ComputerUseRepl } = await import('../../../src/utils/computerUse/replRuntime.ts')
|
||||
const { callDaemon, shutdownDaemon } = await import('../../../src/utils/computerUse/cuHelperDaemon.ts')
|
||||
const { ensureInstalledHelper } = await import('../../../src/utils/computerUse/cuHelperInstall.ts')
|
||||
const logger = { silly() {}, debug() {}, info() {}, warn() {}, error() {} }
|
||||
let dispatch: ReturnType<typeof bindSessionContext> | undefined
|
||||
const report: Record<string, unknown> = { stage: 'start' }
|
||||
try {
|
||||
report.packagedInstall = ensureInstalledHelper({ sourceApp: process.env.CU_FIXTURE_NESTED_HELPER }) !== null
|
||||
if (process.env.CU_FIXTURE_REQUIRE_PACKAGED_INSTALL === '1' && !report.packagedInstall) {
|
||||
throw new Error('The production packaged-helper installation rejected the explicitly signed fixture')
|
||||
}
|
||||
report.stage = 'daemon-attestation'
|
||||
report.ping = await callDaemon('ping')
|
||||
const { readdir, readFile, writeFile } = await import('node:fs/promises')
|
||||
const { join } = await import('node:path')
|
||||
const { getRuntimePaths } = await import('../../../src/utils/computerUse/pythonBridge.ts')
|
||||
const stateRoot = getRuntimePaths().runtimeStateRoot
|
||||
const pidfile = (await readdir(stateRoot)).find(file => file.startsWith(`cu-helper.daemon.${process.pid}.`) && file.endsWith('.sock.pid'))
|
||||
if (!pidfile) throw new Error('Authenticated helper PID file was not found')
|
||||
const helperPID = (await readFile(join(stateRoot, pidfile), 'utf8')).trim()
|
||||
await writeFile(process.env.CU_FIXTURE_HELPER_PID_FILE!, helperPID)
|
||||
report.helperPID = Number(helperPID)
|
||||
const permissions = await callDaemon<{ accessibility: boolean, screenRecording: boolean }>('check_permissions')
|
||||
report.permissions = permissions
|
||||
report.stage = 'javascript'
|
||||
const executor = createCliExecutor({ getMouseAnimationEnabled: () => false, getHideBeforeActionEnabled: () => false })
|
||||
let screenshot: { width: number, height: number } | undefined
|
||||
if (executor.engine) {
|
||||
const getAppState = executor.engine.getAppState.bind(executor.engine)
|
||||
executor.engine.getAppState = async (...args) => {
|
||||
const state = await getAppState(...args)
|
||||
screenshot = state.screenshot
|
||||
return state
|
||||
}
|
||||
}
|
||||
dispatch = bindSessionContext({
|
||||
serverName: 'native-chain-fixture', logger,
|
||||
executor,
|
||||
// No permission card and no prompting in this deterministic fixture.
|
||||
// The authoritative probe above comes from the actual signed helper.
|
||||
ensureOsPermissions: async () => permissions.accessibility && permissions.screenRecording
|
||||
? { granted: true } : { granted: false, ...permissions },
|
||||
isDisabled: () => false,
|
||||
getAutoUnhideEnabled: () => true,
|
||||
getSubGates: () => ({ pixelValidation: false, clipboardPasteMultiline: false, mouseAnimation: false, hideBeforeAction: false, autoTargetDisplay: false, clipboardGuard: false }),
|
||||
cropRawPatch: () => null,
|
||||
createReplRuntime: () => new ComputerUseRepl(),
|
||||
}, 'pixels', {
|
||||
getAllowedApps: () => [],
|
||||
getGrantFlags: () => ({ clipboardRead: false, clipboardWrite: false, systemKeyCombos: false }),
|
||||
getUserDeniedBundleIds: () => [],
|
||||
getSelectedDisplayId: () => undefined,
|
||||
})
|
||||
report.first = await dispatch('js', { code: 'let count = 1; nodeRepl.write(count)' })
|
||||
report.second = await dispatch('js', { code: 'nodeRepl.write(++count)' })
|
||||
const observation = await dispatch('js', {
|
||||
code: `let app = await cua.getApp(${JSON.stringify(process.env.CU_FIXTURE_TARGET_APP)})`,
|
||||
})
|
||||
report.nativeObservation = observation
|
||||
if (!observation.isError && screenshot) {
|
||||
const x = Math.floor(screenshot.width / 2)
|
||||
const y = Math.floor(screenshot.height / 2)
|
||||
const replay = await dispatch('js', {
|
||||
code: `for (let i = 0; i < 12; i++) await app.drag([${x},${y}], [${x} + i % 2,${y}]); await app.getAXStateAndScreenshot()`,
|
||||
})
|
||||
report.replay = {
|
||||
...replay,
|
||||
content: replay.content.map(content => content.type === 'image'
|
||||
? { type: 'image', mimeType: content.mimeType, base64Length: content.data.length } : content),
|
||||
}
|
||||
}
|
||||
report.stage = 'complete'
|
||||
} catch (error) {
|
||||
report.error = error instanceof Error ? error.message : String(error)
|
||||
} finally {
|
||||
await dispatch?.('js_reset', {})
|
||||
await shutdownDaemon()
|
||||
console.log(JSON.stringify(report))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
#include <spawn.h>
|
||||
#include <sys/wait.h>
|
||||
#include <unistd.h>
|
||||
|
||||
extern char **environ;
|
||||
|
||||
// This tiny signed parent supplies the required host ancestry without starting
|
||||
// Electron, loading user configuration, or invoking any model.
|
||||
int main(int argc, char **argv) {
|
||||
if (argc < 2) return 2;
|
||||
pid_t child;
|
||||
int status = posix_spawn(&child, argv[1], NULL, NULL, &argv[1], environ);
|
||||
if (status != 0) return status;
|
||||
if (waitpid(child, &status, 0) < 0) return 3;
|
||||
return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status);
|
||||
}
|
||||
@@ -0,0 +1,146 @@
|
||||
import AppKit
|
||||
|
||||
final class ReceiptView: NSView {
|
||||
var completed = 0
|
||||
var held = false
|
||||
var unpaired = 0
|
||||
var otherEvents = 0
|
||||
var sources: [Int64] = []
|
||||
private var events: [[String: Any]] = []
|
||||
private var eventsTruncated = 0
|
||||
private var launchBoundaryPassed = false
|
||||
private var lastSaved: Data?
|
||||
private let maximumRecordedEvents = 256
|
||||
let receipt: URL
|
||||
let helperPIDFile: URL
|
||||
|
||||
init(receipt: URL, helperPIDFile: URL) {
|
||||
self.receipt = receipt
|
||||
self.helperPIDFile = helperPIDFile
|
||||
super.init(frame: NSRect(x: 0, y: 0, width: 360, height: 280))
|
||||
setAccessibilityElement(true)
|
||||
setAccessibilityRole(.group)
|
||||
setAccessibilityLabel("Temporary drag canvas")
|
||||
}
|
||||
|
||||
required init?(coder: NSCoder) { fatalError("not used") }
|
||||
override func acceptsFirstMouse(for event: NSEvent?) -> Bool { true }
|
||||
override func mouseDown(with event: NSEvent) {
|
||||
guard belongsToHelper(event) else { return }
|
||||
if held { unpaired += 1 }
|
||||
held = true
|
||||
save()
|
||||
}
|
||||
override func mouseDragged(with event: NSEvent) {
|
||||
guard belongsToHelper(event) else { return }
|
||||
if !held { unpaired += 1 }
|
||||
save()
|
||||
}
|
||||
override func mouseUp(with event: NSEvent) {
|
||||
guard belongsToHelper(event) else { return }
|
||||
if !held { unpaired += 1 }
|
||||
held = false
|
||||
completed += 1
|
||||
save()
|
||||
}
|
||||
override func draw(_ dirtyRect: NSRect) {
|
||||
NSColor(calibratedRed: 0.15, green: 0.22, blue: 0.30, alpha: 1).setFill()
|
||||
bounds.fill()
|
||||
("Temporary Computer Use fixture" as NSString).draw(at: NSPoint(x: 35, y: 130), withAttributes: [.foregroundColor: NSColor.white])
|
||||
}
|
||||
func publishReady() {
|
||||
launchBoundaryPassed = true
|
||||
save()
|
||||
}
|
||||
func refresh() { save() }
|
||||
private func save() {
|
||||
guard launchBoundaryPassed else { return }
|
||||
let running = NSRunningApplication.current
|
||||
let frame = window?.frame ?? .zero
|
||||
let launchTime = running.launchDate?.timeIntervalSinceReferenceDate
|
||||
let ready = running.processIdentifier > 0
|
||||
&& running.bundleIdentifier.map { !$0.isEmpty } == true
|
||||
&& running.executableURL.map { !$0.path.isEmpty } == true
|
||||
&& launchTime.map { $0.isFinite && $0 > 0 } == true
|
||||
&& (window?.windowNumber ?? 0) > 0
|
||||
&& frame.width > 0 && frame.height > 0
|
||||
&& window?.isVisible == true
|
||||
let value: [String: Any] = [
|
||||
"ready": ready,
|
||||
"pid": Int(running.processIdentifier),
|
||||
"bundleID": running.bundleIdentifier as Any? ?? NSNull(),
|
||||
"executable": running.executableURL?.path as Any? ?? NSNull(),
|
||||
"launchTime": launchTime as Any? ?? NSNull(),
|
||||
"windowID": window?.windowNumber ?? 0,
|
||||
"windowFrame": ["x": frame.origin.x, "y": frame.origin.y, "width": frame.width, "height": frame.height],
|
||||
"visible": window?.isVisible ?? false,
|
||||
"hidden": NSApplication.shared.isHidden,
|
||||
"onActiveSpace": window?.isOnActiveSpace ?? false,
|
||||
"miniaturized": window?.isMiniaturized ?? false,
|
||||
"completed": completed, "held": held, "unpaired": unpaired,
|
||||
"otherEvents": otherEvents, "sources": sources,
|
||||
"events": events, "eventsTruncated": eventsTruncated,
|
||||
]
|
||||
if let data = try? JSONSerialization.data(withJSONObject: value, options: [.sortedKeys]), data != lastSaved {
|
||||
do {
|
||||
try data.write(to: receipt, options: .atomic)
|
||||
lastSaved = data
|
||||
} catch {
|
||||
// Leave lastSaved unchanged so the next timer tick retries.
|
||||
}
|
||||
}
|
||||
}
|
||||
private func belongsToHelper(_ event: NSEvent) -> Bool {
|
||||
let source = event.cgEvent?.getIntegerValueField(.eventSourceUnixProcessID) ?? -1
|
||||
let text = (try? String(contentsOf: helperPIDFile, encoding: .utf8)) ?? ""
|
||||
let expected = Int64(text.trimmingCharacters(in: .whitespacesAndNewlines))
|
||||
let accepted = expected.map { $0 > 0 && source == $0 } ?? false
|
||||
if events.count < maximumRecordedEvents {
|
||||
sources.append(source)
|
||||
events.append([
|
||||
"type": event.type.rawValue,
|
||||
"sourcePID": source,
|
||||
"expectedHelperPID": expected as Any? ?? NSNull(),
|
||||
"accepted": accepted,
|
||||
"marker": event.cgEvent?.getIntegerValueField(.eventSourceUserData) ?? -1,
|
||||
"windowNumber": event.windowNumber,
|
||||
"eventNumber": event.eventNumber,
|
||||
"localX": event.locationInWindow.x,
|
||||
"localY": event.locationInWindow.y,
|
||||
"timestamp": event.timestamp,
|
||||
"receivedUptime": ProcessInfo.processInfo.systemUptime,
|
||||
])
|
||||
} else {
|
||||
eventsTruncated += 1
|
||||
}
|
||||
if !accepted {
|
||||
otherEvents += 1
|
||||
save()
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
|
||||
let app = NSApplication.shared
|
||||
app.setActivationPolicy(.accessory)
|
||||
app.finishLaunching()
|
||||
let view = ReceiptView(receipt: URL(fileURLWithPath: CommandLine.arguments[1]), helperPIDFile: URL(fileURLWithPath: CommandLine.arguments[2]))
|
||||
let stop = URL(fileURLWithPath: CommandLine.arguments[3])
|
||||
let window = NSWindow(contentRect: view.frame, styleMask: [.titled, .closable], backing: .buffered, defer: false)
|
||||
window.title = "Temporary Computer Use fixture"
|
||||
window.contentView = view
|
||||
window.setFrameOrigin(NSPoint(x: 60, y: 50))
|
||||
window.orderFrontRegardless()
|
||||
// LaunchServices applies background-launch ordering after main starts. Publish
|
||||
// readiness only after that boundary; never raise the window for each action.
|
||||
let readyTimer = Timer.scheduledTimer(withTimeInterval: 0.2, repeats: false) { _ in
|
||||
window.orderFrontRegardless()
|
||||
view.publishReady()
|
||||
}
|
||||
let stopTimer = Timer.scheduledTimer(withTimeInterval: 0.02, repeats: true) { _ in
|
||||
view.refresh()
|
||||
if FileManager.default.fileExists(atPath: stop.path) { app.terminate(nil) }
|
||||
}
|
||||
let lifetimeTimer = Timer.scheduledTimer(withTimeInterval: 90, repeats: false) { _ in app.terminate(nil) }
|
||||
app.run()
|
||||
@@ -33,6 +33,13 @@ if (feature('ABLATION_BASELINE') && process.env.CLAUDE_CODE_ABLATION_BASELINE) {
|
||||
async function main(): Promise<void> {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
// Sandboxed CU worker: no user configuration, providers or CLI startup.
|
||||
if (args[0] === '--computer-use-repl-worker') {
|
||||
const { runComputerUseReplWorker } = await import('../utils/computerUse/replWorker.js');
|
||||
await runComputerUseReplWorker();
|
||||
return;
|
||||
}
|
||||
|
||||
// Fast-path for --version/-v: zero module loading needed
|
||||
if (args.length === 1 && (args[0] === '--version' || args[0] === '-v' || args[0] === '-V')) {
|
||||
// MACRO.VERSION is inlined at build time
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
|
||||
import { getBundledSkills } from '../bundledSkills.js'
|
||||
import { buildPlatformComputerUseTools } from '../../vendor/computer-use-mcp/mcpServer.js'
|
||||
import {
|
||||
getComputerUsePrompt,
|
||||
getComputerUseToolAllowlist,
|
||||
@@ -35,7 +36,7 @@ describe('computer-use skill content', () => {
|
||||
// hatch has to be enumerated.
|
||||
const prompt = await computerUsePrompt()
|
||||
expect(prompt).toContain('will never fill in')
|
||||
for (const tool of ['click', 'drag', 'press_key', 'type_text', 'paste']) {
|
||||
for (const tool of ['app.click', 'app.drag', 'app.pressKey', 'app.typeText', 'app.paste']) {
|
||||
expect(prompt).toContain(tool)
|
||||
}
|
||||
expect(prompt).toContain('menu bar')
|
||||
@@ -44,7 +45,7 @@ describe('computer-use skill content', () => {
|
||||
test('treats a timed-out paste as result-unknown and refreshes before retry', async () => {
|
||||
const prompt = await computerUsePrompt()
|
||||
expect(prompt).toContain('may have consumed the paste late')
|
||||
expect(prompt).toContain('get_app_state')
|
||||
expect(prompt).toContain('app.getAXStateAndScreenshot()')
|
||||
})
|
||||
|
||||
test('caps repetition and closes the shell escape hatch', async () => {
|
||||
@@ -105,10 +106,14 @@ describe('computer-use skill registration', () => {
|
||||
const skill = getBundledSkills().find(s => s.name === 'computer-use')
|
||||
const platform = process.platform === 'win32' ? 'win32' : 'darwin'
|
||||
expect(skill!.allowedTools).toEqual(getComputerUseToolAllowlist(platform))
|
||||
expect(skill!.allowedTools).toEqual(buildPlatformComputerUseTools({
|
||||
platform,
|
||||
screenshotFiltering: platform === 'win32' ? 'none' : 'native',
|
||||
}, 'pixels').map(tool => `mcp__computer-use__${tool.name}`))
|
||||
expect(skill!.allowedTools).toContain(
|
||||
process.platform === 'win32'
|
||||
? 'mcp__computer-use__screenshot'
|
||||
: 'mcp__computer-use__get_app_state',
|
||||
: 'mcp__computer-use__js',
|
||||
)
|
||||
expect(skill!.allowedTools).not.toContain('mcp__computer-use__request_access')
|
||||
expect(
|
||||
@@ -139,3 +144,41 @@ describe('computer-use Windows guidance', () => {
|
||||
expect(tools).not.toContain('mcp__computer-use__get_app_state')
|
||||
})
|
||||
})
|
||||
|
||||
describe('computer-use observation batching', () => {
|
||||
test('batches decisions from the current state without requiring one call per click', () => {
|
||||
const prompt = getComputerUsePrompt('darwin')
|
||||
expect(prompt).toContain('sequence')
|
||||
expect(prompt).toContain('stable canvas')
|
||||
expect(prompt).toContain('Do not force one model round trip per click')
|
||||
expect(prompt).toContain('stop and re-observe')
|
||||
expect(prompt).not.toContain('one-step sequence is the normal')
|
||||
expect(prompt).not.toContain('about a second')
|
||||
expect(prompt).toContain('Do not add a fixed sleep')
|
||||
expect(getComputerUseToolAllowlist('darwin')).toEqual([
|
||||
'mcp__computer-use__js',
|
||||
'mcp__computer-use__js_reset',
|
||||
])
|
||||
expect(getComputerUseToolAllowlist('win32')).not.toContain('mcp__computer-use__sequence')
|
||||
expect(getComputerUsePrompt('win32')).not.toContain('sequence({')
|
||||
})
|
||||
|
||||
test('teaches persistent app methods and explicit observation without promising broader runtime parity', () => {
|
||||
const prompt = getComputerUsePrompt('darwin')
|
||||
expect(prompt).toContain('Prefer `js({code})`')
|
||||
expect(prompt).toContain('var app = await cua.getApp("App Name")')
|
||||
expect(prompt).toContain('for (const x of [240, 280]) await app.drag')
|
||||
expect(prompt).toContain('getAXStateAndScreenshot()')
|
||||
expect(prompt).toContain('{emit:false}')
|
||||
expect(prompt).toContain('Integer indices map only to')
|
||||
expect(prompt).toContain('same native capture')
|
||||
expect(prompt).toContain('Ordinary\nscript errors retain bindings')
|
||||
expect(prompt).toContain('Timeout, cancellation, and `js_reset` discard')
|
||||
expect(prompt).toContain('Browser/DOM APIs, imports, Node, filesystem, and networking are unavailable')
|
||||
expect(prompt).toContain('256 native calls')
|
||||
expect(prompt).toContain('compatibility interfaces')
|
||||
expect(prompt).not.toContain('launches the app')
|
||||
expect(prompt).not.toContain('```json')
|
||||
expect(getComputerUsePrompt('win32')).not.toContain('cua.getApp')
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,19 +1,11 @@
|
||||
import { COMPUTER_USE_BATCHING_GUIDANCE } from '../../vendor/computer-use-mcp/instructions.js'
|
||||
import { isComputerUseSkillEnabled } from '../../utils/computerUse/skillGate.js'
|
||||
import { buildPlatformComputerUseTools } from '../../vendor/computer-use-mcp/mcpServer.js'
|
||||
import { registerBundledSkill } from '../bundledSkills.js'
|
||||
|
||||
const MAC_COMPUTER_USE_TOOLS = [
|
||||
'list_apps',
|
||||
'get_app_state',
|
||||
'click',
|
||||
'set_value',
|
||||
'select_text',
|
||||
'perform_secondary_action',
|
||||
'scroll',
|
||||
'drag',
|
||||
'press_key',
|
||||
'type_text',
|
||||
'paste',
|
||||
'js',
|
||||
'js_reset',
|
||||
].map(name => `mcp__computer-use__${name}`)
|
||||
|
||||
export function getComputerUseToolAllowlist(
|
||||
@@ -44,37 +36,39 @@ const COMPUTER_USE_PROMPT = `# Operating Mac apps
|
||||
You are driving real applications on the user's Mac through the accessibility
|
||||
engine. Work in this loop:
|
||||
|
||||
1. \`get_app_state({ app })\` — returns the app's accessibility tree AND a
|
||||
screenshot of its window. It launches the app in the background if it is not
|
||||
running, so there is no separate "open" step.
|
||||
2. Act.
|
||||
3. \`get_app_state\` again before deciding anything else. Element handles are only
|
||||
valid inside the snapshot that produced them; re-read to get fresh ones.
|
||||
1. Use \`js\` to bind an app with \`cua.getApp("App Name")\` and read its initial
|
||||
AX state. Request \`app.getScreenshot()\` when visual information is needed.
|
||||
2. Perform known actions in that persistent JS session, then observe at the next
|
||||
decision point.
|
||||
3. Inspect state and screenshots before choosing further actions.
|
||||
|
||||
Do not sleep between an action and the next \`get_app_state\`. The engine already
|
||||
waits for the UI to settle — about a second, longer while the app shows a
|
||||
progress indicator.
|
||||
${COMPUTER_USE_BATCHING_GUIDANCE}
|
||||
|
||||
Do not add a fixed sleep before an observation. The observation path waits
|
||||
for UI changes when needed. Read its result before deciding whether more context
|
||||
is necessary.
|
||||
|
||||
## Naming the app
|
||||
|
||||
Pass the app name straight to \`get_app_state\` — display name, bundle identifier,
|
||||
or full path all work. Do NOT call \`list_apps\` first just to look up an
|
||||
Pass the app name straight to \`cua.getApp\` — display name, bundle identifier,
|
||||
or full path all work. Do NOT call \`cua.listApps()\` first just to look up an
|
||||
identifier. If a call fails with a display name, immediately retry the same call
|
||||
with the bundle identifier before investigating anything else. Reach for
|
||||
\`list_apps\` only when you genuinely cannot name the app.
|
||||
\`cua.listApps()\` only when you genuinely cannot name the app.
|
||||
|
||||
## Choosing how to act
|
||||
|
||||
Prefer \`element_index\` while the tree still describes the control you want: it
|
||||
targets the element directly and survives the window moving.
|
||||
Prefer \`app.click("gN:id")\` while the tree
|
||||
describes the control you want: it targets the element directly and survives
|
||||
the window moving.
|
||||
|
||||
Switch to \`x\`/\`y\` read off the screenshot when either is true:
|
||||
Switch to \`[x,y]\` read off the screenshot when either is true:
|
||||
|
||||
- **the tree is a dead end.** Many Chromium/Electron apps expose only their
|
||||
window frame and menu bar. \`get_app_state\` says so explicitly when it detects
|
||||
window frame and menu bar. AX state says so explicitly when it detects
|
||||
this. That is not a slow-loading tree — it will never fill in. Five tools
|
||||
still work there: \`click\` with x/y, \`drag\` with x/y, \`press_key\`, and
|
||||
\`type_text\`/\`paste\`. Everything else needs a handle it cannot get. The menu bar stays
|
||||
still work there: \`app.click([x,y])\`, \`app.drag(from,to)\`, \`app.pressKey(key)\`, and
|
||||
\`app.typeText(text)\`/\`app.paste(text)\`. Element methods need an observed handle. The menu bar stays
|
||||
fully addressable, so a menu path is often the shortest route.
|
||||
- **element actions run but the UI does not change.**
|
||||
|
||||
@@ -83,14 +77,15 @@ them as-is; do not convert them.
|
||||
|
||||
## Knowing whether it worked
|
||||
|
||||
Mutating tools return a fixed receipt. The receipt means the action was
|
||||
**dispatched**, never that it had the intended effect. Only the next
|
||||
\`get_app_state\` tells you what actually happened. Judge the screenshot as well
|
||||
as AX text: Chromium/CEF content can visibly change while the AX diff stays empty.
|
||||
Awaiting a JS action or receiving a standalone receipt means the action was
|
||||
**dispatched**, never that it had the intended effect. Observe through the JS
|
||||
App to learn what actually happened.
|
||||
Judge the screenshot as well as AX text: Chromium/CEF content
|
||||
can visibly change while the AX diff stays empty.
|
||||
|
||||
If two consecutive screenshots leave the relevant UI unchanged, the approach is wrong.
|
||||
Change something real: switch from handle to coordinates, target a different
|
||||
element, re-read the full tree with \`disableDiff: true\`, or take a different
|
||||
element, re-read the full tree with \`app.getAXState({disableDiffing:true})\`, or take a different
|
||||
route through the UI such as the menu bar. Repeating the same call a third time
|
||||
never helps.
|
||||
|
||||
@@ -102,19 +97,19 @@ session.
|
||||
|
||||
## Tool notes
|
||||
|
||||
- \`get_app_state\` returns a diff against the previous read by default. Pass
|
||||
\`disableDiff: true\` when you need the whole tree — after acting from a
|
||||
- \`app.getAXState()\` returns a diff against the previous read by default. Pass
|
||||
\`{disableDiffing:true}\` when you need the whole tree — after acting from a
|
||||
screenshot alone, or whenever the diff has left you unsure of the state.
|
||||
- \`perform_secondary_action\` only accepts an action actually listed for that
|
||||
- \`app.performSecondaryAction(element,action)\` only accepts an action actually listed for that
|
||||
element in the tree. Do not guess action names.
|
||||
- \`press_key\` and \`type_text\` are delivered to the named app, so they cannot
|
||||
- \`app.pressKey\` and \`app.typeText\` are delivered to the named app, so they cannot
|
||||
trigger global system shortcuts.
|
||||
- If \`type_text\` does not visibly change a Chromium/CEF field, use
|
||||
\`paste({ app, text, format: "text" })\`. It restores the user's prior clipboard.
|
||||
If it times out after dispatch, call \`get_app_state\` before retrying: the app
|
||||
- If \`app.typeText\` does not visibly change a Chromium/CEF field, use
|
||||
\`app.paste(text,{format:"text"})\`. It restores the user's prior clipboard.
|
||||
If it times out after dispatch, rebind and call \`app.getAXStateAndScreenshot()\` before retrying: the app
|
||||
may have consumed the paste late.
|
||||
- \`press_key\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c".
|
||||
- \`select_text\` works inside editable elements; use \`prefix\`/\`suffix\` to
|
||||
- \`app.pressKey\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c".
|
||||
- \`app.selectText\` works inside editable elements; use \`prefix\`/\`suffix\` to
|
||||
disambiguate repeated matches.
|
||||
|
||||
## Trust
|
||||
|
||||
@@ -4,6 +4,7 @@ import { lstatSync, mkdirSync, mkdtempSync, rmSync, symlinkSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
import {
|
||||
__connectWithRetryForTests,
|
||||
__prepareDaemonSocketDirectoryForTests,
|
||||
__resetDaemonClientForTests,
|
||||
__daemonStartCountForTests,
|
||||
@@ -69,6 +70,48 @@ afterEach(() => {
|
||||
})
|
||||
|
||||
describe('cu-helper daemon system commands', () => {
|
||||
test('readiness deadline also bounds a socket that never emits connect or error', async () => {
|
||||
const socket = new FakeSocket()
|
||||
let attempts = 0
|
||||
const request = __connectWithRetryForTests('/fixture/never-connects.sock', 20, () => {
|
||||
attempts++
|
||||
return socket as never
|
||||
}).catch(error => error)
|
||||
let guard: ReturnType<typeof setTimeout> | undefined
|
||||
const result = await Promise.race([
|
||||
request,
|
||||
new Promise<string>(resolve => { guard = setTimeout(() => resolve('still pending'), 150) }),
|
||||
])
|
||||
clearTimeout(guard)
|
||||
expect(result).toBeInstanceOf(Error)
|
||||
expect(result.message).toMatch(/socket not ready within 20ms/)
|
||||
expect(attempts).toBe(1)
|
||||
expect(socket.destroyed).toBe(true)
|
||||
expect(socket.listenerCount('connect')).toBe(0)
|
||||
})
|
||||
|
||||
test('readiness deadline is cleared after connecting successfully', async () => {
|
||||
const socket = new FakeSocket()
|
||||
const request = __connectWithRetryForTests('/fixture/ready.sock', 20, () => {
|
||||
queueMicrotask(() => socket.emit('connect'))
|
||||
return socket as never
|
||||
})
|
||||
expect(await request).toBe(socket as never)
|
||||
await new Promise(resolve => setTimeout(resolve, 35))
|
||||
expect(socket.destroyed).toBe(false)
|
||||
})
|
||||
|
||||
test('a socket that closes before connecting is retired within the readiness deadline', async () => {
|
||||
const socket = new FakeSocket()
|
||||
const request = __connectWithRetryForTests('/fixture/closed.sock', 20, () => {
|
||||
queueMicrotask(() => socket.emit('close'))
|
||||
return socket as never
|
||||
})
|
||||
await expect(request).rejects.toThrow(/socket closed before connecting/)
|
||||
expect(socket.destroyed).toBe(true)
|
||||
expect(socket.listenerCount('connect')).toBe(0)
|
||||
})
|
||||
|
||||
test('uses trusted absolute binaries for process probing and LaunchServices', async () => {
|
||||
const { __daemonProcessCommandsForTests } = await import('./cuHelperDaemon.js')
|
||||
expect(
|
||||
@@ -179,13 +222,14 @@ describe('cu-helper daemon failure classification', () => {
|
||||
await waitForWrite(socket)
|
||||
const id = JSON.parse(socket.writes[0]!).id
|
||||
socket.emit('data', Buffer.from(
|
||||
`${JSON.stringify({ id, ok: false, error: { message: 'grant_flag_required' } })}\n`,
|
||||
`${JSON.stringify({ id, ok: false, error: { message: 'grant_flag_required', code: 'not_trusted' } })}\n`,
|
||||
))
|
||||
|
||||
const error = await request
|
||||
expect(error).toBeInstanceOf(Error)
|
||||
expect(error).not.toBeInstanceOf(DaemonUnavailableError)
|
||||
expect(error.message).toBe('grant_flag_required')
|
||||
expect(error.nativeCode).toBe('not_trusted')
|
||||
})
|
||||
|
||||
test('every request carries negotiated protocol, deadline, and stable turn identity', async () => {
|
||||
|
||||
@@ -9,6 +9,7 @@ import { logForDebugging } from '../debug.js'
|
||||
import { ensureInstalledHelper } from './cuHelperInstall.js'
|
||||
import { attestDaemonSocketPeer } from './cuHelperPeerAttestation.js'
|
||||
import { getRuntimePaths } from './pythonBridge.js'
|
||||
import { NativeCommandError } from '../../vendor/computer-use-mcp/nativeError.js'
|
||||
|
||||
/**
|
||||
* Long-lived `cu-helper daemon` client (macOS only).
|
||||
@@ -48,7 +49,7 @@ const CONNECTION_SCOPED_COMMANDS = new Set([
|
||||
* command was dispatched — the daemon couldn't install/start/connect, or the
|
||||
* socket rejected the write synchronously.
|
||||
* A command that the daemon ran and rejected (e.g. `not_trusted`, `unknown_key`)
|
||||
* rejects with a plain `Error` instead. The bridge uses this distinction to fall
|
||||
* rejects with `NativeCommandError` instead. The bridge uses this distinction to fall
|
||||
* back to the one-shot CLI ONLY on infra failure — never silently swallowing a
|
||||
* real command error (which would just fail the same way on the CLI, minus the
|
||||
* overlay). See helperBridge.ts.
|
||||
@@ -358,6 +359,7 @@ async function connectWithRetry(
|
||||
sock: string,
|
||||
timeoutMs: number,
|
||||
getLaunchError: () => Error | undefined = () => undefined,
|
||||
connect: (socketPath: string) => net.Socket = socketPath => net.connect(socketPath),
|
||||
): Promise<net.Socket> {
|
||||
const deadline = Date.now() + timeoutMs
|
||||
let lastErr: Error | undefined
|
||||
@@ -366,16 +368,35 @@ async function connectWithRetry(
|
||||
if (launchError) throw launchError
|
||||
try {
|
||||
return await new Promise<net.Socket>((resolve, reject) => {
|
||||
const s = net.connect(sock)
|
||||
s.once('connect', () => resolve(s))
|
||||
s.once('error', err => {
|
||||
const s = connect(sock)
|
||||
const cleanup = () => {
|
||||
clearTimeout(timer)
|
||||
s.removeListener('connect', onConnect)
|
||||
s.removeListener('close', onClose)
|
||||
}
|
||||
const onConnect = () => {
|
||||
cleanup()
|
||||
resolve(s)
|
||||
}
|
||||
const fail = (error: Error) => {
|
||||
cleanup()
|
||||
s.destroy()
|
||||
reject(err instanceof Error ? err : new Error(String(err)))
|
||||
})
|
||||
reject(error)
|
||||
}
|
||||
const onClose = () => fail(new Error('daemon socket closed before connecting'))
|
||||
// The outer retry deadline cannot interrupt an unresolved connect.
|
||||
// Bound the actual attempt and retire its socket before retrying.
|
||||
const timer = setTimeout(() => fail(new Error('daemon socket connection timed out')), Math.max(0, deadline - Date.now()))
|
||||
s.once('connect', onConnect)
|
||||
s.once('close', onClose)
|
||||
// Keep an error listener after connect until the daemon state attaches
|
||||
// its own handlers; attestation happens between those two steps.
|
||||
s.once('error', err => fail(err instanceof Error ? err : new Error(String(err))))
|
||||
})
|
||||
} catch (err) {
|
||||
lastErr = err instanceof Error ? err : new Error(String(err))
|
||||
await new Promise(r => setTimeout(r, 100))
|
||||
const remaining = deadline - Date.now()
|
||||
if (remaining > 0) await new Promise(r => setTimeout(r, Math.min(100, remaining)))
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
@@ -383,6 +404,14 @@ async function connectWithRetry(
|
||||
)
|
||||
}
|
||||
|
||||
export function __connectWithRetryForTests(
|
||||
sock: string,
|
||||
timeoutMs: number,
|
||||
connect: (socketPath: string) => net.Socket,
|
||||
): Promise<net.Socket> {
|
||||
return connectWithRetry(sock, timeoutMs, undefined, connect)
|
||||
}
|
||||
|
||||
/** Tear down current state (on death/error) so the next call respawns fresh. */
|
||||
function resetState(reason: string, expectedGeneration?: number): void {
|
||||
if (
|
||||
@@ -545,7 +574,7 @@ function attachSocketHandlers(state: DaemonState): void {
|
||||
const line = state.buf.slice(0, nl)
|
||||
state.buf = state.buf.slice(nl + 1)
|
||||
if (!line.trim()) continue
|
||||
let msg: { id?: string; ok?: boolean; result?: unknown; error?: { message?: string } }
|
||||
let msg: { id?: string; ok?: boolean; result?: unknown; error?: { message?: string; code?: unknown } }
|
||||
try { msg = JSON.parse(line) } catch { continue }
|
||||
const id = msg.id
|
||||
if (!id) continue
|
||||
@@ -554,7 +583,10 @@ function attachSocketHandlers(state: DaemonState): void {
|
||||
state.pending.delete(id)
|
||||
clearTimeout(pending.timer)
|
||||
if (msg.ok) pending.resolve(msg.result)
|
||||
else pending.reject(new Error(msg.error?.message || 'cu-helper daemon command failed'))
|
||||
else pending.reject(new NativeCommandError(
|
||||
msg.error?.message || 'cu-helper daemon command failed',
|
||||
typeof msg.error?.code === 'string' ? msg.error.code : undefined,
|
||||
))
|
||||
}
|
||||
})
|
||||
// The daemon's death is observed via the socket closing — NOT via `proc`,
|
||||
|
||||
@@ -313,6 +313,24 @@ describe('CLI executor Codex engine — daemon payload alignment', () => {
|
||||
const text = await exec.engine!.listApps()
|
||||
expect(text).toBe('No running applications are available to control.')
|
||||
})
|
||||
|
||||
itEngine('preserves native inventory metadata without guessing running state', async () => {
|
||||
const exec = await loadExecutor()
|
||||
nextResult = [
|
||||
{ id: 'dev.test.Editor', displayName: 'Editor', isRunning: false, lastUsedDate: '2026-09-09T01:00:00Z', useCount: 4 },
|
||||
{ id: 'dev.test.Unknown' },
|
||||
]
|
||||
expect(await exec.engine!.listAppsInfo!()).toEqual(nextResult)
|
||||
expect(calls).toEqual([{ command: 'list_apps', payload: {} }])
|
||||
})
|
||||
|
||||
itEngine('accepts the previous helper inventory while upgrading the host', async () => {
|
||||
const exec = await loadExecutor()
|
||||
nextResult = [{ bundleId: 'com.apple.finder', displayName: 'Finder' }]
|
||||
expect(await exec.engine!.listAppsInfo!()).toEqual([
|
||||
{ id: 'com.apple.finder', displayName: 'Finder', isRunning: true },
|
||||
])
|
||||
})
|
||||
})
|
||||
|
||||
describe('Windows virtual cursor motion', () => {
|
||||
|
||||
@@ -24,6 +24,7 @@ import type {
|
||||
SetValueResult,
|
||||
} from '../../vendor/computer-use-mcp/index.js'
|
||||
import { API_RESIZE_PARAMS, targetImageSize } from '../../vendor/computer-use-mcp/index.js'
|
||||
import { formatNativeAppList, type NativeAppInfo } from '../../vendor/computer-use-mcp/executor.js'
|
||||
import { sleep } from '../sleep.js'
|
||||
import {
|
||||
CLI_HOST_BUNDLE_ID,
|
||||
@@ -102,9 +103,8 @@ async function writeClipboard(text: string): Promise<void> {
|
||||
// - drag: `from{x,y}`, `to{x,y}`, `button`
|
||||
// - press_key: `key` ; type_text: `text`
|
||||
//
|
||||
// `list_apps` is the one shape that needs reconciliation: the daemon returns a
|
||||
// structured `AppRef[]` (`{bundleId, displayName}`), but the MCP tool face
|
||||
// (toolCalls.handleListApps) expects the rendered text block. We format it here.
|
||||
// Preserve structured list_apps rows for the JavaScript client. Older helpers
|
||||
// returned only running AppRef rows; normalize those during a host upgrade.
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
/** A daemon `list_apps` row — `Apps.AppRef` (Geometry.swift). */
|
||||
@@ -132,19 +132,13 @@ function appTargetPayload(target: AppTarget): Record<string, unknown> {
|
||||
return { ...identity }
|
||||
}
|
||||
|
||||
/**
|
||||
* Render the daemon's `AppRef[]` into the `list_apps` text block the tool face
|
||||
* surfaces verbatim. The native daemon does not track last-used / use-count, so
|
||||
* we emit the stable subset: "<Display Name> — <bundle.id>", most-recent-first
|
||||
* ordering being whatever the daemon returned (it sorts by display name).
|
||||
*/
|
||||
function formatAppList(apps: readonly DaemonAppRef[]): string {
|
||||
if (apps.length === 0) {
|
||||
return 'No running applications are available to control.'
|
||||
}
|
||||
return apps
|
||||
.map(a => `${a.displayName} — ${a.bundleId}`)
|
||||
.join('\n')
|
||||
async function listNativeApps(): Promise<NativeAppInfo[]> {
|
||||
const apps = await callHelper<Array<NativeAppInfo | DaemonAppRef>>('list_apps', {})
|
||||
return apps.map(app => 'id' in app ? app : {
|
||||
id: app.bundleId,
|
||||
displayName: app.displayName,
|
||||
isRunning: true,
|
||||
})
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -157,10 +151,11 @@ function formatAppList(apps: readonly DaemonAppRef[]): string {
|
||||
export function createCodexEngine(): CodexComputerEngine {
|
||||
return {
|
||||
async listApps(): Promise<string> {
|
||||
const apps = await callHelper<DaemonAppRef[]>('list_apps', {})
|
||||
return formatAppList(apps)
|
||||
return formatNativeAppList(await listNativeApps())
|
||||
},
|
||||
|
||||
listAppsInfo: listNativeApps,
|
||||
|
||||
async resolveTarget(target: AppTarget): Promise<ResolvedAppTarget> {
|
||||
// Sent as-is: the daemon owns the selector→process mapping, and it must
|
||||
// never launch anything to satisfy a match.
|
||||
|
||||
@@ -11,6 +11,7 @@ import { normalizeOsPermissions } from './permissions.js'
|
||||
// Platform-routed helper: macOS → native cu-helper, Windows → Python helper.
|
||||
import { callHelper } from './helperBridge.js'
|
||||
import { maybeShowNativePermissionCard } from './nativePermissionCard.js'
|
||||
import { ComputerUseRepl } from './replRuntime.js'
|
||||
|
||||
class DebugLogger implements Logger {
|
||||
silly(message: string, ...args: unknown[]): void {
|
||||
@@ -37,6 +38,7 @@ export function getComputerUseHostAdapter(): ComputerUseHostAdapter {
|
||||
cached = {
|
||||
serverName: COMPUTER_USE_MCP_SERVER_NAME,
|
||||
logger: new DebugLogger(),
|
||||
createReplRuntime: () => new ComputerUseRepl(),
|
||||
executor: createCliExecutor({
|
||||
getMouseAnimationEnabled: () => getChicagoSubGates().mouseAnimation,
|
||||
getHideBeforeActionEnabled: () => getChicagoSubGates().hideBeforeAction,
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import { expect, test } from 'bun:test'
|
||||
import type { ToolUseContext } from '../../Tool.js'
|
||||
import { MCPTool } from '../../tools/MCPTool/MCPTool.js'
|
||||
import { frameAppStateEnvelope } from '../../vendor/computer-use-mcp/toolCalls.js'
|
||||
import { anthropicToOpenaiChat } from '../../server/proxy/transform/anthropicToOpenaiChat.js'
|
||||
import { anthropicToOpenaiResponses } from '../../server/proxy/transform/anthropicToOpenaiResponses.js'
|
||||
import type { AnthropicContentBlock, AnthropicRequest } from '../../server/proxy/transform/types.js'
|
||||
import { dispatchComputerUseCall } from './wrapper.js'
|
||||
|
||||
// Opaque fixture bytes: the transport must preserve them, not decode, resize
|
||||
// or relabel them after the native helper has encoded its screenshot.
|
||||
const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 4, 0x4a, 0x46, 0xff, 0xd9]).toString('base64')
|
||||
|
||||
test('a JPEG native screenshot keeps its bytes and MIME through the real CLI tool result and both model protocols', async () => {
|
||||
const nativeResult = frameAppStateEnvelope({
|
||||
pid: 42, elementCount: 1, truncated: false, durationMs: 1,
|
||||
axText: 'App=Fixture\nWindow: Canvas',
|
||||
screenshot: { base64: jpeg, width: 360, height: 280, mimeType: 'image/jpeg' },
|
||||
})
|
||||
const result = await dispatchComputerUseCall(async () => nativeResult, 'js', {
|
||||
code: 'await app.getAXStateAndScreenshot()',
|
||||
}, { abortController: new AbortController() } as ToolUseContext)
|
||||
const toolResult = MCPTool.mapToolResultToToolResultBlockParam(result.data as never, 'cu-shot')
|
||||
const image = (toolResult.content as AnthropicContentBlock[]).find(block => block.type === 'image')
|
||||
expect(image).toEqual({ type: 'image', source: { type: 'base64', media_type: 'image/jpeg', data: jpeg } })
|
||||
|
||||
const request: AnthropicRequest = {
|
||||
model: 'fixture', max_tokens: 100,
|
||||
messages: [
|
||||
{ role: 'assistant', content: [{ type: 'tool_use', id: 'cu-shot', name: 'computer_use', input: {} }] },
|
||||
{ role: 'user', content: [toolResult as AnthropicContentBlock] },
|
||||
],
|
||||
}
|
||||
for (const converted of [anthropicToOpenaiChat(request), anthropicToOpenaiResponses(request)]) {
|
||||
const wire = JSON.stringify(converted)
|
||||
expect(wire).toContain(`data:image/jpeg;base64,${jpeg}`)
|
||||
expect(wire).not.toContain('data:image/png;')
|
||||
}
|
||||
})
|
||||
|
||||
test('older native helpers without screenshot MIME retain their PNG transport contract', async () => {
|
||||
const nativeResult = frameAppStateEnvelope({
|
||||
pid: 42, elementCount: 0, truncated: false, durationMs: 1, axText: 'App=Fixture',
|
||||
screenshot: { base64: 'iVBORw==', width: 1, height: 1 },
|
||||
})
|
||||
const result = await dispatchComputerUseCall(async () => nativeResult, 'js', {}, {
|
||||
abortController: new AbortController(),
|
||||
} as ToolUseContext)
|
||||
expect(result.data).toContainEqual({
|
||||
type: 'image', source: { type: 'base64', media_type: 'image/png', data: 'iVBORw==' },
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,124 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { pathToFileURL } from 'node:url'
|
||||
import { createComputerUseReplWorker } from './replWorker'
|
||||
import type { ReplOutput } from '../../vendor/computer-use-mcp/replProtocol'
|
||||
|
||||
test('minified bootstrap source remains self-contained in an isolated realm', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'cu-repl-minified-'))
|
||||
try {
|
||||
const entrypoint = join(directory, 'bootstrap.ts')
|
||||
const apiPath = new URL('../../vendor/computer-use-mcp/replApi.ts', import.meta.url).pathname
|
||||
await writeFile(entrypoint, `export {REPL_BOOTSTRAP_SOURCE} from ${JSON.stringify(apiPath)}`)
|
||||
const build = await Bun.build({
|
||||
entrypoints: [entrypoint],
|
||||
outdir: join(directory, 'out'),
|
||||
minify: { whitespace: true, identifiers: true, syntax: true },
|
||||
target: 'bun',
|
||||
})
|
||||
expect(build.success).toBe(true)
|
||||
const { REPL_BOOTSTRAP_SOURCE } = await import(pathToFileURL(build.outputs[0]!.path).href)
|
||||
const messages: ReplOutput[] = []
|
||||
const worker = createComputerUseReplWorker(message => {
|
||||
messages.push(message)
|
||||
if (message.type === 'invoke') {
|
||||
queueMicrotask(() => {
|
||||
void worker.receive({
|
||||
type: 'response', cellId: message.cellId, requestId: message.requestId,
|
||||
result: { content: [{ type: 'text', text: 'App=Fixture\n[g1:1] button Test' }] },
|
||||
})
|
||||
})
|
||||
}
|
||||
})
|
||||
await worker.receive({ type: 'init', bootstrap: REPL_BOOTSTRAP_SOURCE })
|
||||
await worker.receive({ type: 'run', cellId: 1, code: 'let app = await cua.getApp("Fixture"); let count = 0' })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'for(let i=0;i<3;i++){await app.click([10,10]);count++};nodeRepl.write("done="+count)' })
|
||||
expect(messages.filter(message => message.type === 'done' && message.error)).toEqual([])
|
||||
expect(messages).toContainEqual({ type: 'emit', cellId: 2, content: { type: 'text', text: 'done=3' } })
|
||||
expect(messages.filter(message => message.type === 'invoke').map(message => message.name)).toEqual(['get_app_state', 'click', 'click', 'click'])
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
|
||||
const suite = process.platform === 'darwin' ? describe : describe.skip
|
||||
|
||||
suite('Computer Use compiled worker entrypoint', () => {
|
||||
test('minified standalone binaries run persistent cells without depending on embedded assets', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'cu-repl-compiled-'))
|
||||
const entrypoint = join(directory, 'fixture.ts')
|
||||
const executable = join(directory, 'fixture')
|
||||
const runtimePath = new URL('./replRuntime.ts', import.meta.url).pathname
|
||||
const workerPath = new URL('./replWorker.ts', import.meta.url).pathname
|
||||
const bundledPath = new URL('../bundledMode.ts', import.meta.url).pathname
|
||||
await writeFile(entrypoint, `
|
||||
if (process.argv[2] === '--computer-use-repl-worker') {
|
||||
const {runComputerUseReplWorker} = await import(${JSON.stringify(workerPath)})
|
||||
await runComputerUseReplWorker()
|
||||
} else {
|
||||
const {ComputerUseRepl} = await import(${JSON.stringify(runtimePath)})
|
||||
const {isInBundledMode} = await import(${JSON.stringify(bundledPath)})
|
||||
const runtime = new ComputerUseRepl()
|
||||
const calls = []
|
||||
const invoke = async (name, args) => {
|
||||
calls.push({name, args})
|
||||
return {content:[{type:'text',text:'App=Fixture\\n[g1:1] button Test'}]}
|
||||
}
|
||||
try {
|
||||
const first = await runtime.run({code:'let app = await cua.getApp("Fixture"); let count = 0',timeoutMs:5000},invoke)
|
||||
const second = await runtime.run({code:'for(let i=0;i<3;i++){await app.click([10,10]);count++};nodeRepl.write("done="+count)',timeoutMs:5000},invoke)
|
||||
console.log(JSON.stringify({embedded:Bun.embeddedFiles.length,bundled:isInBundledMode(),main:Bun.main,url:import.meta.url,argv1:process.argv[1],first,second,calls}))
|
||||
} finally {
|
||||
await runtime.reset()
|
||||
}
|
||||
}
|
||||
`)
|
||||
let child: ReturnType<typeof Bun.spawn> | undefined
|
||||
try {
|
||||
const build = await Bun.build({
|
||||
entrypoints: [entrypoint],
|
||||
minify: { whitespace: true, identifiers: true, syntax: true },
|
||||
sourcemap: 'none',
|
||||
target: 'bun',
|
||||
compile: { outfile: executable },
|
||||
})
|
||||
expect(build.success).toBe(true)
|
||||
// Match the sidecar build's required signature repair using ad-hoc
|
||||
// signing only. No developer certificate or keychain is consulted.
|
||||
for (const args of [
|
||||
['--remove-signature', executable],
|
||||
['--sign', '-', '--force', '--timestamp=none', executable],
|
||||
]) {
|
||||
const signing = Bun.spawn(['/usr/bin/codesign', ...args], { stdout: 'ignore', stderr: 'pipe' })
|
||||
expect(await signing.exited).toBe(0)
|
||||
}
|
||||
child = Bun.spawn([executable], {
|
||||
cwd: directory,
|
||||
env: { HOME: directory, TMPDIR: directory, CLAUDE_CONFIG_DIR: join(directory, '.claude'), PATH: '/usr/bin:/bin' },
|
||||
stdin: 'ignore',
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
})
|
||||
const [stdout, stderr, exitCode] = await Promise.all([
|
||||
new Response(child.stdout).text(),
|
||||
new Response(child.stderr).text(),
|
||||
child.exited,
|
||||
])
|
||||
expect({ exitCode, stderr }).toEqual({ exitCode: 0, stderr: '' })
|
||||
const result = JSON.parse(stdout)
|
||||
expect(result.embedded).toBe(0)
|
||||
const errors = [result.first, result.second].filter(item => item.isError)
|
||||
expect(errors).toEqual([])
|
||||
expect(result.second.content).toContainEqual({ type: 'text', text: 'done=3' })
|
||||
expect(result.calls.map((call: { name: string }) => call.name)).toEqual(['get_app_state', 'click', 'click', 'click'])
|
||||
} finally {
|
||||
child?.kill()
|
||||
if (child) {
|
||||
await child.exited
|
||||
}
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
}, 30_000)
|
||||
})
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { compileReplCell } from './replCompiler'
|
||||
|
||||
describe('computer use REPL compiler', () => {
|
||||
test('collects persistent declarations without capturing nested lexical scopes', () => {
|
||||
const compiled = compileReplCell(`
|
||||
const { x, nested: { y = 2 }, ...rest } = source
|
||||
let [first, ...tail] = values
|
||||
async function act() { const local = 1; return local }
|
||||
class Target {}
|
||||
for (var index of [1, 2]) { var visited = index; let scoped = 3 }
|
||||
{ let hidden = 4; function nested() {} }
|
||||
`, [])
|
||||
expect(compiled.bindings).toEqual([
|
||||
{ name: 'x', kind: 'const' },
|
||||
{ name: 'y', kind: 'const' },
|
||||
{ name: 'rest', kind: 'const' },
|
||||
{ name: 'first', kind: 'let' },
|
||||
{ name: 'tail', kind: 'let' },
|
||||
{ name: 'act', kind: 'function' },
|
||||
{ name: 'Target', kind: 'class' },
|
||||
{ name: 'index', kind: 'var' },
|
||||
{ name: 'visited', kind: 'var' },
|
||||
])
|
||||
})
|
||||
|
||||
test('accepts top level await and redeclarations of prior bindings', () => {
|
||||
const compiled = compileReplCell('const app = await select(); let next = app', [
|
||||
{ name: 'app', kind: 'const' },
|
||||
{ name: 'previous', kind: 'let' },
|
||||
])
|
||||
expect(compiled.bindings).toEqual([
|
||||
{ name: 'previous', kind: 'let' },
|
||||
{ name: 'app', kind: 'const' },
|
||||
{ name: 'next', kind: 'let' },
|
||||
])
|
||||
})
|
||||
|
||||
test.each([
|
||||
'import fs from "node:fs"',
|
||||
'await import("node:fs")',
|
||||
'async function later() { return import("node:fs") }',
|
||||
'export const x = 1',
|
||||
'import.meta.url',
|
||||
])('rejects module access: %s', code => {
|
||||
expect(() => compileReplCell(code, [])).toThrow('not available')
|
||||
})
|
||||
|
||||
test('rejects syntax errors before any execution', () => {
|
||||
expect(() => compileReplCell('await action(); const broken =', [])).toThrow()
|
||||
})
|
||||
|
||||
test('warns for writes to previous const bindings without confusing local shadows or object properties', () => {
|
||||
const prior = [{ name: 'count', kind: 'const' as const }]
|
||||
expect(compileReplCell('count++', prior).warnings).toHaveLength(1)
|
||||
expect(compileReplCell('({value: count} = source)', prior).warnings).toHaveLength(1)
|
||||
expect(compileReplCell('function increment() { count++ }', prior).warnings).toHaveLength(1)
|
||||
for (const code of [
|
||||
'count.value++',
|
||||
'function local(count) { count++ }',
|
||||
'function local() { count++; var count = 0 }',
|
||||
'{ let count = 0; count++ }',
|
||||
'for (let count = 0; count < 2; count++) {}',
|
||||
'try {} catch (count) { count++ }',
|
||||
]) {
|
||||
expect(compileReplCell(code, prior).warnings).toEqual([])
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,442 @@
|
||||
import { parse, type Node } from 'acorn'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
|
||||
export interface ReplBinding {
|
||||
name: string
|
||||
kind: 'const' | 'let' | 'var' | 'function' | 'class'
|
||||
}
|
||||
|
||||
interface SyntaxNode extends Node {
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
function syntaxNode(value: unknown): value is SyntaxNode {
|
||||
return !!value && typeof value === 'object' && typeof (value as Node).type === 'string'
|
||||
}
|
||||
|
||||
function children(node: SyntaxNode): SyntaxNode[] {
|
||||
return Object.values(node).flatMap(value => {
|
||||
if (syntaxNode(value)) {
|
||||
return [value]
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.filter(syntaxNode)
|
||||
}
|
||||
return []
|
||||
})
|
||||
}
|
||||
|
||||
function bindingNames(pattern: SyntaxNode): string[] {
|
||||
switch (pattern.type) {
|
||||
case 'Identifier':
|
||||
return [pattern.name as string]
|
||||
case 'RestElement':
|
||||
return bindingNames(pattern.argument as SyntaxNode)
|
||||
case 'AssignmentPattern':
|
||||
return bindingNames(pattern.left as SyntaxNode)
|
||||
case 'ArrayPattern':
|
||||
return (pattern.elements as unknown[]).filter(syntaxNode).flatMap(bindingNames)
|
||||
case 'ObjectPattern':
|
||||
return (pattern.properties as SyntaxNode[]).flatMap(property =>
|
||||
bindingNames((property.type === 'RestElement' ? property.argument : property.value) as SyntaxNode),
|
||||
)
|
||||
default:
|
||||
throw new Error(`Unsupported REPL binding pattern: ${pattern.type}`)
|
||||
}
|
||||
}
|
||||
|
||||
function assignmentNames(target: SyntaxNode): string[] {
|
||||
if (target.type === 'MemberExpression') {
|
||||
return []
|
||||
}
|
||||
if (target.type === 'ObjectPattern') {
|
||||
return (target.properties as SyntaxNode[]).flatMap(property =>
|
||||
assignmentNames((property.type === 'RestElement' ? property.argument : property.value) as SyntaxNode),
|
||||
)
|
||||
}
|
||||
if (target.type === 'ArrayPattern') {
|
||||
return (target.elements as unknown[]).filter(syntaxNode).flatMap(assignmentNames)
|
||||
}
|
||||
if (target.type === 'AssignmentPattern') {
|
||||
return assignmentNames(target.left as SyntaxNode)
|
||||
}
|
||||
if (target.type === 'RestElement') {
|
||||
return assignmentNames(target.argument as SyntaxNode)
|
||||
}
|
||||
return target.type === 'Identifier' ? [target.name as string] : []
|
||||
}
|
||||
|
||||
function reassignedPriorConstants(ast: SyntaxNode, names: Set<string>) {
|
||||
const assigned = new Set<string>()
|
||||
function visit(node: SyntaxNode, outer: Set<string>) {
|
||||
const shadowed = new Set(outer)
|
||||
const addPattern = (pattern: SyntaxNode) => {
|
||||
for (const name of bindingNames(pattern)) {
|
||||
shadowed.add(name)
|
||||
}
|
||||
}
|
||||
if (node.type === 'BlockStatement') {
|
||||
for (const statement of node.body as SyntaxNode[]) {
|
||||
if (statement.type === 'VariableDeclaration' && statement.kind !== 'var') {
|
||||
for (const declaration of statement.declarations as SyntaxNode[]) {
|
||||
addPattern(declaration.id as SyntaxNode)
|
||||
}
|
||||
} else if (statement.type === 'FunctionDeclaration' || statement.type === 'ClassDeclaration') {
|
||||
addPattern(statement.id as SyntaxNode)
|
||||
}
|
||||
}
|
||||
}
|
||||
if (/Function/.test(node.type)) {
|
||||
if (syntaxNode(node.id)) {
|
||||
addPattern(node.id)
|
||||
}
|
||||
for (const parameter of node.params as SyntaxNode[]) {
|
||||
addPattern(parameter)
|
||||
}
|
||||
const collectVars = (body: SyntaxNode) => {
|
||||
if (/Function/.test(body.type) || body.type === 'StaticBlock') {
|
||||
return
|
||||
}
|
||||
if (body.type === 'VariableDeclaration' && body.kind === 'var') {
|
||||
for (const declaration of body.declarations as SyntaxNode[]) {
|
||||
addPattern(declaration.id as SyntaxNode)
|
||||
}
|
||||
}
|
||||
for (const child of children(body)) {
|
||||
collectVars(child)
|
||||
}
|
||||
}
|
||||
collectVars(node.body as SyntaxNode)
|
||||
}
|
||||
if (node.type === 'CatchClause' && syntaxNode(node.param)) {
|
||||
addPattern(node.param)
|
||||
}
|
||||
if (node.type.startsWith('For')) {
|
||||
const declaration = node.init ?? node.left
|
||||
if (syntaxNode(declaration) && declaration.type === 'VariableDeclaration' && declaration.kind !== 'var') {
|
||||
for (const item of declaration.declarations as SyntaxNode[]) {
|
||||
addPattern(item.id as SyntaxNode)
|
||||
}
|
||||
}
|
||||
}
|
||||
let target: unknown
|
||||
if (node.type === 'AssignmentExpression') {
|
||||
target = node.left
|
||||
}
|
||||
if (node.type === 'UpdateExpression') {
|
||||
target = node.argument
|
||||
}
|
||||
if ((node.type === 'ForOfStatement' || node.type === 'ForInStatement') && (node.left as SyntaxNode).type !== 'VariableDeclaration') {
|
||||
target = node.left
|
||||
}
|
||||
if (syntaxNode(target)) {
|
||||
for (const name of assignmentNames(target)) {
|
||||
if (names.has(name) && !shadowed.has(name)) {
|
||||
assigned.add(name)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const child of children(node)) {
|
||||
visit(child, shadowed)
|
||||
}
|
||||
}
|
||||
visit(ast, new Set())
|
||||
return assigned
|
||||
}
|
||||
|
||||
interface SourceEdit { start: number; end: number; text: string }
|
||||
|
||||
/** Rewrite free references through the persistent program scope. Native
|
||||
* declarations remain intact, including destructuring initialization and TDZ.
|
||||
* A realm-local accessor resolves the same lexical slot for old closures and
|
||||
* new cells. Local scopes must never be rewritten into that shared namespace.
|
||||
*/
|
||||
function persistentReferences(ast: SyntaxNode, scope: string): SourceEdit[] {
|
||||
const edits: SourceEdit[] = []
|
||||
const addPattern = (target: Set<string>, pattern: SyntaxNode) => {
|
||||
for (const name of bindingNames(pattern)) target.add(name)
|
||||
}
|
||||
const blockNames = (body: SyntaxNode[], outer: Set<string>) => {
|
||||
const result = new Set(outer)
|
||||
for (const node of body) {
|
||||
if (node.type === 'VariableDeclaration' && node.kind !== 'var') {
|
||||
for (const declaration of node.declarations as SyntaxNode[]) addPattern(result, declaration.id as SyntaxNode)
|
||||
} else if (node.type === 'FunctionDeclaration' || node.type === 'ClassDeclaration') {
|
||||
addPattern(result, node.id as SyntaxNode)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
function functionVars(node: SyntaxNode, target: Set<string>) {
|
||||
if (/Function/.test(node.type) || node.type === 'StaticBlock') return
|
||||
if (node.type === 'VariableDeclaration' && node.kind === 'var') {
|
||||
for (const declaration of node.declarations as SyntaxNode[]) addPattern(target, declaration.id as SyntaxNode)
|
||||
}
|
||||
for (const child of children(node)) functionVars(child, target)
|
||||
}
|
||||
function patternExpressions(pattern: SyntaxNode, shadowed: Set<string>) {
|
||||
if (pattern.type === 'AssignmentPattern') {
|
||||
patternExpressions(pattern.left as SyntaxNode, shadowed)
|
||||
visit(pattern.right as SyntaxNode, shadowed)
|
||||
} else if (pattern.type === 'ObjectPattern') {
|
||||
for (const property of pattern.properties as SyntaxNode[]) {
|
||||
if (property.type === 'RestElement') patternExpressions(property.argument as SyntaxNode, shadowed)
|
||||
else {
|
||||
if (property.computed) visit(property.key as SyntaxNode, shadowed)
|
||||
patternExpressions(property.value as SyntaxNode, shadowed)
|
||||
}
|
||||
}
|
||||
} else if (pattern.type === 'ArrayPattern') {
|
||||
for (const element of (pattern.elements as unknown[]).filter(syntaxNode)) patternExpressions(element, shadowed)
|
||||
} else if (pattern.type === 'RestElement') patternExpressions(pattern.argument as SyntaxNode, shadowed)
|
||||
}
|
||||
function visit(node: SyntaxNode, outer: Set<string>, parent?: SyntaxNode, key?: string, shorthand?: string) {
|
||||
if (node.type === 'UnaryExpression' && node.operator === 'typeof' && (node.argument as SyntaxNode).type === 'Identifier') {
|
||||
const name = (node.argument as SyntaxNode).name as string
|
||||
if (!outer.has(name)) {
|
||||
edits.push({ start: node.start, end: node.end, text: `${scope}.typeOf(${JSON.stringify(name)})` })
|
||||
return
|
||||
}
|
||||
}
|
||||
if (node.type === 'Identifier') {
|
||||
const name = node.name as string
|
||||
if (outer.has(name)) return
|
||||
let text = `${scope}.values[${JSON.stringify(name)}]`
|
||||
// Preserve an ordinary lexical call's undefined receiver. A member call
|
||||
// would otherwise expose the private accessor object as `this`.
|
||||
if ((parent?.type === 'CallExpression' && key === 'callee') || (parent?.type === 'TaggedTemplateExpression' && key === 'tag')) text = `(0, ${text})`
|
||||
if (shorthand) text = `${shorthand}: ${text}`
|
||||
edits.push({ start: node.start, end: node.end, text })
|
||||
return
|
||||
}
|
||||
if (/Function/.test(node.type)) {
|
||||
const parameters = new Set(outer)
|
||||
if (node.type !== 'ArrowFunctionExpression') parameters.add('arguments')
|
||||
if (node.type === 'FunctionExpression' && syntaxNode(node.id)) addPattern(parameters, node.id)
|
||||
for (const parameter of node.params as SyntaxNode[]) addPattern(parameters, parameter)
|
||||
for (const parameter of node.params as SyntaxNode[]) patternExpressions(parameter, parameters)
|
||||
const bodyScope = new Set(parameters)
|
||||
functionVars(node.body as SyntaxNode, bodyScope)
|
||||
visit(node.body as SyntaxNode, bodyScope)
|
||||
return
|
||||
}
|
||||
if (node.type === 'VariableDeclaration') {
|
||||
for (const declaration of node.declarations as SyntaxNode[]) {
|
||||
patternExpressions(declaration.id as SyntaxNode, outer)
|
||||
if (syntaxNode(declaration.init)) visit(declaration.init, outer)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (node.type === 'ClassDeclaration' || node.type === 'ClassExpression') {
|
||||
const classScope = new Set(outer)
|
||||
if (syntaxNode(node.id)) addPattern(classScope, node.id)
|
||||
if (syntaxNode(node.superClass)) visit(node.superClass, classScope)
|
||||
visit(node.body as SyntaxNode, classScope)
|
||||
return
|
||||
}
|
||||
if (node.type === 'BlockStatement' || node.type === 'StaticBlock') {
|
||||
const scope = blockNames(node.body as SyntaxNode[], outer)
|
||||
if (node.type === 'StaticBlock') for (const item of node.body as SyntaxNode[]) functionVars(item, scope)
|
||||
for (const statement of node.body as SyntaxNode[]) visit(statement, scope)
|
||||
return
|
||||
}
|
||||
if (node.type === 'CatchClause') {
|
||||
const scope = new Set(outer)
|
||||
if (syntaxNode(node.param)) {
|
||||
addPattern(scope, node.param)
|
||||
patternExpressions(node.param, scope)
|
||||
}
|
||||
visit(node.body as SyntaxNode, scope)
|
||||
return
|
||||
}
|
||||
let shadowed = outer
|
||||
if (node.type.startsWith('For')) {
|
||||
const declaration = node.init ?? node.left
|
||||
if (syntaxNode(declaration) && declaration.type === 'VariableDeclaration' && declaration.kind !== 'var') {
|
||||
shadowed = new Set(outer)
|
||||
for (const item of declaration.declarations as SyntaxNode[]) addPattern(shadowed, item.id as SyntaxNode)
|
||||
}
|
||||
}
|
||||
if (node.type === 'SwitchStatement') {
|
||||
visit(node.discriminant as SyntaxNode, outer)
|
||||
const cases = node.cases as SyntaxNode[]
|
||||
const scope = blockNames(cases.flatMap(item => item.consequent as SyntaxNode[]), outer)
|
||||
for (const item of cases) visit(item, scope)
|
||||
return
|
||||
}
|
||||
if (node.type === 'Property') {
|
||||
if (node.computed) visit(node.key as SyntaxNode, shadowed)
|
||||
const value = node.value as SyntaxNode
|
||||
if (node.shorthand) {
|
||||
const name = (node.key as SyntaxNode).name as string
|
||||
if (value.type === 'AssignmentPattern') {
|
||||
visit(value.left as SyntaxNode, shadowed, value, 'left', name)
|
||||
visit(value.right as SyntaxNode, shadowed, value, 'right')
|
||||
} else visit(value, shadowed, node, 'value', name)
|
||||
} else visit(value, shadowed, node, 'value')
|
||||
return
|
||||
}
|
||||
for (const [childKey, value] of Object.entries(node)) {
|
||||
if ((node.type === 'MemberExpression' && childKey === 'property' && !node.computed)
|
||||
|| ((node.type === 'MethodDefinition' || node.type === 'PropertyDefinition') && childKey === 'key' && !node.computed)
|
||||
|| ((node.type === 'LabeledStatement' || node.type === 'BreakStatement' || node.type === 'ContinueStatement') && childKey === 'label')) continue
|
||||
if (syntaxNode(value)) visit(value, shadowed, node, childKey)
|
||||
else if (Array.isArray(value)) for (const child of value.filter(syntaxNode)) visit(child, shadowed, node, childKey)
|
||||
}
|
||||
}
|
||||
visit(ast, new Set())
|
||||
return edits
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile a cell into an async function while retaining real lexical bindings.
|
||||
* Native lexical accessors are shared with later cells and prior closures;
|
||||
* new declarations may replace prior names, as in a notebook. Accessors registered
|
||||
* before execution let the worker retain initialized bindings after an error.
|
||||
* This is our own small AST transform, not the official Node REPL compiler.
|
||||
*/
|
||||
export function compileReplCell(code: string, prior: readonly ReplBinding[]) {
|
||||
const ast = parse(code, { ecmaVersion: 'latest', sourceType: 'module' }) as unknown as SyntaxNode
|
||||
const current = new Map<string, ReplBinding>()
|
||||
const identifiers = new Set<string>()
|
||||
const hoistedDeclarations: Array<{ node: SyntaxNode; parent?: SyntaxNode }> = []
|
||||
|
||||
function inspect(node: SyntaxNode, depth: number, inFunctionScope: boolean, parent?: SyntaxNode) {
|
||||
if (node.type === 'Identifier') {
|
||||
identifiers.add(node.name as string)
|
||||
}
|
||||
const importMeta = node.type === 'MetaProperty' && (node.meta as SyntaxNode).name === 'import'
|
||||
if (node.type.startsWith('Import') || node.type.startsWith('Export') || importMeta) {
|
||||
throw new Error('Module imports and exports are not available in Computer Use JavaScript')
|
||||
}
|
||||
if (node.type === 'VariableDeclaration' && !inFunctionScope && (depth === 1 || node.kind === 'var')) {
|
||||
for (const declaration of node.declarations as SyntaxNode[]) {
|
||||
for (const name of bindingNames(declaration.id as SyntaxNode)) {
|
||||
current.set(name, { name, kind: node.kind as ReplBinding['kind'] })
|
||||
}
|
||||
}
|
||||
if (node.kind === 'var') {
|
||||
hoistedDeclarations.push({ node, parent })
|
||||
}
|
||||
}
|
||||
if (depth === 1 && (node.type === 'FunctionDeclaration' || node.type === 'ClassDeclaration')) {
|
||||
const name = (node.id as SyntaxNode).name as string
|
||||
current.set(name, { name, kind: node.type === 'FunctionDeclaration' ? 'function' : 'class' })
|
||||
if (node.type === 'FunctionDeclaration') {
|
||||
hoistedDeclarations.push({ node, parent })
|
||||
}
|
||||
}
|
||||
const nestedScope = inFunctionScope || /Function/.test(node.type) || node.type === 'StaticBlock'
|
||||
for (const child of children(node)) {
|
||||
inspect(child, depth + 1, nestedScope, node)
|
||||
}
|
||||
}
|
||||
inspect(ast, 0, false)
|
||||
|
||||
const carried = prior.filter(binding => !current.has(binding.name))
|
||||
const priorConstants = new Set(carried.filter(binding => binding.kind === 'const').map(binding => binding.name))
|
||||
const reassigned = reassignedPriorConstants(ast, priorConstants)
|
||||
const warnedNames = new Set(reassigned)
|
||||
for (const binding of current.values()) {
|
||||
if (binding.kind === 'const' && prior.some(previous => previous.name === binding.name && previous.kind === 'const')) {
|
||||
warnedNames.add(binding.name)
|
||||
}
|
||||
}
|
||||
const bindings = [...carried, ...current.values()]
|
||||
let prefix: string
|
||||
do {
|
||||
prefix = `__cu_cell_${randomUUID().replaceAll('-', '_')}`
|
||||
} while ([`${prefix}_scopeFactory`, `${prefix}_scope`, `${prefix}_register`, `${prefix}_mark`, `${prefix}_value`].some(name => identifiers.has(name)))
|
||||
const scopeFactory = `${prefix}_scopeFactory`
|
||||
const scope = `${prefix}_scope`
|
||||
const register = `${prefix}_register`
|
||||
const mark = `${prefix}_mark`
|
||||
const value = `${prefix}_value`
|
||||
const registrations = [...current.values()].map(binding =>
|
||||
`[${JSON.stringify(binding.name)}, ${JSON.stringify(binding.kind)}, () => ${binding.name}, ${value} => { ${binding.name} = ${value} }, ${binding.kind === 'var' || binding.kind === 'function'}]`,
|
||||
).join(',\n')
|
||||
|
||||
// Hoisted values exist before their declaration executes. In a failed cell,
|
||||
// only reached declarations should replace saved bindings. Insert markers
|
||||
// without rewriting user initializers, destructuring or nested function code.
|
||||
const insertions: Array<{ position: number; text: string }> = []
|
||||
let markerCounter = 0
|
||||
for (const { node, parent } of hoistedDeclarations) {
|
||||
if (node.type === 'FunctionDeclaration') {
|
||||
insertions.push({ position: node.end, text: `;${mark}(${JSON.stringify((node.id as SyntaxNode).name)});` })
|
||||
continue
|
||||
}
|
||||
const iteration = parent && (parent.type === 'ForOfStatement' || parent.type === 'ForInStatement') && parent.left === node
|
||||
if (iteration) {
|
||||
const names = (node.declarations as SyntaxNode[]).flatMap(declaration => bindingNames(declaration.id as SyntaxNode))
|
||||
const marker = `${mark}(${names.map(name => JSON.stringify(name)).join(',')});`
|
||||
const body = parent.body as SyntaxNode
|
||||
if (body.type === 'BlockStatement') {
|
||||
insertions.push({ position: body.start + 1, text: marker })
|
||||
} else {
|
||||
insertions.push({ position: body.start, text: `{${marker}` })
|
||||
insertions.push({ position: body.end, text: '}' })
|
||||
}
|
||||
continue
|
||||
}
|
||||
for (const declaration of node.declarations as SyntaxNode[]) {
|
||||
const names = bindingNames(declaration.id as SyntaxNode)
|
||||
insertions.push({
|
||||
position: declaration.end,
|
||||
text: `, ${prefix}_reached_${markerCounter++} = ${mark}(${names.map(name => JSON.stringify(name)).join(',')})`,
|
||||
})
|
||||
}
|
||||
}
|
||||
// Direct writes can initialize a hoisted var before its declaration site.
|
||||
// Keep those writes on failure too. A logical assignment marks only when
|
||||
// its RHS actually evaluates, preserving short-circuit behavior.
|
||||
const futureVars = new Map<string, number>()
|
||||
for (const { node } of hoistedDeclarations) {
|
||||
if (node.type !== 'VariableDeclaration') {
|
||||
continue
|
||||
}
|
||||
for (const declaration of node.declarations as SyntaxNode[]) {
|
||||
for (const name of bindingNames(declaration.id as SyntaxNode)) {
|
||||
futureVars.set(name, Math.min(futureVars.get(name) ?? Infinity, declaration.start))
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const statement of ast.body as SyntaxNode[]) {
|
||||
if (statement.type !== 'ExpressionStatement') {
|
||||
continue
|
||||
}
|
||||
const expression = statement.expression as SyntaxNode
|
||||
const target = expression.type === 'AssignmentExpression' ? expression.left
|
||||
: expression.type === 'UpdateExpression' ? expression.argument : undefined
|
||||
if (!syntaxNode(target)) {
|
||||
continue
|
||||
}
|
||||
const names = assignmentNames(target).filter(name => (futureVars.get(name) ?? -1) > expression.start)
|
||||
if (names.length === 0) {
|
||||
continue
|
||||
}
|
||||
const marker = `${mark}(${names.map(name => JSON.stringify(name)).join(',')})`
|
||||
if (['&&=', '||=', '??='].includes(expression.operator as string)) {
|
||||
const right = expression.right as SyntaxNode
|
||||
const value = `${prefix}_value`
|
||||
insertions.push({ position: right.start, text: `((${value}) => (${marker}, ${value}))(` })
|
||||
insertions.push({ position: right.end, text: ')' })
|
||||
} else {
|
||||
insertions.push({ position: statement.end, text: `;${marker};` })
|
||||
}
|
||||
}
|
||||
let instrumented = code
|
||||
const edits = [
|
||||
...persistentReferences(ast, scope),
|
||||
...insertions.map(({ position, text }) => ({ start: position, end: position, text })),
|
||||
]
|
||||
for (const edit of edits.sort((left, right) => right.start - left.start || right.end - left.end)) {
|
||||
instrumented = instrumented.slice(0, edit.start) + edit.text + instrumented.slice(edit.end)
|
||||
}
|
||||
|
||||
return {
|
||||
bindings,
|
||||
warnings: [...warnedNames].map(name => `${name} was declared with const; use let for reassignable variables.`),
|
||||
source: `(async (${scopeFactory}, ${register}, ${mark}) => {\n"use strict";\nconst ${scope} = ${scopeFactory}(${JSON.stringify([...reassigned])});\n${register}([${registrations}]);\n${instrumented}\n})`,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test'
|
||||
import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
import { ComputerUseRepl } from './replRuntime.js'
|
||||
|
||||
const runtimes: ComputerUseRepl[] = []
|
||||
afterEach(async () => {
|
||||
await Promise.all(runtimes.splice(0).map(runtime => runtime.reset()))
|
||||
})
|
||||
|
||||
const suite = process.platform === 'darwin' ? describe : describe.skip
|
||||
test('unsupported platforms fail closed without starting an unsandboxed worker', async () => {
|
||||
const runtime = new ComputerUseRepl()
|
||||
runtimes.push(runtime)
|
||||
const result = await runtime.run({ code: '1', timeoutMs: process.platform === 'darwin' ? 0 : 1000 }, async () => {
|
||||
throw new Error('must not dispatch')
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.content[0]).toMatchObject({ type: 'text', text: expect.stringMatching(process.platform === 'darwin' ? /Invalid/ : /macOS only/) })
|
||||
})
|
||||
suite('Computer Use isolated persistent runtime', () => {
|
||||
function createRuntime() {
|
||||
const runtime = new ComputerUseRepl()
|
||||
runtimes.push(runtime)
|
||||
return runtime
|
||||
}
|
||||
|
||||
test('a known pre-dispatch rejection is neither a completed action nor an unknown result', async () => {
|
||||
const runtime = createRuntime()
|
||||
const result = await runtime.run({ code: 'await cua.getApp("Fixture")', timeoutMs: 5000 }, async () => ({
|
||||
isError: true, nativeCallNotDispatched: true,
|
||||
content: [{ type: 'text', text: 'permissions required' }],
|
||||
}))
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.structuredContent).toMatchObject({
|
||||
nativeCallsStarted: 1, nativeCallsCompleted: 0, nativeCallsRejectedBeforeDispatch: 1, nativeResultUnknown: false,
|
||||
})
|
||||
expect(result.structuredContent?.recovery).toContain('No native actions were dispatched')
|
||||
})
|
||||
|
||||
test('keeps App bindings and performs sequential loops without model round trips', async () => {
|
||||
const runtime = createRuntime()
|
||||
const calls: string[] = []
|
||||
const invoke = async (name: string) => {
|
||||
calls.push(name)
|
||||
return { content: [{ type: 'text' as const, text: 'App=Fixture\n[g1:1] button Test' }] }
|
||||
}
|
||||
const first = await runtime.run({ code: 'let app = await cua.getApp("Fixture")', timeoutMs: 5000 }, invoke)
|
||||
expect(first.isError).not.toBe(true)
|
||||
const second = await runtime.run({ code: 'for (let i = 0; i < 3; i++) await app.click([10, 10]); nodeRepl.write("done")', timeoutMs: 5000 }, invoke)
|
||||
expect(second.isError).not.toBe(true)
|
||||
expect(second.content).toContainEqual({ type: 'text', text: 'done' })
|
||||
expect(calls).toEqual(['get_app_state', 'click', 'click', 'click'])
|
||||
})
|
||||
|
||||
test('a reused worker invokes each cell in its current caller context across awaits', async () => {
|
||||
const contexts = new AsyncLocalStorage<{ turn: string; displayId: number; aborted: boolean }>()
|
||||
const observed: Array<{ method: string; turn: string | undefined; displayId: number | undefined; afterAwait: string | undefined }> = []
|
||||
const runtime = createRuntime()
|
||||
const firstContext = { turn: 'first-turn', displayId: 1, aborted: false }
|
||||
const secondContext = { turn: 'second-turn', displayId: 2, aborted: false }
|
||||
const invoke = async (method: string) => {
|
||||
const context = contexts.getStore()
|
||||
await Promise.resolve()
|
||||
observed.push({ method, turn: context?.turn, displayId: context?.displayId, afterAwait: contexts.getStore()?.turn })
|
||||
if (context?.aborted) {
|
||||
return { isError: true, nativeCallNotDispatched: true, content: [{ type: 'text' as const, text: 'The caller turn has ended' }] }
|
||||
}
|
||||
return { content: [{ type: 'text' as const, text: 'App=Fixture\n[g1:1] button Test' }] }
|
||||
}
|
||||
|
||||
const first = await contexts.run(firstContext, () => runtime.run({ code: 'let app = await cua.getApp("Fixture")', timeoutMs: 5000 }, invoke))
|
||||
expect(first.isError).not.toBe(true)
|
||||
// The warmed worker's stdout listener belongs to this completed turn.
|
||||
// Its next native callback must use the new turn's state and cancellation.
|
||||
firstContext.aborted = true
|
||||
const second = await contexts.run(secondContext, () => runtime.run({ code: 'await app.click([10,10])', timeoutMs: 5000 }, invoke))
|
||||
|
||||
expect(second.isError).not.toBe(true)
|
||||
expect(observed).toEqual([
|
||||
{ method: 'get_app_state', turn: 'first-turn', displayId: 1, afterAwait: 'first-turn' },
|
||||
{ method: 'click', turn: 'second-turn', displayId: 2, afterAwait: 'second-turn' },
|
||||
])
|
||||
})
|
||||
|
||||
test('terminates CPU loops, discards bindings, and starts a fresh kernel', async () => {
|
||||
const runtime = createRuntime()
|
||||
const invoke = async () => ({ content: [] })
|
||||
await runtime.run({ code: 'let saved = 42', timeoutMs: 5000 }, invoke)
|
||||
const started = Date.now()
|
||||
const result = await runtime.run({ code: 'while (true) {}', timeoutMs: 150 }, invoke)
|
||||
expect(result.isError).toBe(true)
|
||||
expect(Date.now() - started).toBeLessThan(3000)
|
||||
const next = await runtime.run({ code: 'nodeRepl.write(typeof saved)', timeoutMs: 5000 }, invoke)
|
||||
expect(next.content).toContainEqual({ type: 'text', text: 'undefined' })
|
||||
})
|
||||
|
||||
test('cancellation drains an already dispatched action and stops later actions', async () => {
|
||||
const runtime = createRuntime()
|
||||
const abort = new AbortController()
|
||||
let release!: () => void
|
||||
const actionPending = new Promise<void>(resolve => { release = resolve })
|
||||
let started!: () => void
|
||||
const actionStarted = new Promise<void>(resolve => { started = resolve })
|
||||
const calls: string[] = []
|
||||
const result = runtime.run({
|
||||
code: 'let app = await cua.getApp("Fixture"); await app.click([1,1]); await app.click([2,2])',
|
||||
timeoutMs: 5000,
|
||||
signal: abort.signal,
|
||||
}, async name => {
|
||||
calls.push(name)
|
||||
if (name === 'click') {
|
||||
started()
|
||||
await actionPending
|
||||
}
|
||||
return { content: [{ type: 'text', text: 'state' }] }
|
||||
})
|
||||
await actionStarted
|
||||
abort.abort()
|
||||
let finished = false
|
||||
void result.then(() => { finished = true })
|
||||
await new Promise(resolve => setTimeout(resolve, 30))
|
||||
expect(finished).toBe(false)
|
||||
release()
|
||||
const cancelled = await result
|
||||
expect(cancelled.isError).toBe(true)
|
||||
expect(cancelled.structuredContent).toMatchObject({ nativeCallsStarted: 2, nativeCallsCompleted: 2, nativeResultUnknown: false, bindingsReset: true })
|
||||
expect(calls).toEqual(['get_app_state', 'click'])
|
||||
})
|
||||
|
||||
test('a detached microtask loop cannot report success and is terminated by the parent', async () => {
|
||||
const runtime = createRuntime()
|
||||
const invoke = async () => ({ content: [] })
|
||||
const result = await runtime.run({
|
||||
code: 'let spin; spin = () => { Promise.resolve().then(spin) }; Promise.resolve().then(spin)',
|
||||
timeoutMs: 150,
|
||||
}, invoke)
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.content).toEqual([{ type: 'text', text: expect.stringContaining('timed out') }])
|
||||
const fresh = await runtime.run({ code: 'nodeRepl.write(typeof spin)', timeoutMs: 5000 }, invoke)
|
||||
expect(fresh.content).toEqual([{ type: 'text', text: 'undefined' }])
|
||||
})
|
||||
|
||||
test('idle health checks retain a responsive kernel and its variables', async () => {
|
||||
const runtime = new ComputerUseRepl({ maxRssBytes: 768 * 1024 * 1024, healthCheckMs: 50 })
|
||||
runtimes.push(runtime)
|
||||
const invoke = async () => ({ content: [] })
|
||||
expect((await runtime.run({ code: 'let retained = 7', timeoutMs: 5000 }, invoke)).isError).not.toBe(true)
|
||||
await new Promise(resolve => setTimeout(resolve, 2200))
|
||||
const result = await runtime.run({ code: 'nodeRepl.write(retained)', timeoutMs: 5000 }, invoke)
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(result.content).toEqual([{ type: 'text', text: '7' }])
|
||||
})
|
||||
|
||||
test('the parent memory watchdog terminates a child independently of its VM', async () => {
|
||||
// A tiny injected budget tests real RSS measurement without allocating
|
||||
// hundreds of megabytes on the developer's machine.
|
||||
const runtime = new ComputerUseRepl({ maxRssBytes: 1, healthCheckMs: 50 })
|
||||
runtimes.push(runtime)
|
||||
const result = await runtime.run({ code: 'await new Promise(() => {})', timeoutMs: 5000 }, async () => ({ content: [] }))
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.content).toEqual([{ type: 'text', text: expect.stringContaining('memory budget') }])
|
||||
})
|
||||
|
||||
test('an ignored failing App action cannot be reported as successful execution', async () => {
|
||||
const runtime = createRuntime()
|
||||
const result = await runtime.run({
|
||||
code: 'let app = await cua.getApp("Fixture"); app.click([1,1]); await app.getAXState()',
|
||||
timeoutMs: 5000,
|
||||
}, async name => name === 'click'
|
||||
? { isError: true, content: [{ type: 'text', text: 'fixture click failed' }] }
|
||||
: { content: [{ type: 'text', text: 'fixture state' }] })
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.structuredContent).toMatchObject({ nativeResultUnknown: true })
|
||||
})
|
||||
|
||||
test('explicitly handled App failures can observe the outcome and continue', async () => {
|
||||
const runtime = createRuntime()
|
||||
const result = await runtime.run({
|
||||
code: 'let app = await cua.getApp("Fixture"); try { await app.click([1,1]) } catch(error) { nodeRepl.write(error.message) }; await app.getAXState()',
|
||||
timeoutMs: 5000,
|
||||
}, async name => name === 'click'
|
||||
? { isError: true, content: [{ type: 'text', text: 'fixture click failed' }] }
|
||||
: { content: [{ type: 'text', text: 'fixture state' }] })
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(result.content).toContainEqual({ type: 'text', text: 'fixture click failed' })
|
||||
expect(result.content.at(-1)).toEqual({ type: 'text', text: 'fixture state' })
|
||||
})
|
||||
|
||||
test('invalid bridge operations and output floods stop the kernel without native calls', async () => {
|
||||
for (const code of [
|
||||
'await __cuInvoke("js", {code:"1"})',
|
||||
'__cuEmit({type:"image",data:"AA==",mimeType:"text/html"})',
|
||||
'for(let i=0;i<129;i++) nodeRepl.write(i)',
|
||||
]) {
|
||||
const runtime = createRuntime()
|
||||
let calls = 0
|
||||
const result = await runtime.run({ code, timeoutMs: 5000 }, async () => { ++calls; return { content: [] } })
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.structuredContent).toMatchObject({ bindingsReset: true, nativeCallsStarted: 0 })
|
||||
expect(calls).toBe(0)
|
||||
}
|
||||
})
|
||||
|
||||
test('the native call budget stops an oversized loop without replaying actions', async () => {
|
||||
const runtime = createRuntime()
|
||||
let clicks = 0
|
||||
const result = await runtime.run({
|
||||
code: 'let app = await cua.getApp("Fixture"); for(let i=0;i<300;i++) await app.click([1,1])',
|
||||
timeoutMs: 5000,
|
||||
}, async name => {
|
||||
if (name === 'click') ++clicks
|
||||
return { content: [{ type: 'text', text: 'fixture state' }] }
|
||||
})
|
||||
expect(result.isError).toBe(true)
|
||||
expect(clicks).toBe(255)
|
||||
expect(result.structuredContent).toMatchObject({ nativeCallsStarted: 256, nativeCallsCompleted: 256, bindingsReset: true })
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,393 @@
|
||||
import { execFile, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'
|
||||
import { AsyncResource } from 'node:async_hooks'
|
||||
import { mkdtemp, realpath, rm } from 'node:fs/promises'
|
||||
import { rmSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
import { wrapCommandWithSandboxMacOS } from '@anthropic-ai/sandbox-runtime/dist/sandbox/macos-sandbox-utils.js'
|
||||
import { isInBundledMode } from '../bundledMode.js'
|
||||
import { REPL_BOOTSTRAP_SOURCE } from '../../vendor/computer-use-mcp/replApi.js'
|
||||
import {
|
||||
REPL_MAX_ACTIONS,
|
||||
REPL_MAX_CODE_BYTES,
|
||||
REPL_MAX_OUTPUT_BYTES,
|
||||
type ComputerUseReplRuntime,
|
||||
type ReplContent,
|
||||
type ReplInput,
|
||||
type ReplInvoke,
|
||||
type ReplOutput,
|
||||
} from '../../vendor/computer-use-mcp/replProtocol.js'
|
||||
import type { CuCallToolResult } from '../../vendor/computer-use-mcp/toolCalls.js'
|
||||
|
||||
const INVOCABLE_TOOLS = new Set([
|
||||
'list_apps', 'get_app_state', 'click', 'drag', 'scroll', 'press_key',
|
||||
'type_text', 'paste', 'set_value', 'select_text', 'perform_secondary_action',
|
||||
])
|
||||
const errorText = (error: unknown) => error instanceof Error ? error.message : String(error)
|
||||
const quote = (value: string) => `'${value.replaceAll("'", "'\\''")}'`
|
||||
|
||||
export function createComputerUseReplSandboxCommand(options: { args: string[], readable: string[], directory: string }): string {
|
||||
// sandbox-runtime adds its own TMPDIR outside this command. Override it
|
||||
// inside the sandbox without changing the parent process's environment.
|
||||
const args = ['/usr/bin/env', ...['TMPDIR', 'TMP', 'TEMP'].map(name => `${name}=${options.directory}`), ...options.args]
|
||||
return wrapCommandWithSandboxMacOS({
|
||||
command: `exec ${args.map(quote).join(' ')}`,
|
||||
// /bin/sh is a selector stub on recent macOS and may open
|
||||
// /private/var/select/sh. Execute the concrete system shell directly.
|
||||
binShell: '/bin/bash',
|
||||
needsNetworkRestriction: true,
|
||||
readConfig: {
|
||||
denyOnly: ['/'],
|
||||
allowWithinDeny: [...options.readable, '/System', '/usr/lib', '/usr/share', '/bin/bash', '/usr/bin/env', '/dev', '/private/etc/localtime'],
|
||||
},
|
||||
writeConfig: { allowOnly: [options.directory, '/dev/null'], denyWithinAllow: [] },
|
||||
})
|
||||
}
|
||||
|
||||
interface Kernel {
|
||||
process: ChildProcessWithoutNullStreams
|
||||
directory: string
|
||||
ready: Promise<void>
|
||||
rejectReady: (error: Error) => void
|
||||
buffer: string
|
||||
stderr: string
|
||||
health?: ReturnType<typeof setInterval>
|
||||
heartbeat?: { nonce: number, sentAt: number }
|
||||
}
|
||||
|
||||
interface Cell {
|
||||
id: number
|
||||
accepting: boolean
|
||||
controller: AbortController
|
||||
content: ReplContent[]
|
||||
bytes: number
|
||||
actions: number
|
||||
started: number
|
||||
completed: number
|
||||
rejectedBeforeDispatch: number
|
||||
actionFailed: boolean
|
||||
requests: Set<number>
|
||||
tail: Promise<void>
|
||||
finish: (error?: string, invalidate?: boolean) => void
|
||||
result: Promise<CuCallToolResult>
|
||||
invoke: ReplInvoke
|
||||
}
|
||||
|
||||
// Idle kernels must not keep the CLI alive. Exit also removes their disposable
|
||||
// files; neither compilation nor execution uses the user's home or config.
|
||||
const liveKernels = new Set<Kernel>()
|
||||
let exitHookInstalled = false
|
||||
function killKernel(kernel: Kernel) {
|
||||
liveKernels.delete(kernel)
|
||||
clearInterval(kernel.health)
|
||||
try { if (kernel.process.pid) process.kill(-kernel.process.pid, 'SIGKILL') } catch {}
|
||||
kernel.process.kill('SIGKILL')
|
||||
kernel.process.stdin.destroy()
|
||||
kernel.process.stdout.destroy()
|
||||
kernel.process.stderr.destroy()
|
||||
kernel.rejectReady(new Error('Computer Use JavaScript kernel stopped.'))
|
||||
}
|
||||
|
||||
/** Persistent JS lives in a disposable sandboxed child, never in the CLI VM. */
|
||||
export class ComputerUseRepl implements ComputerUseReplRuntime {
|
||||
private kernel?: Kernel
|
||||
private cell?: Cell
|
||||
private nextCellId = 0
|
||||
private idleFailure?: string
|
||||
|
||||
constructor(private readonly limits = { maxRssBytes: 768 * 1024 * 1024, healthCheckMs: 1000 }) {}
|
||||
|
||||
private async start(signal: AbortSignal): Promise<Kernel> {
|
||||
if (this.kernel) return this.kernel
|
||||
if (process.platform !== 'darwin') throw new Error('Computer Use JavaScript is currently available on macOS only.')
|
||||
const directory = await realpath(await mkdtemp(join(tmpdir(), 'cc-haha-cu-repl-')))
|
||||
try {
|
||||
const executable = await realpath(process.execPath)
|
||||
let args: string[]
|
||||
let readable: string[]
|
||||
// A compiled Bun program with no embedded asset imports reports an
|
||||
// empty embeddedFiles array. Its modules still live in Bun's virtual FS.
|
||||
if (isInBundledMode() || import.meta.url.startsWith('file:///$bunfs/')) {
|
||||
args = [executable, '--computer-use-repl-worker']
|
||||
readable = [executable, directory]
|
||||
} else {
|
||||
// One self-contained script keeps the sandbox from reading the source
|
||||
// checkout, node_modules, .env files or provider configuration.
|
||||
const build = await Bun.build({
|
||||
entrypoints: [fileURLToPath(new URL('./replWorker.ts', import.meta.url))],
|
||||
outdir: directory,
|
||||
target: 'bun',
|
||||
format: 'esm',
|
||||
packages: 'bundle',
|
||||
})
|
||||
if (!build.success) throw new Error(`Could not build Computer Use kernel: ${build.logs.join('\n')}`)
|
||||
args = [executable, '--no-env-file', join(directory, 'replWorker.js')]
|
||||
readable = [executable, directory]
|
||||
}
|
||||
if (signal.aborted) throw new Error('Computer Use JavaScript cancelled before startup.')
|
||||
// Use the existing pure macOS wrapper, not SandboxManager's shared Bash
|
||||
// configuration/proxies. No network, Unix sockets, PTY or user files.
|
||||
const command = createComputerUseReplSandboxCommand({ args, readable, directory })
|
||||
const child = spawn('/bin/sh', ['-c', `exec ${command}`], {
|
||||
cwd: directory,
|
||||
detached: true,
|
||||
stdio: 'pipe',
|
||||
env: {
|
||||
HOME: directory, TMPDIR: directory, TMP: directory, TEMP: directory,
|
||||
PATH: '/usr/bin:/bin', LANG: 'en_US.UTF-8', BUN_OPTIONS: '--no-env-file',
|
||||
CLAUDE_CONFIG_DIR: join(directory, '.claude'),
|
||||
},
|
||||
})
|
||||
let ready!: () => void
|
||||
let rejectReady!: (error: Error) => void
|
||||
const readyPromise = new Promise<void>((resolve, reject) => { ready = resolve; rejectReady = reject })
|
||||
// An exit before run() starts awaiting readiness must not be unhandled.
|
||||
void readyPromise.catch(() => {})
|
||||
const kernel: Kernel = { process: child, directory, ready: readyPromise, rejectReady, buffer: '', stderr: '' }
|
||||
this.kernel = kernel
|
||||
liveKernels.add(kernel)
|
||||
if (!exitHookInstalled) {
|
||||
exitHookInstalled = true
|
||||
process.once('exit', () => {
|
||||
for (const item of liveKernels) {
|
||||
killKernel(item)
|
||||
rmSync(item.directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
}
|
||||
const failed = (message: string) => {
|
||||
rejectReady(new Error(message))
|
||||
if (this.kernel !== kernel) return
|
||||
if (this.cell) this.cell.finish(message, true)
|
||||
else {
|
||||
this.idleFailure = `${message} Bindings were reset. Select the app again.`
|
||||
this.kernel = undefined
|
||||
killKernel(kernel)
|
||||
void rm(kernel.directory, { recursive: true, force: true }).catch(() => {})
|
||||
}
|
||||
}
|
||||
child.on('error', error => failed(errorText(error)))
|
||||
child.on('exit', (code, sig) => failed(`Computer Use JavaScript kernel exited (${sig ?? code}). ${kernel.stderr}`))
|
||||
child.stdin.on('error', error => failed(errorText(error)))
|
||||
child.stderr.setEncoding('utf8')
|
||||
child.stderr.on('data', (chunk: string) => {
|
||||
kernel.stderr = (kernel.stderr + chunk).slice(-4096)
|
||||
})
|
||||
child.stdout.setEncoding('utf8')
|
||||
child.stdout.on('data', (chunk: string) => {
|
||||
if (this.kernel !== kernel) return
|
||||
kernel.buffer += chunk
|
||||
if (Buffer.byteLength(kernel.buffer) > REPL_MAX_OUTPUT_BYTES) {
|
||||
failed('Computer Use JavaScript exceeded the output limit.')
|
||||
return
|
||||
}
|
||||
let newline: number
|
||||
while ((newline = kernel.buffer.indexOf('\n')) >= 0) {
|
||||
const line = kernel.buffer.slice(0, newline)
|
||||
kernel.buffer = kernel.buffer.slice(newline + 1)
|
||||
try {
|
||||
const message = JSON.parse(line) as ReplOutput
|
||||
if (message.type === 'ready') ready()
|
||||
else if (message.type === 'pong') {
|
||||
if (message.nonce === kernel.heartbeat?.nonce) kernel.heartbeat = undefined
|
||||
}
|
||||
else this.receive(kernel, message)
|
||||
} catch {
|
||||
failed('Computer Use JavaScript kernel sent invalid protocol data.')
|
||||
}
|
||||
}
|
||||
})
|
||||
let nonce = 0
|
||||
let checkingMemory = false
|
||||
kernel.health = setInterval(() => {
|
||||
if (this.kernel !== kernel) return
|
||||
// Bun has no supported hard heap cap. This independent RSS watchdog is
|
||||
// a soft resource limit; a runaway VM cannot prevent the parent check.
|
||||
if (!checkingMemory && child.pid) {
|
||||
checkingMemory = true
|
||||
execFile('/bin/ps', ['-o', 'rss=', '-p', String(child.pid)], { timeout: 1000, maxBuffer: 1024 }, (error, stdout) => {
|
||||
checkingMemory = false
|
||||
if (error || this.kernel !== kernel) return
|
||||
const rssBytes = Number(stdout.trim()) * 1024
|
||||
if (Number.isFinite(rssBytes) && rssBytes > this.limits.maxRssBytes) failed('Computer Use JavaScript exceeded its memory budget.')
|
||||
})
|
||||
}
|
||||
if (this.cell) {
|
||||
kernel.heartbeat = undefined
|
||||
return
|
||||
}
|
||||
if (kernel.heartbeat) {
|
||||
if (Date.now() - kernel.heartbeat.sentAt > 2000) failed('Computer Use JavaScript left an unresponsive background task.')
|
||||
} else {
|
||||
kernel.heartbeat = { nonce: ++nonce, sentAt: Date.now() }
|
||||
this.send(kernel, { type: 'ping', nonce })
|
||||
}
|
||||
}, this.limits.healthCheckMs)
|
||||
kernel.health.unref()
|
||||
this.send(kernel, { type: 'init', bootstrap: REPL_BOOTSTRAP_SOURCE })
|
||||
return kernel
|
||||
} catch (error) {
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
private send(kernel: Kernel, message: ReplInput) {
|
||||
if (this.kernel === kernel && !kernel.process.stdin.destroyed) {
|
||||
kernel.process.stdin.write(`${JSON.stringify(message)}\n`)
|
||||
}
|
||||
}
|
||||
|
||||
private receive(kernel: Kernel, message: ReplOutput) {
|
||||
const cell = this.cell
|
||||
if (!cell || !cell.accepting || !('cellId' in message) || message.cellId !== cell.id) return
|
||||
if (message.type === 'done') {
|
||||
cell.finish(message.error)
|
||||
} else if (message.type === 'emit') {
|
||||
const content = message.content
|
||||
if (!content || (content.type !== 'text' && content.type !== 'image') ||
|
||||
(content.type === 'text' ? typeof content.text !== 'string' :
|
||||
typeof content.data !== 'string' || typeof content.mimeType !== 'string' || !/^image\/(png|jpeg|webp)$/.test(content.mimeType))) {
|
||||
cell.finish('Computer Use JavaScript emitted invalid content.', true)
|
||||
return
|
||||
}
|
||||
cell.bytes += Buffer.byteLength(JSON.stringify(content))
|
||||
if (cell.bytes > REPL_MAX_OUTPUT_BYTES || cell.content.length >= 128) {
|
||||
cell.finish('Computer Use JavaScript exceeded the output limit.', true)
|
||||
} else cell.content.push(content)
|
||||
} else if (message.type === 'invoke') {
|
||||
if (!Number.isSafeInteger(message.requestId) || cell.requests.has(message.requestId) ||
|
||||
!INVOCABLE_TOOLS.has(message.name) || ++cell.actions > REPL_MAX_ACTIONS) {
|
||||
cell.finish('Computer Use JavaScript exceeded its action limit or requested an invalid operation.', true)
|
||||
return
|
||||
}
|
||||
cell.requests.add(message.requestId)
|
||||
// Even Promise.all cannot interleave target acquisition, injection and
|
||||
// observation. A cell is one item in the outer semantic session queue.
|
||||
cell.tail = cell.tail.then(async () => {
|
||||
if (!cell.accepting || cell.controller.signal.aborted) return
|
||||
++cell.started
|
||||
try {
|
||||
const result = await cell.invoke(message.name, message.args, cell.controller.signal)
|
||||
if (result.isError) {
|
||||
if (result.nativeCallNotDispatched === true) ++cell.rejectedBeforeDispatch
|
||||
else cell.actionFailed = true
|
||||
} else ++cell.completed
|
||||
if (!cell.accepting) return
|
||||
// A completed command rejection is structured API data. The facade
|
||||
// recreates its typed error in the guest; transport failures below
|
||||
// still reject the RPC itself.
|
||||
this.send(kernel, { type: 'response', cellId: cell.id, requestId: message.requestId, result })
|
||||
} catch (error) {
|
||||
cell.actionFailed = true
|
||||
if (cell.accepting) this.send(kernel, { type: 'response', cellId: cell.id, requestId: message.requestId, error: errorText(error) })
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
async run(options: Parameters<ComputerUseReplRuntime['run']>[0], invoke: ReplInvoke): Promise<CuCallToolResult> {
|
||||
if (this.cell) return { isError: true, content: [{ type: 'text', text: 'A Computer Use JavaScript cell is already running.' }] }
|
||||
if (this.idleFailure) {
|
||||
const text = this.idleFailure
|
||||
this.idleFailure = undefined
|
||||
return { isError: true, content: [{ type: 'text', text }] }
|
||||
}
|
||||
if (typeof options.code !== 'string' || Buffer.byteLength(options.code) > REPL_MAX_CODE_BYTES ||
|
||||
!Number.isInteger(options.timeoutMs) || options.timeoutMs < 1 || options.timeoutMs > 60000) {
|
||||
return { isError: true, content: [{ type: 'text', text: 'Invalid Computer Use JavaScript code or timeout (maximum 60 seconds).' }] }
|
||||
}
|
||||
const controller = new AbortController()
|
||||
let resolve!: (result: CuCallToolResult) => void
|
||||
const result = new Promise<CuCallToolResult>(done => { resolve = done })
|
||||
let deadline: ReturnType<typeof setTimeout>
|
||||
let poll: ReturnType<typeof setInterval>
|
||||
const cancelled = () => cell.finish('Computer Use JavaScript cancelled. Bindings were reset; observe the app before continuing.', true)
|
||||
const cell: Cell = {
|
||||
id: ++this.nextCellId, accepting: true, controller, content: [], bytes: 0,
|
||||
actions: 0, started: 0, completed: 0, rejectedBeforeDispatch: 0, actionFailed: false, requests: new Set(), tail: Promise.resolve(), result,
|
||||
// The warm worker's stdout listener retains its startup async context.
|
||||
// Native dispatch must instead read this cell caller's state and aborts.
|
||||
invoke: AsyncResource.bind(invoke),
|
||||
finish: (error, invalidate = false) => {
|
||||
if (!cell.accepting) return
|
||||
cell.accepting = false
|
||||
clearTimeout(deadline)
|
||||
clearInterval(poll)
|
||||
options.signal?.removeEventListener('abort', cancelled)
|
||||
if (invalidate) {
|
||||
controller.abort()
|
||||
const kernel = this.kernel
|
||||
this.kernel = undefined
|
||||
if (kernel) {
|
||||
killKernel(kernel)
|
||||
void rm(kernel.directory, { recursive: true, force: true }).catch(() => {})
|
||||
}
|
||||
}
|
||||
// A native event already being injected cannot be undone. Drain that
|
||||
// one operation before releasing the session queue; queued RPCs stop.
|
||||
void cell.tail.then(() => {
|
||||
if (this.cell === cell) this.cell = undefined
|
||||
const kernel = this.kernel
|
||||
if (kernel) {
|
||||
kernel.process.unref()
|
||||
for (const stream of [kernel.process.stdin, kernel.process.stdout, kernel.process.stderr]) {
|
||||
(stream as typeof stream & { unref?: () => void }).unref?.()
|
||||
}
|
||||
}
|
||||
resolve({
|
||||
content: [...cell.content, ...(error ? [{ type: 'text' as const, text: error }] : [])],
|
||||
...(error ? {
|
||||
isError: true,
|
||||
structuredContent: {
|
||||
nativeCallsStarted: cell.started,
|
||||
nativeCallsCompleted: cell.completed,
|
||||
nativeCallsRejectedBeforeDispatch: cell.rejectedBeforeDispatch,
|
||||
nativeResultUnknown: cell.actionFailed || cell.started !== cell.completed + cell.rejectedBeforeDispatch,
|
||||
bindingsReset: invalidate,
|
||||
recovery: cell.started === cell.rejectedBeforeDispatch
|
||||
? 'No native actions were dispatched. Resolve the reported error before retrying.'
|
||||
: 'Observe the app before retrying. Completed calls were dispatched, but their visual effect still needs verification.',
|
||||
},
|
||||
} : {}),
|
||||
})
|
||||
})
|
||||
},
|
||||
}
|
||||
this.cell = cell
|
||||
deadline = setTimeout(() => cell.finish('Computer Use JavaScript timed out. Bindings were reset; observe the app before continuing.', true), options.timeoutMs)
|
||||
poll = setInterval(() => { if (options.isAborted?.()) cancelled() }, 50)
|
||||
options.signal?.addEventListener('abort', cancelled, { once: true })
|
||||
if (options.signal?.aborted || options.isAborted?.()) cancelled()
|
||||
else {
|
||||
try {
|
||||
const kernel = await this.start(controller.signal)
|
||||
if (cell.accepting) {
|
||||
kernel.process.ref()
|
||||
for (const stream of [kernel.process.stdin, kernel.process.stdout, kernel.process.stderr]) {
|
||||
(stream as typeof stream & { ref?: () => void }).ref?.()
|
||||
}
|
||||
await kernel.ready
|
||||
if (cell.accepting) this.send(kernel, { type: 'run', cellId: cell.id, code: options.code })
|
||||
}
|
||||
} catch (error) {
|
||||
cell.finish(`Computer Use JavaScript startup failed: ${errorText(error)}`, true)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
async reset() {
|
||||
this.idleFailure = undefined
|
||||
const cell = this.cell
|
||||
const kernel = this.kernel
|
||||
cell?.finish('Computer Use JavaScript reset. Bindings were discarded.', true)
|
||||
if (kernel && this.kernel === kernel) {
|
||||
this.kernel = undefined
|
||||
killKernel(kernel)
|
||||
await rm(kernel.directory, { recursive: true, force: true })
|
||||
}
|
||||
if (cell) await cell.result
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,218 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test'
|
||||
import { spawn, type ChildProcess } from 'node:child_process'
|
||||
import { mkdir, mkdtemp, readFile, realpath, rm, stat, symlink, writeFile } from 'node:fs/promises'
|
||||
import { connect, createServer, type NetConnectOpts, type Server } from 'node:net'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { ComputerUseRepl, createComputerUseReplSandboxCommand } from './replRuntime.js'
|
||||
|
||||
const directories: string[] = []
|
||||
const children = new Set<ChildProcess>()
|
||||
const servers: Server[] = []
|
||||
const runtimes: ComputerUseRepl[] = []
|
||||
|
||||
afterEach(async () => {
|
||||
for (const child of children) child.kill('SIGKILL')
|
||||
children.clear()
|
||||
await Promise.all(runtimes.splice(0).map(runtime => runtime.reset()))
|
||||
await Promise.all(servers.splice(0).map(server => new Promise<void>(resolve => server.close(() => resolve()))))
|
||||
await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true })))
|
||||
})
|
||||
|
||||
async function temporaryDirectory() {
|
||||
const directory = await realpath(await mkdtemp(join(tmpdir(), 'cu-repl-sandbox-test-')))
|
||||
directories.push(directory)
|
||||
return directory
|
||||
}
|
||||
|
||||
async function verifyListening(options: NetConnectOpts) {
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
const socket = connect(options)
|
||||
let connected = false
|
||||
const deadline = setTimeout(() => { socket.destroy(); reject(new Error('Fixture listener did not accept the control connection')) }, 1000)
|
||||
socket.once('error', error => { clearTimeout(deadline); reject(error) })
|
||||
socket.once('connect', () => { connected = true })
|
||||
// The fixture server closes only after recording its accepted connection.
|
||||
socket.once('close', () => {
|
||||
clearTimeout(deadline)
|
||||
if (connected) resolve()
|
||||
else reject(new Error('Fixture control connection closed before connecting'))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
function runSandbox(command: string, directory: string) {
|
||||
return new Promise<{ code: number | null, stdout: string, stderr: string }>((resolve, reject) => {
|
||||
const child = spawn('/bin/sh', ['-c', `exec ${command}`], {
|
||||
cwd: directory,
|
||||
stdio: 'pipe',
|
||||
env: {
|
||||
HOME: directory, TMPDIR: directory, TMP: directory, TEMP: directory,
|
||||
PATH: '/usr/bin:/bin', LANG: 'en_US.UTF-8', BUN_OPTIONS: '--no-env-file',
|
||||
CLAUDE_CONFIG_DIR: join(directory, '.claude'),
|
||||
},
|
||||
})
|
||||
children.add(child)
|
||||
let stdout = ''
|
||||
let stderr = ''
|
||||
const deadline = setTimeout(() => child.kill('SIGKILL'), 5000)
|
||||
child.stdout?.on('data', chunk => { stdout += chunk.toString() })
|
||||
child.stderr?.on('data', chunk => { stderr += chunk.toString() })
|
||||
child.once('error', error => { clearTimeout(deadline); children.delete(child); reject(error) })
|
||||
child.once('close', code => {
|
||||
clearTimeout(deadline)
|
||||
children.delete(child)
|
||||
resolve({ code, stdout, stderr })
|
||||
})
|
||||
child.stdin?.end()
|
||||
})
|
||||
}
|
||||
|
||||
const suite = process.platform === 'darwin' ? describe : describe.skip
|
||||
suite('Computer Use OS sandbox', () => {
|
||||
test('production child receives a private and usable temporary directory', async () => {
|
||||
const directory = join(await temporaryDirectory(), "child's temporary directory")
|
||||
await mkdir(directory)
|
||||
const script = join(directory, 'temporary-directory-probe.mjs')
|
||||
await writeFile(script, `
|
||||
import fs from 'node:fs'
|
||||
import os from 'node:os'
|
||||
import path from 'node:path'
|
||||
const expected = ${JSON.stringify(directory)}
|
||||
const result = {
|
||||
TMPDIR: process.env.TMPDIR, TMP: process.env.TMP, TEMP: process.env.TEMP,
|
||||
osTmpdir: os.tmpdir(), temporaryFile: 'not attempted outside fixture',
|
||||
}
|
||||
// Do not write a test file if the sandbox wrapper redirected temp state
|
||||
// anywhere outside this disposable directory (for example /tmp/claude).
|
||||
if ([result.TMPDIR, result.TMP, result.TEMP, result.osTmpdir].every(value => value === expected)) {
|
||||
const created = fs.mkdtempSync(path.join(os.tmpdir(), 'private-temp-'))
|
||||
const file = path.join(created, 'sentinel.txt')
|
||||
fs.writeFileSync(file, 'private temporary file')
|
||||
result.temporaryFile = fs.readFileSync(file, 'utf8')
|
||||
// afterEach owns the whole fixture directory, including this child.
|
||||
}
|
||||
console.log(JSON.stringify(result))
|
||||
`)
|
||||
const executable = await realpath(process.execPath)
|
||||
const command = createComputerUseReplSandboxCommand({
|
||||
args: [executable, '--no-env-file', script], readable: [executable, directory], directory,
|
||||
})
|
||||
const output = await runSandbox(command, directory)
|
||||
expect(output.code, output.stderr).toBe(0)
|
||||
expect(JSON.parse(output.stdout.trim())).toEqual({
|
||||
TMPDIR: directory, TMP: directory, TEMP: directory, osTmpdir: directory,
|
||||
temporaryFile: 'private temporary file',
|
||||
})
|
||||
}, 15000)
|
||||
|
||||
test('production profile denies external files, symlink and Data-volume aliases, and local network', async () => {
|
||||
const inside = await temporaryDirectory()
|
||||
const outside = await temporaryDirectory()
|
||||
const sentinel = join(outside, 'sentinel.txt')
|
||||
await writeFile(sentinel, 'disposable outside sentinel')
|
||||
let dataVolumeAlias: string | undefined = `/System/Volumes/Data${sentinel}`
|
||||
try {
|
||||
// Probe only this disposable file. Confirm the alias is the same inode
|
||||
// before using it to test the sandbox's handling of APFS firmlinks.
|
||||
expect(await readFile(dataVolumeAlias, 'utf8')).toBe('disposable outside sentinel')
|
||||
const [original, alias] = await Promise.all([stat(sentinel), stat(dataVolumeAlias)])
|
||||
expect({ device: alias.dev, inode: alias.ino }).toEqual({ device: original.dev, inode: original.ino })
|
||||
} catch (error) {
|
||||
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error
|
||||
dataVolumeAlias = undefined
|
||||
}
|
||||
const link = join(inside, 'escape-link')
|
||||
await symlink(sentinel, link)
|
||||
|
||||
let accepted = 0
|
||||
const tcp = createServer(socket => { accepted++; socket.destroy() })
|
||||
const unix = createServer(socket => { accepted++; socket.destroy() })
|
||||
servers.push(tcp, unix)
|
||||
await new Promise<void>((resolve, reject) => { tcp.once('error', reject); tcp.listen(0, '127.0.0.1', resolve) })
|
||||
const address = tcp.address()
|
||||
if (!address || typeof address === 'string') throw new Error('missing loopback fixture address')
|
||||
const unixPath = join(inside, 'server.sock')
|
||||
await new Promise<void>((resolve, reject) => { unix.once('error', reject); unix.listen(unixPath, resolve) })
|
||||
await verifyListening({ host: '127.0.0.1', port: address.port })
|
||||
await verifyListening({ path: unixPath })
|
||||
expect(accepted).toBe(2)
|
||||
accepted = 0
|
||||
|
||||
const script = join(inside, 'probe.mjs')
|
||||
await writeFile(script, `
|
||||
import fs from 'node:fs'
|
||||
import net from 'node:net'
|
||||
const attempt = operation => { try { operation(); return 'allowed' } catch (error) { return error.code } }
|
||||
const connect = options => new Promise(resolve => {
|
||||
const socket = net.connect(options)
|
||||
const finish = value => { clearTimeout(timer); socket.destroy(); resolve(value) }
|
||||
const timer = setTimeout(() => finish('timeout'), 1000)
|
||||
socket.once('connect', () => finish('allowed'))
|
||||
socket.once('error', error => finish(error.code))
|
||||
})
|
||||
const result = {
|
||||
started: true,
|
||||
read: attempt(() => fs.readFileSync(${JSON.stringify(sentinel)}, 'utf8')),
|
||||
write: attempt(() => fs.writeFileSync(${JSON.stringify(sentinel)}, 'changed')),
|
||||
symlinkRead: attempt(() => fs.readFileSync(${JSON.stringify(link)}, 'utf8')),
|
||||
symlinkWrite: attempt(() => fs.writeFileSync(${JSON.stringify(link)}, 'changed')),
|
||||
dataVolumeRead: ${dataVolumeAlias ? `attempt(() => fs.readFileSync(${JSON.stringify(dataVolumeAlias)}, 'utf8'))` : "'unavailable'"},
|
||||
dataVolumeWrite: ${dataVolumeAlias ? `attempt(() => fs.writeFileSync(${JSON.stringify(dataVolumeAlias)}, 'changed'))` : "'unavailable'"},
|
||||
internalWrite: attempt(() => fs.writeFileSync(${JSON.stringify(join(inside, 'allowed.txt'))}, 'ok')),
|
||||
unixExists: fs.statSync(${JSON.stringify(unixPath)}).isSocket(),
|
||||
tcp: await connect({host:'127.0.0.1', port:${address.port}}),
|
||||
unix: await connect({path:${JSON.stringify(unixPath)}}),
|
||||
}
|
||||
console.log(JSON.stringify(result))
|
||||
`)
|
||||
const executable = await realpath(process.execPath)
|
||||
const command = createComputerUseReplSandboxCommand({
|
||||
args: [executable, '--no-env-file', script], readable: [executable, inside], directory: inside,
|
||||
})
|
||||
const output = await runSandbox(command, inside)
|
||||
expect(output.code, output.stderr).toBe(0)
|
||||
const result = JSON.parse(output.stdout.trim())
|
||||
expect(result.started).toBe(true)
|
||||
expect(result.internalWrite).toBe('allowed')
|
||||
expect(result.unixExists).toBe(true)
|
||||
for (const operation of ['read', 'write', 'symlinkRead', 'symlinkWrite']) {
|
||||
expect(['EPERM', 'EACCES'], `${operation}: ${result[operation]}`).toContain(result[operation])
|
||||
}
|
||||
console.info(`[sandbox Data-volume alias] control=${dataVolumeAlias ? 'same inode' : 'unavailable'}, read=${result.dataVolumeRead}, write=${result.dataVolumeWrite}`)
|
||||
if (dataVolumeAlias) {
|
||||
for (const operation of ['dataVolumeRead', 'dataVolumeWrite']) {
|
||||
expect(['EPERM', 'EACCES'], `${operation}: ${result[operation]}`).toContain(result[operation])
|
||||
}
|
||||
}
|
||||
// macOS can report sandbox-denied connect as ECONNREFUSED. The controls
|
||||
// above prove both listeners are live, and neither may receive this probe.
|
||||
for (const operation of ['tcp', 'unix']) {
|
||||
expect(['EPERM', 'EACCES', 'ECONNREFUSED', 'ENOENT'], `${operation}: ${result[operation]}`).toContain(result[operation])
|
||||
}
|
||||
expect(accepted).toBe(0)
|
||||
expect(await readFile(sentinel, 'utf8')).toBe('disposable outside sentinel')
|
||||
expect(await readFile(join(inside, 'allowed.txt'), 'utf8')).toBe('ok')
|
||||
}, 15000)
|
||||
|
||||
})
|
||||
|
||||
describe('Computer Use OS sandbox availability', () => {
|
||||
test('actual worker bootstraps on macOS and fails closed on unsupported platforms', async () => {
|
||||
const runtime = new ComputerUseRepl()
|
||||
runtimes.push(runtime)
|
||||
let invoked = false
|
||||
const result = await runtime.run({ code: 'nodeRepl.write("sandbox worker ready")', timeoutMs: 5000 }, async () => {
|
||||
invoked = true
|
||||
return { content: [] }
|
||||
})
|
||||
if (process.platform === 'darwin') {
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(result.content).toContainEqual({ type: 'text', text: 'sandbox worker ready' })
|
||||
} else {
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.content).toEqual([{ type: 'text', text: expect.stringContaining('available on macOS only') }])
|
||||
}
|
||||
expect(invoked).toBe(false)
|
||||
}, 15000)
|
||||
})
|
||||
@@ -0,0 +1,476 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { mkdtemp, rm } from 'node:fs/promises'
|
||||
import { tmpdir } from 'node:os'
|
||||
import { join } from 'node:path'
|
||||
import { createComputerUseReplWorker } from './replWorker'
|
||||
import { REPL_BOOTSTRAP_SOURCE } from '../../vendor/computer-use-mcp/replApi'
|
||||
|
||||
type Message = Record<string, any>
|
||||
|
||||
function fixture() {
|
||||
const messages: Message[] = []
|
||||
const worker = createComputerUseReplWorker(message => messages.push(message))
|
||||
return { worker, messages }
|
||||
}
|
||||
|
||||
async function init(worker: ReturnType<typeof createComputerUseReplWorker>) {
|
||||
await worker.receive({ type: 'init', bootstrap: `
|
||||
const nodeRepl = Object.freeze({
|
||||
write(value) { __cuEmit({type:'text', text: JSON.stringify(value)}) }
|
||||
})
|
||||
` })
|
||||
}
|
||||
|
||||
function texts(messages: Message[]) {
|
||||
return messages.filter(message => message.type === 'emit').map(message => message.content.text)
|
||||
}
|
||||
|
||||
async function until(predicate: () => boolean) {
|
||||
for (let attempt = 0; attempt < 200; attempt++) {
|
||||
if (predicate()) {
|
||||
return
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 10))
|
||||
}
|
||||
throw new Error('worker event did not arrive')
|
||||
}
|
||||
|
||||
describe('computer use persistent REPL worker', () => {
|
||||
test('closures and later cells read and write one persistent lexical binding', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: 'let count=0; function step(){count+=1}; function current(){return count}' })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'step(); nodeRepl.write({count,current:current()})' })
|
||||
await worker.receive({ type: 'run', cellId: 3, code: 'count=10; step(); nodeRepl.write({count,current:current()})' })
|
||||
expect(texts(messages)).toEqual(['{"count":1,"current":1}', '{"count":11,"current":11}'])
|
||||
expect(messages.filter(message => message.type === 'done').every(message => !message.error)).toBe(true)
|
||||
})
|
||||
|
||||
test('helpers resolve a future top-level App without replacing the VM global object', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: `
|
||||
function target(){return app}
|
||||
function futureMath(){return Math}
|
||||
function missing(){return typeof absent}
|
||||
const originalMath = globalThis.Math
|
||||
nodeRepl.write(missing())
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: `
|
||||
let app = {name:'Fixture'}
|
||||
let Math = {name:'local'}
|
||||
let absent = 3
|
||||
nodeRepl.write([target().name, futureMath().name, missing(), globalThis.Math===originalMath])
|
||||
` })
|
||||
expect(texts(messages)).toEqual(['"undefined"', '["Fixture","local","number",true]'])
|
||||
expect(messages.filter(message => message.type === 'done').every(message => !message.error)).toBe(true)
|
||||
})
|
||||
|
||||
test('shared references preserve shorthand, destructuring, shadowing and lexical call receivers', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: `
|
||||
let count=1
|
||||
function read(){return count}
|
||||
function receiver(){return this}
|
||||
function tagged(){return this}
|
||||
function defaults(value=count){var count=7; return [value,count,arguments.length]}
|
||||
class View { #value=3; current(){return count+this.#value} self(){return View} }
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: `
|
||||
({count}= {count:4})
|
||||
let output={count}
|
||||
let [a,b=count] = [1]
|
||||
{ let count=8; output.block=count }
|
||||
for(let count=0;count<2;count++){output.loop=count}
|
||||
function local(count){return (()=>count)()}
|
||||
function hoisted(){count=5;var count;return count}
|
||||
const named=function count(){return count.name}
|
||||
try {throw 6} catch(count){output.caught=count}
|
||||
let view = new View()
|
||||
nodeRepl.write([output,a,b,read(),local(9),hoisted(),named(),view.current(),view.self()===View,
|
||||
receiver()===undefined,tagged\`x\`===undefined,defaults()])
|
||||
` })
|
||||
expect(texts(messages)).toEqual(['[{"count":4,"block":8,"loop":1,"caught":6},1,4,4,9,5,"count",7,true,true,true,[4,7,0]]'])
|
||||
expect(messages.filter(message => message.type === 'done').every(message => !message.error)).toBe(true)
|
||||
})
|
||||
|
||||
test('failed declarations retain partial initialization and restore slots visible to old closures', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: 'let app="old"; function selected(){return app}; const fixed=1; function constant(){return fixed}' })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'let app=selected()' })
|
||||
expect(messages.find(message => message.type === 'done' && message.cellId===2).error).toMatch(/initializ|before/i)
|
||||
await worker.receive({ type: 'run', cellId: 3, code: 'nodeRepl.write(selected()); const [first,second=(()=>{throw Error("stop")})()] = [7]' })
|
||||
await worker.receive({ type: 'run', cellId: 4, code: 'fixed=2; nodeRepl.write([first,typeof second,constant()]); const same=1; same=2' })
|
||||
expect(texts(messages)).toEqual(['"old"', 'Warning: fixed was declared with const; use let for reassignable variables.', '[7,"undefined",2]'])
|
||||
expect(messages.find(message => message.type === 'done' && message.cellId===4).error).toContain('constant')
|
||||
})
|
||||
|
||||
test('an existing App helper follows rebinding and never dispatches to the old App', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await worker.receive({ type: 'init', bootstrap: REPL_BOOTSTRAP_SOURCE })
|
||||
const run = async (cellId: number, code: string) => {
|
||||
const running = worker.receive({ type: 'run', cellId, code })
|
||||
const replied = new Set<number>()
|
||||
while (!messages.some(message => message.type === 'done' && message.cellId === cellId)) {
|
||||
for (const request of messages.filter(message => message.type === 'invoke' && message.cellId === cellId && !replied.has(message.requestId))) {
|
||||
replied.add(request.requestId)
|
||||
await worker.receive({ type: 'response', cellId, requestId: request.requestId,
|
||||
result: { app: request.args.app, content: [{ type: 'text', text: 'fixture state' }] } })
|
||||
}
|
||||
await new Promise(resolve => setTimeout(resolve, 1))
|
||||
}
|
||||
await running
|
||||
}
|
||||
await run(1, 'let app = await cua.getApp("First"); async function build(){await app.click([1,2])}')
|
||||
await run(2, 'app = await cua.getApp("Second"); await build()')
|
||||
await run(3, 'let app = await cua.getApp("Third"); await build()')
|
||||
expect(messages.filter(message => message.type === 'invoke' && message.name === 'click').map(message => message.args.app)).toEqual(['Second', 'Third'])
|
||||
expect(messages.filter(message => message.type === 'done').every(message => !message.error)).toBe(true)
|
||||
})
|
||||
|
||||
test('retains top level await, variables, functions and object identity across cells', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: `
|
||||
const app = await Promise.resolve({clicks: 0})
|
||||
let count = 2
|
||||
function label() { return app.clicks }
|
||||
class View { constructor() { this.app = app } }
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: `
|
||||
app.clicks++; count += 3
|
||||
nodeRepl.write([count, label(), new View().app === app])
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 3, code: 'const app = {clicks: 9}; nodeRepl.write(app.clicks)' })
|
||||
expect(texts(messages)).toEqual(['[5,1,true]', 'Warning: app was declared with const; use let for reassignable variables.', '9'])
|
||||
expect(messages.filter(message => message.type === 'done').every(message => !message.error)).toBe(true)
|
||||
})
|
||||
|
||||
test('tolerates prior const reassignment with a warning and saves initialized bindings after failure', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: 'let count = 1; const fixed = 5' })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'count = 3; const reached = 7; throw new Error("stop"); const unreached = 8' })
|
||||
await worker.receive({ type: 'run', cellId: 3, code: 'nodeRepl.write([count, reached, typeof unreached]); fixed = 4' })
|
||||
expect(texts(messages)).toEqual(['Warning: fixed was declared with const; use let for reassignable variables.', '[3,7,"undefined"]'])
|
||||
expect(messages.find(message => message.type === 'done' && message.cellId === 2).error).toContain('stop')
|
||||
expect(messages.find(message => message.type === 'done' && message.cellId === 3).error).toBeUndefined()
|
||||
await worker.receive({ type: 'run', cellId: 4, code: 'nodeRepl.write(fixed); const local = 1; local = 2' })
|
||||
expect(texts(messages).at(-1)).toBe('4')
|
||||
expect(messages.find(message => message.type === 'done' && message.cellId === 4).error).toMatch(/constant|readonly/i)
|
||||
})
|
||||
|
||||
test('failed cells retain reached declarations but discard unreached var and function bindings', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: `
|
||||
var before = 1
|
||||
var declared
|
||||
function beforeFn() { return 2 }
|
||||
throw new Error('stop')
|
||||
var after = 3
|
||||
function afterFn() { return 4 }
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: `
|
||||
nodeRepl.write([before, beforeFn(), declared, typeof afterFn,
|
||||
(() => { try { after; return 'BOUND' } catch { return 'ABSENT' } })()])
|
||||
` })
|
||||
expect(texts(messages)).toEqual(['[1,2,null,"undefined","ABSENT"]'])
|
||||
})
|
||||
|
||||
test('failed loop declarations commit only when their initialization or iteration executes', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: `
|
||||
for (var index = 0; index < 2; index++) {}
|
||||
for (var { value, nested: [item] } of [{ value: 3, nested: [4] }]) {}
|
||||
for (var key in {a: 1}) {}
|
||||
for (var single of [6]) if (single === 6) continue
|
||||
for (var empty of []) {}
|
||||
for (var emptyKey in {}) {}
|
||||
throw new Error('stop')
|
||||
for (var unseen = 0; unseen < 2; unseen++) {}
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: `
|
||||
nodeRepl.write([index, value, item, key, single,
|
||||
(() => { try { empty; return 'BOUND' } catch { return 'ABSENT' } })(),
|
||||
(() => { try { emptyKey; return 'BOUND' } catch { return 'ABSENT' } })(),
|
||||
(() => { try { unseen; return 'BOUND' } catch { return 'ABSENT' } })()])
|
||||
` })
|
||||
expect(texts(messages)).toEqual(['[2,3,4,"a",6,"ABSENT","ABSENT","ABSENT"]'])
|
||||
})
|
||||
|
||||
test('failed replacements retain old values while successful cells preserve ordinary hoisting', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: 'let app = {value: 7}; var previous = 8; function existing() { return 9 }' })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: `
|
||||
let app = await Promise.reject(new Error('replacement failed'))
|
||||
var previous = 99
|
||||
function existing() { return 99 }
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 3, code: `
|
||||
nodeRepl.write([app.value, previous, existing()])
|
||||
if (false) { var skipped = 1 }
|
||||
for (var empty of []) {}
|
||||
function local() { var neverGlobal = 1; return neverGlobal }
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 4, code: 'nodeRepl.write([skipped, empty, typeof local, typeof neverGlobal])' })
|
||||
expect(texts(messages)).toEqual(['[7,8,9]', '[null,null,"function","undefined"]'])
|
||||
})
|
||||
|
||||
test('preserves executed writes before a future var declaration without committing short-circuited writes', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: `
|
||||
future = 9
|
||||
orValue ||= 3
|
||||
nullishValue ??= 4
|
||||
andValue &&= 5
|
||||
throw new Error('stop')
|
||||
var future, orValue, nullishValue, andValue
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: `
|
||||
nodeRepl.write([future, orValue, nullishValue,
|
||||
(() => { try { andValue; return 'BOUND' } catch { return 'ABSENT' } })()])
|
||||
` })
|
||||
expect(texts(messages)).toEqual(['[9,3,4,"ABSENT"]'])
|
||||
})
|
||||
|
||||
test('serializes bridge results and errors into the VM realm', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: `
|
||||
const result = await __cuInvoke('get_app_state', {app:'Fixture'})
|
||||
nodeRepl.write([result.value, Object.getPrototypeOf(result) === Object.prototype])
|
||||
try { await __cuInvoke('click', {x:1,y:2}) }
|
||||
catch (error) { nodeRepl.write([error instanceof Error, error.message]) }
|
||||
` })
|
||||
await until(() => messages.some(message => message.type === 'invoke'))
|
||||
const first = messages.find(message => message.type === 'invoke')
|
||||
expect(first).toMatchObject({ cellId: 1, name: 'get_app_state', args: { app: 'Fixture' } })
|
||||
await worker.receive({ type: 'response', cellId: 99, requestId: first.requestId, result: { value: 99 } })
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: first.requestId, result: { value: 4 } })
|
||||
await until(() => messages.filter(message => message.type === 'invoke').length === 2)
|
||||
const second = messages.filter(message => message.type === 'invoke')[1]!
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: second.requestId, error: 'fixture failure' })
|
||||
await running
|
||||
expect(texts(messages)).toEqual(['[4,true]', '[true,"fixture failure"]'])
|
||||
})
|
||||
|
||||
test('awaits already dispatched work but prevents detached calls after submitted code completes', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: `
|
||||
void __cuInvoke('click', {x:1}).then(() => __cuInvoke('click', {x:2})).catch(error => nodeRepl.write(error.message))
|
||||
` })
|
||||
await until(() => messages.some(message => message.type === 'invoke'))
|
||||
expect(messages.some(message => message.type === 'done')).toBe(false)
|
||||
const first = messages.find(message => message.type === 'invoke')
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: first.requestId, result: {} })
|
||||
await running
|
||||
expect(messages.filter(message => message.type === 'invoke')).toHaveLength(1)
|
||||
expect(messages.filter(message => message.type === 'done')).toHaveLength(1)
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'nodeRepl.write("next")' })
|
||||
expect(messages.filter(message => message.type === 'invoke')).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('reports failure of an unawaited dispatched operation instead of claiming success', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: 'void __cuInvoke("click", {})' })
|
||||
await until(() => messages.some(message => message.type === 'invoke'))
|
||||
await Promise.resolve()
|
||||
const request = messages.find(message => message.type === 'invoke')
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: request.requestId, error: 'target moved' })
|
||||
await running
|
||||
expect(messages.find(message => message.type === 'done').error).toContain('target moved')
|
||||
})
|
||||
|
||||
test('retains an unobserved failure received while a different awaited call is pending', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: `
|
||||
void __cuInvoke('click', {})
|
||||
await __cuInvoke('get_app_state', {})
|
||||
` })
|
||||
await until(() => messages.filter(message => message.type === 'invoke').length === 2)
|
||||
const [first, second] = messages.filter(message => message.type === 'invoke')
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: first!.requestId, error: 'first action failed' })
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: second!.requestId, result: {} })
|
||||
await running
|
||||
expect(messages.find(message => message.type === 'done').error).toContain('first action failed')
|
||||
})
|
||||
|
||||
test('an ignored structured native rejection cannot become a successful cell', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: 'void __cuInvoke("click", {})' })
|
||||
await until(() => messages.some(message => message.type === 'invoke'))
|
||||
const request = messages.find(message => message.type === 'invoke')!
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: request.requestId, result: {
|
||||
isError: true, content: [{ type: 'text', text: 'native command rejected' }],
|
||||
nativeError: { name: 'SkyComputerUseError', message: 'native command rejected', code: -10007 },
|
||||
} })
|
||||
await running
|
||||
expect(messages.find(message => message.type === 'done').error).toContain('native command rejected')
|
||||
})
|
||||
|
||||
test('allows explicit error handling of a returned operation', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: `
|
||||
const recovery = __cuInvoke('click', {}).catch(() => 'handled')
|
||||
nodeRepl.write(await recovery)
|
||||
` })
|
||||
await until(() => messages.some(message => message.type === 'invoke'))
|
||||
const request = messages.find(message => message.type === 'invoke')
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: request.requestId, error: 'expected' })
|
||||
await running
|
||||
expect(messages.find(message => message.type === 'done').error).toBeUndefined()
|
||||
expect(texts(messages)).toEqual(['"handled"'])
|
||||
})
|
||||
|
||||
test('waits for the host rejection checkpoint before reporting cell completion', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: 'await Promise.resolve()' })
|
||||
// Flushing only promise continuations must not publish done. The host
|
||||
// needs a complete turn to report ignored async App-method rejections.
|
||||
for (let index = 0; index < 20; index++) {
|
||||
await Promise.resolve()
|
||||
}
|
||||
expect(messages.some(message => message.type === 'done')).toBe(false)
|
||||
await running
|
||||
expect(messages.some(message => message.type === 'done')).toBe(true)
|
||||
})
|
||||
|
||||
test('exposes no Node host objects, module loader, timers or code generation', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: `
|
||||
nodeRepl.write([typeof process, typeof require, typeof Buffer, typeof setTimeout, typeof arguments])
|
||||
for (const candidate of [() => Function('return 1')(), () => __cuInvoke.constructor('return process')(), () => console.log.constructor('return process')()]) {
|
||||
try { candidate(); nodeRepl.write('escaped') } catch { nodeRepl.write('blocked') }
|
||||
}
|
||||
` })
|
||||
expect(texts(messages)).toEqual(['["undefined","undefined","undefined","undefined","undefined"]', '"blocked"', '"blocked"', '"blocked"'])
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'await import("node:fs")' })
|
||||
expect(messages.find(message => message.type === 'done' && message.cellId === 2).error).toContain('not available')
|
||||
})
|
||||
|
||||
test('rejects concurrent cells instead of sharing active context', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: 'await __cuInvoke("click", {})' })
|
||||
await until(() => messages.some(message => message.type === 'invoke'))
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'nodeRepl.write("overlap")' })
|
||||
expect(messages.find(message => message.type === 'done' && message.cellId === 2).error).toContain('already running')
|
||||
const request = messages.find(message => message.type === 'invoke')
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: request.requestId, result: {} })
|
||||
await running
|
||||
expect(texts(messages)).toEqual([])
|
||||
})
|
||||
|
||||
test('does not retain a phantom request after a transport failure', async () => {
|
||||
const messages: Message[] = []
|
||||
const worker = createComputerUseReplWorker(message => {
|
||||
if (message.type === 'invoke') throw new Error('transport unavailable')
|
||||
messages.push(message)
|
||||
})
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: 'await __cuInvoke("click", {})' })
|
||||
expect(messages.find(message => message.type === 'done').error).toContain('transport unavailable')
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'nodeRepl.write("recovered")' })
|
||||
expect(texts(messages)).toEqual(['"recovered"'])
|
||||
})
|
||||
|
||||
test('cannot move a previous cell continuation into a later cell', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
await worker.receive({ type: 'run', cellId: 1, code: `
|
||||
let release
|
||||
const gate = new Promise(resolve => { release = resolve })
|
||||
gate.then(() => __cuInvoke('click', {stale: true})).catch(() => {})
|
||||
` })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'release(); await Promise.resolve(); nodeRepl.write("current")' })
|
||||
expect(messages.filter(message => message.type === 'invoke')).toEqual([])
|
||||
expect(texts(messages)).toEqual(['"current"'])
|
||||
})
|
||||
|
||||
test('provides the actual CUA bootstrap in the same persistent realm', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await worker.receive({ type: 'init', bootstrap: REPL_BOOTSTRAP_SOURCE })
|
||||
await worker.receive({ type: 'run', cellId: 1, code: 'let value = await Promise.resolve(4); nodeRepl.write(value)' })
|
||||
await worker.receive({ type: 'run', cellId: 2, code: 'nodeRepl.write([typeof cua.getApp, value])' })
|
||||
expect(texts(messages)).toHaveLength(2)
|
||||
expect(texts(messages).join(' ')).toContain('function')
|
||||
expect(messages.filter(message => message.type === 'done').every(message => !message.error)).toBe(true)
|
||||
})
|
||||
|
||||
test('responds to host health checks even while a cell awaits a UI response', async () => {
|
||||
const { worker, messages } = fixture()
|
||||
await init(worker)
|
||||
const running = worker.receive({ type: 'run', cellId: 1, code: 'await __cuInvoke("click", {})' })
|
||||
await until(() => messages.some(message => message.type === 'invoke'))
|
||||
await worker.receive({ type: 'ping', nonce: 42 })
|
||||
expect(messages.at(-1)).toEqual({ type: 'pong', nonce: 42 })
|
||||
const request = messages.find(message => message.type === 'invoke')
|
||||
await worker.receive({ type: 'response', cellId: 1, requestId: request.requestId, result: {} })
|
||||
await running
|
||||
})
|
||||
|
||||
test('runs the same persistent protocol in a separate process with a disposable home', async () => {
|
||||
const directory = await mkdtemp(join(tmpdir(), 'cu-repl-worker-'))
|
||||
const child = Bun.spawn([process.execPath, '--no-env-file', new URL('./replWorker.ts', import.meta.url).pathname], {
|
||||
cwd: directory,
|
||||
env: { HOME: directory, TMPDIR: directory, CLAUDE_CONFIG_DIR: directory },
|
||||
stdin: 'pipe',
|
||||
stdout: 'pipe',
|
||||
stderr: 'pipe',
|
||||
})
|
||||
const messages: Message[] = []
|
||||
const write = (message: Message) => child.stdin.write(`${JSON.stringify(message)}\n`)
|
||||
const reading = (async () => {
|
||||
let buffered = ''
|
||||
const decoder = new TextDecoder()
|
||||
for await (const bytes of child.stdout) {
|
||||
buffered += decoder.decode(bytes, { stream: true })
|
||||
let newline: number
|
||||
while ((newline = buffered.indexOf('\n')) !== -1) {
|
||||
const message = JSON.parse(buffered.slice(0, newline))
|
||||
buffered = buffered.slice(newline + 1)
|
||||
messages.push(message)
|
||||
if (message.type === 'invoke') {
|
||||
write({ type: 'response', cellId: message.cellId, requestId: message.requestId, result: { value: message.args.value } })
|
||||
}
|
||||
}
|
||||
}
|
||||
})()
|
||||
try {
|
||||
write({ type: 'init', bootstrap: '' })
|
||||
await until(() => messages.some(message => message.type === 'ready'))
|
||||
write({ type: 'ping', nonce: 7 })
|
||||
await until(() => messages.some(message => message.type === 'pong' && message.nonce === 7))
|
||||
write({ type: 'run', cellId: 1, code: 'let total = 2' })
|
||||
await until(() => messages.some(message => message.type === 'done' && message.cellId === 1))
|
||||
write({ type: 'run', cellId: 2, code: `
|
||||
for (const value of [3, 4]) { total += (await __cuInvoke('fixture', {value})).value }
|
||||
__cuEmit({type:'text', text:String(total)})
|
||||
` })
|
||||
await until(() => messages.some(message => message.type === 'done' && message.cellId === 2))
|
||||
expect(messages.filter(message => message.type === 'invoke').map(message => message.args)).toEqual([{ value: 3 }, { value: 4 }])
|
||||
expect(texts(messages)).toEqual(['9'])
|
||||
expect(messages.filter(message => message.type === 'done').every(message => !message.error)).toBe(true)
|
||||
child.stdin.end()
|
||||
await reading
|
||||
expect(await child.exited).toBe(0)
|
||||
} finally {
|
||||
child.kill()
|
||||
await child.exited
|
||||
await rm(directory, { recursive: true, force: true })
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,394 @@
|
||||
import { AsyncLocalStorage } from 'node:async_hooks'
|
||||
import { randomUUID } from 'node:crypto'
|
||||
import { createInterface } from 'node:readline'
|
||||
import { createContext, Script } from 'node:vm'
|
||||
import { compileReplCell, type ReplBinding } from './replCompiler'
|
||||
import type { ReplInput, ReplOutput } from '../../vendor/computer-use-mcp/replProtocol'
|
||||
|
||||
interface CellState {
|
||||
cellId: number
|
||||
accepting: boolean
|
||||
finished: boolean
|
||||
pending: Map<number, { settled: Promise<void>; settle: () => void }>
|
||||
}
|
||||
|
||||
interface RealmRuntime {
|
||||
begin(): void
|
||||
scope(writableConstants: string[]): unknown
|
||||
register(entries: unknown): void
|
||||
mark(...names: string[]): void
|
||||
commit(succeeded: boolean): string
|
||||
response(message: string): void
|
||||
unobservedError(): string | undefined
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
try {
|
||||
return String((error as { message?: unknown })?.message ?? error).slice(0, 8_000)
|
||||
} catch {
|
||||
return 'JavaScript execution failed'
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Persistent JavaScript kernel. Production runs this in a disposable process;
|
||||
* vm is a language boundary, not the OS sandbox or hard time/memory limit.
|
||||
* Only JSON strings cross its host bridge. In particular no host Promise,
|
||||
* Error, timers, module loader, or Node object is passed into the realm.
|
||||
*/
|
||||
export function createComputerUseReplWorker(send: (message: ReplOutput) => void) {
|
||||
const context = createContext(Object.create(null), {
|
||||
codeGeneration: { strings: false, wasm: false },
|
||||
})
|
||||
const execution = new AsyncLocalStorage<CellState>()
|
||||
let active: CellState | undefined
|
||||
let initialized = false
|
||||
let initializing = false
|
||||
let bindings: ReplBinding[] = []
|
||||
const bridgeName = `__cu_host_${randomUUID().replaceAll('-', '_')}`
|
||||
|
||||
context[bridgeName] = (serialized: string): string | undefined => {
|
||||
try {
|
||||
const message = JSON.parse(serialized)
|
||||
const cell = execution.getStore()
|
||||
if (!cell || cell.finished || cell !== active) {
|
||||
// Late output is discarded. Throwing here would turn an otherwise
|
||||
// handled rejection in detached user code into an unhandled rejection.
|
||||
if (message.type === 'emit') {
|
||||
return
|
||||
}
|
||||
return 'This JavaScript cell has ended; await every Computer Use operation'
|
||||
}
|
||||
if (message.type === 'emit') {
|
||||
send({ type: 'emit', cellId: cell.cellId, content: message.content })
|
||||
return
|
||||
}
|
||||
if (message.type !== 'invoke' || !cell.accepting) {
|
||||
return 'Submitted JavaScript has completed; await every Computer Use operation'
|
||||
}
|
||||
if (
|
||||
!Number.isSafeInteger(message.requestId)
|
||||
|| typeof message.name !== 'string'
|
||||
|| !message.args
|
||||
|| typeof message.args !== 'object'
|
||||
|| Array.isArray(message.args)
|
||||
) {
|
||||
return 'Invalid Computer Use bridge request'
|
||||
}
|
||||
if (cell.pending.has(message.requestId)) {
|
||||
return 'Duplicate Computer Use bridge request'
|
||||
}
|
||||
let settle!: () => void
|
||||
const settled = new Promise<void>(resolve => {
|
||||
settle = resolve
|
||||
})
|
||||
cell.pending.set(message.requestId, { settled, settle })
|
||||
// The drain waits for requests that were sent before the submitted cell
|
||||
// completed. New requests from detached continuations are rejected.
|
||||
try {
|
||||
send({
|
||||
type: 'invoke',
|
||||
cellId: cell.cellId,
|
||||
requestId: message.requestId,
|
||||
name: message.name,
|
||||
args: message.args,
|
||||
})
|
||||
} catch (error) {
|
||||
cell.pending.delete(message.requestId)
|
||||
settle()
|
||||
return errorMessage(error)
|
||||
}
|
||||
return
|
||||
} catch (error) {
|
||||
return errorMessage(error)
|
||||
}
|
||||
}
|
||||
|
||||
const realm = new Script(`(() => {
|
||||
const hostSend = globalThis[${JSON.stringify(bridgeName)}]
|
||||
delete globalThis[${JSON.stringify(bridgeName)}]
|
||||
const pending = new Map()
|
||||
let operations = []
|
||||
let requestCounter = 0
|
||||
const saved = new Map()
|
||||
const previous = new Map()
|
||||
const reached = new Set()
|
||||
let candidate = []
|
||||
const invoke = (name, args) => {
|
||||
const requestId = ++requestCounter
|
||||
const operation = {observed: false, error: undefined}
|
||||
const promise = new Promise((resolve, reject) => {
|
||||
pending.set(requestId, {resolve, reject, operation})
|
||||
let failure
|
||||
try {
|
||||
failure = hostSend(JSON.stringify({type: 'invoke', requestId, name, args}))
|
||||
} catch {
|
||||
failure = 'Could not serialize Computer Use arguments'
|
||||
}
|
||||
if (failure) {
|
||||
pending.delete(requestId)
|
||||
reject(new Error(failure))
|
||||
} else {
|
||||
operations.push(operation)
|
||||
}
|
||||
})
|
||||
promise.catch(() => {})
|
||||
// Track whether submitted code consumes this operation. A separate
|
||||
// successful await must not hide an earlier ignored native failure.
|
||||
return {
|
||||
then(onFulfilled, onRejected) {
|
||||
operation.observed = true
|
||||
return promise.then(onFulfilled, onRejected)
|
||||
},
|
||||
catch(onRejected) {
|
||||
operation.observed = true
|
||||
return promise.catch(onRejected)
|
||||
},
|
||||
finally(onFinally) {
|
||||
operation.observed = true
|
||||
return promise.finally(onFinally)
|
||||
},
|
||||
}
|
||||
}
|
||||
const emit = content => {
|
||||
const failure = hostSend(JSON.stringify({type: 'emit', content}))
|
||||
if (failure) {
|
||||
throw new Error(failure)
|
||||
}
|
||||
}
|
||||
Object.defineProperties(globalThis, {
|
||||
__cuInvoke: {value: invoke},
|
||||
__cuEmit: {value: emit},
|
||||
})
|
||||
return {
|
||||
begin() {
|
||||
operations = []
|
||||
reached.clear()
|
||||
previous.clear()
|
||||
candidate = []
|
||||
},
|
||||
scope(writableConstants) {
|
||||
const writable = new Set(writableConstants)
|
||||
const values = new Proxy(Object.create(null), {
|
||||
get(_target, name) {
|
||||
const entry = saved.get(name)
|
||||
if (!entry && Reflect.has(globalThis, name)) return Reflect.get(globalThis, name)
|
||||
if (!entry) throw new ReferenceError(String(name) + ' is not defined')
|
||||
return entry.get()
|
||||
},
|
||||
set(_target, name, value) {
|
||||
const entry = saved.get(name)
|
||||
if (!entry) {
|
||||
if (!Reflect.has(globalThis, name)) throw new ReferenceError(String(name) + ' is not defined')
|
||||
if (!Reflect.set(globalThis, name, value)) throw new TypeError('Assignment to readonly global.')
|
||||
return true
|
||||
}
|
||||
if (entry.kind === 'const' && writable.has(name)) {
|
||||
// A prior cell's const may be reassigned with the compiler's
|
||||
// warning. Keep the shared slot, including readers in closures.
|
||||
entry.get()
|
||||
entry.get = () => value
|
||||
entry.set = next => { value = next }
|
||||
} else {
|
||||
if (entry.kind === 'const') {
|
||||
entry.get() // Preserve TDZ before the immutable-binding error.
|
||||
throw new TypeError('Assignment to constant variable.')
|
||||
}
|
||||
entry.set(value)
|
||||
}
|
||||
return true
|
||||
},
|
||||
})
|
||||
return {
|
||||
values,
|
||||
typeOf(name) {
|
||||
if (!saved.has(name) && !Reflect.has(globalThis, name)) return 'undefined'
|
||||
return typeof values[name]
|
||||
},
|
||||
}
|
||||
},
|
||||
register(entries) {
|
||||
candidate = entries
|
||||
for (const [name, kind, get, set] of entries) {
|
||||
previous.set(name, saved.get(name))
|
||||
saved.set(name, {kind, get, set})
|
||||
}
|
||||
},
|
||||
mark(...names) {
|
||||
for (const name of names) {
|
||||
reached.add(name)
|
||||
}
|
||||
},
|
||||
commit(succeeded) {
|
||||
for (const [name, _kind, getter, _setter, newlyHoisted] of candidate) {
|
||||
let retain = succeeded || !newlyHoisted || reached.has(name)
|
||||
try {
|
||||
getter()
|
||||
} catch {
|
||||
retain = false // The declaration was not initialized before failure.
|
||||
}
|
||||
if (!retain) {
|
||||
const old = previous.get(name)
|
||||
if (old) saved.set(name, old)
|
||||
else saved.delete(name)
|
||||
}
|
||||
}
|
||||
candidate = []
|
||||
previous.clear()
|
||||
return JSON.stringify([...saved].map(([name, entry]) => ({name, kind: entry.kind})))
|
||||
},
|
||||
response(serialized) {
|
||||
const message = JSON.parse(serialized)
|
||||
const entry = pending.get(message.requestId)
|
||||
if (!entry) {
|
||||
return
|
||||
}
|
||||
pending.delete(message.requestId)
|
||||
if (typeof message.error === 'string') {
|
||||
entry.operation.error = message.error
|
||||
entry.reject(new Error(message.error))
|
||||
} else {
|
||||
if (message.result?.isError === true) {
|
||||
const text = Array.isArray(message.result.content)
|
||||
? message.result.content.filter(item => item?.type === 'text' && typeof item.text === 'string').map(item => item.text).join('\\n')
|
||||
: ''
|
||||
entry.operation.error = text || 'Computer Use action failed.'
|
||||
}
|
||||
entry.resolve(message.result)
|
||||
}
|
||||
},
|
||||
unobservedError() {
|
||||
const failed = operations.find(operation => !operation.observed && typeof operation.error === 'string')
|
||||
return failed?.error
|
||||
},
|
||||
}
|
||||
})()`).runInContext(context) as RealmRuntime
|
||||
|
||||
async function run(cellId: number, code: string, bootstrap = false) {
|
||||
if (active) {
|
||||
send({ type: 'done', cellId, error: 'A JavaScript cell is already running' })
|
||||
return
|
||||
}
|
||||
const cell: CellState = { cellId, accepting: true, finished: false, pending: new Map() }
|
||||
active = cell
|
||||
realm.begin()
|
||||
let error: string | undefined
|
||||
let submittedSucceeded = false
|
||||
await execution.run(cell, async () => {
|
||||
try {
|
||||
const compiled = compileReplCell(code, bindings)
|
||||
const execute = new Script(compiled.source, { filename: 'computer-use-cell.js' }).runInContext(context)
|
||||
for (const warning of compiled.warnings) {
|
||||
send({ type: 'emit', cellId, content: { type: 'text', text: `Warning: ${warning}` } })
|
||||
}
|
||||
await execute(realm.scope, realm.register, realm.mark)
|
||||
submittedSucceeded = true
|
||||
} catch (failure) {
|
||||
error = errorMessage(failure)
|
||||
} finally {
|
||||
cell.accepting = false
|
||||
// A response can settle a pending call on another stdin callback. Keep
|
||||
// this cell alive until those calls finish, without reopening dispatch.
|
||||
while (cell.pending.size > 0) {
|
||||
await Promise.all([...cell.pending.values()].map(request => request.settled))
|
||||
}
|
||||
// The facade awaits its bridge internally, but a caller can still
|
||||
// ignore the outer async App-method promise. Its rejection is reported
|
||||
// by the host only after microtasks finish. Do not publish success
|
||||
// before that checkpoint; the process-level rejection handler resets
|
||||
// the kernel while the parent still considers this cell active.
|
||||
// A detached infinite microtask chain also remains under its deadline.
|
||||
await new Promise<void>(resolve => setImmediate(resolve))
|
||||
const backgroundError = realm.unobservedError()
|
||||
if (!error && backgroundError) {
|
||||
error = `An unawaited Computer Use operation failed: ${backgroundError}. Observe the current state before continuing; do not replay prior actions.`
|
||||
}
|
||||
bindings = JSON.parse(realm.commit(submittedSucceeded)) as ReplBinding[]
|
||||
cell.finished = true
|
||||
active = undefined
|
||||
}
|
||||
})
|
||||
if (bootstrap) {
|
||||
if (error) {
|
||||
throw new Error(error)
|
||||
}
|
||||
} else {
|
||||
send({ type: 'done', cellId, ...(error ? { error } : {}) })
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
async receive(message: ReplInput): Promise<void> {
|
||||
if (message.type === 'ping') {
|
||||
send({ type: 'pong', nonce: message.nonce })
|
||||
return
|
||||
}
|
||||
if (message.type === 'init') {
|
||||
if (initialized || initializing) {
|
||||
throw new Error('Computer Use JavaScript worker is already initialized')
|
||||
}
|
||||
initializing = true
|
||||
try {
|
||||
await run(0, message.bootstrap, true)
|
||||
initialized = true
|
||||
send({ type: 'ready' })
|
||||
} finally {
|
||||
initializing = false
|
||||
}
|
||||
return
|
||||
}
|
||||
if (message.type === 'response') {
|
||||
const cell = active
|
||||
if (!cell || cell.cellId !== message.cellId) {
|
||||
return
|
||||
}
|
||||
const pending = cell.pending.get(message.requestId)
|
||||
if (!pending) {
|
||||
return
|
||||
}
|
||||
cell.pending.delete(message.requestId)
|
||||
realm.response(JSON.stringify(message))
|
||||
pending.settle()
|
||||
return
|
||||
}
|
||||
if (!initialized) {
|
||||
send({ type: 'done', cellId: message.cellId, error: 'Computer Use JavaScript worker is not initialized' })
|
||||
return
|
||||
}
|
||||
await run(message.cellId, message.code)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export async function runComputerUseReplWorker(): Promise<void> {
|
||||
const stopAfterAsyncFailure = () => {
|
||||
process.stderr.write('Computer Use JavaScript worker stopped after an unhandled asynchronous error.\n')
|
||||
process.exit(1)
|
||||
}
|
||||
process.on('unhandledRejection', stopAfterAsyncFailure)
|
||||
process.on('uncaughtException', stopAfterAsyncFailure)
|
||||
process.stdout.on('error', error => {
|
||||
process.exit((error as NodeJS.ErrnoException).code === 'EPIPE' ? 0 : 1)
|
||||
})
|
||||
const worker = createComputerUseReplWorker(message => {
|
||||
process.stdout.write(`${JSON.stringify(message)}\n`)
|
||||
})
|
||||
const input = createInterface({ input: process.stdin, crlfDelay: Infinity })
|
||||
for await (const line of input) {
|
||||
try {
|
||||
const message = JSON.parse(line) as ReplInput
|
||||
// Do not block stdin: responses resolve the currently running cell.
|
||||
void worker.receive(message).catch(() => process.exit(1))
|
||||
} catch {
|
||||
process.exitCode = 1
|
||||
input.close()
|
||||
break
|
||||
}
|
||||
}
|
||||
// The parent owns the kernel. A closed input pipe cannot service new calls.
|
||||
process.exit(process.exitCode || 0)
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
await runComputerUseReplWorker()
|
||||
}
|
||||
@@ -17,8 +17,7 @@ describe('setupComputerUseMCP runtime capability', () => {
|
||||
})
|
||||
|
||||
expect(Object.keys(result.mcpConfig)).toEqual(['computer-use'])
|
||||
expect(result.allowedTools).toContain('mcp__computer-use__get_app_state')
|
||||
expect(result.allowedTools).not.toContain('mcp__computer-use__screenshot')
|
||||
expect(result.allowedTools).toEqual(['mcp__computer-use__js', 'mcp__computer-use__js_reset'])
|
||||
})
|
||||
|
||||
test('keeps the Windows compatibility engine available without a macOS helper', () => {
|
||||
@@ -32,6 +31,8 @@ describe('setupComputerUseMCP runtime capability', () => {
|
||||
expect(Object.keys(result.mcpConfig)).toEqual(['computer-use'])
|
||||
expect(result.allowedTools).toContain('mcp__computer-use__screenshot')
|
||||
expect(result.allowedTools).not.toContain('mcp__computer-use__get_app_state')
|
||||
expect(result.allowedTools).not.toContain('mcp__computer-use__sequence')
|
||||
expect(result.allowedTools).not.toContain('mcp__computer-use__js')
|
||||
})
|
||||
|
||||
test('does not resolve a helper or advertise tools on unsupported platforms', () => {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { buildSessionContext } from './wrapper.js'
|
||||
import { buildSessionContext, createComputerUseEscapeHandler, dispatchComputerUseCall } from './wrapper.js'
|
||||
import type { ToolUseContext } from '../../Tool.js'
|
||||
|
||||
describe('Computer Use session authorization', () => {
|
||||
test('enables every supported app without exposing a runtime permission callback', () => {
|
||||
@@ -15,3 +16,81 @@ describe('Computer Use session authorization', () => {
|
||||
expect(context.onPermissionRequest).toBeUndefined()
|
||||
})
|
||||
})
|
||||
|
||||
describe('Computer Use CLI dispatch boundary', () => {
|
||||
const context = () => ({ abortController: new AbortController() }) as ToolUseContext
|
||||
|
||||
test('pins cancellation to this call rather than the latest queued context', async () => {
|
||||
const callContext = context()
|
||||
const result = await dispatchComputerUseCall(async (_tool, _args, signal) => {
|
||||
expect(signal).toBe(callContext.abortController.signal)
|
||||
callContext.abortController.abort()
|
||||
expect(signal?.aborted).toBe(true)
|
||||
return { content: [{ type: 'text', text: 'cancelled' }] }
|
||||
}, 'sequence', { app: 'Finder', steps: [] }, callContext)
|
||||
expect(result.data).toEqual([{ type: 'text', text: 'cancelled' }])
|
||||
})
|
||||
|
||||
test('preserves partial sequence progress for both the model and SDK', async () => {
|
||||
const summary = { status: 'completed', completedSteps: 2, totalSteps: 2 }
|
||||
const result = await dispatchComputerUseCall(async () => ({
|
||||
structuredContent: summary,
|
||||
content: [{ type: 'text', text: 'Current app state' }],
|
||||
}), 'sequence', {}, context())
|
||||
expect(result.data).toContainEqual({ type: 'text', text: JSON.stringify(summary) })
|
||||
expect(result.mcpMeta?.structuredContent).toEqual(summary)
|
||||
})
|
||||
|
||||
test('a failed sequence remains a tool error with the completed-step evidence', async () => {
|
||||
const summary = { status: 'failed', completedSteps: 1, failedStepIndex: 1, resultUnknown: true }
|
||||
try {
|
||||
await dispatchComputerUseCall(async () => ({
|
||||
isError: true,
|
||||
structuredContent: summary,
|
||||
content: [{ type: 'text', text: 'Inspect state before retrying; do not replay completed steps.' }],
|
||||
}), 'sequence', {}, context())
|
||||
throw new Error('failed tool was returned as success')
|
||||
} catch (error) {
|
||||
expect(String(error)).toContain(JSON.stringify(summary))
|
||||
expect(String(error)).toContain('do not replay completed steps')
|
||||
}
|
||||
})
|
||||
})
|
||||
|
||||
test('queued CLI calls keep their AppState accessors isolated', async () => {
|
||||
let release!: () => void
|
||||
const waiting = new Promise<void>(resolve => { release = resolve })
|
||||
const firstContext = {
|
||||
abortController: new AbortController(),
|
||||
getAppState: () => ({ computerUseMcpState: { selectedDisplayId: 1 } }),
|
||||
} as ToolUseContext
|
||||
const secondContext = {
|
||||
abortController: new AbortController(),
|
||||
getAppState: () => ({ computerUseMcpState: { selectedDisplayId: 2 } }),
|
||||
} as ToolUseContext
|
||||
const session = buildSessionContext()
|
||||
const first = dispatchComputerUseCall(async () => {
|
||||
await waiting
|
||||
return { content: [{ type: 'text', text: String(session.getSelectedDisplayId()) }] }
|
||||
}, 'sequence', {}, firstContext)
|
||||
const second = dispatchComputerUseCall(async () => ({
|
||||
content: [{ type: 'text', text: String(session.getSelectedDisplayId()) }],
|
||||
}), 'click', {}, secondContext)
|
||||
expect((await second).data).toEqual([{ type: 'text', text: '2' }])
|
||||
release()
|
||||
expect((await first).data).toEqual([{ type: 'text', text: '1' }])
|
||||
})
|
||||
|
||||
|
||||
test('a host Escape callback aborts its turn outside any dispatch async context', async () => {
|
||||
const turnController = new AbortController()
|
||||
const nextController = new AbortController()
|
||||
const onEscape = createComputerUseEscapeHandler(turnController)
|
||||
await dispatchComputerUseCall(async () => ({ content: [] }), 'click', {}, {
|
||||
abortController: nextController,
|
||||
} as ToolUseContext)
|
||||
// Simulate a host callback delivered after the promise context has gone away.
|
||||
onEscape()
|
||||
expect(turnController.signal.aborted).toBe(true)
|
||||
expect(nextController.signal.aborted).toBe(false)
|
||||
})
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -3,6 +3,7 @@ import { readFileSync } from 'node:fs'
|
||||
import { resolve } from 'node:path'
|
||||
|
||||
import { _test as deniedApps } from './deniedApps.js'
|
||||
import { NATIVE_FORBIDDEN_BUNDLE_IDS } from './nativeAppPolicy.js'
|
||||
|
||||
const SWIFT_SET_MARKER = 'static let deniedBundleIDs: Set<String> = ['
|
||||
const SWIFT_INTRINSIC_SET_MARKER = 'static let intrinsicDeniedBundleIDs: Set<String> = ['
|
||||
@@ -19,13 +20,8 @@ function parseNativeDeniedBundleIds(source: string, marker: string): string[] {
|
||||
return [...body.matchAll(/^\s*"([^"]+)",?\s*(?:\/\/.*)?$/gm)].map(match => match[1])
|
||||
}
|
||||
|
||||
test('native AppTargetPolicy deny set stays in exact parity with deniedApps bundle sets', () => {
|
||||
const tsEntries = [
|
||||
deniedApps.BROWSER_BUNDLE_IDS,
|
||||
deniedApps.TERMINAL_BUNDLE_IDS,
|
||||
deniedApps.TRADING_BUNDLE_IDS,
|
||||
deniedApps.POLICY_DENIED_BUNDLE_IDS,
|
||||
].flatMap(entries => [...entries])
|
||||
test('native deny policy matches the official 24 exact forbidden identities', () => {
|
||||
const tsEntries = [...NATIVE_FORBIDDEN_BUNDLE_IDS]
|
||||
const expected = new Set(tsEntries)
|
||||
|
||||
const swiftPath = resolve(
|
||||
@@ -43,11 +39,15 @@ test('native AppTargetPolicy deny set stays in exact parity with deniedApps bund
|
||||
|
||||
expect(tsEntries).toHaveLength(expected.size)
|
||||
expect(nativeEntries).toHaveLength(actual.size)
|
||||
expect(expected.size).toBe(107)
|
||||
expect(actual.size).toBe(107)
|
||||
expect(expected.size).toBe(24)
|
||||
expect(actual.size).toBe(24)
|
||||
expect(missing).toEqual([])
|
||||
expect(extra).toEqual([])
|
||||
expect([...actual].sort()).toEqual([...expected].sort())
|
||||
// Browser classification still exists for the Windows tool tier. Native
|
||||
// macOS control may use the same browser as the official Codex app surface.
|
||||
expect(deniedApps.BROWSER_BUNDLE_IDS.size).toBe(29)
|
||||
expect([...deniedApps.BROWSER_BUNDLE_IDS].filter(bundleId => actual.has(bundleId))).toEqual([])
|
||||
})
|
||||
|
||||
test('native intrinsic deny set stays separate and matches the TS host/helper defaults', () => {
|
||||
|
||||
+21
-2
@@ -32,6 +32,20 @@ export interface RunningApp {
|
||||
displayName: string
|
||||
}
|
||||
|
||||
/** Native macOS inventory, preserving optional usage and running metadata. */
|
||||
export interface NativeAppInfo {
|
||||
id: string
|
||||
displayName?: string
|
||||
isRunning?: boolean
|
||||
lastUsedDate?: string
|
||||
useCount?: number
|
||||
}
|
||||
|
||||
export function formatNativeAppList(apps: readonly NativeAppInfo[]): string {
|
||||
if (apps.length === 0) return 'No running applications are available to control.'
|
||||
return apps.map(app => `${app.displayName ?? app.id} — ${app.id}`).join('\n')
|
||||
}
|
||||
|
||||
// ----------------------------------------------------------------------------
|
||||
// Codex semantic engine — AX-tree-aware app state + element-indexed injection.
|
||||
//
|
||||
@@ -47,12 +61,15 @@ export interface RunningApp {
|
||||
// is the Swift format authority; see blueprint §1).
|
||||
// ----------------------------------------------------------------------------
|
||||
|
||||
/** A captured window screenshot returned alongside an app-state snapshot.
|
||||
* Base64 PNG in capture-pixel space (left-top origin, scaled per blueprint §5). */
|
||||
/** A captured window screenshot in capture-pixel space (left-top origin). */
|
||||
export interface AppStateScreenshot {
|
||||
base64: string
|
||||
width: number
|
||||
height: number
|
||||
/** Older helpers omit this and return PNG. */
|
||||
mimeType?: 'image/png' | 'image/jpeg'
|
||||
/** Uniform capture scale before encoded pixel dimensions are rounded up. */
|
||||
pixelsPerPoint?: number
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -151,6 +168,8 @@ export interface ResolvedAppTarget {
|
||||
export interface CodexComputerEngine {
|
||||
/** Enumerate running/recent apps as the `list_apps` text block expects. */
|
||||
listApps(): Promise<string>
|
||||
/** Structured inventory for JavaScript clients; legacy engines may omit it. */
|
||||
listAppsInfo?(): Promise<NativeAppInfo[]>
|
||||
/**
|
||||
* Resolve a loose selector to one running process plus its lifetime identity.
|
||||
* Never launches anything — a selector that matches nothing is an error, not
|
||||
|
||||
+75
-35
@@ -13,48 +13,88 @@
|
||||
* - inserted its own waits between an action and the next state read;
|
||||
* - and finally abandoned the toolset for `osascript` and Python.
|
||||
*
|
||||
* None of that is fixed by a better tool description: it is workflow knowledge —
|
||||
* what to do first, when a route is a dead end, how to tell whether an action
|
||||
* landed. Codex ships exactly this as a skill document its model reads before
|
||||
* acting; the MCP protocol has the same slot (`ServerOptions.instructions`,
|
||||
* delivered in the initialize handshake) and we were leaving it empty.
|
||||
* This is our operating guidance for the native JavaScript and semantic tools.
|
||||
* The facade follows the inspected official native App API; it does not expose
|
||||
* the official browser provider or general Node runtime.
|
||||
* The bundled skill is the enabled product entry point; this export also serves
|
||||
* hosts and tests that consume the guidance directly.
|
||||
*
|
||||
* Keep this SHORT and behavioural. Every line should change what the model does
|
||||
* at a specific decision point — this text is paid for on every session, and a
|
||||
* paragraph the model cannot act on is pure cost.
|
||||
*/
|
||||
/** Shared with the bundled skill so both entry points teach the same observation boundary. */
|
||||
export const COMPUTER_USE_BATCHING_GUIDANCE = `## Persistent JavaScript and known actions
|
||||
|
||||
Prefer \`js({code})\` for native app work. Start with
|
||||
\`var app = await cua.getApp("App Name")\`: it displays initial AX state and, once
|
||||
per session, App API guidance. Variables, App bindings, calculations, loops, and
|
||||
top-level await work across cells. Do not force one model round trip per click.
|
||||
Await every action. In a stable canvas, after observing the actual coordinates,
|
||||
one cell can perform known drags and inspect their result:
|
||||
|
||||
\`\`\`javascript
|
||||
for (const x of [240, 280]) await app.drag([x, 320], [x + 1, 320])
|
||||
await app.getAXStateAndScreenshot()
|
||||
\`\`\`
|
||||
|
||||
Use \`app.getAXState()\` for text, \`app.getScreenshot()\` for an image, or
|
||||
\`app.getAXStateAndScreenshot()\` for both. \`{emit:false}\` returns data without
|
||||
displaying it; use \`nodeRepl.write(value)\` or \`await nodeRepl.emitImage(bytes)\`
|
||||
when needed. These methods select output; all still use the same native capture.
|
||||
Use copied \`gN:id\` handles for element actions. Integer indices map only to
|
||||
observed handles; after image-only capture, call
|
||||
\`app.getAXState({disableDiffing:true})\` before using integers again.
|
||||
|
||||
Batch only while prerequisite state remains known. An unfamiliar menu, dialog,
|
||||
navigation, changed layout, or any error requires you to stop and re-observe.
|
||||
A click may do nothing in some canvas apps; after inspecting that result, a short
|
||||
0–1 pixel drag can be an alternative. Inspect before repeating the strategy.
|
||||
Do not add a fixed sleep before observing, or blindly replay a partial batch.
|
||||
|
||||
Each JS cell allows 256 native calls, 256 KiB of code, and 128 output blocks up
|
||||
to 16 MiB. \`timeout_ms\` defaults to 30000 and cannot exceed 60000. Ordinary
|
||||
script errors retain bindings. Timeout, cancellation, and \`js_reset\` discard
|
||||
them: select the app again, observe, and account for actions that already ran.
|
||||
Browser/DOM APIs, imports, Node, filesystem, and networking are unavailable.
|
||||
|
||||
Only \`js\` and \`js_reset\` are advertised on macOS. The older semantic tools and
|
||||
\`sequence\` remain compatibility interfaces for existing clients; use the App
|
||||
methods above in this session.
|
||||
`.trim()
|
||||
|
||||
export const COMPUTER_USE_INSTRUCTIONS = `
|
||||
Operate macOS apps through the accessibility engine. Read this before your first action.
|
||||
|
||||
## Loop
|
||||
|
||||
1. \`get_app_state({ app })\` — returns the app's accessibility tree AND a screenshot
|
||||
of its window. It launches the app in the background if it is not running, so
|
||||
there is no separate "open" step.
|
||||
2. Act.
|
||||
3. \`get_app_state\` again before deciding what to do next. Element handles are only
|
||||
valid for the snapshot they came from; re-read to get fresh ones.
|
||||
1. Use \`js\` to bind an app with \`cua.getApp("App Name")\` and read its initial
|
||||
AX state. Request \`app.getScreenshot()\` when visual information is needed.
|
||||
2. Perform known actions in the same JS session, then observe at a decision point.
|
||||
3. Inspect state and screenshots before choosing further actions.
|
||||
|
||||
Do not sleep between an action and the next \`get_app_state\`. The engine already
|
||||
waits for the UI to settle (about a second, longer while the app shows a progress
|
||||
indicator).
|
||||
${COMPUTER_USE_BATCHING_GUIDANCE}
|
||||
|
||||
Do not add a fixed sleep before an observation. The observation path waits
|
||||
for UI changes when needed. Read its result before deciding whether more context
|
||||
is necessary.
|
||||
|
||||
## Naming the app
|
||||
|
||||
Pass the app name directly — display name, bundle identifier, or full path all work.
|
||||
Do NOT call \`list_apps\` just to look up an identifier; try \`get_app_state({ app: "Safari" })\`
|
||||
first. If a call fails by display name, retry the same call with the bundle
|
||||
identifier before investigating anything else. Use \`list_apps\` only when you
|
||||
Pass the app name directly to \`cua.getApp\` — display name, bundle identifier,
|
||||
or full path all work. Do NOT call \`cua.listApps()\` just to look up an identifier.
|
||||
If a call fails by display name, retry the same call with the bundle
|
||||
identifier before investigating anything else. Use \`cua.listApps()\` only when you
|
||||
genuinely cannot name the app.
|
||||
|
||||
## Choosing between element handles and coordinates
|
||||
|
||||
Prefer \`element_index\` when the thing you want is actually in the tree: it targets
|
||||
the element directly and survives the window moving.
|
||||
Prefer \`app.click("gN:id")\` when the control is
|
||||
actually in the tree: it targets the element directly and survives window movement.
|
||||
|
||||
Switch to \`x\`/\`y\` read off the screenshot when either is true:
|
||||
Switch to \`[x,y]\` read off the screenshot when either is true:
|
||||
- the tree does not contain what you need (many Chromium/Electron apps expose
|
||||
only their window frame and menu bar — \`get_app_state\` says so explicitly when
|
||||
only their window frame and menu bar — AX state says so explicitly when
|
||||
it detects this), or
|
||||
- element actions run but the UI does not change.
|
||||
|
||||
@@ -63,15 +103,15 @@ as-is. Do not convert them.
|
||||
|
||||
## Telling whether an action worked
|
||||
|
||||
Mutating tools return a fixed receipt. The receipt means "the action was
|
||||
dispatched", NOT "it had the intended effect" — you must look at the next
|
||||
\`get_app_state\` to know.
|
||||
Awaiting a JS action or receiving a standalone dispatch receipt means the action
|
||||
was dispatched, NOT that it had the intended effect. Inspect a fresh observation
|
||||
from the JS App to know.
|
||||
|
||||
Judge success from the screenshot as well as the AX text. An empty AX diff does
|
||||
not mean a Chromium/CEF interface stayed unchanged. If two consecutive screenshots
|
||||
leave the relevant UI unchanged, the approach is wrong.
|
||||
Change something real: switch from element handle to coordinates, target a
|
||||
different element, re-read the full tree with \`disableDiff: true\`, or take a
|
||||
different element, re-read the full tree with \`app.getAXState({disableDiffing:true})\`, or take a
|
||||
different route through the UI. Repeating the same call a third time never helps.
|
||||
|
||||
If you cannot make progress after a few genuinely different attempts, say so and
|
||||
@@ -81,18 +121,18 @@ waste the user's time.
|
||||
|
||||
## Tool notes
|
||||
|
||||
- \`get_app_state\` returns a diff against the previous read by default. Pass
|
||||
\`disableDiff: true\` when you need the full tree — for example after acting on a
|
||||
- \`app.getAXState()\` returns a diff against the previous read by default. Pass
|
||||
\`{disableDiffing:true}\` when you need the full tree — for example after acting on a
|
||||
screenshot alone, or when the diff has left you unsure of the current state.
|
||||
- \`perform_secondary_action\` only accepts an action actually listed for that
|
||||
- \`app.performSecondaryAction(element,action)\` only accepts an action actually listed for that
|
||||
element in the tree. Do not guess action names.
|
||||
- \`press_key\` and \`type_text\` are delivered to the named app, so they cannot
|
||||
- \`app.pressKey\` and \`app.typeText\` are delivered to the named app, so they cannot
|
||||
trigger global system shortcuts.
|
||||
- If \`type_text\` does not visually change a Chromium/CEF field, use
|
||||
\`paste({ app, text, format: "text" })\`; it restores the user's prior clipboard.
|
||||
- If \`app.typeText\` does not visually change a Chromium/CEF field, use
|
||||
\`app.paste(text,{format:"text"})\`; it restores the user's prior clipboard.
|
||||
If paste times out after dispatch, treat the result as unknown and call
|
||||
\`get_app_state\` before retrying, because the target may have consumed it late.
|
||||
- \`press_key\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c".
|
||||
- \`select_text\` works inside editable elements; use \`prefix\`/\`suffix\` to
|
||||
\`app.getAXStateAndScreenshot()\` after rebinding before retrying, because the target may have consumed it late.
|
||||
- \`app.pressKey\` uses xdotool key names: "a", "Return", "Tab", "Up", "super+c".
|
||||
- \`app.selectText\` works inside editable elements; use \`prefix\`/\`suffix\` to
|
||||
disambiguate repeated matches.
|
||||
`.trim()
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { expect, test } from 'bun:test'
|
||||
import { isSystemKeyCombo, normalizeKeySequence } from './keyBlocklist.js'
|
||||
|
||||
test('native left and right modifier aliases pass through the same shortcut grant gate', () => {
|
||||
for (const command of ['Super_L', 'Super_R', 'Meta_L', 'Meta_R']) {
|
||||
expect(isSystemKeyCombo(`${command}+q`, 'darwin')).toBe(true)
|
||||
expect(isSystemKeyCombo(`Control_R+${command}+q`, 'darwin')).toBe(true)
|
||||
expect(isSystemKeyCombo(`Shift_L+${command}+Tab`, 'darwin')).toBe(true)
|
||||
expect(isSystemKeyCombo(`${command}+Alt_R+Escape`, 'darwin')).toBe(true)
|
||||
expect(isSystemKeyCombo(`${command}+c`, 'darwin')).toBe(false)
|
||||
}
|
||||
expect(normalizeKeySequence('Control_L+Alt_L+Shift_R+Meta_R+a')).toBe('ctrl+alt+shift+meta+a')
|
||||
})
|
||||
+10
@@ -22,7 +22,11 @@
|
||||
const CANONICAL_MODIFIER: Readonly<Record<string, string>> = {
|
||||
// Key::Meta — "meta"|"super"|"command"|"cmd"|"windows"|"win"
|
||||
meta: "meta",
|
||||
meta_l: 'meta',
|
||||
meta_r: 'meta',
|
||||
super: "meta",
|
||||
super_l: 'meta',
|
||||
super_r: 'meta',
|
||||
command: "meta",
|
||||
cmd: "meta",
|
||||
windows: "meta",
|
||||
@@ -30,18 +34,24 @@ const CANONICAL_MODIFIER: Readonly<Record<string, string>> = {
|
||||
// Key::Control + LControl + RControl
|
||||
ctrl: "ctrl",
|
||||
control: "ctrl",
|
||||
control_l: 'ctrl',
|
||||
control_r: 'ctrl',
|
||||
lctrl: "ctrl",
|
||||
lcontrol: "ctrl",
|
||||
rctrl: "ctrl",
|
||||
rcontrol: "ctrl",
|
||||
// Key::Shift + LShift + RShift
|
||||
shift: "shift",
|
||||
shift_l: 'shift',
|
||||
shift_r: 'shift',
|
||||
lshift: "shift",
|
||||
rshift: "shift",
|
||||
// Key::Alt and Key::Option — distinct Rust variants but same keycode on
|
||||
// darwin (kVK_Option). Collapse: cmd+alt+escape and cmd+option+escape
|
||||
// both Force Quit.
|
||||
alt: "alt",
|
||||
alt_l: 'alt',
|
||||
alt_r: 'alt',
|
||||
option: "alt",
|
||||
opt: "alt",
|
||||
};
|
||||
|
||||
+124
-5
@@ -26,17 +26,25 @@ import {
|
||||
|
||||
import type { ScreenshotResult } from "./executor.js";
|
||||
import type { CuCallToolResult } from "./toolCalls.js";
|
||||
import { NATIVE_ERROR } from './nativeError.js'
|
||||
import {
|
||||
APP_INVENTORY,
|
||||
NATIVE_CALL_NOT_DISPATCHED,
|
||||
defersLockAcquire,
|
||||
handleToolCall,
|
||||
resetMouseButtonHeld,
|
||||
RESOLVED_APP_PATH,
|
||||
staticRequestError,
|
||||
} from "./toolCalls.js";
|
||||
import { buildComputerUseTools } from "./tools.js";
|
||||
import {
|
||||
defersLockAcquire as legacyDefersLockAcquire,
|
||||
handleToolCall as legacyHandleToolCall,
|
||||
hasHeldMouseForSession,
|
||||
releaseHeldMouseForSession,
|
||||
resetMouseButtonHeld as legacyResetMouseButtonHeld,
|
||||
WINDOWS_MOUSE_OWNER,
|
||||
type WindowsMouseOwner,
|
||||
} from "./windowsLegacyToolCalls.js";
|
||||
import { buildComputerUseTools as buildLegacyComputerUseTools } from "./windowsLegacyTools.js";
|
||||
import type {
|
||||
@@ -49,6 +57,7 @@ import type {
|
||||
CuPermissionResponse,
|
||||
} from "./types.js";
|
||||
import { DEFAULT_GRANT_FLAGS } from "./types.js";
|
||||
import { REPL_MAX_CODE_BYTES, type ComputerUseReplRuntime } from './replProtocol.js'
|
||||
|
||||
const DEFAULT_LOCK_HELD_MESSAGE =
|
||||
"Another Claude session is currently using the computer. Wait for that " +
|
||||
@@ -103,7 +112,8 @@ export function buildPlatformComputerUseTools(
|
||||
): Tool[] {
|
||||
return caps.platform === "win32"
|
||||
? buildLegacyComputerUseTools(caps, coordinateMode, installedAppNames)
|
||||
: buildComputerUseTools(caps, coordinateMode, installedAppNames);
|
||||
: buildComputerUseTools(caps, coordinateMode, installedAppNames)
|
||||
.filter(tool => tool.name === 'js' || tool.name === 'js_reset');
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -120,12 +130,15 @@ export function bindSessionContext(
|
||||
adapter: ComputerUseHostAdapter,
|
||||
coordinateMode: CoordinateMode,
|
||||
ctx: ComputerUseSessionContext,
|
||||
): (name: string, args: unknown) => Promise<CuCallToolResult> {
|
||||
): (name: string, args: unknown, signal?: AbortSignal) => Promise<CuCallToolResult> {
|
||||
const { logger, serverName } = adapter;
|
||||
|
||||
// Screenshot blob persists here across calls — NOT on `ctx`. Hosts hold
|
||||
// onto the returned dispatcher; that's the identity that matters.
|
||||
let lastScreenshot: ScreenshotResult | undefined;
|
||||
let repl: ComputerUseReplRuntime | undefined
|
||||
let replEpoch = 0
|
||||
const replError = (text: string): CuCallToolResult => ({ isError: true, content: [{ type: 'text', text }] })
|
||||
|
||||
const wrapPermission = ctx.onPermissionRequest
|
||||
? async (
|
||||
@@ -184,8 +197,44 @@ export function bindSessionContext(
|
||||
const clearStaleMouseState = legacyPixelFace
|
||||
? legacyResetMouseButtonHeld
|
||||
: resetMouseButtonHeld;
|
||||
const mouseOwner: WindowsMouseOwner = {
|
||||
canRelease: async () => !ctx.checkCuLock || (await ctx.checkCuLock()).isSelf,
|
||||
}
|
||||
|
||||
return async (name, args) => {
|
||||
const cancellationResult = async (
|
||||
message: string,
|
||||
beforeDispatch: boolean,
|
||||
checkedLock?: { holder: string | undefined; isSelf: boolean },
|
||||
): Promise<CuCallToolResult> => {
|
||||
let cleanupMessage = ''
|
||||
if (legacyPixelFace && hasHeldMouseForSession(mouseOwner)) {
|
||||
try {
|
||||
const released = checkedLock && !checkedLock.isSelf
|
||||
? false
|
||||
: await releaseHeldMouseForSession(adapter, mouseOwner)
|
||||
if (!released && hasHeldMouseForSession(mouseOwner)) {
|
||||
cleanupMessage = ' Held mouse cleanup was skipped because this session no longer owns the Computer Use lock.'
|
||||
}
|
||||
} catch (error) {
|
||||
cleanupMessage = ` Could not release this session's held mouse: ${error instanceof Error ? error.message : String(error)}`
|
||||
}
|
||||
}
|
||||
return {
|
||||
isError: true,
|
||||
content: [{ type: 'text', text: message + cleanupMessage }],
|
||||
...(beforeDispatch ? { [NATIVE_CALL_NOT_DISPATCHED]: true } : {}),
|
||||
}
|
||||
}
|
||||
|
||||
const dispatch = async (
|
||||
name: string,
|
||||
args: unknown,
|
||||
signal?: AbortSignal,
|
||||
): Promise<CuCallToolResult> => {
|
||||
const isAborted = () => signal?.aborted === true || ctx.isAborted?.() === true;
|
||||
if (isAborted()) {
|
||||
return cancellationResult('Computer Use cancelled before dispatch.', true)
|
||||
}
|
||||
// ─── Static request validation (semantic face only) ───────────────────
|
||||
// Runs before the lock so a malformed call costs nothing: no cross-process
|
||||
// lock acquisition, no TCC probe, no approval dialog. The legacy face
|
||||
@@ -207,6 +256,9 @@ export function bindSessionContext(
|
||||
// instead of pre-computing + feeding a fake sync result.
|
||||
if (ctx.checkCuLock) {
|
||||
const lock = await ctx.checkCuLock();
|
||||
if (isAborted()) {
|
||||
return cancellationResult('Computer Use cancelled before lock acquisition.', true, lock)
|
||||
}
|
||||
if (lock.holder !== undefined && !lock.isSelf) {
|
||||
const text =
|
||||
ctx.formatLockHeldMessage?.(lock.holder) ?? DEFAULT_LOCK_HELD_MESSAGE;
|
||||
@@ -214,6 +266,7 @@ export function bindSessionContext(
|
||||
content: [{ type: "text", text }],
|
||||
isError: true,
|
||||
telemetry: { error_kind: "cu_lock_held" },
|
||||
[NATIVE_CALL_NOT_DISPATCHED]: true,
|
||||
};
|
||||
}
|
||||
if (lock.holder === undefined && !toolDefersLockAcquire(name)) {
|
||||
@@ -226,6 +279,9 @@ export function bindSessionContext(
|
||||
// acquire instead; this re-check is a belt-and-suspenders for that
|
||||
// path too.
|
||||
const recheck = await ctx.checkCuLock();
|
||||
if (isAborted()) {
|
||||
return cancellationResult('Computer Use cancelled before dispatch.', true, recheck)
|
||||
}
|
||||
if (recheck.holder !== undefined && !recheck.isSelf) {
|
||||
const text =
|
||||
ctx.formatLockHeldMessage?.(recheck.holder) ??
|
||||
@@ -234,6 +290,7 @@ export function bindSessionContext(
|
||||
content: [{ type: "text", text }],
|
||||
isError: true,
|
||||
telemetry: { error_kind: "cu_lock_held" },
|
||||
[NATIVE_CALL_NOT_DISPATCHED]: true,
|
||||
};
|
||||
}
|
||||
// Fresh holder → any prior session's mouseButtonHeld is stale.
|
||||
@@ -287,7 +344,8 @@ export function bindSessionContext(
|
||||
// above already ran.
|
||||
checkCuLock: undefined,
|
||||
acquireCuLock: undefined,
|
||||
isAborted: ctx.isAborted,
|
||||
isAborted,
|
||||
...(legacyPixelFace ? { [WINDOWS_MOUSE_OWNER]: mouseOwner } : {}),
|
||||
};
|
||||
|
||||
logger.debug(
|
||||
@@ -305,11 +363,71 @@ export function bindSessionContext(
|
||||
ctx.onScreenshotCaptured?.(dims);
|
||||
}
|
||||
|
||||
if (legacyPixelFace && isAborted()) {
|
||||
// An in-flight mouse-down may have completed after cancellation. Its
|
||||
// press belongs to this binder, but the action did already dispatch.
|
||||
const cancelled = await cancellationResult('Computer Use cancelled after dispatch. Inspect the current state before continuing.', false)
|
||||
return { ...result, ...cancelled, content: [...result.content, ...cancelled.content] }
|
||||
}
|
||||
|
||||
return result;
|
||||
} finally {
|
||||
dialogAbort.abort();
|
||||
}
|
||||
};
|
||||
// A daemon serializes single commands, not whole sequences. Keep every
|
||||
// semantic call behind one session queue so ordinary tools cannot interleave.
|
||||
let tail: Promise<unknown> = Promise.resolve();
|
||||
return (name, args, signal) => {
|
||||
if (legacyPixelFace) {
|
||||
return dispatch(name, args, signal);
|
||||
}
|
||||
if (name === 'js_reset') {
|
||||
if (args === null || typeof args !== 'object' || Array.isArray(args) || Object.keys(args).length > 0) {
|
||||
return Promise.resolve(replError('js_reset takes an empty object.'))
|
||||
}
|
||||
++replEpoch
|
||||
return (async () => {
|
||||
await repl?.reset()
|
||||
return { content: [{ type: 'text' as const, text: 'Computer Use JavaScript reset. Select an app again to continue.' }] }
|
||||
})()
|
||||
}
|
||||
const epoch = replEpoch
|
||||
const pending = tail.then(async () => {
|
||||
if (name !== 'js') return dispatch(name, args, signal)
|
||||
if (epoch !== replEpoch) return replError('Computer Use JavaScript was reset before this queued cell started.')
|
||||
if (adapter.isDisabled() || signal?.aborted || ctx.isAborted?.()) {
|
||||
await repl?.reset()
|
||||
return replError('Computer Use JavaScript is disabled or cancelled.')
|
||||
}
|
||||
if (args === null || typeof args !== 'object' || Array.isArray(args)) return replError('js requires an object with code.')
|
||||
const input = args as Record<string, unknown>
|
||||
const timeoutMs = input.timeout_ms ?? 30000
|
||||
if (typeof input.code !== 'string' || Buffer.byteLength(input.code) > REPL_MAX_CODE_BYTES ||
|
||||
Object.keys(input).some(key => !['code', 'title', 'timeout_ms'].includes(key)) ||
|
||||
(input.title !== undefined && typeof input.title !== 'string') ||
|
||||
typeof timeoutMs !== 'number' || !Number.isInteger(timeoutMs) || timeoutMs < 1 || timeoutMs > 60000) {
|
||||
return replError('js requires code (up to 256 KiB), an optional title, and timeout_ms between 1 and 60000.')
|
||||
}
|
||||
if (!adapter.createReplRuntime) return replError('This Computer Use host does not provide an isolated JavaScript kernel.')
|
||||
repl ??= adapter.createReplRuntime()
|
||||
return repl.run({ code: input.code, timeoutMs, signal, isAborted: () => adapter.isDisabled() || ctx.isAborted?.() === true },
|
||||
async (method, parameters, innerSignal) => {
|
||||
// Inner operations use the existing guarded dispatcher directly;
|
||||
// re-entering the outer session queue would deadlock the cell.
|
||||
const result = await dispatch(method, parameters, innerSignal)
|
||||
return {
|
||||
...result,
|
||||
...(result[RESOLVED_APP_PATH] === undefined ? {} : { app: result[RESOLVED_APP_PATH] }),
|
||||
...(result[APP_INVENTORY] === undefined ? {} : { apps: result[APP_INVENTORY] }),
|
||||
...(result[NATIVE_ERROR] === undefined ? {} : { nativeError: result[NATIVE_ERROR] }),
|
||||
...(result[NATIVE_CALL_NOT_DISPATCHED] === true ? { nativeCallNotDispatched: true } : {}),
|
||||
}
|
||||
})
|
||||
});
|
||||
tail = pending.catch(() => {});
|
||||
return pending;
|
||||
};
|
||||
}
|
||||
|
||||
export function createComputerUseMcpServer(
|
||||
@@ -343,10 +461,11 @@ export function createComputerUseMcpServer(
|
||||
const dispatch = bindSessionContext(adapter, coordinateMode, context);
|
||||
server.setRequestHandler(
|
||||
CallToolRequestSchema,
|
||||
async (request): Promise<CallToolResult> => {
|
||||
async (request, extra): Promise<CallToolResult> => {
|
||||
const { screenshot: _s, telemetry: _t, ...result } = await dispatch(
|
||||
request.params.name,
|
||||
request.params.arguments ?? {},
|
||||
extra.signal,
|
||||
);
|
||||
return result;
|
||||
},
|
||||
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
import { isIntrinsicAppDenied } from './deniedApps.js'
|
||||
|
||||
// Official macOS SkyComputerUseService 26.831.1000926:
|
||||
// isForbiddenComputerUseTarget checks these four sets with exact equality.
|
||||
// Legacy cross-platform tiers and display-name matches do not apply here.
|
||||
export const NATIVE_FORBIDDEN_BUNDLE_IDS: ReadonlySet<string> = new Set([
|
||||
'com.apple.Terminal',
|
||||
'com.googlecode.iterm2',
|
||||
'org.alacritty',
|
||||
'dev.warp.Warp-Stable',
|
||||
'net.kovidgoyal.kitty',
|
||||
'co.zeit.hyper',
|
||||
'com.github.wez.wezterm',
|
||||
'org.tabby',
|
||||
'com.mitchellh.ghostty',
|
||||
'com.raphaelamorim.rio',
|
||||
'dev.commandline.waveterm',
|
||||
'com.openai.codex',
|
||||
'com.openai.codex.alpha',
|
||||
'com.openai.codex.beta',
|
||||
'com.openai.codex.dev',
|
||||
'com.openai.codex.nightly',
|
||||
'com.openai.chat',
|
||||
'com.openai.chat.alpha',
|
||||
'com.openai.chat.beta',
|
||||
'com.openai.chat.nightly',
|
||||
'com.openai.chat.mac-debug',
|
||||
'com.apple.UserNotificationCenter',
|
||||
'com.apple.LocalAuthenticationRemoteService',
|
||||
'com.apple.SecurityAgent',
|
||||
])
|
||||
|
||||
export function isNativeAppDenied(bundleId: string | undefined, hostBundleId?: string): boolean {
|
||||
return isIntrinsicAppDenied(bundleId, hostBundleId) ||
|
||||
(bundleId !== undefined && NATIVE_FORBIDDEN_BUNDLE_IDS.has(bundleId))
|
||||
}
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { NativeCommandError, NATIVE_HELPER_ERROR_MAP, NATIVE_SERVER_ERROR_CODES, toNativeErrorMetadata } from './nativeError.js'
|
||||
|
||||
describe('native error metadata boundary', () => {
|
||||
test('preserves all 21 official code names and only maps proven native causes', () => {
|
||||
expect(Object.keys(NATIVE_SERVER_ERROR_CODES)).toHaveLength(21)
|
||||
expect(new Set(Object.values(NATIVE_SERVER_ERROR_CODES)).size).toBe(21)
|
||||
for (const [nativeCode, errorName] of Object.entries(NATIVE_HELPER_ERROR_MAP)) {
|
||||
expect(toNativeErrorMetadata(new NativeCommandError('Fixture message', nativeCode))).toEqual({
|
||||
name: 'SkyComputerUseError', message: 'Fixture message', nativeCode,
|
||||
code: NATIVE_SERVER_ERROR_CODES[errorName], errorName, request: null, requestType: 'jsonRPC',
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
test('unknown, ambiguous, stale and result-unknown causes never become guessed server errors', () => {
|
||||
for (const nativeCode of ['app_denied', 'stale_process', 'stale_snapshot', 'bad_payload', 'user_interference_result_unknown', 'future_error', '__proto__']) {
|
||||
expect(toNativeErrorMetadata(new NativeCommandError('Fixture message', nativeCode))).toEqual({
|
||||
name: 'Error', message: 'Fixture message', nativeCode,
|
||||
})
|
||||
}
|
||||
expect(toNativeErrorMetadata(new NativeCommandError('message'))).toEqual({ name: 'Error', message: 'message' })
|
||||
})
|
||||
|
||||
test('client TypeErrors remain distinct and plain Error properties do not forge a native code', () => {
|
||||
expect(toNativeErrorMetadata(new TypeError('pages must be a finite number > 0'))).toEqual({ name: 'TypeError', message: 'pages must be a finite number > 0' })
|
||||
const error = Object.assign(new Error('not_trusted'), { code: -10009, nativeCode: 'not_trusted' })
|
||||
expect(toNativeErrorMetadata(error)).toEqual({ name: 'Error', message: 'not_trusted' })
|
||||
})
|
||||
})
|
||||
+84
@@ -0,0 +1,84 @@
|
||||
/** Installed @oai/sky targets/mac/errors.js, CodexComputerUseIPC-5. */
|
||||
export const NATIVE_SERVER_ERROR_CODES = {
|
||||
senderProcessNotAuthenticated: -10000,
|
||||
couldNotGetRequestData: -10001,
|
||||
couldNotGetRequestTypeName: -10002,
|
||||
couldNotResolveRequestType: -10003,
|
||||
unhandledEvent: -10004,
|
||||
unknownError: -10005,
|
||||
appNotAllowed: -10006,
|
||||
runningApplicationNotFound: -10007,
|
||||
accessibilityError: -10008,
|
||||
permissionsNotGranted: -10009,
|
||||
invalidApp: -10010,
|
||||
noActiveSession: -10011,
|
||||
userStoppedSession: -10012,
|
||||
incompatibleClientVersion: -10013,
|
||||
permissionsPending: -10014,
|
||||
blockedURL: -10015,
|
||||
userIntervened: -10016,
|
||||
couldNotGetSenderPID: -10017,
|
||||
ambiguousApp: -10018,
|
||||
couldNotGetBootstrapPort: -10019,
|
||||
screenLocked: -10020,
|
||||
} as const
|
||||
|
||||
/** Only mappings whose native throw sites establish the corresponding cause.
|
||||
* In particular app_denied includes failed identity proof, and stale_* differs
|
||||
* from a missing process; neither is guessed from the human-readable message.
|
||||
*/
|
||||
export const NATIVE_HELPER_ERROR_MAP = {
|
||||
not_trusted: 'permissionsNotGranted',
|
||||
screen_recording_denied: 'permissionsNotGranted',
|
||||
ax_failed: 'accessibilityError',
|
||||
process_gone: 'runningApplicationNotFound',
|
||||
target_not_running: 'runningApplicationNotFound',
|
||||
ambiguous_target: 'ambiguousApp',
|
||||
protocol_mismatch: 'incompatibleClientVersion',
|
||||
screen_locked: 'screenLocked',
|
||||
user_interference: 'userIntervened',
|
||||
} as const satisfies Record<string, keyof typeof NATIVE_SERVER_ERROR_CODES>
|
||||
|
||||
export interface NativeErrorMetadata {
|
||||
name: 'Error' | 'TypeError' | 'SkyComputerUseError'
|
||||
message: string
|
||||
code?: number
|
||||
errorName?: string
|
||||
/** The helper discriminator is retained even when no official code is known. */
|
||||
nativeCode?: string
|
||||
request?: null
|
||||
requestType?: 'jsonRPC'
|
||||
}
|
||||
|
||||
/** Internal only; the binder deliberately projects this metadata into JSON. */
|
||||
export const NATIVE_ERROR = Symbol('computer-use-native-error')
|
||||
|
||||
/** A rejected command is never an infrastructure failure eligible for replay. */
|
||||
export class NativeCommandError extends Error {
|
||||
readonly nativeCode?: string
|
||||
|
||||
constructor(message: string, nativeCode?: string) {
|
||||
super(message)
|
||||
this.name = 'NativeCommandError'
|
||||
this.nativeCode = nativeCode
|
||||
}
|
||||
}
|
||||
|
||||
export function toNativeErrorMetadata(error: unknown): NativeErrorMetadata {
|
||||
const message = error instanceof Error ? error.message : String(error)
|
||||
if (error instanceof TypeError) return { name: 'TypeError', message }
|
||||
if (error instanceof NativeCommandError && error.nativeCode !== undefined) {
|
||||
const nativeCode = error.nativeCode
|
||||
const errorName = Object.hasOwn(NATIVE_HELPER_ERROR_MAP, nativeCode)
|
||||
? NATIVE_HELPER_ERROR_MAP[nativeCode as keyof typeof NATIVE_HELPER_ERROR_MAP]
|
||||
: undefined
|
||||
if (errorName !== undefined) return {
|
||||
name: 'SkyComputerUseError', message, nativeCode,
|
||||
code: NATIVE_SERVER_ERROR_CODES[errorName], errorName,
|
||||
// The official native pipe wraps JSON-RPC errors with these values.
|
||||
request: null, requestType: 'jsonRPC',
|
||||
}
|
||||
return { name: 'Error', message, nativeCode }
|
||||
}
|
||||
return { name: 'Error', message }
|
||||
}
|
||||
+291
-13
@@ -1,30 +1,23 @@
|
||||
import { Client } from '@modelcontextprotocol/sdk/client/index.js'
|
||||
import { InMemoryTransport } from '@modelcontextprotocol/sdk/inMemory.js'
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { afterEach, describe, expect, test } from 'bun:test'
|
||||
|
||||
import type {
|
||||
ComputerExecutor,
|
||||
DisplayGeometry,
|
||||
ScreenshotResult,
|
||||
} from './executor.js'
|
||||
import { createComputerUseMcpServer } from './mcpServer.js'
|
||||
import { bindSessionContext, createComputerUseMcpServer } from './mcpServer.js'
|
||||
import { NATIVE_CALL_NOT_DISPATCHED } from './toolCalls.js'
|
||||
import { resetMouseButtonHeld } from './windowsLegacyToolCalls.js'
|
||||
import type {
|
||||
ComputerUseHostAdapter,
|
||||
ComputerUseSessionContext,
|
||||
} from './types.js'
|
||||
|
||||
const DARWIN_TOOL_NAMES = [
|
||||
'click',
|
||||
'drag',
|
||||
'get_app_state',
|
||||
'list_apps',
|
||||
'perform_secondary_action',
|
||||
'paste',
|
||||
'press_key',
|
||||
'scroll',
|
||||
'select_text',
|
||||
'set_value',
|
||||
'type_text',
|
||||
'js',
|
||||
'js_reset',
|
||||
].sort()
|
||||
|
||||
const WINDOWS_LEGACY_TOOL_NAMES = [
|
||||
@@ -63,6 +56,8 @@ const logger = {
|
||||
error() {},
|
||||
}
|
||||
|
||||
afterEach(() => resetMouseButtonHeld())
|
||||
|
||||
function makeSessionContext(
|
||||
overrides: Partial<ComputerUseSessionContext> = {},
|
||||
): ComputerUseSessionContext {
|
||||
@@ -276,6 +271,288 @@ function makeWindowsAdapter(calls: string[]): ComputerUseHostAdapter {
|
||||
}
|
||||
|
||||
describe('Computer Use platform routing', () => {
|
||||
test('win32 pre-cancelled single click and key calls never dispatch input', async () => {
|
||||
const calls: string[] = []
|
||||
let tccChecks = 0
|
||||
let lockChecks = 0
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
adapter.ensureOsPermissions = async () => {
|
||||
tccChecks += 1
|
||||
return { granted: true }
|
||||
}
|
||||
const dispatch = bindSessionContext(
|
||||
adapter,
|
||||
'pixels',
|
||||
makeSessionContext({
|
||||
checkCuLock: async () => {
|
||||
lockChecks += 1
|
||||
return { holder: undefined, isSelf: false }
|
||||
},
|
||||
}),
|
||||
)
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
const results = await Promise.all([
|
||||
dispatch('left_click', { coordinate: [10, 20] }, controller.signal),
|
||||
dispatch('key', { text: 'ctrl+a' }, controller.signal),
|
||||
])
|
||||
|
||||
expect(calls).not.toContain('click:10,20,left,1')
|
||||
expect(calls).not.toContain('key:ctrl+a')
|
||||
expect(results.every(result => result.isError === true)).toBe(true)
|
||||
expect(tccChecks).toBe(0)
|
||||
expect(lockChecks).toBe(0)
|
||||
})
|
||||
|
||||
test('win32 cancellation while awaiting the lock check never acquires or dispatches', async () => {
|
||||
const calls: string[] = []
|
||||
let releaseLockCheck!: (value: { holder: undefined; isSelf: false }) => void
|
||||
const lockCheck = new Promise<{ holder: undefined; isSelf: false }>(resolve => {
|
||||
releaseLockCheck = resolve
|
||||
})
|
||||
let acquireCount = 0
|
||||
const dispatch = bindSessionContext(
|
||||
makeWindowsAdapter(calls),
|
||||
'pixels',
|
||||
makeSessionContext({
|
||||
checkCuLock: async () => lockCheck,
|
||||
acquireCuLock: async () => { acquireCount += 1 },
|
||||
}),
|
||||
)
|
||||
const controller = new AbortController()
|
||||
const pending = dispatch('key', { text: 'ctrl+a' }, controller.signal)
|
||||
|
||||
controller.abort()
|
||||
releaseLockCheck({ holder: undefined, isSelf: false })
|
||||
const result = await pending
|
||||
|
||||
expect(acquireCount).toBe(0)
|
||||
expect(calls).not.toContain('key:ctrl+a')
|
||||
expect(result.isError).toBe(true)
|
||||
})
|
||||
|
||||
test('win32 cancellation during lock acquisition never dispatches new input', async () => {
|
||||
const calls: string[] = []
|
||||
let acquired = false
|
||||
let enteredAcquire!: () => void
|
||||
let finishAcquire!: () => void
|
||||
const acquiring = new Promise<void>(resolve => { enteredAcquire = resolve })
|
||||
const acquireGate = new Promise<void>(resolve => { finishAcquire = resolve })
|
||||
const dispatch = bindSessionContext(makeWindowsAdapter(calls), 'pixels', makeSessionContext({
|
||||
checkCuLock: async () => acquired
|
||||
? { holder: 'self', isSelf: true }
|
||||
: { holder: undefined, isSelf: false },
|
||||
acquireCuLock: async () => {
|
||||
enteredAcquire()
|
||||
await acquireGate
|
||||
acquired = true
|
||||
},
|
||||
}))
|
||||
const controller = new AbortController()
|
||||
const pending = dispatch('key', { text: 'ctrl+a' }, controller.signal)
|
||||
await acquiring
|
||||
controller.abort()
|
||||
finishAcquire()
|
||||
|
||||
const result = await pending
|
||||
expect(result.isError).toBe(true)
|
||||
expect(calls).not.toContain('key:ctrl+a')
|
||||
})
|
||||
|
||||
test('win32 cancellation while awaiting OS permissions never dispatches click or key input', async () => {
|
||||
const calls: string[] = []
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
let permissionsEntered!: () => void
|
||||
let finishPermissions!: (value: { granted: true }) => void
|
||||
const checking = new Promise<void>(resolve => { permissionsEntered = resolve })
|
||||
const permissionGate = new Promise<{ granted: true }>(resolve => { finishPermissions = resolve })
|
||||
adapter.ensureOsPermissions = async () => {
|
||||
permissionsEntered()
|
||||
return permissionGate
|
||||
}
|
||||
const dispatch = bindSessionContext(adapter, 'pixels', makeSessionContext())
|
||||
const controller = new AbortController()
|
||||
const pending = Promise.all([
|
||||
dispatch('left_click', { coordinate: [10, 20] }, controller.signal),
|
||||
dispatch('key', { text: 'ctrl+a' }, controller.signal),
|
||||
])
|
||||
await checking
|
||||
controller.abort()
|
||||
finishPermissions({ granted: true })
|
||||
|
||||
const results = await pending
|
||||
expect(calls).not.toContain('click:10,20,left,1')
|
||||
expect(calls).not.toContain('key:ctrl+a')
|
||||
expect(results.every(result => result.isError === true)).toBe(true)
|
||||
})
|
||||
|
||||
test('win32 cancellation releases this binder held mouse once without dispatching the requested key', async () => {
|
||||
const calls: string[] = []
|
||||
const dispatch = bindSessionContext(makeWindowsAdapter(calls), 'pixels', makeSessionContext())
|
||||
expect((await dispatch('left_mouse_down', {})).isError).toBeFalsy()
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
expect((await dispatch('key', { text: 'ctrl+a' }, controller.signal)).isError).toBe(true)
|
||||
expect((await dispatch('left_mouse_up', {}, controller.signal)).isError).toBe(true)
|
||||
expect(calls.filter(call => call === 'mouseUp')).toHaveLength(1)
|
||||
expect(calls).not.toContain('key:ctrl+a')
|
||||
})
|
||||
|
||||
test('win32 cancellation cannot release another binder held mouse', async () => {
|
||||
const ownerCalls: string[] = []
|
||||
const otherCalls: string[] = []
|
||||
const owner = bindSessionContext(makeWindowsAdapter(ownerCalls), 'pixels', makeSessionContext())
|
||||
const other = bindSessionContext(makeWindowsAdapter(otherCalls), 'pixels', makeSessionContext())
|
||||
expect((await owner('left_mouse_down', {})).isError).toBeFalsy()
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
expect((await other('left_mouse_up', {}, controller.signal)).isError).toBe(true)
|
||||
expect(otherCalls).not.toContain('mouseUp')
|
||||
await owner('left_mouse_up', {}, controller.signal)
|
||||
expect(ownerCalls.filter(call => call === 'mouseUp')).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('win32 cancellation does not release a held mouse after losing the global lock', async () => {
|
||||
const calls: string[] = []
|
||||
let isSelf = true
|
||||
const dispatch = bindSessionContext(makeWindowsAdapter(calls), 'pixels', makeSessionContext({
|
||||
checkCuLock: async () => ({ holder: isSelf ? 'self' : 'other', isSelf }),
|
||||
}))
|
||||
expect((await dispatch('left_mouse_down', {})).isError).toBeFalsy()
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
isSelf = false
|
||||
|
||||
expect((await dispatch('left_mouse_up', {}, controller.signal)).isError).toBe(true)
|
||||
expect(calls).not.toContain('mouseUp')
|
||||
isSelf = true
|
||||
await dispatch('left_mouse_up', {}, controller.signal)
|
||||
expect(calls.filter(call => call === 'mouseUp')).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('win32 cancelled mouse release failures stay visible and preserve recovery ownership', async () => {
|
||||
const calls: string[] = []
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
let releaseAttempts = 0
|
||||
adapter.executor.mouseUp = async () => {
|
||||
releaseAttempts += 1
|
||||
if (releaseAttempts === 1) throw new Error('Windows release fixture failed')
|
||||
calls.push('mouseUp')
|
||||
}
|
||||
const dispatch = bindSessionContext(adapter, 'pixels', makeSessionContext())
|
||||
expect((await dispatch('left_mouse_down', {})).isError).toBeFalsy()
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
|
||||
const failed = await dispatch('key', { text: 'ctrl+a' }, controller.signal)
|
||||
expect(failed.isError).toBe(true)
|
||||
expect(failed.content).toContainEqual(expect.objectContaining({
|
||||
type: 'text', text: expect.stringContaining('Windows release fixture failed'),
|
||||
}))
|
||||
await dispatch('left_mouse_up', {}, controller.signal)
|
||||
await dispatch('left_mouse_up', {}, controller.signal)
|
||||
expect(releaseAttempts).toBe(2)
|
||||
expect(calls.filter(call => call === 'mouseUp')).toHaveLength(1)
|
||||
expect(calls).not.toContain('key:ctrl+a')
|
||||
})
|
||||
|
||||
test('win32 simultaneous cancelled calls share one owned mouse release', async () => {
|
||||
const calls: string[] = []
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
let releaseEntered!: () => void
|
||||
let finishRelease!: () => void
|
||||
const releasing = new Promise<void>(resolve => { releaseEntered = resolve })
|
||||
const releaseGate = new Promise<void>(resolve => { finishRelease = resolve })
|
||||
adapter.executor.mouseUp = async () => {
|
||||
calls.push('mouseUp')
|
||||
releaseEntered()
|
||||
await releaseGate
|
||||
}
|
||||
const dispatch = bindSessionContext(adapter, 'pixels', makeSessionContext())
|
||||
await dispatch('left_mouse_down', {})
|
||||
const controller = new AbortController()
|
||||
controller.abort()
|
||||
const first = dispatch('left_mouse_up', {}, controller.signal)
|
||||
await releasing
|
||||
const second = dispatch('left_mouse_up', {}, controller.signal)
|
||||
finishRelease()
|
||||
const results = await Promise.all([first, second])
|
||||
|
||||
expect(results.every(result => result.isError === true)).toBe(true)
|
||||
expect(calls.filter(call => call === 'mouseUp')).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('win32 cancellation during mouse-down releases after completion without claiming no dispatch', async () => {
|
||||
const calls: string[] = []
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
let downEntered!: () => void
|
||||
let finishDown!: () => void
|
||||
const entering = new Promise<void>(resolve => { downEntered = resolve })
|
||||
const downGate = new Promise<void>(resolve => { finishDown = resolve })
|
||||
adapter.executor.mouseDown = async () => {
|
||||
calls.push('mouseDown')
|
||||
downEntered()
|
||||
await downGate
|
||||
}
|
||||
const dispatch = bindSessionContext(adapter, 'pixels', makeSessionContext())
|
||||
const controller = new AbortController()
|
||||
const pending = dispatch('left_mouse_down', {}, controller.signal)
|
||||
await entering
|
||||
controller.abort()
|
||||
finishDown()
|
||||
|
||||
const result = await pending
|
||||
expect(calls.filter(call => ['mouseDown', 'mouseUp'].includes(call))).toEqual(['mouseDown', 'mouseUp'])
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result[NATIVE_CALL_NOT_DISPATCHED]).not.toBe(true)
|
||||
})
|
||||
|
||||
test('win32 mid-batch cancellation releases its held mouse and skips later actions', async () => {
|
||||
const calls: string[] = []
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
const controller = new AbortController()
|
||||
adapter.executor.mouseDown = async () => {
|
||||
calls.push('mouseDown')
|
||||
controller.abort()
|
||||
}
|
||||
const dispatch = bindSessionContext(adapter, 'pixels', makeSessionContext())
|
||||
const result = await dispatch('computer_batch', {
|
||||
actions: [{ action: 'left_mouse_down' }, { action: 'key', text: 'ctrl+a' }],
|
||||
}, controller.signal)
|
||||
|
||||
expect(result.isError).toBe(true)
|
||||
expect(calls.filter(call => ['mouseDown', 'mouseUp'].includes(call))).toEqual(['mouseDown', 'mouseUp'])
|
||||
expect(calls).not.toContain('key:ctrl+a')
|
||||
})
|
||||
|
||||
test('win32 mid-batch cancellation cannot release its mouse after losing the global lock', async () => {
|
||||
const calls: string[] = []
|
||||
const adapter = makeWindowsAdapter(calls)
|
||||
const controller = new AbortController()
|
||||
let isSelf = true
|
||||
adapter.executor.mouseDown = async () => {
|
||||
calls.push('mouseDown')
|
||||
isSelf = false
|
||||
controller.abort()
|
||||
}
|
||||
const dispatch = bindSessionContext(adapter, 'pixels', makeSessionContext({
|
||||
checkCuLock: async () => ({ holder: isSelf ? 'self' : 'other', isSelf }),
|
||||
}))
|
||||
const result = await dispatch('computer_batch', {
|
||||
actions: [{ action: 'left_mouse_down' }, { action: 'key', text: 'ctrl+a' }],
|
||||
}, controller.signal)
|
||||
|
||||
expect(result.isError).toBe(true)
|
||||
expect(calls).not.toContain('mouseUp')
|
||||
expect(calls).not.toContain('key:ctrl+a')
|
||||
isSelf = true
|
||||
await dispatch('left_mouse_up', {}, controller.signal)
|
||||
expect(calls.filter(call => call === 'mouseUp')).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('darwin ListTools advertises the current semantic tools', async () => {
|
||||
const connection = await connect(makeDarwinAdapter())
|
||||
try {
|
||||
@@ -336,6 +613,7 @@ describe('Computer Use platform routing', () => {
|
||||
WINDOWS_LEGACY_TOOL_NAMES,
|
||||
)
|
||||
expect(result.tools.some(tool => tool.name === 'get_app_state')).toBe(false)
|
||||
expect(result.tools.some(tool => tool.name === 'sequence')).toBe(false)
|
||||
expect(result.tools.some(tool => tool.name === 'request_access')).toBe(false)
|
||||
expect(result.tools.some(tool => tool.name === 'list_granted_applications')).toBe(false)
|
||||
} finally {
|
||||
|
||||
+430
@@ -0,0 +1,430 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { createContext, runInContext } from 'node:vm'
|
||||
import { COMPUTER_USE_BATCHING_GUIDANCE } from './instructions.js'
|
||||
import {
|
||||
createReplApi,
|
||||
REPL_BOOTSTRAP_SOURCE,
|
||||
type ReplApiBridge,
|
||||
type ReplToolResult,
|
||||
} from './replApi.js'
|
||||
|
||||
const full = '<app_state>\nWindow: "Fixture"\ng7:0 window\n\tg7:1 button Save\n\tg7:4 text field Name\n</app_state>'
|
||||
const diffHeader = 'The following is a diff from the previous accessibility tree for Window: "Fixture" with ~ and + representing changed and added elements, respectively. Removed elements are summarized by ID range.'
|
||||
// A complete 1×1 RGB JPEG encoded at quality 80; no native capture or user data.
|
||||
const jpeg = '/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAYEBQYFBAYGBQYHBwYIChAKCgkJChQODwwQFxQYGBcUFhYaHSUfGhsjHBYWICwgIyYnKSopGR8tMC0oMCUoKSj/2wBDAQcHBwoIChMKChMoGhYaKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCgoKCj/wAARCAABAAEDASIAAhEBAxEB/8QAFQABAQAAAAAAAAAAAAAAAAAAAAX/xAAUEAEAAAAAAAAAAAAAAAAAAAAA/8QAFQEBAQAAAAAAAAAAAAAAAAAABgf/xAAUEQEAAAAAAAAAAAAAAAAAAAAA/9oADAMBAAIRAxEAPwCWAFLQ/9k='
|
||||
|
||||
function jpegState(): ReplToolResult {
|
||||
return { ...state(), content: [{ type: 'text', text: full }, { type: 'image', data: jpeg, mimeType: 'image/jpeg' }] }
|
||||
}
|
||||
|
||||
function state(text = full, image = true): ReplToolResult {
|
||||
return {
|
||||
app: '/Applications/Fixture.app',
|
||||
content: [
|
||||
{ type: 'text', text },
|
||||
...(image ? [{ type: 'image' as const, data: 'AAH+/w==', mimeType: 'image/png' }] : []),
|
||||
],
|
||||
}
|
||||
}
|
||||
|
||||
function fixture() {
|
||||
const calls: Array<{ method: string; args: Record<string, unknown> }> = []
|
||||
const emitted: Array<{ type: string; value: unknown }> = []
|
||||
const results: ReplToolResult[] = []
|
||||
const bridge: ReplApiBridge = {
|
||||
async invoke(method, args) {
|
||||
calls.push({ method, args })
|
||||
return results.shift() ?? (method === 'get_app_state' ? state() : { content: [] })
|
||||
},
|
||||
emit(type: string, value: unknown) {
|
||||
emitted.push({ type, value })
|
||||
},
|
||||
}
|
||||
return { cua: createReplApi(bridge), calls, emitted, results }
|
||||
}
|
||||
|
||||
describe('native Computer Use JavaScript facade', () => {
|
||||
test('binding observes once, emits only AX text, and retains the approved selector', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture alias')
|
||||
expect(f.calls).toEqual([{ method: 'get_app_state', args: { app: 'Fixture alias', disableDiff: true } }])
|
||||
expect(f.emitted).toHaveLength(1)
|
||||
expect(f.emitted[0]?.type).toBe('text')
|
||||
expect(f.emitted[0]?.value).toContain('Native App API')
|
||||
expect(String(f.emitted[0]?.value).endsWith(full)).toBe(true)
|
||||
await app.click([15, 20])
|
||||
await app.pressKey('Cmd+A')
|
||||
expect(f.calls.slice(1)).toEqual([
|
||||
{ method: 'click', args: { app: '/Applications/Fixture.app', x: 15, y: 20 } },
|
||||
{ method: 'press_key', args: { app: '/Applications/Fixture.app', key: 'Cmd+A' } },
|
||||
])
|
||||
expect(f.emitted).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('observations use the same native request but separate outputs and return values', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
f.emitted.length = 0
|
||||
expect(await app.getAXState({ disableDiffing: true })).toBe(full)
|
||||
expect(f.calls.at(-1)).toEqual({ method: 'get_app_state', args: { app: '/Applications/Fixture.app', disableDiff: true } })
|
||||
expect(f.emitted).toEqual([{ type: 'text', value: full }])
|
||||
f.emitted.length = 0
|
||||
expect(Array.from(await app.getScreenshot())).toEqual([0, 1, 254, 255])
|
||||
expect(f.calls.at(-1)).toEqual({ method: 'get_app_state', args: { app: '/Applications/Fixture.app' } })
|
||||
expect(f.emitted).toEqual([{ type: 'image', value: { data: 'AAH+/w==', mimeType: 'image/png' } }])
|
||||
f.emitted.length = 0
|
||||
const both = await app.getAXStateAndScreenshot({ disableDiffing: false })
|
||||
expect(both.state).toBe(full)
|
||||
expect(Array.from(both.screenshot!)).toEqual([0, 1, 254, 255])
|
||||
expect(f.emitted.map(item => item.type)).toEqual(['text', 'image'])
|
||||
expect(f.calls.at(-1)?.args.disableDiff).toBe(false)
|
||||
})
|
||||
|
||||
test('emit:false suppresses every observation output while returning usable data', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
f.emitted.length = 0
|
||||
expect(await app.getAXState({ emit: false })).toBe(full)
|
||||
await app.click(1)
|
||||
expect(f.calls.at(-1)?.args.element_index).toBe('g7:1')
|
||||
expect(Array.from(await app.getScreenshot({ emit: false }))).toEqual([0, 1, 254, 255])
|
||||
const both = await app.getAXStateAndScreenshot({ emit: false })
|
||||
expect(both.state).toBe(full)
|
||||
expect(both.screenshot).toBeInstanceOf(Uint8Array)
|
||||
expect(f.emitted).toEqual([])
|
||||
})
|
||||
|
||||
test('missing images preserve AX state but image-only calls reject', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
f.results.push(state(full, false), state(full, false))
|
||||
expect(await app.getAXStateAndScreenshot({ emit: false })).toEqual({ state: full })
|
||||
await expect(app.getScreenshot()).rejects.toThrow('Screenshot unavailable')
|
||||
await expect(app.click(1)).rejects.toThrow('no current observed handle')
|
||||
})
|
||||
|
||||
test('every native action maps names and parameters without implicit observations', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
f.calls.length = 0
|
||||
await app.click(1, { mouseButton: 1, clickCount: 2 })
|
||||
await app.drag([1, 2], [30, 40])
|
||||
await app.pressKey('Meta+Shift+S')
|
||||
await app.scroll(4, 'down', 0.5)
|
||||
await app.scroll([50, 60], 'u')
|
||||
await app.paste('one')
|
||||
await app.paste('**two**', { format: 'md' })
|
||||
await app.typeText('three')
|
||||
await app.selectText(4, 'word', { prefix: 'a ', suffix: ' b', selectionType: 'cursor_after' })
|
||||
await app.setValue('g7:4', 'new')
|
||||
await app.performSecondaryAction(1, 'show menu')
|
||||
const appArg = { app: '/Applications/Fixture.app' }
|
||||
expect(f.calls).toEqual([
|
||||
{ method: 'click', args: { ...appArg, element_index: 'g7:1', mouse_button: 'right', click_count: 2 } },
|
||||
{ method: 'drag', args: { ...appArg, from_x: 1, from_y: 2, to_x: 30, to_y: 40 } },
|
||||
{ method: 'press_key', args: { ...appArg, key: 'Meta+Shift+S' } },
|
||||
{ method: 'scroll', args: { ...appArg, element_index: 'g7:4', direction: 'down', pages: 0.5 } },
|
||||
{ method: 'scroll', args: { ...appArg, x: 50, y: 60, direction: 'up' } },
|
||||
{ method: 'paste', args: { ...appArg, text: 'one', format: 'text' } },
|
||||
{ method: 'paste', args: { ...appArg, text: '**two**', format: 'md' } },
|
||||
{ method: 'type_text', args: { ...appArg, text: 'three' } },
|
||||
{ method: 'select_text', args: { ...appArg, element_index: 'g7:4', text: 'word', prefix: 'a ', suffix: ' b', selection_type: 'cursor_after' } },
|
||||
{ method: 'set_value', args: { ...appArg, element_index: 'g7:4', value: 'new' } },
|
||||
{ method: 'perform_secondary_action', args: { ...appArg, element_index: 'g7:1', action: 'show menu' } },
|
||||
])
|
||||
})
|
||||
|
||||
test('official mouse aliases do not enter the older numeric convention', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
for (const [input, output] of [[0, 'left'], [1, 'right'], [2, 'middle'], ['l', 'left'], ['r', 'right'], ['m', 'middle']] as const) {
|
||||
await app.click([1, 2], { mouseButton: input })
|
||||
expect(f.calls.at(-1)?.args.mouse_button).toBe(output)
|
||||
}
|
||||
})
|
||||
|
||||
test('macOS accepts trimmed case-insensitive mouse aliases like its actual native client', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
for (const [mouseButton, expected] of [[' RIGHT ', 'right'], ['Left', 'left'], [' M ', 'middle']] as const) {
|
||||
await app.click([1, 2], { mouseButton: mouseButton as 'right' })
|
||||
expect(f.calls.at(-1)?.args.mouse_button).toBe(expected)
|
||||
}
|
||||
})
|
||||
|
||||
test('macOS scroll aliases normalize without accepting invalid pages or directions', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
await app.scroll([1, 2], ' U ', 0.5)
|
||||
expect(f.calls.at(-1)?.args).toEqual({ app: '/Applications/Fixture.app', x: 1, y: 2, direction: 'up', pages: 0.5 })
|
||||
const calls = f.calls.length
|
||||
await expect(app.scroll([1, 2], 'diagonal')).rejects.toThrow('direction must be up, down, left, or right')
|
||||
for (const pages of [0, -1, Infinity, NaN]) {
|
||||
await expect(app.scroll([1, 2], 'down', pages)).rejects.toThrow('pages must be a finite number > 0')
|
||||
}
|
||||
expect(f.calls).toHaveLength(calls)
|
||||
})
|
||||
|
||||
test('listApps preserves trusted inventory fields and does not infer running status', async () => {
|
||||
const f = fixture()
|
||||
const apps = [
|
||||
{ id: 'fixture.running', displayName: 'Running', isRunning: true, lastUsedDate: '2026-01-02T03:04:05Z', useCount: 4 },
|
||||
{ id: 'fixture.stopped', displayName: 'Stopped', isRunning: false },
|
||||
{ id: 'fixture.unknown' },
|
||||
]
|
||||
f.results.push({ content: [{ type: 'text', text: 'Legacy formatter must not override metadata' }], apps })
|
||||
expect(await f.cua.listApps({ emit: false })).toEqual(apps)
|
||||
expect(f.emitted).toEqual([])
|
||||
})
|
||||
|
||||
test('computer exposes every actual macOS window member without browser or phantom initialization methods', async () => {
|
||||
const f = fixture()
|
||||
expect(Object.keys(f.cua).sort()).toEqual(['computer', 'getApp', 'getState', 'listApps'])
|
||||
const computer = f.cua.computer
|
||||
expect(Object.keys(computer).sort()).toEqual([
|
||||
'target', 'list_apps', 'get_app_state', 'click', 'drag', 'paste',
|
||||
'perform_secondary_action', 'press_key', 'scroll', 'select_text', 'set_value', 'type_text',
|
||||
].sort())
|
||||
expect(computer.target).toBe('mac')
|
||||
const observation = await computer.get_app_state({ app: 'Fixture alias', disableDiff: true })
|
||||
expect(observation).toEqual({ app: '/Applications/Fixture.app', text: full, screenshot: { url: 'data:image/png;base64,AAH+/w==' } })
|
||||
await computer.click({ app: 'Fixture alias', element_index: 1, mouse_button: 1 })
|
||||
expect(f.calls.at(-1)).toEqual({ method: 'click', args: { app: 'Fixture alias', element_index: 'g7:1', mouse_button: 'right' } })
|
||||
expect(await computer.press_key({ app: 'Fixture alias', key: 'Cmd+A' })).toBeUndefined()
|
||||
expect(f.emitted).toEqual([])
|
||||
})
|
||||
|
||||
test('all raw window actions keep native arguments, return undefined and never insert an observation', async () => {
|
||||
const f = fixture()
|
||||
const computer = f.cua.computer
|
||||
const target = { app: '/Applications/Fixture.app' }
|
||||
await computer.get_app_state(target)
|
||||
f.calls.length = 0
|
||||
const cases = [
|
||||
['click', { ...target, element_index: 1, mouse_button: ' RIGHT ', click_count: 2 }, { ...target, element_index: 'g7:1', mouse_button: 'right', click_count: 2 }],
|
||||
['drag', { ...target, from_x: 1, from_y: 2, to_x: 3, to_y: 4 }],
|
||||
['paste', { ...target, text: '<p>html</p>', format: 'html' }],
|
||||
['perform_secondary_action', { ...target, element_index: 1, action: 'AXShowMenu' }, { ...target, element_index: 'g7:1', action: 'AXShowMenu' }],
|
||||
['press_key', { ...target, key: 'Cmd+Shift+A' }],
|
||||
['scroll', { ...target, x: 1, y: 2, direction: ' U ', pages: 0.5 }, { ...target, x: 1, y: 2, direction: 'up', pages: 0.5 }],
|
||||
['select_text', { ...target, element_index: 4, text: 'word', prefix: 'a ', suffix: ' b', selection_type: 'cursor_before' }, { ...target, element_index: 'g7:4', text: 'word', prefix: 'a ', suffix: ' b', selection_type: 'cursor_before' }],
|
||||
['set_value', { ...target, element_index: 4, value: 'new' }, { ...target, element_index: 'g7:4', value: 'new' }],
|
||||
['type_text', { ...target, text: 'typed' }],
|
||||
] as const
|
||||
for (const [method, input, expected] of cases) {
|
||||
const call = computer[method] as (value: unknown) => Promise<void>
|
||||
expect(await call(input)).toBeUndefined()
|
||||
expect(f.calls.at(-1)).toEqual({ method, args: expected ?? input })
|
||||
}
|
||||
expect(f.calls).toHaveLength(cases.length)
|
||||
expect(f.emitted).toEqual([])
|
||||
})
|
||||
|
||||
test('a failed raw observation invalidates integer aliases shared with bound Apps', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture alias')
|
||||
f.results.push({ isError: true, content: [{ type: 'text', text: 'Fixture capture failed' }] })
|
||||
await expect(f.cua.computer.get_app_state({ app: 'Fixture alias' })).rejects.toThrow('Fixture capture failed')
|
||||
const count = f.calls.length
|
||||
await expect(app.click(1)).rejects.toThrow('no current observed handle')
|
||||
await expect(f.cua.computer.click({ app: 'Fixture alias', element_index: 1 })).rejects.toThrow('no current observed handle')
|
||||
expect(f.calls).toHaveLength(count)
|
||||
await f.cua.computer.get_app_state({ app: 'Fixture alias', disableDiff: true })
|
||||
await app.click(1)
|
||||
expect(f.calls.at(-1)?.args.element_index).toBe('g7:1')
|
||||
})
|
||||
|
||||
test('numeric indices follow diffs, sparse removals, full replacements and new generations', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
f.results.push(state(`<app_state>\n${diffHeader}\nRemoved element IDs: 1-2\n~\tg7:4 text field Updated\n+\tg7:8 button New\n</app_state>`))
|
||||
await app.getAXState({ emit: false })
|
||||
await expect(app.click(1)).rejects.toThrow('no current observed handle')
|
||||
await app.click(4)
|
||||
expect(f.calls.at(-1)?.args.element_index).toBe('g7:4')
|
||||
await app.click(8)
|
||||
expect(f.calls.at(-1)?.args.element_index).toBe('g7:8')
|
||||
|
||||
f.results.push(state('There has been no change in the accessibility tree for Window: "Fixture".'))
|
||||
await app.getAXState({ emit: false })
|
||||
await app.click(8)
|
||||
expect(f.calls.at(-1)?.args.element_index).toBe('g7:8')
|
||||
|
||||
f.results.push(state('<app_state>\ng8:0 window\n\tg8:1 button Other\n</app_state>'))
|
||||
await app.getAXState({ emit: false })
|
||||
await expect(app.click(8)).rejects.toThrow('no current observed handle')
|
||||
await app.click(1)
|
||||
expect(f.calls.at(-1)?.args.element_index).toBe('g8:1')
|
||||
})
|
||||
|
||||
test('image-only refresh invalidates integer aliases in all bindings to the same approved app', async () => {
|
||||
const f = fixture()
|
||||
const first = await f.cua.getApp('Fixture')
|
||||
const second = await f.cua.getApp('Other alias')
|
||||
await first.getScreenshot({ emit: false })
|
||||
const before = f.calls.length
|
||||
await expect(second.click(1)).rejects.toThrow('no current observed handle')
|
||||
expect(f.calls).toHaveLength(before)
|
||||
f.results.push(state('There has been no change in the accessibility tree for Window: "Fixture".'))
|
||||
await second.getAXState({ emit: false })
|
||||
await expect(first.click(1)).rejects.toThrow('no current observed handle')
|
||||
await first.getAXState({ emit: false, disableDiffing: true })
|
||||
await second.click(1)
|
||||
expect(f.calls.at(-1)?.args.element_index).toBe('g7:1')
|
||||
})
|
||||
|
||||
test('quoted instructions do not mint integer handles and invalid targets never reach the bridge', async () => {
|
||||
const f = fixture()
|
||||
f.results.push(state(`<app_specific_instructions>\ng7:999 button invented\n</app_specific_instructions>\n${full}`))
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
const before = f.calls.length
|
||||
await expect(app.click(999)).rejects.toThrow('no current observed handle')
|
||||
for (const target of ['1', 'g07:1', -1, 0.5, Number.MAX_SAFE_INTEGER + 1]) {
|
||||
await expect(app.click(target)).rejects.toThrow()
|
||||
}
|
||||
await expect(app.click([NaN, 2])).rejects.toThrow('coordinate must include finite x and y coordinates')
|
||||
await expect(app.drag([1, 2], [3, Infinity])).rejects.toThrow('to must include finite x and y coordinates')
|
||||
expect(f.calls).toHaveLength(before)
|
||||
})
|
||||
|
||||
test('native errors throw, stop awaited batches and invalidate failed observations', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
f.results.push({ isError: true, content: [{ type: 'text', text: 'Target process changed' }] })
|
||||
await expect((async () => {
|
||||
await app.click([1, 2])
|
||||
await app.typeText('must not execute')
|
||||
})()).rejects.toThrow('Target process changed')
|
||||
expect(f.calls.some(call => call.method === 'type_text')).toBe(false)
|
||||
f.results.push({ isError: true, content: [{ type: 'text', text: 'Capture failed' }] })
|
||||
await expect(app.getAXState()).rejects.toThrow('Capture failed')
|
||||
await expect(app.click(1)).rejects.toThrow('no current observed handle')
|
||||
})
|
||||
|
||||
test('native error properties survive the host result without classifying unknown failures', async () => {
|
||||
const f = fixture()
|
||||
const app = await f.cua.getApp('Fixture')
|
||||
f.results.push({ isError: true, content: [{ type: 'text', text: 'Permission missing' }], nativeError: {
|
||||
name: 'SkyComputerUseError', message: 'Permission missing', code: -10009,
|
||||
errorName: 'permissionsNotGranted', nativeCode: 'not_trusted', request: null, requestType: 'jsonRPC',
|
||||
} })
|
||||
const denied = await app.click([1, 2]).catch(error => error)
|
||||
expect(denied).toBeInstanceOf(Error)
|
||||
expect(denied).toMatchObject({ name: 'SkyComputerUseError', message: 'Permission missing', code: -10009,
|
||||
errorName: 'permissionsNotGranted', nativeCode: 'not_trusted', request: null, requestType: 'jsonRPC' })
|
||||
f.results.push({ isError: true, content: [], nativeError: { name: 'Error', message: 'Unknown outcome', nativeCode: 'stale_process' } })
|
||||
const unknown = await app.drag([1, 2], [3, 4]).catch(error => error)
|
||||
expect(unknown).toMatchObject({ name: 'Error', message: 'Unknown outcome', nativeCode: 'stale_process' })
|
||||
expect(unknown.code).toBeUndefined()
|
||||
expect(unknown.errorName).toBeUndefined()
|
||||
f.results.push({ isError: true, content: [], nativeError: { name: 'TypeError', message: 'Invalid client argument' } })
|
||||
expect(await app.pressKey('Cmd+A').catch(error => error)).toBeInstanceOf(TypeError)
|
||||
})
|
||||
|
||||
test('inventory returns native app data and emits only when requested', async () => {
|
||||
const f = fixture()
|
||||
f.results.push({ content: [{ type: 'text', text: 'Fixture — com.test.fixture\nOther — com.test.other' }] })
|
||||
expect(await f.cua.listApps({ emit: false })).toEqual([
|
||||
{ id: 'com.test.fixture', displayName: 'Fixture', isRunning: true },
|
||||
{ id: 'com.test.other', displayName: 'Other', isRunning: true },
|
||||
])
|
||||
expect(f.emitted).toEqual([])
|
||||
f.results.push({ content: [{ type: 'text', text: 'No running applications are available to control.' }] })
|
||||
expect(await f.cua.getState()).toEqual({ apps: [], browsers: [] })
|
||||
expect(f.emitted).toHaveLength(1)
|
||||
expect(f.emitted[0]?.value).toContain('Native App API')
|
||||
expect(String(f.emitted[0]?.value).endsWith('{"apps":[],"browsers":[]}')).toBe(true)
|
||||
})
|
||||
|
||||
test('API guidance appears on the first visible selection only, never at bootstrap', async () => {
|
||||
const f = fixture()
|
||||
expect(f.emitted).toEqual([])
|
||||
f.results.push({ content: [] })
|
||||
await f.cua.getState({ emit: false })
|
||||
expect(f.emitted).toEqual([])
|
||||
await f.cua.getApp('Fixture')
|
||||
expect(f.emitted[0]?.value).toContain('Native App API')
|
||||
await f.cua.getApp('Fixture again')
|
||||
expect(f.emitted[1]?.value).toBe(full)
|
||||
f.results.push({ content: [] })
|
||||
await f.cua.getState()
|
||||
expect(f.emitted[2]?.value).toBe('{"apps":[],"browsers":[]}')
|
||||
})
|
||||
})
|
||||
|
||||
describe('worker bootstrap source', () => {
|
||||
test('JPEG captures retain bytes and MIME through automatic, raw and deferred output', async () => {
|
||||
const output: Array<Record<string, unknown>> = []
|
||||
const context = createContext({ __cuInvoke: async () => jpegState(), __cuEmit: (value: Record<string, unknown>) => output.push(value) })
|
||||
runInContext(REPL_BOOTSTRAP_SOURCE, context)
|
||||
await runInContext(`(async () => {
|
||||
const app = await cua.getApp('Fixture')
|
||||
await app.getScreenshot()
|
||||
await app.getAXStateAndScreenshot()
|
||||
const bytes = await app.getScreenshot({emit:false})
|
||||
await nodeRepl.emitImage(bytes)
|
||||
await nodeRepl.emitImage({bytes})
|
||||
const raw = await cua.computer.get_app_state({app:'Fixture'})
|
||||
nodeRepl.write(raw.screenshot.url)
|
||||
})()`, context)
|
||||
const images = output.filter(item => item.type === 'image')
|
||||
expect(images).toHaveLength(4)
|
||||
for (const image of images) {
|
||||
expect(image).toEqual({ type: 'image', data: jpeg, mimeType: 'image/jpeg' })
|
||||
expect(Buffer.from(image.data as string, 'base64')).toEqual(Buffer.from(jpeg, 'base64'))
|
||||
}
|
||||
expect(output.at(-1)).toEqual({ type: 'text', text: `data:image/jpeg;base64,${jpeg}` })
|
||||
})
|
||||
|
||||
test('the shared JS example performs two known drags and observes only once', async () => {
|
||||
const example = COMPUTER_USE_BATCHING_GUIDANCE.match(/```javascript\n([^`]+)\n```/)
|
||||
expect(example).not.toBeNull()
|
||||
const calls: Array<{ method: string; args: unknown }> = []
|
||||
const context = createContext({
|
||||
__cuInvoke: async (method: string, args: unknown) => {
|
||||
calls.push({ method, args })
|
||||
return state()
|
||||
},
|
||||
__cuEmit: () => {},
|
||||
})
|
||||
runInContext(REPL_BOOTSTRAP_SOURCE, context)
|
||||
await runInContext('cua.getApp("Canvas App").then(value => globalThis.app = value)', context)
|
||||
calls.length = 0
|
||||
await runInContext(`(async () => {${example![1]}})()`, context)
|
||||
expect(calls.map(call => call.method)).toEqual(['drag', 'drag', 'get_app_state'])
|
||||
expect(calls[0]?.args).toEqual({ app: '/Applications/Fixture.app', from_x: 240, from_y: 320, to_x: 241, to_y: 320 })
|
||||
expect(calls[1]?.args).toEqual({ app: '/Applications/Fixture.app', from_x: 280, from_y: 320, to_x: 281, to_y: 320 })
|
||||
})
|
||||
|
||||
test('runs with standard JavaScript globals, preserves app bindings and emits image bytes without Node', async () => {
|
||||
const calls: Array<{ method: string; args: unknown }> = []
|
||||
const output: Array<Record<string, unknown>> = []
|
||||
const context = createContext({
|
||||
__cuInvoke: async (method: string, args: unknown) => {
|
||||
calls.push({ method, args })
|
||||
return state()
|
||||
},
|
||||
__cuEmit: (content: Record<string, unknown>) => output.push(content),
|
||||
})
|
||||
runInContext(REPL_BOOTSTRAP_SOURCE, context)
|
||||
await runInContext('globalThis.app = awaitPromise = cua.getApp("Fixture")', context)
|
||||
await runInContext('app = awaitPromise.then(async a => { await a.click([1,2]); return a })', context)
|
||||
// Separate cells share variables, while the facade itself has no Node dependency.
|
||||
await runInContext('(async () => { const a = await app; const bytes = await a.getScreenshot({emit:false}); await nodeRepl.emitImage({bytes,mimeType:"image/png"}); await nodeRepl.write({length:bytes.length}); })()', context)
|
||||
expect(calls.map(call => call.method)).toEqual(['get_app_state', 'click', 'get_app_state'])
|
||||
expect(output.at(-2)).toEqual({ type: 'image', data: 'AAH+/w==', mimeType: 'image/png' })
|
||||
expect(output.at(-1)).toEqual({ type: 'text', text: '{"length":4}' })
|
||||
expect(runInContext('[typeof process, typeof Buffer, typeof require]', context)).toEqual(['undefined', 'undefined', 'undefined'])
|
||||
})
|
||||
|
||||
test('image emission roundtrips empty, padded and unpadded byte groups', async () => {
|
||||
const output: Array<Record<string, unknown>> = []
|
||||
const context = createContext({ __cuInvoke: async () => state(), __cuEmit: (value: Record<string, unknown>) => output.push(value) })
|
||||
runInContext(REPL_BOOTSTRAP_SOURCE, context)
|
||||
await runInContext('(async () => { for (let n = 0; n < 8; n++) await nodeRepl.emitImage(new Uint8Array(Array.from({length:n}, (_,i) => i * 31))); })()', context)
|
||||
for (let n = 0; n < 8; n++) {
|
||||
expect(output[n]?.data).toBe(Buffer.from(Array.from({ length: n }, (_, i) => i * 31)).toString('base64'))
|
||||
}
|
||||
await expect(runInContext('nodeRepl.emitImage("file:///private/fixture.png")', context)).rejects.toThrow('requires Uint8Array')
|
||||
})
|
||||
})
|
||||
+491
@@ -0,0 +1,491 @@
|
||||
import type { NativeErrorMetadata } from './nativeError.js'
|
||||
|
||||
/**
|
||||
* Native-app facade for the isolated Computer Use JavaScript worker.
|
||||
*
|
||||
* The official @oai/cua native facade uses get_app_state for all three
|
||||
* observations. Output selection and emit:false live here, not in the native
|
||||
* capture implementation. Every invocation still crosses the host's existing
|
||||
* semantic tool policy and process-identity validation.
|
||||
*/
|
||||
export interface ReplImage {
|
||||
data: string
|
||||
mimeType: string
|
||||
}
|
||||
|
||||
export interface ReplToolResult {
|
||||
content: Array<{ type: string; text?: string; data?: string; mimeType?: string }>
|
||||
isError?: boolean
|
||||
/** The approved app selector, when supplied by the host broker. */
|
||||
app?: string
|
||||
/** Structured, trusted app inventory; legacy hosts may supply only text. */
|
||||
apps?: Array<{ id: string; displayName?: string; isRunning?: boolean; lastUsedDate?: string; useCount?: number }>
|
||||
nativeError?: NativeErrorMetadata
|
||||
}
|
||||
|
||||
export interface ReplApiBridge {
|
||||
invoke(method: string, args: Record<string, unknown>): Promise<ReplToolResult>
|
||||
emit(type: 'text', value: string): void | Promise<void>
|
||||
emit(type: 'image', value: ReplImage): void | Promise<void>
|
||||
}
|
||||
|
||||
export interface ReplObservationOptions {
|
||||
emit?: boolean
|
||||
}
|
||||
|
||||
export interface ReplStateOptions extends ReplObservationOptions {
|
||||
disableDiffing?: boolean
|
||||
}
|
||||
|
||||
export type ReplPoint = [number, number]
|
||||
export type ReplElement = number | string
|
||||
export type ReplClickTarget = ReplElement | ReplPoint
|
||||
|
||||
export interface ReplClickOptions {
|
||||
mouseButton?: 'left' | 'right' | 'middle' | 'l' | 'r' | 'm' | 0 | 1 | 2
|
||||
clickCount?: number
|
||||
}
|
||||
|
||||
export interface ReplSelectTextOptions {
|
||||
prefix?: string
|
||||
suffix?: string
|
||||
selectionType?: 'text' | 'cursor_before' | 'cursor_after'
|
||||
}
|
||||
|
||||
/** Keep this function self-contained: its source executes in a separate realm. */
|
||||
export function createReplApi(bridge: ReplApiBridge) {
|
||||
type Indices = { generation?: string; handles: Map<number, string> }
|
||||
const indicesByApp = new Map<string, Indices>()
|
||||
const appAliases = new Map<string, string>()
|
||||
const handlePattern = /^g(0|[1-9]\d*):(0|[1-9]\d*)$/u
|
||||
let hasEmittedGuidance = false
|
||||
const appGuidance = `Native App API
|
||||
Bind with await cua.getApp("App name, bundle ID, or path"). Keep the returned app between cells.
|
||||
Observe: app.getAXState({disableDiffing?,emit?}), app.getScreenshot({emit?}), app.getAXStateAndScreenshot({disableDiffing?,emit?}). emit:false returns data without displaying it.
|
||||
Act: app.click([x,y] or element,{mouseButton?,clickCount?}), app.drag([x,y],[x,y]), app.pressKey(key), app.scroll([x,y] or element,direction,pages?), app.paste(text,{format?}), app.typeText(text), app.selectText(element,text,{prefix?,suffix?,selectionType?}), app.setValue(element,value), app.performSecondaryAction(element,action).
|
||||
The raw macOS window API is also available as cua.computer (target:"mac"): list_apps(), get_app_state({app,disableDiff?}), and the corresponding snake_case action methods with an explicit app. Raw methods return data without displaying it.
|
||||
Use observed gN:id handles. Integer indices require a current AX observation; after image-only capture use getAXState({disableDiffing:true}) before using integers. Coordinates refer to the returned screenshot.
|
||||
Await actions in loops, then observe at the next decision point. nodeRepl.write(value) emits text; nodeRepl.emitImage(bytes) emits an image. Browser/DOM, imports, Node, filesystem, and networking APIs are unavailable.`
|
||||
|
||||
function textOf(result: ReplToolResult): string {
|
||||
return result.content
|
||||
.filter(block => block.type === 'text' && typeof block.text === 'string')
|
||||
.map(block => block.text)
|
||||
.join('\n')
|
||||
}
|
||||
|
||||
async function invoke(method: string, args: Record<string, unknown>) {
|
||||
const result = await bridge.invoke(method, args)
|
||||
if (result.isError) {
|
||||
const metadata = result.nativeError
|
||||
const message = metadata?.message ?? (textOf(result) || `Computer Use ${method} failed`)
|
||||
const error = metadata?.name === 'TypeError' ? new TypeError(message) : new Error(message)
|
||||
if (metadata?.name === 'SkyComputerUseError') {
|
||||
error.name = metadata.name
|
||||
if (Number.isInteger(metadata.code)) Reflect.set(error, 'code', metadata.code)
|
||||
if (typeof metadata.errorName === 'string') Reflect.set(error, 'errorName', metadata.errorName)
|
||||
if (metadata.request === null) Reflect.set(error, 'request', null)
|
||||
if (metadata.requestType === 'jsonRPC') Reflect.set(error, 'requestType', metadata.requestType)
|
||||
}
|
||||
if (typeof metadata?.nativeCode === 'string') Reflect.set(error, 'nativeCode', metadata.nativeCode)
|
||||
throw error
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
function imageOf(result: ReplToolResult): ReplImage | undefined {
|
||||
const block = result.content.find(item => item.type === 'image')
|
||||
return block && typeof block.data === 'string' && typeof block.mimeType === 'string'
|
||||
? { data: block.data, mimeType: block.mimeType }
|
||||
: undefined
|
||||
}
|
||||
|
||||
function decodeImage(image: ReplImage): Uint8Array {
|
||||
const base64 = image.data
|
||||
if (!/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/u.test(base64)) {
|
||||
throw new Error('The screenshot contains invalid base64 data')
|
||||
}
|
||||
const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
|
||||
const padding = base64.endsWith('==') ? 2 : base64.endsWith('=') ? 1 : 0
|
||||
const bytes = new Uint8Array(base64.length / 4 * 3 - padding)
|
||||
let offset = 0
|
||||
for (let i = 0; i < base64.length; i += 4) {
|
||||
const n = (alphabet.indexOf(base64[i]!) << 18)
|
||||
| (alphabet.indexOf(base64[i + 1]!) << 12)
|
||||
| (Math.max(0, alphabet.indexOf(base64[i + 2]!)) << 6)
|
||||
| Math.max(0, alphabet.indexOf(base64[i + 3]!))
|
||||
if (offset < bytes.length) bytes[offset++] = (n >> 16) & 255
|
||||
if (offset < bytes.length) bytes[offset++] = (n >> 8) & 255
|
||||
if (offset < bytes.length) bytes[offset++] = n & 255
|
||||
}
|
||||
return bytes
|
||||
}
|
||||
|
||||
function clearIndices(indices: Indices) {
|
||||
indices.generation = undefined
|
||||
indices.handles.clear()
|
||||
}
|
||||
|
||||
function rememberState(indices: Indices, text: string) {
|
||||
// Read only the engine's AX envelope, never app-specific instructions that
|
||||
// happen to quote something resembling a handle.
|
||||
const start = text.indexOf('<app_state>\n')
|
||||
const end = text.lastIndexOf('\n</app_state>')
|
||||
const state = start >= 0 && end > start ? text.slice(start + 12, end) : text
|
||||
const lines = state.split('\n')
|
||||
const isDiff = lines.some(line => line.startsWith('The following is a diff from the previous accessibility tree ')
|
||||
|| line.startsWith('There has been no change in the accessibility tree '))
|
||||
if (!isDiff) clearIndices(indices)
|
||||
|
||||
for (const line of lines) {
|
||||
// A tree row starts with indentation and optionally the native diff marker.
|
||||
const match = /^[+~]?[\t ]*(g(0|[1-9]\d*):(0|[1-9]\d*))(?=\s|$)/u.exec(line)
|
||||
if (!match) continue
|
||||
const index = Number(match[3])
|
||||
if (!Number.isSafeInteger(index)) continue
|
||||
if (indices.generation !== match[2]) {
|
||||
clearIndices(indices)
|
||||
indices.generation = match[2]
|
||||
}
|
||||
indices.handles.set(index, match[1]!)
|
||||
}
|
||||
|
||||
// Iterate observed IDs rather than expanding arbitrary sparse ranges.
|
||||
for (const line of lines) {
|
||||
if (!line.startsWith('Removed element IDs: ')) continue
|
||||
for (const range of line.slice(21).split(', ')) {
|
||||
const match = /^(\d+)(?:-(\d+))?$/u.exec(range)
|
||||
if (!match) continue
|
||||
const low = Number(match[1])
|
||||
const high = Number(match[2] ?? match[1])
|
||||
for (const id of indices.handles.keys()) {
|
||||
if (id >= low && id <= high) indices.handles.delete(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function element(indices: Indices, target: ReplElement): string {
|
||||
if (typeof target === 'string' && handlePattern.test(target)) return target
|
||||
if (typeof target === 'number' && Number.isSafeInteger(target) && target >= 0) {
|
||||
const handle = indices.handles.get(target)
|
||||
if (handle) return handle
|
||||
throw new Error(`Element ${target} has no current observed handle. Call getAXState({disableDiffing:true}) before using an integer index.`)
|
||||
}
|
||||
throw new Error('Expected an observed integer element index or an opaque gN:id handle')
|
||||
}
|
||||
|
||||
function point(value: ReplPoint, name = 'coordinate'): ReplPoint {
|
||||
if (!Array.isArray(value) || !Number.isFinite(value[0]) || !Number.isFinite(value[1])) {
|
||||
throw new TypeError(`${name} must include finite x and y coordinates`)
|
||||
}
|
||||
// The official facade selects the first two tuple entries before passing
|
||||
// them to the macOS client; extra array properties are not native inputs.
|
||||
return [value[0], value[1]]
|
||||
}
|
||||
|
||||
function targetArgs(indices: Indices, target: ReplClickTarget) {
|
||||
if (Array.isArray(target)) {
|
||||
const [x, y] = point(target)
|
||||
return { x, y }
|
||||
}
|
||||
return { element_index: element(indices, target) }
|
||||
}
|
||||
|
||||
function mouseButton(value: ReplClickOptions['mouseButton']) {
|
||||
if (value === undefined) return undefined
|
||||
// @oai/sky uses 0/1/2. The older semantic tool uses 1/2/3: normalize
|
||||
// to names at this boundary so right clicks cannot become left clicks.
|
||||
const normalized = typeof value === 'string' ? value.trim().toLowerCase() : value
|
||||
if (normalized === 0 || normalized === 'left' || normalized === 'l') return 'left'
|
||||
if (normalized === 1 || normalized === 'right' || normalized === 'r') return 'right'
|
||||
if (normalized === 2 || normalized === 'middle' || normalized === 'm') return 'middle'
|
||||
throw new TypeError(typeof value === 'number'
|
||||
? 'mouseButton number must be 0, 1, or 2'
|
||||
: 'mouseButton must be left, right, middle, l, r, m, 0, 1, or 2')
|
||||
}
|
||||
|
||||
function scrollDirection(value: string) {
|
||||
const direction = value.trim().toLowerCase()
|
||||
if (direction === 'u' || direction === 'up') return 'up'
|
||||
if (direction === 'd' || direction === 'down') return 'down'
|
||||
if (direction === 'l' || direction === 'left') return 'left'
|
||||
if (direction === 'r' || direction === 'right') return 'right'
|
||||
throw new TypeError('direction must be up, down, left, or right')
|
||||
}
|
||||
|
||||
function validatePages(pages?: number) {
|
||||
if (pages !== undefined && (!Number.isFinite(pages) || pages <= 0)) {
|
||||
throw new TypeError('pages must be a finite number > 0')
|
||||
}
|
||||
}
|
||||
|
||||
function indicesFor(selector: string) {
|
||||
const app = appAliases.get(selector) ?? selector
|
||||
let indices = indicesByApp.get(app)
|
||||
if (!indices) {
|
||||
indices = { handles: new Map() }
|
||||
indicesByApp.set(app, indices)
|
||||
}
|
||||
return indices
|
||||
}
|
||||
|
||||
async function emitText(text: string, options?: ReplObservationOptions) {
|
||||
if (options?.emit !== false) await bridge.emit('text', text)
|
||||
}
|
||||
|
||||
async function emitImage(image: ReplImage, options?: ReplObservationOptions) {
|
||||
if (options?.emit !== false) await bridge.emit('image', image)
|
||||
}
|
||||
|
||||
async function emitSelectionText(text: string, options?: ReplObservationOptions) {
|
||||
if (options?.emit === false) return
|
||||
const prefix = hasEmittedGuidance ? '' : `${appGuidance}\n\n`
|
||||
hasEmittedGuidance = true
|
||||
await emitText(prefix + text)
|
||||
}
|
||||
|
||||
async function listApps(options?: ReplObservationOptions) {
|
||||
const result = await invoke('list_apps', {})
|
||||
const text = textOf(result)
|
||||
// The existing engine's formatter emits exactly "displayName — bundleId".
|
||||
// No extra application metadata is inferred from this text interface.
|
||||
const apps = result.apps ?? text.split('\n').flatMap(line => {
|
||||
const separator = line.lastIndexOf(' — ')
|
||||
if (separator < 0) return []
|
||||
const id = line.slice(separator + 3)
|
||||
if (!id) return []
|
||||
return [{ id, displayName: line.slice(0, separator), isRunning: true }]
|
||||
})
|
||||
await emitText(JSON.stringify(apps), options)
|
||||
return apps
|
||||
}
|
||||
|
||||
async function getState(options?: ReplObservationOptions) {
|
||||
const state = { apps: await listApps({ emit: false }), browsers: [] }
|
||||
await emitSelectionText(JSON.stringify(state), options)
|
||||
return state
|
||||
}
|
||||
|
||||
async function getApp(selector: string) {
|
||||
if (typeof selector !== 'string' || !selector.trim()) {
|
||||
throw new Error('getApp requires an explicit app name, bundle ID, or path')
|
||||
}
|
||||
const first = await invoke('get_app_state', { app: selector, disableDiff: true })
|
||||
// Official getApp binds the policy-approved app path returned by its first
|
||||
// observation, rather than caching a PID or trusting a guest host object.
|
||||
const app = typeof first.app === 'string' && first.app ? first.app : selector
|
||||
appAliases.set(selector, app)
|
||||
const observed = indicesFor(app)
|
||||
const firstText = textOf(first)
|
||||
rememberState(observed, firstText)
|
||||
await emitSelectionText(firstText)
|
||||
|
||||
async function observe(options?: ReplStateOptions) {
|
||||
try {
|
||||
return await invoke('get_app_state', {
|
||||
app,
|
||||
...(options?.disableDiffing === undefined ? {} : { disableDiff: options.disableDiffing }),
|
||||
})
|
||||
} catch (error) {
|
||||
clearIndices(observed)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
async function action(method: string, args: Record<string, unknown>) {
|
||||
await invoke(method, { app, ...args })
|
||||
}
|
||||
|
||||
return {
|
||||
async getAXState(options?: ReplStateOptions) {
|
||||
const text = textOf(await observe(options))
|
||||
rememberState(observed, text)
|
||||
await emitText(text, options)
|
||||
return text
|
||||
},
|
||||
async getScreenshot(options?: ReplObservationOptions) {
|
||||
// A hidden AX refresh may change generations. Do not create numeric
|
||||
// aliases for elements the caller did not receive in an AX observation.
|
||||
clearIndices(observed)
|
||||
const image = imageOf(await observe())
|
||||
if (!image) throw new Error(`Screenshot unavailable for ${app}.`)
|
||||
const bytes = decodeImage(image)
|
||||
await emitImage(image, options)
|
||||
return bytes
|
||||
},
|
||||
async getAXStateAndScreenshot(options?: ReplStateOptions) {
|
||||
const result = await observe(options)
|
||||
const state = textOf(result)
|
||||
rememberState(observed, state)
|
||||
await emitText(state, options)
|
||||
const image = imageOf(result)
|
||||
if (!image) return { state }
|
||||
const screenshot = decodeImage(image)
|
||||
await emitImage(image, options)
|
||||
return { state, screenshot }
|
||||
},
|
||||
async click(target: ReplClickTarget, options?: ReplClickOptions) {
|
||||
const button = mouseButton(options?.mouseButton)
|
||||
await action('click', {
|
||||
...targetArgs(observed, target),
|
||||
...(button === undefined ? {} : { mouse_button: button }),
|
||||
...(options?.clickCount === undefined ? {} : { click_count: options.clickCount }),
|
||||
})
|
||||
},
|
||||
async drag(from: ReplPoint, to: ReplPoint) {
|
||||
const [from_x, from_y] = point(from, 'from')
|
||||
const [to_x, to_y] = point(to, 'to')
|
||||
await action('drag', { from_x, from_y, to_x, to_y })
|
||||
},
|
||||
async pressKey(key: string) {
|
||||
if (key.trim() === '') throw new TypeError('key is required')
|
||||
await action('press_key', { key })
|
||||
},
|
||||
async scroll(target: ReplClickTarget, direction: string, pages?: number) {
|
||||
validatePages(pages)
|
||||
await action('scroll', {
|
||||
...targetArgs(observed, target), direction: scrollDirection(direction),
|
||||
...(pages === undefined ? {} : { pages }),
|
||||
})
|
||||
},
|
||||
async paste(text: string, options?: { format?: 'text' | 'md' | 'html' }) {
|
||||
await action('paste', { text, format: options?.format ?? 'text' })
|
||||
},
|
||||
async typeText(text: string) {
|
||||
await action('type_text', { text })
|
||||
},
|
||||
async selectText(index: ReplElement, text: string, options?: ReplSelectTextOptions) {
|
||||
await action('select_text', {
|
||||
element_index: element(observed, index), text,
|
||||
...(options?.prefix === undefined ? {} : { prefix: options.prefix }),
|
||||
...(options?.suffix === undefined ? {} : { suffix: options.suffix }),
|
||||
...(options?.selectionType === undefined ? {} : { selection_type: options.selectionType }),
|
||||
})
|
||||
},
|
||||
async setValue(index: ReplElement, value: string) {
|
||||
await action('set_value', { element_index: element(observed, index), value })
|
||||
},
|
||||
async performSecondaryAction(index: ReplElement, actionName: string) {
|
||||
await action('perform_secondary_action', { element_index: element(observed, index), action: actionName })
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
type NativeTarget = { app: string }
|
||||
type NativePointTarget = NativeTarget & { element_index?: ReplElement; x?: number; y?: number }
|
||||
async function nativeAction(method: string, input: NativeTarget & Record<string, unknown>) {
|
||||
if (typeof input.app !== 'string' || input.app.trim() === '') throw new TypeError('app is required')
|
||||
const args = { ...input }
|
||||
if (args.element_index !== undefined) args.element_index = element(indicesFor(input.app), args.element_index as ReplElement)
|
||||
await invoke(method, args)
|
||||
}
|
||||
function nativePoint(input: NativePointTarget) {
|
||||
return input.element_index !== undefined
|
||||
? { element_index: input.element_index }
|
||||
: { x: point([input.x!, input.y!])[0], y: input.y }
|
||||
}
|
||||
// The macOS sky client exposes this window API in addition to bound Apps.
|
||||
// It still uses the same guarded host operations; it cannot access a socket,
|
||||
// native object, audio capability, or another provider directly.
|
||||
const computer = {
|
||||
target: 'mac' as const,
|
||||
list_apps: () => listApps({ emit: false }),
|
||||
async get_app_state(input: NativeTarget & { disableDiff?: boolean }) {
|
||||
if (typeof input.app !== 'string' || input.app.trim() === '') throw new TypeError('app is required')
|
||||
const result = await invoke('get_app_state', {
|
||||
app: input.app,
|
||||
...(input.disableDiff === undefined ? {} : { disableDiff: input.disableDiff }),
|
||||
}).catch(error => {
|
||||
clearIndices(indicesFor(input.app))
|
||||
throw error
|
||||
})
|
||||
const app = typeof result.app === 'string' && result.app ? result.app : input.app
|
||||
appAliases.set(input.app, app)
|
||||
const text = textOf(result)
|
||||
rememberState(indicesFor(app), text)
|
||||
const image = imageOf(result)
|
||||
return { app, text, screenshot: image ? { url: `data:${image.mimeType};base64,${image.data}` } : null }
|
||||
},
|
||||
click: async (input: NativePointTarget & { mouse_button?: ReplClickOptions['mouseButton']; click_count?: number }) =>
|
||||
nativeAction('click', { app: input.app, ...nativePoint(input),
|
||||
...(input.mouse_button === undefined ? {} : { mouse_button: mouseButton(input.mouse_button) }),
|
||||
...(input.click_count === undefined ? {} : { click_count: input.click_count }) }),
|
||||
async drag(input: NativeTarget & { from_x: number; from_y: number; to_x: number; to_y: number }) {
|
||||
point([input.from_x, input.from_y], 'from')
|
||||
point([input.to_x, input.to_y], 'to')
|
||||
return nativeAction('drag', { app: input.app, from_x: input.from_x, from_y: input.from_y, to_x: input.to_x, to_y: input.to_y })
|
||||
},
|
||||
paste: async (input: NativeTarget & { text: string; format: 'text' | 'md' | 'html' }) =>
|
||||
nativeAction('paste', { app: input.app, text: input.text, format: input.format }),
|
||||
perform_secondary_action: async (input: NativeTarget & { element_index: ReplElement; action: string }) =>
|
||||
nativeAction('perform_secondary_action', { app: input.app, element_index: input.element_index, action: input.action }),
|
||||
async press_key(input: NativeTarget & { key: string }) {
|
||||
if (input.key.trim() === '') throw new TypeError('key is required')
|
||||
await nativeAction('press_key', { app: input.app, key: input.key })
|
||||
},
|
||||
async scroll(input: NativePointTarget & { direction: string; pages?: number }) {
|
||||
validatePages(input.pages)
|
||||
await nativeAction('scroll', { app: input.app, ...nativePoint(input), direction: scrollDirection(input.direction),
|
||||
...(input.pages === undefined ? {} : { pages: input.pages }) })
|
||||
},
|
||||
select_text: async (input: NativeTarget & { element_index: ReplElement; text: string; prefix?: string; suffix?: string; selection_type?: ReplSelectTextOptions['selectionType'] }) =>
|
||||
nativeAction('select_text', { app: input.app, element_index: input.element_index, text: input.text,
|
||||
...(input.prefix === undefined ? {} : { prefix: input.prefix }),
|
||||
...(input.suffix === undefined ? {} : { suffix: input.suffix }),
|
||||
...(input.selection_type === undefined ? {} : { selection_type: input.selection_type }) }),
|
||||
set_value: async (input: NativeTarget & { element_index: ReplElement; value: string }) =>
|
||||
nativeAction('set_value', { app: input.app, element_index: input.element_index, value: input.value }),
|
||||
type_text: async (input: NativeTarget & { text: string }) => nativeAction('type_text', { app: input.app, text: input.text }),
|
||||
}
|
||||
|
||||
return { getApp, listApps, getState, computer }
|
||||
}
|
||||
|
||||
/** Only guest-created data crosses this interface; it never reads a file/URL. */
|
||||
function createReplOutput(emit: (content: Record<string, unknown>) => void) {
|
||||
return {
|
||||
async write(value: unknown) {
|
||||
emit({ type: 'text', text: typeof value === 'string' ? value : JSON.stringify(value) ?? String(value) })
|
||||
},
|
||||
async emitImage(value: Uint8Array | { bytes: Uint8Array; mimeType?: string } | ReplImage) {
|
||||
if (value && typeof value === 'object' && 'data' in value && typeof value.data === 'string') {
|
||||
emit({ type: 'image', data: value.data, mimeType: value.mimeType })
|
||||
return
|
||||
}
|
||||
const bytes = value instanceof Uint8Array ? value
|
||||
: value && typeof value === 'object' && 'bytes' in value ? value.bytes : undefined
|
||||
if (!(bytes instanceof Uint8Array)) throw new Error('emitImage requires Uint8Array bytes or {bytes, mimeType}')
|
||||
const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
|
||||
const parts: string[] = []
|
||||
for (let i = 0; i < bytes.length; i += 3) {
|
||||
const a = bytes[i]!
|
||||
const b = bytes[i + 1] ?? 0
|
||||
const c = bytes[i + 2] ?? 0
|
||||
parts.push(alphabet[a >> 2]! + alphabet[((a & 3) << 4) | (b >> 4)]!
|
||||
+ (i + 1 < bytes.length ? alphabet[((b & 15) << 2) | (c >> 6)] : '=')
|
||||
+ (i + 2 < bytes.length ? alphabet[c & 63] : '='))
|
||||
}
|
||||
emit({
|
||||
type: 'image', data: parts.join(''),
|
||||
// A screenshot returned as bytes has no MIME property. macOS captures
|
||||
// are JPEG; keep the encoded image type when displaying those bytes.
|
||||
mimeType: 'mimeType' in value && value.mimeType ? value.mimeType
|
||||
: bytes[0] === 0xff && bytes[1] === 0xd8 && bytes[2] === 0xff ? 'image/jpeg' : 'image/png',
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
export const REPL_API_SOURCE = `(${createReplApi.toString()})`
|
||||
|
||||
/** The worker supplies these two realm-local, JSON-only bridge functions. */
|
||||
export const REPL_BOOTSTRAP_SOURCE = `
|
||||
globalThis.cua = ${REPL_API_SOURCE}({
|
||||
invoke: __cuInvoke,
|
||||
emit(type, value) {
|
||||
__cuEmit(type === 'text' ? { type: 'text', text: value } : { type: 'image', ...value })
|
||||
}
|
||||
});
|
||||
globalThis.nodeRepl = (${createReplOutput.toString()})(__cuEmit);
|
||||
`
|
||||
+219
@@ -0,0 +1,219 @@
|
||||
import { afterEach, describe, expect, test } from 'bun:test'
|
||||
import { bindSessionContext } from './mcpServer.js'
|
||||
import type { ComputerUseHostAdapter, ComputerUseSessionContext } from './types.js'
|
||||
import type { AppTarget, CodexComputerEngine, ComputerExecutor } from './executor.js'
|
||||
import { ComputerUseRepl } from '../../utils/computerUse/replRuntime.js'
|
||||
import { NativeCommandError } from './nativeError.js'
|
||||
|
||||
const runtimes: ComputerUseRepl[] = []
|
||||
afterEach(async () => { await Promise.all(runtimes.splice(0).map(runtime => runtime.reset())) })
|
||||
const suite = process.platform === 'darwin' ? describe : describe.skip
|
||||
|
||||
function fixture() {
|
||||
const calls: Array<{ name: string, target?: AppTarget }> = []
|
||||
const engine = {
|
||||
async resolveTarget(target: AppTarget) {
|
||||
calls.push({ name: 'resolve', target })
|
||||
return {
|
||||
pid: 123, bundleId: 'dev.test.Fixture', displayName: 'Fixture', path: '/Applications/Fixture.app',
|
||||
executablePath: '/Applications/Fixture.app/Contents/MacOS/Fixture', launchTime: 100,
|
||||
processIdentity: { pid: 123, bundleId: 'dev.test.Fixture', executablePath: '/Applications/Fixture.app/Contents/MacOS/Fixture', launchTime: 100 },
|
||||
}
|
||||
},
|
||||
async getAppState(target: AppTarget) {
|
||||
calls.push({ name: 'state', target })
|
||||
return {
|
||||
pid: 123, elementCount: 1, truncated: false, durationMs: 1,
|
||||
axText: 'App=Fixture\n\tg1:1 button Test',
|
||||
screenshot: { base64: 'AQID', width: 10, height: 10 },
|
||||
}
|
||||
},
|
||||
async click({ target }: { target: AppTarget }) { calls.push({ name: 'click', target }) },
|
||||
} as unknown as CodexComputerEngine
|
||||
let created = 0
|
||||
const adapter: ComputerUseHostAdapter = {
|
||||
serverName: 'test', logger: { silly() {}, debug() {}, info() {}, warn() {}, error() {} },
|
||||
executor: { capabilities: { platform: 'darwin', screenshotFiltering: 'native', hostBundleId: 'dev.test.host' }, engine } as ComputerExecutor,
|
||||
ensureOsPermissions: async () => ({ granted: true }),
|
||||
isDisabled: () => false, getAutoUnhideEnabled: () => true,
|
||||
getSubGates: () => ({ pixelValidation: false, clipboardPasteMultiline: false, mouseAnimation: false, hideBeforeAction: false, autoTargetDisplay: false, clipboardGuard: false }),
|
||||
cropRawPatch: () => null,
|
||||
createReplRuntime: () => {
|
||||
++created
|
||||
const runtime = new ComputerUseRepl()
|
||||
runtimes.push(runtime)
|
||||
return runtime
|
||||
},
|
||||
}
|
||||
const context: ComputerUseSessionContext = {
|
||||
getAllowedApps: () => [],
|
||||
getGrantFlags: () => ({ clipboardRead: false, clipboardWrite: false, systemKeyCombos: false }),
|
||||
getUserDeniedBundleIds: () => [], getSelectedDisplayId: () => undefined,
|
||||
}
|
||||
return { calls, engine, adapter, context, created: () => created }
|
||||
}
|
||||
|
||||
test('invalid JavaScript requests cannot create a kernel on any host platform', async () => {
|
||||
const f = fixture()
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
expect((await dispatch('js', { code: null })).isError).toBe(true)
|
||||
expect(f.created()).toBe(0)
|
||||
expect(f.calls).toEqual([])
|
||||
})
|
||||
|
||||
test('the session rejects invalid resets, disabled execution and hosts without an isolated runtime', async () => {
|
||||
const f = fixture()
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
for (const args of [null, [], { unexpected: true }]) expect((await dispatch('js_reset', args)).isError).toBe(true)
|
||||
f.adapter.isDisabled = () => true
|
||||
expect((await dispatch('js', { code: '1' })).isError).toBe(true)
|
||||
f.adapter.isDisabled = () => false
|
||||
delete f.adapter.createReplRuntime
|
||||
expect((await dispatch('js', { code: '1' })).content).toEqual([{ type: 'text', text: expect.stringContaining('does not provide an isolated') }])
|
||||
expect(f.created()).toBe(0)
|
||||
expect(f.calls).toEqual([])
|
||||
})
|
||||
|
||||
suite('persistent JavaScript through guarded session dispatch', () => {
|
||||
test('both lock conflict paths reject before native dispatch without an unknown action result', async () => {
|
||||
for (const conflictAfterAcquire of [false, true]) {
|
||||
const f = fixture()
|
||||
let checks = 0
|
||||
f.context.checkCuLock = async () => (++checks === 1 && conflictAfterAcquire)
|
||||
? { holder: undefined, isSelf: false } : { holder: 'another session', isSelf: false }
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
const result = await dispatch('js', { code: 'await cua.getApp("Fixture")' })
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.structuredContent).toMatchObject({
|
||||
nativeCallsCompleted: 0, nativeCallsRejectedBeforeDispatch: 1, nativeResultUnknown: false,
|
||||
})
|
||||
expect(f.calls).toEqual([])
|
||||
}
|
||||
})
|
||||
|
||||
test('a TCC refusal preserves its typed cause and reports that no action was dispatched', async () => {
|
||||
const f = fixture()
|
||||
f.adapter.ensureOsPermissions = async () => ({ granted: false })
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
const result = await dispatch('js', { code: 'await cua.getApp("Fixture")' })
|
||||
expect(result.isError).toBe(true)
|
||||
expect(result.structuredContent).toMatchObject({
|
||||
nativeCallsCompleted: 0, nativeCallsRejectedBeforeDispatch: 1, nativeResultUnknown: false,
|
||||
})
|
||||
const caught = await dispatch('js', { code: 'try { await cua.getApp("Fixture") } catch (e) { nodeRepl.write([e.name, e.code, e.errorName]) }' })
|
||||
expect(caught.content).toEqual([{ type: 'text', text: '["SkyComputerUseError",-10009,"permissionsNotGranted"]' }])
|
||||
expect(f.calls).toEqual([])
|
||||
})
|
||||
|
||||
test('preserves typed native errors across the worker and allows explicit recovery without replay', async () => {
|
||||
const f = fixture()
|
||||
let clicks = 0
|
||||
f.engine.click = async () => { ++clicks; throw new NativeCommandError('receiver disappeared', 'process_gone') }
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
expect((await dispatch('js', { code: 'let app = await cua.getApp("Fixture", { emit: false })' })).isError).not.toBe(true)
|
||||
const result = await dispatch('js', { code: 'try { await app.click([1,2]) } catch (e) { nodeRepl.write({ name: e.name, code: e.code, errorName: e.errorName, message: e.message, nativeCode: e.nativeCode, request: e.request, requestType: e.requestType }) }' })
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(result.content).toEqual([{ type: 'text', text: JSON.stringify({
|
||||
name: 'SkyComputerUseError', code: -10007, errorName: 'runningApplicationNotFound',
|
||||
message: 'receiver disappeared', nativeCode: 'process_gone', request: null, requestType: 'jsonRPC',
|
||||
}) }])
|
||||
expect(clicks).toBe(1)
|
||||
expect((await dispatch('js', { code: 'await app.getAXState({ emit: false })' })).isError).not.toBe(true)
|
||||
expect(clicks).toBe(1)
|
||||
})
|
||||
|
||||
test('native inventory survives the isolated worker RPC without adding observations', async () => {
|
||||
const f = fixture()
|
||||
const apps = [
|
||||
{ id: 'dev.test.Editor', displayName: 'Editor', isRunning: false, lastUsedDate: '2026-09-09T01:00:00Z', useCount: 4 },
|
||||
{ id: 'dev.test.Unknown' },
|
||||
]
|
||||
let enumerations = 0
|
||||
f.engine.listAppsInfo = async () => { ++enumerations; return apps }
|
||||
f.engine.listApps = async () => { throw new Error('must not enumerate twice') }
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
const result = await dispatch('js', { code: 'nodeRepl.write(await cua.listApps({ emit: false }))' })
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(result.content).toEqual([{ type: 'text', text: JSON.stringify(apps) }])
|
||||
expect(enumerations).toBe(1)
|
||||
expect(f.calls).toEqual([])
|
||||
})
|
||||
|
||||
test('binds the approved app path, rechecks every action and emits only requested observations', async () => {
|
||||
const f = fixture()
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
const first = await dispatch('js', { code: 'let app = await cua.getApp("Fixture")' })
|
||||
expect(first.isError).not.toBe(true)
|
||||
expect(first.content.filter(block => block.type === 'image')).toHaveLength(0)
|
||||
const next = await dispatch('js', {
|
||||
code: 'for (let i = 0; i < 3; i++) await app.click([1,2]); await app.getAXStateAndScreenshot()',
|
||||
})
|
||||
expect(next.isError).not.toBe(true)
|
||||
expect(f.created()).toBe(1)
|
||||
expect(f.calls.filter(call => call.name === 'resolve').map(call => call.target)).toEqual([
|
||||
{ app: 'Fixture' }, ...Array(4).fill({ app: '/Applications/Fixture.app' }),
|
||||
])
|
||||
expect(f.calls.filter(call => call.name === 'click')).toHaveLength(3)
|
||||
expect(f.calls.filter(call => call.name === 'state')).toHaveLength(2)
|
||||
expect(next.content.filter(block => block.type === 'image')).toEqual([{ type: 'image', data: 'AQID', mimeType: 'image/png' }])
|
||||
})
|
||||
|
||||
test('validates the cell before creating a kernel and keeps pure JS out of TCC and lock gates', async () => {
|
||||
const f = fixture()
|
||||
f.adapter.ensureOsPermissions = async () => { throw new Error('unexpected TCC') }
|
||||
f.context.checkCuLock = async () => { throw new Error('unexpected lock') }
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
for (const args of [{ code: '1', timeout_ms: 0 }, { code: null }, { code: '1', injected: true }]) {
|
||||
expect((await dispatch('js', args)).isError).toBe(true)
|
||||
}
|
||||
expect(f.created()).toBe(0)
|
||||
const pure = await dispatch('js', { code: 'let counter = 1; nodeRepl.write(++counter)' })
|
||||
expect(pure.isError).not.toBe(true)
|
||||
expect(pure.content).toEqual([{ type: 'text', text: '2' }])
|
||||
expect(f.calls).toEqual([])
|
||||
})
|
||||
|
||||
test('a JavaScript cell owns the outer queue while its internal guarded actions run', async () => {
|
||||
const f = fixture()
|
||||
let entered!: () => void
|
||||
const enteredPromise = new Promise<void>(resolve => { entered = resolve })
|
||||
let release!: () => void
|
||||
const barrier = new Promise<void>(resolve => { release = resolve })
|
||||
const order: number[] = []
|
||||
f.engine.click = async options => {
|
||||
order.push(options.x!)
|
||||
if (options.x === 1) { entered(); await barrier }
|
||||
}
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
const cell = dispatch('js', { code: 'let app = await cua.getApp("Fixture"); await app.click([1,1]); await app.click([2,2])' })
|
||||
await enteredPromise
|
||||
const ordinary = dispatch('click', { app: 'Fixture', x: 3, y: 3 })
|
||||
expect(order).toEqual([1])
|
||||
release()
|
||||
expect((await cell).isError).not.toBe(true)
|
||||
expect((await ordinary).isError).not.toBe(true)
|
||||
expect(order).toEqual([1, 2, 3])
|
||||
})
|
||||
|
||||
test('reset interrupts the active cell and invalidates queued cells without replaying input', async () => {
|
||||
const f = fixture()
|
||||
let entered!: () => void
|
||||
const enteredPromise = new Promise<void>(resolve => { entered = resolve })
|
||||
let release!: () => void
|
||||
const barrier = new Promise<void>(resolve => { release = resolve })
|
||||
let clicked = 0
|
||||
f.engine.click = async () => { ++clicked; entered(); await barrier }
|
||||
const dispatch = bindSessionContext(f.adapter, 'pixels', f.context)
|
||||
const running = dispatch('js', { code: 'let app = await cua.getApp("Fixture"); await app.click([1,1]); await app.click([2,2])' })
|
||||
await enteredPromise
|
||||
const queued = dispatch('js', { code: 'await app.click([3,3])' })
|
||||
const reset = dispatch('js_reset', {})
|
||||
release()
|
||||
expect((await running).isError).toBe(true)
|
||||
expect((await queued).isError).toBe(true)
|
||||
expect((await reset).isError).not.toBe(true)
|
||||
expect(clicked).toBe(1)
|
||||
const next = await dispatch('js', { code: 'nodeRepl.write(typeof app)' })
|
||||
expect(next.content).toEqual([{ type: 'text', text: 'undefined' }])
|
||||
})
|
||||
})
|
||||
+33
@@ -0,0 +1,33 @@
|
||||
import type { CuCallToolResult } from './toolCalls.js'
|
||||
|
||||
export type ReplContent = { type: 'text', text: string } | { type: 'image', data: string, mimeType: string }
|
||||
|
||||
/** Only JSON crosses the untrusted kernel boundary. No native/host objects. */
|
||||
export type ReplInput =
|
||||
| { type: 'init', bootstrap: string }
|
||||
| { type: 'ping', nonce: number }
|
||||
| { type: 'run', cellId: number, code: string }
|
||||
| { type: 'response', cellId: number, requestId: number, result?: unknown, error?: string }
|
||||
|
||||
export type ReplOutput =
|
||||
| { type: 'ready' }
|
||||
| { type: 'pong', nonce: number }
|
||||
| { type: 'invoke', cellId: number, requestId: number, name: string, args: unknown }
|
||||
| { type: 'emit', cellId: number, content: ReplContent }
|
||||
| { type: 'done', cellId: number, error?: string }
|
||||
|
||||
export type ReplInvoke = (name: string, args: unknown, signal: AbortSignal) => Promise<CuCallToolResult>
|
||||
|
||||
export interface ComputerUseReplRuntime {
|
||||
run(options: {
|
||||
code: string
|
||||
timeoutMs: number
|
||||
signal?: AbortSignal
|
||||
isAborted?: () => boolean
|
||||
}, invoke: ReplInvoke): Promise<CuCallToolResult>
|
||||
reset(): Promise<void>
|
||||
}
|
||||
|
||||
export const REPL_MAX_CODE_BYTES = 256 * 1024
|
||||
export const REPL_MAX_OUTPUT_BYTES = 16 * 1024 * 1024
|
||||
export const REPL_MAX_ACTIONS = 256
|
||||
+315
-10
@@ -1,4 +1,4 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { describe, expect, spyOn, test } from 'bun:test'
|
||||
|
||||
import type {
|
||||
AppStateResult,
|
||||
@@ -10,6 +10,7 @@ import type {
|
||||
} from './executor.js'
|
||||
import {
|
||||
_test,
|
||||
APP_INVENTORY,
|
||||
CUA_APP_VERSION,
|
||||
defersLockAcquire,
|
||||
frameAppStateEnvelope,
|
||||
@@ -17,11 +18,13 @@ import {
|
||||
resetMouseButtonHeld,
|
||||
} from './toolCalls.js'
|
||||
import { buildComputerUseTools } from './tools.js'
|
||||
import { bindSessionContext } from './mcpServer.js'
|
||||
import { COMPUTER_USE_INSTRUCTIONS } from './instructions.js'
|
||||
import { isSystemKeyCombo } from './keyBlocklist.js'
|
||||
import type {
|
||||
ComputerUseHostAdapter,
|
||||
ComputerUseOverrides,
|
||||
ComputerUseSessionContext,
|
||||
} from './types.js'
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -216,9 +219,12 @@ describe('buildComputerUseTools — current Codex tool face', () => {
|
||||
test('exposes the current Codex tools, including explicit paste', () => {
|
||||
expect(tools.map(t => t.name).sort()).toEqual(
|
||||
[
|
||||
'js',
|
||||
'js_reset',
|
||||
'click',
|
||||
'drag',
|
||||
'get_app_state',
|
||||
'sequence',
|
||||
'list_apps',
|
||||
'perform_secondary_action',
|
||||
'paste',
|
||||
@@ -250,7 +256,7 @@ describe('buildComputerUseTools — current Codex tool face', () => {
|
||||
test('every targeted tool requires an explicit app while list_apps does not', () => {
|
||||
for (const tool of tools) {
|
||||
const required = ((tool.inputSchema as any).required ?? []) as string[]
|
||||
if (tool.name === 'list_apps') {
|
||||
if (['list_apps', 'js', 'js_reset'].includes(tool.name)) {
|
||||
expect(required).not.toContain('app')
|
||||
} else {
|
||||
expect(required).toContain('app')
|
||||
@@ -389,10 +395,11 @@ describe('buildComputerUseTools — current Codex tool face', () => {
|
||||
}
|
||||
})
|
||||
|
||||
test('list_apps describes the native alphabetical running-app order honestly', () => {
|
||||
test('list_apps describes native running and recent inventory without implying access', () => {
|
||||
const description = tools.find(t => t.name === 'list_apps')!.description ?? ''
|
||||
expect(description).toContain('alphabetical')
|
||||
expect(description).not.toContain('most-recently-used')
|
||||
expect(description).toContain('running and recently used')
|
||||
expect(description).toContain('Discovery does not grant access')
|
||||
expect(description).not.toContain('alphabetical')
|
||||
})
|
||||
|
||||
test('every schema rejects unknown properties', () => {
|
||||
@@ -410,8 +417,8 @@ describe('buildComputerUseTools — current Codex tool face', () => {
|
||||
expect(annotations, tool.name).toBeDefined()
|
||||
const readOnly = tool.name === 'list_apps' || tool.name === 'get_app_state'
|
||||
expect(annotations.readOnlyHint, tool.name).toBe(readOnly)
|
||||
expect(annotations.idempotentHint, tool.name).toBe(readOnly)
|
||||
expect(annotations.destructiveHint, tool.name).toBe(false)
|
||||
expect(annotations.idempotentHint, tool.name).toBe(readOnly || tool.name === 'js_reset')
|
||||
expect(annotations.destructiveHint, tool.name).toBe(tool.name === 'js')
|
||||
expect(annotations.openWorldHint, tool.name).toBe(false)
|
||||
}
|
||||
})
|
||||
@@ -437,7 +444,7 @@ describe('buildComputerUseTools — current Codex tool face', () => {
|
||||
'normalized_0_100',
|
||||
['Finder', 'Slack'],
|
||||
)
|
||||
expect(withArgs).toHaveLength(11)
|
||||
expect(withArgs).toHaveLength(14)
|
||||
})
|
||||
})
|
||||
|
||||
@@ -498,6 +505,16 @@ describe('frameAppStateEnvelope', () => {
|
||||
expect(imgs[0].data).toBe('AAAA')
|
||||
})
|
||||
|
||||
test('preserves JPEG bytes and their declared MIME type from native capture', () => {
|
||||
const result = frameAppStateEnvelope({
|
||||
pid: 1, elementCount: 0, truncated: false, durationMs: 1, axText: 'Fixture',
|
||||
screenshot: { base64: '/9j/2Q==', width: 1397, height: 768, mimeType: 'image/jpeg' },
|
||||
})
|
||||
expect(result.content.filter(block => block.type === 'image')).toEqual([
|
||||
{ type: 'image', data: '/9j/2Q==', mimeType: 'image/jpeg' },
|
||||
])
|
||||
})
|
||||
|
||||
test('no image block when screenshot is absent (capture failed)', () => {
|
||||
const out = frameAppStateEnvelope({
|
||||
pid: 1, elementCount: 0, truncated: false, durationMs: 1, axText: 'App=x (pid 1)',
|
||||
@@ -587,6 +604,15 @@ describe('arg parsing helpers', () => {
|
||||
})).toBeUndefined()
|
||||
})
|
||||
|
||||
test('native policy uses the official exact identities instead of legacy app categories or names', () => {
|
||||
for (const bundleId of ['com.microsoft.VSCode', 'com.apple.Music', 'com.spotify.client', 'com.tradingview.tradingviewapp.desktop', 'dev.test.Terminal']) {
|
||||
expect(_test.policyDenyMessage({ bundleId, displayName: 'Terminal Music Editor' })).toBeUndefined()
|
||||
}
|
||||
for (const bundleId of ['com.raphaelamorim.rio', 'dev.commandline.waveterm', 'com.openai.codex.beta', 'com.openai.chat.mac-debug', 'com.apple.SecurityAgent']) {
|
||||
expect(_test.policyDenyMessage({ bundleId, displayName: 'Fixture' })).toContain('not allowed')
|
||||
}
|
||||
})
|
||||
|
||||
test('policyDenyMessage permanently denies the host and helper while a host override only adds', () => {
|
||||
const customHostBundleId = 'com.example.custom-host'
|
||||
for (const bundleId of [
|
||||
@@ -685,6 +711,24 @@ describe('handleToolCall — gates', () => {
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
})
|
||||
|
||||
test('native browser actions follow the observed official Chrome App path with normal identity checks', async () => {
|
||||
const executablePath = '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
|
||||
const identity = { pid: 812, bundleId: 'com.google.Chrome', executablePath, launchTime: 1812 }
|
||||
for (const app of ['Google Chrome', 'com.google.Chrome', '/Applications/Google Chrome.app', '812']) {
|
||||
const { engine, calls } = makeEngine({ resolveTarget: async () => ({
|
||||
pid: 812, bundleId: 'com.google.Chrome', displayName: 'Google Chrome',
|
||||
path: '/Applications/Google Chrome.app', executablePath, launchTime: 1812,
|
||||
processIdentity: identity,
|
||||
}) })
|
||||
const adapter = makeAdapter({ engine })
|
||||
expect((await handleToolCall(adapter, 'get_app_state', { app }, baseOverrides())).isError).not.toBe(true)
|
||||
const result = await handleToolCall(adapter, 'drag', { app, from_x: 10, from_y: 20, to_x: 11, to_y: 20 }, baseOverrides())
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'getAppState', 'resolveTarget', 'drag'])
|
||||
expect(calls.at(-1)?.args).toMatchObject({ target: { pid: 812, expectedProcessIdentity: identity } })
|
||||
}
|
||||
})
|
||||
|
||||
test('configured host bundle is refused before permission or engine dispatch', async () => {
|
||||
const customHostBundleId = 'com.example.custom-host'
|
||||
const { engine, calls } = makeEngine({
|
||||
@@ -1142,7 +1186,7 @@ describe('handleToolCall — gates', () => {
|
||||
})
|
||||
|
||||
test('press_key blocks every dangerous macOS shortcut alias when systemKeyCombos is false', async () => {
|
||||
const metaAliases = ['cmd', 'super', 'command', 'meta']
|
||||
const metaAliases = ['cmd', 'super', 'command', 'meta', 'Super_L', 'Super_R', 'Meta_L', 'Meta_R']
|
||||
const dangerous = [
|
||||
...metaAliases.flatMap(meta => [
|
||||
`${meta}+q`,
|
||||
@@ -1302,6 +1346,19 @@ describe('handleToolCall — gates', () => {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
describe('handleToolCall — tool dispatch', () => {
|
||||
test('keeps structured app inventory across dispatch with one enumeration', async () => {
|
||||
const apps = [{ id: 'dev.test.Editor', displayName: 'Editor', isRunning: false, useCount: 4 }]
|
||||
const { engine, calls } = makeEngine()
|
||||
let enumerations = 0
|
||||
engine.listAppsInfo = async () => { ++enumerations; return apps }
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'list_apps', {}, baseOverrides())
|
||||
expect(r.isError).not.toBe(true)
|
||||
expect(r[APP_INVENTORY]).toEqual(apps)
|
||||
expect(textOf(r)).toBe('Editor — dev.test.Editor')
|
||||
expect(enumerations).toBe(1)
|
||||
expect(calls).toEqual([])
|
||||
})
|
||||
|
||||
test('list_apps returns the engine text verbatim', async () => {
|
||||
const { engine } = makeEngine({ listApps: async () => 'A — a.b [running]' })
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'list_apps', {}, baseOverrides())
|
||||
@@ -1548,7 +1605,7 @@ describe('handleToolCall — tool dispatch', () => {
|
||||
|
||||
test('server guidance treats AX diffs and timed-out paste as non-authoritative', () => {
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('An empty AX diff does')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('paste({ app, text, format: "text" })')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('app.paste(text,{format:"text"})')
|
||||
expect(COMPUTER_USE_INSTRUCTIONS).toContain('treat the result as unknown')
|
||||
})
|
||||
|
||||
@@ -1730,3 +1787,251 @@ describe('resetMouseButtonHeld', () => {
|
||||
expect(() => resetMouseButtonHeld()).not.toThrow()
|
||||
})
|
||||
})
|
||||
|
||||
// A sequence is a bounded set of existing native operations, not a script.
|
||||
describe('same-app sequence', () => {
|
||||
const steps = [{ tool: 'press_key', key: 's x 1 period 3 5 Return' }, { tool: 'click', x: 12, y: 24 }]
|
||||
test('runs in order against one proven identity and returns one final screenshot', async () => {
|
||||
const { engine, calls } = makeEngine({ getAppState: () => ({ pid: 1234, appName: 'Finder', bundleId: 'com.apple.finder', windowTitle: 'Docs', elementCount: 0, truncated: false, durationMs: 1, axText: '', screenshot: { base64: 'PNG', width: 10, height: 10 } }) })
|
||||
const result = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps }, baseOverrides())
|
||||
expect(result.isError).not.toBe(true)
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey', 'click', 'getAppState'])
|
||||
const first = (calls[1]!.args as any).target
|
||||
expect(first.expectedProcessIdentity).toBeDefined()
|
||||
expect((calls[2]!.args as any).target).toEqual(first)
|
||||
expect(calls[3]!.args).toEqual(first)
|
||||
expect(result.structuredContent?.completedSteps).toBe(2)
|
||||
expect(imageBlocks(result)).toHaveLength(1)
|
||||
})
|
||||
test('validates all steps before any engine or permission call', async () => {
|
||||
for (const invalid of [
|
||||
{ tool: 'click', x: -1, y: 0 },
|
||||
{ tool: 'press_key', key: 'a super+q' },
|
||||
{ tool: 'press_key', key: 'a', app: 'Terminal' },
|
||||
{ tool: 'sequence', steps: [] },
|
||||
{ tool: 'press_key', key: Array(129).fill('a').join(' ') },
|
||||
]) {
|
||||
const { engine, calls } = makeEngine()
|
||||
const adapter = makeAdapter({ engine })
|
||||
adapter.ensureOsPermissions = async () => { throw new Error('must preflight first') }
|
||||
const r = await handleToolCall(adapter, 'sequence', { app: 'Finder', steps: [steps[0], invalid] }, baseOverrides())
|
||||
expect(r.isError).toBe(true)
|
||||
expect(calls).toHaveLength(0)
|
||||
}
|
||||
})
|
||||
test('stops at first native failure without replay or subsequent input', async () => {
|
||||
const { engine, calls } = makeEngine({ click: () => { throw new Error('execution result is unknown: timed out') } })
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps: [...steps, steps[0]] }, baseOverrides())
|
||||
expect(r.isError).toBe(true)
|
||||
expect(r.structuredContent).toMatchObject({ completedSteps: 1, failedStepIndex: 1, resultUnknown: true })
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey', 'click'])
|
||||
})
|
||||
test('cancellation after an awaited action prevents every subsequent dispatch', async () => {
|
||||
let aborted = false
|
||||
const { engine, calls } = makeEngine({ pressKey: () => { aborted = true } })
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps }, baseOverrides({ isAborted: () => aborted }))
|
||||
expect(r.isError).toBe(true)
|
||||
expect(r.structuredContent).toMatchObject({ completedSteps: 1, status: 'cancelled' })
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey'])
|
||||
})
|
||||
test('revoked kill switch stops a sequence and a fresh call never dispatches', async () => {
|
||||
let disabled = false
|
||||
const { engine, calls } = makeEngine({ pressKey: () => { disabled = true } })
|
||||
const adapter = makeAdapter({ engine }); adapter.isDisabled = () => disabled
|
||||
const r = await handleToolCall(adapter, 'sequence', { app: 'Finder', steps }, baseOverrides())
|
||||
expect(r.isError).toBe(true)
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey'])
|
||||
})
|
||||
test('enforces step and aggregate chord budgets before dispatch', async () => {
|
||||
for (const list of [[], Array(257).fill(steps[0]), Array(17).fill({ tool: 'press_key', key: Array(128).fill('a').join(' ') })]) {
|
||||
const { engine, calls } = makeEngine()
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps: list }, baseOverrides())
|
||||
expect(r.isError).toBe(true)
|
||||
expect(calls).toHaveLength(0)
|
||||
}
|
||||
})
|
||||
test('standalone macOS keyboard macros use the same 128-chord preflight limit', async () => {
|
||||
const { engine, calls } = makeEngine()
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'press_key', {
|
||||
app: 'Finder', key: Array(129).fill('a').join(' '),
|
||||
}, baseOverrides())
|
||||
expect(r.isError).toBe(true)
|
||||
expect(calls).toHaveLength(0)
|
||||
})
|
||||
})
|
||||
|
||||
function sessionContext(extra: Partial<ComputerUseSessionContext> = {}): ComputerUseSessionContext {
|
||||
return {
|
||||
getAllowedApps: () => [], getGrantFlags: () => ({ clipboardRead: true, clipboardWrite: true, systemKeyCombos: false }),
|
||||
getUserDeniedBundleIds: () => [], getSelectedDisplayId: () => undefined,
|
||||
...extra,
|
||||
}
|
||||
}
|
||||
|
||||
describe('sequence session coordination', () => {
|
||||
test('ordinary actions cannot interleave; queued cancellation never reaches the engine', async () => {
|
||||
let release!: () => void
|
||||
let entered!: () => void
|
||||
const waiting = new Promise<void>(resolve => { release = resolve })
|
||||
const started = new Promise<void>(resolve => { entered = resolve })
|
||||
const { engine, calls } = makeEngine({ pressKey: async () => { entered(); await waiting } })
|
||||
const dispatch = bindSessionContext(makeAdapter({ engine }), 'pixels', sessionContext())
|
||||
const sequence = dispatch('sequence', { app: 'Finder', steps: [{ tool: 'press_key', key: 'a' }, { tool: 'click', x: 1, y: 2 }] })
|
||||
await started
|
||||
const abort = new AbortController()
|
||||
const cancelled = dispatch('click', { app: 'TextEdit', x: 9, y: 9 }, abort.signal)
|
||||
const ordinary = dispatch('click', { app: 'Finder', x: 3, y: 4 })
|
||||
abort.abort()
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey'])
|
||||
release()
|
||||
await sequence
|
||||
expect((await cancelled).isError).toBe(true)
|
||||
await ordinary
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey', 'click', 'getAppState', 'resolveTarget', 'click'])
|
||||
expect((calls.at(-1)!.args as any).x).toBe(3)
|
||||
})
|
||||
test('explicit call signal stays cancelled and stops a running sequence', async () => {
|
||||
const abort = new AbortController()
|
||||
const { engine, calls } = makeEngine({ pressKey: () => abort.abort() })
|
||||
const dispatch = bindSessionContext(makeAdapter({ engine }), 'pixels', sessionContext({ isAborted: () => false }))
|
||||
const r = await dispatch('sequence', { app: 'Finder', steps: [{ tool: 'press_key', key: 'a' }, { tool: 'press_key', key: 'b' }] }, abort.signal)
|
||||
expect(r.structuredContent).toMatchObject({ status: 'cancelled', completedSteps: 1 })
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey'])
|
||||
})
|
||||
test('another session holder prevents a sequence from resolving or acting', async () => {
|
||||
const { engine, calls } = makeEngine()
|
||||
const dispatch = bindSessionContext(makeAdapter({ engine }), 'pixels', sessionContext({ checkCuLock: async () => ({ holder: 'other', isSelf: false }) }))
|
||||
const r = await dispatch('sequence', { app: 'Finder', steps: [{ tool: 'press_key', key: 'a' }] })
|
||||
expect(r.telemetry?.error_kind).toBe('cu_lock_held')
|
||||
expect(calls).toHaveLength(0)
|
||||
})
|
||||
test('deadline is cooperative: no detached or subsequent mutation after in-flight completion', async () => {
|
||||
let now = 0
|
||||
const clock = spyOn(performance, 'now').mockImplementation(() => now)
|
||||
try {
|
||||
const { engine, calls } = makeEngine({ pressKey: () => { now = 60_001 } })
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps: [{ tool: 'press_key', key: 'a' }, { tool: 'press_key', key: 'b' }] }, baseOverrides())
|
||||
expect(r.structuredContent).toMatchObject({ status: 'deadline_exceeded', completedSteps: 1 })
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey'])
|
||||
} finally { clock.mockRestore() }
|
||||
})
|
||||
test('native stale identity stops input without resolving a replacement process', async () => {
|
||||
const { engine, calls } = makeEngine({ click: () => { throw new Error('The target process changed') } })
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps: [{ tool: 'click', x: 1, y: 2 }, { tool: 'press_key', key: 'a' }] }, baseOverrides())
|
||||
expect(r.isError).toBe(true)
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'click'])
|
||||
})
|
||||
test('reports successful mutations separately from a missing final screenshot', async () => {
|
||||
const { engine } = makeEngine()
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'Finder', steps: [{ tool: 'press_key', key: 'a' }] }, baseOverrides())
|
||||
expect(r.isError).toBe(true)
|
||||
expect(r.structuredContent).toMatchObject({ status: 'observation_failed', completedSteps: 1 })
|
||||
})
|
||||
})
|
||||
|
||||
test('abort while awaiting lock check never acquires a new turn lock', async () => {
|
||||
const abort = new AbortController()
|
||||
let acquired = false
|
||||
const { engine, calls } = makeEngine()
|
||||
const dispatch = bindSessionContext(makeAdapter({ engine }), 'pixels', sessionContext({
|
||||
checkCuLock: async () => { abort.abort(); return { holder: undefined, isSelf: false } },
|
||||
acquireCuLock: async () => { acquired = true },
|
||||
}))
|
||||
expect((await dispatch('sequence', { app: 'Finder', steps: [{ tool: 'press_key', key: 'a' }] }, abort.signal)).isError).toBe(true)
|
||||
expect(acquired).toBe(false)
|
||||
expect(calls).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('sequence treats a generic exception after mutation as result-unknown', async () => {
|
||||
let effectApplied = false
|
||||
const { engine, calls } = makeEngine({ click: () => {
|
||||
effectApplied = true
|
||||
throw new Error('reply decoding failed')
|
||||
} })
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'sequence', {
|
||||
app: 'Finder', steps: [{ tool: 'click', x: 1, y: 2 }, { tool: 'press_key', key: 'a' }],
|
||||
}, baseOverrides())
|
||||
expect(effectApplied).toBe(true)
|
||||
expect(r.structuredContent).toMatchObject({ completedSteps: 0, failedStepIndex: 0, resultUnknown: true })
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'click'])
|
||||
})
|
||||
|
||||
test('cancellation after final observation reports it was captured but not delivered', async () => {
|
||||
let aborted = false
|
||||
const { engine, calls } = makeEngine({ getAppState: () => {
|
||||
aborted = true
|
||||
return { pid: 1234, appName: 'Finder', bundleId: 'com.apple.finder', windowTitle: 'Docs', elementCount: 0, truncated: false, durationMs: 1, axText: '', screenshot: { base64: 'PNG', width: 10, height: 10 } }
|
||||
} })
|
||||
const r = await handleToolCall(makeAdapter({ engine }), 'sequence', {
|
||||
app: 'Finder', steps: [{ tool: 'press_key', key: 'a' }],
|
||||
}, baseOverrides({ isAborted: () => aborted }))
|
||||
expect(r.structuredContent).toMatchObject({ status: 'cancelled', completedSteps: 1, observationSkipped: false, observationDelivered: false })
|
||||
expect(calls.map(c => c.method)).toEqual(['resolveTarget', 'pressKey', 'getAppState'])
|
||||
expect(imageBlocks(r)).toHaveLength(0)
|
||||
})
|
||||
|
||||
describe('canvas action batches', () => {
|
||||
const state: AppStateResult = {
|
||||
pid: 1234, appName: 'X', bundleId: 'com.test.X', elementCount: 0,
|
||||
truncated: false, durationMs: 1, axText: 'Canvas changed',
|
||||
screenshot: { base64: 'PNG', width: 600, height: 400 },
|
||||
}
|
||||
|
||||
test('the compatibility sequence executes short drags and observes only once', async () => {
|
||||
const args = { app: 'X', steps: [
|
||||
{ tool: 'drag', from_x: 240, from_y: 320, to_x: 241, to_y: 320 },
|
||||
{ tool: 'drag', from_x: 280, from_y: 320, to_x: 281, to_y: 320 },
|
||||
] }
|
||||
const { engine, calls } = makeEngine({ getAppState: () => state })
|
||||
const result = await handleToolCall(makeAdapter({ engine }), 'sequence', args, baseOverrides())
|
||||
expect(result.isError).toBeFalsy()
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'drag', 'drag', 'getAppState'])
|
||||
expect(calls[1]!.args).toMatchObject({ from: { x: 240, y: 320 }, to: { x: 241, y: 320 } })
|
||||
expect(calls[2]!.args).toMatchObject({ from: { x: 280, y: 320 }, to: { x: 281, y: 320 } })
|
||||
expect(imageBlocks(result)).toHaveLength(1)
|
||||
})
|
||||
|
||||
test('repeated coordinates and a zero-distance drag retain their ordered input', async () => {
|
||||
const { engine, calls } = makeEngine({ getAppState: () => state })
|
||||
const steps = [
|
||||
{ tool: 'click', x: 100, y: 200 },
|
||||
{ tool: 'click', x: 100, y: 200 },
|
||||
{ tool: 'drag', from_x: 100, from_y: 200, to_x: 100, to_y: 200 },
|
||||
]
|
||||
const result = await handleToolCall(makeAdapter({ engine }), 'sequence', { app: 'X', steps }, baseOverrides())
|
||||
expect(result.structuredContent).toMatchObject({ status: 'completed', completedSteps: 3 })
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'click', 'click', 'drag', 'getAppState'])
|
||||
expect(calls[1]!.args).toEqual(calls[2]!.args)
|
||||
expect(calls[3]!.args).toMatchObject({ from: { x: 100, y: 200 }, to: { x: 100, y: 200 } })
|
||||
})
|
||||
|
||||
test('a batch uses the existing resolved-target policy before any mutation', async () => {
|
||||
const { engine, calls } = makeEngine()
|
||||
const result = await handleToolCall(makeAdapter({ engine }), 'sequence', {
|
||||
app: 'com.test.host', steps: [{ tool: 'click', x: 1, y: 2 }],
|
||||
}, baseOverrides())
|
||||
expect(result.telemetry?.error_kind).toBe('app_denied')
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
})
|
||||
|
||||
test('Windows never advertises or dispatches the macOS sequence face', async () => {
|
||||
expect(buildComputerUseTools({ platform: 'win32', screenshotFiltering: 'none' }).some(tool => tool.name === 'sequence')).toBe(false)
|
||||
const { engine, calls } = makeEngine()
|
||||
const adapter = makeAdapter({ engine, platform: 'win32' })
|
||||
adapter.ensureOsPermissions = async () => { throw new Error('must not reach permissions') }
|
||||
const result = await handleToolCall(adapter, 'sequence', {
|
||||
app: 'X', steps: [{ tool: 'click', x: 1, y: 2 }],
|
||||
}, baseOverrides())
|
||||
expect(result.telemetry?.error_kind).toBe('bad_args')
|
||||
expect(calls).toHaveLength(0)
|
||||
})
|
||||
|
||||
test('final capture failure never repeats already completed canvas actions', async () => {
|
||||
const { engine, calls } = makeEngine({ getAppState: () => { throw new Error('capture unavailable') } })
|
||||
const result = await handleToolCall(makeAdapter({ engine }), 'sequence', {
|
||||
app: 'X', steps: [{ tool: 'click', x: 1, y: 2 }],
|
||||
}, baseOverrides())
|
||||
expect(result.structuredContent).toMatchObject({ status: 'observation_failed', completedSteps: 1, resultUnknown: false })
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'click', 'getAppState'])
|
||||
})
|
||||
})
|
||||
|
||||
+272
-47
@@ -1,12 +1,16 @@
|
||||
/**
|
||||
* Tool dispatch for the Codex-compatible computer-use face (blueprint §7).
|
||||
* Tool dispatch for CC-haha's macOS native semantic API and batch sequence.
|
||||
*
|
||||
* Ten semantic tools, dispatched to the native `cu-helper` AX engine
|
||||
* Semantic tools dispatched to the native `cu-helper` AX engine
|
||||
* (`adapter.executor.engine`, a thin wrapper over the daemon's NDJSON
|
||||
* commands):
|
||||
*
|
||||
* list_apps, get_app_state, click, perform_secondary_action, set_value,
|
||||
* select_text, scroll, drag, press_key, type_text, paste
|
||||
* select_text, scroll, drag, press_key, type_text, paste, sequence
|
||||
*
|
||||
* The historical third-party blueprint informed the original API names. It is
|
||||
* not the current official Codex contract; sequence is our bounded batch API,
|
||||
* not Codex's persistent JavaScript entry point.
|
||||
*
|
||||
* ## Three properties this file exists to hold
|
||||
*
|
||||
@@ -24,10 +28,11 @@
|
||||
* the model typed. Checking the string instead would let "friendly alias"
|
||||
* walk past a denylist that names the bundle id.
|
||||
*
|
||||
* **3. Mutating tools never take an implicit snapshot.** They return a fixed
|
||||
* **3. Standalone mutations never take an implicit snapshot.** They return a fixed
|
||||
* receipt and the model calls `get_app_state` when it wants to see the result.
|
||||
* An implicit re-snapshot after every action doubles the AX traversals and the
|
||||
* window captures for a state the model often doesn't read.
|
||||
* window captures for a state the model often doesn't read. `sequence` explicitly
|
||||
* opts into one final observation after its known action batch.
|
||||
*
|
||||
* ## Enforcement order (every call)
|
||||
*
|
||||
@@ -43,8 +48,7 @@
|
||||
* 8. Proven process lifetime — mutating tools only.
|
||||
* 9. Engine dispatch.
|
||||
*
|
||||
* The Codex `<app_state>` envelope is framed HERE, in TS, not in Swift
|
||||
* (blueprint §7). Swift renders the inner tree text (the format authority);
|
||||
* The `<app_state>` envelope is framed here in TS. Swift renders the inner tree;
|
||||
* we wrap it in the version banner + optional <app_specific_instructions> +
|
||||
* <app_state> tags, and attach the window screenshot as a second content
|
||||
* block when one came back.
|
||||
@@ -52,20 +56,20 @@
|
||||
|
||||
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
|
||||
|
||||
import {
|
||||
getDeniedCategoryForApp,
|
||||
isIntrinsicAppDenied,
|
||||
isPolicyDenied,
|
||||
} from "./deniedApps.js";
|
||||
import { isNativeAppDenied } from './nativeAppPolicy.js'
|
||||
import { NATIVE_ERROR, NATIVE_SERVER_ERROR_CODES, toNativeErrorMetadata, type NativeErrorMetadata } from './nativeError.js'
|
||||
import type {
|
||||
AppStateResult,
|
||||
AppTarget,
|
||||
CodexComputerEngine,
|
||||
CodexMouseButton,
|
||||
NativeAppInfo,
|
||||
ProcessIdentity,
|
||||
ResolvedAppTarget,
|
||||
ScreenshotResult,
|
||||
} from "./executor.js";
|
||||
import { formatNativeAppList } from './executor.js'
|
||||
import { buildComputerUseTools } from "./tools.js";
|
||||
import { isSystemKeyCombo } from "./keyBlocklist.js";
|
||||
import type {
|
||||
ComputerUseHostAdapter,
|
||||
@@ -113,26 +117,34 @@ export interface CuCallTelemetry {
|
||||
}
|
||||
|
||||
/**
|
||||
* `CallToolResult` augmented with piggybacked telemetry. The Codex face attaches
|
||||
* `CallToolResult` augmented with piggybacked telemetry. The semantic API attaches
|
||||
* its window screenshot as a normal `image` content block inside `content`, NOT
|
||||
* via this out-of-band `screenshot` stash — that field is retained only so the
|
||||
* host wrapper's screenshot-stash plumbing (`bindSessionContext` in mcpServer.ts)
|
||||
* keeps type-checking; the semantic engine never populates it, so that branch is
|
||||
* inert (no pixel-compare consumer remains).
|
||||
*/
|
||||
export const RESOLVED_APP_PATH = Symbol('computer-use-resolved-app-path')
|
||||
export const APP_INVENTORY = Symbol('computer-use-app-inventory')
|
||||
export const NATIVE_CALL_NOT_DISPATCHED = Symbol('computer-use-native-call-not-dispatched')
|
||||
|
||||
export type CuCallToolResult = CallToolResult & {
|
||||
/** Internal binding metadata. Symbols are never serialized into MCP output. */
|
||||
[RESOLVED_APP_PATH]?: string
|
||||
[APP_INVENTORY]?: NativeAppInfo[]
|
||||
[NATIVE_ERROR]?: NativeErrorMetadata
|
||||
[NATIVE_CALL_NOT_DISPATCHED]?: true
|
||||
screenshot?: ScreenshotResult;
|
||||
telemetry?: CuCallTelemetry;
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Codex envelope framing (blueprint §7)
|
||||
// App-state envelope framing
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Our Computer Use "CUA App Version", surfaced in the `get_app_state` banner.
|
||||
* Codex prints a monotonic build number here (observed 750 / 770). This is our
|
||||
* own product's value; it only needs to be stable and present so harnesses that
|
||||
* This is our own product's value; it only needs to be stable and present so harnesses that
|
||||
* key off the banner have something to read.
|
||||
*/
|
||||
export const CUA_APP_VERSION = "1";
|
||||
@@ -147,7 +159,7 @@ const MUTATION_RECEIPT =
|
||||
"Action completed. Call `get_app_state` to fetch the updated UI state.";
|
||||
|
||||
/**
|
||||
* Frame a daemon `AppStateResult` into the Codex content blocks.
|
||||
* Frame a daemon `AppStateResult` into the app-state content blocks.
|
||||
*
|
||||
* Text block:
|
||||
* Computer Use state (CUA App Version: <v>)
|
||||
@@ -189,7 +201,7 @@ export function frameAppStateEnvelope(state: AppStateResult): CuCallToolResult {
|
||||
content.push({
|
||||
type: "image",
|
||||
data: state.screenshot.base64,
|
||||
mimeType: "image/png",
|
||||
mimeType: state.screenshot.mimeType ?? 'image/png',
|
||||
});
|
||||
}
|
||||
|
||||
@@ -201,10 +213,20 @@ export function frameAppStateEnvelope(state: AppStateResult): CuCallToolResult {
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function errorResult(text: string, errorKind?: CuErrorKind): CuCallToolResult {
|
||||
// The official client rejects a forbidden app with a plain Error before
|
||||
// sending input. Only the actual permission failure has this server cause.
|
||||
const errorName = errorKind === 'tcc_not_granted' ? 'permissionsNotGranted' : undefined
|
||||
return {
|
||||
content: [{ type: "text", text }],
|
||||
isError: true,
|
||||
telemetry: errorKind ? { error_kind: errorKind } : undefined,
|
||||
...(['app_denied', 'bad_args', 'tcc_not_granted', 'grant_flag_required', 'cu_lock_held', 'feature_unavailable'].includes(errorKind ?? '')
|
||||
? { [NATIVE_CALL_NOT_DISPATCHED]: true as const } : {}),
|
||||
...(errorName === undefined ? {} : { [NATIVE_ERROR]: {
|
||||
name: 'SkyComputerUseError' as const, message: text,
|
||||
code: NATIVE_SERVER_ERROR_CODES[errorName], errorName,
|
||||
request: null, requestType: 'jsonRPC' as const,
|
||||
} }),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -318,7 +340,7 @@ function optionalBoolean(value: unknown, key: string): boolean | undefined {
|
||||
}
|
||||
|
||||
/** Parse `{x, y}` from either a nested object `{from:{x,y}}` or flat
|
||||
* `from_x/from_y` (Codex's drag sample uses the flat form). */
|
||||
* `from_x/from_y` (the public schema uses the flat form). */
|
||||
function parsePoint(
|
||||
args: Record<string, unknown>,
|
||||
nestedKey: string,
|
||||
@@ -341,8 +363,7 @@ function parsePoint(
|
||||
}
|
||||
|
||||
/**
|
||||
* Mouse buttons, in Codex's vocabulary plus the single-letter aliases the
|
||||
* official face accepts and the 1..5 numeric convention.
|
||||
* Accepted mouse-button names, single-letter aliases and legacy numeric values.
|
||||
*/
|
||||
const MOUSE_BUTTON_ALIASES: Readonly<Record<string, CodexMouseButton>> = {
|
||||
left: "left",
|
||||
@@ -355,7 +376,7 @@ const MOUSE_BUTTON_ALIASES: Readonly<Record<string, CodexMouseButton>> = {
|
||||
r: "right",
|
||||
};
|
||||
|
||||
/** Map mouse_button (string name, short alias, or Codex's numeric 1..5). */
|
||||
/** Map mouse_button (string name, short alias, or legacy numeric 1..5). */
|
||||
function parseMouseButton(value: unknown): CodexMouseButton | undefined {
|
||||
if (value === undefined || value === null) return undefined;
|
||||
if (typeof value === "string") {
|
||||
@@ -462,25 +483,22 @@ export function defersLockAcquire(toolName: string): boolean {
|
||||
}
|
||||
|
||||
/** Preserved no-op export (mcpServer.ts + the old lock path call it on a fresh
|
||||
* lock holder). The Codex face holds no cross-call mouse-button state, so there
|
||||
* lock holder). The semantic API holds no cross-call mouse-button state, so there
|
||||
* is nothing to clear — kept for call-site compatibility. */
|
||||
export function resetMouseButtonHeld(): void {
|
||||
/* no cross-call mouse state in the semantic engine */
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Safety denylist (Codex's exact refusal text)
|
||||
// Native app policy
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Refusal check against a RESOLVED target — the real bundle id and display
|
||||
* name of the process we are about to drive, not the string the model typed.
|
||||
*
|
||||
* "Denied" spans three lists:
|
||||
* • the intrinsic set (our own app and helper) — permanent, ungrantable;
|
||||
* • the category denylist (browsers, terminals/IDEs, trading apps), matching
|
||||
* Codex's behavior of refusing iTerm/Chrome;
|
||||
* • the media/DRM policy list (Netflix, Spotify, Kindle, …).
|
||||
* Official native forbidden identities plus our own app and helper. The
|
||||
* resolved bundle ID is authoritative; display-name substrings are not policy.
|
||||
*
|
||||
* `requestedApp` only shapes the message — the model should see the name it
|
||||
* used. `hostBundleId` extends the intrinsic set for builds whose bundle id
|
||||
@@ -493,14 +511,9 @@ function policyDenyMessage(
|
||||
): string | undefined {
|
||||
const bundleId = resolved.bundleId;
|
||||
const displayName = resolved.displayName ?? bundleId ?? requestedApp ?? "";
|
||||
const denied =
|
||||
isIntrinsicAppDenied(bundleId, hostBundleId) ||
|
||||
getDeniedCategoryForApp(bundleId, displayName) !== null ||
|
||||
isPolicyDenied(bundleId, displayName);
|
||||
if (!denied) return undefined;
|
||||
if (!isNativeAppDenied(bundleId, hostBundleId)) return undefined;
|
||||
const shown = requestedApp ?? displayName ?? bundleId ?? "";
|
||||
// Matches Codex: "Computer Use is not allowed to use the app '<app>' for
|
||||
// safety reasons."
|
||||
// Preserve the existing product refusal message.
|
||||
return `Computer Use is not allowed to use the app '${shown}' for safety reasons.`;
|
||||
}
|
||||
|
||||
@@ -509,11 +522,11 @@ function policyDenyMessage(
|
||||
*
|
||||
* The guidance is static per app, so re-sending it on every `get_app_state`
|
||||
* spends tokens to restate something the model already has — and in a long
|
||||
* session it crowds out the state the model actually asked for. Codex tracks
|
||||
* the same thing per client session and only ever emits the block once.
|
||||
* session it crowds out the state the model actually asked for. Track this
|
||||
* per client session and emit the block once.
|
||||
*
|
||||
* Keyed by bundle id where the daemon resolved one, else by what the model
|
||||
* asked for, mirroring Codex's own fallback.
|
||||
* asked for.
|
||||
*/
|
||||
const deliveredAppInstructions = new Set<string>();
|
||||
|
||||
@@ -549,6 +562,7 @@ interface ParsedRequest {
|
||||
/** What the model typed, for messages and the approval dialog. */
|
||||
requestedApp?: string;
|
||||
mutating: boolean;
|
||||
steps?: ParsedRequest[];
|
||||
run(engine: CodexComputerEngine, target: AppTarget): Promise<CuCallToolResult>;
|
||||
}
|
||||
|
||||
@@ -568,6 +582,7 @@ const KNOWN_TOOLS: ReadonlySet<string> = new Set([
|
||||
"list_apps",
|
||||
"get_app_state",
|
||||
...MUTATING_TOOLS,
|
||||
"sequence",
|
||||
]);
|
||||
|
||||
function parseRequest(
|
||||
@@ -579,7 +594,59 @@ function parseRequest(
|
||||
if (name === "list_apps") {
|
||||
return {
|
||||
mutating: false,
|
||||
run: async engine => okText(await engine.listApps()),
|
||||
run: async engine => {
|
||||
if (!engine.listAppsInfo) return okText(await engine.listApps())
|
||||
const apps = await engine.listAppsInfo()
|
||||
return { ...okText(formatNativeAppList(apps)), [APP_INVENTORY]: apps }
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
if (name === "sequence") {
|
||||
if (platform !== "darwin") {
|
||||
throw new BadArgs("sequence is only available on macOS");
|
||||
}
|
||||
if (Object.keys(args).some(key => key !== "app" && key !== "steps")) {
|
||||
throw new BadArgs("sequence accepts only app and steps");
|
||||
}
|
||||
const target = parseTarget(args);
|
||||
if (!Array.isArray(args.steps) || args.steps.length < 1 || args.steps.length > 256) {
|
||||
throw new BadArgs("sequence requires 1–256 steps");
|
||||
}
|
||||
const schemas = buildComputerUseTools({ platform: "darwin" });
|
||||
let chords = 0;
|
||||
const steps = args.steps.map((value, index) => {
|
||||
const step = asRecord(value);
|
||||
const tool = step.tool;
|
||||
if (typeof tool !== "string" || !MUTATING_TOOLS.has(tool)) {
|
||||
throw new BadArgs(`sequence step ${index}: expected an existing mutation tool`);
|
||||
}
|
||||
const schema = schemas.find(t => t.name === tool)!.inputSchema;
|
||||
const allowed = new Set([
|
||||
"tool",
|
||||
...Object.keys(schema.properties ?? {}).filter(key => key !== "app"),
|
||||
]);
|
||||
if (Object.keys(step).some(key => !allowed.has(key))) {
|
||||
throw new BadArgs(`sequence step ${index}: unknown argument or per-step app override`);
|
||||
}
|
||||
if (tool === "press_key" && typeof step.key === "string") {
|
||||
const count = step.key.replace(/\s*\+\s*/g, "+").trim().split(/\s+/).length;
|
||||
chords += count;
|
||||
if (count > 128 || chords > 2048) {
|
||||
throw new BadArgs("sequence exceeds its keyboard chord budget");
|
||||
}
|
||||
}
|
||||
const { tool: _tool, ...input } = step;
|
||||
return parseRequest(tool, { ...input, app: args.app }, grantFlags, platform);
|
||||
});
|
||||
return {
|
||||
target,
|
||||
requestedApp: (args.app as string).trim(),
|
||||
mutating: true,
|
||||
steps,
|
||||
run: async () => {
|
||||
throw new Error("sequence requires guarded dispatch");
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@@ -765,6 +832,9 @@ function parseRequest(
|
||||
|
||||
case "press_key": {
|
||||
const key = requiredString(args, "key");
|
||||
if (platform === "darwin" && key.replace(/\s*\+\s*/g, "+").trim().split(/\s+/).length > 128) {
|
||||
throw new BadArgs("press_key supports at most 128 key chords; use shorter sequences");
|
||||
}
|
||||
// The grant bit comes from the SESSION, never from the model's arguments —
|
||||
// otherwise the model could authorize its own cmd+q by passing a flag.
|
||||
const systemKeyCombos = grantFlags?.systemKeyCombos === true;
|
||||
@@ -929,7 +999,11 @@ export async function handleToolCall(
|
||||
);
|
||||
}
|
||||
|
||||
// Unknown tool → clean error (defensive; ListTools only advertises the ten).
|
||||
if (overrides.isAborted?.()) {
|
||||
return errorResult("Computer Use was cancelled; no action was dispatched.", "other");
|
||||
}
|
||||
|
||||
// Unknown tool → clean error; ListTools advertises only supported tools.
|
||||
if (!KNOWN_TOOLS.has(name)) {
|
||||
return errorResult(`Unknown computer-use tool "${name}".`, "bad_args");
|
||||
}
|
||||
@@ -970,6 +1044,10 @@ export async function handleToolCall(
|
||||
);
|
||||
}
|
||||
|
||||
if (overrides.isAborted?.() || adapter.isDisabled()) {
|
||||
return errorResult("Computer Use stopped before dispatch.", "other");
|
||||
}
|
||||
|
||||
// ─── Gate 4: global CU lock ──────────────────────────────────────────
|
||||
const deferAcquire = defersLockAcquire(name);
|
||||
const lock = overrides.checkCuLock?.();
|
||||
@@ -1027,16 +1105,163 @@ export async function handleToolCall(
|
||||
}
|
||||
|
||||
// ─── Dispatch ──────────────────────────────────────────────────────
|
||||
return await request.run(engine, dispatchTarget(resolved, request.target));
|
||||
if (overrides.isAborted?.() || adapter.isDisabled()) {
|
||||
return errorResult("Computer Use stopped before dispatch.", "other");
|
||||
}
|
||||
const pinned = dispatchTarget(resolved, request.target);
|
||||
if (request.steps) {
|
||||
return await runSequence(request.steps, engine, pinned, requestedApp, adapter, overrides);
|
||||
}
|
||||
const result = await request.run(engine, pinned)
|
||||
if (name === 'get_app_state' && resolved.path) result[RESOLVED_APP_PATH] = resolved.path
|
||||
return result
|
||||
} catch (err) {
|
||||
// A daemon-level error (staleness warning, not_trusted, invalid action,
|
||||
// not-settable, …). Surface its message verbatim — these strings are part
|
||||
// of the Codex-compatible protocol (e.g. the §6 "user changed '<app>'"
|
||||
// warning, "Element handle g17:99 not found in snapshot (has M elements).",
|
||||
// "<action> is not a valid secondary action for <handle>").
|
||||
// not-settable, …). Preserve its diagnostic so the caller can inspect the
|
||||
// current state before choosing another action.
|
||||
const msg = err instanceof Error ? err.message : String(err);
|
||||
logger.error(`[${serverName}] tool=${name} failed: ${msg}`, err);
|
||||
return errorResult(msg, "executor_threw");
|
||||
return { ...errorResult(msg, 'executor_threw'), [NATIVE_ERROR]: toNativeErrorMetadata(err) }
|
||||
}
|
||||
}
|
||||
|
||||
/** Never detach this loop with Promise.race: a timed-out loop must not keep injecting. */
|
||||
async function runSequence(
|
||||
steps: ParsedRequest[],
|
||||
engine: CodexComputerEngine,
|
||||
target: AppTarget,
|
||||
requestedApp: string,
|
||||
adapter: ComputerUseHostAdapter,
|
||||
overrides: ComputerUseOverrides,
|
||||
): Promise<CuCallToolResult> {
|
||||
const started = performance.now();
|
||||
let completedSteps = 0;
|
||||
const stepDurationsMs: number[] = [];
|
||||
const summary = (status: string, extra: Record<string, unknown> = {}) => ({
|
||||
status,
|
||||
completedSteps,
|
||||
totalSteps: steps.length,
|
||||
elapsedMs: Math.round(performance.now() - started),
|
||||
stepDurationsMs,
|
||||
...extra,
|
||||
});
|
||||
const stop = (observationSkipped = true): CuCallToolResult | undefined => {
|
||||
let status: string | undefined;
|
||||
if (overrides.isAborted?.()) {
|
||||
status = "cancelled";
|
||||
} else if (adapter.isDisabled()) {
|
||||
status = "disabled";
|
||||
} else if (performance.now() - started >= 60_000) {
|
||||
status = "deadline_exceeded";
|
||||
}
|
||||
if (!status) {
|
||||
return undefined;
|
||||
}
|
||||
return {
|
||||
...errorResult(
|
||||
`Sequence ${status}; stopped after ${completedSteps} completed steps. ` +
|
||||
"No further actions dispatched. Inspect the current state before continuing; " +
|
||||
"do not replay completed steps.",
|
||||
"other",
|
||||
),
|
||||
structuredContent: summary(status, {
|
||||
resultUnknown: false,
|
||||
observationSkipped,
|
||||
observationDelivered: false,
|
||||
}),
|
||||
};
|
||||
};
|
||||
|
||||
for (const step of steps) {
|
||||
const stopped = stop();
|
||||
if (stopped) {
|
||||
return stopped;
|
||||
}
|
||||
const stepStarted = performance.now();
|
||||
try {
|
||||
const result = await step.run(engine, target);
|
||||
stepDurationsMs.push(Math.round(performance.now() - stepStarted));
|
||||
if (result.isError) {
|
||||
return {
|
||||
...result,
|
||||
structuredContent: summary("failed", {
|
||||
failedStepIndex: completedSteps,
|
||||
resultUnknown: true,
|
||||
observationSkipped: true,
|
||||
}),
|
||||
};
|
||||
}
|
||||
completedSteps++;
|
||||
} catch (err) {
|
||||
stepDurationsMs.push(Math.round(performance.now() - stepStarted));
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
...errorResult(
|
||||
`Sequence step ${completedSteps} failed: ${message}. ` +
|
||||
"Inspect the app before retrying; completed steps were not rolled back.",
|
||||
"executor_threw",
|
||||
),
|
||||
structuredContent: summary("failed", {
|
||||
failedStepIndex: completedSteps,
|
||||
resultUnknown: true,
|
||||
observationSkipped: true,
|
||||
}),
|
||||
};
|
||||
}
|
||||
const stoppedAfter = stop();
|
||||
if (stoppedAfter) {
|
||||
return stoppedAfter;
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const state = await engine.getAppState(target);
|
||||
const stopped = stop(false);
|
||||
if (stopped) {
|
||||
return stopped;
|
||||
}
|
||||
const result = frameAppStateEnvelope(instructionsOncePerApp(state, requestedApp));
|
||||
if (!state.screenshot?.base64) {
|
||||
return {
|
||||
...result,
|
||||
isError: true,
|
||||
telemetry: { error_kind: "capture_failed" },
|
||||
structuredContent: summary("observation_failed", { resultUnknown: false }),
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `All ${completedSteps} steps completed, but the final screenshot is unavailable. ` +
|
||||
"The AX state alone does not verify the visual result. " +
|
||||
"Call get_app_state before continuing; do not replay completed steps.",
|
||||
},
|
||||
...result.content,
|
||||
],
|
||||
};
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
structuredContent: summary("completed", { resultUnknown: false }),
|
||||
content: [
|
||||
{
|
||||
type: "text",
|
||||
text: `Sequence completed ${completedSteps} steps. The following is the actual final app state.`,
|
||||
},
|
||||
...result.content,
|
||||
],
|
||||
};
|
||||
} catch (err) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
return {
|
||||
...errorResult(
|
||||
`All ${completedSteps} sequence steps completed, but final observation failed: ${message}. ` +
|
||||
"Call get_app_state before continuing; do not replay completed steps.",
|
||||
"capture_failed",
|
||||
),
|
||||
structuredContent: summary("observation_failed", {
|
||||
resultUnknown: false,
|
||||
observationSkipped: false,
|
||||
}),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
|
||||
import { buildPlatformComputerUseTools } from './mcpServer.js'
|
||||
import { buildComputerUseTools } from './tools.js'
|
||||
|
||||
/**
|
||||
* Cross-provider portability rules for the Computer Use tool schemas.
|
||||
@@ -89,7 +90,7 @@ describe('computer use tool schema portability', () => {
|
||||
// take coordinates alone — so keying by name across both platforms would
|
||||
// silently assert against whichever came last.
|
||||
const darwinTools = new Map(
|
||||
buildPlatformComputerUseTools(
|
||||
buildComputerUseTools(
|
||||
{ screenshotFiltering: 'native', platform: 'darwin' },
|
||||
'pixels',
|
||||
).map(tool => [tool.name, tool]),
|
||||
|
||||
Vendored
+87
-18
@@ -1,19 +1,17 @@
|
||||
/**
|
||||
* MCP tool schemas for the computer-use server — Codex-compatible semantic
|
||||
* face (blueprint §7). Eleven tools, named verbatim after Codex's public
|
||||
* computer-use MCP:
|
||||
* MCP schemas for CC-haha's macOS native semantic API and batch sequence:
|
||||
*
|
||||
* list_apps, get_app_state, click, perform_secondary_action, set_value,
|
||||
* select_text, scroll, drag, press_key, type_text, paste
|
||||
* select_text, scroll, drag, press_key, type_text, paste, sequence
|
||||
*
|
||||
* This replaces the prior 27-tool pixel face. The legacy `coordinateMode` /
|
||||
* `screenshotFiltering` parameters are accepted for call-site compatibility
|
||||
* but no longer influence the schema — the tool shapes are static.
|
||||
* but no longer influence the semantic action schemas. Platform selection
|
||||
* controls sequence availability; Windows keeps its separate pixel tool API.
|
||||
*
|
||||
* Param names mirror Codex exactly (verified against iFurySt's reverse-
|
||||
* engineered ToolDefinitions.swift and the 2026-04-17 tool-call samples):
|
||||
* `app`, `element_index`, `click_count`, `mouse_button`, `direction`, `pages`,
|
||||
* `value`, `action`, `key`, `text`.
|
||||
* The original names came from a historical third-party reconstruction, not
|
||||
* the current official Codex source. The current official interface uses a
|
||||
* persistent JavaScript entry point; these MCP tools are our compatibility API.
|
||||
*
|
||||
* Three things here are load-bearing and easy to erode:
|
||||
*
|
||||
@@ -83,11 +81,11 @@ function coordinateProp(axis: "x" | "y", role: string) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the Codex computer-use face.
|
||||
* Build the native semantic tools and optional macOS batch tool.
|
||||
*
|
||||
* Signature is preserved from the legacy pixel builder so existing call sites
|
||||
* (`setup.ts`, host `mcpServer.ts`) keep compiling. All three parameters are
|
||||
* ignored — the tool list is static.
|
||||
* (`setup.ts`, host `mcpServer.ts`) keep compiling. Only the platform capability
|
||||
* affects this tool list; coordinate mode and installed-app hints are ignored.
|
||||
*/
|
||||
export function buildComputerUseTools(
|
||||
_caps?: {
|
||||
@@ -98,13 +96,34 @@ export function buildComputerUseTools(
|
||||
_coordinateMode?: CoordinateMode,
|
||||
_installedAppNames?: string[],
|
||||
): Tool[] {
|
||||
return [
|
||||
const tools: Tool[] = [
|
||||
{
|
||||
name: 'js',
|
||||
annotations: { readOnlyHint: false, destructiveHint: true, idempotentHint: false, openWorldHint: false },
|
||||
description: 'Run JavaScript in a persistent, isolated Computer Use session. Use this for native app automation: let app = await cua.getApp("App Name"); then await app.click([x,y]), app.drag([x,y],[x,y]), app.pressKey("CMD+A"), or loops of known actions. App selection emits initial state and API guidance. Observe with app.getAXState(), app.getScreenshot(), or app.getAXStateAndScreenshot(); emit:false returns data without displaying it. Use nodeRepl.write(value) for text and await nodeRepl.emitImage(bytes) for images. Top-level variables persist between calls, including after ordinary script errors. Await all actions. Imports, Node APIs, filesystem and networking are unavailable. Timeout/cancellation resets bindings; observe before retrying partial work. Native apps only; browser DOM/tab APIs are not implemented.',
|
||||
inputSchema: {
|
||||
type: 'object',
|
||||
additionalProperties: false,
|
||||
required: ['code'],
|
||||
properties: {
|
||||
code: { type: 'string', description: 'JavaScript with top-level await. Up to 256 KiB and 256 native calls per cell.' },
|
||||
title: { type: 'string', description: 'Short description of the operation.' },
|
||||
timeout_ms: { type: 'integer', minimum: 1, maximum: 60000, default: 30000 },
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'js_reset',
|
||||
annotations: { readOnlyHint: false, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
||||
description: 'Reset the persistent Computer Use JavaScript session and discard all App handles and variables. Select an app again before continuing.',
|
||||
inputSchema: { type: 'object', additionalProperties: false, properties: {} },
|
||||
},
|
||||
{
|
||||
name: "list_apps",
|
||||
annotations: { readOnlyHint: true, destructiveHint: false, idempotentHint: true, openWorldHint: false },
|
||||
description:
|
||||
"List the running applications available to control, in alphabetical " +
|
||||
"order. Each line is \"<App Name> — <bundle.id>\". Use this to discover " +
|
||||
'List running and recently used applications, with running apps first. ' +
|
||||
"Each line is \"<App Name> — <bundle.id>\". Discovery does not grant access. Use this to discover " +
|
||||
"the exact app name or bundle id to pass to get_app_state.",
|
||||
inputSchema: {
|
||||
type: "object" as const,
|
||||
@@ -121,8 +140,8 @@ export function buildComputerUseTools(
|
||||
"Read the accessibility (AX) state of an application: an indented tree " +
|
||||
"of every interactive UI element, each tagged with the opaque handle to " +
|
||||
"pass as `element_index`, plus a screenshot of the app's key window. " +
|
||||
"Call this first, and again after any action whose result you need to " +
|
||||
"see — actions return a receipt, not a new state.",
|
||||
"Call this first, then after one or more standalone actions before deciding what to do next. " +
|
||||
"A sequence already returns its final state and screenshot; do not repeat an observation that is current.",
|
||||
inputSchema: {
|
||||
type: "object" as const,
|
||||
additionalProperties: false,
|
||||
@@ -379,7 +398,8 @@ export function buildComputerUseTools(
|
||||
"named keys like \"Return\", \"Tab\", \"Escape\", \"BackSpace\", " +
|
||||
"\"Up\"/\"Down\"/\"Left\"/\"Right\", \"Prior\"/\"Next\" (page up/down), " +
|
||||
"\"Home\"/\"End\", \"F1\"–\"F12\", \"KP_0\"–\"KP_9\". \"super\" maps to " +
|
||||
"Command. Call get_app_state afterwards to see the result.",
|
||||
"Command. Space-separated macros support at most 128 chords, executed in order. " +
|
||||
"Observe after the known action batch, using sequence or get_app_state.",
|
||||
inputSchema: {
|
||||
type: "object" as const,
|
||||
additionalProperties: false,
|
||||
@@ -445,4 +465,53 @@ export function buildComputerUseTools(
|
||||
},
|
||||
},
|
||||
];
|
||||
if (_caps?.platform === "win32") {
|
||||
return tools.filter(tool => !['js', 'js_reset'].includes(tool.name));
|
||||
}
|
||||
const variants = tools
|
||||
.filter(tool => !["list_apps", "get_app_state", 'js', 'js_reset'].includes(tool.name))
|
||||
.map(tool => {
|
||||
const { app: _app, ...properties } = tool.inputSchema.properties ?? {};
|
||||
return {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: { tool: { type: "string", const: tool.name }, ...properties },
|
||||
required: [
|
||||
"tool",
|
||||
...(tool.inputSchema.required ?? []).filter(key => key !== "app"),
|
||||
],
|
||||
};
|
||||
});
|
||||
tools.push({
|
||||
name: "sequence",
|
||||
annotations: {
|
||||
readOnlyHint: false,
|
||||
destructiveHint: false,
|
||||
idempotentHint: false,
|
||||
openWorldHint: false,
|
||||
},
|
||||
description:
|
||||
"Run a bounded sequence of known UI actions in order against ONE app, then return its real AX state and screenshot. " +
|
||||
"Batch actions determined from the current observation, including repeated coordinates on a stable canvas, then inspect the final state. " +
|
||||
"Do not force one model round trip per click. If a step needs a new decision or changes the prerequisite state, stop and re-observe before continuing. " +
|
||||
"Each step uses tool plus that tool's arguments, without app. Stops on the first error or cancellation; never retries. " +
|
||||
"At most 256 steps, 128 space-separated chords per press_key step, 2048 total chords, and a 60-second cooperative deadline " +
|
||||
"(wait for the current native action to finish, then stop). " +
|
||||
"No nested sequences, code, sleeps or per-step app changes. A failed or timed-out step may have partially executed; inspect state before retrying.",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
additionalProperties: false,
|
||||
properties: {
|
||||
app: APP_PROP,
|
||||
steps: {
|
||||
type: "array",
|
||||
minItems: 1,
|
||||
maxItems: 256,
|
||||
items: { anyOf: variants },
|
||||
},
|
||||
},
|
||||
required: ["app", "steps"],
|
||||
},
|
||||
});
|
||||
return tools;
|
||||
}
|
||||
|
||||
Vendored
+3
@@ -222,6 +222,9 @@ export interface ComputerUseHostAdapter {
|
||||
logger: Logger;
|
||||
executor: ComputerExecutor;
|
||||
|
||||
/** A fresh persistent kernel per bound session. The host owns isolation. */
|
||||
createReplRuntime?(): import('./replProtocol.js').ComputerUseReplRuntime;
|
||||
|
||||
/**
|
||||
* TCC state check — Accessibility + Screen Recording on macOS. Pure check,
|
||||
* no dialog, no relaunch. When either is missing, `request_access` threads
|
||||
|
||||
+65
-20
@@ -666,6 +666,31 @@ function parseKeyChord(text: string): string[] {
|
||||
* can't. The per-turn reset is the correctness boundary.
|
||||
*/
|
||||
let mouseButtonHeld = false;
|
||||
// Internal binder identity. It is carried on overrides, never on the wire, so
|
||||
// a cancelled session cannot release another session's synthetic mouse press.
|
||||
export const WINDOWS_MOUSE_OWNER = Symbol('windowsComputerUseMouseOwner')
|
||||
export interface WindowsMouseOwner {
|
||||
canRelease(): Promise<boolean>
|
||||
}
|
||||
let mouseButtonOwner: WindowsMouseOwner | undefined
|
||||
let mouseHoldGeneration = 0
|
||||
let pendingMouseRelease: { generation: number; promise: Promise<boolean> } | undefined
|
||||
|
||||
function mouseOwner(overrides: ComputerUseOverrides): WindowsMouseOwner | undefined {
|
||||
return (overrides as ComputerUseOverrides & { [WINDOWS_MOUSE_OWNER]?: WindowsMouseOwner })[WINDOWS_MOUSE_OWNER]
|
||||
}
|
||||
|
||||
export function hasHeldMouseForSession(owner: WindowsMouseOwner): boolean {
|
||||
return mouseButtonHeld && mouseButtonOwner === owner
|
||||
}
|
||||
|
||||
/** Re-checks the owner and its host lock before sending the matching release. */
|
||||
export async function releaseHeldMouseForSession(
|
||||
adapter: ComputerUseHostAdapter,
|
||||
owner: WindowsMouseOwner,
|
||||
): Promise<boolean> {
|
||||
return releaseHeldMouse(adapter, owner)
|
||||
}
|
||||
/** Whether mouse_move occurred between left_mouse_down and left_mouse_up.
|
||||
* When false at mouseUp, the decomposed sequence is a click-release (not a
|
||||
* drop) — hit-test at "mouse", not "mouse_full". */
|
||||
@@ -677,6 +702,8 @@ let mouseMoved = false;
|
||||
export function resetMouseButtonHeld(): void {
|
||||
mouseButtonHeld = false;
|
||||
mouseMoved = false;
|
||||
mouseButtonOwner = undefined
|
||||
mouseHoldGeneration += 1
|
||||
}
|
||||
|
||||
/** If a left_mouse_down set the OS button without a matching left_mouse_up
|
||||
@@ -684,11 +711,28 @@ export function resetMouseButtonHeld(): void {
|
||||
* handleClick. No-op when not held — callers don't need to check. */
|
||||
async function releaseHeldMouse(
|
||||
adapter: ComputerUseHostAdapter,
|
||||
): Promise<void> {
|
||||
if (!mouseButtonHeld) return;
|
||||
await adapter.executor.mouseUp();
|
||||
mouseButtonHeld = false;
|
||||
mouseMoved = false;
|
||||
owner?: WindowsMouseOwner,
|
||||
): Promise<boolean> {
|
||||
if (!mouseButtonHeld || (owner !== undefined && mouseButtonOwner !== owner)) return false
|
||||
const generation = mouseHoldGeneration
|
||||
if (pendingMouseRelease?.generation === generation) return pendingMouseRelease.promise
|
||||
// Share only cleanup, not the Windows action dispatcher. Keep ownership on
|
||||
// failure so a later cancellation can retry releasing this same press.
|
||||
const promise = Promise.resolve().then(async () => {
|
||||
if (owner && !await owner.canRelease()) return false
|
||||
// Checking the host lock yields. A new lock holder can reset the old
|
||||
// state and start a different press while that check is in flight.
|
||||
if (!mouseButtonHeld || mouseHoldGeneration !== generation || (owner && mouseButtonOwner !== owner)) return false
|
||||
await adapter.executor.mouseUp()
|
||||
if (mouseHoldGeneration === generation) resetMouseButtonHeld()
|
||||
return true
|
||||
})
|
||||
pendingMouseRelease = { generation, promise }
|
||||
try {
|
||||
return await promise
|
||||
} finally {
|
||||
if (pendingMouseRelease?.promise === promise) pendingMouseRelease = undefined
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1614,7 +1658,7 @@ async function executeTeachStep(
|
||||
// The host's Exit handler also calls stopSession, so the turn is
|
||||
// already unwinding. Caller decides what to return for the transcript.
|
||||
// A PREVIOUS step's left_mouse_down may have left the OS button held.
|
||||
await releaseHeldMouse(adapter);
|
||||
await releaseHeldMouse(adapter, mouseOwner(overrides));
|
||||
return { kind: "exit" };
|
||||
}
|
||||
|
||||
@@ -1644,7 +1688,7 @@ async function executeTeachStep(
|
||||
// this IS the exit path, just caught mid-dispatch instead of at the
|
||||
// onTeachStep await above. Callers already handle { kind: "exit" }.
|
||||
if (overrides.isAborted?.()) {
|
||||
await releaseHeldMouse(adapter);
|
||||
await releaseHeldMouse(adapter, mouseOwner(overrides));
|
||||
return { kind: "exit" };
|
||||
}
|
||||
// Same inter-step settle as handleComputerBatch.
|
||||
@@ -1667,7 +1711,7 @@ async function executeTeachStep(
|
||||
results.push(result);
|
||||
|
||||
if (inner.isError) {
|
||||
await releaseHeldMouse(adapter);
|
||||
await releaseHeldMouse(adapter, mouseOwner(overrides));
|
||||
return {
|
||||
kind: "action_error",
|
||||
executed: results.length - 1,
|
||||
@@ -2161,9 +2205,7 @@ async function handleClickVariant(
|
||||
// click-tier and read-tier windows. Release first so click() gets a clean
|
||||
// slate.
|
||||
if (mouseButtonHeld) {
|
||||
await adapter.executor.mouseUp();
|
||||
mouseButtonHeld = false;
|
||||
mouseMoved = false;
|
||||
await releaseHeldMouse(adapter, mouseOwner(overrides));
|
||||
}
|
||||
|
||||
const coord = extractCoordinate(args);
|
||||
@@ -2496,9 +2538,7 @@ async function handleDrag(
|
||||
// the handleClickVariant clear above. Release first so drag() gets a
|
||||
// clean slate.
|
||||
if (mouseButtonHeld) {
|
||||
await adapter.executor.mouseUp();
|
||||
mouseButtonHeld = false;
|
||||
mouseMoved = false;
|
||||
await releaseHeldMouse(adapter, mouseOwner(overrides));
|
||||
}
|
||||
|
||||
// `coordinate` is the END point
|
||||
@@ -3009,6 +3049,8 @@ async function handleLeftMouseDown(
|
||||
await adapter.executor.mouseDown();
|
||||
mouseButtonHeld = true;
|
||||
mouseMoved = false;
|
||||
mouseButtonOwner = mouseOwner(overrides)
|
||||
mouseHoldGeneration += 1
|
||||
return okText("Mouse button pressed.");
|
||||
}
|
||||
|
||||
@@ -3035,8 +3077,7 @@ async function handleLeftMouseUp(
|
||||
err: CuCallToolResult,
|
||||
): Promise<CuCallToolResult> => {
|
||||
await adapter.executor.mouseUp();
|
||||
mouseButtonHeld = false;
|
||||
mouseMoved = false;
|
||||
resetMouseButtonHeld();
|
||||
return err;
|
||||
};
|
||||
|
||||
@@ -3062,8 +3103,7 @@ async function handleLeftMouseUp(
|
||||
if (hitGate) return releaseFirst(hitGate);
|
||||
|
||||
await adapter.executor.mouseUp();
|
||||
mouseButtonHeld = false;
|
||||
mouseMoved = false;
|
||||
resetMouseButtonHeld();
|
||||
return okText("Mouse button released.");
|
||||
}
|
||||
|
||||
@@ -3185,7 +3225,7 @@ async function handleComputerBatch(
|
||||
// host's await but not this loop — without this check the remaining
|
||||
// actions fire into a dead session.
|
||||
if (overrides.isAborted?.()) {
|
||||
await releaseHeldMouse(adapter);
|
||||
await releaseHeldMouse(adapter, mouseOwner(overrides));
|
||||
return errorResult(
|
||||
`Batch aborted after ${results.length} of ${actions.length} actions (user interrupt).`,
|
||||
);
|
||||
@@ -3221,7 +3261,7 @@ async function handleComputerBatch(
|
||||
// Release held mouse: the error may be a mid-grapheme abort in
|
||||
// handleType, or a frontmost gate, landing between mouse_down and
|
||||
// mouse_up.
|
||||
await releaseHeldMouse(adapter);
|
||||
await releaseHeldMouse(adapter, mouseOwner(overrides));
|
||||
return okJson(
|
||||
{
|
||||
completed: results.slice(0, -1),
|
||||
@@ -3390,6 +3430,11 @@ export async function handleToolCall(
|
||||
// state through to the renderer, which shows a TCC toggle panel instead
|
||||
// of the app list. Every other tool short-circuits here.
|
||||
const osPerms = await adapter.ensureOsPermissions();
|
||||
if (overrides.isAborted?.()) {
|
||||
// Permission checks can yield after the binder's cancellation check. The
|
||||
// binder still owns held-mouse cleanup; do not dispatch a new input here.
|
||||
return errorResult('Computer Use cancelled before dispatch.')
|
||||
}
|
||||
let tccState:
|
||||
| { accessibility: boolean; screenRecording: boolean }
|
||||
| undefined;
|
||||
|
||||
Reference in New Issue
Block a user