fix(computer-use): support cell-scoped JavaScript timers

Keep timers inside the guest realm and cancel them when a cell ends.
Preserve parsed statement boundaries when rewriting persistent calls.
This commit is contained in:
程序员阿江(Relakkes)
2026-09-11 16:34:32 +08:00
parent d5c02b43d9
commit 6d09d9f899
9 changed files with 438 additions and 8 deletions
@@ -2,6 +2,39 @@ import { describe, expect, test } from 'bun:test'
import { compileReplCell } from './replCompiler'
describe('computer use REPL compiler', () => {
test.each([
'first()\nsecond()',
'first() // a comment must not consume the boundary\nsecond()',
'let value = first()\nsecond(value)',
'var value = first()\nsecond(value)',
'function run() { return first()\nsecond() }\nsecond(run())',
'try { throw first()\nsecond() } catch (value) { second(value) }',
'if (true) first()\nelse second()\nsecond()',
'for (let i = 0; i < 2; i++) first()\nsecond()',
'for (var value of [1, 2]) first()\nsecond(value)',
'for (var value in {a: 1}) { first()\nsecond(value) }',
'first()\nsecond`text`',
])('preserves JavaScript statement boundaries when rewriting free calls: %s', async code => {
async function execute(compiled: boolean) {
const calls: unknown[][] = []
const values: Record<string, unknown> = {
first: () => { calls.push(['first']); return 7 },
second: (...args: unknown[]) => { calls.push(['second', ...args]); return 9 },
}
if (compiled) {
const cell = compileReplCell(code, [])
const run = new Function(`return ${cell.source}`)()
await run(() => ({ values }), (bindings: Array<[string, string, () => unknown, (value: unknown) => void]>) => {
for (const [name, , get, set] of bindings) Object.defineProperty(values, name, { get, set, configurable: true })
}, () => {})
} else {
await new Function('first', 'second', `return (async () => { ${code}\n})()`)(values.first, values.second)
}
return calls
}
expect(await execute(true)).toEqual(await execute(false))
})
test('collects persistent declarations without capturing nested lexical scopes', () => {
const compiled = compileReplCell(`
const { x, nested: { y = 2 }, ...rest } = source
+12 -1
View File
@@ -300,8 +300,20 @@ export function compileReplCell(code: string, prior: readonly ReplBinding[]) {
const current = new Map<string, ReplBinding>()
const identifiers = new Set<string>()
const hoistedDeclarations: Array<{ node: SyntaxNode; parent?: SyntaxNode }> = []
const insertions: Array<{ position: number; text: string }> = []
function inspect(node: SyntaxNode, depth: number, inFunctionScope: boolean, parent?: SyntaxNode) {
// A free call becomes `(0, scope.values[name])(...)`. Its new leading
// parenthesis can undo ASI after the preceding expression, initializer,
// return, or throw. Preserve boundaries from the original parsed program,
// including nested function bodies, without altering for-loop headers.
const variableStatement = node.type === 'VariableDeclaration'
&& !(parent?.type === 'ForStatement' && parent.init === node)
&& !((parent?.type === 'ForOfStatement' || parent?.type === 'ForInStatement') && parent.left === node)
if ((variableStatement || node.type === 'ExpressionStatement' || node.type === 'ReturnStatement' || node.type === 'ThrowStatement')
&& code[node.end - 1] !== ';') {
insertions.push({ position: node.end, text: ';' })
}
if (node.type === 'Identifier') {
identifiers.add(node.name as string)
}
@@ -359,7 +371,6 @@ export function compileReplCell(code: string, prior: readonly ReplBinding[]) {
// Hoisted values exist before their declaration executes. In a failed cell,
// only reached declarations should replace saved bindings. Insert markers
// without rewriting user initializers, destructuring or nested function code.
const insertions: Array<{ position: number; text: string }> = []
let markerCounter = 0
for (const { node, parent } of hoistedDeclarations) {
if (node.type === 'FunctionDeclaration') {
+68
View File
@@ -82,6 +82,74 @@ suite('Computer Use isolated persistent runtime', () => {
])
})
test('the sandboxed worker completes paste, timer, and observation without replaying paste', async () => {
const runtime = createRuntime()
const calls: string[] = []
const image = { type: 'image' as const, data: 'AAH+/w==', mimeType: 'image/png' }
const invoke = async (name: string) => {
calls.push(name)
return { content: [{ type: 'text' as const, text: 'fixture state' }, image] }
}
const selected = await runtime.run({ code: 'var app = await cua.getApp("Fixture")', timeoutMs: 5000 }, invoke)
expect(selected.isError).not.toBe(true)
const result = await runtime.run({ code: `
await app.paste('query')
await new Promise(resolve => setTimeout(resolve, 0))
await app.getAXStateAndScreenshot()
`, timeoutMs: 5000 }, invoke)
expect(result.isError).not.toBe(true)
expect(calls).toEqual(['get_app_state', 'paste', 'get_app_state'])
expect(result.content).toEqual([{ type: 'text', text: 'fixture state' }, image])
})
test('cancellation during a timer wait resets its bindings and prevents the following action', async () => {
const runtime = createRuntime()
const abort = new AbortController()
const calls: string[] = []
let scheduled!: () => void
const timerScheduled = new Promise<void>(resolve => { scheduled = resolve })
const result = runtime.run({ code: `
var app = await cua.getApp('Fixture')
var waiting = new Promise(resolve => setTimeout(resolve, 10000))
await app.getAXState()
await waiting
await app.click([2,2])
`, timeoutMs: 5000, signal: abort.signal }, async name => {
calls.push(name)
if (calls.length === 2) scheduled()
return { content: [{ type: 'text', text: 'fixture state' }] }
})
// Surface a worker startup failure directly instead of waiting for a
// marker that can no longer arrive.
expect(await Promise.race([timerScheduled.then(() => true), result.then(() => false)])).toBe(true)
abort.abort()
const cancelled = await result
expect(cancelled.isError).toBe(true)
expect(cancelled.structuredContent).toMatchObject({ bindingsReset: true, nativeResultUnknown: false })
expect(calls).toEqual(['get_app_state', 'get_app_state'])
const fresh = await runtime.run({ code: 'nodeRepl.write(typeof waiting)', timeoutMs: 5000 }, async () => {
throw new Error('No native action expected')
})
expect(fresh.content).toEqual([{ type: 'text', text: 'undefined' }])
})
test('the parent deadline also terminates an awaited timer and its pending action', async () => {
const runtime = createRuntime()
const invoke = async () => ({ content: [] })
expect((await runtime.run({ code: 'var retained = 1', timeoutMs: 5000 }, invoke)).isError).not.toBe(true)
const result = await runtime.run({ code: `
await new Promise(resolve => setTimeout(resolve, 10000))
await cua.getApp('Must not dispatch')
`, timeoutMs: 150 }, async () => {
throw new Error('No native action expected')
})
expect(result.isError).toBe(true)
expect(result.content).toEqual([{ type: 'text', text: expect.stringContaining('timed out') }])
expect(result.structuredContent).toMatchObject({ bindingsReset: true, nativeCallsStarted: 0 })
const fresh = await runtime.run({ code: 'nodeRepl.write(typeof retained)', timeoutMs: 5000 }, invoke)
expect(fresh.content).toEqual([{ type: 'text', text: 'undefined' }])
})
test('terminates CPU loops, discards bindings, and starts a fresh kernel', async () => {
const runtime = createRuntime()
const invoke = async () => ({ content: [] })
+196 -4
View File
@@ -347,16 +347,204 @@ describe('computer use persistent REPL worker', () => {
expect(messages.some(message => message.type === 'done')).toBe(true)
})
test('exposes no Node host objects, module loader, timers or code generation', async () => {
test('observes after an awaited timer between paste and observation', async () => {
const messages: Message[] = []
const worker = createComputerUseReplWorker(message => {
messages.push(message)
if (message.type === 'invoke') queueMicrotask(() => worker.receive({
type: 'response', cellId: message.cellId, requestId: message.requestId,
result: { app: 'Fixture', content: [{ type: 'text', text: 'fixture state' }] },
}))
})
await worker.receive({ type: 'init', bootstrap: REPL_BOOTSTRAP_SOURCE })
await worker.receive({ type: 'run', cellId: 1, code: `
let app = await cua.getApp('Fixture')
await app.paste('query')
await new Promise(resolve => setTimeout(resolve, 0))
await app.getAXState()
` })
expect(messages.find(message => message.type === 'done').error).toBeUndefined()
expect(messages.filter(message => message.type === 'invoke').map(message => message.name))
.toEqual(['get_app_state', 'paste', 'get_app_state'])
})
test('supports timer callback arguments, asynchronous zero delay, and either clear function', async () => {
const { worker, messages } = fixture()
await init(worker)
await worker.receive({ type: 'run', cellId: 1, code: `
let order = ['sync']
let original = {name: 'same object'}
let canceled = setTimeout(() => order.push('canceled'), 0)
clearInterval(canceled)
clearTimeout(undefined)
let ticks = 0
await new Promise(resolve => {
const id = setInterval((text, object) => {
order.push([text, object === original, typeof id])
if (++ticks === 2) { clearTimeout(id); resolve() }
}, 0, 'tick', original)
})
await new Promise(resolve => setTimeout((value) => {order.push(value); resolve()}, undefined, 'last'))
nodeRepl.write(order)
` })
expect(texts(messages)).toEqual(['["sync",["tick",true,"number"],["tick",true,"number"],"last"]'])
expect(messages.find(message => message.type === 'done').error).toBeUndefined()
})
test('allows an awaited timer callback to dispatch in its active cell', async () => {
const { worker, messages } = fixture()
await init(worker)
const running = worker.receive({ type: 'run', cellId: 1, code: `
await new Promise(resolve => {
const id = setInterval(async () => {
clearInterval(id)
await __cuInvoke('click', {x: 12, y: 34})
resolve()
}, 0)
})
nodeRepl.write('finished')
` })
await until(() => messages.some(message => message.type === 'invoke'))
const request = messages.find(message => message.type === 'invoke')!
expect(request).toMatchObject({ cellId: 1, name: 'click', args: { x: 12, y: 34 } })
await worker.receive({ type: 'response', cellId: 1, requestId: request.requestId, result: {} })
await running
expect(messages.filter(message => message.type === 'invoke')).toHaveLength(1)
expect(messages.find(message => message.type === 'done').error).toBeUndefined()
expect(texts(messages)).toEqual(['"finished"'])
})
test('clears detached timers on successful and failed cells before another cell can run', async () => {
const { worker, messages } = fixture()
await init(worker)
await worker.receive({ type: 'run', cellId: 1, code: `
let late = 0
setTimeout(() => { late++; __cuInvoke('click', {}) }, 0)
setInterval(() => { late++; __cuInvoke('paste', {}) }, 0)
` })
await worker.receive({ type: 'run', cellId: 2, code: `
setTimeout(() => { late++; __cuInvoke('click', {}) }, 0)
setInterval(() => { late++; __cuInvoke('paste', {}) }, 0)
throw Error('expected cell failure')
` })
await worker.receive({ type: 'run', cellId: 3, code: `
await new Promise(resolve => setTimeout(resolve, 30))
nodeRepl.write(late)
` })
expect(messages.find(message => message.type === 'done' && message.cellId === 2).error).toContain('expected cell failure')
expect(messages.find(message => message.type === 'done' && message.cellId === 3).error).toBeUndefined()
expect(messages.filter(message => message.type === 'invoke')).toEqual([])
expect(texts(messages)).toEqual(['0'])
})
test('rejects timer scheduling or clearing from a previous cell continuation', async () => {
const { worker, messages } = fixture()
await init(worker)
await worker.receive({ type: 'run', cellId: 1, code: `
let release, currentTimer, staleErrors = []
const gate = new Promise(resolve => { release = resolve })
gate.then(() => {
try { clearTimeout(currentTimer) } catch (error) { staleErrors.push(error.message) }
try { setTimeout(() => __cuInvoke('click', {}), 0) } catch (error) { staleErrors.push(error.message) }
})
` })
await worker.receive({ type: 'run', cellId: 2, code: `
await new Promise(resolve => {
currentTimer = setTimeout(resolve, 5)
release()
})
nodeRepl.write(staleErrors)
` })
expect(messages.find(message => message.type === 'done' && message.cellId === 2).error).toBeUndefined()
expect(JSON.parse(texts(messages)[0]!)).toEqual([
expect.stringContaining('cell has ended'), expect.stringContaining('cell has ended'),
])
expect(messages.filter(message => message.type === 'invoke')).toEqual([])
})
test.each(['throw Error("timer broke")', 'await Promise.resolve(); throw Error("timer broke")'])(
'fails the awaiting cell on timer callback error: %s', async body => {
const { worker, messages } = fixture()
await init(worker)
await worker.receive({ type: 'run', cellId: 1, code: `
setTimeout(${body.includes('await') ? 'async ' : ''}() => { ${body} }, 0)
setTimeout(() => __cuInvoke('click', {}), 10)
await new Promise(() => {})
` })
await worker.receive({ type: 'run', cellId: 2, code: `
await new Promise(resolve => setTimeout(resolve, 20))
nodeRepl.write('recovered')
` })
expect(messages.find(message => message.type === 'done' && message.cellId === 1).error).toContain('timer broke')
expect(messages.find(message => message.type === 'done' && message.cellId === 2).error).toBeUndefined()
expect(messages.filter(message => message.type === 'invoke')).toEqual([])
expect(texts(messages)).toEqual(['"recovered"'])
},
)
test('does not dispatch a suspended timer callback after its cell has ended', async () => {
const { worker, messages } = fixture()
await init(worker)
await worker.receive({ type: 'run', cellId: 1, code: `
let release
await new Promise(started => {
setTimeout(async () => {
started()
await new Promise(resolve => { release = resolve })
await __cuInvoke('click', {})
}, 0)
})
` })
await worker.receive({ type: 'run', cellId: 2, code: `
release()
await new Promise(resolve => setTimeout(resolve, 10))
nodeRepl.write('current cell')
` })
expect(messages.filter(message => message.type === 'invoke')).toEqual([])
expect(messages.filter(message => message.type === 'done').every(message => !message.error)).toBe(true)
expect(texts(messages)).toEqual(['"current cell"'])
})
test('rejects string callbacks and bounds active timers while retaining a usable cell', async () => {
const { worker, messages } = fixture()
await init(worker)
await worker.receive({ type: 'run', cellId: 1, code: `
try { setTimeout('globalThis.escaped=true', 0) } catch (error) { nodeRepl.write(error.message) }
try { setInterval(null, 0) } catch (error) { nodeRepl.write(error.message) }
let timers = []
for (let i=0; i<1024; i++) timers.push(setTimeout(() => {}, 60000))
try { setTimeout(() => {}, 0) } catch (error) { nodeRepl.write(error.message) }
timers.forEach(clearTimeout)
await new Promise(resolve => setTimeout(resolve, 0))
nodeRepl.write(typeof escaped)
` })
expect(texts(messages)).toEqual([
'"Timer callback must be a function"', '"Timer callback must be a function"',
'"Too many active JavaScript timers (maximum 1024)"', '"undefined"',
])
expect(messages.find(message => message.type === 'done').error).toBeUndefined()
})
test('exposes realm timers without Node host objects, a module loader or code generation', async () => {
const { worker, messages } = fixture()
await init(worker)
await worker.receive({ type: 'run', cellId: 1, code: `
nodeRepl.write([typeof process, typeof require, typeof Buffer, typeof setTimeout, typeof arguments])
for (const candidate of [() => Function('return 1')(), () => __cuInvoke.constructor('return process')(), () => console.log.constructor('return process')()]) {
for (const candidate of [() => Function('return 1')(), () => __cuInvoke.constructor('return process')(),
() => console.log.constructor('return process')(), ...[setTimeout, clearTimeout, setInterval, clearInterval]
.map(timer => () => timer.constructor('return process')())]) {
try { candidate(); nodeRepl.write('escaped') } catch { nodeRepl.write('blocked') }
}
await new Promise(resolve => setTimeout(function () {
nodeRepl.write(this === globalThis)
const handle = setTimeout(resolve, 0)
nodeRepl.write([typeof handle, typeof handle.ref, setTimeout instanceof Function])
}, 0))
` })
expect(texts(messages)).toEqual(['["undefined","undefined","undefined","undefined","undefined"]', '"blocked"', '"blocked"', '"blocked"'])
expect(texts(messages)).toEqual([
'["undefined","undefined","undefined","function","undefined"]',
...Array(7).fill('"blocked"'), 'false', '["number","undefined",true]',
])
await worker.receive({ type: 'run', cellId: 2, code: 'await import("node:fs")' })
expect(messages.find(message => message.type === 'done' && message.cellId === 2).error).toContain('not available')
})
@@ -432,7 +620,10 @@ describe('computer use persistent REPL worker', () => {
stderr: 'pipe',
})
const messages: Message[] = []
const write = (message: Message) => child.stdin.write(`${JSON.stringify(message)}\n`)
const write = (message: Message) => {
child.stdin.write(`${JSON.stringify(message)}\n`)
child.stdin.flush()
}
const reading = (async () => {
let buffered = ''
const decoder = new TextDecoder()
@@ -458,6 +649,7 @@ describe('computer use persistent REPL worker', () => {
await until(() => messages.some(message => message.type === 'done' && message.cellId === 1))
write({ type: 'run', cellId: 2, code: `
for (const value of [3, 4]) { total += (await __cuInvoke('fixture', {value})).value }
await new Promise(resolve => setTimeout(resolve, 0))
__cuEmit({type:'text', text:String(total)})
` })
await until(() => messages.some(message => message.type === 'done' && message.cellId === 2))
+118 -3
View File
@@ -10,6 +10,9 @@ interface CellState {
accepting: boolean
finished: boolean
pending: Map<number, { settled: Promise<void>; settle: () => void }>
timers: Map<number, ReturnType<typeof setTimeout>>
timerError?: string
failTimer(message: string): void
}
interface RealmRuntime {
@@ -20,8 +23,13 @@ interface RealmRuntime {
commit(succeeded: boolean): string
response(message: string): void
unobservedError(): string | undefined
fireTimer(id: number): void
clearTimers(): void
execute(fn: unknown): Promise<unknown>
}
const MAX_ACTIVE_TIMERS = 1024
function errorMessage(error: unknown): string {
try {
return String((error as { message?: unknown })?.message ?? error).slice(0, 8_000)
@@ -34,7 +42,7 @@ function errorMessage(error: unknown): string {
* Persistent JavaScript kernel. Production runs this in a disposable process;
* vm is a language boundary, not the OS sandbox or hard time/memory limit.
* Only JSON strings cross its host bridge. In particular no host Promise,
* Error, timers, module loader, or Node object is passed into the realm.
* Error, timer handle/function, module loader, or Node object is passed into the realm.
*/
export function createComputerUseReplWorker(send: (message: ReplOutput) => void) {
const context = createContext(Object.create(null), {
@@ -63,6 +71,42 @@ export function createComputerUseReplWorker(send: (message: ReplOutput) => void)
send({ type: 'emit', cellId: cell.cellId, content: message.content })
return
}
if (message.type === 'submitted_done') {
cell.accepting = false
return
}
if (message.type === 'timer_error' && typeof message.message === 'string') {
cell.failTimer(message.message.slice(0, 8_000))
return
}
if (message.type === 'timer' && cell.accepting) {
const { id } = message
if (!Number.isSafeInteger(id) || id < 1) return 'Invalid JavaScript timer ID'
if (message.action === 'clear') {
const handle = cell.timers.get(id)
if (handle !== undefined) clearTimeout(handle)
cell.timers.delete(id)
return
}
if (message.action !== 'schedule' || typeof message.interval !== 'boolean'
|| !Number.isInteger(message.delay) || message.delay < 0 || message.delay > 2_147_483_647) {
return 'Invalid JavaScript timer request'
}
if (cell.timers.has(id)) return 'Duplicate JavaScript timer ID'
if (cell.timers.size >= MAX_ACTIVE_TIMERS) return 'Too many active JavaScript timers (maximum 1024)'
const fire = () => {
if (cell !== active || cell.finished || !cell.accepting) return
if (!message.interval) cell.timers.delete(id)
try {
realm.fireTimer(id)
} catch (error) {
cell.failTimer(errorMessage(error))
}
}
const handle = message.interval ? setInterval(fire, message.delay) : setTimeout(fire, message.delay)
cell.timers.set(id, handle)
return
}
if (message.type !== 'invoke' || !cell.accepting) {
return 'Submitted JavaScript has completed; await every Computer Use operation'
}
@@ -114,6 +158,30 @@ export function createComputerUseReplWorker(send: (message: ReplOutput) => void)
const previous = new Map()
const reached = new Set()
let candidate = []
const timers = new Map()
let timerCounter = 0
const scheduleTimer = (callback, delay, args, interval) => {
if (typeof callback !== 'function') throw new TypeError('Timer callback must be a function')
delay = Number(delay)
// Match Node's asynchronous minimum and integer delay normalization,
// without exposing Node functions or its Timeout objects to the guest.
delay = !Number.isFinite(delay) || delay < 1 || delay > 2147483647 ? 1 : Math.trunc(delay)
const id = ++timerCounter
timers.set(id, {callback, args, interval})
const failure = hostSend(JSON.stringify({type: 'timer', action: 'schedule', id, delay, interval}))
if (failure) {
timers.delete(id)
throw new Error(failure)
}
return id
}
const clearTimer = value => {
const id = typeof value === 'string' ? Number(value) : value
if (!timers.has(id)) return
const failure = hostSend(JSON.stringify({type: 'timer', action: 'clear', id}))
if (failure) throw new Error(failure)
timers.delete(id)
}
const invoke = (name, args) => {
const requestId = ++requestCounter
const operation = {observed: false, error: undefined}
@@ -159,8 +227,41 @@ export function createComputerUseReplWorker(send: (message: ReplOutput) => void)
Object.defineProperties(globalThis, {
__cuInvoke: {value: invoke},
__cuEmit: {value: emit},
setTimeout: {value: (callback, delay = 0, ...args) => scheduleTimer(callback, delay, args, false)},
clearTimeout: {value: clearTimer},
setInterval: {value: (callback, delay = 0, ...args) => scheduleTimer(callback, delay, args, true)},
clearInterval: {value: clearTimer},
})
return {
execute(fn) {
// Close dispatch at the submitted promise's first settlement, before
// the host's timer-error race adds another promise checkpoint. Keep
// these handlers guest-created; no host callbacks enter the realm.
return fn(this.scope, this.register, this.mark).then(value => {
hostSend(JSON.stringify({type: 'submitted_done'}))
return value
}, error => {
hostSend(JSON.stringify({type: 'submitted_done'}))
throw error
})
},
fireTimer(id) {
const entry = timers.get(id)
if (!entry) return
if (!entry.interval) timers.delete(id)
const result = Reflect.apply(entry.callback, undefined, entry.args)
// Async callbacks can reject while the submitted code awaits another
// event. Attribute that error to this cell instead of crashing the host.
Promise.resolve(result).catch(error => {
let message
try { message = String(error?.message ?? error).slice(0, 8000) }
catch { message = 'JavaScript timer callback failed' }
hostSend(JSON.stringify({type: 'timer_error', message}))
})
},
clearTimers() {
timers.clear()
},
begin() {
operations = []
reached.clear()
@@ -269,7 +370,17 @@ export function createComputerUseReplWorker(send: (message: ReplOutput) => void)
send({ type: 'done', cellId, error: 'A JavaScript cell is already running' })
return
}
const cell: CellState = { cellId, accepting: true, finished: false, pending: new Map() }
let rejectTimer!: (error: Error) => void
const timerFailure = new Promise<never>((_resolve, reject) => { rejectTimer = reject })
const cell: CellState = {
cellId, accepting: true, finished: false, pending: new Map(), timers: new Map(),
failTimer(message) {
if (cell.timerError !== undefined) return
cell.timerError = message || 'JavaScript timer callback failed'
cell.accepting = false
rejectTimer(new Error(cell.timerError))
},
}
active = cell
realm.begin()
let error: string | undefined
@@ -281,12 +392,15 @@ export function createComputerUseReplWorker(send: (message: ReplOutput) => void)
for (const warning of compiled.warnings) {
send({ type: 'emit', cellId, content: { type: 'text', text: `Warning: ${warning}` } })
}
await execute(realm.scope, realm.register, realm.mark)
await Promise.race([realm.execute(execute), timerFailure])
submittedSucceeded = true
} catch (failure) {
error = errorMessage(failure)
} finally {
cell.accepting = false
for (const handle of cell.timers.values()) clearTimeout(handle)
cell.timers.clear()
realm.clearTimers()
// A response can settle a pending call on another stdin callback. Keep
// this cell alive until those calls finish, without reopening dispatch.
while (cell.pending.size > 0) {
@@ -300,6 +414,7 @@ export function createComputerUseReplWorker(send: (message: ReplOutput) => void)
// A detached infinite microtask chain also remains under its deadline.
await new Promise<void>(resolve => setImmediate(resolve))
const backgroundError = realm.unobservedError()
if (!error && cell.timerError !== undefined) error = cell.timerError
if (!error && backgroundError) {
error = `An unawaited Computer Use operation failed: ${backgroundError}. Observe the current state before continuing; do not replay prior actions.`
}
+2
View File
@@ -7,6 +7,8 @@ describe('macOS action and observation guidance', () => {
expect(COMPUTER_USE_INSTRUCTIONS).toContain('then observe at a decision point')
expect(COMPUTER_USE_INSTRUCTIONS).toContain('Do not add a fixed sleep before observing')
expect(COMPUTER_USE_INSTRUCTIONS).toContain('Use copied `gN:id` handles')
expect(COMPUTER_USE_INSTRUCTIONS).toContain('new Promise(resolve => setTimeout(resolve, milliseconds))')
expect(COMPUTER_USE_INSTRUCTIONS).toContain('Timers belong to the current cell and are cleared when it')
})
test('keeps standalone receipts and unknown paste results subject to observation', () => {
+6
View File
@@ -52,6 +52,12 @@ A click may do nothing in some canvas apps; after inspecting that result, a shor
0–1 pixel drag can be an alternative. Inspect before repeating the strategy.
Do not add a fixed sleep before observing, or blindly replay a partial batch.
When a task explicitly needs a delay, await
\`new Promise(resolve => setTimeout(resolve, milliseconds))\`.
\`setTimeout\`/\`clearTimeout\` and \`setInterval\`/\`clearInterval\` are available
with numeric handles. Timers belong to the current cell and are cleared when it
ends; do not schedule actions to run after the cell returns.
Each JS cell allows 256 native calls, 256 KiB of code, and 128 output blocks up
to 16 MiB. \`timeout_ms\` defaults to 30000 and cannot exceed 60000. Ordinary
script errors retain bindings. Timeout, cancellation, and \`js_reset\` discard
+2
View File
@@ -344,6 +344,8 @@ describe('native Computer Use JavaScript facade', () => {
expect(f.emitted).toEqual([])
await f.cua.getApp('Fixture')
expect(f.emitted[0]?.value).toContain('Native App API')
expect(f.emitted[0]?.value).toContain('setTimeout/clearTimeout and setInterval/clearInterval')
expect(f.emitted[0]?.value).toContain('Do not schedule actions after the cell returns')
await f.cua.getApp('Fixture again')
expect(f.emitted[1]?.value).toBe(full)
f.results.push({ content: [] })
+1
View File
@@ -65,6 +65,7 @@ Observe: app.getAXState({disableDiffing?,emit?}), app.getScreenshot({emit?}), ap
Act: app.click([x,y] or element,{mouseButton?,clickCount?}), app.drag([x,y],[x,y]), app.pressKey(key), app.scroll([x,y] or element,direction,pages?), app.paste(text,{format?}), app.typeText(text), app.selectText(element,text,{prefix?,suffix?,selectionType?}), app.setValue(element,value), app.performSecondaryAction(element,action).
The raw macOS window API is also available as cua.computer (target:"mac"): list_apps(), get_app_state({app,disableDiff?}), and the corresponding snake_case action methods with an explicit app. Raw methods return data without displaying it.
Use observed gN:id handles. Integer indices require a current AX observation; after image-only capture use getAXState({disableDiffing:true}) before using integers. Coordinates refer to the returned screenshot.
Timers: setTimeout/clearTimeout and setInterval/clearInterval use numeric handles and are cleared when the cell ends. If a delay is required, await new Promise(resolve => setTimeout(resolve, milliseconds)); observations normally need no added sleep. Do not schedule actions after the cell returns.
Await actions in loops, then observe at the next decision point. nodeRepl.write(value) emits text; nodeRepl.emitImage(bytes) emits an image. Browser/DOM, imports, Node, filesystem, and networking APIs are unavailable.`
function textOf(result: ReplToolResult): string {