feat(computer-use): align native app automation with Codex

Add a persistent isolated JavaScript worker for native app actions and batch
known operations without a model round trip between each input. Preserve
per-cell context, native errors, screenshot coordinates, and image types.

Align macOS gesture, key, inventory, capture, scroll, and clipboard behavior;
include a signed native receiver fixture and compiled sidecar regression tests.
Keep the Windows pixel route and fix cancellation with session-owned mouse
cleanup, lock revalidation, and portable signing-fixture tests.
This commit is contained in:
程序员阿江(Relakkes)
2026-09-10 03:34:39 +08:00
parent a2b3f59aaf
commit 56c6a9aa8e
84 changed files with 8719 additions and 726 deletions
+5
View File
@@ -137,6 +137,11 @@ describe('this repository', () => {
// desktop/tsconfig.json, so `check:desktop` alone cannot prove it still builds.
expect(dependentsOf(['desktop/src/lib/browserSafePort.ts'], graph))
.toContain('desktop/electron/services/sidecarManager.ts')
// The desktop CU worker enters the merged sidecar before normal routing.
expect(dependentsOf(['src/utils/computerUse/replWorker.ts'], graph))
.toContain('desktop/sidecars/claude-sidecar.ts')
expect(dependentsOf(['preload.ts'], graph))
.toContain('desktop/sidecars/claude-sidecar.ts')
})
test('resolves every repo-local module specifier so selection stays trustworthy', () => {
+4 -2
View File
@@ -22,7 +22,7 @@ const RESOLUTION_EXTENSIONS = ['.ts', '.tsx', '.mts', '.cts', '.js', '.jsx', '.m
const INDEX_BASENAMES = ['index.ts', 'index.tsx', 'index.js', 'index.jsx', 'index.mjs', 'index.cjs'] as const
const SKIPPED_DIRECTORIES = new Set(['node_modules', '.git', 'dist', 'build', 'coverage', 'artifacts', 'electron-dist', 'build-artifacts', 'target'])
export const GRAPH_SOURCE_ROOTS = ['src', 'desktop/src', 'desktop/electron', 'adapters', 'scripts'] as const
export const GRAPH_SOURCE_ROOTS = ['src', 'desktop/src', 'desktop/electron', 'desktop/sidecars', 'adapters', 'scripts', 'preload.ts'] as const
/**
* Matches `import ... from 'x'`, `export ... from 'x'`, bare `import 'x'`,
@@ -80,7 +80,9 @@ export function listGraphSourceFiles(rootDir: string, roots: readonly string[] =
for (const root of roots) {
const absolute = join(rootDir, root)
if (existsSync(absolute)) walk(absolute)
if (!existsSync(absolute)) continue
if (statSync(absolute).isFile() && isSourceFile(root)) files.push(normalizeGraphPath(root))
else walk(absolute)
}
return files.sort()