mirror of
https://github.com/NanmiCoder/claude-code-haha.git
synced 2026-10-10 03:43:11 +08:00
fix(computer-use): remove per-app approval prompts
This commit is contained in:
@@ -1,202 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest'
|
||||
import { act, fireEvent, render, screen } from '@testing-library/react'
|
||||
|
||||
const { sendMock, openSettingsMock } = vi.hoisted(() => ({
|
||||
sendMock: vi.fn(),
|
||||
openSettingsMock: vi.fn(async () => ({ ok: true })),
|
||||
}))
|
||||
|
||||
vi.mock('../../api/websocket', () => ({
|
||||
wsManager: {
|
||||
connect: vi.fn(),
|
||||
disconnect: vi.fn(),
|
||||
onConnectionState: vi.fn((_sessionId: string, handler: (state: string) => void) => {
|
||||
handler('connecting')
|
||||
return () => {}
|
||||
}),
|
||||
onMessage: vi.fn(() => () => {}),
|
||||
clearHandlers: vi.fn(),
|
||||
send: sendMock,
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('../../api/sessions', () => ({
|
||||
sessionsApi: {
|
||||
getMessages: vi.fn(async () => ({ messages: [] })),
|
||||
getSlashCommands: vi.fn(async () => ({ commands: [] })),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('../../stores/teamStore', () => ({
|
||||
useTeamStore: {
|
||||
getState: () => ({
|
||||
getMemberBySessionId: vi.fn(() => null),
|
||||
sendMessageToMember: vi.fn(async () => {}),
|
||||
handleTeamCreated: vi.fn(),
|
||||
handleTeamUpdate: vi.fn(),
|
||||
handleTeamDeleted: vi.fn(),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('../../stores/tabStore', () => ({
|
||||
useTabStore: {
|
||||
getState: () => ({
|
||||
updateTabStatus: vi.fn(),
|
||||
updateTabTitle: vi.fn(),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('../../stores/sessionStore', () => ({
|
||||
useSessionStore: {
|
||||
getState: () => ({
|
||||
updateSessionTitle: vi.fn(),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('../../stores/cliTaskStore', () => ({
|
||||
useCLITaskStore: {
|
||||
getState: () => ({
|
||||
fetchSessionTasks: vi.fn(),
|
||||
tasks: [],
|
||||
clearTasks: vi.fn(),
|
||||
setTasksFromTodos: vi.fn(),
|
||||
markCompletedAndDismissed: vi.fn(),
|
||||
resetCompletedTasks: vi.fn(async () => {}),
|
||||
refreshTasks: vi.fn(),
|
||||
}),
|
||||
},
|
||||
}))
|
||||
|
||||
vi.mock('../../api/computerUse', () => ({
|
||||
computerUseApi: {
|
||||
openSettings: openSettingsMock,
|
||||
},
|
||||
}))
|
||||
|
||||
import { useChatStore } from '../../stores/chatStore'
|
||||
import { useSettingsStore } from '../../stores/settingsStore'
|
||||
import { ComputerUsePermissionModal } from './ComputerUsePermissionModal'
|
||||
|
||||
describe('ComputerUsePermissionModal', () => {
|
||||
beforeEach(() => {
|
||||
sendMock.mockReset()
|
||||
openSettingsMock.mockReset()
|
||||
useSettingsStore.setState({ locale: 'en' })
|
||||
useChatStore.setState({ sessions: {} })
|
||||
})
|
||||
|
||||
it('returns a full approval payload for resolved apps and requested flags', () => {
|
||||
render(
|
||||
<ComputerUsePermissionModal
|
||||
sessionId="session-1"
|
||||
request={{
|
||||
requestId: 'cu-1',
|
||||
reason: 'Open Finder and inspect a file',
|
||||
apps: [
|
||||
{
|
||||
requestedName: 'Finder',
|
||||
resolved: {
|
||||
bundleId: 'com.apple.finder',
|
||||
displayName: 'Finder',
|
||||
},
|
||||
isSentinel: false,
|
||||
alreadyGranted: false,
|
||||
proposedTier: 'full',
|
||||
},
|
||||
{
|
||||
requestedName: 'Missing App',
|
||||
isSentinel: false,
|
||||
alreadyGranted: false,
|
||||
proposedTier: 'full',
|
||||
},
|
||||
],
|
||||
requestedFlags: {
|
||||
clipboardRead: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
screenshotFiltering: 'native',
|
||||
willHide: [{ bundleId: 'com.apple.TextEdit', displayName: 'TextEdit' }],
|
||||
autoUnhideEnabled: true,
|
||||
}}
|
||||
/>,
|
||||
)
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Allow for session' }))
|
||||
|
||||
expect(sendMock).toHaveBeenCalledTimes(1)
|
||||
expect(sendMock).toHaveBeenCalledWith('session-1', {
|
||||
type: 'computer_use_permission_response',
|
||||
requestId: 'cu-1',
|
||||
response: {
|
||||
granted: [
|
||||
expect.objectContaining({
|
||||
bundleId: 'com.apple.finder',
|
||||
displayName: 'Finder',
|
||||
tier: 'full',
|
||||
}),
|
||||
],
|
||||
denied: [
|
||||
{
|
||||
bundleId: 'Missing App',
|
||||
reason: 'not_installed',
|
||||
},
|
||||
],
|
||||
flags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
userConsented: true,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('opens System Settings from the macOS permission panel', async () => {
|
||||
render(
|
||||
<ComputerUsePermissionModal
|
||||
sessionId="session-1"
|
||||
request={{
|
||||
requestId: 'cu-1',
|
||||
reason: '',
|
||||
apps: [],
|
||||
requestedFlags: {},
|
||||
screenshotFiltering: 'native',
|
||||
tccState: {
|
||||
accessibility: false,
|
||||
screenRecording: true,
|
||||
},
|
||||
}}
|
||||
/>,
|
||||
)
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Open Accessibility' }))
|
||||
})
|
||||
|
||||
expect(openSettingsMock).toHaveBeenCalledWith('Privacy_Accessibility')
|
||||
})
|
||||
|
||||
it('discloses that Windows screenshots include ungranted visible apps', () => {
|
||||
render(
|
||||
<ComputerUsePermissionModal
|
||||
sessionId="session-1"
|
||||
request={{
|
||||
requestId: 'cu-windows',
|
||||
reason: 'Inspect Explorer',
|
||||
apps: [],
|
||||
requestedFlags: {},
|
||||
screenshotFiltering: 'none',
|
||||
}}
|
||||
/>,
|
||||
)
|
||||
|
||||
const disclosure = screen.getByRole('note').textContent ?? ''
|
||||
expect(disclosure).toContain(
|
||||
'screenshots can include every visible window on this display',
|
||||
)
|
||||
expect(disclosure).toContain('Input remains limited to the apps you allow')
|
||||
})
|
||||
})
|
||||
@@ -1,318 +0,0 @@
|
||||
import { useMemo, useState } from 'react'
|
||||
import { useTranslation } from '../../i18n'
|
||||
import { computerUseApi } from '../../api/computerUse'
|
||||
import { useChatStore } from '../../stores/chatStore'
|
||||
import type {
|
||||
ComputerUsePermissionRequest,
|
||||
ComputerUsePermissionResponse,
|
||||
} from '../../types/chat'
|
||||
import { Badge } from '@/components/ui/Badge'
|
||||
import { Button } from '@/components/ui/Button'
|
||||
import { Modal } from '@/components/ui/Modal'
|
||||
|
||||
type Props = {
|
||||
sessionId: string
|
||||
request: ComputerUsePermissionRequest | null
|
||||
}
|
||||
|
||||
const DEFAULT_GRANT_FLAGS = {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
} as const
|
||||
|
||||
function denyAllResponse(): ComputerUsePermissionResponse {
|
||||
return {
|
||||
granted: [],
|
||||
denied: [],
|
||||
flags: { ...DEFAULT_GRANT_FLAGS },
|
||||
userConsented: false,
|
||||
}
|
||||
}
|
||||
|
||||
function buildAllowResponse(
|
||||
request: ComputerUsePermissionRequest,
|
||||
): ComputerUsePermissionResponse {
|
||||
const now = Date.now()
|
||||
const granted = request.apps.flatMap((app) => {
|
||||
if (!app.resolved || app.alreadyGranted) return []
|
||||
return [{
|
||||
bundleId: app.resolved.bundleId,
|
||||
displayName: app.resolved.displayName,
|
||||
grantedAt: now,
|
||||
tier: app.proposedTier,
|
||||
}]
|
||||
})
|
||||
|
||||
const denied = request.apps.flatMap((app) => {
|
||||
if (app.resolved) return []
|
||||
return [{
|
||||
bundleId: app.requestedName,
|
||||
reason: 'not_installed' as const,
|
||||
}]
|
||||
})
|
||||
|
||||
const flags = {
|
||||
...DEFAULT_GRANT_FLAGS,
|
||||
...Object.fromEntries(
|
||||
Object.entries(request.requestedFlags).filter(([, value]) => value === true),
|
||||
),
|
||||
}
|
||||
|
||||
return {
|
||||
granted,
|
||||
denied,
|
||||
flags,
|
||||
userConsented: true,
|
||||
}
|
||||
}
|
||||
|
||||
export function ComputerUsePermissionModal({ sessionId, request }: Props) {
|
||||
const t = useTranslation()
|
||||
const respondToComputerUsePermission = useChatStore(
|
||||
(s) => s.respondToComputerUsePermission,
|
||||
)
|
||||
const [openingPane, setOpeningPane] = useState<
|
||||
'Privacy_Accessibility' | 'Privacy_ScreenCapture' | null
|
||||
>(null)
|
||||
|
||||
const requestedFlags = useMemo(
|
||||
() =>
|
||||
request
|
||||
? Object.entries(request.requestedFlags)
|
||||
.filter(([, enabled]) => enabled)
|
||||
.map(([flag]) => flag)
|
||||
: [],
|
||||
[request],
|
||||
)
|
||||
|
||||
if (!request) return null
|
||||
|
||||
const handleDeny = () => {
|
||||
respondToComputerUsePermission(
|
||||
sessionId,
|
||||
request.requestId,
|
||||
denyAllResponse(),
|
||||
)
|
||||
}
|
||||
|
||||
const handleAllow = () => {
|
||||
respondToComputerUsePermission(
|
||||
sessionId,
|
||||
request.requestId,
|
||||
buildAllowResponse(request),
|
||||
)
|
||||
}
|
||||
|
||||
const openSettings = async (
|
||||
pane: 'Privacy_Accessibility' | 'Privacy_ScreenCapture',
|
||||
) => {
|
||||
setOpeningPane(pane)
|
||||
try {
|
||||
await computerUseApi.openSettings(pane)
|
||||
} finally {
|
||||
setOpeningPane(null)
|
||||
}
|
||||
}
|
||||
|
||||
const tccState = request.tccState
|
||||
|
||||
return (
|
||||
<Modal
|
||||
open
|
||||
onClose={handleDeny}
|
||||
title={
|
||||
tccState
|
||||
? t('computerUseApproval.titleTcc')
|
||||
: t('computerUseApproval.titleApps')
|
||||
}
|
||||
width={640}
|
||||
footer={
|
||||
tccState ? (
|
||||
<Button variant="ghost" onClick={handleDeny}>
|
||||
{t('computerUseApproval.deny')}
|
||||
</Button>
|
||||
) : (
|
||||
<>
|
||||
<Button variant="ghost" onClick={handleDeny}>
|
||||
{t('computerUseApproval.deny')}
|
||||
</Button>
|
||||
<Button variant="primary" onClick={handleAllow}>
|
||||
{t('computerUseApproval.allow')}
|
||||
</Button>
|
||||
</>
|
||||
)
|
||||
}
|
||||
>
|
||||
{tccState ? (
|
||||
<div className="space-y-4">
|
||||
<p className="text-sm text-[var(--color-text-secondary)]">
|
||||
{t('computerUseApproval.tccHint')}
|
||||
</p>
|
||||
|
||||
<div className="space-y-3">
|
||||
<PermissionRow
|
||||
label={t('computerUseApproval.accessibility')}
|
||||
granted={tccState.accessibility}
|
||||
actionLabel={t('computerUseApproval.openAccessibility')}
|
||||
actionLoading={openingPane === 'Privacy_Accessibility'}
|
||||
onAction={() => openSettings('Privacy_Accessibility')}
|
||||
/>
|
||||
<PermissionRow
|
||||
label={t('computerUseApproval.screenRecording')}
|
||||
granted={tccState.screenRecording}
|
||||
actionLabel={t('computerUseApproval.openScreenRecording')}
|
||||
actionLoading={openingPane === 'Privacy_ScreenCapture'}
|
||||
onAction={() => openSettings('Privacy_ScreenCapture')}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="rounded-[var(--radius-lg)] border border-[var(--color-border)] bg-[var(--color-surface-container-low)] p-3 text-xs text-[var(--color-text-tertiary)]">
|
||||
{t('computerUseApproval.tryAgainHint')}
|
||||
</div>
|
||||
|
||||
<div className="flex justify-end">
|
||||
<Button variant="secondary" onClick={handleDeny}>
|
||||
{t('computerUseApproval.tryAgain')}
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-4">
|
||||
{request.screenshotFiltering === 'none' ? (
|
||||
<div
|
||||
role="note"
|
||||
className="rounded-[var(--radius-lg)] border border-[var(--color-warning)] bg-[var(--color-warning-container)] p-3 text-sm text-[var(--color-on-warning-container)]"
|
||||
>
|
||||
{t('computerUseApproval.unfilteredScreenshots')}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{request.reason ? (
|
||||
<div className="rounded-[var(--radius-lg)] border border-[var(--color-border)] bg-[var(--color-surface-container-low)] p-3">
|
||||
<div className="text-xs font-semibold uppercase tracking-wide text-[var(--color-text-tertiary)]">
|
||||
{t('computerUseApproval.reason')}
|
||||
</div>
|
||||
<div className="mt-1 text-sm text-[var(--color-text-primary)]">
|
||||
{request.reason}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
<div className="space-y-2">
|
||||
{request.apps.map((app) => {
|
||||
const resolved = app.resolved
|
||||
return (
|
||||
<div
|
||||
key={resolved?.bundleId ?? app.requestedName}
|
||||
className="rounded-[var(--radius-lg)] border border-[var(--color-border)] bg-[var(--color-surface-container-low)] p-3"
|
||||
>
|
||||
<div className="flex items-start justify-between gap-3">
|
||||
<div>
|
||||
<div className="text-sm font-semibold text-[var(--color-text-primary)]">
|
||||
{resolved?.displayName ?? app.requestedName}
|
||||
</div>
|
||||
<div className="mt-1 text-xs text-[var(--color-text-tertiary)]">
|
||||
{resolved?.bundleId ?? t('computerUseApproval.notInstalled')}
|
||||
</div>
|
||||
</div>
|
||||
<Badge className="font-semibold uppercase tracking-wide">
|
||||
{app.proposedTier}
|
||||
</Badge>
|
||||
</div>
|
||||
|
||||
{!resolved ? (
|
||||
<p className="mt-2 text-xs text-[var(--color-error)]">
|
||||
{t('computerUseApproval.notInstalled')}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{app.alreadyGranted ? (
|
||||
<p className="mt-2 text-xs text-[var(--color-success)]">
|
||||
{t('computerUseApproval.alreadyGranted')}
|
||||
</p>
|
||||
) : null}
|
||||
|
||||
{app.isSentinel ? (
|
||||
<p className="mt-2 text-xs text-[var(--color-warning)]">
|
||||
{t('computerUseApproval.sensitiveApp')}
|
||||
</p>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
|
||||
{requestedFlags.length > 0 ? (
|
||||
<div className="rounded-[var(--radius-lg)] border border-[var(--color-border)] bg-[var(--color-surface-container-low)] p-3">
|
||||
<div className="text-xs font-semibold uppercase tracking-wide text-[var(--color-text-tertiary)]">
|
||||
{t('computerUseApproval.alsoRequested')}
|
||||
</div>
|
||||
<div className="mt-2 flex flex-wrap gap-2">
|
||||
{requestedFlags.map((flag) => (
|
||||
<Badge key={flag} size="md" mono wrap>
|
||||
{flag}
|
||||
</Badge>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{request.willHide && request.willHide.length > 0 ? (
|
||||
<div className="rounded-[var(--radius-lg)] border border-[var(--color-border)] bg-[var(--color-surface-container-low)] p-3 text-sm text-[var(--color-text-secondary)]">
|
||||
{request.autoUnhideEnabled
|
||||
? t('computerUseApproval.hideWhileWorkingRestore', {
|
||||
count: request.willHide.length,
|
||||
})
|
||||
: t('computerUseApproval.hideWhileWorking', {
|
||||
count: request.willHide.length,
|
||||
})}
|
||||
</div>
|
||||
) : null}
|
||||
</div>
|
||||
)}
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
function PermissionRow({
|
||||
label,
|
||||
granted,
|
||||
actionLabel,
|
||||
actionLoading,
|
||||
onAction,
|
||||
}: {
|
||||
label: string
|
||||
granted: boolean
|
||||
actionLabel: string
|
||||
actionLoading: boolean
|
||||
onAction: () => void
|
||||
}) {
|
||||
const t = useTranslation()
|
||||
|
||||
return (
|
||||
<div className="flex items-center justify-between gap-4 rounded-[var(--radius-lg)] border border-[var(--color-border)] bg-[var(--color-surface-container-low)] p-3">
|
||||
<div>
|
||||
<div className="text-sm font-semibold text-[var(--color-text-primary)]">
|
||||
{label}
|
||||
</div>
|
||||
<div className="mt-1 text-xs text-[var(--color-text-tertiary)]">
|
||||
{granted
|
||||
? t('computerUseApproval.granted')
|
||||
: t('computerUseApproval.notGranted')}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{!granted ? (
|
||||
<Button
|
||||
variant="secondary"
|
||||
size="sm"
|
||||
loading={actionLoading}
|
||||
onClick={onAction}
|
||||
>
|
||||
{actionLabel}
|
||||
</Button>
|
||||
) : null}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
import { fireEvent, render, screen } from '@testing-library/react'
|
||||
import { describe, expect, it, vi } from 'vitest'
|
||||
import '@testing-library/jest-dom'
|
||||
import { ComputerUseEnableDialog } from './ComputerUseEnableDialog'
|
||||
|
||||
describe('ComputerUseEnableDialog', () => {
|
||||
it('explains full-computer access and confirms explicitly', () => {
|
||||
const onConfirm = vi.fn()
|
||||
render(
|
||||
<ComputerUseEnableDialog
|
||||
open
|
||||
platform="win32"
|
||||
onClose={() => {}}
|
||||
onConfirm={onConfirm}
|
||||
/>,
|
||||
)
|
||||
|
||||
expect(screen.getByRole('dialog')).toHaveTextContent('all supported applications')
|
||||
expect(screen.getByRole('dialog')).toHaveTextContent('screenshots')
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Enable Computer Use' }))
|
||||
expect(onConfirm).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
it('adds the macOS system-permission distinction', () => {
|
||||
render(
|
||||
<ComputerUseEnableDialog
|
||||
open
|
||||
platform="darwin"
|
||||
onClose={() => {}}
|
||||
onConfirm={() => {}}
|
||||
/>,
|
||||
)
|
||||
|
||||
expect(screen.getByRole('dialog')).toHaveTextContent('Accessibility and Screen Recording')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,62 @@
|
||||
import { useTranslation } from '@/i18n'
|
||||
import { ActionDialog } from '@/components/ui/ActionDialog'
|
||||
|
||||
type Props = {
|
||||
open: boolean
|
||||
loading?: boolean
|
||||
platform: 'darwin' | 'win32'
|
||||
onClose: () => void
|
||||
onConfirm: () => void | Promise<void>
|
||||
}
|
||||
|
||||
export function ComputerUseEnableDialog({
|
||||
open,
|
||||
loading = false,
|
||||
platform,
|
||||
onClose,
|
||||
onConfirm,
|
||||
}: Props) {
|
||||
const t = useTranslation()
|
||||
|
||||
return (
|
||||
<ActionDialog
|
||||
open={open}
|
||||
onClose={onClose}
|
||||
title={t('settings.computerUse.enableRiskTitle')}
|
||||
width={500}
|
||||
loading={loading}
|
||||
body={(
|
||||
<div className="space-y-4 text-sm leading-6 text-[var(--color-text-secondary)]">
|
||||
<div className="flex items-start gap-3 rounded-[var(--radius-lg)] border border-[var(--color-warning)] bg-[var(--color-warning-container)] px-3 py-3 text-[var(--color-on-warning-container)]">
|
||||
<span className="material-symbols-outlined mt-0.5 text-[20px] text-[var(--color-warning)]">warning</span>
|
||||
<p className="font-semibold">{t('settings.computerUse.enableRiskSummary')}</p>
|
||||
</div>
|
||||
<ul className="list-disc space-y-1.5 pl-5">
|
||||
<li>{t('settings.computerUse.enableRiskScreen')}</li>
|
||||
<li>{t('settings.computerUse.enableRiskActions')}</li>
|
||||
<li>{t('settings.computerUse.enableRiskAllApps')}</li>
|
||||
<li>{t('settings.computerUse.enableRiskStop')}</li>
|
||||
</ul>
|
||||
{platform === 'darwin' && (
|
||||
<p className="rounded-[var(--radius-lg)] bg-[var(--color-surface-container-low)] px-3 py-2 text-xs text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.enableRiskMacos')}
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
actions={[
|
||||
{
|
||||
label: t('common.cancel'),
|
||||
onClick: onClose,
|
||||
variant: 'secondary',
|
||||
},
|
||||
{
|
||||
label: t('settings.computerUse.enableRiskConfirm'),
|
||||
onClick: onConfirm,
|
||||
variant: 'primary',
|
||||
loading,
|
||||
},
|
||||
]}
|
||||
/>
|
||||
)
|
||||
}
|
||||
@@ -1389,6 +1389,14 @@ Row 9, all 8 cells: continuing from straight down, turning left through lower-le
|
||||
'settings.computerUse.title': 'Computer Use',
|
||||
'settings.computerUse.description': 'Allow Claude to take screenshots, click, type, and control your computer. Requires Python 3. On macOS, accessibility permissions are also needed.',
|
||||
'settings.computerUse.enabledToggle': 'Enabled',
|
||||
'settings.computerUse.enableRiskTitle': 'Enable Computer Use?',
|
||||
'settings.computerUse.enableRiskSummary': 'Computer Use gives Claude broad control of this computer. Only enable it when you understand and accept these risks.',
|
||||
'settings.computerUse.enableRiskScreen': 'Claude can take screenshots and may see sensitive or private information on screen.',
|
||||
'settings.computerUse.enableRiskActions': 'Claude can click, type, use the clipboard and system shortcuts, and may send, change, or delete content.',
|
||||
'settings.computerUse.enableRiskAllApps': 'Once enabled, Claude can control all supported applications without asking for permission app by app. Product safety restrictions still apply.',
|
||||
'settings.computerUse.enableRiskStop': 'You can interrupt control at any time with Stop or Esc, and disable Computer Use here.',
|
||||
'settings.computerUse.enableRiskMacos': 'macOS still requires the system-level Accessibility and Screen Recording permissions. Those operating-system prompts are separate from app authorization.',
|
||||
'settings.computerUse.enableRiskConfirm': 'Enable Computer Use',
|
||||
'settings.computerUse.disabledHint': 'Computer Use is off. New sessions will not inject the computer-use MCP server or expose desktop-control tools to the Coding Agent.',
|
||||
'settings.computerUse.notSupported': 'Computer Use is only supported on macOS and Windows.',
|
||||
'settings.computerUse.macosUnsupportedTitle': 'Computer Use requires macOS {version} or later',
|
||||
@@ -2027,27 +2035,6 @@ Row 9, all 8 cells: continuing from straight down, turning left through lower-le
|
||||
'permission.planRejected': 'Plan rejected',
|
||||
|
||||
// ─── Computer Use Approval ──────────────────────────────────────
|
||||
'computerUseApproval.titleApps': 'Computer Use wants to control these apps',
|
||||
'computerUseApproval.titleTcc': 'Computer Use needs macOS permissions',
|
||||
'computerUseApproval.reason': 'Why Claude is asking',
|
||||
'computerUseApproval.allow': 'Allow for session',
|
||||
'computerUseApproval.deny': 'Deny',
|
||||
'computerUseApproval.alreadyGranted': 'Already granted for this session',
|
||||
'computerUseApproval.notInstalled': 'App not installed',
|
||||
'computerUseApproval.unfilteredScreenshots': 'On Windows, screenshots can include every visible window on this display, including apps not listed below. Input remains limited to the apps you allow.',
|
||||
'computerUseApproval.sensitiveApp': 'This app is treated as sensitive and deserves extra review.',
|
||||
'computerUseApproval.alsoRequested': 'Also requested',
|
||||
'computerUseApproval.hideWhileWorking': '{count} other apps will be hidden while Claude works.',
|
||||
'computerUseApproval.hideWhileWorkingRestore': '{count} other apps will be hidden while Claude works, then restored when Claude is done.',
|
||||
'computerUseApproval.accessibility': 'Accessibility',
|
||||
'computerUseApproval.screenRecording': 'Screen Recording',
|
||||
'computerUseApproval.granted': 'Granted',
|
||||
'computerUseApproval.notGranted': 'Not granted',
|
||||
'computerUseApproval.openAccessibility': 'Open Accessibility',
|
||||
'computerUseApproval.openScreenRecording': 'Open Screen Recording',
|
||||
'computerUseApproval.tryAgain': 'Try again',
|
||||
'computerUseApproval.tccHint': 'Grant the missing permissions in System Settings, then come back and choose "Try again".',
|
||||
'computerUseApproval.tryAgainHint': 'Try again returns control to Claude so it can call request_access once more after macOS permission changes take effect.',
|
||||
|
||||
// ─── Ask User Question ──────────────────────────────────────
|
||||
'question.needsInput': 'Claude needs your input',
|
||||
|
||||
@@ -1391,6 +1391,14 @@ export const jp: Record<TranslationKey, string> = {
|
||||
'settings.computerUse.title': 'コンピューター操作',
|
||||
'settings.computerUse.description': 'Claude がスクリーンショットを撮影し、クリック、入力を行い、コンピューターを操作できるようにします。Python 3 が必要です。macOS ではアクセシビリティ権限も必要です。',
|
||||
'settings.computerUse.enabledToggle': '有効',
|
||||
'settings.computerUse.enableRiskTitle': 'Computer Use を有効にしますか?',
|
||||
'settings.computerUse.enableRiskSummary': 'Computer Use は Claude にこのコンピューターを幅広く操作する権限を与えます。以下のリスクを理解し、同意した場合のみ有効にしてください。',
|
||||
'settings.computerUse.enableRiskScreen': 'Claude はスクリーンショットを取得し、画面上の機密情報や個人情報を見る可能性があります。',
|
||||
'settings.computerUse.enableRiskActions': 'Claude はクリック、入力、クリップボード、システムショートカットを使用し、内容を送信、変更、削除する可能性があります。',
|
||||
'settings.computerUse.enableRiskAllApps': '有効にすると、Claude はアプリごとの許可を求めず、対応するすべてのアプリを操作できます。製品の安全制限は引き続き適用されます。',
|
||||
'settings.computerUse.enableRiskStop': '停止ボタンまたは Esc でいつでも操作を中断でき、ここで Computer Use を無効にできます。',
|
||||
'settings.computerUse.enableRiskMacos': 'macOS では、システムレベルのアクセシビリティと画面収録の許可が引き続き必要です。これらはアプリの許可とは別です。',
|
||||
'settings.computerUse.enableRiskConfirm': 'Computer Use を有効にする',
|
||||
'settings.computerUse.disabledHint': 'コンピューター操作はオフです。新しいセッションでは、computer-use MCP サーバーを注入したり、デスクトップ操作ツールをコーディングエージェントに公開したりしません。',
|
||||
'settings.computerUse.notSupported': 'コンピューター操作は macOS と Windows でのみサポートされます。',
|
||||
'settings.computerUse.macosUnsupportedTitle': 'Computer Use には macOS {version} 以降が必要です',
|
||||
@@ -2029,27 +2037,6 @@ export const jp: Record<TranslationKey, string> = {
|
||||
'permission.planRejected': '計画が拒否されました',
|
||||
|
||||
// ─── Computer Use Approval ──────────────────────────────────────
|
||||
'computerUseApproval.titleApps': 'コンピューター操作がこれらのアプリを操作しようとしています',
|
||||
'computerUseApproval.titleTcc': 'コンピューター操作には macOS の権限が必要です',
|
||||
'computerUseApproval.reason': 'Claude が要求している理由',
|
||||
'computerUseApproval.allow': 'このセッションで許可',
|
||||
'computerUseApproval.deny': '拒否',
|
||||
'computerUseApproval.alreadyGranted': 'このセッションでは既に許可されています',
|
||||
'computerUseApproval.notInstalled': 'アプリがインストールされていません',
|
||||
'computerUseApproval.unfilteredScreenshots': 'Windows では、このディスプレイに表示されているすべてのウィンドウ(下にないアプリを含む)がスクリーンショットに写る場合があります。入力操作は許可したアプリだけに制限されます。',
|
||||
'computerUseApproval.sensitiveApp': 'このアプリは機密として扱われ、追加の確認が必要です。',
|
||||
'computerUseApproval.alsoRequested': '同時に要求中',
|
||||
'computerUseApproval.hideWhileWorking': 'Claude の作業中、他の {count} 個のアプリが非表示になります。',
|
||||
'computerUseApproval.hideWhileWorkingRestore': 'Claude の作業中、他の {count} 個のアプリが非表示になり、Claude の完了後に復元されます。',
|
||||
'computerUseApproval.accessibility': 'アクセシビリティ',
|
||||
'computerUseApproval.screenRecording': '画面収録',
|
||||
'computerUseApproval.granted': '許可済み',
|
||||
'computerUseApproval.notGranted': '未許可',
|
||||
'computerUseApproval.openAccessibility': 'アクセシビリティを開く',
|
||||
'computerUseApproval.openScreenRecording': '画面収録を開く',
|
||||
'computerUseApproval.tryAgain': '再試行',
|
||||
'computerUseApproval.tccHint': 'システム設定で不足している権限を許可してから、ここに戻って「再試行」を選択してください。',
|
||||
'computerUseApproval.tryAgainHint': '再試行すると Claude に制御が戻り、macOS の権限変更が反映された後に request_access をもう一度呼び出せます。',
|
||||
|
||||
// ─── Ask User Question ──────────────────────────────────────
|
||||
'question.needsInput': 'Claude が入力を必要としています',
|
||||
|
||||
@@ -1391,6 +1391,14 @@ export const kr: Record<TranslationKey, string> = {
|
||||
'settings.computerUse.title': '컴퓨터 사용',
|
||||
'settings.computerUse.description': 'Claude가 스크린샷을 찍고 클릭, 입력하며 컴퓨터를 제어할 수 있도록 허용합니다. Python 3가 필요합니다. macOS에서는 접근성 권한도 필요합니다.',
|
||||
'settings.computerUse.enabledToggle': '사용',
|
||||
'settings.computerUse.enableRiskTitle': 'Computer Use를 활성화할까요?',
|
||||
'settings.computerUse.enableRiskSummary': 'Computer Use는 Claude에게 이 컴퓨터를 폭넓게 제어할 권한을 부여합니다. 다음 위험을 이해하고 동의하는 경우에만 활성화하세요.',
|
||||
'settings.computerUse.enableRiskScreen': 'Claude는 스크린샷을 찍고 화면의 민감한 정보나 개인정보를 볼 수 있습니다.',
|
||||
'settings.computerUse.enableRiskActions': 'Claude는 클릭, 입력, 클립보드와 시스템 단축키를 사용하고 콘텐츠를 전송, 변경 또는 삭제할 수 있습니다.',
|
||||
'settings.computerUse.enableRiskAllApps': '활성화하면 Claude는 앱별 권한을 다시 묻지 않고 지원되는 모든 앱을 제어할 수 있습니다. 제품 안전 제한은 계속 적용됩니다.',
|
||||
'settings.computerUse.enableRiskStop': '중지 버튼이나 Esc로 언제든 제어를 중단하고 여기에서 Computer Use를 비활성화할 수 있습니다.',
|
||||
'settings.computerUse.enableRiskMacos': 'macOS에서는 시스템 수준의 손쉬운 사용 및 화면 기록 권한이 계속 필요합니다. 이 운영 체제 권한은 앱 승인과 별개입니다.',
|
||||
'settings.computerUse.enableRiskConfirm': 'Computer Use 활성화',
|
||||
'settings.computerUse.disabledHint': '컴퓨터 사용이 꺼져 있습니다. 새 세션은 computer-use MCP 서버를 주입하거나 데스크톱 제어 도구를 코딩 에이전트에 노출하지 않습니다.',
|
||||
'settings.computerUse.notSupported': '컴퓨터 사용은 macOS와 Windows에서만 지원됩니다.',
|
||||
'settings.computerUse.macosUnsupportedTitle': 'Computer Use에는 macOS {version} 이상이 필요합니다',
|
||||
@@ -2029,27 +2037,6 @@ export const kr: Record<TranslationKey, string> = {
|
||||
'permission.planRejected': '계획 거부됨',
|
||||
|
||||
// ─── Computer Use Approval ──────────────────────────────────────
|
||||
'computerUseApproval.titleApps': '컴퓨터 사용이 이 앱들을 제어하려고 합니다',
|
||||
'computerUseApproval.titleTcc': '컴퓨터 사용에 macOS 권한이 필요합니다',
|
||||
'computerUseApproval.reason': 'Claude가 요청하는 이유',
|
||||
'computerUseApproval.allow': '이 세션에서 허용',
|
||||
'computerUseApproval.deny': '거부',
|
||||
'computerUseApproval.alreadyGranted': '이 세션에서는 이미 허용됨',
|
||||
'computerUseApproval.notInstalled': '앱이 설치되지 않음',
|
||||
'computerUseApproval.unfilteredScreenshots': 'Windows에서는 아래에 나열되지 않은 앱을 포함해 이 디스플레이에 보이는 모든 창이 스크린샷에 포함될 수 있습니다. 입력 동작은 허용한 앱으로만 제한됩니다.',
|
||||
'computerUseApproval.sensitiveApp': '이 앱은 민감한 것으로 처리되며 추가 검토가 필요합니다.',
|
||||
'computerUseApproval.alsoRequested': '함께 요청됨',
|
||||
'computerUseApproval.hideWhileWorking': 'Claude가 작업하는 동안 다른 {count}개의 앱이 숨겨집니다.',
|
||||
'computerUseApproval.hideWhileWorkingRestore': 'Claude가 작업하는 동안 다른 {count}개의 앱이 숨겨졌다가 Claude가 완료되면 복원됩니다.',
|
||||
'computerUseApproval.accessibility': '접근성',
|
||||
'computerUseApproval.screenRecording': '화면 기록',
|
||||
'computerUseApproval.granted': '허용됨',
|
||||
'computerUseApproval.notGranted': '허용되지 않음',
|
||||
'computerUseApproval.openAccessibility': '접근성 열기',
|
||||
'computerUseApproval.openScreenRecording': '화면 기록 열기',
|
||||
'computerUseApproval.tryAgain': '다시 시도',
|
||||
'computerUseApproval.tccHint': '시스템 설정에서 부족한 권한을 허용한 후 돌아와 "다시 시도"를 선택하세요.',
|
||||
'computerUseApproval.tryAgainHint': '다시 시도하면 Claude에 제어가 돌아가 macOS 권한 변경이 적용된 후 request_access를 한 번 더 호출할 수 있습니다.',
|
||||
|
||||
// ─── Ask User Question ──────────────────────────────────────
|
||||
'question.needsInput': 'Claude가 입력을 필요로 합니다',
|
||||
|
||||
@@ -1390,6 +1390,14 @@ export const zh: Record<TranslationKey, string> = {
|
||||
'settings.computerUse.title': 'Computer Use',
|
||||
'settings.computerUse.description': '允許 Claude 截圖、點選、打字並控制你的電腦。需要 Python 3,macOS 上還需要輔助功能許可權。',
|
||||
'settings.computerUse.enabledToggle': '啟用',
|
||||
'settings.computerUse.enableRiskTitle': '開啟 Computer Use?',
|
||||
'settings.computerUse.enableRiskSummary': 'Computer Use 會給予 Claude 較廣泛的電腦控制能力。請僅在理解並接受以下風險後開啟。',
|
||||
'settings.computerUse.enableRiskScreen': 'Claude 可以擷取螢幕畫面,並可能看到螢幕上的敏感或隱私資訊。',
|
||||
'settings.computerUse.enableRiskActions': 'Claude 可以點擊、輸入、使用剪貼簿和系統快捷鍵,也可能傳送、修改或刪除內容。',
|
||||
'settings.computerUse.enableRiskAllApps': '開啟後,Claude 可以直接控制所有受支援的應用程式,不再逐一要求 App 權限;產品安全限制仍然生效。',
|
||||
'settings.computerUse.enableRiskStop': '你可以隨時點擊停止或按 Esc 中斷控制,也可以在此關閉 Computer Use。',
|
||||
'settings.computerUse.enableRiskMacos': 'macOS 仍要求系統層級的「輔助使用」和「螢幕錄製」權限;這些系統提示與 App 授權不同。',
|
||||
'settings.computerUse.enableRiskConfirm': '確認開啟',
|
||||
'settings.computerUse.disabledHint': 'Computer Use 已關閉。新會話不會注入 computer-use MCP,也不會把桌面控制工具暴露給 Coding Agent。',
|
||||
'settings.computerUse.notSupported': 'Computer Use 僅支援 macOS 和 Windows。',
|
||||
'settings.computerUse.macosUnsupportedTitle': 'Computer Use 需要 macOS {version} 或更新版本',
|
||||
@@ -2028,27 +2036,6 @@ export const zh: Record<TranslationKey, string> = {
|
||||
'permission.planRejected': '計劃已拒絕',
|
||||
|
||||
// ─── Computer Use Approval ──────────────────────────────────────
|
||||
'computerUseApproval.titleApps': 'Computer Use 想控制這些應用',
|
||||
'computerUseApproval.titleTcc': 'Computer Use 需要 macOS 許可權',
|
||||
'computerUseApproval.reason': '請求原因',
|
||||
'computerUseApproval.allow': '本次會話允許',
|
||||
'computerUseApproval.deny': '拒絕',
|
||||
'computerUseApproval.alreadyGranted': '本次會話已授權',
|
||||
'computerUseApproval.notInstalled': '應用未安裝',
|
||||
'computerUseApproval.unfilteredScreenshots': '在 Windows 上,截圖會包含此顯示器上的所有可見視窗,包括下方未列出的應用。輸入操作仍只限於你允許的應用。',
|
||||
'computerUseApproval.sensitiveApp': '該應用屬於高敏感類別,請額外確認後再授權。',
|
||||
'computerUseApproval.alsoRequested': '同時請求了',
|
||||
'computerUseApproval.hideWhileWorking': 'Claude 工作時會隱藏另外 {count} 個應用。',
|
||||
'computerUseApproval.hideWhileWorkingRestore': 'Claude 工作時會隱藏另外 {count} 個應用,結束後會自動恢復。',
|
||||
'computerUseApproval.accessibility': '輔助功能',
|
||||
'computerUseApproval.screenRecording': '螢幕錄製',
|
||||
'computerUseApproval.granted': '已授權',
|
||||
'computerUseApproval.notGranted': '未授權',
|
||||
'computerUseApproval.openAccessibility': '開啟輔助功能設定',
|
||||
'computerUseApproval.openScreenRecording': '開啟螢幕錄製設定',
|
||||
'computerUseApproval.tryAgain': '稍後重試',
|
||||
'computerUseApproval.tccHint': '先在系統設定裡授予缺失許可權,返回後再點“稍後重試”。',
|
||||
'computerUseApproval.tryAgainHint': '“稍後重試”會把控制權交還給 Claude,讓它在 macOS 許可權生效後重新呼叫 request_access。',
|
||||
|
||||
// ─── Ask User Question ──────────────────────────────────────
|
||||
'question.needsInput': 'Claude 需要你的輸入',
|
||||
|
||||
@@ -1390,6 +1390,14 @@ export const zh: Record<TranslationKey, string> = {
|
||||
'settings.computerUse.title': 'Computer Use',
|
||||
'settings.computerUse.description': '允许 Claude 截屏、点击、打字并控制你的电脑。需要 Python 3,macOS 上还需要辅助功能权限。',
|
||||
'settings.computerUse.enabledToggle': '启用',
|
||||
'settings.computerUse.enableRiskTitle': '开启 Computer Use?',
|
||||
'settings.computerUse.enableRiskSummary': 'Computer Use 会给予 Claude 较广泛的电脑控制能力。请仅在理解并接受以下风险后开启。',
|
||||
'settings.computerUse.enableRiskScreen': 'Claude 可以截取屏幕画面,并可能看到屏幕上的敏感或隐私信息。',
|
||||
'settings.computerUse.enableRiskActions': 'Claude 可以点击、输入、使用剪贴板和系统快捷键,也可能发送、修改或删除内容。',
|
||||
'settings.computerUse.enableRiskAllApps': '开启后,Claude 可以直接控制所有受支持的应用,不再逐个请求 App 权限;产品安全限制仍然生效。',
|
||||
'settings.computerUse.enableRiskStop': '你可以随时点击停止或按 Esc 中断控制,也可以在这里关闭 Computer Use。',
|
||||
'settings.computerUse.enableRiskMacos': 'macOS 仍要求系统级的「辅助功能」和「屏幕录制」权限;这些系统提示与 App 授权不同。',
|
||||
'settings.computerUse.enableRiskConfirm': '确认开启',
|
||||
'settings.computerUse.disabledHint': 'Computer Use 已关闭。新会话不会注入 computer-use MCP,也不会把桌面控制工具暴露给 Coding Agent。',
|
||||
'settings.computerUse.notSupported': 'Computer Use 仅支持 macOS 和 Windows。',
|
||||
'settings.computerUse.macosUnsupportedTitle': 'Computer Use 需要 macOS {version} 或更高版本',
|
||||
@@ -2028,27 +2036,6 @@ export const zh: Record<TranslationKey, string> = {
|
||||
'permission.planRejected': '计划已拒绝',
|
||||
|
||||
// ─── Computer Use Approval ──────────────────────────────────────
|
||||
'computerUseApproval.titleApps': 'Computer Use 想控制这些应用',
|
||||
'computerUseApproval.titleTcc': 'Computer Use 需要 macOS 权限',
|
||||
'computerUseApproval.reason': '请求原因',
|
||||
'computerUseApproval.allow': '本次会话允许',
|
||||
'computerUseApproval.deny': '拒绝',
|
||||
'computerUseApproval.alreadyGranted': '本次会话已授权',
|
||||
'computerUseApproval.notInstalled': '应用未安装',
|
||||
'computerUseApproval.unfilteredScreenshots': '在 Windows 上,截图会包含此显示器上的所有可见窗口,包括下方未列出的应用。输入操作仍只限于你允许的应用。',
|
||||
'computerUseApproval.sensitiveApp': '该应用属于高敏感类别,请额外确认后再授权。',
|
||||
'computerUseApproval.alsoRequested': '同时请求了',
|
||||
'computerUseApproval.hideWhileWorking': 'Claude 工作时会隐藏另外 {count} 个应用。',
|
||||
'computerUseApproval.hideWhileWorkingRestore': 'Claude 工作时会隐藏另外 {count} 个应用,结束后会自动恢复。',
|
||||
'computerUseApproval.accessibility': '辅助功能',
|
||||
'computerUseApproval.screenRecording': '屏幕录制',
|
||||
'computerUseApproval.granted': '已授权',
|
||||
'computerUseApproval.notGranted': '未授权',
|
||||
'computerUseApproval.openAccessibility': '打开辅助功能设置',
|
||||
'computerUseApproval.openScreenRecording': '打开屏幕录制设置',
|
||||
'computerUseApproval.tryAgain': '稍后重试',
|
||||
'computerUseApproval.tccHint': '先在系统设置里授予缺失权限,返回后再点“稍后重试”。',
|
||||
'computerUseApproval.tryAgainHint': '“稍后重试”会把控制权交还给 Claude,让它在 macOS 权限生效后重新调用 request_access。',
|
||||
|
||||
// ─── Ask User Question ──────────────────────────────────────
|
||||
'question.needsInput': 'Claude 需要你的输入',
|
||||
|
||||
@@ -34,7 +34,6 @@ import {
|
||||
type SessionHeaderMetaItem,
|
||||
} from '@/components/chat/SessionChatSurface'
|
||||
import { getWorktreeDisplayName, WorktreeDetails } from '../components/chat/WorktreeDetails'
|
||||
import { ComputerUsePermissionModal } from '../components/chat/ComputerUsePermissionModal'
|
||||
import { WorkbenchPanel } from '../components/workbench/WorkbenchPanel'
|
||||
import { AgentTeamsStrip } from '../components/agentTeams/AgentTeamsSummary'
|
||||
import { snapshotWithHistoricalMembers } from '../components/agentTeams/agentTeamsModel'
|
||||
@@ -321,7 +320,6 @@ export function ActiveSession() {
|
||||
const connectToSession = useChatStore((s) => s.connectToSession)
|
||||
const stopBackgroundTask = useChatStore((s) => s.stopBackgroundTask)
|
||||
const sessionState = useChatStore((s) => activeTabId ? s.sessions[activeTabId] : undefined)
|
||||
const pendingComputerUsePermission = sessionState?.pendingComputerUsePermission ?? null
|
||||
const fetchSessionTasks = useCLITaskStore((s) => s.fetchSessionTasks)
|
||||
const trackedTaskSessionId = useCLITaskStore((s) => s.sessionId)
|
||||
const cliTasks = useCLITaskStore((s) => s.tasks)
|
||||
@@ -769,12 +767,6 @@ export function ActiveSession() {
|
||||
</aside>
|
||||
</>
|
||||
) : null}
|
||||
overlay={(
|
||||
<ComputerUsePermissionModal
|
||||
sessionId={activeTabId}
|
||||
request={pendingComputerUsePermission?.request ?? null}
|
||||
/>
|
||||
)}
|
||||
>
|
||||
{isEmpty ? (
|
||||
<div
|
||||
|
||||
@@ -129,6 +129,53 @@ describe('ComputerUseSettings', () => {
|
||||
})
|
||||
})
|
||||
|
||||
it('requires explicit risk confirmation before enabling all supported apps', async () => {
|
||||
computerUseApiMock.getAuthorizedApps.mockResolvedValue({
|
||||
...enabledConfig,
|
||||
enabled: false,
|
||||
})
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
|
||||
const toggle = await screen.findByLabelText('Enabled')
|
||||
fireEvent.click(toggle)
|
||||
|
||||
expect(toggle).not.toBeChecked()
|
||||
expect(computerUseApiMock.setAuthorizedApps).not.toHaveBeenCalled()
|
||||
expect(screen.getByRole('dialog')).toHaveTextContent('all supported applications')
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Enable Computer Use' }))
|
||||
|
||||
await waitFor(() => {
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({
|
||||
enabled: true,
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
})
|
||||
})
|
||||
expect(toggle).toBeChecked()
|
||||
})
|
||||
|
||||
it('keeps Computer Use disabled when the risk dialog is cancelled', async () => {
|
||||
computerUseApiMock.getAuthorizedApps.mockResolvedValue({
|
||||
...enabledConfig,
|
||||
enabled: false,
|
||||
})
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
|
||||
const toggle = await screen.findByLabelText('Enabled')
|
||||
fireEvent.click(toggle)
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Cancel' }))
|
||||
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument()
|
||||
expect(toggle).not.toBeChecked()
|
||||
expect(computerUseApiMock.setAuthorizedApps).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('saves a custom Python interpreter path and rechecks status', async () => {
|
||||
render(<ComputerUseSettings />)
|
||||
|
||||
@@ -205,116 +252,6 @@ describe('ComputerUseSettings', () => {
|
||||
expect(computerUseApiMock.setAuthorizedApps).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
it('keeps the user-selected enablement when a stale refresh resolves later', async () => {
|
||||
const staleRefresh = deferred<typeof enabledConfig>()
|
||||
computerUseApiMock.getStatus.mockResolvedValue({
|
||||
...readyStatus,
|
||||
venv: {
|
||||
...readyStatus.venv,
|
||||
created: true,
|
||||
},
|
||||
dependencies: {
|
||||
...readyStatus.dependencies,
|
||||
installed: true,
|
||||
},
|
||||
})
|
||||
computerUseApiMock.getInstalledApps.mockResolvedValue({ apps: [] })
|
||||
computerUseApiMock.getAuthorizedApps
|
||||
.mockResolvedValueOnce({
|
||||
...enabledConfig,
|
||||
enabled: false,
|
||||
})
|
||||
.mockReturnValueOnce(staleRefresh.promise)
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
|
||||
const toggle = await screen.findByLabelText('Enabled')
|
||||
await waitFor(() => expect(toggle).not.toBeChecked())
|
||||
await waitFor(() => expect(computerUseApiMock.getInstalledApps).toHaveBeenCalled())
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(toggle)
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
expect(toggle).toBeChecked()
|
||||
|
||||
await act(async () => {
|
||||
staleRefresh.resolve({
|
||||
...enabledConfig,
|
||||
enabled: false,
|
||||
})
|
||||
await staleRefresh.promise
|
||||
})
|
||||
|
||||
expect(toggle).toBeChecked()
|
||||
})
|
||||
|
||||
it('saves app and grant flag changes from the ready environment view', async () => {
|
||||
computerUseApiMock.getStatus.mockResolvedValue({
|
||||
...readyStatus,
|
||||
venv: {
|
||||
...readyStatus.venv,
|
||||
created: true,
|
||||
},
|
||||
dependencies: {
|
||||
...readyStatus.dependencies,
|
||||
installed: true,
|
||||
},
|
||||
})
|
||||
computerUseApiMock.getInstalledApps.mockResolvedValue({
|
||||
apps: [
|
||||
{
|
||||
bundleId: 'com.example.Preview',
|
||||
displayName: 'Preview',
|
||||
path: '/Applications/Preview.app',
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
|
||||
await screen.findByText('Preview')
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(screen.getByText('Preview'))
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({
|
||||
authorizedApps: [
|
||||
expect.objectContaining({
|
||||
bundleId: 'com.example.Preview',
|
||||
displayName: 'Preview',
|
||||
}),
|
||||
],
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
})
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(screen.getByLabelText('Clipboard Access'))
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({
|
||||
authorizedApps: [
|
||||
expect.objectContaining({
|
||||
bundleId: 'com.example.Preview',
|
||||
displayName: 'Preview',
|
||||
}),
|
||||
],
|
||||
grantFlags: {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
describe('native cu-helper branch', () => {
|
||||
const nativeStatus: ComputerUseStatus = {
|
||||
...readyStatus,
|
||||
@@ -479,6 +416,7 @@ describe('ComputerUseSettings', () => {
|
||||
expect(toggle).not.toBeChecked()
|
||||
|
||||
fireEvent.click(toggle)
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Enable Computer Use' }))
|
||||
|
||||
expect(
|
||||
await screen.findByText(
|
||||
@@ -555,72 +493,6 @@ describe('ComputerUseSettings', () => {
|
||||
expect(screen.getByRole('heading', { name: 'Computer Control' })).toBeInTheDocument()
|
||||
})
|
||||
|
||||
/**
|
||||
* Rows show the application's own icon, served per bundle id. The letter
|
||||
* tile is the fallback for bundles that ship no icon, so it must appear on
|
||||
* image error and NOT before — a page that renders letters while perfectly
|
||||
* good icons exist looks broken.
|
||||
*/
|
||||
describe('app icons', () => {
|
||||
const authorizedConfig = {
|
||||
...enabledConfig,
|
||||
authorizedApps: [
|
||||
{
|
||||
bundleId: 'com.example.Preview',
|
||||
displayName: 'Preview',
|
||||
authorizedAt: '2026-01-01T00:00:00.000Z',
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
it('renders the icon it loaded for the row bundle id', async () => {
|
||||
computerUseApiMock.getStatus.mockResolvedValue(nativeStatus)
|
||||
computerUseApiMock.getAuthorizedApps.mockResolvedValue(authorizedConfig)
|
||||
computerUseApiMock.loadAppIcon.mockResolvedValue('blob:icon-preview')
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
await screen.findByText('Preview')
|
||||
|
||||
await waitFor(() => {
|
||||
expect(document.querySelector('img[src="blob:icon-preview"]')).not.toBeNull()
|
||||
})
|
||||
expect(computerUseApiMock.loadAppIcon).toHaveBeenCalledWith('com.example.Preview')
|
||||
// The letter tile is the fallback, so it must be gone once the icon
|
||||
// arrives — otherwise both would render.
|
||||
expect(screen.queryByText('P')).toBeNull()
|
||||
})
|
||||
|
||||
it('keeps the letter tile when the bundle has no icon', async () => {
|
||||
computerUseApiMock.getStatus.mockResolvedValue(nativeStatus)
|
||||
computerUseApiMock.getAuthorizedApps.mockResolvedValue(authorizedConfig)
|
||||
// null is the ordinary "this bundle ships no icon" answer.
|
||||
computerUseApiMock.loadAppIcon.mockResolvedValue(null)
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
await screen.findByText('Preview')
|
||||
|
||||
await waitFor(() => expect(screen.getByText('P')).toBeInTheDocument())
|
||||
expect(document.querySelector('img')).toBeNull()
|
||||
})
|
||||
|
||||
it('never points an img straight at the endpoint', async () => {
|
||||
// The packaged renderer is a file:// page, so a cross-origin <img> to
|
||||
// /api/... is refused by the server and silently shows nothing. Icons
|
||||
// must arrive as blob URLs through the authenticated channel.
|
||||
computerUseApiMock.getStatus.mockResolvedValue(nativeStatus)
|
||||
computerUseApiMock.getAuthorizedApps.mockResolvedValue(authorizedConfig)
|
||||
computerUseApiMock.loadAppIcon.mockResolvedValue('blob:icon-preview')
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
await screen.findByText('Preview')
|
||||
await waitFor(() => {
|
||||
expect(document.querySelector('img[src="blob:icon-preview"]')).not.toBeNull()
|
||||
})
|
||||
|
||||
expect(document.querySelector('img[src*="/api/"]')).toBeNull()
|
||||
})
|
||||
})
|
||||
|
||||
it('pops the native permission card when enabling Computer Use with missing permissions', async () => {
|
||||
computerUseApiMock.getStatus.mockResolvedValue(nativeStatus)
|
||||
computerUseApiMock.getAuthorizedApps.mockResolvedValue({
|
||||
@@ -635,10 +507,18 @@ describe('ComputerUseSettings', () => {
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(toggle)
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Enable Computer Use' }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({ enabled: true })
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({
|
||||
enabled: true,
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
})
|
||||
expect(computerUseApiMock.openPermissionCard).toHaveBeenCalledTimes(1)
|
||||
})
|
||||
|
||||
@@ -659,10 +539,18 @@ describe('ComputerUseSettings', () => {
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(toggle)
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Enable Computer Use' }))
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({ enabled: true })
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({
|
||||
enabled: true,
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
})
|
||||
expect(computerUseApiMock.openPermissionCard).not.toHaveBeenCalled()
|
||||
})
|
||||
|
||||
@@ -696,7 +584,7 @@ describe('ComputerUseSettings', () => {
|
||||
).toBeInTheDocument()
|
||||
})
|
||||
|
||||
it('removes an always-allowed app via the trash button', async () => {
|
||||
it('does not render an app-by-app authorization list', async () => {
|
||||
computerUseApiMock.getStatus.mockResolvedValue(nativeStatus)
|
||||
computerUseApiMock.getAuthorizedApps.mockResolvedValue({
|
||||
...enabledConfig,
|
||||
@@ -711,66 +599,10 @@ describe('ComputerUseSettings', () => {
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
|
||||
const remove = await screen.findByLabelText('Remove Preview')
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(remove)
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({
|
||||
authorizedApps: [],
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
it('adds an app from the picker fed by getInstalledApps', async () => {
|
||||
computerUseApiMock.getStatus.mockResolvedValue(nativeStatus)
|
||||
computerUseApiMock.getInstalledApps.mockResolvedValue({
|
||||
apps: [
|
||||
{
|
||||
bundleId: 'com.example.Notes',
|
||||
displayName: 'Notes',
|
||||
path: '/Applications/Notes.app',
|
||||
},
|
||||
],
|
||||
})
|
||||
|
||||
render(<ComputerUseSettings />)
|
||||
|
||||
const addButton = await screen.findByText('Add App')
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(addButton)
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
await waitFor(() => expect(computerUseApiMock.getInstalledApps).toHaveBeenCalled())
|
||||
|
||||
const notesEntry = await screen.findByText('Notes')
|
||||
|
||||
await act(async () => {
|
||||
fireEvent.click(notesEntry)
|
||||
await Promise.resolve()
|
||||
})
|
||||
|
||||
expect(computerUseApiMock.setAuthorizedApps).toHaveBeenCalledWith({
|
||||
authorizedApps: [
|
||||
expect.objectContaining({
|
||||
bundleId: 'com.example.Notes',
|
||||
displayName: 'Notes',
|
||||
}),
|
||||
],
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
})
|
||||
await screen.findByRole('heading', { name: 'Computer Control' })
|
||||
expect(screen.queryByText('Preview')).not.toBeInTheDocument()
|
||||
expect(screen.queryByText('Add App')).not.toBeInTheDocument()
|
||||
expect(computerUseApiMock.getInstalledApps).not.toHaveBeenCalled()
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { useState, useEffect, useCallback, useMemo, useRef } from 'react'
|
||||
import { computerUseApi, type ComputerUseStatus, type SetupResult, type InstalledApp, type AuthorizedApp } from '../api/computerUse'
|
||||
import { useState, useEffect, useCallback, useRef } from 'react'
|
||||
import { computerUseApi, type ComputerUseStatus, type SetupResult } from '../api/computerUse'
|
||||
import { useTranslation } from '../i18n'
|
||||
import { ComputerUseEnableDialog } from '@/components/computer-use/ComputerUseEnableDialog'
|
||||
import { Button } from '@/components/ui/Button'
|
||||
import { ErrorState } from '@/components/ui/ErrorState'
|
||||
import { LoadingState } from '@/components/ui/LoadingState'
|
||||
import { Modal } from '@/components/ui/Modal'
|
||||
import { Switch } from '@/components/ui/Switch'
|
||||
import { getDesktopHost } from '../lib/desktopHost'
|
||||
|
||||
@@ -59,22 +59,14 @@ export function ComputerUseSettings() {
|
||||
const [setupRunning, setSetupRunning] = useState(false)
|
||||
const [setupResult, setSetupResult] = useState<SetupResult | null>(null)
|
||||
|
||||
// App authorization state
|
||||
const [installedApps, setInstalledApps] = useState<InstalledApp[]>([])
|
||||
const [authorizedBundleIds, setAuthorizedBundleIds] = useState<Set<string>>(new Set())
|
||||
const [authorizedApps, setAuthorizedApps] = useState<AuthorizedApp[]>([])
|
||||
const [appsLoading, setAppsLoading] = useState(false)
|
||||
const [appsSaved, setAppsSaved] = useState(false)
|
||||
const [searchQuery, setSearchQuery] = useState('')
|
||||
const [computerUseEnabled, setComputerUseEnabled] = useState(true)
|
||||
const [clipboardAccess, setClipboardAccess] = useState(true)
|
||||
const [systemKeys, setSystemKeys] = useState(true)
|
||||
const [computerUseEnabled, setComputerUseEnabled] = useState(false)
|
||||
const [enableConfirmOpen, setEnableConfirmOpen] = useState(false)
|
||||
const [enableSaving, setEnableSaving] = useState(false)
|
||||
const [pythonPathDraft, setPythonPathDraft] = useState('')
|
||||
const [pythonPathSaved, setPythonPathSaved] = useState('')
|
||||
const [pythonPathSaving, setPythonPathSaving] = useState(false)
|
||||
const [pythonPathMessage, setPythonPathMessage] = useState<string | null>(null)
|
||||
// Native (cu-helper) Codex-style UI state
|
||||
const [pickerOpen, setPickerOpen] = useState(false)
|
||||
const [cardOpening, setCardOpening] = useState(false)
|
||||
const [cardError, setCardError] = useState<string | null>(null)
|
||||
const configMutationSeqRef = useRef(0)
|
||||
@@ -100,10 +92,6 @@ export function ComputerUseSettings() {
|
||||
) => {
|
||||
if (requestSeq !== configMutationSeqRef.current) return
|
||||
setComputerUseEnabled(configResult.enabled)
|
||||
setAuthorizedApps(configResult.authorizedApps)
|
||||
setAuthorizedBundleIds(new Set(configResult.authorizedApps.map(a => a.bundleId)))
|
||||
setClipboardAccess(configResult.grantFlags.clipboardRead)
|
||||
setSystemKeys(configResult.grantFlags.systemKeyCombos)
|
||||
setPythonPathDraft(configResult.pythonPath ?? '')
|
||||
setPythonPathSaved(configResult.pythonPath ?? '')
|
||||
}, [])
|
||||
@@ -122,33 +110,12 @@ export function ComputerUseSettings() {
|
||||
}
|
||||
}, [applyConfig])
|
||||
|
||||
const fetchApps = useCallback(async () => {
|
||||
const requestSeq = configMutationSeqRef.current
|
||||
setAppsLoading(true)
|
||||
try {
|
||||
const [appsResult, configResult] = await Promise.all([
|
||||
computerUseApi.getInstalledApps(),
|
||||
computerUseApi.getAuthorizedApps(),
|
||||
])
|
||||
setInstalledApps(appsResult.apps)
|
||||
applyConfig(configResult, requestSeq)
|
||||
} catch {
|
||||
// API not ready
|
||||
} finally {
|
||||
setAppsLoading(false)
|
||||
}
|
||||
}, [applyConfig])
|
||||
|
||||
useEffect(() => {
|
||||
fetchStatus()
|
||||
fetchConfig()
|
||||
}, [fetchStatus, fetchConfig])
|
||||
|
||||
// Load apps when environment is ready
|
||||
const envReady = status?.venv.created && status?.dependencies.installed
|
||||
useEffect(() => {
|
||||
if (envReady) fetchApps()
|
||||
}, [envReady, fetchApps])
|
||||
|
||||
const handleSetup = async () => {
|
||||
setSetupRunning(true)
|
||||
@@ -157,7 +124,6 @@ export function ComputerUseSettings() {
|
||||
const result = await computerUseApi.runSetup()
|
||||
setSetupResult(result)
|
||||
await fetchStatus()
|
||||
if (result.success) await fetchApps()
|
||||
} catch {
|
||||
setSetupResult({ success: false, steps: [{ name: 'error', ok: false, message: 'Request failed' }] })
|
||||
} finally {
|
||||
@@ -165,59 +131,23 @@ export function ComputerUseSettings() {
|
||||
}
|
||||
}
|
||||
|
||||
const toggleApp = (app: InstalledApp) => {
|
||||
configMutationSeqRef.current += 1
|
||||
const newSet = new Set(authorizedBundleIds)
|
||||
let newAuthorized = [...authorizedApps]
|
||||
if (newSet.has(app.bundleId)) {
|
||||
newSet.delete(app.bundleId)
|
||||
newAuthorized = newAuthorized.filter(a => a.bundleId !== app.bundleId)
|
||||
} else {
|
||||
newSet.add(app.bundleId)
|
||||
newAuthorized.push({
|
||||
bundleId: app.bundleId,
|
||||
displayName: app.displayName,
|
||||
authorizedAt: new Date().toISOString(),
|
||||
})
|
||||
}
|
||||
setAuthorizedBundleIds(newSet)
|
||||
setAuthorizedApps(newAuthorized)
|
||||
|
||||
// Auto-save
|
||||
computerUseApi.setAuthorizedApps({
|
||||
authorizedApps: newAuthorized,
|
||||
grantFlags: { clipboardRead: clipboardAccess, clipboardWrite: clipboardAccess, systemKeyCombos: systemKeys },
|
||||
}).then(() => {
|
||||
setAppsSaved(true)
|
||||
setTimeout(() => setAppsSaved(false), 1500)
|
||||
})
|
||||
}
|
||||
|
||||
const toggleFlag = (flag: 'clipboard' | 'systemKeys', value: boolean) => {
|
||||
configMutationSeqRef.current += 1
|
||||
if (flag === 'clipboard') setClipboardAccess(value)
|
||||
else setSystemKeys(value)
|
||||
|
||||
computerUseApi.setAuthorizedApps({
|
||||
authorizedApps,
|
||||
grantFlags: {
|
||||
clipboardRead: flag === 'clipboard' ? value : clipboardAccess,
|
||||
clipboardWrite: flag === 'clipboard' ? value : clipboardAccess,
|
||||
systemKeyCombos: flag === 'systemKeys' ? value : systemKeys,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const toggleComputerUseEnabled = async (value: boolean): Promise<boolean> => {
|
||||
const requestSeq = ++configMutationSeqRef.current
|
||||
const previous = computerUseEnabled
|
||||
setConfigError(null)
|
||||
setComputerUseEnabled(value)
|
||||
try {
|
||||
await computerUseApi.setAuthorizedApps({ enabled: value })
|
||||
await computerUseApi.setAuthorizedApps(value
|
||||
? {
|
||||
enabled: true,
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
}
|
||||
: { enabled: false })
|
||||
if (requestSeq !== configMutationSeqRef.current) return true
|
||||
setAppsSaved(true)
|
||||
setTimeout(() => setAppsSaved(false), 1500)
|
||||
return true
|
||||
} catch {
|
||||
if (requestSeq === configMutationSeqRef.current) {
|
||||
@@ -248,66 +178,29 @@ export function ComputerUseSettings() {
|
||||
}
|
||||
}, [t, fetchStatus])
|
||||
|
||||
// Master Computer Use toggle on the native path. Mirrors toggleComputerUseEnabled
|
||||
// for persistence, but additionally pops the native OS-permission card when
|
||||
// turning ON while macOS permissions are still missing (the headline flow).
|
||||
const toggleAnyApp = (value: boolean) => {
|
||||
void (async () => {
|
||||
const saved = await toggleComputerUseEnabled(value)
|
||||
const requestComputerUseEnabled = (value: boolean) => {
|
||||
if (value) {
|
||||
setEnableConfirmOpen(true)
|
||||
return
|
||||
}
|
||||
void toggleComputerUseEnabled(false)
|
||||
}
|
||||
|
||||
const confirmComputerUseEnabled = async () => {
|
||||
setEnableSaving(true)
|
||||
const saved = await toggleComputerUseEnabled(true)
|
||||
setEnableSaving(false)
|
||||
if (!saved) return
|
||||
setEnableConfirmOpen(false)
|
||||
if (
|
||||
saved &&
|
||||
value &&
|
||||
(status?.permissions.accessibility === false ||
|
||||
status?.permissions.screenRecording === false)
|
||||
status?.engine === 'macos-native'
|
||||
&& (status.permissions.accessibility === false
|
||||
|| status.permissions.screenRecording === false)
|
||||
) {
|
||||
await openPermissionCard()
|
||||
}
|
||||
})()
|
||||
}
|
||||
|
||||
// Persist an authorized-apps list change (native add/remove). Reuses the same
|
||||
// setAuthorizedApps shape + saved-flash + stale-guard discipline as toggleApp.
|
||||
const persistAuthorizedApps = useCallback((next: AuthorizedApp[]) => {
|
||||
configMutationSeqRef.current += 1
|
||||
setAuthorizedApps(next)
|
||||
setAuthorizedBundleIds(new Set(next.map(a => a.bundleId)))
|
||||
computerUseApi.setAuthorizedApps({
|
||||
authorizedApps: next,
|
||||
grantFlags: { clipboardRead: clipboardAccess, clipboardWrite: clipboardAccess, systemKeyCombos: systemKeys },
|
||||
}).then(() => {
|
||||
setAppsSaved(true)
|
||||
setTimeout(() => setAppsSaved(false), 1500)
|
||||
})
|
||||
}, [clipboardAccess, systemKeys])
|
||||
|
||||
const removeAuthorizedApp = (bundleId: string) => {
|
||||
persistAuthorizedApps(authorizedApps.filter(a => a.bundleId !== bundleId))
|
||||
}
|
||||
|
||||
const addAuthorizedApp = (app: InstalledApp) => {
|
||||
if (authorizedBundleIds.has(app.bundleId)) {
|
||||
setPickerOpen(false)
|
||||
return
|
||||
}
|
||||
persistAuthorizedApps([
|
||||
...authorizedApps,
|
||||
{
|
||||
bundleId: app.bundleId,
|
||||
displayName: app.displayName,
|
||||
authorizedAt: new Date().toISOString(),
|
||||
},
|
||||
])
|
||||
setPickerOpen(false)
|
||||
}
|
||||
|
||||
// Lazy-load installed apps the first time the picker opens (the native path
|
||||
// has no Python env-ready gate, so fetchApps' envReady effect never fires).
|
||||
const openPicker = useCallback(() => {
|
||||
setSearchQuery('')
|
||||
setPickerOpen(true)
|
||||
if (installedApps.length === 0) void fetchApps()
|
||||
}, [installedApps.length, fetchApps])
|
||||
|
||||
const savePythonPath = async (value = pythonPathDraft) => {
|
||||
configMutationSeqRef.current += 1
|
||||
const normalized = value.trim()
|
||||
@@ -367,35 +260,19 @@ export function ComputerUseSettings() {
|
||||
? `${t('settings.computerUse.pythonCustomInvalid')} — ${status.python.path}${status.python.error ? `: ${status.python.error}` : ''}`
|
||||
: t('settings.computerUse.pythonNotFound')
|
||||
|
||||
// Filter apps by search query
|
||||
const filteredApps = useMemo(() => {
|
||||
if (!searchQuery) return installedApps
|
||||
const q = searchQuery.toLowerCase()
|
||||
return installedApps.filter(
|
||||
a => a.displayName.toLowerCase().includes(q) || a.bundleId.toLowerCase().includes(q)
|
||||
)
|
||||
}, [installedApps, searchQuery])
|
||||
|
||||
// Sort: authorized apps first, then alphabetical
|
||||
const sortedApps = useMemo(() => {
|
||||
return [...filteredApps].sort((a, b) => {
|
||||
const aAuth = authorizedBundleIds.has(a.bundleId) ? 0 : 1
|
||||
const bAuth = authorizedBundleIds.has(b.bundleId) ? 0 : 1
|
||||
if (aAuth !== bAuth) return aAuth - bAuth
|
||||
return a.displayName.localeCompare(b.displayName)
|
||||
})
|
||||
}, [filteredApps, authorizedBundleIds])
|
||||
|
||||
// Native (cu-helper) path: drop the entire Python setup flow in favor of the
|
||||
// Codex-style page. Branch ONLY when on macOS AND the Swift helper resolves.
|
||||
const native = status?.engine === 'macos-native'
|
||||
|
||||
// Picker list (native "+ 添加应用"): installed apps not yet authorized, sorted.
|
||||
const pickerApps = useMemo(() => {
|
||||
return [...filteredApps]
|
||||
.filter(a => !authorizedBundleIds.has(a.bundleId))
|
||||
.sort((a, b) => a.displayName.localeCompare(b.displayName))
|
||||
}, [filteredApps, authorizedBundleIds])
|
||||
const enableDialog = (
|
||||
<ComputerUseEnableDialog
|
||||
open={enableConfirmOpen}
|
||||
loading={enableSaving}
|
||||
platform={status?.platform === 'darwin' ? 'darwin' : 'win32'}
|
||||
onClose={() => setEnableConfirmOpen(false)}
|
||||
onConfirm={confirmComputerUseEnabled}
|
||||
/>
|
||||
)
|
||||
|
||||
// The renderer cannot choose between the native macOS page and the
|
||||
// compatibility page until the capability probe finishes. Rendering the
|
||||
@@ -477,29 +354,21 @@ export function ComputerUseSettings() {
|
||||
|
||||
if (native && status) {
|
||||
return (
|
||||
<>
|
||||
<NativeComputerUse
|
||||
t={t}
|
||||
status={status}
|
||||
enabled={computerUseEnabled}
|
||||
onToggleEnabled={toggleAnyApp}
|
||||
authorizedApps={authorizedApps}
|
||||
onRemoveApp={removeAuthorizedApp}
|
||||
appsSaved={appsSaved}
|
||||
onToggleEnabled={requestComputerUseEnabled}
|
||||
configError={configError}
|
||||
statusError={checkState === 'error'}
|
||||
cardOpening={cardOpening}
|
||||
cardError={cardError}
|
||||
onOpenCard={openPermissionCard}
|
||||
onRecheck={fetchStatus}
|
||||
pickerOpen={pickerOpen}
|
||||
onOpenPicker={openPicker}
|
||||
onClosePicker={() => setPickerOpen(false)}
|
||||
onAddApp={addAuthorizedApp}
|
||||
appsLoading={appsLoading}
|
||||
pickerApps={pickerApps}
|
||||
searchQuery={searchQuery}
|
||||
onSearch={setSearchQuery}
|
||||
/>
|
||||
{enableDialog}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -522,6 +391,7 @@ export function ComputerUseSettings() {
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="max-w-2xl space-y-6">
|
||||
{/* Title */}
|
||||
<div>
|
||||
@@ -531,7 +401,7 @@ export function ComputerUseSettings() {
|
||||
</h2>
|
||||
<Switch
|
||||
checked={computerUseEnabled}
|
||||
onChange={value => { void toggleComputerUseEnabled(value) }}
|
||||
onChange={requestComputerUseEnabled}
|
||||
label={t('settings.computerUse.enabledToggle')}
|
||||
size="sm"
|
||||
/>
|
||||
@@ -746,106 +616,11 @@ export function ComputerUseSettings() {
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{/* ─── App Authorization Section ─── */}
|
||||
{envReady && (
|
||||
<div className="space-y-4 pt-4 border-t border-[var(--color-border)]">
|
||||
<div>
|
||||
<h3 className="text-base font-semibold text-[var(--color-text-primary)] flex items-center gap-2" style={{ fontFamily: 'var(--font-headline)' }}>
|
||||
{t('settings.computerUse.appsTitle')}
|
||||
{appsSaved && (
|
||||
<span className="text-xs font-normal text-[var(--color-success)] flex items-center gap-1">
|
||||
<span className="material-symbols-outlined text-[14px]" style={{ fontVariationSettings: "'FILL' 1" }}>check</span>
|
||||
{t('settings.computerUse.appsSaved')}
|
||||
</span>
|
||||
)}
|
||||
</h3>
|
||||
<p className="mt-1 text-sm text-[var(--color-text-secondary)]">
|
||||
{t('settings.computerUse.appsDescription')}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Grant flags */}
|
||||
<div className="flex gap-4">
|
||||
<label className="flex items-center gap-2 text-sm text-[var(--color-text-secondary)] cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={clipboardAccess}
|
||||
onChange={e => toggleFlag('clipboard', e.target.checked)}
|
||||
className="rounded border-[var(--color-border)] accent-[var(--color-brand)]"
|
||||
/>
|
||||
{t('settings.computerUse.flagClipboard')}
|
||||
</label>
|
||||
<label className="flex items-center gap-2 text-sm text-[var(--color-text-secondary)] cursor-pointer">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={systemKeys}
|
||||
onChange={e => toggleFlag('systemKeys', e.target.checked)}
|
||||
className="rounded border-[var(--color-border)] accent-[var(--color-brand)]"
|
||||
/>
|
||||
{t('settings.computerUse.flagSystemKeys')}
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{/* Search */}
|
||||
<div className="relative">
|
||||
<span className="material-symbols-outlined text-[18px] text-[var(--color-text-tertiary)] absolute left-3 top-1/2 -translate-y-1/2">search</span>
|
||||
<input
|
||||
type="text"
|
||||
value={searchQuery}
|
||||
onChange={e => setSearchQuery(e.target.value)}
|
||||
placeholder={t('settings.computerUse.appsSearch')}
|
||||
className="w-full pl-9 pr-4 py-2 text-sm bg-[var(--color-surface-container-low)] border border-[var(--color-border)] rounded-[var(--radius-lg)] text-[var(--color-text-primary)] placeholder:text-[var(--color-text-tertiary)] focus:outline-none focus:border-[var(--color-brand)]"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* App list */}
|
||||
{appsLoading ? (
|
||||
<div className="py-6 text-center text-sm text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.appsLoading')}
|
||||
</div>
|
||||
) : installedApps.length === 0 ? (
|
||||
<div className="py-6 text-center text-sm text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.appsEmpty')}
|
||||
</div>
|
||||
) : (
|
||||
<div className="max-h-[400px] overflow-y-auto rounded-[var(--radius-lg)] border border-[var(--color-border)]">
|
||||
{sortedApps.map(app => {
|
||||
const isAuthorized = authorizedBundleIds.has(app.bundleId)
|
||||
return (
|
||||
<button
|
||||
key={app.bundleId}
|
||||
onClick={() => toggleApp(app)}
|
||||
className={`w-full flex items-center gap-3 px-4 py-2.5 text-left transition-colors hover:bg-[var(--color-surface-hover)] border-b border-[var(--color-border)] last:border-b-0 ${
|
||||
isAuthorized ? 'bg-[var(--color-brand-soft)]' : ''
|
||||
}`}
|
||||
>
|
||||
<div className={`w-5 h-5 rounded flex items-center justify-center flex-shrink-0 border ${
|
||||
isAuthorized
|
||||
? 'bg-[var(--color-brand)] border-[var(--color-brand)]'
|
||||
: 'border-[var(--color-border)]'
|
||||
}`}>
|
||||
{isAuthorized && (
|
||||
<span className="material-symbols-outlined text-[14px] text-[var(--color-on-primary)]" style={{ fontVariationSettings: "'FILL' 1" }}>check</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="flex-1 min-w-0">
|
||||
<div className="text-sm font-medium text-[var(--color-text-primary)] truncate">
|
||||
{app.displayName}
|
||||
</div>
|
||||
<div className="text-[11px] text-[var(--color-text-tertiary)] truncate font-mono">
|
||||
{app.bundleId}
|
||||
</div>
|
||||
</div>
|
||||
</button>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
) : null}
|
||||
</div>
|
||||
{enableDialog}
|
||||
</>
|
||||
)
|
||||
}
|
||||
|
||||
@@ -855,65 +630,6 @@ export function ComputerUseSettings() {
|
||||
|
||||
type Translate = ReturnType<typeof useTranslation>
|
||||
|
||||
/**
|
||||
* An app's real icon, falling back to a letter tile.
|
||||
*
|
||||
* The `/apps` payload deliberately carries no icon bytes — it lists every
|
||||
* installed application, and inlining hundreds of PNGs would bloat one
|
||||
* response. Each row instead points an `<img>` at the icon endpoint, which
|
||||
* reads the bundle's own `.icns` the same way Finder does. `loading="lazy"`
|
||||
* matters here: the picker is a long scroller, and without it every row off
|
||||
* screen would still cost a request and a rasterisation.
|
||||
*
|
||||
* A bundle with no icon 404s, which is an ordinary outcome rather than a
|
||||
* failure — that is what the letter tile is for.
|
||||
*/
|
||||
function AppIcon({ name, bundleId }: { name: string; bundleId?: string }) {
|
||||
const [iconUrl, setIconUrl] = useState<string | null>(null)
|
||||
|
||||
useEffect(() => {
|
||||
if (!bundleId) {
|
||||
setIconUrl(null)
|
||||
return
|
||||
}
|
||||
let cancelled = false
|
||||
setIconUrl(null)
|
||||
void computerUseApi.loadAppIcon(bundleId).then(url => {
|
||||
if (!cancelled) setIconUrl(url)
|
||||
})
|
||||
return () => {
|
||||
cancelled = true
|
||||
}
|
||||
}, [bundleId])
|
||||
|
||||
const tileClass =
|
||||
'flex h-9 w-9 flex-shrink-0 items-center justify-center rounded-[10px] border border-[var(--color-border)] bg-[var(--color-surface-container-high)] shadow-[inset_0_1px_0_rgba(255,255,255,0.04)]'
|
||||
|
||||
if (iconUrl) {
|
||||
return (
|
||||
<div className={tileClass}>
|
||||
<img
|
||||
src={iconUrl}
|
||||
alt=""
|
||||
aria-hidden="true"
|
||||
draggable={false}
|
||||
className="block h-7 w-7 object-contain"
|
||||
/>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
// Shown both while the icon is in flight and when the bundle has none. The
|
||||
// letter is a stable placeholder rather than a spinner, so a list of
|
||||
// iconless utilities does not read as permanently loading.
|
||||
const letter = name.trim().charAt(0).toUpperCase() || '?'
|
||||
return (
|
||||
<div className={`${tileClass} text-[13px] font-semibold text-[var(--color-text-secondary)]`}>
|
||||
{letter}
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
/** macOS OS-permission status row (辅助功能 / 屏幕录制): a refined row with a
|
||||
* status dot (granted=emerald, needed=amber, checking=neutral) + label + state,
|
||||
* built to live inside a divide-y group rather than as a standalone boxy card. */
|
||||
@@ -974,45 +690,23 @@ function NativeComputerUse({
|
||||
status,
|
||||
enabled,
|
||||
onToggleEnabled,
|
||||
authorizedApps,
|
||||
onRemoveApp,
|
||||
appsSaved,
|
||||
configError,
|
||||
statusError,
|
||||
cardOpening,
|
||||
cardError,
|
||||
onOpenCard,
|
||||
onRecheck,
|
||||
pickerOpen,
|
||||
onOpenPicker,
|
||||
onClosePicker,
|
||||
onAddApp,
|
||||
appsLoading,
|
||||
pickerApps,
|
||||
searchQuery,
|
||||
onSearch,
|
||||
}: {
|
||||
t: Translate
|
||||
status: ComputerUseStatus
|
||||
enabled: boolean
|
||||
onToggleEnabled: (value: boolean) => void
|
||||
authorizedApps: AuthorizedApp[]
|
||||
onRemoveApp: (bundleId: string) => void
|
||||
appsSaved: boolean
|
||||
configError: string | null
|
||||
statusError: boolean
|
||||
cardOpening: boolean
|
||||
cardError: string | null
|
||||
onOpenCard: () => void
|
||||
onRecheck: () => void
|
||||
pickerOpen: boolean
|
||||
onOpenPicker: () => void
|
||||
onClosePicker: () => void
|
||||
onAddApp: (app: InstalledApp) => void
|
||||
appsLoading: boolean
|
||||
pickerApps: InstalledApp[]
|
||||
searchQuery: string
|
||||
onSearch: (value: string) => void
|
||||
}) {
|
||||
const accessibility = status.permissions.accessibility
|
||||
const screenRecording = status.permissions.screenRecording
|
||||
@@ -1141,153 +835,6 @@ function NativeComputerUse({
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* ─── 始终允许的应用 (Always-allowed apps) ─── */}
|
||||
<section className="space-y-3">
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<h3 className="flex items-center gap-2 text-[11px] font-semibold uppercase tracking-[0.08em] text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.allowedAppsTitle')}
|
||||
{appsSaved && (
|
||||
<span className="flex items-center gap-1 text-[11px] font-medium normal-case tracking-normal text-[var(--color-success)]">
|
||||
<span
|
||||
className="material-symbols-outlined text-[14px]"
|
||||
style={{ fontVariationSettings: "'FILL' 1" }}
|
||||
>
|
||||
check_circle
|
||||
</span>
|
||||
{t('settings.computerUse.appsSaved')}
|
||||
</span>
|
||||
)}
|
||||
</h3>
|
||||
<button
|
||||
onClick={onOpenPicker}
|
||||
className="flex items-center gap-1.5 rounded-lg bg-[var(--color-brand)] px-3 py-1.5 text-xs font-semibold text-[var(--color-btn-primary-fg)] shadow-[0_1px_2px_rgba(0,0,0,0.08)] transition hover:opacity-90 active:scale-[0.98]"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[16px]">add</span>
|
||||
{t('settings.computerUse.addApp')}
|
||||
</button>
|
||||
</div>
|
||||
<p className="text-xs leading-relaxed text-[var(--color-text-secondary)]">
|
||||
{t('settings.computerUse.allowedAppsDesc')}
|
||||
</p>
|
||||
|
||||
{authorizedApps.length === 0 ? (
|
||||
<div className="flex flex-col items-center gap-3 rounded-xl border border-dashed border-[var(--color-border)] bg-[var(--color-surface-container-low)]/40 px-6 py-10 text-center">
|
||||
<div className="flex h-11 w-11 items-center justify-center rounded-xl border border-[var(--color-border)] bg-[var(--color-surface-container)] text-[var(--color-text-tertiary)]">
|
||||
<span className="material-symbols-outlined text-[22px]">apps</span>
|
||||
</div>
|
||||
<p className="max-w-xs text-xs leading-relaxed text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.allowedAppsEmpty')}
|
||||
</p>
|
||||
</div>
|
||||
) : (
|
||||
<div className="overflow-hidden rounded-xl border border-[var(--color-border)] bg-[var(--color-surface-container-low)] shadow-[0_1px_2px_rgba(0,0,0,0.03)]">
|
||||
<div className="divide-y divide-[var(--color-border)]">
|
||||
{authorizedApps.map(app => (
|
||||
<div
|
||||
key={app.bundleId}
|
||||
className="group flex items-center gap-3 px-3 py-2.5 transition-colors hover:bg-[var(--color-surface-hover)]"
|
||||
>
|
||||
<AppIcon name={app.displayName} bundleId={app.bundleId} />
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="truncate text-sm font-medium text-[var(--color-text-primary)]">
|
||||
{app.displayName}
|
||||
</div>
|
||||
<div className="truncate font-mono text-[11px] text-[var(--color-text-tertiary)]">
|
||||
{app.bundleId}
|
||||
</div>
|
||||
</div>
|
||||
<button
|
||||
onClick={() => onRemoveApp(app.bundleId)}
|
||||
aria-label={`${t('settings.computerUse.removeApp')} ${app.displayName}`}
|
||||
title={t('settings.computerUse.removeApp')}
|
||||
className="flex h-8 w-8 flex-shrink-0 items-center justify-center rounded-lg text-[var(--color-text-tertiary)] opacity-0 transition group-hover:opacity-100 hover:bg-[var(--color-error-container)] hover:text-[var(--color-error)] focus-visible:opacity-100 active:scale-90"
|
||||
>
|
||||
<span className="material-symbols-outlined text-[18px]">delete</span>
|
||||
</button>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
{/* App picker dialog.
|
||||
Uses the shared Modal rather than a hand-rolled overlay: Modal portals
|
||||
to document.body, so it cannot be trapped by an ancestor's stacking
|
||||
context, and it sits on `--z-dialog` instead of a bare `z-50` that the
|
||||
rest of the `--z-*` scale does not know about. It also brings the focus
|
||||
trap, Escape-to-close and focus restore this picker used to lack. */}
|
||||
<Modal
|
||||
open={pickerOpen}
|
||||
onClose={onClosePicker}
|
||||
title={t('settings.computerUse.addAppTitle')}
|
||||
width={448}
|
||||
>
|
||||
{/* Cancel Modal's content padding so the app rows stay edge-to-edge —
|
||||
their divider lines are the list's structure, and inset dividers
|
||||
would read as a nested card. The search field keeps the padding. */}
|
||||
<div className="-mx-6 -my-4 flex max-h-[60vh] flex-col">
|
||||
<div className="shrink-0 px-6 pb-3 pt-4">
|
||||
<div className="relative">
|
||||
<span className="material-symbols-outlined pointer-events-none absolute left-3 top-1/2 -translate-y-1/2 text-[18px] text-[var(--color-text-tertiary)]">
|
||||
search
|
||||
</span>
|
||||
<input
|
||||
type="text"
|
||||
autoFocus
|
||||
value={searchQuery}
|
||||
onChange={e => onSearch(e.target.value)}
|
||||
placeholder={t('settings.computerUse.appsSearch')}
|
||||
className="w-full rounded-lg border border-[var(--color-border)] bg-[var(--color-surface-container-low)] py-2 pl-9 pr-4 text-sm text-[var(--color-text-primary)] placeholder:text-[var(--color-text-tertiary)] transition focus:border-[var(--color-brand)] focus:outline-none focus:ring-1 focus:ring-[var(--color-brand)]"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{/* The list scrolls, not the dialog — otherwise the search field
|
||||
scrolls out of view exactly when a long list needs it most. */}
|
||||
<div className="min-h-0 flex-1 divide-y divide-[var(--color-border)] overflow-y-auto border-t border-[var(--color-border)]">
|
||||
{appsLoading ? (
|
||||
<div className="flex flex-col items-center gap-2 py-12 text-center">
|
||||
<span className="material-symbols-outlined animate-spin text-[20px] text-[var(--color-text-tertiary)]">
|
||||
progress_activity
|
||||
</span>
|
||||
<span className="text-sm text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.appsPickerLoading')}
|
||||
</span>
|
||||
</div>
|
||||
) : pickerApps.length === 0 ? (
|
||||
<div className="flex flex-col items-center gap-2 py-12 text-center">
|
||||
<span className="material-symbols-outlined text-[20px] text-[var(--color-text-tertiary)]">
|
||||
search_off
|
||||
</span>
|
||||
<span className="text-sm text-[var(--color-text-tertiary)]">
|
||||
{t('settings.computerUse.appsPickerEmpty')}
|
||||
</span>
|
||||
</div>
|
||||
) : (
|
||||
pickerApps.map(app => (
|
||||
<button
|
||||
key={app.bundleId}
|
||||
onClick={() => onAddApp(app)}
|
||||
className="group flex w-full items-center gap-3 px-4 py-2.5 text-left transition-colors hover:bg-[var(--color-surface-hover)]"
|
||||
>
|
||||
<AppIcon name={app.displayName} bundleId={app.bundleId} />
|
||||
<div className="min-w-0 flex-1">
|
||||
<div className="truncate text-sm font-medium text-[var(--color-text-primary)]">
|
||||
{app.displayName}
|
||||
</div>
|
||||
<div className="truncate font-mono text-[11px] text-[var(--color-text-tertiary)]">
|
||||
{app.bundleId}
|
||||
</div>
|
||||
</div>
|
||||
<span className="material-symbols-outlined text-[18px] text-[var(--color-text-tertiary)] transition-colors group-hover:text-[var(--color-brand)]">
|
||||
add_circle
|
||||
</span>
|
||||
</button>
|
||||
))
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</Modal>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
|
||||
+33
-1
@@ -250,7 +250,8 @@ class TestMutatingCommandsAreGuarded(unittest.TestCase):
|
||||
"""Coordinate actions must not jump while mouse animation is enabled."""
|
||||
source = _win_source()
|
||||
self.assertIn("def _move_cursor_to", source)
|
||||
self.assertIn("1 - (1 - progress) ** 3", source)
|
||||
self.assertIn("def _spring_cursor_path", source)
|
||||
self.assertNotIn("1 - (1 - progress) ** 3", source)
|
||||
dispatcher = source.index("def main()")
|
||||
for command in ("click", "drag", "move_mouse", "scroll"):
|
||||
marker = f'if command == "{command}":'
|
||||
@@ -262,6 +263,28 @@ class TestMutatingCommandsAreGuarded(unittest.TestCase):
|
||||
f"{command} must honor the mouse-animation gate",
|
||||
)
|
||||
|
||||
@unittest.skipUnless(IS_WINDOWS, "requires the Windows helper module")
|
||||
def test_spring_cursor_path_starts_smoothly_and_lands_exactly(self):
|
||||
spec = importlib.util.spec_from_file_location("win_helper_motion", WIN_HELPER)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
points = module._spring_cursor_path(0, 0, 1000, 500)
|
||||
self.assertGreater(len(points), 12)
|
||||
self.assertEqual(points[-1], (1000, 500))
|
||||
first_distance = (points[0][0] ** 2 + points[0][1] ** 2) ** 0.5
|
||||
total_distance = (1000 ** 2 + 500 ** 2) ** 0.5
|
||||
self.assertLess(first_distance / total_distance, 0.10)
|
||||
self.assertTrue(all(a != b for a, b in zip(points, points[1:])))
|
||||
|
||||
def test_drag_reuses_the_shared_cursor_motion(self):
|
||||
source = _win_source()
|
||||
dispatcher = source.index('if command == "drag":')
|
||||
body = source[dispatcher:source.index('if command == "move_mouse":', dispatcher)]
|
||||
self.assertGreaterEqual(body.count("_move_cursor_to("), 2)
|
||||
self.assertNotIn("for step in range", body)
|
||||
|
||||
def test_helper_uses_per_monitor_dpi_coordinates(self):
|
||||
source = _win_source()
|
||||
self.assertIn("DPI_AWARENESS_CONTEXT_PER_MONITOR_AWARE_V2", source)
|
||||
@@ -577,6 +600,15 @@ class TestCursorBadge(unittest.TestCase):
|
||||
self.assertIn("WindowFromPoint", source)
|
||||
self.assertIn("GetForegroundWindow", source)
|
||||
|
||||
def test_overlay_activity_does_not_hide_before_animated_motion(self):
|
||||
source = CURSOR_BADGE.read_text(encoding="utf-8")
|
||||
start = source.index(" def _on_activity")
|
||||
end = source.index(" def _read_parent", start)
|
||||
body = source[start:end]
|
||||
self.assertNotIn("ShowWindow", body)
|
||||
self.assertIn("GetCursorPos", body)
|
||||
self.assertIn("self._requested_visible = self._target_pid is not None", body)
|
||||
|
||||
def test_overlay_readiness_precedes_the_first_action(self):
|
||||
source = CURSOR_BADGE.read_text(encoding="utf-8")
|
||||
self.assertIn('print("READY", flush=True)', source)
|
||||
|
||||
@@ -569,14 +569,18 @@ class VirtualCursorOverlay:
|
||||
|
||||
with self._lock:
|
||||
if point is not None:
|
||||
# Hide for the target lookup. A click-through layered window can
|
||||
# still be returned by WindowFromPoint on some Windows builds.
|
||||
if self.hwnd and self._shown:
|
||||
user32.ShowWindow(self.hwnd, SW_HIDE)
|
||||
self._shown = False
|
||||
# Keep the cursor visible between related actions. Hiding here
|
||||
# made every movement reappear only after the first injected
|
||||
# frame, which looked like a jump into the middle of the path.
|
||||
# `_pid_at_point` already skips this transparent overlay.
|
||||
self._destination = point
|
||||
self._target_pid = target_pid or self._pid_at_point(point)
|
||||
self._requested_visible = False # reveal on injected motion
|
||||
if self._agent_position is None:
|
||||
current = POINT()
|
||||
if user32.GetCursorPos(ctypes.byref(current)):
|
||||
self._agent_position = (int(current.x), int(current.y))
|
||||
self._requested_visible = self._target_pid is not None
|
||||
self._hidden_for_user = False
|
||||
elif target_pid is not None:
|
||||
self._target_pid = target_pid
|
||||
|
||||
|
||||
+54
-21
@@ -1196,30 +1196,70 @@ def _absolute_mouse_move(x: int, y: int) -> _INPUT:
|
||||
)
|
||||
|
||||
|
||||
def _spring_cursor_path(
|
||||
start_x: int,
|
||||
start_y: int,
|
||||
target_x: int,
|
||||
target_y: int,
|
||||
) -> list[tuple[int, int]]:
|
||||
"""Sample the same damped-spring motion used by the macOS cursor."""
|
||||
distance = ((target_x - start_x) ** 2 + (target_y - start_y) ** 2) ** 0.5
|
||||
if distance < 2:
|
||||
return [(target_x, target_y)]
|
||||
|
||||
# CursorMotionState.swift uses k=196 and a damping ratio of 0.85. A
|
||||
# 60-Hz fixed step gives Windows the same zero-velocity start and gentle
|
||||
# settle while keeping physical-pointer actions bounded.
|
||||
frame_interval = 1.0 / 60.0
|
||||
stiffness = 196.0
|
||||
damping = 2.0 * 0.85 * stiffness ** 0.5
|
||||
max_duration = min(0.45, max(0.20, distance / 3000.0))
|
||||
sample_count = max(1, round(max_duration / frame_interval))
|
||||
|
||||
pos_x, pos_y = float(start_x), float(start_y)
|
||||
vel_x = vel_y = 0.0
|
||||
points: list[tuple[int, int]] = []
|
||||
for _ in range(sample_count):
|
||||
vel_x += (stiffness * (target_x - pos_x) - damping * vel_x) * frame_interval
|
||||
vel_y += (stiffness * (target_y - pos_y) - damping * vel_y) * frame_interval
|
||||
pos_x += vel_x * frame_interval
|
||||
pos_y += vel_y * frame_interval
|
||||
point = (round(pos_x), round(pos_y))
|
||||
if not points or point != points[-1]:
|
||||
points.append(point)
|
||||
|
||||
remaining = ((target_x - pos_x) ** 2 + (target_y - pos_y) ** 2) ** 0.5
|
||||
speed = (vel_x ** 2 + vel_y ** 2) ** 0.5
|
||||
if remaining < 0.5 and speed < 6.0:
|
||||
break
|
||||
|
||||
target = (target_x, target_y)
|
||||
if not points or points[-1] != target:
|
||||
points.append(target)
|
||||
return points
|
||||
|
||||
|
||||
def _move_cursor_to(x: int, y: int, animate: bool) -> None:
|
||||
"""Move the shared pointer along a short eased path for overlay parity."""
|
||||
"""Move the shared pointer with the macOS virtual-cursor spring."""
|
||||
current = wintypes.POINT()
|
||||
if not _user32.GetCursorPos(ctypes.byref(current)):
|
||||
_send_inputs([_absolute_mouse_move(x, y)])
|
||||
return
|
||||
|
||||
start_x, start_y = int(current.x), int(current.y)
|
||||
distance = ((x - start_x) ** 2 + (y - start_y) ** 2) ** 0.5
|
||||
if not animate or distance < 2:
|
||||
if not animate:
|
||||
_send_inputs([_absolute_mouse_move(x, y)])
|
||||
return
|
||||
|
||||
# 100-260ms is long enough to read as motion without slowing the agent.
|
||||
duration = min(0.26, max(0.10, distance / 4000.0))
|
||||
steps = max(6, min(18, round(duration * 60)))
|
||||
for step in range(1, steps + 1):
|
||||
progress = step / steps
|
||||
eased = 1 - (1 - progress) ** 3
|
||||
next_x = round(start_x + (x - start_x) * eased)
|
||||
next_y = round(start_y + (y - start_y) * eased)
|
||||
points = _spring_cursor_path(start_x, start_y, x, y)
|
||||
started = time.perf_counter()
|
||||
for index, (next_x, next_y) in enumerate(points):
|
||||
_send_inputs([_absolute_mouse_move(next_x, next_y)])
|
||||
if step < steps:
|
||||
time.sleep(duration / steps)
|
||||
if index < len(points) - 1:
|
||||
deadline = started + (index + 1) / 60.0
|
||||
delay = deadline - time.perf_counter()
|
||||
if delay > 0:
|
||||
time.sleep(delay)
|
||||
|
||||
|
||||
_VIRTUAL_KEYS = {
|
||||
@@ -1859,14 +1899,7 @@ def main() -> int:
|
||||
animate = bool(payload.get("animate", True))
|
||||
_move_cursor_to(start_x, start_y, animate)
|
||||
_send_inputs([_mouse_input(MOUSEEVENTF_LEFTDOWN)])
|
||||
duration = 0.18 if animate else 0
|
||||
for step in range(1, 13):
|
||||
_send_inputs([_absolute_mouse_move(
|
||||
round(start_x + (target_x - start_x) * step / 12),
|
||||
round(start_y + (target_y - start_y) * step / 12),
|
||||
)])
|
||||
if duration and step < 12:
|
||||
time.sleep(duration / 12)
|
||||
_move_cursor_to(target_x, target_y, animate)
|
||||
_send_inputs([_mouse_input(MOUSEEVENTF_LEFTUP)])
|
||||
return _finish(lease, True)
|
||||
if command == "move_mouse":
|
||||
|
||||
@@ -73,13 +73,18 @@ afterAll(async () => {
|
||||
})
|
||||
|
||||
describe('Computer Use API authorized app config', () => {
|
||||
it('defaults Computer Use enabled for existing users without config', async () => {
|
||||
it('defaults Computer Use off until the risk confirmation is accepted', async () => {
|
||||
const res = await callAuthorizedApps('GET')
|
||||
|
||||
expect(res.status).toBe(200)
|
||||
expect(await res.json()).toMatchObject({
|
||||
enabled: true,
|
||||
enabled: false,
|
||||
authorizedApps: [],
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
@@ -484,63 +489,21 @@ describe('runPipInstallWithFallback', () => {
|
||||
})
|
||||
})
|
||||
|
||||
describe('computeRuntimeGrantAdditions', () => {
|
||||
it('maps new grants to AuthorizedApp entries with ISO authorizedAt', async () => {
|
||||
const { computeRuntimeGrantAdditions } = await importComputerUseApi()
|
||||
const grantedAt = Date.UTC(2026, 0, 2, 3, 4, 5)
|
||||
|
||||
const additions = computeRuntimeGrantAdditions(
|
||||
[],
|
||||
[{ bundleId: 'com.apple.Notes', displayName: 'Notes', grantedAt }],
|
||||
describe('retired per-app authorization endpoint', () => {
|
||||
it('cannot emit a runtime approval request', async () => {
|
||||
const response = await callComputerUseAction(
|
||||
'request-access',
|
||||
'POST',
|
||||
JSON.stringify({
|
||||
sessionId: 'session-1',
|
||||
request: { requestId: 'request-1', apps: [] },
|
||||
}),
|
||||
)
|
||||
|
||||
expect(additions).toEqual([
|
||||
{
|
||||
bundleId: 'com.apple.Notes',
|
||||
displayName: 'Notes',
|
||||
authorizedAt: new Date(grantedAt).toISOString(),
|
||||
},
|
||||
])
|
||||
expect(response.status).toBe(410)
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
error: 'APP_AUTHORIZATION_REMOVED',
|
||||
})
|
||||
|
||||
it('dedupes grants already present in the stored config by bundleId', async () => {
|
||||
const { computeRuntimeGrantAdditions } = await importComputerUseApi()
|
||||
|
||||
const additions = computeRuntimeGrantAdditions(
|
||||
[{ bundleId: 'com.apple.Notes', displayName: 'Notes' }],
|
||||
[
|
||||
{ bundleId: 'com.apple.Notes', displayName: 'Notes', grantedAt: 1 },
|
||||
{ bundleId: 'com.apple.Safari', displayName: 'Safari', grantedAt: 2 },
|
||||
],
|
||||
)
|
||||
|
||||
expect(additions.map((a) => a.bundleId)).toEqual(['com.apple.Safari'])
|
||||
})
|
||||
|
||||
it('dedupes duplicate bundleIds within the same grant batch', async () => {
|
||||
const { computeRuntimeGrantAdditions } = await importComputerUseApi()
|
||||
|
||||
const additions = computeRuntimeGrantAdditions(
|
||||
[],
|
||||
[
|
||||
{ bundleId: 'com.apple.Safari', displayName: 'Safari', grantedAt: 1 },
|
||||
{ bundleId: 'com.apple.Safari', displayName: 'Safari (dup)', grantedAt: 2 },
|
||||
],
|
||||
)
|
||||
|
||||
expect(additions).toHaveLength(1)
|
||||
expect(additions[0]).toMatchObject({ bundleId: 'com.apple.Safari', displayName: 'Safari' })
|
||||
})
|
||||
|
||||
it('skips grants without a bundleId and returns [] for an empty batch', async () => {
|
||||
const { computeRuntimeGrantAdditions } = await importComputerUseApi()
|
||||
|
||||
expect(computeRuntimeGrantAdditions([], [])).toEqual([])
|
||||
expect(
|
||||
computeRuntimeGrantAdditions([], [
|
||||
{ bundleId: '', displayName: 'No Bundle', grantedAt: 1 },
|
||||
]),
|
||||
).toEqual([])
|
||||
})
|
||||
})
|
||||
|
||||
|
||||
@@ -12,8 +12,6 @@ import { access, readFile, mkdir, writeFile, rm } from 'fs/promises'
|
||||
import { createHash } from 'crypto'
|
||||
import path from 'path'
|
||||
import { fileURLToPath } from 'url'
|
||||
import type { AppGrant, CuPermissionRequest } from '../../vendor/computer-use-mcp/types.js'
|
||||
import { computerUseApprovalService } from '../services/computerUseApprovalService.js'
|
||||
import { diagnosticsService } from '../services/diagnosticsService.js'
|
||||
import { normalizeIconSize, readAppIconPng } from '../services/macAppIcon.js'
|
||||
import { listInstalledMacApps } from './macInstalledApps.js'
|
||||
@@ -910,13 +908,8 @@ export type ComputerUseConfigPatch = {
|
||||
pythonPath?: string | null
|
||||
}
|
||||
|
||||
type RequestAccessBody = {
|
||||
sessionId?: string
|
||||
request?: CuPermissionRequest
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG: ComputerUseConfig = {
|
||||
enabled: true,
|
||||
enabled: false,
|
||||
authorizedApps: [],
|
||||
grantFlags: DEFAULT_DESKTOP_GRANT_FLAGS,
|
||||
pythonPath: null,
|
||||
@@ -1042,56 +1035,6 @@ export async function openComputerUseSettings(
|
||||
: { ok: false, message: result.stderr || `Failed to run ${command.cmd}` }
|
||||
}
|
||||
|
||||
/**
|
||||
* Compute the AuthorizedApp entries to APPEND for a batch of runtime grants,
|
||||
* deduped by bundleId against the already-stored apps. Pure (no I/O) so it can
|
||||
* be unit-tested directly.
|
||||
*
|
||||
* Mapping: AppGrant.grantedAt (epoch ms) -> AuthorizedApp.authorizedAt (ISO
|
||||
* string). NOTE the stored schema (StoredAuthorizedApp) carries no `tier`, so
|
||||
* a runtime read/click-tier grant is reloaded as full-tier next session — a
|
||||
* pre-existing limitation of the stored schema, out of scope here.
|
||||
*/
|
||||
export function computeRuntimeGrantAdditions(
|
||||
existingApps: AuthorizedApp[],
|
||||
granted: AppGrant[],
|
||||
): AuthorizedApp[] {
|
||||
const existing = new Set(existingApps.map((app) => app.bundleId))
|
||||
const additions: AuthorizedApp[] = []
|
||||
// Dedupe against BOTH the stored set and within this batch, so a request
|
||||
// listing the same bundleId twice can't double-append.
|
||||
const seen = new Set(existing)
|
||||
for (const grant of granted) {
|
||||
if (!grant.bundleId || seen.has(grant.bundleId)) continue
|
||||
seen.add(grant.bundleId)
|
||||
additions.push({
|
||||
bundleId: grant.bundleId,
|
||||
displayName: grant.displayName,
|
||||
authorizedAt: new Date(grant.grantedAt).toISOString(),
|
||||
})
|
||||
}
|
||||
return additions
|
||||
}
|
||||
|
||||
/**
|
||||
* Append newly-granted apps (from a runtime request_access approval) to the
|
||||
* persisted config. Best-effort: any failure is swallowed so it can never tank
|
||||
* the grant response the model is awaiting. Idempotent across repeat calls in a
|
||||
* long session because additions are deduped by bundleId.
|
||||
*/
|
||||
async function persistRuntimeGrants(granted: AppGrant[]): Promise<void> {
|
||||
if (!granted || granted.length === 0) return
|
||||
try {
|
||||
const config = await loadConfig()
|
||||
const additions = computeRuntimeGrantAdditions(config.authorizedApps, granted)
|
||||
if (additions.length === 0) return
|
||||
config.authorizedApps = [...config.authorizedApps, ...additions]
|
||||
await saveConfig(config)
|
||||
} catch {
|
||||
// best-effort — a config-write failure must NOT fail the grant response
|
||||
}
|
||||
}
|
||||
|
||||
async function listInstalledApps(): Promise<{ bundleId: string; displayName: string; path: string }[]> {
|
||||
// macOS: enumerate the application roots directly. This needs neither a
|
||||
// Python venv nor a running helper, so the app picker populates on a cold
|
||||
@@ -1351,38 +1294,15 @@ export async function handleComputerUseApi(
|
||||
}
|
||||
|
||||
if (action === 'request-access' && req.method === 'POST') {
|
||||
try {
|
||||
const body = (await req.json()) as RequestAccessBody
|
||||
if (!body.sessionId || !body.request?.requestId) {
|
||||
return Response.json(
|
||||
{ error: 'BAD_REQUEST', message: 'sessionId and request are required' },
|
||||
{ status: 400 },
|
||||
{
|
||||
error: 'APP_AUTHORIZATION_REMOVED',
|
||||
message: 'Per-application Computer Use authorization is no longer required.',
|
||||
},
|
||||
{ status: 410 },
|
||||
)
|
||||
}
|
||||
|
||||
const response = await computerUseApprovalService.requestApproval(
|
||||
body.sessionId,
|
||||
body.request,
|
||||
)
|
||||
|
||||
// Runtime auto-add: persist apps the user just granted into the stored
|
||||
// config so they appear under "始终允许的应用" in Settings and survive
|
||||
// into the next session. This is the SINGLE server-side runtime-grant
|
||||
// ingress (no teach-access route exists), runs in the process that owns
|
||||
// the config writer, and does NOT touch the per-session allowlist (that
|
||||
// lives in the separate CLI subprocess). Best-effort: a config-write
|
||||
// failure must never fail the grant the model is awaiting.
|
||||
await persistRuntimeGrants(response.granted)
|
||||
|
||||
return Response.json(response)
|
||||
} catch (error) {
|
||||
const message =
|
||||
error instanceof Error ? error.message : 'Computer Use approval failed'
|
||||
const status = message.includes('not connected') ? 409 : 500
|
||||
return Response.json({ error: 'COMPUTER_USE_APPROVAL_FAILED', message }, { status })
|
||||
}
|
||||
}
|
||||
|
||||
return Response.json(
|
||||
{ error: 'NOT_FOUND', message: `Unknown computer-use action: ${action}` },
|
||||
{ status: 404 },
|
||||
|
||||
@@ -107,9 +107,10 @@ describe('computer-use skill registration', () => {
|
||||
expect(skill!.allowedTools).toEqual(getComputerUseToolAllowlist(platform))
|
||||
expect(skill!.allowedTools).toContain(
|
||||
process.platform === 'win32'
|
||||
? 'mcp__computer-use__request_access'
|
||||
? 'mcp__computer-use__screenshot'
|
||||
: 'mcp__computer-use__get_app_state',
|
||||
)
|
||||
expect(skill!.allowedTools).not.toContain('mcp__computer-use__request_access')
|
||||
expect(
|
||||
skill!.allowedTools!.every(t => t.startsWith('mcp__computer-use__')),
|
||||
).toBe(true)
|
||||
@@ -123,9 +124,10 @@ describe('computer-use skill registration', () => {
|
||||
})
|
||||
|
||||
describe('computer-use Windows guidance', () => {
|
||||
test('matches the unfiltered, permission-gated pixel tool face', () => {
|
||||
test('matches the unfiltered, feature-authorized pixel tool face', () => {
|
||||
const prompt = getComputerUsePrompt('win32')
|
||||
expect(prompt).toContain('request_access')
|
||||
expect(prompt).not.toContain('request_access')
|
||||
expect(prompt).toContain('without an app-by-app approval prompt')
|
||||
expect(prompt).toContain('screenshots are NOT filtered')
|
||||
expect(prompt).toContain('most recent full screenshot')
|
||||
expect(prompt).toContain('UNKNOWN result')
|
||||
|
||||
@@ -96,8 +96,8 @@ never helps.
|
||||
|
||||
If three genuinely different approaches fail, stop and tell the user what you
|
||||
tried and what you observed. Do not drive the UI with \`osascript\`, AppleScript,
|
||||
System Events, JXA, Python, or shell commands — those bypass the permission
|
||||
model the user granted, do not work on these apps, and burn the rest of the
|
||||
System Events, JXA, Python, or shell commands — those bypass Computer Use's
|
||||
target and interference safeguards, do not work on these apps, and burn the rest of the
|
||||
session.
|
||||
|
||||
## Tool notes
|
||||
@@ -156,30 +156,26 @@ const WINDOWS_COMPUTER_USE_PROMPT = `# Operating Windows apps
|
||||
You are driving real applications on the user's Windows desktop through the
|
||||
Computer Use pixel tools. Work in this loop:
|
||||
|
||||
1. \`request_access({ apps, reason })\` once, naming every app the task needs.
|
||||
It must run before every other Computer Use tool. If another app becomes
|
||||
necessary later, request access to add it.
|
||||
2. \`screenshot()\` and inspect the current display.
|
||||
3. Act using coordinates from that exact full-display screenshot.
|
||||
4. Take another \`screenshot()\` before deciding whether the action worked.
|
||||
1. \`screenshot()\` and inspect the current display.
|
||||
2. Act using coordinates from that exact full-display screenshot.
|
||||
3. Take another \`screenshot()\` before deciding whether the action worked.
|
||||
|
||||
On Windows screenshots are NOT filtered: every visible window on the captured
|
||||
display can appear, including apps that were not granted. Permission limits
|
||||
input, not visibility. Never interact with an ungranted app; request access or
|
||||
ask the user first.
|
||||
display can appear. Enabling Computer Use authorizes control of supported apps
|
||||
without an app-by-app approval prompt. Product safety restrictions still apply.
|
||||
|
||||
Use \`zoom\` to read small details, but never use coordinates from a zoom image
|
||||
for actions. Coordinates always refer to the most recent full screenshot. Use
|
||||
\`open_application\` to launch or foreground a granted app. Input actions are
|
||||
also checked against the frontmost app and the window under the target point;
|
||||
if either is ungranted, stop and refresh state instead of trying to bypass the
|
||||
gate.
|
||||
\`open_application\` to launch or foreground an installed app. Input actions
|
||||
are checked against the frontmost app and the window under the target point;
|
||||
if a target cannot be identified or is safety-restricted, stop and refresh
|
||||
state instead of trying to bypass the gate.
|
||||
|
||||
Mutating tools return a dispatch receipt, not proof of the intended result.
|
||||
Only the next screenshot proves what happened. If two attempts leave the UI
|
||||
unchanged, change approach. Do not repeat an identical action a third time.
|
||||
Do not fall back to PowerShell, Python, AutoHotkey, or another UI automation
|
||||
path; those bypass the permission and interference safeguards the user granted.
|
||||
path; those bypass Computer Use's target, product-safety, and interference safeguards.
|
||||
|
||||
The helper shares Windows' real mouse and keyboard stream. If it reports user
|
||||
interference or an UNKNOWN result, do not repeat the action. Take a screenshot
|
||||
@@ -217,7 +213,7 @@ export function registerComputerUseSkill(): void {
|
||||
// competes with the Chrome extension and purpose-built MCP servers on web
|
||||
// tasks, where they are faster and more precise.
|
||||
description: isWindows
|
||||
? "Operate apps on the user's Windows desktop — click, type, scroll and inspect the display through permission-gated pixel tools. For native desktop apps and cross-app workflows. Prefer a purpose-built MCP server, browser integration, or CLI when one covers the task."
|
||||
? "Operate apps on the user's Windows desktop — click, type, scroll and inspect the display through safety-gated pixel tools. For native desktop apps and cross-app workflows. Prefer a purpose-built MCP server, browser integration, or CLI when one covers the task."
|
||||
: "Operate apps on the user's Mac — click, type, scroll and read app state through the accessibility engine. For native desktop apps and cross-app workflows. Prefer a purpose-built MCP server, the Chrome extension, or a CLI when one covers the task.",
|
||||
whenToUse: isWindows
|
||||
? 'When the user wants something done inside a Windows application. Invoke this BEFORE the first mcp__computer-use__* call; it carries the approval, screenshot, and pixel-action workflow those tools assume.'
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
/**
|
||||
* Filter and sanitize installed-app data for inclusion in the `request_access`
|
||||
* tool description. Ported from Cowork's appNames.ts. Two
|
||||
* Filter and sanitize installed-app data for inclusion in Computer Use tool
|
||||
* descriptions. Ported from Cowork's appNames.ts. Two
|
||||
* concerns: noise filtering (Spotlight returns every bundle on disk — XPC
|
||||
* helpers, daemons, input methods) and prompt-injection hardening (app names
|
||||
* are attacker-controlled; anyone can ship an app named anything).
|
||||
@@ -8,8 +8,8 @@
|
||||
* Residual risk: short benign-char adversarial names ("grant all") can't be
|
||||
* filtered programmatically. The tool description's structural framing
|
||||
* ("Available applications:") makes it clear these are app names, and the
|
||||
* downstream permission dialog requires explicit user approval — a bad name
|
||||
* can't auto-grant anything.
|
||||
* runtime resolves the chosen app against the installed inventory and still
|
||||
* applies product-safety restrictions before acting.
|
||||
*/
|
||||
|
||||
/** Minimal shape — matches what `listInstalledApps` returns. */
|
||||
|
||||
@@ -47,7 +47,7 @@ async function tryGetInstalledAppNames(): Promise<string[] | undefined> {
|
||||
* Construct the in-process server. Delegates to the package's
|
||||
* `createComputerUseMcpServer` for the Server object + stub CallTool handler,
|
||||
* then REPLACES the ListTools handler with one that includes installed-app
|
||||
* names in the `request_access` description (the package's factory doesn't
|
||||
* names in platform-specific tool descriptions (the package's factory doesn't
|
||||
* take `installedAppNames`, and Cowork builds its own tool array in
|
||||
* serverDef.ts for the same reason).
|
||||
*
|
||||
|
||||
@@ -10,14 +10,14 @@ import {
|
||||
} from './preauthorizedConfig.js'
|
||||
|
||||
describe('resolveStoredComputerUseConfig', () => {
|
||||
test('keeps desktop grant flags disabled until explicitly granted', () => {
|
||||
test('starts disabled and prepares full grants for explicit enablement', () => {
|
||||
expect(resolveStoredComputerUseConfig()).toEqual({
|
||||
enabled: true,
|
||||
enabled: false,
|
||||
authorizedApps: [],
|
||||
grantFlags: {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
pythonPath: null,
|
||||
})
|
||||
@@ -30,7 +30,7 @@ describe('resolveStoredComputerUseConfig', () => {
|
||||
})
|
||||
})
|
||||
|
||||
test('honors explicit grant flags without enabling unspecified grants', () => {
|
||||
test('honors explicit grant flags while defaulting unspecified grants on', () => {
|
||||
expect(
|
||||
resolveStoredComputerUseConfig({
|
||||
grantFlags: {
|
||||
@@ -38,12 +38,12 @@ describe('resolveStoredComputerUseConfig', () => {
|
||||
},
|
||||
}),
|
||||
).toEqual({
|
||||
enabled: true,
|
||||
enabled: false,
|
||||
authorizedApps: [],
|
||||
grantFlags: {
|
||||
clipboardRead: true,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
pythonPath: null,
|
||||
})
|
||||
@@ -57,12 +57,12 @@ describe('resolveStoredComputerUseConfig', () => {
|
||||
try {
|
||||
await expect(loadStoredComputerUseConfigResult()).resolves.toEqual({
|
||||
config: {
|
||||
enabled: true,
|
||||
enabled: false,
|
||||
authorizedApps: [],
|
||||
grantFlags: {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
},
|
||||
pythonPath: null,
|
||||
},
|
||||
|
||||
@@ -26,12 +26,14 @@ export type StoredComputerUseConfig = {
|
||||
[key: string]: unknown
|
||||
}
|
||||
|
||||
export const DEFAULT_COMPUTER_USE_ENABLED = true
|
||||
// Computer Use starts off so the first enablement always crosses the explicit
|
||||
// risk-confirmation boundary in the desktop settings page.
|
||||
export const DEFAULT_COMPUTER_USE_ENABLED = false
|
||||
|
||||
export const DEFAULT_DESKTOP_GRANT_FLAGS: CuGrantFlags = {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
}
|
||||
|
||||
const FAIL_CLOSED_GRANT_FLAGS: CuGrantFlags = {
|
||||
|
||||
@@ -20,8 +20,8 @@ type SetupComputerUseDeps = {
|
||||
/**
|
||||
* Build the dynamic MCP config + allowed tool names. Mirror of
|
||||
* `setupClaudeInChrome`. The `mcp__computer-use__*` tools are added to
|
||||
* `allowedTools` so they bypass the normal permission prompt — the package's
|
||||
* `request_access` handles approval for the whole session.
|
||||
* `allowedTools` so they bypass the normal tool prompt. The settings-page risk
|
||||
* confirmation is the authorization boundary for the whole session.
|
||||
*
|
||||
* The MCP layer isn't ceremony: the API backend detects `mcp__computer-use__*`
|
||||
* tool names and emits a CU availability hint into the system prompt
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { describe, expect, test } from 'bun:test'
|
||||
import { buildSessionContext } from './wrapper.js'
|
||||
|
||||
describe('Computer Use session authorization', () => {
|
||||
test('enables every supported app without exposing a runtime permission callback', () => {
|
||||
const context = buildSessionContext()
|
||||
|
||||
expect(context.getAllowedApps()).toEqual([])
|
||||
expect(context.getUserDeniedBundleIds()).toEqual([])
|
||||
expect(context.getGrantFlags()).toEqual({
|
||||
clipboardRead: true,
|
||||
clipboardWrite: true,
|
||||
systemKeyCombos: true,
|
||||
})
|
||||
expect(context.onPermissionRequest).toBeUndefined()
|
||||
})
|
||||
})
|
||||
File diff suppressed because one or more lines are too long
+17
-37
@@ -28,7 +28,6 @@ const DARWIN_TOOL_NAMES = [
|
||||
].sort()
|
||||
|
||||
const WINDOWS_LEGACY_TOOL_NAMES = [
|
||||
'request_access',
|
||||
'screenshot',
|
||||
'zoom',
|
||||
'left_click',
|
||||
@@ -43,7 +42,6 @@ const WINDOWS_LEGACY_TOOL_NAMES = [
|
||||
'mouse_move',
|
||||
'open_application',
|
||||
'switch_display',
|
||||
'list_granted_applications',
|
||||
'read_clipboard',
|
||||
'write_clipboard',
|
||||
'wait',
|
||||
@@ -69,14 +67,7 @@ function makeSessionContext(
|
||||
overrides: Partial<ComputerUseSessionContext> = {},
|
||||
): ComputerUseSessionContext {
|
||||
return {
|
||||
getAllowedApps: () => [
|
||||
{
|
||||
bundleId: 'com.example.allowed',
|
||||
displayName: 'Allowed App',
|
||||
tier: 'full',
|
||||
grantedAt: 1,
|
||||
},
|
||||
],
|
||||
getAllowedApps: () => [],
|
||||
getGrantFlags: () => ({
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
@@ -84,15 +75,9 @@ function makeSessionContext(
|
||||
}),
|
||||
getUserDeniedBundleIds: () => [],
|
||||
getSelectedDisplayId: () => undefined,
|
||||
onPermissionRequest: async () => ({
|
||||
granted: [],
|
||||
denied: [],
|
||||
flags: {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
onPermissionRequest: async () => {
|
||||
throw new Error('per-app permission prompts must not run')
|
||||
},
|
||||
}),
|
||||
...overrides,
|
||||
}
|
||||
}
|
||||
@@ -266,7 +251,9 @@ function makeWindowsAdapter(calls: string[]): ComputerUseHostAdapter {
|
||||
},
|
||||
]
|
||||
},
|
||||
async openApp() {},
|
||||
async openApp(bundleId: string) {
|
||||
calls.push(`openApp:${bundleId}`)
|
||||
},
|
||||
} as unknown as ComputerExecutor
|
||||
|
||||
return {
|
||||
@@ -341,7 +328,7 @@ describe('Computer Use platform routing', () => {
|
||||
}
|
||||
})
|
||||
|
||||
test('win32 ListTools advertises the complete legacy pixel face', async () => {
|
||||
test('win32 ListTools advertises pixel controls without app-authorization tools', async () => {
|
||||
const connection = await connect(makeWindowsAdapter([]))
|
||||
try {
|
||||
const result = await connection.client.listTools()
|
||||
@@ -349,6 +336,8 @@ describe('Computer Use platform routing', () => {
|
||||
WINDOWS_LEGACY_TOOL_NAMES,
|
||||
)
|
||||
expect(result.tools.some(tool => tool.name === 'get_app_state')).toBe(false)
|
||||
expect(result.tools.some(tool => tool.name === 'request_access')).toBe(false)
|
||||
expect(result.tools.some(tool => tool.name === 'list_granted_applications')).toBe(false)
|
||||
} finally {
|
||||
await connection.close()
|
||||
}
|
||||
@@ -361,8 +350,6 @@ describe('Computer Use platform routing', () => {
|
||||
makeSessionContext(),
|
||||
)
|
||||
try {
|
||||
// Two captures make the second hidden-app note exercise
|
||||
// executor.listRunningApps (the Python `list_running_apps` handler).
|
||||
expect((await connection.client.callTool({ name: 'screenshot' })).isError).toBeFalsy()
|
||||
expect((await connection.client.callTool({ name: 'screenshot' })).isError).toBeFalsy()
|
||||
expect(
|
||||
@@ -391,7 +378,7 @@ describe('Computer Use platform routing', () => {
|
||||
).toBeFalsy()
|
||||
|
||||
expect(calls).toContain('screenshot')
|
||||
expect(calls).toContain('listRunningApps')
|
||||
expect(calls).not.toContain('listRunningApps')
|
||||
expect(calls).toContain('click:10,20,left,1')
|
||||
expect(calls).toContain('key:ctrl+a')
|
||||
expect(calls).toContain('type:ok')
|
||||
@@ -410,6 +397,13 @@ describe('Computer Use platform routing', () => {
|
||||
expect(blockedHold.isError).toBe(true)
|
||||
expect(calls).not.toContain('key:ctrl+alt+delete')
|
||||
expect(calls).not.toContain('holdKey:alt+f4')
|
||||
|
||||
const opened = await connection.client.callTool({
|
||||
name: 'open_application',
|
||||
arguments: { app: 'Allowed App' },
|
||||
})
|
||||
expect(opened.isError).toBeFalsy()
|
||||
expect(calls).toContain('openApp:com.example.allowed')
|
||||
} finally {
|
||||
await connection.close()
|
||||
}
|
||||
@@ -448,20 +442,6 @@ describe('Computer Use platform routing', () => {
|
||||
}),
|
||||
)
|
||||
try {
|
||||
const access = await connection.client.callTool({
|
||||
name: 'request_access',
|
||||
arguments: {
|
||||
apps: ['Allowed App'],
|
||||
reason: 'Exercise Windows lock routing.',
|
||||
},
|
||||
})
|
||||
const list = await connection.client.callTool({
|
||||
name: 'list_granted_applications',
|
||||
})
|
||||
expect(access.isError).toBeFalsy()
|
||||
expect(list.isError).toBeFalsy()
|
||||
expect(acquireCount).toBe(0)
|
||||
|
||||
// The first action takes the lock and must clear the prior session's
|
||||
// module-level mouseButtonHeld flag before dispatching.
|
||||
const screenshot = await connection.client.callTool({ name: 'screenshot' })
|
||||
|
||||
+48
-114
@@ -805,9 +805,9 @@ describe('handleToolCall — gates', () => {
|
||||
}
|
||||
})
|
||||
|
||||
test('resolves alias and PID targets before permission, then dispatches to the canonical running PID', async () => {
|
||||
test('resolves alias and PID targets and dispatches without an app permission prompt', async () => {
|
||||
for (const requestedApp of ['Notes', '812']) {
|
||||
let permissionRequest: any
|
||||
let permissionCalls = 0
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async (target: AppTarget) => {
|
||||
expect(target).toEqual(requestedApp === '812' ? { pid: 812 } : { app: 'Notes' })
|
||||
@@ -833,37 +833,15 @@ describe('handleToolCall — gates', () => {
|
||||
{ app: requestedApp, element_index: 'g17:1' },
|
||||
baseOverrides({
|
||||
allowedApps: [],
|
||||
onPermissionRequest: async req => {
|
||||
permissionRequest = req
|
||||
return {
|
||||
granted: [{
|
||||
bundleId: 'com.apple.Notes',
|
||||
displayName: 'Notes',
|
||||
grantedAt: 2,
|
||||
tier: 'full',
|
||||
}],
|
||||
denied: [],
|
||||
flags: {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
},
|
||||
}
|
||||
onPermissionRequest: async () => {
|
||||
permissionCalls++
|
||||
throw new Error('must not prompt')
|
||||
},
|
||||
}),
|
||||
)
|
||||
|
||||
expect(r.isError).toBeFalsy()
|
||||
expect(permissionRequest.apps).toHaveLength(1)
|
||||
expect(permissionRequest.apps[0]).toMatchObject({
|
||||
requestedName: requestedApp,
|
||||
alreadyGranted: false,
|
||||
resolved: {
|
||||
bundleId: 'com.apple.Notes',
|
||||
displayName: 'Notes',
|
||||
path: '/System/Applications/Notes.app',
|
||||
},
|
||||
})
|
||||
expect(permissionCalls).toBe(0)
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'click'])
|
||||
expect(calls[1].args).toMatchObject({ target: { pid: 812 } })
|
||||
}
|
||||
@@ -898,8 +876,9 @@ describe('handleToolCall — gates', () => {
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
})
|
||||
|
||||
test('authorizes an installed path without launching it and dispatches get_app_state by exact path', async () => {
|
||||
test('dispatches get_app_state by exact installed path without an app prompt', async () => {
|
||||
const path = '/Applications/Acme Notes.app'
|
||||
let permissionCalls = 0
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async () => ({
|
||||
bundleId: 'com.acme.notes',
|
||||
@@ -913,67 +892,20 @@ describe('handleToolCall — gates', () => {
|
||||
{ app: path },
|
||||
baseOverrides({
|
||||
allowedApps: [],
|
||||
onPermissionRequest: async req => ({
|
||||
granted: [{
|
||||
bundleId: req.apps[0].resolved!.bundleId,
|
||||
displayName: req.apps[0].resolved!.displayName,
|
||||
grantedAt: 2,
|
||||
tier: 'full',
|
||||
}],
|
||||
denied: [],
|
||||
flags: {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
onPermissionRequest: async () => {
|
||||
permissionCalls++
|
||||
throw new Error('must not prompt')
|
||||
},
|
||||
}),
|
||||
}),
|
||||
)
|
||||
|
||||
expect(r.isError).toBeFalsy()
|
||||
expect(permissionCalls).toBe(0)
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'getAppState'])
|
||||
expect(calls[1].args).toEqual({ app: path })
|
||||
})
|
||||
|
||||
test('a permission denial or mismatched grant never reaches the mutation engine', async () => {
|
||||
for (const granted of [[], [{
|
||||
bundleId: 'com.other.app',
|
||||
displayName: 'Other',
|
||||
grantedAt: 2,
|
||||
tier: 'full' as const,
|
||||
}]]) {
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async () => ({
|
||||
pid: 812,
|
||||
bundleId: 'com.apple.Notes',
|
||||
displayName: 'Notes',
|
||||
}),
|
||||
})
|
||||
const r = await handleToolCall(
|
||||
makeAdapter({ engine }),
|
||||
'type_text',
|
||||
{ app: 'Notes', text: 'secret' },
|
||||
baseOverrides({
|
||||
allowedApps: [],
|
||||
onPermissionRequest: async () => ({
|
||||
granted,
|
||||
denied: [{ bundleId: 'com.apple.Notes', reason: 'user_denied' }],
|
||||
flags: {
|
||||
clipboardRead: false,
|
||||
clipboardWrite: false,
|
||||
systemKeyCombos: false,
|
||||
},
|
||||
}),
|
||||
}),
|
||||
)
|
||||
|
||||
expect(r.isError).toBe(true)
|
||||
expect(r.telemetry?.error_kind).toBe('app_not_granted')
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
}
|
||||
})
|
||||
|
||||
test('an ungranted target without a permission handler fails closed', async () => {
|
||||
test('an empty legacy allowlist still permits a supported target without a permission handler', async () => {
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async () => ({
|
||||
pid: 812,
|
||||
@@ -990,12 +922,11 @@ describe('handleToolCall — gates', () => {
|
||||
baseOverrides({ allowedApps: [], onPermissionRequest: undefined }),
|
||||
)
|
||||
|
||||
expect(r.isError).toBe(true)
|
||||
expect(r.telemetry?.error_kind).toBe('app_not_granted')
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
expect(r.isError).toBeFalsy()
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'click'])
|
||||
})
|
||||
|
||||
test('an exact preauthorization bypasses the permission handler', async () => {
|
||||
test('legacy preauthorization data is not consulted or prompted for', async () => {
|
||||
let permissionCalls = 0
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async () => ({
|
||||
@@ -1024,42 +955,45 @@ describe('handleToolCall — gates', () => {
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'typeText'])
|
||||
})
|
||||
|
||||
test('resolved user-denied and policy-denied aliases fail before mutation', async () => {
|
||||
const cases = [
|
||||
{
|
||||
resolved: { pid: 812, bundleId: 'com.apple.Notes', displayName: 'Notes' },
|
||||
overrides: { userDeniedBundleIds: ['com.apple.Notes'] },
|
||||
},
|
||||
{
|
||||
resolved: { pid: 900, bundleId: 'com.googlecode.iterm2', displayName: 'iTerm2' },
|
||||
overrides: {},
|
||||
},
|
||||
]
|
||||
for (const entry of cases) {
|
||||
let permissionCalls = 0
|
||||
const { engine, calls } = makeEngine({ resolveTarget: async () => entry.resolved })
|
||||
test('legacy user-denied app state no longer blocks a supported app', async () => {
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async () => ({
|
||||
pid: 812,
|
||||
bundleId: 'com.apple.Notes',
|
||||
displayName: 'Notes',
|
||||
executablePath: '/System/Applications/Notes.app/Contents/MacOS/Notes',
|
||||
launchTime: 1812,
|
||||
}),
|
||||
})
|
||||
const r = await handleToolCall(
|
||||
makeAdapter({ engine }),
|
||||
'type_text',
|
||||
{ app: 'friendly alias', text: 'blocked' },
|
||||
baseOverrides({
|
||||
allowedApps: [{
|
||||
bundleId: entry.resolved.bundleId,
|
||||
displayName: entry.resolved.displayName,
|
||||
grantedAt: 1,
|
||||
}],
|
||||
...entry.overrides,
|
||||
onPermissionRequest: async () => {
|
||||
permissionCalls++
|
||||
throw new Error('must not prompt')
|
||||
},
|
||||
}),
|
||||
{ app: 'Notes', text: 'allowed' },
|
||||
baseOverrides({ allowedApps: [], userDeniedBundleIds: ['com.apple.Notes'] }),
|
||||
)
|
||||
|
||||
expect(r.isError).toBeFalsy()
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget', 'typeText'])
|
||||
})
|
||||
|
||||
test('the product denylist still blocks a resolved app before mutation', async () => {
|
||||
const { engine, calls } = makeEngine({
|
||||
resolveTarget: async () => ({
|
||||
pid: 900,
|
||||
bundleId: 'com.googlecode.iterm2',
|
||||
displayName: 'iTerm2',
|
||||
}),
|
||||
})
|
||||
const r = await handleToolCall(
|
||||
makeAdapter({ engine }),
|
||||
'type_text',
|
||||
{ app: 'iTerm2', text: 'blocked' },
|
||||
baseOverrides({ allowedApps: [] }),
|
||||
)
|
||||
|
||||
expect(r.isError).toBe(true)
|
||||
expect(r.telemetry?.error_kind).toBe('app_denied')
|
||||
expect(permissionCalls).toBe(0)
|
||||
expect(calls.map(call => call.method)).toEqual(['resolveTarget'])
|
||||
}
|
||||
})
|
||||
|
||||
test('invalid app values return bad_args without resolving a target', async () => {
|
||||
|
||||
+5
-97
@@ -17,7 +17,7 @@
|
||||
* before any mutation, because a bare pid can be recycled between the moment
|
||||
* we resolved it and the moment we act on it.
|
||||
*
|
||||
* **2. Authorization happens after resolution, on the resolved identity.**
|
||||
* **2. Safety checks happen after resolution, on the resolved identity.**
|
||||
* The model names an app loosely ("Notes", a path, a pid). `resolveTarget` is
|
||||
* the single seam that turns that into one real running process; every policy
|
||||
* check then runs against *that* process's bundle id, not against the string
|
||||
@@ -39,10 +39,9 @@
|
||||
* 4. Global CU lock (`overrides.checkCuLock`).
|
||||
* 5. Engine presence.
|
||||
* 6. `resolveTarget` → one running process + its lifetime identity.
|
||||
* 7. Denylists against the RESOLVED identity (policy, intrinsic, user).
|
||||
* 8. Per-app grant (allowlist, else the host's approval dialog).
|
||||
* 9. Proven process lifetime — mutating tools only.
|
||||
* 10. Engine dispatch.
|
||||
* 7. Product/intrinsic denylists against the RESOLVED identity.
|
||||
* 8. Proven process lifetime — mutating tools only.
|
||||
* 9. Engine dispatch.
|
||||
*
|
||||
* The Codex `<app_state>` envelope is framed HERE, in TS, not in Swift
|
||||
* (blueprint §7). Swift renders the inner tree text (the format authority);
|
||||
@@ -54,7 +53,6 @@
|
||||
import type { CallToolResult } from "@modelcontextprotocol/sdk/types.js";
|
||||
|
||||
import {
|
||||
getDefaultTierForApp,
|
||||
getDeniedCategoryForApp,
|
||||
isIntrinsicAppDenied,
|
||||
isPolicyDenied,
|
||||
@@ -69,12 +67,10 @@ import type {
|
||||
ScreenshotResult,
|
||||
} from "./executor.js";
|
||||
import { isSystemKeyCombo } from "./keyBlocklist.js";
|
||||
import { SENTINEL_BUNDLE_IDS } from "./sentinelApps.js";
|
||||
import type {
|
||||
ComputerUseHostAdapter,
|
||||
ComputerUseOverrides,
|
||||
CuGrantFlags,
|
||||
CuPermissionRequest,
|
||||
} from "./types.js";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -903,35 +899,6 @@ function dispatchTarget(
|
||||
return identity ? { pid, expectedProcessIdentity: identity } : { pid };
|
||||
}
|
||||
|
||||
/** Build the approval-dialog request for a single resolved target. */
|
||||
function buildPermissionRequest(
|
||||
resolved: ResolvedAppTarget,
|
||||
requestedApp: string,
|
||||
screenshotFiltering: "native" | "none",
|
||||
): CuPermissionRequest {
|
||||
const bundleId = resolved.bundleId ?? "";
|
||||
const displayName = resolved.displayName ?? requestedApp;
|
||||
return {
|
||||
requestId: `cu-${bundleId || requestedApp}`,
|
||||
reason: `Computer Use needs access to ${displayName}.`,
|
||||
apps: [
|
||||
{
|
||||
requestedName: requestedApp,
|
||||
resolved: {
|
||||
bundleId,
|
||||
displayName,
|
||||
path: resolved.path ?? "",
|
||||
},
|
||||
isSentinel: SENTINEL_BUNDLE_IDS.has(bundleId),
|
||||
alreadyGranted: false,
|
||||
proposedTier: getDefaultTierForApp(bundleId, displayName),
|
||||
},
|
||||
],
|
||||
requestedFlags: {},
|
||||
screenshotFiltering,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Top-level dispatch
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -1049,33 +1016,7 @@ export async function handleToolCall(
|
||||
);
|
||||
if (denied) return errorResult(denied, "app_denied");
|
||||
|
||||
if (
|
||||
resolved.bundleId !== undefined &&
|
||||
overrides.userDeniedBundleIds?.includes(resolved.bundleId)
|
||||
) {
|
||||
return errorResult(
|
||||
`Computer Use is not allowed to use the app '${requestedApp}' — it is ` +
|
||||
"on your deny list. Remove it in Settings if access is needed.",
|
||||
"app_denied",
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Gate 8: per-app grant ─────────────────────────────────────────
|
||||
const authorized = await authorizeResolvedTarget(
|
||||
resolved,
|
||||
requestedApp,
|
||||
overrides,
|
||||
adapter,
|
||||
);
|
||||
if (!authorized) {
|
||||
return errorResult(
|
||||
`Computer Use is not authorized to control '${requestedApp}'. Ask the ` +
|
||||
"user to grant access, then retry.",
|
||||
"app_not_granted",
|
||||
);
|
||||
}
|
||||
|
||||
// ─── Gate 9: proven process lifetime (mutations only) ──────────────
|
||||
// ─── Gate 8: proven process lifetime (mutations only) ──────────────
|
||||
if (request.mutating && !provenIdentity(resolved)) {
|
||||
return errorResult(
|
||||
`Resolving '${requestedApp}' did not prove the running process ` +
|
||||
@@ -1099,39 +1040,6 @@ export async function handleToolCall(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the resolved process is allowed to be driven — already granted, or
|
||||
* granted by the user in response to the approval dialog.
|
||||
*
|
||||
* Fails closed in both no-answer cases: no handler wired, or a handler whose
|
||||
* response doesn't name this exact bundle id.
|
||||
*/
|
||||
async function authorizeResolvedTarget(
|
||||
resolved: ResolvedAppTarget,
|
||||
requestedApp: string,
|
||||
overrides: ComputerUseOverrides,
|
||||
adapter: ComputerUseHostAdapter,
|
||||
): Promise<boolean> {
|
||||
const bundleId = resolved.bundleId;
|
||||
if (bundleId === undefined) return false;
|
||||
|
||||
if (overrides.allowedApps?.some(app => app.bundleId === bundleId)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const request = overrides.onPermissionRequest;
|
||||
if (!request) return false;
|
||||
|
||||
const response = await request(
|
||||
buildPermissionRequest(
|
||||
resolved,
|
||||
requestedApp,
|
||||
adapter.executor.capabilities.screenshotFiltering,
|
||||
),
|
||||
);
|
||||
return response.granted.some(grant => grant.bundleId === bundleId);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Test surface (mirrors the old `_test` export shape; unit tests import this)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
+109
-248
@@ -4,31 +4,15 @@
|
||||
*
|
||||
* Enforcement order, every call:
|
||||
* 1. Kill switch (`adapter.isDisabled()`).
|
||||
* 2. TCC gate (`adapter.ensureOsPermissions()`). `request_access` is
|
||||
* exempted — it threads the ungranted state to the renderer so the
|
||||
* user can grant TCC perms from inside the approval dialog.
|
||||
* 2. OS permission gate (`adapter.ensureOsPermissions()`).
|
||||
* 3. Tool-specific gates (see dispatch table) — ANY exception in a gate
|
||||
* returns a tool error, executor never called.
|
||||
* 4. Executor call.
|
||||
*
|
||||
* For input actions (click/type/key/scroll/drag/move_mouse) the tool-specific
|
||||
* gates are, in order:
|
||||
* a. `prepareForAction` — hide every non-allowlisted app, then defocus us
|
||||
* (battle-tested pre-action sequence from the Vercept acquisition).
|
||||
* Sub-gated via `hideBeforeAction`. After this runs the screenshot is
|
||||
* TRUE (what the
|
||||
* model sees IS what's at each pixel) and we are not keyboard-focused.
|
||||
* b. Frontmost gate — branched by actionKind:
|
||||
* mouse: frontmost ∈ allowlist ∪ {hostBundleId, Finder} → pass.
|
||||
* hostBundleId passes because the executor's
|
||||
* `withClickThrough` bracket makes us click-through.
|
||||
* keyboard: frontmost ∈ allowlist ∪ {Finder} → pass.
|
||||
* hostBundleId → ERROR (safety net — defocus should have
|
||||
* moved us off; if it didn't, typing would go into our
|
||||
* own chat box).
|
||||
* After step (a) this gate fires RARELY — only when something popped
|
||||
* up between prepare and action, or the 5-try hide loop gave up.
|
||||
* Checked FRESH on every call, not cached across calls.
|
||||
* a. `prepareForAction` — platform preparation and host defocus.
|
||||
* b. Resolve the frontmost app and apply product/intrinsic safety tiers.
|
||||
*
|
||||
* For click variants only, AFTER the above gates but BEFORE the executor call:
|
||||
* c. Pixel-validation staleness check (sub-gated).
|
||||
@@ -334,6 +318,10 @@ function tierSatisfies(
|
||||
return tier === "click" || tier === "full";
|
||||
}
|
||||
|
||||
function automaticTier(bundleId: string | undefined, displayName: string): CuAppPermTier {
|
||||
return getDefaultTierForApp(bundleId, displayName);
|
||||
}
|
||||
|
||||
// Appended to every tier_insufficient error. The model may try to route
|
||||
// around the gate (osascript, System Events, cliclick via Bash) — this
|
||||
// closes that door explicitly. Leading space so it concatenates cleanly.
|
||||
@@ -418,45 +406,15 @@ async function runInputActionGates(
|
||||
subGates: CuSubGates,
|
||||
actionKind: CuActionKind,
|
||||
): Promise<CuCallToolResult | null> {
|
||||
// Step A+B — hide non-allowlisted apps + defocus us. Sub-gated. After this
|
||||
// runs, the frontmost gate below becomes a rare edge-case detector (something
|
||||
// popped up between prepare and action) rather than a normal-path blocker.
|
||||
// ALL grant tiers stay visible — visibility is the baseline (tier "read").
|
||||
// Windows shows the full desktop. Preparation gets an empty filter so it may
|
||||
// perform platform bookkeeping without hiding apps based on an allowlist.
|
||||
if (subGates.hideBeforeAction) {
|
||||
const hidden = await adapter.executor.prepareForAction(
|
||||
overrides.allowedApps.map((a) => a.bundleId),
|
||||
overrides.selectedDisplayId,
|
||||
);
|
||||
// Empty-check so we don't spam the callback on every action when nothing
|
||||
// was hidden (the common case after the first action of a turn).
|
||||
if (hidden.length > 0) {
|
||||
overrides.onAppsHidden?.(hidden);
|
||||
}
|
||||
await adapter.executor.prepareForAction([], overrides.selectedDisplayId);
|
||||
}
|
||||
|
||||
// Frontmost gate. Check FRESH on every call.
|
||||
const frontmost = await adapter.executor.getFrontmostApp();
|
||||
|
||||
const tierByBundleId = new Map(
|
||||
overrides.allowedApps.map((a) => [a.bundleId, a.tier] as const),
|
||||
);
|
||||
|
||||
// After handleToolCall's tier backfill, every grant has a concrete tier —
|
||||
// .get() returning undefined means the app is not in the allowlist at all.
|
||||
const frontmostTier = frontmost
|
||||
? tierByBundleId.get(frontmost.bundleId)
|
||||
: undefined;
|
||||
|
||||
// Clipboard guard. Per-action, not per-tool-call — runs for every sub-action
|
||||
// inside computer_batch and teach_step/teach_batch, so clicking into a
|
||||
// click-tier app mid-batch stashes+clears before the next click lands.
|
||||
// Lives here (not in handleToolCall) so deferAcquire tools (request_access,
|
||||
// list_granted_applications), `wait`, and the teach_step blocking-dialog
|
||||
// phase don't trigger a sync — only input actions do.
|
||||
if (subGates.clipboardGuard) {
|
||||
await syncClipboardStash(adapter, overrides, frontmostTier === "click");
|
||||
}
|
||||
|
||||
if (!frontmost) {
|
||||
// Refuse rather than let it through. This path derives its target from
|
||||
// whatever is in front, so "we could not tell what is in front" means we
|
||||
@@ -469,19 +427,36 @@ async function runInputActionGates(
|
||||
// it silently applies to nothing.
|
||||
return errorResult(
|
||||
"The foreground application could not be identified. Refusing input " +
|
||||
"until a granted application is brought to the front.",
|
||||
"until a supported application is brought to the front.",
|
||||
"state_conflict",
|
||||
);
|
||||
}
|
||||
|
||||
const { hostBundleId } = adapter.executor.capabilities;
|
||||
|
||||
if (frontmostTier !== undefined) {
|
||||
if (tierSatisfies(frontmostTier, actionKind)) return null;
|
||||
// In the allowlist but tier doesn't cover this action. Tailor the
|
||||
// guidance to the actual tier — at "read", suggesting left_click or Bash
|
||||
// is wrong (nothing is allowed; use Chrome MCP). At "click", the
|
||||
// mouse_full/keyboard-specific messages apply.
|
||||
if (frontmost.bundleId === hostBundleId) {
|
||||
if (actionKind !== "keyboard") return null;
|
||||
return errorResult(
|
||||
"Claude's own window still has keyboard focus. Click on the target " +
|
||||
"application first so typing cannot land in the chat box.",
|
||||
"state_conflict",
|
||||
);
|
||||
}
|
||||
|
||||
if (isPolicyDenied(frontmost.bundleId, frontmost.displayName)) {
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is not supported by Computer Use for product-safety reasons.`,
|
||||
"app_denied",
|
||||
);
|
||||
}
|
||||
|
||||
const frontmostTier = automaticTier(frontmost.bundleId, frontmost.displayName);
|
||||
|
||||
if (subGates.clipboardGuard) {
|
||||
await syncClipboardStash(adapter, overrides, frontmostTier === "click");
|
||||
}
|
||||
|
||||
if (!tierSatisfies(frontmostTier, actionKind)) {
|
||||
if (frontmostTier === "read") {
|
||||
// tier "read" is not category-unique (browser AND trading map to it) —
|
||||
// re-look-up so the CiC hint only shows for actual browsers.
|
||||
@@ -489,7 +464,7 @@ async function runInputActionGates(
|
||||
getDeniedCategoryForApp(frontmost.bundleId, frontmost.displayName) ===
|
||||
"browser";
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is granted at tier "read" — ` +
|
||||
`"${frontmost.displayName}" is restricted to tier "read" — ` +
|
||||
`visible in screenshots only, no clicks or typing.` +
|
||||
(isBrowser
|
||||
? " Use the Claude-in-Chrome MCP for browser interaction (tools " +
|
||||
@@ -501,10 +476,9 @@ async function runInputActionGates(
|
||||
"tier_insufficient",
|
||||
);
|
||||
}
|
||||
// frontmostTier === "click" (tier === "full" would have passed tierSatisfies)
|
||||
if (actionKind === "keyboard") {
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is granted at tier "click" — ` +
|
||||
`"${frontmost.displayName}" is restricted to tier "click" — ` +
|
||||
`typing, key presses, and paste require tier "full". The keys ` +
|
||||
`would go to this app's text fields or integrated terminal. To ` +
|
||||
`type into a different app, click it first to bring it forward. ` +
|
||||
@@ -514,7 +488,7 @@ async function runInputActionGates(
|
||||
}
|
||||
// actionKind === "mouse_full" ("mouse" and "mouse_position" pass at "click")
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is granted at tier "click" — ` +
|
||||
`"${frontmost.displayName}" is restricted to tier "click" — ` +
|
||||
`right-click, middle-click, and clicks with modifier keys require ` +
|
||||
`tier "full". Right-click opens a context menu with Paste/Cut, and ` +
|
||||
`modifier chords fire as keystrokes before the click. Plain ` +
|
||||
@@ -522,32 +496,8 @@ async function runInputActionGates(
|
||||
"tier_insufficient",
|
||||
);
|
||||
}
|
||||
// Finder is never-hide, always allowed.
|
||||
if (frontmost.bundleId === FINDER_BUNDLE_ID) return null;
|
||||
|
||||
if (frontmost.bundleId === hostBundleId) {
|
||||
if (actionKind !== "keyboard") {
|
||||
// mouse and mouse_full are both click events — click-through works.
|
||||
// We're click-through (executor's withClickThrough). Pass.
|
||||
return null;
|
||||
}
|
||||
// Keyboard safety net — defocus (prepareForAction step B) should have
|
||||
// moved us off. If we're still here, typing would go to our chat box.
|
||||
return errorResult(
|
||||
"Claude's own window still has keyboard focus. This should not happen " +
|
||||
"after the pre-action defocus. Click on the target application first.",
|
||||
"state_conflict",
|
||||
);
|
||||
}
|
||||
|
||||
// Non-allowlisted, non-us, non-Finder. RARE after the hide loop — means
|
||||
// something popped up between prepare and action, or the 5-try loop gave up.
|
||||
return errorResult(
|
||||
`"${frontmost.displayName}" is not in the allowed applications and is ` +
|
||||
`currently in front. Take a new screenshot — it may have appeared ` +
|
||||
`since your last one.`,
|
||||
"app_not_granted",
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -575,28 +525,18 @@ async function runHitTestGate(
|
||||
const target = await adapter.executor.appUnderPoint(x, y);
|
||||
if (!target) return null; // desktop / nothing under point / platform no-op
|
||||
|
||||
// Finder (desktop, file dialogs) is always clickable — same exemption as
|
||||
// runInputActionGates. Our own overlay is filtered by Swift (pid != self).
|
||||
// Finder (desktop, file dialogs) and our click-through overlay are valid.
|
||||
if (target.bundleId === FINDER_BUNDLE_ID) return null;
|
||||
|
||||
const tierByBundleId = new Map(
|
||||
overrides.allowedApps.map((a) => [a.bundleId, a.tier] as const),
|
||||
);
|
||||
|
||||
if (!tierByBundleId.has(target.bundleId)) {
|
||||
// Not in the allowlist at all. The frontmost check would catch this if
|
||||
// the target were frontmost, but here a different app is in front. This
|
||||
// is the "something popped up" edge case — a new window appeared between
|
||||
// screenshot and click, or a background app's window overlaps the target.
|
||||
if (target.bundleId === adapter.executor.capabilities.hostBundleId) return null;
|
||||
if (isPolicyDenied(target.bundleId, target.displayName)) {
|
||||
return errorResult(
|
||||
`Click at these coordinates would land on "${target.displayName}", ` +
|
||||
`which is not in the allowed applications. Take a fresh screenshot ` +
|
||||
`to see the current window layout.`,
|
||||
"app_not_granted",
|
||||
"which Computer Use does not support for product-safety reasons.",
|
||||
"app_denied",
|
||||
);
|
||||
}
|
||||
|
||||
const targetTier = tierByBundleId.get(target.bundleId);
|
||||
const targetTier = automaticTier(target.bundleId, target.displayName);
|
||||
|
||||
// Frontmost-based sync (runInputActionGates) misses the case where
|
||||
// the click lands on a NON-FRONTMOST click-tier window. Re-sync by
|
||||
@@ -615,7 +555,7 @@ async function runHitTestGate(
|
||||
if (actionKind === "mouse_full" && targetTier === "click") {
|
||||
return errorResult(
|
||||
`Click at these coordinates would land on "${target.displayName}", ` +
|
||||
`which is granted at tier "click" — right-click, middle-click, and ` +
|
||||
`which is restricted to tier "click" — right-click, middle-click, and ` +
|
||||
`clicks with modifier keys require tier "full" (they can Paste via ` +
|
||||
`the context menu or fire modifier-chord keystrokes). Plain ` +
|
||||
`left_click is allowed here.` + TIER_ANTI_SUBVERSION,
|
||||
@@ -626,7 +566,7 @@ async function runHitTestGate(
|
||||
getDeniedCategoryForApp(target.bundleId, target.displayName) === "browser";
|
||||
return errorResult(
|
||||
`Click at these coordinates would land on "${target.displayName}", ` +
|
||||
`which is granted at tier "read" (screenshots only, no interaction). ` +
|
||||
`which is restricted to tier "read" (screenshots only, no interaction). ` +
|
||||
(isBrowser
|
||||
? "Use the Claude-in-Chrome MCP for browser interaction."
|
||||
: "Ask the user to take any actions in this app themselves.") +
|
||||
@@ -1686,13 +1626,7 @@ async function executeTeachStep(
|
||||
}
|
||||
|
||||
if (subGates.hideBeforeAction) {
|
||||
const hidden = await adapter.executor.prepareForAction(
|
||||
overrides.allowedApps.map((a) => a.bundleId),
|
||||
overrides.selectedDisplayId,
|
||||
);
|
||||
if (hidden.length > 0) {
|
||||
overrides.onAppsHidden?.(hidden);
|
||||
}
|
||||
await adapter.executor.prepareForAction([], overrides.selectedDisplayId);
|
||||
}
|
||||
|
||||
const stepSubGates: CuSubGates = {
|
||||
@@ -1921,29 +1855,6 @@ async function handleTeachBatch(
|
||||
return appendTeachScreenshot(resultJson, adapter, overrides, subGates);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the hidden-apps note that accompanies a screenshot. Tells the model
|
||||
* which apps got hidden (not in allowlist) and how to add them. Returns
|
||||
* undefined when nothing was hidden since the last screenshot.
|
||||
*/
|
||||
async function buildHiddenNote(
|
||||
adapter: ComputerUseHostAdapter,
|
||||
hiddenSinceLastSeen: string[],
|
||||
): Promise<string | undefined> {
|
||||
if (hiddenSinceLastSeen.length === 0) return undefined;
|
||||
const running = await adapter.executor.listRunningApps();
|
||||
const nameOf = new Map(running.map((a) => [a.bundleId, a.displayName]));
|
||||
const names = hiddenSinceLastSeen.map((id) => nameOf.get(id) ?? id);
|
||||
const list = names.map((n) => `"${n}"`).join(", ");
|
||||
const one = names.length === 1;
|
||||
return (
|
||||
`${list} ${one ? "was" : "were"} open and got hidden before this screenshot ` +
|
||||
`(not in the session allowlist). If a previous action was meant to open ` +
|
||||
`${one ? "it" : "one of them"}, that's why you don't see it — call ` +
|
||||
`request_access to add ${one ? "it" : "them"} to the allowlist.`
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Assign a human-readable label to each display. Falls back to `display N`
|
||||
* when NSScreen.localizedName is undefined; disambiguates identical labels
|
||||
@@ -2033,14 +1944,6 @@ async function handleScreenshot(
|
||||
overrides: ComputerUseOverrides,
|
||||
subGates: CuSubGates,
|
||||
): Promise<CuCallToolResult> {
|
||||
// §2 — empty allowlist → tool error, no screenshot.
|
||||
if (overrides.allowedApps.length === 0) {
|
||||
return errorResult(
|
||||
"No applications are granted for this session. Call request_access first.",
|
||||
"allowlist_empty",
|
||||
);
|
||||
}
|
||||
|
||||
// Atomic resolve→prepare→capture (one Swift call, no scheduler gap).
|
||||
// Off → fall through to separate-calls path below.
|
||||
if (subGates.autoTargetDisplay) {
|
||||
@@ -2049,8 +1952,8 @@ async function handleScreenshot(
|
||||
// Otherwise sticky display: only auto-resolve when the allowed-app
|
||||
// set has changed since the display was last resolved. Prevents the
|
||||
// resolver yanking the display on every screenshot.
|
||||
const allowedBundleIds = overrides.allowedApps.map((a) => a.bundleId);
|
||||
const currentAppSetKey = allowedBundleIds.slice().sort().join(",");
|
||||
const allowedBundleIds: string[] = [];
|
||||
const currentAppSetKey = "all-supported-apps";
|
||||
const appSetChanged = currentAppSetKey !== overrides.displayResolvedForApps;
|
||||
const autoResolve = !overrides.displayPinnedByModel && appSetChanged;
|
||||
|
||||
@@ -2096,24 +1999,11 @@ async function handleScreenshot(
|
||||
overrides.onDisplayResolvedForApps?.(currentAppSetKey);
|
||||
}
|
||||
|
||||
// Report hidden apps only when the model has already seen the screen.
|
||||
let hiddenSinceLastSeen: string[] = [];
|
||||
if (overrides.lastScreenshot !== undefined) {
|
||||
hiddenSinceLastSeen = result.hidden;
|
||||
}
|
||||
if (result.hidden.length > 0) {
|
||||
overrides.onAppsHidden?.(result.hidden);
|
||||
}
|
||||
|
||||
// Partial-success case: hide succeeded, capture failed (SCK perm
|
||||
// revoked mid-session). onAppsHidden fired above so auto-unhide will
|
||||
// restore hidden apps at turn end. Now surface the error to the model.
|
||||
// Partial-success case: capture failed after preparation.
|
||||
if (result.captureError !== undefined) {
|
||||
return errorResult(result.captureError, "capture_failed");
|
||||
}
|
||||
|
||||
const hiddenNote = await buildHiddenNote(adapter, hiddenSinceLastSeen);
|
||||
|
||||
// Cherry-pick — don't spread `result` (would leak resolver fields into lastScreenshot).
|
||||
const shot: ScreenshotResult = {
|
||||
base64: result.base64,
|
||||
@@ -2136,7 +2026,6 @@ async function handleScreenshot(
|
||||
return {
|
||||
content: [
|
||||
...(monitorNote ? [{ type: "text" as const, text: monitorNote }] : []),
|
||||
...(hiddenNote ? [{ type: "text" as const, text: hiddenNote }] : []),
|
||||
{
|
||||
type: "image",
|
||||
data: shot.base64,
|
||||
@@ -2147,52 +2036,18 @@ async function handleScreenshot(
|
||||
};
|
||||
}
|
||||
|
||||
// Same hide+defocus sequence as input actions. Screenshot needs hide too
|
||||
// — if a non-allowlisted app is on top, SCContentFilter would composite it
|
||||
// out, but the pixels BELOW it are what the model would see, and those are
|
||||
// NOT what's actually there. Hiding first makes the screenshot TRUE.
|
||||
let hiddenSinceLastSeen: string[] = [];
|
||||
// Keep the platform preparation hook, but do not filter or hide applications.
|
||||
if (subGates.hideBeforeAction) {
|
||||
const hidden = await adapter.executor.prepareForAction(
|
||||
overrides.allowedApps.map((a) => a.bundleId),
|
||||
overrides.selectedDisplayId,
|
||||
);
|
||||
// "Something appeared since the model last looked." Report whenever:
|
||||
// (a) prepare hid something AND
|
||||
// (b) the model has ALREADY SEEN the screen (lastScreenshot is set).
|
||||
//
|
||||
// (b) is the discriminator that silences the first screenshot's
|
||||
// expected-noise hide. NOT a delta against a cumulative set — that was
|
||||
// the earlier bug: cuHiddenDuringTurn only grows, so once Preview is in
|
||||
// it (from the first screenshot's hide), subsequent re-hides of Preview
|
||||
// delta to zero. The double-click → Preview opens → re-hide → silent
|
||||
// loop never breaks.
|
||||
//
|
||||
// With this check: every re-hide fires. If the model loops "click → file
|
||||
// opens in Preview → screenshot → Preview hidden", it gets told EVERY
|
||||
// time. Eventually it'll request_access for Preview (or give up).
|
||||
//
|
||||
// False positive: user alt-tabs mid-turn → Safari re-hidden → reported.
|
||||
// Rare, and "Safari appeared" is at worst mild noise — far better than
|
||||
// the false-negative of never explaining why the file vanished.
|
||||
if (overrides.lastScreenshot !== undefined) {
|
||||
hiddenSinceLastSeen = hidden;
|
||||
}
|
||||
if (hidden.length > 0) {
|
||||
overrides.onAppsHidden?.(hidden);
|
||||
}
|
||||
await adapter.executor.prepareForAction([], overrides.selectedDisplayId);
|
||||
}
|
||||
|
||||
const allowedBundleIds = overrides.allowedApps.map((g) => g.bundleId);
|
||||
const shot = await takeScreenshotWithRetry(
|
||||
adapter.executor,
|
||||
allowedBundleIds,
|
||||
[],
|
||||
adapter.logger,
|
||||
overrides.selectedDisplayId,
|
||||
);
|
||||
|
||||
const hiddenNote = await buildHiddenNote(adapter, hiddenSinceLastSeen);
|
||||
|
||||
const monitorNote = await buildMonitorNote(
|
||||
adapter,
|
||||
shot.displayId,
|
||||
@@ -2203,7 +2058,6 @@ async function handleScreenshot(
|
||||
return {
|
||||
content: [
|
||||
...(monitorNote ? [{ type: "text" as const, text: monitorNote }] : []),
|
||||
...(hiddenNote ? [{ type: "text" as const, text: hiddenNote }] : []),
|
||||
{
|
||||
type: "image",
|
||||
data: shot.base64,
|
||||
@@ -2275,12 +2129,11 @@ async function handleZoom(
|
||||
h: (y1 - y0) * ratioY,
|
||||
};
|
||||
|
||||
const allowedIds = overrides.allowedApps.map((g) => g.bundleId);
|
||||
// Crop from the same display as lastScreenshot so the zoom region
|
||||
// matches the image the model is reading coords from.
|
||||
const zoomed = await adapter.executor.zoom(
|
||||
regionLogical,
|
||||
allowedIds,
|
||||
[],
|
||||
last.displayId,
|
||||
);
|
||||
|
||||
@@ -2333,7 +2186,7 @@ async function handleClickVariant(
|
||||
) {
|
||||
return errorResult(
|
||||
`The modifier chord "${args.text}" would fire a system shortcut. ` +
|
||||
"Request the systemKeyCombos grant flag via request_access, or use " +
|
||||
"Enable Computer Use again to accept system-shortcut access, or use " +
|
||||
"only modifier keys (shift, ctrl, alt, cmd) in the text parameter.",
|
||||
"grant_flag_required",
|
||||
);
|
||||
@@ -2379,12 +2232,11 @@ async function handleClickVariant(
|
||||
async () => {
|
||||
// The fresh screenshot for validation uses the SAME allow-set as
|
||||
// the model's last screenshot did, so we compare like with like.
|
||||
const allowedIds = overrides.allowedApps.map((g) => g.bundleId);
|
||||
try {
|
||||
// Fresh shot must match lastScreenshot's display, not the current
|
||||
// selection — pixel-compare is against the model's last image.
|
||||
return await adapter.executor.screenshot({
|
||||
allowedBundleIds: allowedIds,
|
||||
allowedBundleIds: [],
|
||||
displayId: overrides.lastScreenshot?.displayId,
|
||||
});
|
||||
} catch {
|
||||
@@ -2549,7 +2401,7 @@ async function handleKey(
|
||||
!overrides.grantFlags.systemKeyCombos
|
||||
) {
|
||||
return errorResult(
|
||||
`"${keySequence}" is a system-level shortcut. Request the \`systemKeyCombos\` grant via request_access to use it.`,
|
||||
`"${keySequence}" is a system-level shortcut. Re-enable Computer Use and accept the risk notice to use it.`,
|
||||
"grant_flag_required",
|
||||
);
|
||||
}
|
||||
@@ -2789,27 +2641,27 @@ async function handleOpenApplication(
|
||||
const app = requireString(args, "app");
|
||||
if (app instanceof Error) return errorResult(app.message, "bad_args");
|
||||
|
||||
// Resolve display-name → bundle ID. Same logic as request_access.
|
||||
const allowed = new Set(overrides.allowedApps.map((g) => g.bundleId));
|
||||
let targetBundleId: string | undefined;
|
||||
|
||||
if (looksLikeBundleId(app) && allowed.has(app)) {
|
||||
targetBundleId = app;
|
||||
} else {
|
||||
// Try display name → bundle ID, but ONLY against the allowlist itself.
|
||||
// Avoids paying the listInstalledApps() cost on the hot path and is
|
||||
// arguably more correct: if the user granted "Slack", the model asking
|
||||
// to open "Slack" should match THAT grant.
|
||||
const match = overrides.allowedApps.find(
|
||||
(g) => g.displayName.toLowerCase() === app.toLowerCase(),
|
||||
// Resolve only against the helper's installed-app inventory. Never pass an
|
||||
// arbitrary model string to the Windows shell.
|
||||
const installed = await adapter.executor.listInstalledApps();
|
||||
const wanted = app.trim().toLowerCase();
|
||||
const match = installed.find(
|
||||
candidate =>
|
||||
candidate.bundleId.toLowerCase() === wanted
|
||||
|| candidate.displayName.toLowerCase() === wanted,
|
||||
);
|
||||
|
||||
if (!match) {
|
||||
return errorResult(
|
||||
`"${app}" was not found in the installed application inventory.`,
|
||||
"bad_args",
|
||||
);
|
||||
targetBundleId = match?.bundleId;
|
||||
}
|
||||
|
||||
if (!targetBundleId || !allowed.has(targetBundleId)) {
|
||||
if (isPolicyDenied(match.bundleId, match.displayName)) {
|
||||
return errorResult(
|
||||
`"${app}" is not granted for this session. Call request_access first.`,
|
||||
"app_not_granted",
|
||||
`"${match.displayName}" is not supported by Computer Use for product-safety reasons.`,
|
||||
"app_denied",
|
||||
);
|
||||
}
|
||||
|
||||
@@ -2817,7 +2669,7 @@ async function handleOpenApplication(
|
||||
// what tier "read" enables (you need it on screen to screenshot it). The
|
||||
// tier gates on click/type catch any follow-up interaction.
|
||||
|
||||
await adapter.executor.openApp(targetBundleId);
|
||||
await adapter.executor.openApp(match.bundleId);
|
||||
|
||||
// On multi-monitor setups, macOS may place the opened window on a monitor
|
||||
// the resolver won't pick (e.g. Claude + another allowed app are co-located
|
||||
@@ -2920,7 +2772,7 @@ async function handleReadClipboard(
|
||||
): Promise<CuCallToolResult> {
|
||||
if (!overrides.grantFlags.clipboardRead) {
|
||||
return errorResult(
|
||||
"Clipboard read is not granted. Request `clipboardRead` via request_access.",
|
||||
"Clipboard read is disabled. Re-enable Computer Use and accept the risk notice.",
|
||||
"grant_flag_required",
|
||||
);
|
||||
}
|
||||
@@ -2930,11 +2782,8 @@ async function handleReadClipboard(
|
||||
// (same as what the app's own Paste would see).
|
||||
if (subGates.clipboardGuard) {
|
||||
const frontmost = await adapter.executor.getFrontmostApp();
|
||||
const tierByBundleId = new Map(
|
||||
overrides.allowedApps.map((a) => [a.bundleId, a.tier] as const),
|
||||
);
|
||||
const frontmostTier = frontmost
|
||||
? tierByBundleId.get(frontmost.bundleId)
|
||||
? automaticTier(frontmost.bundleId, frontmost.displayName)
|
||||
: undefined;
|
||||
await syncClipboardStash(adapter, overrides, frontmostTier === "click");
|
||||
}
|
||||
@@ -2953,7 +2802,7 @@ async function handleWriteClipboard(
|
||||
): Promise<CuCallToolResult> {
|
||||
if (!overrides.grantFlags.clipboardWrite) {
|
||||
return errorResult(
|
||||
"Clipboard write is not granted. Request `clipboardWrite` via request_access.",
|
||||
"Clipboard write is disabled. Re-enable Computer Use and accept the risk notice.",
|
||||
"grant_flag_required",
|
||||
);
|
||||
}
|
||||
@@ -2962,11 +2811,8 @@ async function handleWriteClipboard(
|
||||
|
||||
if (subGates.clipboardGuard) {
|
||||
const frontmost = await adapter.executor.getFrontmostApp();
|
||||
const tierByBundleId = new Map(
|
||||
overrides.allowedApps.map((a) => [a.bundleId, a.tier] as const),
|
||||
);
|
||||
const frontmostTier = frontmost
|
||||
? tierByBundleId.get(frontmost.bundleId)
|
||||
? automaticTier(frontmost.bundleId, frontmost.displayName)
|
||||
: undefined;
|
||||
|
||||
// Defense-in-depth for the clipboardGuard bypass: write_clipboard +
|
||||
@@ -3107,7 +2953,7 @@ async function handleHoldKey(
|
||||
!overrides.grantFlags.systemKeyCombos
|
||||
) {
|
||||
return errorResult(
|
||||
`"${text}" is a system-level shortcut. Request the \`systemKeyCombos\` grant via request_access to use it.`,
|
||||
`"${text}" is a system-level shortcut. Re-enable Computer Use and accept the risk notice to use it.`,
|
||||
"grant_flag_required",
|
||||
);
|
||||
}
|
||||
@@ -3316,13 +3162,7 @@ async function handleComputerBatch(
|
||||
// prepareForAction ONCE. After this, inner dispatches skip it via
|
||||
// hideBeforeAction:false.
|
||||
if (subGates.hideBeforeAction) {
|
||||
const hidden = await adapter.executor.prepareForAction(
|
||||
overrides.allowedApps.map((a) => a.bundleId),
|
||||
overrides.selectedDisplayId,
|
||||
);
|
||||
if (hidden.length > 0) {
|
||||
overrides.onAppsHidden?.(hidden);
|
||||
}
|
||||
await adapter.executor.prepareForAction([], overrides.selectedDisplayId);
|
||||
}
|
||||
|
||||
// Inner actions: skip prepare (already ran), skip pixelCompare (stale by
|
||||
@@ -3642,16 +3482,37 @@ export async function handleToolCall(
|
||||
// ANY exception below → tool error, executor never left in a half-called
|
||||
// state. Explicit inversion of the prior `catch → return true` fail-open.
|
||||
try {
|
||||
// request_access / request_teach_access: need tccState thread-through;
|
||||
// dispatchAction never sees them (not batchable).
|
||||
// Compatibility for a stale client that cached the removed app-permission
|
||||
// tool. Enabling Computer Use is now the authorization boundary, so this
|
||||
// call must never open an app-by-app approval dialog.
|
||||
if (name === "request_access") {
|
||||
return okJson({
|
||||
enabled: true,
|
||||
appAuthorization: "all_supported_apps",
|
||||
screenshotFiltering: adapter.executor.capabilities.screenshotFiltering,
|
||||
});
|
||||
}
|
||||
|
||||
// Teach mode still needs an explicit tool transition because it hides the
|
||||
// main window, but it no longer asks for per-app permission.
|
||||
if (name === "request_teach_access") {
|
||||
if (overrides.getTeachModeActive?.()) {
|
||||
return errorResult("Teach mode is already active.", "teach_mode_conflict");
|
||||
}
|
||||
const reason = requireString(a, "reason");
|
||||
if (reason instanceof Error) return errorResult(reason.message, "bad_args");
|
||||
if (!Array.isArray(a.apps) || !a.apps.every(app => typeof app === "string")) {
|
||||
return errorResult('"apps" must be an array of strings.', "bad_args");
|
||||
}
|
||||
if (!overrides.onTeachModeActivated || !overrides.onTeachStep) {
|
||||
return errorResult("Teach mode is not available in this session.", "feature_unavailable");
|
||||
}
|
||||
overrides.onTeachModeActivated();
|
||||
return okJson({ teachModeActive: true, reason });
|
||||
}
|
||||
|
||||
// teach_step: blocking UI tool, also not batchable; needs subGates for
|
||||
// its action-execution phase.
|
||||
if (name === "request_access") {
|
||||
return await handleRequestAccess(adapter, a, overrides, tccState);
|
||||
}
|
||||
if (name === "request_teach_access") {
|
||||
return await handleRequestTeachAccess(adapter, a, overrides, tccState);
|
||||
}
|
||||
if (name === "teach_step") {
|
||||
return await handleTeachStep(adapter, a, overrides, subGates);
|
||||
}
|
||||
|
||||
+12
-68
@@ -29,7 +29,7 @@ const COORD_DESC: Record<CoordinateMode, { x: string; y: string }> = {
|
||||
};
|
||||
|
||||
const FRONTMOST_GATE_DESC =
|
||||
"The frontmost application must be in the session allowlist at the time of this call, or this tool returns an error and does nothing.";
|
||||
"The target is resolved at call time and remains subject to product safety restrictions.";
|
||||
|
||||
/**
|
||||
* Item schema for the `actions` array in `computer_batch`, `teach_step`, and
|
||||
@@ -110,7 +110,7 @@ const BATCH_ACTION_ITEM_SCHEMA = {
|
||||
* -call time. Both should read the same frozen-at-load gate constant.
|
||||
*
|
||||
* `installedAppNames` — optional pre-sanitized list of app display names to
|
||||
* enumerate in the `request_access` description. The caller is responsible
|
||||
* enumerate in platform-specific tool descriptions. The caller is responsible
|
||||
* for sanitization (length cap, character allowlist, sort, count cap) —
|
||||
* this function just splices the list into the description verbatim. Omit
|
||||
* to fall back to the generic "display names or bundle IDs" wording.
|
||||
@@ -127,9 +127,7 @@ export function buildComputerUseTools(
|
||||
): Tool[] {
|
||||
const coord = COORD_DESC[coordinateMode];
|
||||
|
||||
// Shared hint suffix for BOTH request_access and request_teach_access —
|
||||
// they use the same resolveRequestedApps path, so the model should get
|
||||
// the same enumeration for both.
|
||||
// Teach mode uses this optional list to help the model name installed apps.
|
||||
const installedAppsHint =
|
||||
installedAppNames && installedAppNames.length > 0
|
||||
? ` Available applications on this machine: ${installedAppNames.join(", ")}.`
|
||||
@@ -153,57 +151,15 @@ export function buildComputerUseTools(
|
||||
|
||||
const screenshotDesc =
|
||||
caps.screenshotFiltering === "native"
|
||||
? "Take a screenshot of the primary display. Applications not in the session allowlist are excluded at the compositor level — only granted apps and the desktop are visible."
|
||||
: "Take a screenshot of the primary display. On this platform, screenshots are NOT filtered — all open windows are visible. Input actions targeting apps not in the session allowlist are rejected.";
|
||||
? "Take a screenshot of the primary display."
|
||||
: "Take a screenshot of the primary display. On this platform, screenshots are NOT filtered — all open windows are visible.";
|
||||
|
||||
return [
|
||||
{
|
||||
name: "request_access",
|
||||
description:
|
||||
"Request user permission to control a set of applications for this session. Must be called before any other tool in this server. " +
|
||||
"The user sees a single dialog listing all requested apps and either allows the whole set or denies it. " +
|
||||
"Call this again mid-session to add more apps; previously granted apps remain granted. " +
|
||||
"Returns the granted apps, denied apps, and screenshot filtering capability.",
|
||||
inputSchema: {
|
||||
type: "object" as const,
|
||||
properties: {
|
||||
apps: {
|
||||
type: "array",
|
||||
items: { type: "string" },
|
||||
description:
|
||||
"Application display names (e.g. \"Slack\", \"Calendar\") or bundle identifiers (e.g. \"com.tinyspeck.slackmacgap\"). Display names are resolved case-insensitively against installed apps." +
|
||||
installedAppsHint,
|
||||
},
|
||||
reason: {
|
||||
type: "string",
|
||||
description:
|
||||
"One-sentence explanation shown to the user in the approval dialog. Explain the task, not the mechanism.",
|
||||
},
|
||||
clipboardRead: {
|
||||
type: "boolean",
|
||||
description:
|
||||
"Also request permission to read the user's clipboard (separate checkbox in the dialog).",
|
||||
},
|
||||
clipboardWrite: {
|
||||
type: "boolean",
|
||||
description:
|
||||
"Also request permission to write the user's clipboard. When granted, multi-line `type` calls use the clipboard fast path.",
|
||||
},
|
||||
systemKeyCombos: {
|
||||
type: "boolean",
|
||||
description:
|
||||
"Also request permission to send system-level key combos (quit app, switch app, lock screen). Without this, those specific combos are blocked.",
|
||||
},
|
||||
},
|
||||
required: ["apps", "reason"],
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: "screenshot",
|
||||
description:
|
||||
screenshotDesc +
|
||||
" Returns an error if the allowlist is empty. The returned image is what subsequent click coordinates are relative to.",
|
||||
" The returned image is what subsequent click coordinates are relative to.",
|
||||
inputSchema: {
|
||||
type: "object" as const,
|
||||
properties: {
|
||||
@@ -400,7 +356,7 @@ export function buildComputerUseTools(
|
||||
{
|
||||
name: "open_application",
|
||||
description:
|
||||
"Bring an application to the front, launching it if necessary. The target application must already be in the session allowlist — call request_access first.",
|
||||
"Bring an installed application to the front, launching it if necessary. The application is resolved against the installed-app inventory before launch.",
|
||||
inputSchema: {
|
||||
type: "object" as const,
|
||||
properties: {
|
||||
@@ -437,17 +393,6 @@ export function buildComputerUseTools(
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: "list_granted_applications",
|
||||
description:
|
||||
"List the applications currently in the session allowlist, plus the active grant flags and coordinate mode. No side effects.",
|
||||
inputSchema: {
|
||||
type: "object" as const,
|
||||
properties: {},
|
||||
required: [],
|
||||
},
|
||||
},
|
||||
|
||||
{
|
||||
name: "read_clipboard",
|
||||
description:
|
||||
@@ -575,7 +520,7 @@ export function buildComputerUseTools(
|
||||
* takes `coord` so `teach_step.anchor`'s description uses the same
|
||||
* frozen coordinate-mode phrasing as click coords, and `installedAppsHint`
|
||||
* so `request_teach_access.apps` gets the same enumeration as
|
||||
* `request_access.apps` (same resolution path → same hint).
|
||||
* installed app references (same inventory → same hint).
|
||||
*/
|
||||
function buildTeachTools(
|
||||
coord: { x: string; y: string },
|
||||
@@ -620,12 +565,11 @@ function buildTeachTools(
|
||||
{
|
||||
name: "request_teach_access",
|
||||
description:
|
||||
"Request permission to guide the user through a task step-by-step with on-screen tooltips. " +
|
||||
"Use this INSTEAD OF request_access when the user wants to LEARN how to do something " +
|
||||
"Start guiding the user through a task step-by-step with on-screen tooltips. " +
|
||||
"Use this when the user wants to LEARN how to do something " +
|
||||
'(phrases like "teach me", "walk me through", "show me how", "help me learn"). ' +
|
||||
"On approval the main Claude window hides and a fullscreen tooltip overlay appears. " +
|
||||
"The main Claude window hides and a fullscreen tooltip overlay appears. " +
|
||||
"You then call teach_step repeatedly; each call shows one tooltip and waits for the user to click Next. " +
|
||||
"Same app-allowlist semantics as request_access, but no clipboard/system-key flags. " +
|
||||
"Teach mode ends automatically when your turn ends.",
|
||||
inputSchema: {
|
||||
type: "object" as const,
|
||||
@@ -640,7 +584,7 @@ function buildTeachTools(
|
||||
reason: {
|
||||
type: "string",
|
||||
description:
|
||||
'What you will be teaching. Shown in the approval dialog as "Claude wants to guide you through {reason}". Keep it short and task-focused.',
|
||||
"What you will be teaching. Keep it short and task-focused.",
|
||||
},
|
||||
},
|
||||
required: ["apps", "reason"],
|
||||
|
||||
Reference in New Issue
Block a user