fix(runtime): converge background tasks on Stop and runtime loss (#1441)

* fix(server): reap background shell tasks on Stop and after runtime exit

A session's background shell task (Bash `run_in_background`, Dream, workflow)
is tracked as a non-Agent task, and nothing on the server ever bounded or
reaped one. A user Stop only interrupted the foreground turn and the Agent
tasks, so the shell kept running after the user had asked everything to stop.
A client disconnect let the CLI live forever, because the disconnect watcher
waits for `hasActiveBackgroundTasks` to clear and a task that never emits a
terminal notification never clears it. A hard runtime death published no
terminal state at all, so a later reconnect re-hydrated a ghost "Running"
entry that could never reach a terminal state.

This closes all three from one file of server bookkeeping:

- `handleStopGeneration` now reaps the session's non-Agent background tasks
  through the same `requestControl` stop path the panel's per-task Stop uses,
  and the 3-second force-kill guard watches them too. Programmatic stops
  (turn stop, runtime-config restart) keep the previous narrow behavior.
- A new 31-minute ceiling bounds a session that a disconnected client keeps
  alive through background tasks alone; when it elapses the shared runtime is
  stopped and terminal bookends are published. A reconnect clears the ceiling.
- `close()` and `closeStoppedAgentsAfterRuntimeExit` publish terminal bookends
  whenever the runtime is already gone but task records remain, instead of
  leaving them to be re-hydrated as "Running".

The new timer registry is released by `cleanupSessionRuntimeState` and
classified as `cleared` in the session-state cleanup guard, so a ceiling
scheduled for a session cannot outlive that session.

会话的后台 shell 任务(Bash `run_in_background`、Dream、workflow)登记为非 Agent 任务,
服务端一直没有收掉它们的路径:按停止键只断前台轮和 Agent 任务,shell 还在跑;客户端断开后
CLI 因为 `hasActiveBackgroundTasks` 恒为真而一直存活;运行时崩溃又不写终态,重连后又
变成幽灵 Running。这次在 handler 一处收掉这三种:

- 按停止键时一并停掉本会话的非 Agent 后台任务,走面板单任务 Stop 同一个 stop 通路,
  三秒强杀闸也一并盯住它们;程序化停止(停轮、重启换配置)仍走原来的窄路径。
- 新增 31 分钟硬上限:没有客户端、只靠后台任务续命的会话,到期就停掉共享运行时并写终态;
  重连则撤销该计时。
- 运行时已经没了但任务记录还在时,`close()` 与 `closeStoppedAgentsAfterRuntimeExit`
  直接写终态,不再留着让重连把 Running 灌回来。

Refs #1132

* fix(runtime): converge background tasks and reap orphaned shells

* fix(desktop): restore stopped shell tasks after cold history load

---------

Co-authored-by: gugugaga <267102352+omazili-guga@users.noreply.github.com>
Co-authored-by: 程序员阿江(Relakkes) <relakkes@gmail.com>
This commit is contained in:
gugugaga
2026-10-05 22:33:35 +08:00
committed by GitHub
parent d0ba3d3bd0
commit 8b2f9102a5
11 changed files with 1493 additions and 73 deletions
+84
View File
@@ -1787,6 +1787,90 @@ describe('chatStore history mapping', () => {
.toBeUndefined() .toBeUndefined()
}) })
it.each(['Bash:0', 'root-shell-task'])(
'restores a stopped shell on cold load with terminal toolUseId %s',
async (toolUseId) => {
vi.mocked(sessionsApi.getFullHistory).mockResolvedValueOnce({
messages: [
{
id: 'root-shell-use',
type: 'assistant',
timestamp: '2026-04-06T00:00:00.000Z',
content: [{
type: 'tool_use',
id: 'Bash:0',
name: 'Bash',
input: { command: 'bun test', run_in_background: true },
}],
},
{
id: 'root-shell-result',
type: 'tool_result',
timestamp: '2026-04-06T00:00:01.000Z',
content: [{
type: 'tool_result',
tool_use_id: 'Bash:0',
content: 'Command running in background with ID: root-shell-task',
}],
},
],
taskNotifications: [
{
taskId: 'root-shell-task',
toolUseId,
status: 'stopped',
summary: 'Background task stopped',
timestamp: '2026-04-06T00:00:02.000Z',
},
{
taskId: 'root-shell-task',
toolUseId: 'unrelated-shell-tool',
status: 'failed',
summary: 'A task ID match must not override an unrelated real tool anchor',
timestamp: '2026-04-06T00:00:03.000Z',
},
{
taskId: 'root-shell-task',
toolUseId: 'root-shell-task',
ownerAgentId: 'child-agent',
status: 'failed',
summary: 'An owned child notification must not replace the root outcome',
timestamp: '2026-04-06T00:00:03.000Z',
},
{
taskId: 'unjoined-child-task',
toolUseId: 'unjoined-child-task',
status: 'stopped',
summary: 'An unowned task with no root transcript anchor must stay excluded',
},
],
})
useChatStore.setState({
sessions: {
[TEST_SESSION_ID]: makeSession({ messages: [] }),
},
})
await useChatStore.getState().loadHistory(TEST_SESSION_ID)
const session = useChatStore.getState().sessions[TEST_SESSION_ID]
expect(session?.backgroundAgentTasks?.['root-shell-task']).toMatchObject({
taskId: 'root-shell-task',
toolUseId: 'Bash:0',
status: 'stopped',
summary: 'Background task stopped',
})
expect(session?.agentTaskNotifications?.['Bash:0']).toMatchObject({
taskId: 'root-shell-task',
toolUseId: 'Bash:0',
status: 'stopped',
})
expect(session?.agentTaskNotifications?.['root-shell-task']).toBeUndefined()
expect(session?.agentTaskNotifications?.['unrelated-shell-tool']).toBeUndefined()
expect(session?.backgroundAgentTasks?.['unjoined-child-task']).toBeUndefined()
},
)
it('does not assign an unjoined child notification to the root run', async () => { it('does not assign an unjoined child notification to the root run', async () => {
vi.mocked(sessionsApi.getFullHistory).mockResolvedValueOnce({ vi.mocked(sessionsApi.getFullHistory).mockResolvedValueOnce({
messages: [{ messages: [{
+15 -1
View File
@@ -2333,7 +2333,21 @@ async function fetchAndMapSessionHistory(
// jobs and notifications into the parent rail after every reload. // jobs and notifications into the parent rail after every reload.
const rootRunMessages = messages.filter((message) => !message.parentToolUseId) const rootRunMessages = messages.filter((message) => !message.parentToolUseId)
const rootToolUseIds = transcriptToolUseIds(rootRunMessages) const rootToolUseIds = transcriptToolUseIds(rootRunMessages)
const rootRunNotifications = (taskNotifications ?? []).filter( const rootShellTasks = reconstructBackgroundShellTasks(rootRunMessages)
const rootRunNotifications = (taskNotifications ?? []).map((notification) => {
// After a server restart, a late Stop can only persist the task ID as its
// tool anchor. Recover the real anchor from this root run's Bash result so
// the terminal survives cold history loading and links to the right tool.
const rootShellTask = rootShellTasks[notification.taskId]
if (
!notification.ownerAgentId &&
notification.toolUseId === notification.taskId &&
rootShellTask?.toolUseId
) {
return { ...notification, toolUseId: rootShellTask.toolUseId }
}
return notification
}).filter(
(notification) => ( (notification) => (
!notification.ownerAgentId && ( !notification.ownerAgentId && (
// workflow_run_id predates owner_agent_id. Keep restoring those // workflow_run_id predates owner_agent_id. Keep restoring those
@@ -0,0 +1,411 @@
import { afterEach, describe, expect, it, mock, spyOn } from 'bun:test'
import {
__resetWebSocketHandlerStateForTests,
getSessionChatActivityState,
getSessionTurnState,
handleWebSocket,
stopSessionTurn,
} from '../ws/handler.js'
import { activeNonAgentTasks, authoritativeStoppedTaskIds, clearAgentRuntimeState } from '../ws/agentTaskState.js'
import { conversationService } from '../services/conversationService.js'
import { computerUseApprovalService } from '../services/computerUseApprovalService.js'
import { sessionService } from '../services/sessionService.js'
import * as teamPlanRuntime from '../services/teamPlanRuntime.js'
import { migrationMaintenance } from '../migrationMaintenance.js'
async function flushMicrotasks() {
for (let index = 0; index < 30; index++) await Promise.resolve()
}
function setup() {
const sessionId = `background-cleanup-${crypto.randomUUID()}`
const clients: any[] = []
function client() {
const sent: any[] = []
const ws: any = {
data: { sessionId, channel: 'client', clientKind: 'full', connectedAt: Date.now(), sdkToken: null, serverPort: 0, serverHost: '127.0.0.1' },
send: mock((payload: string) => sent.push(JSON.parse(payload))),
close: mock(() => {}),
sent,
}
clients.push(ws)
return ws
}
const ws = client()
const callbacks = new Set<(message: any) => void>()
const timers: Array<{ id: number; callback: () => void; delay: number }> = []
const cancelled = new Set<number>()
let runtimeAlive = true
spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, delay = 0) => {
const id = timers.length + 1
timers.push({ id, callback, delay })
return id as any
}) as any)
spyOn(globalThis, 'clearTimeout').mockImplementation((id: any) => { cancelled.add(id) })
spyOn(conversationService, 'hasSession').mockImplementation(() => runtimeAlive)
const stop = spyOn(conversationService, 'stopSession').mockImplementation(() => { runtimeAlive = false })
const permissions = spyOn(conversationService, 'getPendingPermissionRequests').mockReturnValue([])
const computerPermissions = spyOn(computerUseApprovalService, 'getPendingRequests').mockReturnValue([])
spyOn(conversationService, 'onOutput').mockImplementation((_id, callback) => { callbacks.add(callback) })
spyOn(conversationService, 'removeOutputCallback').mockImplementation((_id, callback) => { callbacks.delete(callback) })
spyOn(conversationService, 'sendInterrupt').mockReturnValue(true)
spyOn(conversationService, 'sendMessage').mockResolvedValue(true)
const control = spyOn(conversationService, 'requestControl').mockResolvedValue({})
const append = spyOn(sessionService, 'appendSessionTaskNotification').mockResolvedValue()
spyOn(sessionService, 'getCustomTitle').mockResolvedValue('Existing title')
const teamWork = spyOn(teamPlanRuntime, 'hasActiveTeamWorkForParent').mockReturnValue(false)
function dispatch(message: any) { for (const callback of [...callbacks]) callback(message) }
function startTask(id = 'bash-1', type = 'local_bash') {
dispatch({ type: 'system', subtype: 'task_started', task_id: id, tool_use_id: `tool-${id}`, task_type: type })
}
function close() { handleWebSocket.close(ws, 1000, 'test disconnect') }
function ceiling() { return timers.findLast(timer => timer.delay === 31 * 60_000) }
function terminals(socket = ws) {
return socket.sent.filter((message: any) => message.type === 'system_notification' &&
message.subtype === 'task_notification' && ['stopped', 'failed'].includes(message.data?.status))
}
handleWebSocket.open(ws)
return { sessionId, ws, client, timers, cancelled, stop, permissions, computerPermissions, control, append, teamWork, dispatch, startTask, close, ceiling, terminals }
}
afterEach(() => {
__resetWebSocketHandlerStateForTests()
migrationMaintenance.resetForTests()
mock.restore()
})
describe('background task cleanup boundaries', () => {
it('bounds a silent task from disconnect and converges both activity and runtime state', async () => {
const s = setup()
s.startTask()
s.close()
expect(s.ceiling()).toBeDefined()
s.ceiling()!.callback()
await flushMicrotasks()
expect(s.stop).toHaveBeenCalledWith(s.sessionId)
expect(s.append).toHaveBeenCalledWith(s.sessionId, expect.objectContaining({ taskId: 'bash-1', status: 'failed' }))
expect(getSessionChatActivityState(s.sessionId)).toBe('idle')
expect(getSessionTurnState(s.sessionId)).toBe('idle')
expect(activeNonAgentTasks.has(s.sessionId)).toBe(false)
const idleGrace = s.timers.findLast(timer => timer.delay === 30_000)
expect(idleGrace).toBeDefined()
idleGrace!.callback()
expect(authoritativeStoppedTaskIds.has(s.sessionId)).toBe(false)
})
it('starts the shell ceiling after the real foreground turn and CLI run both settle', async () => {
const s = setup()
handleWebSocket.message(s.ws, JSON.stringify({ type: 'user_message', content: 'work' }))
await flushMicrotasks()
s.dispatch({ type: 'system', subtype: 'session_state_changed', state: 'running' })
s.startTask()
s.close()
expect(s.ceiling()).toBeUndefined()
s.dispatch({ type: 'system', subtype: 'session_state_changed', state: 'idle' })
expect(s.ceiling()).toBeUndefined()
s.dispatch({ type: 'result', subtype: 'success', is_error: false, result: 'done' })
expect(s.ceiling()).toBeDefined()
expect(s.stop).not.toHaveBeenCalled()
})
it('does not reap a foreground run that starts after the shell ceiling was queued', async () => {
const s = setup()
s.startTask()
s.close()
const old = s.ceiling()!
s.dispatch({ type: 'system', subtype: 'session_state_changed', state: 'running' })
old.callback()
expect(s.stop).not.toHaveBeenCalled()
expect(getSessionTurnState(s.sessionId)).toBe('running')
s.dispatch({ type: 'system', subtype: 'session_state_changed', state: 'idle' })
expect(s.ceiling()!.id).not.toBe(old.id)
})
it('keeps Agent tasks and approved working teams alive alongside a shell', () => {
const s = setup()
s.startTask()
s.startTask('agent-1', 'local_agent')
s.close()
expect(s.ceiling()).toBeUndefined()
s.dispatch({ type: 'system', subtype: 'task_notification', task_id: 'agent-1', status: 'completed' })
const beforeTeam = s.ceiling()!
expect(beforeTeam).toBeDefined()
s.teamWork.mockReturnValue(true)
beforeTeam.callback()
expect(s.stop).not.toHaveBeenCalled()
})
it('starts the shell ceiling after a detached Agent finishes its durable stop', async () => {
const s = setup()
s.startTask()
s.startTask('agent-1', 'local_agent')
await flushMicrotasks()
let saveAgent!: () => void
s.append.mockImplementation((_id, notification) => notification.taskId === 'agent-1' && notification.status === 'stopped'
? new Promise<void>(resolve => { saveAgent = resolve }) : Promise.resolve())
handleWebSocket.message(s.ws, JSON.stringify({ type: 'stop_background_task', taskId: 'agent-1' }))
await flushMicrotasks()
s.close()
expect(s.ceiling()).toBeUndefined()
saveAgent()
await flushMicrotasks()
expect(s.ceiling()).toBeDefined()
expect(s.stop).not.toHaveBeenCalled()
})
it('does not use the shell ceiling to shorten a pending tool or computer-use prompt', () => {
const s = setup()
s.startTask()
s.close()
const original = s.ceiling()!
s.permissions.mockReturnValue([{ requestId: 'tool-permission', request: { subtype: 'can_use_tool' } }] as any)
original.callback()
expect(s.stop).not.toHaveBeenCalled()
s.permissions.mockReturnValue([])
s.dispatch({ type: 'system', subtype: 'session_state_changed', state: 'idle' })
const next = s.ceiling()!
s.computerPermissions.mockReturnValue([{ requestId: 'computer-permission' }] as any)
next.callback()
expect(s.stop).not.toHaveBeenCalled()
})
it('persists shell bookends when an abandoned permission reaches its own cleanup bound', async () => {
const s = setup()
s.startTask()
s.permissions.mockReturnValue([{ requestId: 'abandoned-permission', request: { subtype: 'can_use_tool' } }] as any)
s.close()
// This is the existing permission ceiling, not the background-only ceiling.
const permissionBound = s.timers.find(timer => timer.delay === 31 * 60_000)!
expect(permissionBound).toBeDefined()
permissionBound.callback()
s.permissions.mockReturnValue([])
await flushMicrotasks()
expect(s.stop).toHaveBeenCalledTimes(1)
expect(s.append).toHaveBeenCalledWith(s.sessionId, expect.objectContaining({ taskId: 'bash-1', status: 'failed' }))
expect(getSessionTurnState(s.sessionId)).toBe('idle')
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).not.toBe(true)
expect(s.timers.findLast(timer => timer.delay === 30_000)).toBeDefined()
})
it('does not kill or persist background tasks during migration maintenance', () => {
const s = setup()
s.startTask()
s.close()
const timer = s.ceiling()!
s.append.mockClear()
migrationMaintenance.begin()
timer.callback()
expect(s.stop).not.toHaveBeenCalled()
expect(s.append).not.toHaveBeenCalled()
})
it('ignores a queued old ceiling after reconnect and a second disconnect', () => {
const s = setup()
s.startTask()
s.close()
const old = s.ceiling()!
const reconnected = s.client()
handleWebSocket.open(reconnected)
expect(s.cancelled.has(old.id)).toBe(true)
handleWebSocket.close(reconnected, 1000, 'second disconnect')
const current = s.ceiling()!
old.callback()
expect(s.stop).not.toHaveBeenCalled()
current.callback()
expect(s.stop).toHaveBeenCalledTimes(1)
})
it('protects a real replacement turn from the previous Stop timeout', async () => {
const s = setup()
s.control.mockImplementation(() => new Promise(() => {}))
s.startTask()
handleWebSocket.message(s.ws, JSON.stringify({ type: 'stop_generation' }))
const previousStop = s.timers.find(timer => timer.delay === 3_000)!
handleWebSocket.message(s.ws, JSON.stringify({ type: 'user_message', content: 'new work' }))
await flushMicrotasks()
previousStop.callback()
expect(s.stop).not.toHaveBeenCalled()
expect(getSessionTurnState(s.sessionId)).toBe('running')
s.startTask('new-shell')
expect(s.control.mock.calls.filter(call => call[1]?.task_id === 'new-shell')).toHaveLength(0)
})
it('stops a late shell once while the original user Stop still owns its scope', async () => {
const s = setup()
handleWebSocket.message(s.ws, JSON.stringify({ type: 'stop_generation' }))
s.startTask('late-shell')
s.dispatch({ type: 'system', subtype: 'task_notification', task_id: 'late-shell', status: 'running' })
expect(s.control.mock.calls).toEqual([[s.sessionId, { subtype: 'stop_task', task_id: 'late-shell' }]])
s.timers.find(timer => timer.delay === 3_000)!.callback()
await flushMicrotasks()
expect(s.stop).toHaveBeenCalledTimes(1)
expect(s.terminals()).toHaveLength(1)
})
it('keeps programmatic turn Stop narrow for independent shell tasks', () => {
const s = setup()
s.startTask()
stopSessionTurn(s.sessionId)
s.startTask('later-shell')
expect(s.control).not.toHaveBeenCalled()
})
it('persists not_found before dropping tracking or publishing a terminal notification', async () => {
const s = setup()
s.startTask()
await flushMicrotasks()
let save!: () => void
s.append.mockImplementation((_id, notification) => notification.status === 'stopped'
? new Promise<void>(resolve => { save = resolve }) : Promise.resolve())
s.control.mockResolvedValue({ reason: 'not_found' })
handleWebSocket.message(s.ws, JSON.stringify({ type: 'stop_background_task', taskId: 'bash-1' }))
await flushMicrotasks()
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).toBe(true)
expect(authoritativeStoppedTaskIds.get(s.sessionId)?.has('bash-1')).not.toBe(true)
expect(s.terminals()).toHaveLength(0)
s.dispatch({ type: 'system', subtype: 'task_notification', task_id: 'bash-1', status: 'running' })
save()
await flushMicrotasks()
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).not.toBe(true)
expect(s.terminals()).toHaveLength(1)
})
it('retries failed terminal persistence without reviving or forgetting an exited task', async () => {
const s = setup()
s.startTask()
await flushMicrotasks()
s.append.mockRejectedValue(new Error('disk unavailable'))
s.close()
s.ceiling()!.callback()
await flushMicrotasks()
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).toBe(true)
expect(authoritativeStoppedTaskIds.get(s.sessionId)?.has('bash-1')).not.toBe(true)
expect(getSessionChatActivityState(s.sessionId)).toBe('idle')
// The ordinary cleanup must retain the pending terminal record as well.
s.timers.findLast(timer => timer.delay === 30_000)!.callback()
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).toBe(true)
s.append.mockResolvedValue()
const returning = s.client()
handleWebSocket.open(returning)
await flushMicrotasks()
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).not.toBe(true)
expect(s.terminals(returning)).toHaveLength(1)
})
for (const wholeSession of [false, true]) {
for (const response of [{}, { reason: 'not_found' }]) {
it(`ignores a stale ${wholeSession ? 'session' : 'task'} Stop response after its task record is replaced (${JSON.stringify(response)})`, async () => {
const s = setup()
let resolveControl!: (response: any) => void
s.control.mockImplementation(() => new Promise(resolve => { resolveControl = resolve }))
s.startTask()
handleWebSocket.message(s.ws, JSON.stringify(wholeSession
? { type: 'stop_generation' }
: { type: 'stop_background_task', taskId: 'bash-1' }))
// A runtime replacement releases the old registry; the replacement CLI
// is allowed to reuse a handle while an old request callback is queued.
clearAgentRuntimeState(s.sessionId)
s.startTask()
const replacement = activeNonAgentTasks.get(s.sessionId)!.get('bash-1')
resolveControl(response)
await flushMicrotasks()
expect(activeNonAgentTasks.get(s.sessionId)!.get('bash-1')).toBe(replacement)
expect(s.terminals()).toHaveLength(0)
})
}
}
it('treats a successful stop control response as confirmation even without a terminal frame', async () => {
const s = setup()
s.startTask()
handleWebSocket.message(s.ws, JSON.stringify({ type: 'stop_generation' }))
await flushMicrotasks()
expect(s.terminals().map((message: any) => message.data.status)).toEqual(['stopped'])
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).not.toBe(true)
s.timers.find(timer => timer.delay === 3_000)!.callback()
expect(s.stop).not.toHaveBeenCalled()
})
it('bounds a hung terminal write and can retry it after the timeout', async () => {
const s = setup()
s.startTask()
await flushMicrotasks()
s.append.mockImplementation(() => new Promise(() => {}))
s.close()
s.ceiling()!.callback()
for (let attempt = 0; attempt < 3; attempt++) {
const timeout = s.timers.filter(timer => timer.delay === 1_000)[attempt]
expect(timeout).toBeDefined()
timeout.callback()
await flushMicrotasks()
}
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).toBe(true)
expect(authoritativeStoppedTaskIds.get(s.sessionId)?.has('bash-1')).not.toBe(true)
s.append.mockResolvedValue()
s.timers.find(timer => timer.delay === 250)!.callback()
await flushMicrotasks()
expect(activeNonAgentTasks.get(s.sessionId)?.has('bash-1')).not.toBe(true)
expect(s.append.mock.calls.filter(call => call[1].status === 'failed')).toHaveLength(4)
})
it('does not infer task termination from a crash merely because Stop was requested', async () => {
const s = setup()
s.control.mockImplementation(() => new Promise(() => {}))
s.startTask()
handleWebSocket.message(s.ws, JSON.stringify({ type: 'stop_generation' }))
s.stop(s.sessionId)
s.dispatch({ type: 'result', subtype: 'error', is_error: true, result: 'runtime crashed before stop acknowledgement' })
await flushMicrotasks()
expect(s.terminals().map((message: any) => message.data.status)).toEqual(['failed'])
})
it('preserves an acknowledged stop outcome while persistence and a crash race', async () => {
const s = setup()
s.startTask()
await flushMicrotasks()
let save!: () => void
s.append.mockImplementation((_id, notification) => notification.status === 'stopped'
? new Promise<void>(resolve => { save = resolve }) : Promise.resolve())
handleWebSocket.message(s.ws, JSON.stringify({ type: 'stop_generation' }))
await flushMicrotasks()
s.stop(s.sessionId)
s.dispatch({ type: 'result', subtype: 'error', is_error: true, result: 'runtime crashed after confirmed stop' })
save()
await flushMicrotasks()
expect(s.terminals().map((message: any) => message.data.status)).toEqual(['stopped'])
expect(s.append.mock.calls.filter(call => call[1].status === 'failed')).toHaveLength(0)
})
it('records an unacknowledged Windows force-stop as failed rather than claiming the shell died', async () => {
const descriptor = Object.getOwnPropertyDescriptor(process, 'platform')!
Object.defineProperty(process, 'platform', { ...descriptor, value: 'win32' })
try {
const s = setup()
s.control.mockImplementation(() => new Promise(() => {}))
s.startTask()
handleWebSocket.message(s.ws, JSON.stringify({ type: 'stop_generation' }))
s.timers.find(timer => timer.delay === 3_000)!.callback()
await flushMicrotasks()
s.startTask('late-shell')
await flushMicrotasks()
expect(s.terminals().map((message: any) => message.data.status)).toEqual(['failed', 'failed'])
} finally {
Object.defineProperty(process, 'platform', descriptor)
}
})
it('records unrequested runtime loss as failed and closes late task records consistently', async () => {
const s = setup()
s.startTask()
s.stop(s.sessionId)
s.dispatch({ type: 'result', subtype: 'error', is_error: true, result: 'runtime crashed' })
await flushMicrotasks()
s.startTask('late-shell')
await flushMicrotasks()
expect(s.terminals().map((message: any) => message.data.status)).toEqual(['failed', 'failed'])
expect(s.append).toHaveBeenCalledWith(s.sessionId, expect.objectContaining({
taskId: 'bash-1', status: 'failed', summary: expect.stringContaining('could not be confirmed'),
}))
expect(getSessionChatActivityState(s.sessionId)).toBe('idle')
})
})
@@ -0,0 +1,109 @@
import { afterEach, expect, test, mock, spyOn } from 'bun:test'
import {
__resetWebSocketHandlerStateForTests,
ensureCliSessionStartedForControl,
getSessionChatActivityState,
getSessionTurnState,
} from '../ws/handler.js'
import { activeNonAgentTasks } from '../ws/agentTaskState.js'
import { conversationService } from '../services/conversationService.js'
import { computerUseApprovalService } from '../services/computerUseApprovalService.js'
import { sessionService } from '../services/sessionService.js'
import { SettingsService } from '../services/settingsService.js'
import { ProviderService } from '../services/providerService.js'
import { migrationMaintenance } from '../migrationMaintenance.js'
async function flushMicrotasks() {
for (let index = 0; index < 30; index++) await Promise.resolve()
}
function setup() {
const sessionId = `headless-background-${crypto.randomUUID()}`
const callbacks = new Set<(message: any) => void>()
const timers: Array<{ callback: () => void; delay: number }> = []
let runtimeAlive = false
spyOn(conversationService, 'hasSession').mockImplementation(() => runtimeAlive)
const start = spyOn(conversationService, 'startSession').mockImplementation(async () => { runtimeAlive = true })
const stop = spyOn(conversationService, 'stopSession').mockImplementation(() => { runtimeAlive = false })
spyOn(conversationService, 'onOutput').mockImplementation((_id, callback) => { callbacks.add(callback) })
spyOn(conversationService, 'removeOutputCallback').mockImplementation((_id, callback) => { callbacks.delete(callback) })
spyOn(conversationService, 'getPendingPermissionRequests').mockReturnValue([])
spyOn(computerUseApprovalService, 'getPendingRequests').mockReturnValue([])
spyOn(sessionService, 'getSessionWorkDir').mockResolvedValue('/tmp/headless-control-fixture')
spyOn(sessionService, 'getSessionLaunchInfo').mockResolvedValue({
runtimeModelId: 'fixture-model', runtimeProviderId: 'fixture-provider', permissionMode: 'default',
} as any)
spyOn(ProviderService.prototype, 'listProviders').mockResolvedValue({
activeId: 'fixture-provider', providers: [{ id: 'fixture-provider' }],
} as any)
spyOn(ProviderService.prototype, 'getProvider').mockResolvedValue(null)
spyOn(SettingsService.prototype, 'getUserSettings').mockResolvedValue({})
const append = spyOn(sessionService, 'appendSessionTaskNotification').mockResolvedValue()
spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, delay = 0) => {
timers.push({ callback, delay })
return timers.length as any
}) as any)
spyOn(globalThis, 'clearTimeout').mockImplementation(() => {})
function dispatch(message: any) { for (const callback of [...callbacks]) callback(message) }
function task(id = 'headless-shell') {
dispatch({ type: 'system', subtype: 'task_started', task_id: id, tool_use_id: `tool-${id}`, task_type: 'local_bash' })
}
const ensure = () => ensureCliSessionStartedForControl(sessionId, new URL('http://127.0.0.1:12345/api/sessions/control'))
return { sessionId, callbacks, timers, start, stop, append, dispatch, task, ensure }
}
afterEach(() => {
__resetWebSocketHandlerStateForTests()
migrationMaintenance.resetForTests()
mock.restore()
})
test('a control-started session settles a runtime crash without ever opening a client socket', async () => {
const s = setup()
await s.ensure()
expect(s.start).toHaveBeenCalledTimes(1)
s.task()
expect(activeNonAgentTasks.get(s.sessionId)?.has('headless-shell')).toBe(true)
s.stop(s.sessionId)
s.dispatch({ type: 'result', is_error: true, result: 'CLI died' })
await flushMicrotasks()
expect(s.append).toHaveBeenCalledWith(s.sessionId, expect.objectContaining({ taskId: 'headless-shell', status: 'failed' }))
expect(activeNonAgentTasks.has(s.sessionId)).toBe(false)
expect(getSessionTurnState(s.sessionId)).toBe('idle')
expect(getSessionChatActivityState(s.sessionId)).not.toBe('running')
})
test('a control-started silent shell gets a ceiling without a client disconnect event', async () => {
const s = setup()
await s.ensure()
expect(s.timers.some(timer => timer.delay === 30_000)).toBe(false)
s.task()
const ceiling = s.timers.find(timer => timer.delay === 31 * 60_000)
expect(ceiling).toBeDefined()
ceiling!.callback()
await flushMicrotasks()
expect(s.stop).toHaveBeenCalledWith(s.sessionId)
expect(s.append).toHaveBeenCalledWith(s.sessionId, expect.objectContaining({ taskId: 'headless-shell', status: 'failed' }))
})
test('initial CLI idle does not arm cleanup ahead of the first headless control request', async () => {
const s = setup()
await s.ensure()
s.dispatch({ type: 'system', subtype: 'session_state_changed', state: 'idle' })
expect(s.timers.some(timer => timer.delay === 30_000)).toBe(false)
expect(s.stop).not.toHaveBeenCalled()
})
test('the independent observer does not persist stale task results after its durable runtime-loss bookend', async () => {
const s = setup()
await s.ensure()
s.task()
s.stop(s.sessionId)
s.dispatch({ type: 'result', is_error: true, result: 'CLI died' })
await flushMicrotasks()
expect(s.append).toHaveBeenCalledWith(s.sessionId, expect.objectContaining({ taskId: 'headless-shell', status: 'failed' }))
s.append.mockClear()
s.dispatch({ type: 'system', subtype: 'task_notification', task_id: 'headless-shell', tool_use_id: 'tool-headless-shell', status: 'completed' })
await flushMicrotasks()
expect(s.append).not.toHaveBeenCalled()
})
@@ -45,7 +45,9 @@ const CONTAINERS: Record<string, Classification> = {
activeNonAgentTasks: { kind: 'cleared' }, activeNonAgentTasks: { kind: 'cleared' },
activeUserTurns: { kind: 'cleared' }, activeUserTurns: { kind: 'cleared' },
agentStopRequestedSessions: { kind: 'cleared' }, agentStopRequestedSessions: { kind: 'cleared' },
nonAgentStopRequestedSessions: { kind: 'cleared' },
authoritativeStoppedTaskIds: { kind: 'cleared' }, authoritativeStoppedTaskIds: { kind: 'cleared' },
backgroundTaskCleanupTimers: { kind: 'cleared' },
deferredPermissionModes: { kind: 'cleared' }, deferredPermissionModes: { kind: 'cleared' },
deferredRuntimeRestarts: { kind: 'cleared' }, deferredRuntimeRestarts: { kind: 'cleared' },
interruptedSessionChats: { kind: 'cleared' }, interruptedSessionChats: { kind: 'cleared' },
@@ -57,6 +59,7 @@ const CONTAINERS: Record<string, Classification> = {
prewarmedSessions: { kind: 'cleared' }, prewarmedSessions: { kind: 'cleared' },
rejectedRuntimeConfigs: { kind: 'cleared' }, rejectedRuntimeConfigs: { kind: 'cleared' },
runtimeExitStoppedSessions: { kind: 'cleared' }, runtimeExitStoppedSessions: { kind: 'cleared' },
runtimeExitFailedSessions: { kind: 'cleared' },
runtimeOverrides: { kind: 'cleared' }, runtimeOverrides: { kind: 'cleared' },
runtimeTransitionPromises: { kind: 'cleared' }, runtimeTransitionPromises: { kind: 'cleared' },
sessionDisconnectWatchers: { kind: 'cleared' }, sessionDisconnectWatchers: { kind: 'cleared' },
+219 -14
View File
@@ -1331,7 +1331,7 @@ describe('WebSocket handler session isolation', () => {
await flushMicrotasks() await flushMicrotasks()
expect(sendInterrupt).toHaveBeenCalledWith(sessionId) expect(sendInterrupt).toHaveBeenCalledWith(sessionId)
expect(requestControl).toHaveBeenCalledTimes(4) expect(requestControl).toHaveBeenCalledTimes(6)
for (const taskId of [ for (const taskId of [
'agent-task-1', 'agent-task-1',
'agent-task-2', 'agent-task-2',
@@ -1343,6 +1343,15 @@ describe('WebSocket handler session isolation', () => {
task_id: taskId, task_id: taskId,
}, 3_000) }, 3_000)
} }
// A user Stop also reaps the non-Agent tasks sharing the runtime, so their
// shell processes are actually asked to stop rather than only being
// bookended when the runtime happens to exit.
for (const taskId of ['bash-collateral-task', 'provider-analyzer-teammate']) {
expect(requestControl).toHaveBeenCalledWith(sessionId, {
subtype: 'stop_task',
task_id: taskId,
})
}
expect(archiveRemoteSession).toHaveBeenCalledWith( expect(archiveRemoteSession).toHaveBeenCalledWith(
'remote-session-agent-task-3', 'remote-session-agent-task-3',
{ timeoutMs: 1_500 }, { timeoutMs: 1_500 },
@@ -1355,13 +1364,13 @@ describe('WebSocket handler session isolation', () => {
expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({ expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({
taskId: 'bash-collateral-task', taskId: 'bash-collateral-task',
toolUseId: 'bash-collateral-tool', toolUseId: 'bash-collateral-tool',
status: 'stopped', status: 'failed',
})) }))
expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({ expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({
taskId: 'provider-analyzer-teammate', taskId: 'provider-analyzer-teammate',
toolUseId: 'provider-analyzer-teammate-tool', toolUseId: 'provider-analyzer-teammate-tool',
ownerAgentId: 'provider-analyzer', ownerAgentId: 'provider-analyzer',
status: 'stopped', status: 'failed',
})) }))
expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({ expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({
type: 'system_notification', type: 'system_notification',
@@ -1376,7 +1385,7 @@ describe('WebSocket handler session isolation', () => {
subtype: 'task_notification', subtype: 'task_notification',
data: expect.objectContaining({ data: expect.objectContaining({
task_id: 'bash-collateral-task', task_id: 'bash-collateral-task',
status: 'stopped', status: 'failed',
}), }),
}) })
expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({ expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({
@@ -1385,7 +1394,7 @@ describe('WebSocket handler session isolation', () => {
data: expect.objectContaining({ data: expect.objectContaining({
task_id: 'provider-analyzer-teammate', task_id: 'provider-analyzer-teammate',
owner_agent_id: 'provider-analyzer', owner_agent_id: 'provider-analyzer',
status: 'stopped', status: 'failed',
}), }),
}) })
@@ -1411,19 +1420,19 @@ describe('WebSocket handler session isolation', () => {
notification.taskId === 'bash-collateral-task')).toHaveLength(1) notification.taskId === 'bash-collateral-task')).toHaveLength(1)
expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({ expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({
taskId: 'late-bash-after-force-kill', taskId: 'late-bash-after-force-kill',
status: 'stopped', status: 'failed',
})) }))
expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({ expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({
type: 'system_notification', type: 'system_notification',
subtype: 'task_notification', subtype: 'task_notification',
data: expect.objectContaining({ data: expect.objectContaining({
task_id: 'late-bash-after-force-kill', task_id: 'late-bash-after-force-kill',
status: 'stopped', status: 'failed',
}), }),
}) })
}) })
it('does not bulk-stop non-Agent background tasks', async () => { it('bulk-stops non-Agent background tasks on user Stop', async () => {
const sessionId = `stop-agent-filter-${crypto.randomUUID()}` const sessionId = `stop-agent-filter-${crypto.randomUUID()}`
const ws = makeClientSocket(sessionId) const ws = makeClientSocket(sessionId)
const outputCallbacks: Array<(cliMsg: any) => void> = [] const outputCallbacks: Array<(cliMsg: any) => void> = []
@@ -1453,8 +1462,13 @@ describe('WebSocket handler session isolation', () => {
handleWebSocket.message(ws, JSON.stringify({ type: 'stop_generation' })) handleWebSocket.message(ws, JSON.stringify({ type: 'stop_generation' }))
await Promise.resolve() await Promise.resolve()
// A user Stop ends the whole session: the Agent task is stopped through the
// Agent finalization path (with its 3s control timeout), and the non-Agent
// shell/dream tasks are reaped through the same control channel without one.
expect(requestControl.mock.calls).toEqual([ expect(requestControl.mock.calls).toEqual([
[sessionId, { subtype: 'stop_task', task_id: 'agent-task-1' }, 3_000], [sessionId, { subtype: 'stop_task', task_id: 'agent-task-1' }, 3_000],
[sessionId, { subtype: 'stop_task', task_id: 'bash-task-1' }],
[sessionId, { subtype: 'stop_task', task_id: 'dream-task-1' }],
]) ])
outputCallbacks[0]?.({ outputCallbacks[0]?.({
type: 'system', type: 'system',
@@ -2667,14 +2681,14 @@ describe('WebSocket handler session isolation', () => {
expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({ expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({
taskId: 'bash-clear-failed', taskId: 'bash-clear-failed',
toolUseId: 'bash-clear-failed-tool', toolUseId: 'bash-clear-failed-tool',
status: 'stopped', status: 'failed',
})) }))
expect(first.sent.map((payload) => JSON.parse(payload))).toContainEqual({ expect(first.sent.map((payload) => JSON.parse(payload))).toContainEqual({
type: 'system_notification', type: 'system_notification',
subtype: 'task_notification', subtype: 'task_notification',
data: expect.objectContaining({ data: expect.objectContaining({
task_id: 'bash-clear-failed', task_id: 'bash-clear-failed',
status: 'stopped', status: 'failed',
}), }),
}) })
}) })
@@ -4766,7 +4780,8 @@ describe('WebSocket handler session isolation', () => {
summary: 'Desktop verification passed', summary: 'Desktop verification passed',
}) })
expect(setTimeoutSpy).not.toHaveBeenCalled() expect(setTimeoutSpy).toHaveBeenCalledTimes(1)
expect(setTimeoutSpy.mock.calls[0]?.[1]).toBe(31 * 60_000)
expect(stopSession).not.toHaveBeenCalled() expect(stopSession).not.toHaveBeenCalled()
outputCallbacks[1]?.({ outputCallbacks[1]?.({
@@ -4779,11 +4794,11 @@ describe('WebSocket handler session isolation', () => {
summary: 'Focused tests passed', summary: 'Focused tests passed',
}) })
expect(setTimeoutSpy).toHaveBeenCalledTimes(1) expect(setTimeoutSpy).toHaveBeenCalledTimes(2)
expect(setTimeoutSpy.mock.calls[0]?.[1]).toBe(30_000) expect(setTimeoutSpy.mock.calls[1]?.[1]).toBe(30_000)
expect(stopSession).not.toHaveBeenCalled() expect(stopSession).not.toHaveBeenCalled()
const expireIdleGrace = setTimeoutSpy.mock.calls[0]?.[0] as (() => void) | undefined const expireIdleGrace = setTimeoutSpy.mock.calls[1]?.[0] as (() => void) | undefined
expireIdleGrace?.() expireIdleGrace?.()
expect(stopSession).toHaveBeenCalledWith(sessionId) expect(stopSession).toHaveBeenCalledWith(sessionId)
}) })
@@ -5153,6 +5168,196 @@ describe('WebSocket handler session isolation', () => {
activeBackgroundTaskIds: [], activeBackgroundTaskIds: [],
}) })
}) })
it('reaps a non-Agent background task when the user stops a background-only session', async () => {
const sessionId = `stop-background-only-${crypto.randomUUID()}`
const ws = makeClientSocket(sessionId)
const outputCallbacks: Array<(cliMsg: any) => void> = []
spyOn(globalThis, 'setTimeout').mockImplementation(() => 1 as any)
spyOn(conversationService, 'hasSession').mockReturnValue(true)
spyOn(conversationService, 'onOutput').mockImplementation((_sid, callback) => {
outputCallbacks.push(callback)
})
spyOn(conversationService, 'removeOutputCallback').mockImplementation(() => {})
// The CLI reports the task as already evicted, so the Stop converges on a
// terminal bookend rather than waiting for a notification that never comes.
const requestControl = spyOn(conversationService, 'requestControl')
.mockResolvedValue({ reason: 'not_found' })
handleWebSocket.open(ws)
outputCallbacks[0]?.({
type: 'system',
subtype: 'task_started',
task_id: 'orphan-bash-1',
tool_use_id: 'orphan-bash-tool-1',
description: 'Background shell that outlived the turn',
task_type: 'local_bash',
})
await flushMicrotasks()
ws.sent.length = 0
// No foreground turn and no Agent task: before this fix the non-Agent task
// was simply never asked to stop.
handleWebSocket.message(ws, JSON.stringify({ type: 'stop_generation' }))
await flushMicrotasks()
expect(requestControl).toHaveBeenCalledWith(sessionId, {
subtype: 'stop_task',
task_id: 'orphan-bash-1',
})
expect(ws.sent.map((payload) => JSON.parse(payload))).toContainEqual({
type: 'system_notification',
subtype: 'task_notification',
message: 'Background shell that outlived the turn stopped',
data: expect.objectContaining({
task_id: 'orphan-bash-1',
tool_use_id: 'orphan-bash-tool-1',
status: 'stopped',
}),
})
})
it('bounds a disconnected session kept alive only by a background task', () => {
const sessionId = `background-task-ceiling-${crypto.randomUUID()}`
const ws = makeClientSocket(sessionId)
let nextTimerId = 0
const timers: Array<{ id: number; callback: () => void; delayMs: number }> = []
spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, delayMs?: number) => {
const id = ++nextTimerId
timers.push({ id, callback, delayMs: delayMs ?? 0 })
return id as any
}) as any)
const stopSession = spyOn(conversationService, 'stopSession').mockImplementation(() => {})
const append = spyOn(sessionService, 'appendSessionTaskNotification').mockResolvedValue()
spyOn(conversationService, 'getPendingPermissionRequests').mockReturnValue([])
spyOn(conversationService, 'hasSession').mockReturnValue(true)
const outputCallbacks: Array<(cliMsg: any) => void> = []
spyOn(conversationService, 'onOutput').mockImplementation((_sid, callback) => {
outputCallbacks.push(callback)
})
spyOn(conversationService, 'removeOutputCallback').mockImplementation(() => {})
handleWebSocket.open(ws)
outputCallbacks[0]?.({
type: 'system',
subtype: 'task_started',
task_id: 'ceiling-bash-1',
tool_use_id: 'ceiling-bash-tool-1',
description: 'Never-ending background shell',
task_type: 'local_bash',
})
handleWebSocket.close(ws, 1000, 'pet closed while a background task runs')
expect(stopSession).not.toHaveBeenCalled()
// The watcher sees the background task still running with no client left.
outputCallbacks.at(-1)?.({
type: 'system',
subtype: 'task_notification',
task_id: 'ceiling-bash-1',
tool_use_id: 'ceiling-bash-tool-1',
task_type: 'local_bash',
status: 'running',
})
const ceiling = timers.find((timer) => timer.delayMs === 31 * 60_000)
expect(ceiling).toBeDefined()
ceiling?.callback()
expect(stopSession).toHaveBeenCalledWith(sessionId)
expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({
taskId: 'ceiling-bash-1',
toolUseId: 'ceiling-bash-tool-1',
status: 'failed',
}))
})
it('cancels the background-task ceiling when a client reconnects', () => {
const sessionId = `background-task-ceiling-reconnect-${crypto.randomUUID()}`
const ws = makeClientSocket(sessionId)
let nextTimerId = 0
const timers: Array<{ id: number; callback: () => void; delayMs: number }> = []
spyOn(globalThis, 'setTimeout').mockImplementation(((callback: () => void, delayMs?: number) => {
const id = ++nextTimerId
timers.push({ id, callback, delayMs: delayMs ?? 0 })
return id as any
}) as any)
const clearTimeoutSpy = spyOn(globalThis, 'clearTimeout').mockImplementation(() => {})
const stopSession = spyOn(conversationService, 'stopSession').mockImplementation(() => {})
spyOn(conversationService, 'getPendingPermissionRequests').mockReturnValue([])
spyOn(conversationService, 'hasSession').mockReturnValue(true)
const outputCallbacks: Array<(cliMsg: any) => void> = []
spyOn(conversationService, 'onOutput').mockImplementation((_sid, callback) => {
outputCallbacks.push(callback)
})
spyOn(conversationService, 'removeOutputCallback').mockImplementation(() => {})
handleWebSocket.open(ws)
outputCallbacks[0]?.({
type: 'system',
subtype: 'task_started',
task_id: 'ceiling-bash-reconnect-1',
tool_use_id: 'ceiling-bash-reconnect-tool-1',
description: 'Still watched by a returning client',
task_type: 'local_bash',
})
handleWebSocket.close(ws, 1000, 'pet closed while a background task runs')
outputCallbacks.at(-1)?.({
type: 'system',
subtype: 'task_notification',
task_id: 'ceiling-bash-reconnect-1',
tool_use_id: 'ceiling-bash-reconnect-tool-1',
task_type: 'local_bash',
status: 'running',
})
const ceiling = timers.find((timer) => timer.delayMs === 31 * 60_000)
expect(ceiling).toBeDefined()
const reconnected = makeClientSocket(sessionId)
handleWebSocket.open(reconnected)
expect(clearTimeoutSpy).toHaveBeenCalledWith(ceiling?.id)
// Even an already-queued ceiling must not kill a session a client is using.
ceiling?.callback()
expect(stopSession).not.toHaveBeenCalled()
})
it('reaps residual background tasks when the runtime is already gone at disconnect', () => {
const sessionId = `dead-runtime-residual-${crypto.randomUUID()}`
const ws = makeClientSocket(sessionId)
spyOn(globalThis, 'setTimeout').mockImplementation(() => 0 as any)
const stopSession = spyOn(conversationService, 'stopSession').mockImplementation(() => {})
const append = spyOn(sessionService, 'appendSessionTaskNotification').mockResolvedValue()
spyOn(conversationService, 'getPendingPermissionRequests').mockReturnValue([])
const hasSession = spyOn(conversationService, 'hasSession').mockReturnValue(true)
const outputCallbacks: Array<(cliMsg: any) => void> = []
spyOn(conversationService, 'onOutput').mockImplementation((_sid, callback) => {
outputCallbacks.push(callback)
})
spyOn(conversationService, 'removeOutputCallback').mockImplementation(() => {})
handleWebSocket.open(ws)
outputCallbacks[0]?.({
type: 'system',
subtype: 'task_started',
task_id: 'residual-bash-1',
tool_use_id: 'residual-bash-tool-1',
description: 'Task record left behind by a dead CLI',
task_type: 'local_bash',
})
// The CLI died before the renderer closed, so no completion event will ever
// arrive to clear this record.
hasSession.mockReturnValue(false)
handleWebSocket.close(ws, 1006, 'renderer closed after the CLI died')
expect(append).toHaveBeenCalledWith(sessionId, expect.objectContaining({
taskId: 'residual-bash-1',
toolUseId: 'residual-bash-tool-1',
status: 'failed',
}))
expect(stopSession).not.toHaveBeenCalled()
})
}) })
describe('prewarm idle timer active-turn guard (issue #865 follow-up)', () => { describe('prewarm idle timer active-turn guard (issue #865 follow-up)', () => {
+32 -4
View File
@@ -2,13 +2,13 @@
* Agent and background-task lifecycle state for the session WebSocket handler. * Agent and background-task lifecycle state for the session WebSocket handler.
* *
* Moved verbatim out of `handler.ts` as the first cut of that file. This slice was * Moved verbatim out of `handler.ts` as the first cut of that file. This slice was
* chosen because it is provably closed: these functions read and write only the six * chosen because it is provably closed: these functions read and write only the lifecycle
* containers declared here and call nothing outside them, so relocating them cannot * containers declared here and call nothing outside them, so relocating them cannot
* change behavior. The functions that emit stop bookends stayed behind on purpose — * change behavior. The functions that emit stop bookends stayed behind on purpose —
* they reach into the broadcast layer and the socket registry, so moving them would * they reach into the broadcast layer and the socket registry, so moving them would
* have required injecting dependencies rather than moving code. * have required injecting dependencies rather than moving code.
* *
* All six containers are per-session and released by `clearAgentRuntimeState`, which * All lifecycle containers are per-session and released by `clearAgentRuntimeState`, which
* `cleanupSessionRuntimeState` calls. `src/server/__tests__/sessionStateCleanup.test.ts` * `cleanupSessionRuntimeState` calls. `src/server/__tests__/sessionStateCleanup.test.ts`
* follows that closure across this module boundary. * follows that closure across this module boundary.
* *
@@ -23,6 +23,15 @@ export type ActiveNonAgentTaskState = {
toolUseId: string toolUseId: string
description?: string description?: string
ownerAgentId?: string ownerAgentId?: string
terminalStatus?: 'stopped' | 'failed'
terminalSummary?: string
terminalMessage?: boolean
stopRequested?: boolean
localStopConfirmed?: boolean
finalizationRetryCount?: number
finalizationRetryTimer?: ReturnType<typeof setTimeout>
finalization?: Promise<boolean>
stopFailureMessage?: string
} }
export type ActiveAgentTaskState = { export type ActiveAgentTaskState = {
@@ -54,8 +63,12 @@ export const authoritativeStoppedTaskIds = new Map<string, Set<string>>()
export const agentStopRequestedSessions = new Set<string>() export const agentStopRequestedSessions = new Set<string>()
export const nonAgentStopRequestedSessions = new Set<string>()
export const runtimeExitStoppedSessions = new Set<string>() export const runtimeExitStoppedSessions = new Set<string>()
export const runtimeExitFailedSessions = new Set<string>()
export type CliBackgroundTaskLifecycle = { export type CliBackgroundTaskLifecycle = {
taskId: string taskId: string
running: boolean running: boolean
@@ -123,6 +136,8 @@ export function untrackCliBackgroundTask(sessionId: string, taskId: string): voi
if (sessionAgentTasks?.size === 0) activeAgentTasks.delete(sessionId) if (sessionAgentTasks?.size === 0) activeAgentTasks.delete(sessionId)
const sessionNonAgentTasks = activeNonAgentTasks.get(sessionId) const sessionNonAgentTasks = activeNonAgentTasks.get(sessionId)
const nonAgentTask = sessionNonAgentTasks?.get(taskId)
if (nonAgentTask) clearAgentStopFinalizationRetry(nonAgentTask)
sessionNonAgentTasks?.delete(taskId) sessionNonAgentTasks?.delete(taskId)
if (sessionNonAgentTasks?.size === 0) activeNonAgentTasks.delete(sessionId) if (sessionNonAgentTasks?.size === 0) activeNonAgentTasks.delete(sessionId)
} }
@@ -139,16 +154,28 @@ export function clearAgentRuntimeState(
) )
: new Map<string, ActiveAgentTaskState>() : new Map<string, ActiveAgentTaskState>()
const retryableNonAgentStops = options?.preserveRetryableStops
? new Map([...(activeNonAgentTasks.get(sessionId)?.entries() ?? [])]
.filter(([, task]) => task.localStopConfirmed))
: new Map<string, ActiveNonAgentTaskState>()
for (const task of activeAgentTasks.get(sessionId)?.values() ?? []) { for (const task of activeAgentTasks.get(sessionId)?.values() ?? []) {
clearAgentStopFinalizationRetry(task) clearAgentStopFinalizationRetry(task)
} }
for (const task of activeNonAgentTasks.get(sessionId)?.values() ?? []) {
clearAgentStopFinalizationRetry(task)
}
activeBackgroundTaskIds.delete(sessionId) activeBackgroundTaskIds.delete(sessionId)
activeAgentTasks.delete(sessionId) activeAgentTasks.delete(sessionId)
activeNonAgentTasks.delete(sessionId) activeNonAgentTasks.delete(sessionId)
authoritativeStoppedTaskIds.delete(sessionId) authoritativeStoppedTaskIds.delete(sessionId)
agentStopRequestedSessions.delete(sessionId) agentStopRequestedSessions.delete(sessionId)
nonAgentStopRequestedSessions.delete(sessionId)
runtimeExitStoppedSessions.delete(sessionId) runtimeExitStoppedSessions.delete(sessionId)
runtimeExitFailedSessions.delete(sessionId)
if (retryableNonAgentStops.size > 0) {
activeNonAgentTasks.set(sessionId, retryableNonAgentStops)
}
if (retryableStops.size > 0) { if (retryableStops.size > 0) {
activeAgentTasks.set(sessionId, retryableStops) activeAgentTasks.set(sessionId, retryableStops)
activeBackgroundTaskIds.set(sessionId, new Set(retryableStops.keys())) activeBackgroundTaskIds.set(sessionId, new Set(retryableStops.keys()))
@@ -171,11 +198,12 @@ export function hasActiveBackgroundTasks(sessionId: string): boolean {
const sessionAgentTasks = activeAgentTasks.get(sessionId) const sessionAgentTasks = activeAgentTasks.get(sessionId)
return [...taskIds].some((taskId) => { return [...taskIds].some((taskId) => {
const agentTask = sessionAgentTasks?.get(taskId) const agentTask = sessionAgentTasks?.get(taskId)
return !agentTask || !agentTask.localStopConfirmed || agentTask.bookendPending if (agentTask) return !agentTask.localStopConfirmed || agentTask.bookendPending
return !activeNonAgentTasks.get(sessionId)?.get(taskId)?.localStopConfirmed
}) })
} }
export function clearAgentStopFinalizationRetry(task: ActiveAgentTaskState): void { export function clearAgentStopFinalizationRetry(task: { finalizationRetryTimer?: ReturnType<typeof setTimeout> }): void {
if (task.finalizationRetryTimer === undefined) return if (task.finalizationRetryTimer === undefined) return
clearTimeout(task.finalizationRetryTimer) clearTimeout(task.finalizationRetryTimer)
task.finalizationRetryTimer = undefined task.finalizationRetryTimer = undefined
+391 -52
View File
@@ -84,7 +84,9 @@ import {
activeNonAgentTasks, activeNonAgentTasks,
authoritativeStoppedTaskIds, authoritativeStoppedTaskIds,
agentStopRequestedSessions, agentStopRequestedSessions,
nonAgentStopRequestedSessions,
runtimeExitStoppedSessions, runtimeExitStoppedSessions,
runtimeExitFailedSessions,
getCliBackgroundTaskLifecycle, getCliBackgroundTaskLifecycle,
isAgentTaskType, isAgentTaskType,
untrackCliBackgroundTask, untrackCliBackgroundTask,
@@ -96,6 +98,7 @@ import {
} from './agentTaskState.js' } from './agentTaskState.js'
import type { import type {
ActiveAgentTaskState, ActiveAgentTaskState,
ActiveNonAgentTaskState,
CliBackgroundTaskLifecycle, CliBackgroundTaskLifecycle,
} from './agentTaskState.js' } from './agentTaskState.js'
import { import {
@@ -165,7 +168,15 @@ const sessionSlashCommands = new Map<string, SessionSlashCommand[]>()
// The longest automatic question wait is 30 minutes; leave room for its // The longest automatic question wait is 30 minutes; leave room for its
// bounded model request before reclaiming a disconnected CLI. // bounded model request before reclaiming a disconnected CLI.
const PENDING_PERMISSION_DISCONNECT_CLEANUP_MS = 31 * 60_000 const PENDING_PERMISSION_DISCONNECT_CLEANUP_MS = 31 * 60_000
// A background shell task may legitimately outlive a disconnected client, but
// nothing else bounds one that never emits a terminal notification — so a
// forgotten run_in_background loop would pin the CLI (and its process group)
// forever. Cap that keep-alive at the same order as the permission bound;
// once it elapses with the client still gone, the shared runtime is stopped and
// terminal bookends are published.
const BACKGROUND_TASK_DISCONNECT_MAX_MS = 31 * 60_000
const sessionCleanupTimers = new Map<string, ReturnType<typeof setTimeout>>() const sessionCleanupTimers = new Map<string, ReturnType<typeof setTimeout>>()
const backgroundTaskCleanupTimers = new Map<string, ReturnType<typeof setTimeout>>()
/** /**
* Per-session removers for the active-work watcher (issue #764). When the last * Per-session removers for the active-work watcher (issue #764). When the last
* client disconnects while a turn or background task is still running, we let * client disconnects while a turn or background task is still running, we let
@@ -304,6 +315,12 @@ function trackCliBackgroundTaskLifecycle(
const lifecycle = getCliBackgroundTaskLifecycle(cliMsg) const lifecycle = getCliBackgroundTaskLifecycle(cliMsg)
if (!lifecycle) return null if (!lifecycle) return null
const existingNonAgentTask = activeNonAgentTasks.get(sessionId)?.get(lifecycle.taskId)
if (existingNonAgentTask?.localStopConfirmed) {
void emitAuthoritativeNonAgentStopped(sessionId, existingNonAgentTask)
return { ...lifecycle, running: false, status: 'stopped', suppressForward: true }
}
const existingAgentTask = activeAgentTasks.get(sessionId)?.get(lifecycle.taskId) const existingAgentTask = activeAgentTasks.get(sessionId)?.get(lifecycle.taskId)
if ( if (
lifecycle.running && lifecycle.running &&
@@ -394,6 +411,11 @@ function trackCliBackgroundTaskLifecycle(
return { ...lifecycle, suppressForward: true } return { ...lifecycle, suppressForward: true }
} }
if (existingNonAgentTask?.stopRequested) {
existingNonAgentTask.localStopConfirmed = true
void emitAuthoritativeNonAgentStopped(sessionId, existingNonAgentTask)
return { ...lifecycle, suppressForward: true }
}
untrackCliBackgroundTask(sessionId, lifecycle.taskId) untrackCliBackgroundTask(sessionId, lifecycle.taskId)
return lifecycle return lifecycle
} }
@@ -617,6 +639,7 @@ export const handleWebSocket = {
// Cancel any "let the running turn finish, then clean up" watcher too — // Cancel any "let the running turn finish, then clean up" watcher too —
// the session is observed again (issue #764). // the session is observed again (issue #764).
cancelSessionDisconnectWatcher(sessionId) cancelSessionDisconnectWatcher(sessionId)
clearBackgroundTaskDisconnectCeiling(sessionId)
addActiveClient(sessionId, ws) addActiveClient(sessionId, ws)
if (prewarmPendingSessions.has(sessionId) || prewarmedSessions.has(sessionId)) { if (prewarmPendingSessions.has(sessionId) || prewarmedSessions.has(sessionId)) {
@@ -647,6 +670,9 @@ export const handleWebSocket = {
turnActive: hasLiveUserTurnForClient(sessionId), turnActive: hasLiveUserTurnForClient(sessionId),
}) })
replayAgentStopFailures(ws, sessionId) replayAgentStopFailures(ws, sessionId)
for (const task of activeNonAgentTasks.get(sessionId)?.values() ?? []) {
if (task.localStopConfirmed) void emitAuthoritativeNonAgentStopped(sessionId, task)
}
}, },
message(ws: SessionConnection, rawMessage: string | Buffer) { message(ws: SessionConnection, rawMessage: string | Buffer) {
@@ -758,7 +784,7 @@ export const handleWebSocket = {
break break
case 'stop_generation': case 'stop_generation':
handleStopGeneration(ws) handleStopGeneration(ws, { reapBackgroundTasks: true })
break break
case 'stop_background_task': case 'stop_background_task':
@@ -804,6 +830,23 @@ export const handleWebSocket = {
// closed. Defer cleanup until all active work completes, then apply the // closed. Defer cleanup until all active work completes, then apply the
// idle grace period. Sessions that are already idle go straight to the timer. // idle grace period. Sessions that are already idle go straight to the timer.
if (hasActiveSessionWork(sessionId)) { if (hasActiveSessionWork(sessionId)) {
// If the CLI runtime is already gone but turn/task bookkeeping lingers,
// no completion event will ever arrive to clear it. Publish terminal
// bookends now so a later reconnect does not re-hydrate a ghost "Running".
// A deliberate restart never reaches here: it stops the runtime without
// emitting an error result, and startSession re-creates the session.
if (!conversationService.hasSession(sessionId) && hasTrackedTaskRecords(sessionId)) {
const knownExit = runtimeExitStoppedSessions.has(sessionId)
runtimeExitStoppedSessions.add(sessionId)
if (!knownExit) runtimeExitFailedSessions.add(sessionId)
activeCliRuns.delete(sessionId)
const turn = activeUserTurns.get(sessionId)
if (turn && !sessionStartupPromises.has(sessionId)) clearActiveUserTurn(sessionId, turn)
void emitStoppedForNonAgentTasksAfterRuntimeExit(sessionId)
void emitAuthoritativeStoppedForActiveAgents(sessionId)
scheduleDisconnectCleanup(sessionId)
return
}
// A turn blocked on permission cannot finish without user input. Keep the // A turn blocked on permission cannot finish without user input. Keep the
// completion watcher for early cleanup, but also enforce the existing // completion watcher for early cleanup, but also enforce the existing
// pending-permission maximum so an abandoned prompt cannot pin the CLI. // pending-permission maximum so an abandoned prompt cannot pin the CLI.
@@ -868,6 +911,8 @@ async function handleUserMessage(
sessionStopRequested.has(sessionId) || agentStopRequestedSessions.has(sessionId) sessionStopRequested.has(sessionId) || agentStopRequestedSessions.has(sessionId)
activeTurn.admissionPending = !collaboration activeTurn.admissionPending = !collaboration
activeUserTurns.set(sessionId, activeTurn) activeUserTurns.set(sessionId, activeTurn)
nonAgentStopRequestedSessions.delete(sessionId)
clearBackgroundTaskDisconnectCeiling(sessionId)
const content = await resolveSessionReferenceContext(message.content, message.sessionReferences, const content = await resolveSessionReferenceContext(message.content, message.sessionReferences,
async id => Boolean(await sessionService.getSessionSummary(id))) async id => Boolean(await sessionService.getSessionSummary(id)))
@@ -1184,9 +1229,25 @@ function bindSessionTurnObserver(sessionId: string): void {
// The observer is independent of renderer subscriptions, so background // The observer is independent of renderer subscriptions, so background
// sessions keep their task/permission lifecycle when no page is open. // sessions keep their task/permission lifecycle when no page is open.
if (!hasActiveClients(sessionId)) { if (!hasActiveClients(sessionId)) {
trackCliRunState(sessionId, message) const cliRunState = trackCliRunState(sessionId, message)
trackCliBackgroundTaskLifecycle(sessionId, message) const taskLifecycle = trackCliBackgroundTaskLifecycle(sessionId, message)
persistThenForwardCliMessage(sessionId, message, () => {}) stopLateAgentTaskIfRequested(sessionId, taskLifecycle)
stopLateNonAgentTaskIfRequested(sessionId, taskLifecycle)
closeLateNonAgentTaskAfterRuntimeExit(sessionId, taskLifecycle)
closeStoppedAgentsAfterRuntimeExit(sessionId, message)
if (!sessionStartupPromises.has(sessionId)) {
refreshBackgroundTaskDisconnectCeiling(sessionId)
// Control-only sessions never had a renderer disconnect. Observe their
// completion after work actually starts; an initial idle frame must
// not arm cleanup ahead of a side-question/Agent control request.
if (!sessionDisconnectWatchers.has(sessionId) &&
(cliRunState === 'running' || taskLifecycle?.running === true)) {
watchTurnCompletionForCleanup(sessionId)
}
}
if (!taskLifecycle?.suppressForward) {
persistThenForwardCliMessage(sessionId, message, () => {})
}
} }
queueMicrotask(() => { queueMicrotask(() => {
if (sessionTurnObservers.get(sessionId) === callback) { if (sessionTurnObservers.get(sessionId) === callback) {
@@ -1207,6 +1268,7 @@ function clearSessionTurnObserver(sessionId: string): void {
function clearActiveUserTurn(sessionId: string, activeTurn: ActiveUserTurnState): void { function clearActiveUserTurn(sessionId: string, activeTurn: ActiveUserTurnState): void {
if (activeUserTurns.get(sessionId) === activeTurn) { if (activeUserTurns.get(sessionId) === activeTurn) {
activeUserTurns.delete(sessionId) activeUserTurns.delete(sessionId)
refreshBackgroundTaskDisconnectCeiling(sessionId)
} }
} }
@@ -1251,7 +1313,13 @@ function forceStopSharedRuntimeForAgentCancellation(sessionId: string): void {
// slash command), otherwise its result can be consumed as the dead turn's. // slash command), otherwise its result can be consumed as the dead turn's.
pendingInterruptedTurnResults.delete(sessionId) pendingInterruptedTurnResults.delete(sessionId)
runtimeExitStoppedSessions.add(sessionId) runtimeExitStoppedSessions.add(sessionId)
// Losing the CLI cannot confirm every shell descendant stopped: POSIX jobs
// can create their own process groups, and Windows has no lifetime guard.
runtimeExitFailedSessions.add(sessionId)
conversationService.stopSession(sessionId) conversationService.stopSession(sessionId)
activeCliRuns.delete(sessionId)
clearBackgroundTaskDisconnectCeiling(sessionId)
cancelSessionDisconnectWatcher(sessionId)
const stoppedTurn = activeUserTurns.get(sessionId) const stoppedTurn = activeUserTurns.get(sessionId)
if ( if (
stoppedTurn?.cancelled && stoppedTurn?.cancelled &&
@@ -1259,7 +1327,9 @@ function forceStopSharedRuntimeForAgentCancellation(sessionId: string): void {
) { ) {
clearActiveUserTurn(sessionId, stoppedTurn) clearActiveUserTurn(sessionId, stoppedTurn)
} }
void emitStoppedForNonAgentTasksAfterRuntimeExit(sessionId) void emitStoppedForNonAgentTasksAfterRuntimeExit(sessionId).then(() => {
scheduleDisconnectedSessionCleanupIfIdle(sessionId)
})
} }
function consumeInterruptedTurnResult(sessionId: string, cliMsg: any): boolean { function consumeInterruptedTurnResult(sessionId: string, cliMsg: any): boolean {
@@ -1298,7 +1368,9 @@ function acknowledgeActiveTurnReplay(sessionId: string, cliMsg: any): boolean {
pendingInterruptedTurnResults.delete(sessionId) pendingInterruptedTurnResults.delete(sessionId)
sessionStopRequested.delete(sessionId) sessionStopRequested.delete(sessionId)
agentStopRequestedSessions.delete(sessionId) agentStopRequestedSessions.delete(sessionId)
nonAgentStopRequestedSessions.delete(sessionId)
runtimeExitStoppedSessions.delete(sessionId) runtimeExitStoppedSessions.delete(sessionId)
runtimeExitFailedSessions.delete(sessionId)
return true return true
} }
@@ -2082,6 +2154,7 @@ async function restartSessionWithPermissionMode(
await conversationService.startSession(sessionId, workDir, sdkUrl, runtimeSettings) await conversationService.startSession(sessionId, workDir, sdkUrl, runtimeSettings)
if (!agentStopRequestedSessions.has(sessionId)) { if (!agentStopRequestedSessions.has(sessionId)) {
runtimeExitStoppedSessions.delete(sessionId) runtimeExitStoppedSessions.delete(sessionId)
runtimeExitFailedSessions.delete(sessionId)
} }
await commitConfirmedPermissionMode(sessionId, mode, workDir) await commitConfirmedPermissionMode(sessionId, mode, workDir)
@@ -2199,6 +2272,7 @@ async function restartSessionWithRuntimeConfig(
const sdkUrl = buildSdkWebSocketUrl(ws, sessionId) const sdkUrl = buildSdkWebSocketUrl(ws, sessionId)
await conversationService.startSession(sessionId, workDir, sdkUrl, runtimeSettings) await conversationService.startSession(sessionId, workDir, sdkUrl, runtimeSettings)
runtimeExitStoppedSessions.delete(sessionId) runtimeExitStoppedSessions.delete(sessionId)
runtimeExitFailedSessions.delete(sessionId)
broadcastAppliedRuntimeConfig(sessionId) broadcastAppliedRuntimeConfig(sessionId)
sendMessage(ws, { type: 'status', state: 'idle' }) sendMessage(ws, { type: 'status', state: 'idle' })
@@ -2227,11 +2301,23 @@ async function restartSessionWithRuntimeConfig(
} }
} }
function handleStopGeneration(ws: SessionConnection) { function handleStopGeneration(
ws: SessionConnection,
options: { reapBackgroundTasks?: boolean } = {},
) {
const { sessionId } = ws.data const { sessionId } = ws.data
emitSessionTurnEvent({ type: 'stopped', sessionId }) emitSessionTurnEvent({ type: 'stopped', sessionId })
const stoppedTurn = activeUserTurns.get(sessionId) const stoppedTurn = activeUserTurns.get(sessionId)
const agentTasks = [...(activeAgentTasks.get(sessionId)?.values() ?? [])] const agentTasks = [...(activeAgentTasks.get(sessionId)?.values() ?? [])]
// A user-issued Stop ends the whole session's work. Background shell tasks
// (Bash/PowerShell run_in_background) are tracked as non-Agent tasks, which
// the Agent-stop bookkeeping below never touches — left alone they keep
// running after the user asked everything to stop, and keep the CLI alive.
// Programmatic stops (stopSessionTurn, runtime-config restart) stay narrow.
const backgroundTasks = options.reapBackgroundTasks === true
? [...(activeNonAgentTasks.get(sessionId)?.values() ?? [])]
: []
const backgroundTaskIds = backgroundTasks.map((task) => task.taskId)
console.log(`[WS] Stop generation requested for session: ${sessionId}`) console.log(`[WS] Stop generation requested for session: ${sessionId}`)
if (stoppedTurn) { if (stoppedTurn) {
@@ -2273,6 +2359,11 @@ function handleStopGeneration(ws: SessionConnection) {
agentTasks.map((task) => requestStopTrackedAgentTask(sessionId, task, ws)), agentTasks.map((task) => requestStopTrackedAgentTask(sessionId, task, ws)),
) )
if (options.reapBackgroundTasks) nonAgentStopRequestedSessions.add(sessionId)
for (const taskId of backgroundTaskIds) {
stopTrackedBackgroundTask(sessionId, taskId)
}
if ( if (
stoppedTurn && stoppedTurn &&
conversationService.hasSession(sessionId) && conversationService.hasSession(sessionId) &&
@@ -2294,7 +2385,10 @@ function handleStopGeneration(ws: SessionConnection) {
conversationService.sendInterrupt(sessionId) conversationService.sendInterrupt(sessionId)
} }
if ((stoppedTurn || agentTasks.length > 0) && conversationService.hasSession(sessionId)) { if (
(stoppedTurn || agentTasks.length > 0 || backgroundTaskIds.length > 0) &&
conversationService.hasSession(sessionId)
) {
// Force-kill if still running after 3 seconds // Force-kill if still running after 3 seconds
setTimeout(() => { setTimeout(() => {
const stoppedForegroundStillCurrent = Boolean( const stoppedForegroundStillCurrent = Boolean(
@@ -2315,8 +2409,17 @@ function handleStopGeneration(ws: SessionConnection) {
[...(activeAgentTasks.get(sessionId)?.values() ?? [])].some( [...(activeAgentTasks.get(sessionId)?.values() ?? [])].some(
(task) => !task.localStopConfirmed, (task) => !task.localStopConfirmed,
) )
const currentTurn = activeUserTurns.get(sessionId)
const stoppedBackgroundStillActive =
nonAgentStopRequestedSessions.has(sessionId) &&
(!currentTurn || currentTurn === stoppedTurn) &&
backgroundTasks.some((task) =>
activeNonAgentTasks.get(sessionId)?.get(task.taskId) === task &&
!task.localStopConfirmed,
)
if ( if (
(stoppedForegroundStillCurrent || stoppedAgentsStillActive) && (stoppedForegroundStillCurrent || stoppedAgentsStillActive || stoppedBackgroundStillActive) &&
!migrationMaintenance.isActive &&
conversationService.hasSession(sessionId) conversationService.hasSession(sessionId)
) { ) {
console.log(`[WS] Force-killing CLI subprocess for session: ${sessionId}`) console.log(`[WS] Force-killing CLI subprocess for session: ${sessionId}`)
@@ -2359,46 +2462,117 @@ async function requestStopBackgroundTask(
return return
} }
const tracked = activeNonAgentTasks.get(sessionId)?.get(taskId)
if (tracked?.localStopConfirmed) {
await emitAuthoritativeNonAgentStopped(sessionId, tracked)
return
}
if (tracked) tracked.stopRequested = true
try { try {
const response = await conversationService.requestControl(sessionId, { const response = await conversationService.requestControl(sessionId, {
subtype: 'stop_task', subtype: 'stop_task',
task_id: taskId, task_id: taskId,
}) })
if (activeNonAgentTasks.get(sessionId)?.get(taskId) !== tracked) return
if (response?.reason === 'not_found') { if (response?.reason === 'not_found') {
convergeEvictedBackgroundTaskStop(sessionId, taskId) await convergeEvictedBackgroundTaskStop(sessionId, taskId)
} else if (tracked && !tracked.localStopConfirmed &&
activeNonAgentTasks.get(sessionId)?.get(taskId) === tracked) {
tracked.localStopConfirmed = true
tracked.terminalStatus = 'stopped'
await emitAuthoritativeNonAgentStopped(sessionId, tracked)
} }
} catch (error) { } catch (error) {
reportBackgroundTaskStopFailure(sessionId, ws, taskId, error) reportBackgroundTaskStopFailure(sessionId, ws, taskId, error)
} }
} }
/**
* Stop a tracked background shell task without an originating socket, for a
* whole-session Stop. Failures are logged rather than answered to one client,
* and a late `not_found` is converged to a terminal bookend just like the
* panel's per-task Stop.
*/
function stopTrackedBackgroundTask(sessionId: string, taskId: string): void {
const task = activeNonAgentTasks.get(sessionId)?.get(taskId)
if (!task || task.stopRequested || task.localStopConfirmed) return
task.stopRequested = true
void migrationMaintenance.track(conversationService
.requestControl(sessionId, { subtype: 'stop_task', task_id: taskId })
.then((response) => {
if (activeNonAgentTasks.get(sessionId)?.get(taskId) !== task) return
if (response?.reason === 'not_found') {
return convergeEvictedBackgroundTaskStop(sessionId, taskId)
}
if (!task.localStopConfirmed && activeNonAgentTasks.get(sessionId)?.get(taskId) === task) {
task.localStopConfirmed = true
task.terminalStatus = 'stopped'
return emitAuthoritativeNonAgentStopped(sessionId, task)
}
})
.catch((error) => {
console.warn(
`[WS] Failed to stop background task ${taskId} for session ${sessionId}:`,
error,
)
}))
}
function stopLateNonAgentTaskIfRequested(
sessionId: string,
lifecycle: CliBackgroundTaskLifecycle | null,
): void {
if (!lifecycle?.running || isAgentTaskType(lifecycle.taskType) ||
!nonAgentStopRequestedSessions.has(sessionId) || runtimeExitStoppedSessions.has(sessionId)) return
const task = activeNonAgentTasks.get(sessionId)?.get(lifecycle.taskId)
if (!task || task.stopRequested) return
if (!conversationService.hasSession(sessionId)) {
runtimeExitStoppedSessions.add(sessionId)
runtimeExitFailedSessions.add(sessionId)
void emitStoppedForNonAgentTasksAfterRuntimeExit(sessionId, 'failed')
return
}
const stoppedTurn = activeUserTurns.get(sessionId)
stopTrackedBackgroundTask(sessionId, lifecycle.taskId)
// A queued shell start can arrive after the original Stop's task snapshot.
// Its fallback belongs to that Stop, never to a newly admitted user turn.
setTimeout(() => {
const currentTurn = activeUserTurns.get(sessionId)
if (migrationMaintenance.isActive || !nonAgentStopRequestedSessions.has(sessionId) ||
(currentTurn && currentTurn !== stoppedTurn) || currentTurn?.replacementAfterStop ||
activeNonAgentTasks.get(sessionId)?.get(task.taskId) !== task ||
task.localStopConfirmed || !conversationService.hasSession(sessionId)) return
forceStopSharedRuntimeForAgentCancellation(sessionId)
void emitAuthoritativeStoppedForActiveAgents(sessionId)
}, 3_000)
}
/** /**
* The CLI evicts a shell task the turn after it terminates (and a process * The CLI evicts a shell task the turn after it terminates (and a process
* restart clears the registry outright), so a Stop that lands late is * restart clears the registry outright), so a Stop that lands late is
* answered with `not_found`. That is the stop's goal state, not a failure: * answered with `not_found`. Persist that terminal state before dropping
* drop the task from local tracking and send the terminal notification * local tracking and sending the terminal notification
* clients need to converge an entry they still show as running. Reporting * clients need to converge an entry they still show as running. Reporting
* `No task found with ID` here only re-arms the stop button for a task that * `No task found with ID` here only re-arms the stop button for a task that
* can never be stopped again. * can never be stopped again.
*/ */
function convergeEvictedBackgroundTaskStop(sessionId: string, taskId: string): void { function convergeEvictedBackgroundTaskStop(sessionId: string, taskId: string): Promise<boolean> {
const tracked = activeNonAgentTasks.get(sessionId)?.get(taskId) let tracked = activeNonAgentTasks.get(sessionId)?.get(taskId)
untrackCliBackgroundTask(sessionId, taskId) if (!tracked) {
const description = tracked?.description tracked = { taskId, toolUseId: taskId }
sendToSession(sessionId, { let tasks = activeNonAgentTasks.get(sessionId)
type: 'system_notification', if (!tasks) {
subtype: 'task_notification', tasks = new Map()
message: description ? `${description} stopped` : 'Background task stopped', activeNonAgentTasks.set(sessionId, tasks)
data: { }
type: 'system', tasks.set(taskId, tracked)
subtype: 'task_notification', }
task_id: taskId, if (!tracked.localStopConfirmed || !tracked.terminalStatus) {
tool_use_id: tracked?.toolUseId, tracked.terminalStatus = 'stopped'
status: 'stopped', tracked.terminalMessage = true
summary: description ? `${description} stopped` : 'Background task stopped', }
timestamp: new Date().toISOString(), tracked.localStopConfirmed = true
}, return emitAuthoritativeNonAgentStopped(sessionId, tracked)
})
} }
const AGENT_STOP_CONTROL_TIMEOUT_MS = 3_000 const AGENT_STOP_CONTROL_TIMEOUT_MS = 3_000
@@ -2797,29 +2971,96 @@ function emitAuthoritativeStoppedForActiveAgents(sessionId: string): Promise<boo
})) }))
} }
function emitStoppedForNonAgentTasksAfterRuntimeExit(sessionId: string): Promise<void[]> { function emitAuthoritativeNonAgentStopped(
sessionId: string,
task: ActiveNonAgentTaskState,
): Promise<boolean> {
if (activeNonAgentTasks.get(sessionId)?.get(task.taskId) !== task ||
sessionClearInProgress.has(sessionId)) return Promise.resolve(false)
if (task.finalization) return task.finalization
clearAgentStopFinalizationRetry(task)
task.localStopConfirmed = true
task.terminalStatus ??= 'stopped'
const status = task.terminalStatus
const cliMsg = {
type: 'system',
subtype: 'task_notification',
task_id: task.taskId,
tool_use_id: task.toolUseId,
...(task.taskType ? { task_type: task.taskType } : {}),
...(task.description ? { description: task.description } : {}),
...(task.ownerAgentId ? { owner_agent_id: task.ownerAgentId } : {}),
status,
...(task.terminalMessage
? { message: task.terminalSummary ?? `${task.description ?? 'Background task'} ${status}` }
: {}),
summary: task.terminalSummary ?? `${task.description ?? 'Background task'} ${status}`,
timestamp: new Date().toISOString(),
}
const finalization = migrationMaintenance.track((async () => {
for (let attempt = 0; attempt < AUTHORITATIVE_STOP_PERSIST_ATTEMPTS; attempt++) {
if (activeNonAgentTasks.get(sessionId)?.get(task.taskId) !== task ||
sessionClearInProgress.has(sessionId)) return false
try {
await (persistCliTaskNotification(sessionId, cliMsg, {
propagateFailure: true,
timeoutMs: AUTHORITATIVE_STOP_PERSIST_TIMEOUT_MS,
}) ?? Promise.resolve())
if (activeNonAgentTasks.get(sessionId)?.get(task.taskId) !== task ||
sessionClearInProgress.has(sessionId)) return false
task.stopFailureMessage = undefined
markTaskAuthoritativelyStopped(sessionId, task.taskId)
untrackCliBackgroundTask(sessionId, task.taskId)
forwardCliMessageToSessionClients(sessionId, cliMsg)
scheduleDisconnectedSessionCleanupIfIdle(sessionId)
return true
} catch {
// A rejected write is evicted from the cache so each attempt retries it.
}
}
if (activeNonAgentTasks.get(sessionId)?.get(task.taskId) !== task ||
sessionClearInProgress.has(sessionId)) return false
task.stopFailureMessage = 'Background task ended, but its terminal state could not be saved'
sendToSession(sessionId, {
type: 'background_task_stop_failed',
taskId: task.taskId,
message: task.stopFailureMessage,
})
const retryCount = task.finalizationRetryCount ?? 0
const delay = AGENT_STOP_FINALIZATION_RETRY_DELAYS_MS[retryCount]
if (delay !== undefined && !migrationMaintenance.isActive) {
task.finalizationRetryCount = retryCount + 1
task.finalizationRetryTimer = setTimeout(() => {
task.finalizationRetryTimer = undefined
if (!migrationMaintenance.isActive) void emitAuthoritativeNonAgentStopped(sessionId, task)
}, delay)
if (typeof task.finalizationRetryTimer === 'object') task.finalizationRetryTimer.unref?.()
}
scheduleDisconnectedSessionCleanupIfIdle(sessionId)
return false
})())
task.finalization = finalization
void finalization.then(() => {
if (task.finalization === finalization) task.finalization = undefined
})
return finalization
}
function emitStoppedForNonAgentTasksAfterRuntimeExit(
sessionId: string,
status: 'stopped' | 'failed' = 'failed',
): Promise<void[]> {
if (status === 'failed') runtimeExitFailedSessions.add(sessionId)
const tasks = [...(activeNonAgentTasks.get(sessionId)?.values() ?? [])] const tasks = [...(activeNonAgentTasks.get(sessionId)?.values() ?? [])]
return Promise.all(tasks.map(async (task) => { return Promise.all(tasks.map(async (task) => {
if (activeNonAgentTasks.get(sessionId)?.get(task.taskId) !== task) return if (!task.localStopConfirmed || !task.terminalStatus) {
// Killing the shared CLI also terminates Bash/Dream/workflow work. Claim task.terminalStatus = status
// each task before awaiting persistence so concurrent force-stop paths task.terminalSummary = status === 'failed'
// cannot publish duplicate terminal bookends. ? `${task.description ?? task.taskId}: runtime connection lost; task completion could not be confirmed`
markTaskAuthoritativelyStopped(sessionId, task.taskId) : `${task.description ?? task.taskId} stopped because the runtime exited`
untrackCliBackgroundTask(sessionId, task.taskId)
const cliMsg = {
type: 'system',
subtype: 'task_notification',
task_id: task.taskId,
tool_use_id: task.toolUseId,
...(task.taskType ? { task_type: task.taskType } : {}),
...(task.description ? { description: task.description } : {}),
...(task.ownerAgentId ? { owner_agent_id: task.ownerAgentId } : {}),
status: 'stopped',
summary: `${task.description ?? task.taskId} stopped because the runtime exited`,
timestamp: new Date().toISOString(),
} }
await (persistCliTaskNotification(sessionId, cliMsg) ?? Promise.resolve()) task.localStopConfirmed = true
forwardCliMessageToSessionClients(sessionId, cliMsg) await emitAuthoritativeNonAgentStopped(sessionId, task)
})) }))
} }
@@ -2854,12 +3095,15 @@ function closeStoppedAgentsAfterRuntimeExit(sessionId: string, cliMsg: any): voi
if ( if (
cliMsg?.type === 'result' && cliMsg?.type === 'result' &&
cliMsg.is_error && cliMsg.is_error &&
agentStopRequestedSessions.has(sessionId) &&
!conversationService.hasSession(sessionId) !conversationService.hasSession(sessionId)
) { ) {
// A vanished runtime cannot deliver further task results. Record failed
// shell outcomes instead of claiming their process termination was confirmed;
// even without a user Stop, reconnects must not rehydrate ghost Running tasks.
runtimeExitStoppedSessions.add(sessionId) runtimeExitStoppedSessions.add(sessionId)
runtimeExitFailedSessions.add(sessionId)
void emitAuthoritativeStoppedForActiveAgents(sessionId) void emitAuthoritativeStoppedForActiveAgents(sessionId)
void emitStoppedForNonAgentTasksAfterRuntimeExit(sessionId) void emitStoppedForNonAgentTasksAfterRuntimeExit(sessionId, 'failed')
} }
} }
@@ -3196,6 +3440,7 @@ function cleanupSessionRuntimeState(
options?: { preserveRetryableAgentStops?: boolean }, options?: { preserveRetryableAgentStops?: boolean },
) { ) {
cancelSessionDisconnectWatcher(sessionId) cancelSessionDisconnectWatcher(sessionId)
clearBackgroundTaskDisconnectCeiling(sessionId)
clearSessionTurnObserver(sessionId) clearSessionTurnObserver(sessionId)
clearAgentRuntimeState(sessionId, { clearAgentRuntimeState(sessionId, {
preserveRetryableStops: options?.preserveRetryableAgentStops, preserveRetryableStops: options?.preserveRetryableAgentStops,
@@ -3363,6 +3608,7 @@ async function ensureCliSessionStarted(
await conversationService.startSession(sessionId, workDir, sdkUrl, startupSettings) await conversationService.startSession(sessionId, workDir, sdkUrl, startupSettings)
bindSessionTurnObserver(sessionId) bindSessionTurnObserver(sessionId)
runtimeExitStoppedSessions.delete(sessionId) runtimeExitStoppedSessions.delete(sessionId)
runtimeExitFailedSessions.delete(sessionId)
})() })()
sessionStartupPromises.set(sessionId, startup) sessionStartupPromises.set(sessionId, startup)
@@ -3411,7 +3657,9 @@ export async function ensureCliSessionStartedForControl(
sdkUrl.toString(), sdkUrl.toString(),
{ ...runtimeSettings, resumeInterruptedTurn: false }, { ...runtimeSettings, resumeInterruptedTurn: false },
) )
bindSessionTurnObserver(sessionId)
runtimeExitStoppedSessions.delete(sessionId) runtimeExitStoppedSessions.delete(sessionId)
runtimeExitFailedSessions.delete(sessionId)
})() })()
sessionStartupPromises.set(sessionId, startup) sessionStartupPromises.set(sessionId, startup)
@@ -4145,6 +4393,16 @@ function scheduleDisconnectCleanup(sessionId: string): void {
} }
console.log(`[WS] Session ${sessionId} not reconnected after ${cleanupDelayMs}ms, stopping CLI subprocess`) console.log(`[WS] Session ${sessionId} not reconnected after ${cleanupDelayMs}ms, stopping CLI subprocess`)
if (permissionBoundExpired) {
const turn = activeUserTurns.get(sessionId)
if (turn) {
turn.cancelled = true
turn.replacementAfterStop = false
}
forceStopSharedRuntimeForAgentCancellation(sessionId)
void emitAuthoritativeStoppedForActiveAgents(sessionId)
return
}
conversationService.stopSession(sessionId) conversationService.stopSession(sessionId)
cleanupSessionRuntimeState(sessionId, { preserveRetryableAgentStops: true }) cleanupSessionRuntimeState(sessionId, { preserveRetryableAgentStops: true })
}, cleanupDelayMs) }, cleanupDelayMs)
@@ -4152,6 +4410,7 @@ function scheduleDisconnectCleanup(sessionId: string): void {
} }
function scheduleDisconnectedSessionCleanupIfIdle(sessionId: string): void { function scheduleDisconnectedSessionCleanupIfIdle(sessionId: string): void {
refreshBackgroundTaskDisconnectCeiling(sessionId)
if ( if (
hasActiveClients(sessionId) || hasActiveClients(sessionId) ||
hasActiveSessionWork(sessionId) hasActiveSessionWork(sessionId)
@@ -4177,8 +4436,10 @@ function watchTurnCompletionForCleanup(sessionId: string): void {
const cliRunState = trackCliRunState(sessionId, cliMsg) const cliRunState = trackCliRunState(sessionId, cliMsg)
const taskLifecycle = trackCliBackgroundTaskLifecycle(sessionId, cliMsg) const taskLifecycle = trackCliBackgroundTaskLifecycle(sessionId, cliMsg)
stopLateAgentTaskIfRequested(sessionId, taskLifecycle) stopLateAgentTaskIfRequested(sessionId, taskLifecycle)
stopLateNonAgentTaskIfRequested(sessionId, taskLifecycle)
closeLateNonAgentTaskAfterRuntimeExit(sessionId, taskLifecycle) closeLateNonAgentTaskAfterRuntimeExit(sessionId, taskLifecycle)
closeStoppedAgentsAfterRuntimeExit(sessionId, cliMsg) closeStoppedAgentsAfterRuntimeExit(sessionId, cliMsg)
refreshBackgroundTaskDisconnectCeiling(sessionId)
if ( if (
(cliRunState === 'running' || taskLifecycle?.running) && (cliRunState === 'running' || taskLifecycle?.running) &&
!hasActiveClients(sessionId) !hasActiveClients(sessionId)
@@ -4191,6 +4452,10 @@ function watchTurnCompletionForCleanup(sessionId: string): void {
const cleanupTimer = sessionCleanupTimers.get(sessionId) const cleanupTimer = sessionCleanupTimers.get(sessionId)
if (cleanupTimer) clearTimeout(cleanupTimer) if (cleanupTimer) clearTimeout(cleanupTimer)
sessionCleanupTimers.delete(sessionId) sessionCleanupTimers.delete(sessionId)
// Arm the hard ceiling for a non-Agent background task that never
// reports back. Running Agent tasks keep their existing handling: only
// they can prove their own stop through the Agent finalization path.
refreshBackgroundTaskDisconnectCeiling(sessionId)
} }
return return
} }
@@ -4211,7 +4476,7 @@ function watchTurnCompletionForCleanup(sessionId: string): void {
const backgroundTaskCompleted = taskLifecycle?.running === false const backgroundTaskCompleted = taskLifecycle?.running === false
if (!foregroundTurnCompleted && !cliRunCompleted && !backgroundTaskCompleted) return if (!foregroundTurnCompleted && !cliRunCompleted && !backgroundTaskCompleted) return
if (hasActiveCliRun(sessionId)) return if (hasActiveCliRun(sessionId)) return
if (hasActiveBackgroundTasks(sessionId)) return if (hasActiveBackgroundTasks(sessionId) || hasActiveTeamWorkForParent(sessionId)) return
if ( if (
!foregroundTurnCompleted && !foregroundTurnCompleted &&
!cliRunCompleted && !cliRunCompleted &&
@@ -4219,6 +4484,7 @@ function watchTurnCompletionForCleanup(sessionId: string): void {
) return ) return
cancelSessionDisconnectWatcher(sessionId) cancelSessionDisconnectWatcher(sessionId)
clearBackgroundTaskDisconnectCeiling(sessionId)
// All observed work finished while still disconnected — fall back to the // All observed work finished while still disconnected — fall back to the
// bounded idle timer rather than stopping the CLI immediately. // bounded idle timer rather than stopping the CLI immediately.
if (!hasActiveClients(sessionId)) { if (!hasActiveClients(sessionId)) {
@@ -4226,6 +4492,7 @@ function watchTurnCompletionForCleanup(sessionId: string): void {
} }
} }
refreshBackgroundTaskDisconnectCeiling(sessionId)
conversationService.onOutput(sessionId, onComplete) conversationService.onOutput(sessionId, onComplete)
sessionDisconnectWatchers.set(sessionId, () => { sessionDisconnectWatchers.set(sessionId, () => {
conversationService.removeOutputCallback(sessionId, onComplete) conversationService.removeOutputCallback(sessionId, onComplete)
@@ -4261,6 +4528,73 @@ function cancelSessionDisconnectWatcher(sessionId: string): void {
} }
} }
/**
* Arm the hard ceiling for a disconnected session kept alive only by background
* shell tasks. Idempotent: an already-armed ceiling is left in place so the
* clock measures from the first observation, not from the latest event.
*/
function isDisconnectedBackgroundOnlySession(sessionId: string): boolean {
return !migrationMaintenance.isActive &&
!hasActiveClients(sessionId) &&
hasTrackedNonAgentTasks(sessionId) &&
!hasPendingOrActiveUserTurn(sessionId) &&
!hasActiveCliRun(sessionId) &&
(activeAgentTasks.get(sessionId)?.size ?? 0) === 0 &&
!hasActiveTeamWorkForParent(sessionId) &&
conversationService.getPendingPermissionRequests(sessionId).length === 0 &&
computerUseApprovalService.getPendingRequests(sessionId).length === 0
}
function refreshBackgroundTaskDisconnectCeiling(sessionId: string): void {
if (isDisconnectedBackgroundOnlySession(sessionId)) {
armBackgroundTaskDisconnectCeiling(sessionId)
} else {
clearBackgroundTaskDisconnectCeiling(sessionId)
}
}
function armBackgroundTaskDisconnectCeiling(sessionId: string): void {
if (backgroundTaskCleanupTimers.has(sessionId)) return
const timer = setTimeout(() => {
if (backgroundTaskCleanupTimers.get(sessionId) !== timer) return
backgroundTaskCleanupTimers.delete(sessionId)
if (!isDisconnectedBackgroundOnlySession(sessionId)) return
console.log(
`[WS] Session ${sessionId} kept alive by background tasks for ${BACKGROUND_TASK_DISCONNECT_MAX_MS}ms without a client; stopping CLI subprocess`,
)
forceStopSharedRuntimeForAgentCancellation(sessionId)
void emitAuthoritativeStoppedForActiveAgents(sessionId)
}, BACKGROUND_TASK_DISCONNECT_MAX_MS)
backgroundTaskCleanupTimers.set(sessionId, timer)
}
function clearBackgroundTaskDisconnectCeiling(sessionId: string): void {
const timer = backgroundTaskCleanupTimers.get(sessionId)
if (timer) {
clearTimeout(timer)
backgroundTaskCleanupTimers.delete(sessionId)
}
}
/**
* Any tracked task record, Agent or not. Used to decide whether a dead runtime
* left behind ghost "Running" entries that need a terminal bookend.
*/
function hasTrackedTaskRecords(sessionId: string): boolean {
return (activeNonAgentTasks.get(sessionId)?.size ?? 0) > 0 ||
(activeAgentTasks.get(sessionId)?.size ?? 0) > 0
}
/**
* Non-Agent background shell tasks only (Bash/Dream/workflow). Unlike
* {@link hasActiveBackgroundTasks}, this never counts an Agent task, whose
* lifecycle is owned by the Agent finalization path.
*/
function hasTrackedNonAgentTasks(sessionId: string): boolean {
return [...(activeNonAgentTasks.get(sessionId)?.values() ?? [])]
.some((task) => !task.localStopConfirmed)
}
function replayPendingPermissionRequests( function replayPendingPermissionRequests(
ws: SessionConnection, ws: SessionConnection,
sessionId: string, sessionId: string,
@@ -4560,6 +4894,7 @@ function bindClientSessionOutput(
trackCliRunState(sessionId, cliMsg) trackCliRunState(sessionId, cliMsg)
const taskLifecycle = trackCliBackgroundTaskLifecycle(sessionId, cliMsg) const taskLifecycle = trackCliBackgroundTaskLifecycle(sessionId, cliMsg)
stopLateAgentTaskIfRequested(sessionId, taskLifecycle) stopLateAgentTaskIfRequested(sessionId, taskLifecycle)
stopLateNonAgentTaskIfRequested(sessionId, taskLifecycle)
closeLateNonAgentTaskAfterRuntimeExit(sessionId, taskLifecycle) closeLateNonAgentTaskAfterRuntimeExit(sessionId, taskLifecycle)
closeStoppedAgentsAfterRuntimeExit(sessionId, cliMsg) closeStoppedAgentsAfterRuntimeExit(sessionId, cliMsg)
if (taskLifecycle?.suppressForward) return if (taskLifecycle?.suppressForward) return
@@ -5191,6 +5526,7 @@ export function getActiveSessionIds(): string[] {
export function __resetWebSocketHandlerStateForTests(): void { export function __resetWebSocketHandlerStateForTests(): void {
for (const timer of sessionCleanupTimers.values()) clearTimeout(timer) for (const timer of sessionCleanupTimers.values()) clearTimeout(timer)
for (const timer of backgroundTaskCleanupTimers.values()) clearTimeout(timer)
for (const timer of prewarmIdleTimers.values()) clearTimeout(timer) for (const timer of prewarmIdleTimers.values()) clearTimeout(timer)
for (const remove of sessionDisconnectWatchers.values()) remove() for (const remove of sessionDisconnectWatchers.values()) remove()
for (const tasks of activeAgentTasks.values()) { for (const tasks of activeAgentTasks.values()) {
@@ -5202,6 +5538,7 @@ export function __resetWebSocketHandlerStateForTests(): void {
taskNotificationPersistence.clear() taskNotificationPersistence.clear()
sessionTranscriptEpochs.clear() sessionTranscriptEpochs.clear()
sessionCleanupTimers.clear() sessionCleanupTimers.clear()
backgroundTaskCleanupTimers.clear()
sessionDisconnectWatchers.clear() sessionDisconnectWatchers.clear()
prewarmPendingSessions.clear() prewarmPendingSessions.clear()
prewarmedSessions.clear() prewarmedSessions.clear()
@@ -5213,7 +5550,9 @@ export function __resetWebSocketHandlerStateForTests(): void {
activeNonAgentTasks.clear() activeNonAgentTasks.clear()
authoritativeStoppedTaskIds.clear() authoritativeStoppedTaskIds.clear()
agentStopRequestedSessions.clear() agentStopRequestedSessions.clear()
nonAgentStopRequestedSessions.clear()
runtimeExitStoppedSessions.clear() runtimeExitStoppedSessions.clear()
runtimeExitFailedSessions.clear()
pendingInterruptedTurnResults.clear() pendingInterruptedTurnResults.clear()
sessionClearInProgress.clear() sessionClearInProgress.clear()
sessionStopRequested.clear() sessionStopRequested.clear()
+3 -2
View File
@@ -1,4 +1,4 @@
import { execFileSync, spawn } from 'child_process' import { execFileSync } from 'child_process'
import { constants as fsConstants, readFileSync, unlinkSync } from 'fs' import { constants as fsConstants, readFileSync, unlinkSync } from 'fs'
import { type FileHandle, mkdir, open, realpath } from 'fs/promises' import { type FileHandle, mkdir, open, realpath } from 'fs/promises'
import memoize from 'lodash-es/memoize.js' import memoize from 'lodash-es/memoize.js'
@@ -35,6 +35,7 @@ import { getPlatform } from './platform.js'
import { SandboxManager } from './sandbox/sandbox-adapter.js' import { SandboxManager } from './sandbox/sandbox-adapter.js'
import { invalidateSessionEnvCache } from './sessionEnvironment.js' import { invalidateSessionEnvCache } from './sessionEnvironment.js'
import { createBashShellProvider } from './shell/bashProvider.js' import { createBashShellProvider } from './shell/bashProvider.js'
import { spawnWithParentProcessGuard } from './shell/parentProcessGuard.js'
import { getCachedPowerShellPath } from './shell/powershellDetection.js' import { getCachedPowerShellPath } from './shell/powershellDetection.js'
import { createPowerShellProvider } from './shell/powershellProvider.js' import { createPowerShellProvider } from './shell/powershellProvider.js'
import type { ShellProvider, ShellType } from './shell/shellProvider.js' import type { ShellProvider, ShellType } from './shell/shellProvider.js'
@@ -313,7 +314,7 @@ export async function exec(
} }
try { try {
const childProcess = spawn(spawnBinary, shellArgs, { const childProcess = spawnWithParentProcessGuard(spawnBinary, shellArgs, {
env: { env: {
...subprocessEnv(), ...subprocessEnv(),
SHELL: shellType === 'bash' ? binShell : undefined, SHELL: shellType === 'bash' ? binShell : undefined,
+175
View File
@@ -0,0 +1,175 @@
import { afterEach, expect, test } from 'bun:test'
import { spawn, type ChildProcess } from 'child_process'
import { closeSync, mkdtempSync, openSync, readFileSync, rmSync, writeFileSync } from 'fs'
import { tmpdir } from 'os'
import { join } from 'path'
import { spawnWithParentProcessGuard } from './parentProcessGuard.js'
const directories: string[] = []
const children: ChildProcess[] = []
const wait = (ms: number) => new Promise(resolve => setTimeout(resolve, ms))
afterEach(() => {
for (const child of children.splice(0)) {
if (!child.pid) continue
try { process.kill(-child.pid, 'SIGKILL') } catch {}
try { child.kill('SIGKILL') } catch {}
}
for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true })
})
function fixtureDirectory(): string {
const directory = mkdtempSync(join(tmpdir(), 'shell-parent-guard-'))
directories.push(directory)
return directory
}
function exited(child: ChildProcess): Promise<number | null> {
if (child.exitCode !== null || child.signalCode !== null) return Promise.resolve(child.exitCode)
return new Promise((resolve, reject) => {
child.once('exit', code => resolve(code))
child.once('error', reject)
})
}
async function waitUntil(predicate: () => boolean): Promise<void> {
const deadline = Date.now() + 3_000
while (!predicate()) {
if (Date.now() >= deadline) throw new Error('Timed out waiting for fixture process')
await wait(10)
}
}
function ticks(path: string): number {
try { return readFileSync(path, 'utf8').trim().split('\n').length } catch { return 0 }
}
test.skipIf(process.platform === 'win32')('kills a detached shell and its descendants after the runtime receives SIGKILL', async () => {
const directory = fixtureDirectory()
const output = join(directory, 'ticks')
const pidPath = join(directory, 'child-pid')
const fixture = join(directory, 'runtime.ts')
writeFileSync(fixture, `
import { openSync, writeFileSync } from 'fs'
import { spawnWithParentProcessGuard } from ${JSON.stringify(import.meta.dir + '/parentProcessGuard.ts')}
const child = spawnWithParentProcessGuard('/bin/sh', ['-c', 'while true; do echo alive; sleep 0.02; done', 'shell'], {
detached: true,
stdio: ['pipe', openSync(${JSON.stringify(output)}, 'a'), 'ignore'],
})
writeFileSync(${JSON.stringify(pidPath)}, String(child.pid))
setInterval(() => {}, 1000)
`)
// Write through an inherited file fd, as background Shell.exec does. Pipe
// output would stop when the CLI dies even if its shell kept running.
const runtime = spawn(process.execPath, ['--no-env-file', fixture], {
cwd: directory,
env: { PATH: process.env.PATH, HOME: directory, CLAUDE_CONFIG_DIR: join(directory, 'config') },
detached: true,
stdio: 'ignore',
})
children.push(runtime)
let shellPid: number | undefined
try {
await waitUntil(() => ticks(output) >= 3)
shellPid = Number(readFileSync(pidPath, 'utf8'))
runtime.kill('SIGKILL')
await exited(runtime)
expect(runtime.signalCode).toBe('SIGKILL')
await wait(150)
const afterExit = ticks(output)
await wait(150)
expect(ticks(output)).toBe(afterExit)
} finally {
if (shellPid) {
try { process.kill(-shellPid, 'SIGKILL') } catch {}
}
}
})
test.skipIf(process.platform === 'win32')('preserves stdin, arguments, bare wait, stdout, stderr and the exit code', async () => {
const child = spawnWithParentProcessGuard('/bin/sh', ['-c',
'IFS= read -r line; sleep 0.02 & wait; printf "%s|%s|%s\\n" "$line" "$1" "$GUARD_TEST_VALUE"; echo err >&2; exit 7',
'sandbox-wrapper', 'literal $value `command` "quotes"',
], {
detached: true,
env: { ...process.env, GUARD_TEST_VALUE: 'env value' },
stdio: ['pipe', 'pipe', 'pipe'],
})
children.push(child)
let stdout = ''
let stderr = ''
child.stdout!.on('data', data => { stdout += data })
child.stderr!.on('data', data => { stderr += data })
child.stdin!.end('user input\n')
expect(await exited(child)).toBe(7)
expect(stdout).toBe('user input|literal $value `command` "quotes"|env value\n')
expect(stderr).toBe('err\n')
})
test.skipIf(process.platform === 'win32')('preserves file output and leaves no process group after normal completion', async () => {
const output = join(fixtureDirectory(), 'output')
const fd = openSync(output, 'a')
let child: ChildProcess
try {
child = spawnWithParentProcessGuard('/bin/sh', ['-c', 'echo stdout; echo stderr >&2; exit 0'], {
detached: true,
stdio: ['pipe', fd, fd],
})
children.push(child)
} finally {
closeSync(fd)
}
expect(await exited(child)).toBe(0)
expect(readFileSync(output, 'utf8')).toBe('stdout\nstderr\n')
expect(() => process.kill(-child.pid!, 0)).toThrow()
})
test.skipIf(process.platform === 'win32')('preserves an existing sandbox wrapper and does not expose the lifetime fd to commands', async () => {
const child = spawnWithParentProcessGuard('/bin/sh', ['-c',
'exec /bin/sh -c "$1" "$2" "$3"', 'sandbox-wrapper',
'if (: <&3) 2>/dev/null; then echo leaked; exit 1; fi; printf "%s\\n" "$1"',
'inner-shell', 'literal $arg `command` "quoted"',
], {
detached: false,
stdio: ['pipe', 'pipe', 'pipe'],
})
children.push(child)
let stdout = ''
child.stdout!.on('data', data => { stdout += data })
expect(await exited(child)).toBe(0)
expect(stdout).toBe('literal $arg `command` "quoted"\n')
})
for (const action of ['terminate', 'abort'] as const) {
test.skipIf(process.platform === 'win32')(`reaps the target and guardian when the supervisor is ${action === 'abort' ? 'aborted' : 'terminated'}`, async () => {
const output = join(fixtureDirectory(), 'ticks')
const fd = openSync(output, 'a')
const controller = new AbortController()
let child: ChildProcess
try {
child = spawnWithParentProcessGuard('/bin/sh', ['-c', 'while true; do echo alive; sleep 0.02; done'], {
detached: true,
signal: controller.signal,
stdio: ['pipe', fd, fd],
})
children.push(child)
} finally {
closeSync(fd)
}
const errors: Error[] = []
child.on('error', error => { errors.push(error) })
const exit = new Promise(resolve => child.once('exit', resolve))
await waitUntil(() => ticks(output) >= 3)
if (action === 'abort') controller.abort()
else child.kill('SIGTERM')
await exit
await waitUntil(() => {
try { process.kill(-child.pid!, 0); return false } catch { return true }
})
const afterExit = ticks(output)
await wait(75)
expect(ticks(output)).toBe(afterExit)
if (action === 'abort') expect(errors[0]?.name).toBe('AbortError')
else expect(errors).toHaveLength(0)
})
}
+51
View File
@@ -0,0 +1,51 @@
import { spawn, type ChildProcess, type SpawnOptions } from 'child_process'
import type { Duplex } from 'stream'
// Keep the guardian outside the target shell's job table: a command's bare
// `wait` must not wait for our lifetime watcher. Arguments reach the original
// shell unchanged, including login flags and sandbox command wrappers.
const PARENT_PROCESS_GUARD = `
(
if ! IFS= read -r lifetime <&3; then
kill -KILL "-$$"
fi
) </dev/null >/dev/null 2>&1 &
lifetime_guard=$!
"$@" 3<&-
lifetime_status=$?
kill "$lifetime_guard" 2>/dev/null || :
wait "$lifetime_guard" 2>/dev/null || :
exit "$lifetime_status"
`
export function spawnWithParentProcessGuard(
command: string,
args: string[],
options: SpawnOptions,
): ChildProcess {
// Windows has no POSIX process group or /bin/sh. Retain its existing spawn
// behavior; callers must not infer that a runtime crash killed its children.
if (process.platform === 'win32') return spawn(command, args, options)
const stdio = Array.isArray(options.stdio)
? [...options.stdio]
: Array(3).fill(options.stdio ?? 'pipe')
stdio[3] = 'pipe'
const child = spawn('/bin/sh', [
'-c', PARENT_PROCESS_GUARD, 'cc-haha-shell-lifetime', command, ...args,
], {
...options,
detached: true,
stdio,
})
// The CLI owns the pipe's write end. Even SIGKILL closes it in the kernel,
// so EOF reaps this command's group without depending on SDK task_started,
// a model loop, or graceful cleanup. Normal completion kills and waits for
// the guardian in the supervisor before returning the original exit code.
const lifetimePipe = child.stdio[3] as Duplex | null
const release = () => lifetimePipe?.destroy()
child.once('exit', release)
child.once('error', release)
return child
}