From eb322bb4cfb8db2526a6c8a6240cb7ad51c353ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Tue, 29 Sep 2026 16:18:31 +0800 Subject: [PATCH] fix(runtime): report measured size on 413 and recover stuck sessions A 413 from the API or a relay was reported as "Request too large (max 20MB)", which is the PDF-only limit, and the only recovery stripped top-level media from the single turn before the error. When the bytes sat in tool results, @-mentioned images or older turns, nothing shrank and every later message failed the same way. The error now reports the size of what was sent and how much of it is images or documents, names the provider or relay as the side that rejected it, and keeps the upstream's own text in errorDetails. After the error, images and documents in everything the failed request carried are replaced with placeholders, including media nested in tool results and from @-mentioned attachments. The rejection carries no sourceModel, so it still applies after a model switch. Transcripts saved with the old wording keep working, and compaction shares the placeholder logic instead of keeping its own copy. Refs #1399 --- src/constants/businessErrors.ts | 14 +- src/services/api/errors.test.ts | 166 ++++++++++++++++++ src/services/api/errors.ts | 113 +++++++++++- src/services/compact/compact.test.ts | 53 +++++- src/services/compact/compact.ts | 58 +------ src/utils/messages.test.ts | 248 +++++++++++++++++++++++++++ src/utils/messages.ts | 129 ++++++++++++-- 7 files changed, 706 insertions(+), 75 deletions(-) diff --git a/src/constants/businessErrors.ts b/src/constants/businessErrors.ts index 542f1a6f..b26ebca8 100644 --- a/src/constants/businessErrors.ts +++ b/src/constants/businessErrors.ts @@ -12,6 +12,9 @@ export const BUSINESS_ERROR_CODES = { export type BusinessErrorCode = (typeof BUSINESS_ERROR_CODES)[keyof typeof BUSINESS_ERROR_CODES] +// Block types to strip from the single turn a rejection followed. +// REQUEST_TOO_LARGE is deliberately absent: a byte limit says nothing about +// which block was at fault, so normalizeMessagesForAPI resolves it history-wide. export const BUSINESS_ERROR_MEDIA_BLOCK_TYPES: Partial< Record > = { @@ -20,5 +23,14 @@ export const BUSINESS_ERROR_MEDIA_BLOCK_TYPES: Partial< [BUSINESS_ERROR_CODES.PDF_INVALID]: ['document'], [BUSINESS_ERROR_CODES.IMAGE_TOO_LARGE]: ['image'], [BUSINESS_ERROR_CODES.IMAGE_UNSUPPORTED]: ['image'], - [BUSINESS_ERROR_CODES.REQUEST_TOO_LARGE]: ['document', 'image'], } + +/** + * Wording of the request-too-large error before it reported measured sizes. + * Transcripts persisted it (the oldest without a businessErrorCode), so history + * normalization must keep recognizing both the SDK and the interactive variant. + */ +export const LEGACY_REQUEST_TOO_LARGE_ERROR_MESSAGES: readonly string[] = [ + 'Request too large (max 20MB). Try with a smaller file.', + 'Request too large (max 20MB). Double press esc to go back and try with a smaller file.', +] diff --git a/src/services/api/errors.test.ts b/src/services/api/errors.test.ts index 226cefa5..f20676aa 100644 --- a/src/services/api/errors.test.ts +++ b/src/services/api/errors.test.ts @@ -1,5 +1,6 @@ import { describe, expect, test } from 'bun:test' import { APIError } from '@anthropic-ai/sdk' +import { getIsInteractive, setIsInteractive } from '../../bootstrap/state.js' import { BUSINESS_ERROR_CODES } from '../../constants/businessErrors.js' import { getAssistantMessageFromError, @@ -7,6 +8,7 @@ import { getImageUnsupportedErrorMessage, isContextOverflowErrorText, isUnsupportedImageInputErrorMessage, + measureRequestPayload, PROMPT_TOO_LONG_ERROR_MESSAGE, parsePromptTooLongTokenCounts, } from './errors.js' @@ -240,3 +242,167 @@ describe('context overflow errors', () => { }) }) }) + +describe('request too large (HTTP 413)', () => { + const MODEL = 'claude-sonnet-5-5' + const nginxHtml = + '\r\n413 Request Entity Too Large\r\n\r\n

413 Request Entity Too Large

\r\n
nginx/1.18.0
\r\n\r\n' + const relayBody = { + error: { message: 'request body too large, limit is 10 MB', type: 'invalid_request_error' }, + } + const anthropicBody = { + type: 'error', + error: { type: 'request_too_large', message: 'Request exceeds the maximum allowed number of bytes.' }, + } + const sources = [ + { + name: 'nginx in front of a relay', + error: new APIError(413, undefined, `413 ${nginxHtml}`, undefined), + upstream: 'nginx/1.18.0', + }, + { + name: 'a relay that names its own limit', + error: new APIError(413, relayBody, `413 ${JSON.stringify(relayBody)}`, undefined), + upstream: 'limit is 10 MB', + }, + { + name: 'the Anthropic API', + error: new APIError(413, anthropicBody, `413 ${JSON.stringify(anthropicBody)}`, undefined), + upstream: 'maximum allowed number of bytes', + }, + ] + const textOf = (msg: { message: { content: unknown[] } }) => + (msg.message.content[0] as { text: string }).text + const image = (chars: number) => ({ + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'A'.repeat(chars) }, + }) + const userTurn = (content: unknown[]) => ({ + type: 'user' as const, + message: { role: 'user' as const, content }, + }) + const relayRejection = sources[1]!.error + + test.each(sources)( + 'does not invent a size limit when $name rejects the request', + ({ error, upstream }) => { + const msg = getAssistantMessageFromError(error, MODEL) + const text = textOf(msg) + + // The old wording hard-coded the PDF limit ("max 20MB") for every 413, + // which is wrong for a relay with its own limit and for the API's real one. + expect(text).not.toMatch(/\d\s?MB/) + expect(text).not.toMatch(/smaller file/i) + expect(text).toContain('HTTP 413') + expect(msg.businessErrorCode).toBe(BUSINESS_ERROR_CODES.REQUEST_TOO_LARGE) + // Whoever rejected the request said something; keep it for diagnosis. + expect(msg.errorDetails).toStartWith('request_too_large: ') + expect(msg.errorDetails).toContain(upstream) + }, + ) + + test('reports how much of the rejected request was images and documents', () => { + const messagesForAPI = [ + userTurn([{ type: 'text', text: 'take a screenshot' }]), + userTurn([ + { type: 'tool_result', tool_use_id: 't1', content: [image(2 * 1024 * 1024)] }, + ]), + ] + + const text = textOf( + getAssistantMessageFromError(relayRejection, MODEL, { + messagesForAPI: messagesForAPI as never, + }), + ) + + expect(text).toContain('about 2MB') + expect(text).toContain('2MB is images or documents') + expect(text).toContain('placeholders') + }) + + test('says so when the rejected request carried no images or documents', () => { + const messagesForAPI = [ + userTurn([ + { type: 'tool_result', tool_use_id: 't1', content: 'x'.repeat(1.5 * 1024 * 1024) }, + ]), + ] + + const text = textOf( + getAssistantMessageFromError(relayRejection, MODEL, { + messagesForAPI: messagesForAPI as never, + }), + ) + + expect(text).toContain('about 1.5MB') + expect(text).toContain('none of it is images or documents') + }) + + test('points interactive users at /compact and non-interactive callers at a new session', () => { + // Session mode is process-global and bun runs every file in one process: + // set both modes explicitly and put back whatever was there. + const original = getIsInteractive() + try { + setIsInteractive(true) + expect(textOf(getAssistantMessageFromError(relayRejection, MODEL))).toContain('/compact') + setIsInteractive(false) + expect(textOf(getAssistantMessageFromError(relayRejection, MODEL))).toContain( + 'start a new session', + ) + } finally { + setIsInteractive(original) + } + }) + + describe('measureRequestPayload', () => { + const toolUse = { + type: 'tool_use', + id: 't', + name: 'Read', + input: { file_path: '/a' }, + } + + test('counts media by data length and keeps it separate from everything else', () => { + const size = measureRequestPayload([ + userTurn([{ type: 'text', text: 'hello' }]), + userTurn([ + { + type: 'tool_result', + tool_use_id: 't', + content: [image(1000), { type: 'text', text: 'abc' }], + }, + ]), + userTurn([ + { + type: 'document', + source: { type: 'base64', media_type: 'application/pdf', data: 'B'.repeat(500) }, + }, + ]), + { type: 'assistant', message: { role: 'assistant', content: [toolUse] } }, + ] as never) + + expect(size?.mediaBytes).toBe(1500) + expect(size?.totalBytes).toBe( + 5 + 1003 + 500 + Buffer.byteLength(JSON.stringify(toolUse)), + ) + }) + + test('never throws while classifying: an unserializable payload falls back to the unmeasured wording', () => { + const circular: Record = {} + circular.self = circular + const messagesForAPI = [ + { + type: 'assistant', + message: { + role: 'assistant', + content: [{ type: 'tool_use', id: 't', name: 'Read', input: circular }], + }, + }, + ] as never + + expect(measureRequestPayload(messagesForAPI)).toBeUndefined() + expect( + textOf(getAssistantMessageFromError(relayRejection, MODEL, { messagesForAPI })), + ).toContain('The size limit is set by that endpoint') + }) + }) +}) diff --git a/src/services/api/errors.ts b/src/services/api/errors.ts index cb6cb5e2..a43a39a9 100644 --- a/src/services/api/errors.ts +++ b/src/services/api/errors.ts @@ -269,11 +269,100 @@ export function getImageUnsupportedErrorMessage(): string { ? 'This model does not support images. Continue with text, or switch to a vision-capable model and send the image again.' : 'This model does not support images. Double press esc to go back, switch to a vision-capable model, or continue with text.' } -export function getRequestTooLargeErrorMessage(): string { - const limits = `max ${formatFileSize(PDF_TARGET_RAW_SIZE)}` - return getIsNonInteractiveSession() - ? `Request too large (${limits}). Try with a smaller file.` - : `Request too large (${limits}). Double press esc to go back and try with a smaller file.` +// A 413 body can be a whole HTML page; keep enough to identify the server +// (e.g. "nginx/1.18.0") without bloating the transcript. +const REQUEST_TOO_LARGE_DETAIL_CHARS = 500 + +/** + * Rough size of a request that was rejected as too large. Only measured on the + * 413 error path, never while sending. Base64 media dominates real payloads, so + * image/document data is counted by string length instead of re-serialized. + * `mediaBytes` is what replacing images and documents with placeholders removes. + */ +export type RequestPayloadSize = { totalBytes: number; mediaBytes: number } + +function measureContent(content: unknown): RequestPayloadSize { + if (typeof content === 'string') { + return { totalBytes: Buffer.byteLength(content), mediaBytes: 0 } + } + if (!Array.isArray(content)) { + return { totalBytes: 0, mediaBytes: 0 } + } + let totalBytes = 0 + let mediaBytes = 0 + for (const block of content) { + const size = measureBlock(block) + totalBytes += size.totalBytes + mediaBytes += size.mediaBytes + } + return { totalBytes, mediaBytes } +} + +function measureBlock(block: unknown): RequestPayloadSize { + if (typeof block === 'string') return measureContent(block) + if (!block || typeof block !== 'object') { + return { totalBytes: 0, mediaBytes: 0 } + } + const { type, text, source, content } = block as { + type?: unknown + text?: unknown + source?: { data?: unknown; url?: unknown } + content?: unknown + } + if (type === 'image' || type === 'document') { + const bytes = + typeof source?.data === 'string' + ? source.data.length + : typeof source?.url === 'string' + ? source.url.length + : 0 + return { totalBytes: bytes, mediaBytes: bytes } + } + if (type === 'text' && typeof text === 'string') return measureContent(text) + if (type === 'tool_result') return measureContent(content) + return { totalBytes: Buffer.byteLength(JSON.stringify(block)), mediaBytes: 0 } +} + +/** + * Returns undefined instead of throwing: this runs while classifying an API + * failure, where a second failure would hide the first. + */ +export function measureRequestPayload( + messages: readonly (UserMessage | AssistantMessage)[], +): RequestPayloadSize | undefined { + try { + let totalBytes = 0 + let mediaBytes = 0 + for (const message of messages) { + const size = measureContent(message.message.content) + totalBytes += size.totalBytes + mediaBytes += size.mediaBytes + } + return { totalBytes, mediaBytes } + } catch { + return undefined + } +} + +export function getRequestTooLargeErrorMessage( + size?: RequestPayloadSize, +): string { + const interactive = !getIsNonInteractiveSession() + const rejected = + 'Request too large: your provider or relay rejected it (HTTP 413).' + const nextStep = interactive + ? 'run /compact or double press esc to go back past the large content' + : 'compact the conversation or start a new session' + const sentenceNextStep = nextStep.charAt(0).toUpperCase() + nextStep.slice(1) + + if (!size) { + return `${rejected} The size limit is set by that endpoint. ${sentenceNextStep}.` + } + const total = formatFileSize(size.totalBytes) + if (size.mediaBytes === 0) { + return `${rejected} This conversation is about ${total} and none of it is images or documents. ${sentenceNextStep}; a relay can enforce a lower request size limit than its upstream provider.` + } + return `${rejected} This conversation is about ${total}, of which ${formatFileSize(size.mediaBytes)} is images or documents. Earlier images and documents are replaced with placeholders in the next request; if it still fails, ${nextStep}.` } export const OAUTH_ORG_NOT_ALLOWED_ERROR_MESSAGE = 'Your account does not have access to Claude Code. Please run /login.' @@ -817,12 +906,20 @@ export function getAssistantMessageFromError( }) } - // Check for request too large errors (413 status) - // This typically happens when a large PDF + conversation context exceeds the 32MB API limit + // 413 comes from whatever sits in front of the model: the API itself (32MB + // request limit) or a relay/gateway with its own, usually smaller, limit. + // Which one and what limit is unknown here, so report what was sent and keep + // the upstream's own words rather than claiming a number. No sourceModel on + // purpose: a byte limit belongs to the provider or relay, not to one model + // (normalizeMessagesForAPI relies on that to keep stripping after a switch). if (error instanceof APIError && error.status === 413) { + const size = options?.messagesForAPI + ? measureRequestPayload(options.messagesForAPI) + : undefined return createAssistantAPIErrorMessage({ - content: getRequestTooLargeErrorMessage(), + content: getRequestTooLargeErrorMessage(size), error: 'invalid_request', + errorDetails: `request_too_large: ${(error.message ?? '').slice(0, REQUEST_TOO_LARGE_DETAIL_CHARS)}`, businessErrorCode: BUSINESS_ERROR_CODES.REQUEST_TOO_LARGE, }) } diff --git a/src/services/compact/compact.test.ts b/src/services/compact/compact.test.ts index 2983f463..3baa6f7b 100644 --- a/src/services/compact/compact.test.ts +++ b/src/services/compact/compact.test.ts @@ -1,6 +1,6 @@ import { describe, expect, test } from 'bun:test' -import { buildPostCompactMessages, truncateHeadForPTLRetry, type CompactionResult } from './compact.js' +import { buildPostCompactMessages, stripImagesFromMessages, truncateHeadForPTLRetry, type CompactionResult } from './compact.js' import { getCurrentUsage } from '../../utils/tokens.js' import type { AssistantMessage, Message } from '../../types/message.js' @@ -189,3 +189,54 @@ describe('oversized compaction recovery (#1373)', () => { expect(second[0]?.type).toBe('user') }) }) + +describe('stripImagesFromMessages', () => { + const image = { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAAA' }, + } + const userWith = (content: unknown) => + ({ + ...makePreservedUser(), + uuid: crypto.randomUUID(), + message: { role: 'user', content }, + }) as Message + + test('replaces images and documents with markers, including inside tool results', () => { + const [stripped] = stripImagesFromMessages([ + userWith([ + { type: 'text', text: 'look' }, + image, + { + type: 'document', + source: { type: 'base64', media_type: 'application/pdf', data: 'JVBERi0=' }, + }, + { type: 'tool_result', tool_use_id: 't1', content: [image, { type: 'text', text: 'caption' }] }, + ]), + ]) + + expect((stripped as { message: { content: unknown } }).message.content).toEqual([ + { type: 'text', text: 'look' }, + { type: 'text', text: '[image]' }, + { type: 'text', text: '[document]' }, + { + type: 'tool_result', + tool_use_id: 't1', + content: [ + { type: 'text', text: '[image]' }, + { type: 'text', text: 'caption' }, + ], + }, + ]) + }) + + test('returns messages without media untouched', () => { + const plain = userWith([{ type: 'text', text: 'plain' }]) + const assistant = makePreservedAssistant() + + const result = stripImagesFromMessages([plain, assistant]) + + expect(result[0]).toBe(plain) + expect(result[1]).toBe(assistant) + }) +}) diff --git a/src/services/compact/compact.ts b/src/services/compact/compact.ts index 29756e16..8e0c122e 100644 --- a/src/services/compact/compact.ts +++ b/src/services/compact/compact.ts @@ -65,6 +65,7 @@ import { getMessagesAfterCompactBoundary, isCompactBoundaryMessage, normalizeMessagesForAPI, + replaceMediaWithPlaceholders, } from '../../utils/messages.js' import { expandPath } from '../../utils/path.js' import { getPlan, getPlanFilePath } from '../../utils/plans.js' @@ -144,60 +145,9 @@ const MAX_COMPACT_STREAMING_RETRIES = 2 * and thinking blocks but not images. */ export function stripImagesFromMessages(messages: Message[]): Message[] { - return messages.map(message => { - if (message.type !== 'user') { - return message - } - - const content = message.message.content - if (!Array.isArray(content)) { - return message - } - - let hasMediaBlock = false - const newContent = content.flatMap(block => { - if (block.type === 'image') { - hasMediaBlock = true - return [{ type: 'text' as const, text: '[image]' }] - } - if (block.type === 'document') { - hasMediaBlock = true - return [{ type: 'text' as const, text: '[document]' }] - } - // Also strip images/documents nested inside tool_result content arrays - if (block.type === 'tool_result' && Array.isArray(block.content)) { - let toolHasMedia = false - const newToolContent = block.content.map(item => { - if (item.type === 'image') { - toolHasMedia = true - return { type: 'text' as const, text: '[image]' } - } - if (item.type === 'document') { - toolHasMedia = true - return { type: 'text' as const, text: '[document]' } - } - return item - }) - if (toolHasMedia) { - hasMediaBlock = true - return [{ ...block, content: newToolContent }] - } - } - return [block] - }) - - if (!hasMediaBlock) { - return message - } - - return { - ...message, - message: { - ...message.message, - content: newContent, - }, - } as typeof message - }) + return messages.map(message => + message.type === 'user' ? replaceMediaWithPlaceholders(message) : message, + ) } /** diff --git a/src/utils/messages.test.ts b/src/utils/messages.test.ts index 818a4735..30740b28 100644 --- a/src/utils/messages.test.ts +++ b/src/utils/messages.test.ts @@ -1,12 +1,21 @@ import { describe, expect, test } from 'bun:test' +import { APIError } from '@anthropic-ai/sdk' import type { ContentBlockParam } from '@anthropic-ai/sdk/resources/index.mjs' +import { BUSINESS_ERROR_CODES } from '../constants/businessErrors.js' +import { + getAssistantMessageFromError, + getImageTooLargeErrorMessage, +} from '../services/api/errors.js' import type { Tool } from '../Tool.js' import type { AssistantMessage } from '../types/message.js' +import { createAttachmentMessage } from './attachments.js' import { + createAssistantAPIErrorMessage, createAssistantMessage, createUserMessage, normalizeMessagesForAPI, normalizeContentFromAPI, + replaceMediaWithPlaceholders, stripSignatureBlocksAfterModelChange, } from './messages.js' @@ -274,3 +283,242 @@ test('legacy ordinary tool-use JSON still round-trips without a migration', () = const replay = normalizeMessagesForAPI([historical, toolResult('legacy-read')], [tool]) expect(replay[0]!.message.content).toEqual(historical.message.content) }) + +type ApiBlock = { type: string; [key: string]: unknown } + +function imageBlock(chars: number) { + return { + type: 'image' as const, + source: { + type: 'base64' as const, + media_type: 'image/png' as const, + data: 'A'.repeat(chars), + }, + } +} + +function screenshotResult(id: string, chars: number) { + return createUserMessage({ + content: [ + { type: 'tool_result', tool_use_id: id, content: [imageBlock(chars)] }, + ] as ContentBlockParam[], + }) +} + +// The real classifier, so the anchor is exactly what a relay's 413 produces. +function requestTooLargeAnchor() { + return getAssistantMessageFromError( + new APIError(413, undefined, '413 Request Entity Too Large', undefined), + 'claude-sonnet-5-5', + ) +} + +function allBlocks( + messages: ReturnType, +): ApiBlock[] { + const blocks: ApiBlock[] = [] + for (const message of messages) { + const content = message.message.content + if (!Array.isArray(content)) continue + for (const block of content as ApiBlock[]) { + blocks.push(block) + if (block.type === 'tool_result' && Array.isArray(block.content)) { + blocks.push(...(block.content as ApiBlock[])) + } + } + } + return blocks +} + +const imagesIn = (blocks: ApiBlock[]) => blocks.filter(b => b.type === 'image') +// Attachment text is wrapped in a system reminder, which adds a trailing newline. +const placeholdersIn = (blocks: ApiBlock[]) => + blocks.filter(b => b.type === 'text' && String(b.text).trim() === '[image]') +const bytesOf = (value: unknown) => Buffer.byteLength(JSON.stringify(value)) + +describe('normalizeMessagesForAPI after a request-too-large rejection', () => { + test('replaces images from every earlier turn, including ones nested in tool results', () => { + const history = [createUserMessage({ content: 'do a long GUI task' })] + for (let i = 0; i < 3; i++) { + history.push(assistant(`a${i}`, [toolUse(`s${i}`)]), screenshotResult(`s${i}`, 300_000)) + } + const fresh = createUserMessage({ + content: [{ type: 'text', text: 'try again' }, imageBlock(1_000)] as ContentBlockParam[], + }) + + const normalized = normalizeMessagesForAPI([...history, requestTooLargeAnchor(), fresh]) + const blocks = allBlocks(normalized) + + // Only the image sent after the rejection is still a real image. + expect(imagesIn(blocks)).toHaveLength(1) + expect(placeholdersIn(blocks)).toHaveLength(3) + // Placeholders replace media in place, so tool_use/tool_result pairing holds. + expect( + blocks.filter(b => b.type === 'tool_result').map(b => b.tool_use_id), + ).toEqual(['s0', 's1', 's2']) + expect(bytesOf(normalized)).toBeLessThan(20_000) + }) + + test('covers images from @-mentioned files, which are rebuilt on every normalization', () => { + const mention = createAttachmentMessage({ + type: 'file', + filename: '/tmp/shot.png', + displayPath: 'shot.png', + content: { + type: 'image', + file: { base64: 'A'.repeat(300_000), type: 'image/png', originalSize: 225_000 }, + }, + } as never) + const history = [ + mention, + createUserMessage({ content: 'what is in @shot.png?' }), + assistant('a0', [{ type: 'text', text: 'A screenshot.' }]), + ] + const followUp = createUserMessage({ content: 'and now?' }) + + // Without a rejection the attachment really does contribute an image. + expect(imagesIn(allBlocks(normalizeMessagesForAPI([...history, followUp])))).toHaveLength(1) + + const blocks = allBlocks( + normalizeMessagesForAPI([...history, requestTooLargeAnchor(), followUp]), + ) + expect(imagesIn(blocks)).toHaveLength(0) + expect(placeholdersIn(blocks)).toHaveLength(1) + }) + + test('leaves a history without media exactly as it was', () => { + const history = [createUserMessage({ content: 'dump the logs' })] + for (let i = 0; i < 3; i++) { + history.push( + assistant(`a${i}`, [toolUse(`t${i}`)]), + createUserMessage({ + content: [ + { type: 'tool_result', tool_use_id: `t${i}`, content: 'x'.repeat(50_000) }, + ] as ContentBlockParam[], + }), + ) + } + const followUp = createUserMessage({ content: 'compact it' }) + + expect( + normalizeMessagesForAPI([...history, requestTooLargeAnchor(), followUp]).map( + m => m.message.content, + ), + ).toEqual(normalizeMessagesForAPI([...history, followUp]).map(m => m.message.content)) + }) + + test('holds for any model: the byte limit belongs to the provider, not to one model', () => { + const history = [ + createUserMessage({ content: 'screenshot it' }), + assistant('a0', [toolUse('s0')]), + screenshotResult('s0', 300_000), + ] + const followUp = createUserMessage({ content: 'continue' }) + + const normalized = normalizeMessagesForAPI( + [...history, requestTooLargeAnchor(), followUp], + [], + 'a-different-model', + ) + + expect(imagesIn(allBlocks(normalized))).toHaveLength(0) + }) + + test.each([ + 'Request too large (max 20MB). Try with a smaller file.', + 'Request too large (max 20MB). Double press esc to go back and try with a smaller file.', + ])('still recognizes transcripts saved with the old wording: %s', legacyText => { + const legacyAnchor = createAssistantAPIErrorMessage({ content: legacyText }) + expect(legacyAnchor.businessErrorCode).toBeUndefined() + const history = [ + createUserMessage({ content: 'screenshot it' }), + assistant('a0', [toolUse('s0')]), + screenshotResult('s0', 300_000), + assistant('a1', [toolUse('s1')]), + screenshotResult('s1', 300_000), + ] + + const blocks = allBlocks( + normalizeMessagesForAPI([...history, legacyAnchor, createUserMessage({ content: 'go on' })]), + ) + + expect(imagesIn(blocks)).toHaveLength(0) + expect(placeholdersIn(blocks)).toHaveLength(2) + }) + + test('other media rejections stay scoped to the turn they followed', () => { + const earlier = createUserMessage({ + content: [{ type: 'text', text: 'first' }, imageBlock(1_000)] as ContentBlockParam[], + }) + const oversized = createUserMessage({ + content: [{ type: 'text', text: 'second' }, imageBlock(1_000)] as ContentBlockParam[], + }) + const anchor = createAssistantAPIErrorMessage({ + content: getImageTooLargeErrorMessage(), + businessErrorCode: BUSINESS_ERROR_CODES.IMAGE_TOO_LARGE, + }) + + const blocks = allBlocks( + normalizeMessagesForAPI([ + earlier, + assistant('a0', [{ type: 'text', text: 'ok' }]), + oversized, + anchor, + createUserMessage({ content: 'retry' }), + ]), + ) + + // Only the rejected turn loses its image; earlier turns are not touched. + expect(imagesIn(blocks)).toHaveLength(1) + }) +}) + +describe('replaceMediaWithPlaceholders', () => { + test('replaces top-level and tool-result media without touching the input', () => { + const message = createUserMessage({ + content: [ + { type: 'text', text: 'see attached' }, + imageBlock(10), + { + type: 'document', + source: { type: 'base64', media_type: 'application/pdf', data: 'JVBERi0=' }, + }, + { + type: 'tool_result', + tool_use_id: 't1', + content: [imageBlock(10), { type: 'text', text: 'caption' }], + }, + ] as ContentBlockParam[], + }) + const before = JSON.stringify(message) + + const replaced = replaceMediaWithPlaceholders(message) + + expect(replaced.message.content).toEqual([ + { type: 'text', text: 'see attached' }, + { type: 'text', text: '[image]' }, + { type: 'text', text: '[document]' }, + { + type: 'tool_result', + tool_use_id: 't1', + content: [ + { type: 'text', text: '[image]' }, + { type: 'text', text: 'caption' }, + ], + }, + ]) + expect(JSON.stringify(message)).toBe(before) + }) + + test('returns the same message when it has no media', () => { + const text = createUserMessage({ content: 'plain text' }) + const blocks = createUserMessage({ + content: [ + { type: 'tool_result', tool_use_id: 't1', content: 'ok' }, + ] as ContentBlockParam[], + }) + + expect(replaceMediaWithPlaceholders(text)).toBe(text) + expect(replaceMediaWithPlaceholders(blocks)).toBe(blocks) + }) +}) diff --git a/src/utils/messages.ts b/src/utils/messages.ts index 812e9449..aa198f55 100644 --- a/src/utils/messages.ts +++ b/src/utils/messages.ts @@ -37,7 +37,9 @@ import { import { OUTPUT_STYLE_CONFIG } from '../constants/outputStyles.js' import { type BusinessErrorCode, + BUSINESS_ERROR_CODES, BUSINESS_ERROR_MEDIA_BLOCK_TYPES, + LEGACY_REQUEST_TOO_LARGE_ERROR_MESSAGES, } from '../constants/businessErrors.js' import { isAutoMemoryEnabled } from '../memdir/paths.js' import { @@ -50,7 +52,6 @@ import { getPdfInvalidErrorMessage, getPdfPasswordProtectedErrorMessage, getPdfTooLargeErrorMessage, - getRequestTooLargeErrorMessage, } from '../services/api/errors.js' import type { AnyObject, Progress } from '../Tool.js' import { isConnectorTextBlock } from '../types/connectorText.js' @@ -2020,6 +2021,80 @@ function relocateToolReferenceSiblings( return result } +/** + * Replaces image and document blocks with short text markers, including media + * nested in tool_result content. Markers (rather than removal) keep turn + * structure and tool_use/tool_result pairing valid, and let the model see that + * media was there. Returns the same object when there is nothing to replace. + * + * Only user messages carry media; assistant messages hold text, tool_use and + * thinking blocks. + */ +export function replaceMediaWithPlaceholders(message: UserMessage): UserMessage { + const content = message.message.content + if (!Array.isArray(content)) { + return message + } + + let hasMediaBlock = false + const newContent = content.flatMap(block => { + if (block.type === 'image') { + hasMediaBlock = true + return [{ type: 'text' as const, text: '[image]' }] + } + if (block.type === 'document') { + hasMediaBlock = true + return [{ type: 'text' as const, text: '[document]' }] + } + // Also replace media nested inside tool_result content arrays + if (block.type === 'tool_result' && Array.isArray(block.content)) { + let toolHasMedia = false + const newToolContent = block.content.map(item => { + if (item.type === 'image') { + toolHasMedia = true + return { type: 'text' as const, text: '[image]' } + } + if (item.type === 'document') { + toolHasMedia = true + return { type: 'text' as const, text: '[document]' } + } + return item + }) + if (toolHasMedia) { + hasMediaBlock = true + return [{ ...block, content: newToolContent }] + } + } + return [block] + }) + + if (!hasMediaBlock) { + return message + } + + return { + ...message, + message: { + ...message.message, + content: newContent, + }, + } as UserMessage +} + +// Legacy transcripts predate businessErrorCode and only carry the wording. +function isRequestTooLargeAnchor( + message: AssistantMessage, + errorText: string | undefined, +): boolean { + if (typeof message.businessErrorCode === 'string') { + return message.businessErrorCode === BUSINESS_ERROR_CODES.REQUEST_TOO_LARGE + } + return ( + errorText !== undefined && + LEGACY_REQUEST_TOO_LARGE_ERROR_MESSAGES.includes(errorText) + ) +} + export function normalizeMessagesForAPI( messages: Message[], tools: Tools = [], @@ -2044,12 +2119,18 @@ export function normalizeMessagesForAPI( [getPdfInvalidErrorMessage()]: new Set(['document']), [getImageTooLargeErrorMessage()]: new Set(['image']), [getImageUnsupportedErrorMessage()]: new Set(['image']), - [getRequestTooLargeErrorMessage()]: new Set(['document', 'image']), } // Walk the reordered messages to build a targeted strip map: // userMessageUUID → set of block types to strip from that message. const stripTargets = new Map>() + // Index of the last request-too-large anchor. Unlike the rejections above it + // says nothing about which block was at fault: the whole conversation was over + // an upstream byte limit. So media is dropped from everything the failed + // request carried, not just the turn before the anchor. It carries no + // sourceModel on purpose: a byte limit belongs to the provider or relay, so + // it must survive a model switch. + let mediaStrippedThrough = -1 for (let i = 0; i < reorderedMessages.length; i++) { const msg = reorderedMessages[i]! if (!isSyntheticApiErrorMessage(msg)) { @@ -2068,6 +2149,17 @@ export function normalizeMessagesForAPI( ) { continue } + const errorText = + Array.isArray(msg.message.content) && + msg.message.content[0]?.type === 'text' + ? msg.message.content[0].text + : undefined + if (isRequestTooLargeAnchor(msg, errorText)) { + // Anchors are visited in order, so the last one wins. + mediaStrippedThrough = i + continue + } + let blockTypesToStrip: Set | undefined const blockTypesFromCode = typeof msg.businessErrorCode === 'string' @@ -2078,11 +2170,6 @@ export function normalizeMessagesForAPI( } // Determine which legacy text error this is. - const errorText = - Array.isArray(msg.message.content) && - msg.message.content[0]?.type === 'text' - ? msg.message.content[0].text - : undefined if (!blockTypesToStrip && errorText) { blockTypesToStrip = errorToBlockTypes[errorText] } @@ -2114,6 +2201,16 @@ export function normalizeMessagesForAPI( } } + // Identity rather than uuid: attachment-derived messages (an @-mentioned + // image) are rebuilt on every normalization, so they have no stable uuid. + const mediaStripped = new Set() + for (let i = 0; i < mediaStrippedThrough; i++) { + const candidate = reorderedMessages[i]! + if (candidate.type === 'user' || candidate.type === 'attachment') { + mediaStripped.add(candidate) + } + } + const result: (UserMessage | AssistantMessage)[] = [] const assistantIndexByMessageId = new Map() let indexedResultLength = 0 @@ -2173,9 +2270,16 @@ export function normalizeMessagesForAPI( ) } + // Before the merge below: the rejected turn and the retry become + // adjacent once the synthetic error is filtered out, and only the + // rejected part may lose its media. + if (mediaStripped.has(message)) { + normalizedMessage = replaceMediaWithPlaceholders(normalizedMessage) + } + // Strip document/image blocks from the specific user message that - // preceded a PDF/image/request-too-large error, to prevent re-sending - // the problematic content on every subsequent API call. + // preceded a PDF/image error, to prevent re-sending the problematic + // content on every subsequent API call. const typesToStrip = stripTargets.get(normalizedMessage.uuid) if (typesToStrip) { const content = normalizedMessage.message.content @@ -2357,11 +2461,14 @@ export function normalizeMessagesForAPI( ) return } + const keptAttachmentMessage = mediaStripped.has(message) + ? rawAttachmentMessage.map(m => replaceMediaWithPlaceholders(m)) + : rawAttachmentMessage const attachmentMessage = checkStatsigFeatureGate_CACHED_MAY_BE_STALE( 'tengu_chair_sermon', ) - ? rawAttachmentMessage.map(ensureSystemReminderWrap) - : rawAttachmentMessage + ? keptAttachmentMessage.map(ensureSystemReminderWrap) + : keptAttachmentMessage // If the last message is also a user message, merge them const lastMessage = last(result)