From 993eb7631d39e81982f070ff8dd61b976ebd5b4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Mon, 14 Sep 2026 15:44:22 +0800 Subject: [PATCH] fix(api): keep provider image rejections from poisoning the session MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When a text-only model rejects an image with wording the classifier doesn't recognize, the 400 fell through to a generic API error with no businessErrorCode, so normalizeMessagesForAPI never stripped the image and every later turn on that model re-failed the same way. And when the wording did match, the strip anchor applied forever — switching to a vision-capable model still replayed history with the image removed. Classify any 400/422 on a request that actually carried image blocks as image_unsupported when no more specific classifier matched, and gate the error-anchored strip to the model that produced the error (sourceModel): the same model keeps stripping and heals, a different model replays the image, and a misclassified anchor only ever affects its own model. --- src/services/api/claude.ts | 6 +- src/services/api/errors.test.ts | 99 ++++++++++++++++++++++++ src/services/api/errors.ts | 44 +++++++++++ src/utils/messages.ts | 20 +++++ tests/mediaRecoveryAndEstimation.test.ts | 71 +++++++++++++++++ 5 files changed, 239 insertions(+), 1 deletion(-) diff --git a/src/services/api/claude.ts b/src/services/api/claude.ts index c4460d6c..e0059ba6 100644 --- a/src/services/api/claude.ts +++ b/src/services/api/claude.ts @@ -1413,7 +1413,11 @@ async function* queryModel( }); queryCheckpoint("query_message_normalization_start"); - let messagesForAPI = normalizeMessagesForAPI(messages, filteredTools); + let messagesForAPI = normalizeMessagesForAPI( + messages, + filteredTools, + options.model, + ); queryCheckpoint("query_message_normalization_end"); // Model-specific post-processing: strip tool-search-specific fields if the diff --git a/src/services/api/errors.test.ts b/src/services/api/errors.test.ts index 8e9b7bb0..390caba7 100644 --- a/src/services/api/errors.test.ts +++ b/src/services/api/errors.test.ts @@ -35,11 +35,110 @@ describe('image unsupported API errors', () => { expect(msg.isApiErrorMessage).toBe(true) expect(msg.businessErrorCode).toBe(BUSINESS_ERROR_CODES.IMAGE_UNSUPPORTED) expect(msg.errorDetails).toBe('This model does not support image blocks') + expect(msg.sourceModel).toBe('mimo-v2.5-pro') expect(msg.message.content[0]).toMatchObject({ type: 'text', text: getImageUnsupportedErrorMessage(), }) }) + + test('falls back to image_unsupported when a 400 with unrecognized wording hit a request carrying images', () => { + const message = 'unsupported content block type: only text is allowed for this model' + const error = new APIError( + 400, + { + type: 'error', + error: { type: 'invalid_request_error', message }, + }, + message, + undefined, + ) + const messagesForAPI = [ + { + type: 'user' as const, + message: { + role: 'user' as const, + content: [ + { type: 'text', text: 'look at this' }, + { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAA' }, + }, + ], + }, + }, + ] + + // The wording alone must not match the text classifier, otherwise this + // test stops exercising the request-context fallback. + expect(isUnsupportedImageInputErrorMessage(message)).toBe(false) + + const msg = getAssistantMessageFromError(error, 'deepseek-v4-pro', { + messagesForAPI: messagesForAPI as never, + }) + + expect(msg.isApiErrorMessage).toBe(true) + expect(msg.businessErrorCode).toBe(BUSINESS_ERROR_CODES.IMAGE_UNSUPPORTED) + expect(msg.sourceModel).toBe('deepseek-v4-pro') + }) + + test('does not fall back to image_unsupported when the failed request carried no images', () => { + const message = 'unsupported content block type: only text is allowed for this model' + const error = new APIError( + 400, + { + type: 'error', + error: { type: 'invalid_request_error', message }, + }, + message, + undefined, + ) + const messagesForAPI = [ + { + type: 'user' as const, + message: { role: 'user' as const, content: 'plain text only' }, + }, + ] + + const msg = getAssistantMessageFromError(error, 'deepseek-v4-pro', { + messagesForAPI: messagesForAPI as never, + }) + + expect(msg.isApiErrorMessage).toBe(true) + expect(msg.businessErrorCode).toBeUndefined() + }) + + test('does not fall back for non-400/422 API errors even when images were sent', () => { + const error = new APIError( + 500, + { + type: 'error', + error: { type: 'api_error', message: 'internal error' }, + }, + 'internal error', + undefined, + ) + const messagesForAPI = [ + { + type: 'user' as const, + message: { + role: 'user' as const, + content: [ + { + type: 'image', + source: { type: 'base64', media_type: 'image/png', data: 'AAA' }, + }, + ], + }, + }, + ] + + const msg = getAssistantMessageFromError(error, 'deepseek-v4-pro', { + messagesForAPI: messagesForAPI as never, + }) + + expect(msg.businessErrorCode).toBeUndefined() + }) }) describe('context overflow errors', () => { diff --git a/src/services/api/errors.ts b/src/services/api/errors.ts index 2d08e6d9..6579fb42 100644 --- a/src/services/api/errors.ts +++ b/src/services/api/errors.ts @@ -493,6 +493,25 @@ function isOpenAIImageUrlTextOnlySchemaError(raw: string): boolean { ) } +// Deep-walk a request payload looking for image blocks. Images can sit at the +// top level of a user message or nested inside a tool_result's content, and +// the payloads here are { type, message: { content } } wrappers, so walk all +// object values rather than only `content`. +function messagesContainImageBlock(messages: readonly unknown[]): boolean { + const walk = (value: unknown): boolean => { + if (Array.isArray(value)) { + return value.some(walk) + } + if (value && typeof value === 'object') { + const record = value as Record + if (record.type === 'image') return true + return Object.values(record).some(walk) + } + return false + } + return walk(messages) +} + export function getAssistantMessageFromError( error: unknown, model: string, @@ -537,6 +556,7 @@ export function getAssistantMessageFromError( error: 'invalid_request', errorDetails: error.message, businessErrorCode: BUSINESS_ERROR_CODES.IMAGE_UNSUPPORTED, + sourceModel: model, }) } @@ -1017,6 +1037,30 @@ export function getAssistantMessageFromError( }) } + // Fallback for image rejections with unrecognized wording. The wording + // classifier above can't enumerate every provider/gateway phrasing, and a + // miss used to poison the session: the rejected image stayed in history and + // every later turn re-failed with the same 400. When a 400/422 lands on a + // request that actually carried image blocks and no more specific classifier + // matched (context overflow, PDF, image size, auth, 404 all handled above), + // treat it as an image rejection so the image is stripped from later turns. + // The strip is gated to sourceModel, so a false positive only affects the + // exact model that failed — switching models replays the image. + if ( + error instanceof APIError && + (error.status === 400 || error.status === 422) && + options?.messagesForAPI && + messagesContainImageBlock(options.messagesForAPI) + ) { + return createAssistantAPIErrorMessage({ + content: getImageUnsupportedErrorMessage(), + error: 'invalid_request', + errorDetails: error.message, + businessErrorCode: BUSINESS_ERROR_CODES.IMAGE_UNSUPPORTED, + sourceModel: model, + }) + } + // Connection errors (non-timeout) — use formatAPIError for detailed messages if (error instanceof APIConnectionError) { return createAssistantAPIErrorMessage({ diff --git a/src/utils/messages.ts b/src/utils/messages.ts index 988166f2..506177b3 100644 --- a/src/utils/messages.ts +++ b/src/utils/messages.ts @@ -377,6 +377,7 @@ function baseCreateAssistantMessage({ error, errorDetails, businessErrorCode, + sourceModel, isVirtual, usage = { input_tokens: 0, @@ -400,6 +401,7 @@ function baseCreateAssistantMessage({ error?: SDKAssistantMessageError errorDetails?: string businessErrorCode?: BusinessErrorCode + sourceModel?: string isVirtual?: true usage?: Usage }): AssistantMessage { @@ -424,6 +426,7 @@ function baseCreateAssistantMessage({ error, errorDetails, businessErrorCode, + sourceModel, isApiErrorMessage, isVirtual, } @@ -459,12 +462,14 @@ export function createAssistantAPIErrorMessage({ error, errorDetails, businessErrorCode, + sourceModel, }: { content: string apiError?: AssistantMessage['apiError'] error?: SDKAssistantMessageError errorDetails?: string businessErrorCode?: BusinessErrorCode + sourceModel?: string }): AssistantMessage { return baseCreateAssistantMessage({ content: [ @@ -478,6 +483,7 @@ export function createAssistantAPIErrorMessage({ error, errorDetails, businessErrorCode, + sourceModel, }) } @@ -2016,6 +2022,7 @@ function relocateToolReferenceSiblings( export function normalizeMessagesForAPI( messages: Message[], tools: Tools = [], + currentModel?: string, ): (UserMessage | AssistantMessage)[] { // Build set of available tool names for filtering unavailable tool references const availableToolNames = new Set(tools.map(t => t.name)) @@ -2047,6 +2054,19 @@ export function normalizeMessagesForAPI( if (!isSyntheticApiErrorMessage(msg)) { continue } + // Error-anchored stripping is scoped to the model that produced the + // error. After the user switches to a different (e.g. vision-capable) + // model, the rejected media must replay normally instead of staying + // stripped forever. Anchors without a sourceModel (legacy transcripts) + // keep the historical strip behavior. + const anchorModel = (msg as { sourceModel?: unknown }).sourceModel + if ( + currentModel && + typeof anchorModel === 'string' && + anchorModel !== currentModel + ) { + continue + } let blockTypesToStrip: Set | undefined const blockTypesFromCode = typeof msg.businessErrorCode === 'string' diff --git a/tests/mediaRecoveryAndEstimation.test.ts b/tests/mediaRecoveryAndEstimation.test.ts index 4b75c102..e6738303 100644 --- a/tests/mediaRecoveryAndEstimation.test.ts +++ b/tests/mediaRecoveryAndEstimation.test.ts @@ -70,6 +70,77 @@ describe('media error recovery', () => { expect(serialized).toContain('describe this screenshot') expect(serialized).toContain('continue with text only') }) + + test('keeps stripping while the failing model is still selected', () => { + const imageUser = createUserMessage({ + content: [ + { type: 'text', text: 'describe this screenshot' }, + imageBlock('base64-image-payload'), + ], + uuid: '00000000-0000-4000-8000-000000000005', + }) + const unsupported = createAssistantAPIErrorMessage({ + content: 'localized display text', + error: 'invalid_request', + businessErrorCode: BUSINESS_ERROR_CODES.IMAGE_UNSUPPORTED, + sourceModel: 'nonvision-model', + }) + + const normalized = normalizeMessagesForAPI( + [imageUser, unsupported], + [], + 'nonvision-model', + ) + + expect(JSON.stringify(normalized)).not.toContain('base64-image-payload') + }) + + test('replays the image after switching to a different model', () => { + const imageUser = createUserMessage({ + content: [ + { type: 'text', text: 'describe this screenshot' }, + imageBlock('base64-image-payload'), + ], + uuid: '00000000-0000-4000-8000-000000000006', + }) + const unsupported = createAssistantAPIErrorMessage({ + content: 'localized display text', + error: 'invalid_request', + businessErrorCode: BUSINESS_ERROR_CODES.IMAGE_UNSUPPORTED, + sourceModel: 'nonvision-model', + }) + + const normalized = normalizeMessagesForAPI( + [imageUser, unsupported], + [], + 'vision-model', + ) + + expect(JSON.stringify(normalized)).toContain('base64-image-payload') + }) + + test('legacy anchors without sourceModel strip regardless of current model', () => { + const imageUser = createUserMessage({ + content: [ + { type: 'text', text: 'describe this screenshot' }, + imageBlock('base64-image-payload'), + ], + uuid: '00000000-0000-4000-8000-000000000007', + }) + const unsupported = createAssistantAPIErrorMessage({ + content: 'localized display text', + error: 'invalid_request', + businessErrorCode: BUSINESS_ERROR_CODES.IMAGE_UNSUPPORTED, + }) + + const normalized = normalizeMessagesForAPI( + [imageUser, unsupported], + [], + 'vision-model', + ) + + expect(JSON.stringify(normalized)).not.toContain('base64-image-payload') + }) }) describe('media context estimation', () => {