fix: coerce Mistral image_url parts from object to string (#3177)

* fix: coerce Mistral image_url parts from object to string

Mistral's API expects image_url content parts to be a plain string URL,
not the OpenAI-style { url: string } object that process_input_messages
produces. Without this coercion, sending an image to any Mistral model
that supports vision (mistral-small, mistral-medium, mistral-large,
ministral-*) results in a request error from the Mistral SDK.

Add #coerceImageUrls() as a private method on MistralAIProvider that
maps { type: 'image_url', image_url: { url } } -> { type: 'image_url',
image_url: url } for every content part in every message. Messages with
plain string content are left untouched, as are parts whose image_url is
already a string.

Add four unit tests covering: object-to-string coercion, already-flat
strings, plain string message content, and mixed text+image content.

* style: use plain ASCII dashes in comment section divider
This commit is contained in:
Dilan Melvin T
2026-06-10 05:42:44 +05:30
committed by GitHub
parent 9c4d1ef535
commit 3e9ceb673b
2 changed files with 150 additions and 0 deletions
@@ -655,6 +655,123 @@ describe('MistralAIProvider.complete streaming', () => {
});
});
// -- Mistral image_url coercion --
describe('MistralAIProvider image_url coercion', () => {
const baseCompletion = {
choices: [
{
message: { content: 'ok', role: 'assistant' },
finishReason: 'stop',
},
],
usage: { promptTokens: 1, completionTokens: 1 },
};
it('flattens image_url objects to plain strings before sending to Mistral', async () => {
const { provider } = makeProvider();
completeMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'mistral-small-2603',
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'describe this image' },
{
type: 'image_url',
image_url: { url: 'https://example.com/img.png' },
},
],
},
],
}),
);
const [args] = completeMock.mock.calls[0]!;
const imagePart = (args.messages[0].content as unknown[]).find(
(p: unknown) => (p as { type: string }).type === 'image_url',
) as { type: string; image_url: unknown };
// Mistral expects a plain string, not { url: string }.
expect(imagePart.image_url).toBe('https://example.com/img.png');
});
it('leaves image_url parts that are already plain strings unchanged', async () => {
const { provider } = makeProvider();
completeMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'mistral-small-2603',
messages: [
{
role: 'user',
content: [
{
type: 'image_url',
image_url: 'https://example.com/already-flat.png',
},
],
},
],
}),
);
const [args] = completeMock.mock.calls[0]!;
const imagePart = (args.messages[0].content as unknown[]).find(
(p: unknown) => (p as { type: string }).type === 'image_url',
) as { type: string; image_url: unknown };
expect(imagePart.image_url).toBe('https://example.com/already-flat.png');
});
it('leaves messages with plain string content untouched', async () => {
const { provider } = makeProvider();
completeMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'mistral-small-2603',
messages: [{ role: 'user', content: 'plain text message' }],
}),
);
const [args] = completeMock.mock.calls[0]!;
expect(args.messages[0].content).toBe('plain text message');
});
it('coerces only image_url parts and leaves other content parts intact', async () => {
const { provider } = makeProvider();
completeMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'mistral-small-2603',
messages: [
{
role: 'user',
content: [
{ type: 'text', text: 'what is in this image?' },
{
type: 'image_url',
image_url: { url: 'https://example.com/photo.jpg' },
},
],
},
],
}),
);
const [args] = completeMock.mock.calls[0]!;
const parts = args.messages[0].content as { type: string; text?: string; image_url?: unknown }[];
const textPart = parts.find((p) => p.type === 'text');
const imgPart = parts.find((p) => p.type === 'image_url');
expect(textPart?.text).toBe('what is in this image?');
expect(imgPart?.image_url).toBe('https://example.com/photo.jpg');
});
});
// ── Error mapping ───────────────────────────────────────────────────
describe('MistralAIProvider.complete error mapping', () => {
@@ -61,6 +61,38 @@ export class MistralAIProvider implements IChatProvider {
return ids;
}
/**
* Mistral's API expects `image_url` content parts to carry a plain
* string URL, not the OpenAI-style `{ url: string }` object.
* This method normalises any `{ type: 'image_url', image_url: { url } }`
* parts to `{ type: 'image_url', image_url: url }` before the request
* is sent. Messages whose `content` is a plain string are left untouched.
*/
#coerceImageUrls(
messages: { role: string; content: unknown }[],
): { role: string; content: unknown }[] {
return messages.map((message) => {
if (!Array.isArray(message.content)) return message;
const content = message.content.map(
(part: { type?: string; image_url?: unknown }) => {
if (
part.type === 'image_url' &&
part.image_url !== null &&
typeof part.image_url === 'object' &&
'url' in (part.image_url as object)
) {
return {
...part,
image_url: (part.image_url as { url: string }).url,
};
}
return part;
},
);
return { ...message, content };
});
}
async complete({
messages,
stream,
@@ -70,6 +102,7 @@ export class MistralAIProvider implements IChatProvider {
temperature,
}: ICompleteArguments): Promise<IChatCompleteResult> {
messages = await OpenAIUtil.process_input_messages(messages);
messages = this.#coerceImageUrls(messages);
for (const message of messages) {
if (message.tool_calls) {
message.toolCalls = message.tool_calls;