mirror of
https://github.com/HeyPuter/puter.git
synced 2026-07-08 08:12:15 +00:00
fix: coerce Mistral image_url parts from object to string (#3177)
* fix: coerce Mistral image_url parts from object to string
Mistral's API expects image_url content parts to be a plain string URL,
not the OpenAI-style { url: string } object that process_input_messages
produces. Without this coercion, sending an image to any Mistral model
that supports vision (mistral-small, mistral-medium, mistral-large,
ministral-*) results in a request error from the Mistral SDK.
Add #coerceImageUrls() as a private method on MistralAIProvider that
maps { type: 'image_url', image_url: { url } } -> { type: 'image_url',
image_url: url } for every content part in every message. Messages with
plain string content are left untouched, as are parts whose image_url is
already a string.
Add four unit tests covering: object-to-string coercion, already-flat
strings, plain string message content, and mixed text+image content.
* style: use plain ASCII dashes in comment section divider
This commit is contained in:
@@ -655,6 +655,123 @@ describe('MistralAIProvider.complete streaming', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// -- Mistral image_url coercion --
|
||||
|
||||
describe('MistralAIProvider image_url coercion', () => {
|
||||
const baseCompletion = {
|
||||
choices: [
|
||||
{
|
||||
message: { content: 'ok', role: 'assistant' },
|
||||
finishReason: 'stop',
|
||||
},
|
||||
],
|
||||
usage: { promptTokens: 1, completionTokens: 1 },
|
||||
};
|
||||
|
||||
it('flattens image_url objects to plain strings before sending to Mistral', async () => {
|
||||
const { provider } = makeProvider();
|
||||
completeMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'mistral-small-2603',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'describe this image' },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'https://example.com/img.png' },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const [args] = completeMock.mock.calls[0]!;
|
||||
const imagePart = (args.messages[0].content as unknown[]).find(
|
||||
(p: unknown) => (p as { type: string }).type === 'image_url',
|
||||
) as { type: string; image_url: unknown };
|
||||
// Mistral expects a plain string, not { url: string }.
|
||||
expect(imagePart.image_url).toBe('https://example.com/img.png');
|
||||
});
|
||||
|
||||
it('leaves image_url parts that are already plain strings unchanged', async () => {
|
||||
const { provider } = makeProvider();
|
||||
completeMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'mistral-small-2603',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: 'https://example.com/already-flat.png',
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const [args] = completeMock.mock.calls[0]!;
|
||||
const imagePart = (args.messages[0].content as unknown[]).find(
|
||||
(p: unknown) => (p as { type: string }).type === 'image_url',
|
||||
) as { type: string; image_url: unknown };
|
||||
expect(imagePart.image_url).toBe('https://example.com/already-flat.png');
|
||||
});
|
||||
|
||||
it('leaves messages with plain string content untouched', async () => {
|
||||
const { provider } = makeProvider();
|
||||
completeMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'mistral-small-2603',
|
||||
messages: [{ role: 'user', content: 'plain text message' }],
|
||||
}),
|
||||
);
|
||||
|
||||
const [args] = completeMock.mock.calls[0]!;
|
||||
expect(args.messages[0].content).toBe('plain text message');
|
||||
});
|
||||
|
||||
it('coerces only image_url parts and leaves other content parts intact', async () => {
|
||||
const { provider } = makeProvider();
|
||||
completeMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'mistral-small-2603',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
content: [
|
||||
{ type: 'text', text: 'what is in this image?' },
|
||||
{
|
||||
type: 'image_url',
|
||||
image_url: { url: 'https://example.com/photo.jpg' },
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const [args] = completeMock.mock.calls[0]!;
|
||||
const parts = args.messages[0].content as { type: string; text?: string; image_url?: unknown }[];
|
||||
const textPart = parts.find((p) => p.type === 'text');
|
||||
const imgPart = parts.find((p) => p.type === 'image_url');
|
||||
expect(textPart?.text).toBe('what is in this image?');
|
||||
expect(imgPart?.image_url).toBe('https://example.com/photo.jpg');
|
||||
});
|
||||
});
|
||||
|
||||
// ── Error mapping ───────────────────────────────────────────────────
|
||||
|
||||
describe('MistralAIProvider.complete error mapping', () => {
|
||||
|
||||
@@ -61,6 +61,38 @@ export class MistralAIProvider implements IChatProvider {
|
||||
return ids;
|
||||
}
|
||||
|
||||
/**
|
||||
* Mistral's API expects `image_url` content parts to carry a plain
|
||||
* string URL, not the OpenAI-style `{ url: string }` object.
|
||||
* This method normalises any `{ type: 'image_url', image_url: { url } }`
|
||||
* parts to `{ type: 'image_url', image_url: url }` before the request
|
||||
* is sent. Messages whose `content` is a plain string are left untouched.
|
||||
*/
|
||||
#coerceImageUrls(
|
||||
messages: { role: string; content: unknown }[],
|
||||
): { role: string; content: unknown }[] {
|
||||
return messages.map((message) => {
|
||||
if (!Array.isArray(message.content)) return message;
|
||||
const content = message.content.map(
|
||||
(part: { type?: string; image_url?: unknown }) => {
|
||||
if (
|
||||
part.type === 'image_url' &&
|
||||
part.image_url !== null &&
|
||||
typeof part.image_url === 'object' &&
|
||||
'url' in (part.image_url as object)
|
||||
) {
|
||||
return {
|
||||
...part,
|
||||
image_url: (part.image_url as { url: string }).url,
|
||||
};
|
||||
}
|
||||
return part;
|
||||
},
|
||||
);
|
||||
return { ...message, content };
|
||||
});
|
||||
}
|
||||
|
||||
async complete({
|
||||
messages,
|
||||
stream,
|
||||
@@ -70,6 +102,7 @@ export class MistralAIProvider implements IChatProvider {
|
||||
temperature,
|
||||
}: ICompleteArguments): Promise<IChatCompleteResult> {
|
||||
messages = await OpenAIUtil.process_input_messages(messages);
|
||||
messages = this.#coerceImageUrls(messages);
|
||||
for (const message of messages) {
|
||||
if (message.tool_calls) {
|
||||
message.toolCalls = message.tool_calls;
|
||||
|
||||
Reference in New Issue
Block a user