Grok image provider mega update

This commit is contained in:
Neal Shah
2026-06-24 10:00:35 -04:00
parent 0c294ae9db
commit 3985fddc11
6 changed files with 336 additions and 62 deletions
@@ -68,6 +68,8 @@ vi.mock('openai', () => {
generate: openaiImagesGenerateMock,
edit: openaiImagesEditMock,
};
// xAI provider reaches its JSON edit endpoint through the SDK's post().
this.post = vi.fn();
this.chat = { completions: { create: vi.fn() } };
this.moderations = { create: vi.fn() };
this.responses = { create: vi.fn() };
@@ -217,8 +219,8 @@ describe('ImageGenerationDriver model catalog', () => {
const ids = all.map((m) => m.id);
// OpenAI catalog: gpt-image-1-mini should be present (lowercased by buildModelMap).
expect(ids).toContain('gpt-image-1-mini');
// xAI catalog: grok-2-image should be present.
expect(ids).toContain('grok-2-image');
// xAI catalog: grok-imagine-image should be present.
expect(ids).toContain('grok-imagine-image');
});
it('list() returns ids/puterIds sorted', async () => {
@@ -272,21 +274,21 @@ describe('ImageGenerationDriver.generate provider routing', () => {
expect(replicateRunMock).not.toHaveBeenCalled();
});
it('routes a known grok-2-image id to the xAI image provider (also OpenAI-SDK shaped)', async () => {
it('routes a known grok-imagine-image id to the xAI image provider (also OpenAI-SDK shaped)', async () => {
openaiImagesGenerateMock.mockResolvedValueOnce({
data: [{ url: 'https://xai/img.png' }],
});
await withActor(() =>
driver.generate({
model: 'grok-2-image',
model: 'grok-imagine-image',
prompt: 'hi',
} as never),
);
// xAI's provider also uses the OpenAI mock — assert via the call args.
const sent = openaiImagesGenerateMock.mock.calls[0]![0];
expect(sent.model).toBe('grok-2-image');
expect(sent.model).toBe('grok-imagine-image');
expect(sent.prompt).toBe('hi');
});
@@ -49,8 +49,9 @@ import { XAIImageProvider } from './XAIImageProvider.js';
// ── OpenAI SDK mock ─────────────────────────────────────────────────
const { generateMock, openAICtor } = vi.hoisted(() => ({
const { generateMock, postMock, openAICtor } = vi.hoisted(() => ({
generateMock: vi.fn(),
postMock: vi.fn(),
openAICtor: vi.fn(),
}));
@@ -61,6 +62,8 @@ vi.mock('openai', () => {
) {
openAICtor(opts);
this.images = { generate: generateMock };
// Low-level post() is how the provider reaches xAI's JSON edit endpoint.
this.post = postMock;
// Some sibling providers boot through the same SDK module.
this.chat = { completions: { create: vi.fn() } };
});
@@ -71,7 +74,9 @@ vi.mock('openai', () => {
let server: PuterServer;
let hasCreditsSpy: MockInstance<MeteringService['hasEnoughCredits']>;
let incrementUsageSpy: MockInstance<MeteringService['incrementUsage']>;
let batchIncrementUsagesSpy: MockInstance<
MeteringService['batchIncrementUsages']
>;
beforeAll(async () => {
server = await setupTestServer();
@@ -86,9 +91,13 @@ const makeProvider = () =>
beforeEach(() => {
generateMock.mockReset();
postMock.mockReset();
openAICtor.mockReset();
hasCreditsSpy = vi.spyOn(server.services.metering, 'hasEnoughCredits');
incrementUsageSpy = vi.spyOn(server.services.metering, 'incrementUsage');
batchIncrementUsagesSpy = vi.spyOn(
server.services.metering,
'batchIncrementUsages',
);
});
afterEach(() => {
@@ -121,15 +130,22 @@ describe('XAIImageProvider construction', () => {
// ── Model catalog ───────────────────────────────────────────────────
describe('XAIImageProvider model catalog', () => {
it('returns grok-2-image as the default', () => {
it('returns grok-imagine-image as the default', () => {
const provider = makeProvider();
expect(provider.getDefaultModel()).toBe('grok-2-image');
expect(provider.getDefaultModel()).toBe('grok-imagine-image');
});
it('exposes the static XAI_IMAGE_GENERATION_MODELS list verbatim', () => {
const provider = makeProvider();
expect(provider.models()).toBe(XAI_IMAGE_GENERATION_MODELS);
});
it('no longer exposes the deprecated grok-2-image model', () => {
const provider = makeProvider();
expect(provider.models().some((m) => m.id === 'grok-2-image')).toBe(
false,
);
});
});
// ── test_mode bypass ────────────────────────────────────────────────
@@ -149,7 +165,7 @@ describe('XAIImageProvider.generate test_mode', () => {
);
expect(hasCreditsSpy).not.toHaveBeenCalled();
expect(generateMock).not.toHaveBeenCalled();
expect(incrementUsageSpy).not.toHaveBeenCalled();
expect(batchIncrementUsagesSpy).not.toHaveBeenCalled();
});
});
@@ -188,7 +204,7 @@ describe('XAIImageProvider.generate credit gate', () => {
).rejects.toMatchObject({ statusCode: 402 });
expect(generateMock).not.toHaveBeenCalled();
expect(incrementUsageSpy).not.toHaveBeenCalled();
expect(batchIncrementUsagesSpy).not.toHaveBeenCalled();
});
});
@@ -208,7 +224,7 @@ describe('XAIImageProvider.generate model resolution', () => {
}),
);
expect(generateMock.mock.calls[0]![0].model).toBe('grok-2-image');
expect(generateMock.mock.calls[0]![0].model).toBe('grok-imagine-image');
});
it('resolves an alias to its canonical id', async () => {
@@ -217,20 +233,20 @@ describe('XAIImageProvider.generate model resolution', () => {
await withTestActor(() =>
provider.generate({
// grok-image is an alias of grok-2-image.
// grok-image is an alias of grok-imagine-image.
model: 'grok-image',
prompt: 'hi',
}),
);
expect(generateMock.mock.calls[0]![0].model).toBe('grok-2-image');
expect(generateMock.mock.calls[0]![0].model).toBe('grok-imagine-image');
});
});
// ── Successful generation ───────────────────────────────────────────
describe('XAIImageProvider.generate success path', () => {
it('returns the URL from response.data[0].url and meters one image', async () => {
it('returns the URL from response.data[0].url and meters one image at the 1k output rate', async () => {
const provider = makeProvider();
generateMock.mockResolvedValueOnce({
data: [{ url: 'https://x.ai/img/abc' }],
@@ -238,22 +254,48 @@ describe('XAIImageProvider.generate success path', () => {
const result = await withTestActor(() =>
provider.generate({
model: 'grok-2-image',
model: 'grok-imagine-image',
prompt: 'a small red dot',
}),
);
expect(result).toBe('https://x.ai/img/abc');
// No input images → generate endpoint, not the edit endpoint.
expect(postMock).not.toHaveBeenCalled();
// One increment, at the model's per-image rate (7 cents → 7,000,000 ucents).
const grok = XAI_IMAGE_GENERATION_MODELS.find(
(m) => m.id === 'grok-2-image',
(m) => m.id === 'grok-imagine-image',
)!;
expect(incrementUsageSpy).toHaveBeenCalledTimes(1);
const [, usageType, count, cost] = incrementUsageSpy.mock.calls[0]!;
expect(usageType).toBe('xai:grok-2-image:output');
expect(count).toBe(1);
expect(cost).toBe(grok.costs.output * 1_000_000);
expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1);
const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!;
expect(entries).toHaveLength(1);
const out = (
entries as Array<{ usageType: string; costOverride: number }>
)[0];
expect(out.usageType).toBe('xai:grok-imagine-image:output:1k');
expect(out.costOverride).toBe(grok.costs['output:1k'] * 1_000_000);
});
it('uses the 2k output rate when quality is "2k"', async () => {
const provider = makeProvider();
generateMock.mockResolvedValueOnce({
data: [{ url: 'https://x.ai/img/2k' }],
});
await withTestActor(() =>
provider.generate({
model: 'grok-imagine-image-quality',
prompt: 'hi',
quality: '2k',
}),
);
const sent = generateMock.mock.calls[0]![0];
expect(sent.resolution).toBe('2k');
const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!;
expect(
(entries as Array<{ usageType: string }>)[0].usageType,
).toBe('xai:grok-imagine-image-quality:output:2k');
});
it('falls back to a base64 data URL when response carries b64_json', async () => {
@@ -264,7 +306,7 @@ describe('XAIImageProvider.generate success path', () => {
const result = await withTestActor(() =>
provider.generate({
model: 'grok-2-image',
model: 'grok-imagine-image',
prompt: 'a small red dot',
}),
);
@@ -279,13 +321,117 @@ describe('XAIImageProvider.generate success path', () => {
await expect(
withTestActor(() =>
provider.generate({
model: 'grok-2-image',
model: 'grok-imagine-image',
prompt: 'a small red dot',
}),
),
).rejects.toThrow(/Failed to extract image URL/);
// Failure path must NOT meter usage.
expect(incrementUsageSpy).not.toHaveBeenCalled();
expect(batchIncrementUsagesSpy).not.toHaveBeenCalled();
});
});
// ── Image-to-image editing (input_images) ───────────────────────────
describe('XAIImageProvider.generate input_images (edit endpoint)', () => {
const PNG = 'data:image/png;base64,iVBORw0KGgo=';
const editResponse = { data: [{ url: 'https://x.ai/img/edited' }] };
it('routes input_images to POST /v1/images/edits (not generate) with a single image object', async () => {
const provider = makeProvider();
postMock.mockResolvedValueOnce(editResponse);
const result = await withTestActor(() =>
provider.generate({
model: 'grok-imagine-image',
prompt: 'add a hat',
input_images: [PNG],
}),
);
expect(result).toBe('https://x.ai/img/edited');
expect(generateMock).not.toHaveBeenCalled();
expect(postMock).toHaveBeenCalledTimes(1);
const [path, opts] = postMock.mock.calls[0]!;
expect(path).toBe('/images/edits');
const body = (opts as { body: Record<string, unknown> }).body;
expect(body.model).toBe('grok-imagine-image');
// Single image → object, not an array.
expect(body.image).toEqual({ type: 'image_url', url: PNG });
});
it('sends an array of image objects for multi-image edits and caps at 3', async () => {
const provider = makeProvider();
postMock.mockResolvedValueOnce(editResponse);
await withTestActor(() =>
provider.generate({
model: 'grok-imagine-image',
prompt: 'merge them',
input_images: [PNG, PNG, PNG, PNG], // 4 → capped to 3
}),
);
const body = (
postMock.mock.calls[0]![1] as { body: Record<string, unknown> }
).body;
expect(Array.isArray(body.image)).toBe(true);
expect(body.image).toHaveLength(3);
});
it('meters output + media_input per input image on edits', async () => {
const provider = makeProvider();
postMock.mockResolvedValueOnce(editResponse);
await withTestActor(() =>
provider.generate({
model: 'grok-imagine-image',
prompt: 'add a hat',
input_images: [PNG, PNG],
}),
);
const grok = XAI_IMAGE_GENERATION_MODELS.find(
(m) => m.id === 'grok-imagine-image',
)!;
const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!;
const types = (entries as Array<{ usageType: string }>).map(
(e) => e.usageType,
);
expect(types).toEqual(
expect.arrayContaining([
'xai:grok-imagine-image:output:1k',
'xai:grok-imagine-image:media_input',
]),
);
const media = (
entries as Array<{
usageType: string;
usageAmount: number;
costOverride: number;
}>
).find((e) => e.usageType.endsWith(':media_input'))!;
expect(media.usageAmount).toBe(2);
expect(media.costOverride).toBe(grok.costs.media_input * 2 * 1_000_000);
});
it('folds singular input_image into the edit path', async () => {
const provider = makeProvider();
postMock.mockResolvedValueOnce(editResponse);
await withTestActor(() =>
provider.generate({
model: 'grok-imagine-image',
prompt: 'add a hat',
input_image: PNG,
}),
);
expect(postMock).toHaveBeenCalledTimes(1);
const body = (
postMock.mock.calls[0]![1] as { body: Record<string, unknown> }
).body;
expect(body.image).toEqual({ type: 'image_url', url: PNG });
});
});
@@ -28,8 +28,13 @@ import type {
import { XAI_IMAGE_GENERATION_MODELS } from './models.js';
import { HttpError } from '../../../../core/http/HttpError.js';
const DEFAULT_MODEL = 'grok-2-image';
const PRICE_KEY = 'output';
const DEFAULT_MODEL = 'grok-imagine-image';
// xAI's Grok Imagine edit endpoint accepts up to 3 source images per request.
const MAX_INPUT_IMAGES = 3;
interface XaiImageResponse {
data?: Array<{ url?: string; b64_json?: string }>;
}
export class XAIImageProvider implements IImageProvider {
#client: OpenAI;
@@ -56,8 +61,9 @@ export class XAIImageProvider implements IImageProvider {
}
async generate(params: IGenerateParams): Promise<string> {
const { prompt, test_mode } = params;
const { model } = params;
const { prompt, test_mode, model, ratio, quality } = params;
let { input_images } = params;
const { input_image, input_image_mime_type } = params;
const selectedModel = this.#getModel(model);
@@ -71,15 +77,33 @@ export class XAIImageProvider implements IImageProvider {
});
}
// Backwards compat: fold singular `input_image` into `input_images`.
if (input_image && (!input_images || input_images.length === 0)) {
input_images = [input_image];
}
// xAI caps edits at 3 source images.
if (input_images && input_images.length > MAX_INPUT_IMAGES) {
input_images = input_images.slice(0, MAX_INPUT_IMAGES);
}
const inputImageCount = input_images?.length ?? 0;
const hasInputImages = inputImageCount > 0;
// xAI uses a `resolution` tier ('1k'/'2k') rather than a pixel size.
const resolution = this.#normalizeResolution(quality);
const aspectRatio = this.#aspectRatio(ratio);
const actor = Context.get('actor');
const userIdentifier =
actor?.user.id + actor?.app?.uid ? `:${actor?.app?.uid}` : '';
const priceInCents = selectedModel.costs[PRICE_KEY];
const costInMicroCents = priceInCents * 1_000_000;
const outputPriceInCents = selectedModel.costs[`output:${resolution}`];
const mediaInputPriceInCents = selectedModel.costs.media_input ?? 0;
const estimatedCostInCents =
outputPriceInCents +
(hasInputImages ? mediaInputPriceInCents * inputImageCount : 0);
const usageAllowed = await this.#meteringService.hasEnoughCredits(
actor,
costInMicroCents,
estimatedCostInCents * 1_000_000,
);
if (!usageAllowed) {
@@ -90,15 +114,27 @@ export class XAIImageProvider implements IImageProvider {
);
}
const response = await this.#client.images.generate({
model: selectedModel.id,
prompt,
user: userIdentifier,
});
const response = hasInputImages
? await this.#edit(
selectedModel.id,
prompt,
input_images!,
input_image_mime_type,
resolution,
aspectRatio,
)
: ((await this.#client.images.generate({
model: selectedModel.id,
prompt,
user: userIdentifier,
// xAI-specific params not in the OpenAI type; passed through.
...(aspectRatio ? { aspect_ratio: aspectRatio } : {}),
resolution,
} as Parameters<
OpenAI['images']['generate']
>[0])) as XaiImageResponse);
const first = response.data?.[0] as
| { url?: string; b64_json?: string }
| undefined;
const first = response.data?.[0];
const url =
first?.url ||
(first?.b64_json
@@ -109,16 +145,75 @@ export class XAIImageProvider implements IImageProvider {
throw new Error('Failed to extract image URL from xAI response');
}
this.#meteringService.incrementUsage(
actor,
`xai:${selectedModel.id}:${PRICE_KEY}`,
1,
costInMicroCents,
);
const usageEntries = [
{
usageType: `xai:${selectedModel.id}:output:${resolution}`,
usageAmount: 1,
costOverride: outputPriceInCents * 1_000_000,
},
];
if (hasInputImages && mediaInputPriceInCents > 0) {
usageEntries.push({
usageType: `xai:${selectedModel.id}:media_input`,
usageAmount: inputImageCount,
costOverride:
mediaInputPriceInCents * inputImageCount * 1_000_000,
});
}
this.#meteringService.batchIncrementUsages(actor, usageEntries);
return url;
}
// Edits go to POST /v1/images/edits as application/json (the OpenAI SDK's
// images.edit() can't be used — it sends multipart/form-data, which xAI
// rejects). We reuse the SDK client's auth + baseURL via its low-level
// post(). Input images are passed as `{ type: 'image_url', url }` objects;
// a single object for one image, an array for multiple.
async #edit(
modelId: string,
prompt: string,
inputImages: string[],
mimeHint: string | undefined,
resolution: string,
aspectRatio: string | undefined,
): Promise<XaiImageResponse> {
const refs = inputImages.map((img) => this.#toImageRef(img, mimeHint));
const body: Record<string, unknown> = {
model: modelId,
prompt,
image: refs.length === 1 ? refs[0] : refs,
resolution,
};
if (aspectRatio) body.aspect_ratio = aspectRatio;
return (await this.#client.post('/images/edits', {
body,
})) as XaiImageResponse;
}
// xAI accepts a public URL or a base64 data URI for input images.
#toImageRef(img: string, mimeHint?: string) {
const url =
img.startsWith('http://') ||
img.startsWith('https://') ||
img.startsWith('data:')
? img
: `data:${mimeHint ?? 'image/png'};base64,${img}`;
return { type: 'image_url', url };
}
#normalizeResolution(quality?: string): '1k' | '2k' {
return (quality ?? '').toLowerCase() === '2k' ? '2k' : '1k';
}
#aspectRatio(ratio?: { w: number; h: number }): string | undefined {
if (!ratio || !ratio.w || !ratio.h) return undefined;
const gcd = (a: number, b: number): number =>
b === 0 ? a : gcd(b, a % b);
const d = gcd(ratio.w, ratio.h) || 1;
return `${ratio.w / d}:${ratio.h / d}`;
}
#getModel(model?: string) {
const models = this.models();
const found = models.find(
@@ -19,17 +19,43 @@
import type { IImageModel } from '../../types.js';
// Costs are in usd-cents (1 = $0.01). xAI's "Grok Imagine" image API bills a
// per-image output rate by resolution tier (1k/2k) plus, for edits, a
// per-input-image "media input" rate. Rates per the xAI Imagine pricing table:
// https://docs.x.ai/developers/pricing
// grok-imagine-image media $0.002 | 1k $0.02 | 2k $0.02
// grok-imagine-image-quality media $0.01 | 1k $0.05 | 2k $0.07
export const XAI_IMAGE_GENERATION_MODELS: IImageModel[] = [
{
puterId: 'x-ai:x-ai/grok-2-image',
id: 'grok-2-image',
aliases: ['grok-image', 'x-ai/grok-image', 'x-ai/grok-2-image'],
name: 'Grok 2 Image',
puterId: 'x-ai:x-ai/grok-imagine-image',
id: 'grok-imagine-image',
aliases: ['grok-image', 'x-ai/grok-image', 'x-ai/grok-imagine-image'],
name: 'Grok Imagine Image',
version: '1.0',
costs_currency: 'usd-cents',
index_cost_key: 'output',
pricing_unit: 'per-image',
index_cost_key: 'output:1k',
costs: {
output: 7, // $0.07 per image
'output:1k': 2, // $0.02 per image
'output:2k': 2, // $0.02 per image
media_input: 0.2, // $0.002 per input image (edits)
},
allowedQualityLevels: ['1k', '2k'],
},
{
puterId: 'x-ai:x-ai/grok-imagine-image-quality',
id: 'grok-imagine-image-quality',
aliases: ['x-ai/grok-imagine-image-quality'],
name: 'Grok Imagine Image (Quality)',
version: '1.0',
costs_currency: 'usd-cents',
pricing_unit: 'per-image',
index_cost_key: 'output:1k',
costs: {
'output:1k': 5, // $0.05 per image
'output:2k': 7, // $0.07 per image
media_input: 1, // $0.01 per input image (edits)
},
allowedQualityLevels: ['1k', '2k'],
},
];
+7 -4
View File
@@ -32,7 +32,7 @@ Additional settings for the generation request. Available options depend on the
|--------|------|-------------|
| `prompt` | `String` | Text description for the image generation |
| `provider` | `String` | The AI provider to use. `'openai-image-generation' (default) \| 'gemini' \| 'together' \| 'xai' \| 'replicate-image-generation'` |
| `model` | `String` | Image model to use (provider-specific). Defaults to `'gpt-image-1-mini'` (OpenAI) or `'grok-2-image'` when `provider: 'xai'` |
| `model` | `String` | Image model to use (provider-specific). Defaults to `'gpt-image-1-mini'` (OpenAI) or `'grok-imagine-image'` when `provider: 'xai'` |
| `test_mode` | `Boolean` | When `true`, returns a sample image without using credits |
| `puter_output_path` | `String` | When set, the generated image is automatically saved to this path on the Puter filesystem. Relative paths are resolved against the app's data directory (or `~/` outside an app). The caller must have write permission to the destination |
@@ -63,12 +63,15 @@ Available when `provider: 'gemini'` or inferred from model:
#### xAI (Grok) Options
Available when `provider: 'xai'` or inferred from model (`grok-2-image`, alias `grok-image`):
Available when `provider: 'xai'` or inferred from model (`grok-imagine-image`, alias `grok-image`):
| Option | Type | Description |
|--------|------|-------------|
| `model` | `String` | Image model to use. Available: `'grok-2-image'` (default) |
| `prompt` | `String` | Text prompt for the image. Grok Image does not support quality/size overrides; pricing is $0.07 per generated image. |
| `model` | `String` | Image model to use. Available: `'grok-imagine-image'` (default), `'grok-imagine-image-quality'` |
| `prompt` | `String` | Text prompt for the image (or the edit instruction when input images are supplied). |
| `quality` | `String` | Output resolution tier: `'1k'` (default) or `'2k'`. |
| `input_image` | `String` | A public URL or base64-encoded (data-URI) input image for image-to-image editing. |
| `input_images` | `Array<String>` | Up to 3 input images (URLs or base64/data-URI) for multi-image editing — combine subjects, transfer styles, compose scenes. Routes through xAI's image edit endpoint. |
#### Together Options
+6 -4
View File
@@ -147,7 +147,7 @@ export interface Txt2ImgOptions {
prompt?: string;
/**
* Image model to use (provider-specific). Defaults to `'gpt-image-1-mini'`
* (OpenAI), or `'grok-2-image'` when `provider` is `'xai'`.
* (OpenAI), or `'grok-imagine-image'` when `provider` is `'xai'`.
*/
model?: string;
/**
@@ -160,14 +160,16 @@ export interface Txt2ImgOptions {
*/
quality?: string;
/**
* An input image for image-to-image generation. Replicate expects a URL;
* Gemini and OpenAI `gpt-image-*` expect a base64-encoded (or data-URI) image.
* An input image for image-to-image generation. Replicate and xAI
* `grok-imagine-*` accept a URL; Gemini and OpenAI `gpt-image-*` expect a
* base64-encoded (or data-URI) image (xAI also accepts base64/data-URI).
*/
input_image?: string;
/**
* Multiple input images for image-to-image / multi-image generation.
* Gemini and OpenAI `gpt-image-*` expect base64-encoded (or data-URI)
* images; Replicate expects image URLs.
* images; Replicate expects image URLs; xAI `grok-imagine-*` accepts either
* (up to 3 images).
*/
input_images?: string[];
/**