per model allowed params on replicate image, normalize ratio (#2898)

* replicate allowed params, normalize ratio

* update docs
This commit is contained in:
Shruc
2026-05-06 21:16:30 +03:00
committed by GitHub
parent 7a1a885e89
commit f00d52f879
5 changed files with 361 additions and 100 deletions
@@ -135,6 +135,9 @@ export class ImageGenerationDriver extends PuterDriver {
throw new HttpError(500, `No provider found for model ${model.id}`);
}
// `width`/`height` or `aspect_ratio` -> `ratio: {w,h}`
this.#normalizeRatio(args);
// Audit log for abuse / billing. Fired before the upstream call
// so a failed generate still shows up in the log (prompt_block
// uses this to track user-by-user image prompts).
@@ -159,6 +162,34 @@ export class ImageGenerationDriver extends PuterDriver {
});
}
#normalizeRatio(parameters: IGenerateParams) {
if (parameters.ratio) return;
const w = parameters.width as number | undefined;
const h = parameters.height as number | undefined;
if (typeof w === 'number' && typeof h === 'number') {
parameters.ratio = { w, h };
delete parameters.width;
delete parameters.height;
return;
}
const ar = parameters.aspect_ratio as string | undefined;
if (typeof ar === 'string' && ar.includes(':')) {
const [aw, ah] = ar.split(':').map(Number);
if (
Number.isFinite(aw) &&
Number.isFinite(ah) &&
aw > 0 &&
ah > 0
) {
parameters.ratio = { w: aw, h: ah };
delete parameters.aspect_ratio;
return;
}
}
}
#registerProviders() {
const providers = this.config.providers ?? {};
const m = this.services.metering;
@@ -33,20 +33,19 @@ import {
const DEFAULT_MODEL = 'black-forest-labs/flux-schnell';
const DEFAULT_RATIO = { w: 1024, h: 1024 };
type ReplicateGenerateParams = IGenerateParams & {
go_fast?: boolean;
seed?: number;
steps?: number;
guidance?: number;
output_quality?: number;
output_megapixels?: string;
prompt_strength?: number;
negative_prompt?: string;
response_format?: string;
disable_safety_checker?: boolean;
};
export class ReplicateImageGenerationProvider implements IImageProvider {
static readonly #CORE_PARAMS: readonly string[] = [
'prompt',
'model',
'ratio',
'quality',
'provider',
'test_mode',
'input_image',
'input_image_mime_type',
'input_images',
];
#client: Replicate;
#meteringService: MeteringService;
@@ -67,11 +66,10 @@ export class ReplicateImageGenerationProvider implements IImageProvider {
}
async generate(params: IGenerateParams): Promise<string> {
const extra = params as ReplicateGenerateParams;
const { prompt, test_mode } = extra;
const { prompt, test_mode } = params;
const selectedModel = this.#getModel(extra.model);
const ratio = this.#normalizeRatio(extra.ratio);
const selectedModel = this.#getModel(params.model);
const ratio = this.#normalizeRatio(params.ratio);
if (test_mode) {
return 'https://puter-sample-data.puter.site/image_example.png';
@@ -90,34 +88,61 @@ export class ReplicateImageGenerationProvider implements IImageProvider {
});
}
const goFast = selectedModel.supportsGoFast
? extra.go_fast !== undefined
? !!extra.go_fast
: (selectedModel.goFastDefault ?? false)
: false;
const filtered = this.#filterAllowedParams(params, selectedModel);
const aliased = this.#applyParamAliases(filtered, selectedModel);
const transformed = this.#applyTransforms(aliased, selectedModel);
const goFast = !!transformed.go_fast;
const generationMode =
typeof transformed.generation_mode === 'string'
? transformed.generation_mode
: undefined;
const inputImages: string[] = [];
if (selectedModel.imageInputKey) {
if (extra.input_image) inputImages.push(extra.input_image);
if (extra.input_images?.length)
inputImages.push(...extra.input_images);
if (params.input_image) inputImages.push(params.input_image);
if (params.input_images?.length)
inputImages.push(...params.input_images);
if (inputImages.length === 0) {
const nativeVal = (params as Record<string, unknown>)[
selectedModel.imageInputKey
];
if (typeof nativeVal === 'string') {
inputImages.push(nativeVal);
} else if (Array.isArray(nativeVal)) {
for (const v of nativeVal) {
if (typeof v === 'string') inputImages.push(v);
}
}
}
}
let singleImage: string | undefined;
if (selectedModel.singleImageInputKey) {
if (typeof params.input_image === 'string') {
singleImage = params.input_image;
} else {
const nativeVal = (params as Record<string, unknown>)[
selectedModel.singleImageInputKey
];
if (typeof nativeVal === 'string') singleImage = nativeVal;
}
}
const singleImage = selectedModel.singleImageInputKey
? extra.input_image
: undefined;
const allInputUrls = singleImage ? [singleImage] : inputImages;
const inputMp =
allInputUrls.length > 0
? await this.#measureInputMegapixels(allInputUrls)
: 0;
const outputMp = this.#resolveOutputMegapixels(extra.output_megapixels);
const outputMp = this.#resolveOutputMegapixels(
params.output_megapixels as string | undefined,
);
const totalCostMicroCents = this.#estimateCost(
selectedModel,
outputMp,
goFast,
inputMp,
generationMode,
);
if (totalCostMicroCents <= 0) {
throw new HttpError(
@@ -140,42 +165,13 @@ export class ReplicateImageGenerationProvider implements IImageProvider {
);
}
const input: Record<string, unknown> = {
const input = this.#buildRequest(selectedModel, {
prompt,
aspect_ratio: this.#toAspectRatio(ratio),
disable_safety_checker: !!extra.disable_safety_checker,
};
if (selectedModel.supportsGoFast) {
input.go_fast = goFast;
}
if (inputImages.length && selectedModel.imageInputKey) {
input[selectedModel.imageInputKey] = inputImages;
} else if (singleImage && selectedModel.singleImageInputKey) {
input[selectedModel.singleImageInputKey] = singleImage;
}
if (Number.isFinite(extra.seed))
input.seed = Math.round(extra.seed as number);
if (Number.isFinite(extra.steps))
input.num_inference_steps = Math.round(extra.steps as number);
if (Number.isFinite(extra.guidance)) input.guidance = extra.guidance;
if (Number.isFinite(extra.output_quality))
input.output_quality = Math.round(extra.output_quality as number);
if (
typeof extra.output_megapixels === 'string' &&
selectedModel.resolutionInputKey
) {
input[selectedModel.resolutionInputKey] =
extra.output_megapixels +
(selectedModel.resolutionSuffix ?? '');
} else if (typeof extra.output_megapixels === 'string') {
input.megapixels = extra.output_megapixels;
}
if (Number.isFinite(extra.prompt_strength))
input.prompt_strength = extra.prompt_strength;
if (typeof extra.negative_prompt === 'string')
input.negative_prompt = extra.negative_prompt;
if (typeof extra.response_format === 'string')
input.output_format = extra.response_format;
ratio,
transformed,
inputImages,
singleImage,
});
const output = await this.#client.run(
selectedModel.replicateId as `${string}/${string}`,
@@ -191,7 +187,14 @@ export class ReplicateImageGenerationProvider implements IImageProvider {
);
}
this.#recordUsage(actor, selectedModel, outputMp, goFast, inputMp);
this.#recordUsage(
actor,
selectedModel,
outputMp,
goFast,
inputMp,
generationMode,
);
return url;
}
@@ -204,6 +207,130 @@ export class ReplicateImageGenerationProvider implements IImageProvider {
return found ?? models.find((m) => m.id === DEFAULT_MODEL)!;
}
/**
* Builds the Replicate API input payload from already-aliased+transformed
* params. Image inputs and prompt/ratio are placed explicitly; everything
* else is spread verbatim so newly-allowed keys flow through without
* needing a code change here.
*/
#buildRequest(
model: ReplicateImageModel,
ctx: {
prompt: string;
ratio: { w: number; h: number };
transformed: Record<string, unknown>;
inputImages: string[];
singleImage?: string;
},
): Record<string, unknown> {
const { prompt, ratio, transformed, inputImages, singleImage } = ctx;
const input: Record<string, unknown> = {
prompt,
aspect_ratio: this.#toAspectRatio(ratio),
};
const handled = new Set<string>(
ReplicateImageGenerationProvider.#CORE_PARAMS,
);
if (model.imageInputKey) handled.add(model.imageInputKey);
if (model.singleImageInputKey) handled.add(model.singleImageInputKey);
if (inputImages.length && model.imageInputKey) {
input[model.imageInputKey] = inputImages;
} else if (singleImage && model.singleImageInputKey) {
input[model.singleImageInputKey] = singleImage;
}
for (const [key, value] of Object.entries(transformed)) {
if (handled.has(key)) continue;
if (value === undefined || value === null) continue;
input[key] = value;
}
return input;
}
/**
* Drops params not in `model.allowed_params` (plus `#CORE_PARAMS` and any
* alias targets, so the native key is also accepted).
*/
#filterAllowedParams(
params: IGenerateParams,
model: ReplicateImageModel,
): IGenerateParams {
const allowedSet = model.allowed_params;
if (!allowedSet) return params;
const aliasTargets = model.param_aliases
? Object.values(model.param_aliases)
: [];
const nativeImageKeys: string[] = [];
if (model.imageInputKey) nativeImageKeys.push(model.imageInputKey);
if (model.singleImageInputKey)
nativeImageKeys.push(model.singleImageInputKey);
const filtered: Record<string, unknown> = {};
for (const key of Object.keys(params)) {
if (
ReplicateImageGenerationProvider.#CORE_PARAMS.includes(key) ||
allowedSet.includes(key) ||
aliasTargets.includes(key) ||
nativeImageKeys.includes(key)
) {
filtered[key] = params[key];
}
}
return filtered as IGenerateParams;
}
/**
* Renames canonical keys to the model's native API names per
* `model.param_aliases` (e.g. `steps` → `num_inference_steps`).
*/
#applyParamAliases(
params: IGenerateParams,
model: ReplicateImageModel,
): Record<string, unknown> {
const aliases = model.param_aliases;
if (!aliases) return params as Record<string, unknown>;
const result: Record<string, unknown> = {};
for (const [key, value] of Object.entries(params)) {
const nativeKey = aliases[key] ?? key;
result[nativeKey] = value;
}
return result;
}
/**
* Applies `param_transforms` on top of the aliased map: injects defaults
* for missing keys, then appends any configured string suffix to the
* value. Returns the original map unchanged when the model declares no
* transforms.
*/
#applyTransforms(
aliased: Record<string, unknown>,
model: ReplicateImageModel,
): Record<string, unknown> {
const transforms = model.param_transforms;
if (!transforms) return aliased;
const result = { ...aliased };
for (const [key, cfg] of Object.entries(transforms)) {
let value = result[key];
if (value === undefined && cfg.default !== undefined) {
value = cfg.default;
}
if (value === undefined) continue;
if (cfg.suffix !== undefined && typeof value === 'string') {
value = value + cfg.suffix;
}
result[key] = value;
}
return result;
}
#normalizeRatio(ratio?: { w: number; h: number }) {
const w = Number(ratio?.w);
const h = Number(ratio?.h);
@@ -253,10 +380,16 @@ export class ReplicateImageGenerationProvider implements IImageProvider {
#resolveCosts(
model: ReplicateImageModel,
goFast: boolean,
generationMode?: string,
): Record<string, number> {
return goFast && model.costs_go_fast
? model.costs_go_fast
: model.costs;
if (goFast && model.costs_go_fast) return model.costs_go_fast;
if (
generationMode &&
model.costs_by_generation_mode?.[generationMode]
) {
return model.costs_by_generation_mode[generationMode];
}
return model.costs;
}
#estimateCost(
@@ -264,8 +397,9 @@ export class ReplicateImageGenerationProvider implements IImageProvider {
outputMp: number,
goFast: boolean,
inputMp: number,
generationMode?: string,
): number {
const costs = this.#resolveCosts(model, goFast);
const costs = this.#resolveCosts(model, goFast, generationMode);
if (model.billingScheme === 'per-image') {
const cents = costs.output;
@@ -300,9 +434,10 @@ export class ReplicateImageGenerationProvider implements IImageProvider {
outputMp: number,
goFast: boolean,
inputMp: number,
generationMode?: string,
) {
const prefix = `replicate:${model.id}`;
const costs = this.#resolveCosts(model, goFast);
const costs = this.#resolveCosts(model, goFast, generationMode);
if (model.billingScheme === 'per-image') {
const cents = costs.output;
@@ -24,15 +24,23 @@ export type ReplicateBillingScheme = 'per-image' | 'megapixel';
export type ReplicateImageModel = IImageModel & {
replicateId: string;
billingScheme: ReplicateBillingScheme;
imageInputKey?: string;
singleImageInputKey?: string;
supportsGoFast?: boolean;
goFastDefault?: boolean;
imageInputKey?: string; // our `input_images`
singleImageInputKey?: string; // our `input_image`
/** Cost map used when `go_fast: true` (e.g. flux-2-dev fast mode). */
costs_go_fast?: Record<string, number>;
resolutionInputKey?: string;
resolutionSuffix?: string;
/** Cost maps keyed by Leonardo `generation_mode`; falls back to `costs`. */
costs_by_generation_mode?: Record<string, Record<string, number>>;
/** Whitelist of caller-supplied params (canonical names) that are forwarded to Replicate. */
allowed_params?: string[];
/** Renames canonical param keys to the model's native API names (e.g. `steps` → `num_inference_steps`). */
param_aliases?: Record<string, string>;
/** Per-key value transforms (default + suffix) applied after `param_aliases`. */
param_transforms?: Record<string, { suffix?: string; default?: unknown }>;
};
const ALIAS_FORMAT = { response_format: 'output_format' };
const ALIAS_FORMAT_STEPS = { ...ALIAS_FORMAT, steps: 'num_inference_steps' };
// Costs are in USD cents.
// Megapixel models: `output_mp` = cost per output megapixel, `input_mp` =
// cost per input megapixel (img2img). Some also carry a flat `run` cost.
@@ -50,8 +58,15 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [
costs: { run: 1.5, input_mp: 1.5, output_mp: 1.5 },
billingScheme: 'megapixel',
imageInputKey: 'input_images',
resolutionInputKey: 'resolution',
resolutionSuffix: ' MP',
allowed_params: [
'seed',
'response_format',
'output_quality',
'output_megapixels',
'safety_tolerance',
],
param_aliases: { ...ALIAS_FORMAT, output_megapixels: 'resolution' },
param_transforms: { resolution: { suffix: ' MP' } },
},
{
id: 'black-forest-labs/flux-2-dev',
@@ -65,9 +80,15 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [
costs_go_fast: { input_mp: 1.2, output_mp: 1.2 },
billingScheme: 'megapixel',
imageInputKey: 'input_images',
supportsGoFast: true,
goFastDefault: true,
resolutionInputKey: 'output_megapixels',
allowed_params: [
'seed',
'response_format',
'output_quality',
'disable_safety_checker',
'go_fast',
],
param_aliases: ALIAS_FORMAT,
param_transforms: { go_fast: { default: true } },
},
{
id: 'black-forest-labs/flux-2-klein-9b-base',
@@ -80,7 +101,15 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [
costs: { input_mp: 1.1, output_mp: 1.1 },
billingScheme: 'megapixel',
imageInputKey: 'images',
resolutionInputKey: 'output_megapixels',
allowed_params: [
'seed',
'guidance',
'response_format',
'output_quality',
'disable_safety_checker',
'output_megapixels',
],
param_aliases: ALIAS_FORMAT,
},
{
id: 'black-forest-labs/flux-2-klein-4b',
@@ -93,7 +122,14 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [
costs: { input_mp: 0.1, output_mp: 0.1 },
billingScheme: 'megapixel',
imageInputKey: 'images',
resolutionInputKey: 'output_megapixels',
allowed_params: [
'seed',
'response_format',
'output_quality',
'disable_safety_checker',
'output_megapixels',
],
param_aliases: ALIAS_FORMAT,
},
// Black Forest Labs FLUX.1
@@ -107,6 +143,18 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [
index_cost_key: 'output',
costs: { output: 0.3 },
billingScheme: 'per-image',
allowed_params: [
'seed',
'steps',
'response_format',
'output_quality',
'disable_safety_checker',
'output_megapixels',
],
param_aliases: {
...ALIAS_FORMAT_STEPS,
output_megapixels: 'megapixels',
},
},
{
id: 'black-forest-labs/flux-1.1-pro',
@@ -119,6 +167,14 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [
costs: { output: 4 },
billingScheme: 'per-image',
singleImageInputKey: 'image_prompt',
allowed_params: [
'seed',
'response_format',
'output_quality',
'safety_tolerance',
'prompt_upsampling',
],
param_aliases: ALIAS_FORMAT,
},
// Leonardo AI
@@ -130,8 +186,20 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [
name: 'Lucid Origin',
costs_currency: 'usd-cents',
index_cost_key: 'output',
costs: { output: 1.65 },
costs: {
output: 1.65, // standard: 11 units * $0.0015/unit = $0.0165
},
costs_by_generation_mode: {
standard: { output: 1.65 },
ultra: { output: 7.65 }, // 51 units * $0.0015/unit = $0.0765
},
billingScheme: 'per-image',
allowed_params: [
'style',
'contrast',
'prompt_enhance',
'generation_mode',
],
},
{
id: 'leonardoai/phoenix-1.0',
@@ -141,7 +209,20 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [
name: 'Phoenix 1.0',
costs_currency: 'usd-cents',
index_cost_key: 'output',
costs: { output: 3.75 },
costs: {
output: 3.75, // quality default: 25 units * $0.0015/unit = $0.0375
},
costs_by_generation_mode: {
fast: { output: 1.8 }, // 12 units * $0.0015/unit = $0.018
quality: { output: 3.75 }, // 25 units * $0.0015/unit = $0.0375
ultra: { output: 7.5 }, // 50 units * $0.0015/unit = $0.075
},
billingScheme: 'per-image',
allowed_params: [
'style',
'contrast',
'prompt_enhance',
'generation_mode',
],
},
];
+1
View File
@@ -63,6 +63,7 @@ export interface IGenerateParams {
input_image?: string;
input_image_mime_type?: string;
input_images?: string[];
[key: string]: unknown;
}
export interface IImageProvider {
+28 -15
View File
@@ -95,24 +95,37 @@ For more details, see the [Together AI API reference](https://docs.together.ai/r
Available when `provider: 'replicate-image-generation'` or inferred from model:
##### Common options
| Option | Type | Description |
|--------|------|-------------|
| `model` | `String` | Image model to use. |
| `ratio` | `Object` | Aspect ratio as `{ w, h }` (e.g., `{ w: 16, h: 9 }`) |
| `seed` | `Number` | Random seed for reproducible generation |
| `steps` | `Number` | Number of inference steps |
| `guidance` | `Number` | Guidance scale for generation |
| `go_fast` | `Boolean` | Use optimized fast mode. Defaults to `true` for `flux-2-dev`. Affects pricing on supported models |
| `output_quality` | `Number` | Output quality (0-100). |
| `output_megapixels` | `String` | Approximate output megapixels (`'0.25'`, `'0.5'`, `'1'`, `'2'`, `'4'`) |
| `input_image` | `String` | URL of an input image for image-to-image generation |
| `input_images` | `Array<String>` | Array of input image URLs for multi-image generation |
| `negative_prompt` | `String` | Text to guide what to avoid in the image |
| `prompt_strength` | `Number` | How strongly the prompt influences the output |
| `disable_safety_checker` | `Boolean` | If `true`, disables the safety checker |
| `response_format` | `String` | Output format: `'webp'`, `'jpg'`, `'png'` |
| `model` | `String` | Model id (e.g. `'black-forest-labs/flux-schnell'`, `'leonardoai/lucid-origin'`). |
| `ratio` | `Object` | Aspect ratio as `{ w, h }` (e.g., `{ w: 16, h: 9 }`). |
| `input_image` | `String` | URL of an input image for image-to-image generation. |
| `input_images` | `Array<String>` | Array of input image URLs for multi-image generation. |
For more details, see the [Replicate API reference](https://replicate.com/docs).
##### Per-model options
These keys are only forwarded for models that whitelist them (see per-model `allowed_params`):
| Option | Type | Models | Description |
|--------|------|--------|-------------|
| `seed` | `Number` | most models | Random seed for reproducible generation. |
| `steps` | `Number` | `flux-schnell` | Number of inference steps. |
| `guidance` | `Number` | `flux-2-klein-9b-base` | Guidance scale. |
| `go_fast` | `Boolean` | `flux-2-dev` | Use optimized fast mode. Defaults to `true` for `flux-2-dev`; affects pricing. |
| `output_quality` | `Number` | flux family | Output quality (0100). |
| `output_megapixels` | `String` | flux family | Approximate output megapixels (e.g. `'0.25'`, `'0.5'`, `'1'`, `'2'`). |
| `disable_safety_checker` | `Boolean` | flux-2-dev / klein / flux-schnell | If `true`, disables the safety checker. |
| `safety_tolerance` | `Number` | `flux-2-pro`, `flux-1.1-pro` | Safety tolerance level. |
| `prompt_upsampling` | `Boolean` | `flux-1.1-pro` | Enable prompt upsampling. |
| `response_format` | `String` | most models | Output format (e.g. `'webp'`, `'jpg'`, `'png'`). |
| `generation_mode` | `String` | Leonardo (`lucid-origin`, `phoenix-1.0`) | Generation tier — affects pricing. e.g. `'standard'`/`'ultra'` (lucid-origin), `'fast'`/`'quality'`/`'ultra'` (phoenix-1.0). |
| `style` | `String` | Leonardo | Stylistic preset. |
| `contrast` | `String` | Leonardo | Contrast preset. |
| `prompt_enhance` | `Boolean` | Leonardo | Server-side prompt enhancement. |
For more details, see the [Replicate API reference](https://replicate.com/docs) and each model's schema page on Replicate.
Any properties not set fall back to provider defaults.