From f00d52f879eefed4adf178734f4f975ff96b8b40 Mon Sep 17 00:00:00 2001 From: Shruc <42489293+P3il4@users.noreply.github.com> Date: Wed, 6 May 2026 21:16:30 +0300 Subject: [PATCH] per model allowed params on replicate image, normalize ratio (#2898) * replicate allowed params, normalize ratio * update docs --- .../drivers/ai-image/ImageGenerationDriver.ts | 31 ++ .../ReplicateImageGenerationProvider.ts | 275 +++++++++++++----- .../ai-image/providers/replicate/models.ts | 111 ++++++- src/backend/drivers/ai-image/types.ts | 1 + src/docs/src/AI/txt2img.md | 43 ++- 5 files changed, 361 insertions(+), 100 deletions(-) diff --git a/src/backend/drivers/ai-image/ImageGenerationDriver.ts b/src/backend/drivers/ai-image/ImageGenerationDriver.ts index 8f70a23b8..bb5e28948 100644 --- a/src/backend/drivers/ai-image/ImageGenerationDriver.ts +++ b/src/backend/drivers/ai-image/ImageGenerationDriver.ts @@ -135,6 +135,9 @@ export class ImageGenerationDriver extends PuterDriver { throw new HttpError(500, `No provider found for model ${model.id}`); } + // `width`/`height` or `aspect_ratio` -> `ratio: {w,h}` + this.#normalizeRatio(args); + // Audit log for abuse / billing. Fired before the upstream call // so a failed generate still shows up in the log (prompt_block // uses this to track user-by-user image prompts). @@ -159,6 +162,34 @@ export class ImageGenerationDriver extends PuterDriver { }); } + #normalizeRatio(parameters: IGenerateParams) { + if (parameters.ratio) return; + + const w = parameters.width as number | undefined; + const h = parameters.height as number | undefined; + if (typeof w === 'number' && typeof h === 'number') { + parameters.ratio = { w, h }; + delete parameters.width; + delete parameters.height; + return; + } + + const ar = parameters.aspect_ratio as string | undefined; + if (typeof ar === 'string' && ar.includes(':')) { + const [aw, ah] = ar.split(':').map(Number); + if ( + Number.isFinite(aw) && + Number.isFinite(ah) && + aw > 0 && + ah > 0 + ) { + parameters.ratio = { w: aw, h: ah }; + delete parameters.aspect_ratio; + return; + } + } + } + #registerProviders() { const providers = this.config.providers ?? {}; const m = this.services.metering; diff --git a/src/backend/drivers/ai-image/providers/replicate/ReplicateImageGenerationProvider.ts b/src/backend/drivers/ai-image/providers/replicate/ReplicateImageGenerationProvider.ts index 36f0c8316..5fe68f45b 100644 --- a/src/backend/drivers/ai-image/providers/replicate/ReplicateImageGenerationProvider.ts +++ b/src/backend/drivers/ai-image/providers/replicate/ReplicateImageGenerationProvider.ts @@ -33,20 +33,19 @@ import { const DEFAULT_MODEL = 'black-forest-labs/flux-schnell'; const DEFAULT_RATIO = { w: 1024, h: 1024 }; -type ReplicateGenerateParams = IGenerateParams & { - go_fast?: boolean; - seed?: number; - steps?: number; - guidance?: number; - output_quality?: number; - output_megapixels?: string; - prompt_strength?: number; - negative_prompt?: string; - response_format?: string; - disable_safety_checker?: boolean; -}; - export class ReplicateImageGenerationProvider implements IImageProvider { + static readonly #CORE_PARAMS: readonly string[] = [ + 'prompt', + 'model', + 'ratio', + 'quality', + 'provider', + 'test_mode', + 'input_image', + 'input_image_mime_type', + 'input_images', + ]; + #client: Replicate; #meteringService: MeteringService; @@ -67,11 +66,10 @@ export class ReplicateImageGenerationProvider implements IImageProvider { } async generate(params: IGenerateParams): Promise { - const extra = params as ReplicateGenerateParams; - const { prompt, test_mode } = extra; + const { prompt, test_mode } = params; - const selectedModel = this.#getModel(extra.model); - const ratio = this.#normalizeRatio(extra.ratio); + const selectedModel = this.#getModel(params.model); + const ratio = this.#normalizeRatio(params.ratio); if (test_mode) { return 'https://puter-sample-data.puter.site/image_example.png'; @@ -90,34 +88,61 @@ export class ReplicateImageGenerationProvider implements IImageProvider { }); } - const goFast = selectedModel.supportsGoFast - ? extra.go_fast !== undefined - ? !!extra.go_fast - : (selectedModel.goFastDefault ?? false) - : false; + const filtered = this.#filterAllowedParams(params, selectedModel); + const aliased = this.#applyParamAliases(filtered, selectedModel); + const transformed = this.#applyTransforms(aliased, selectedModel); + + const goFast = !!transformed.go_fast; + const generationMode = + typeof transformed.generation_mode === 'string' + ? transformed.generation_mode + : undefined; const inputImages: string[] = []; if (selectedModel.imageInputKey) { - if (extra.input_image) inputImages.push(extra.input_image); - if (extra.input_images?.length) - inputImages.push(...extra.input_images); + if (params.input_image) inputImages.push(params.input_image); + if (params.input_images?.length) + inputImages.push(...params.input_images); + if (inputImages.length === 0) { + const nativeVal = (params as Record)[ + selectedModel.imageInputKey + ]; + if (typeof nativeVal === 'string') { + inputImages.push(nativeVal); + } else if (Array.isArray(nativeVal)) { + for (const v of nativeVal) { + if (typeof v === 'string') inputImages.push(v); + } + } + } + } + let singleImage: string | undefined; + if (selectedModel.singleImageInputKey) { + if (typeof params.input_image === 'string') { + singleImage = params.input_image; + } else { + const nativeVal = (params as Record)[ + selectedModel.singleImageInputKey + ]; + if (typeof nativeVal === 'string') singleImage = nativeVal; + } } - const singleImage = selectedModel.singleImageInputKey - ? extra.input_image - : undefined; const allInputUrls = singleImage ? [singleImage] : inputImages; const inputMp = allInputUrls.length > 0 ? await this.#measureInputMegapixels(allInputUrls) : 0; - const outputMp = this.#resolveOutputMegapixels(extra.output_megapixels); + const outputMp = this.#resolveOutputMegapixels( + params.output_megapixels as string | undefined, + ); const totalCostMicroCents = this.#estimateCost( selectedModel, outputMp, goFast, inputMp, + generationMode, ); if (totalCostMicroCents <= 0) { throw new HttpError( @@ -140,42 +165,13 @@ export class ReplicateImageGenerationProvider implements IImageProvider { ); } - const input: Record = { + const input = this.#buildRequest(selectedModel, { prompt, - aspect_ratio: this.#toAspectRatio(ratio), - disable_safety_checker: !!extra.disable_safety_checker, - }; - if (selectedModel.supportsGoFast) { - input.go_fast = goFast; - } - if (inputImages.length && selectedModel.imageInputKey) { - input[selectedModel.imageInputKey] = inputImages; - } else if (singleImage && selectedModel.singleImageInputKey) { - input[selectedModel.singleImageInputKey] = singleImage; - } - if (Number.isFinite(extra.seed)) - input.seed = Math.round(extra.seed as number); - if (Number.isFinite(extra.steps)) - input.num_inference_steps = Math.round(extra.steps as number); - if (Number.isFinite(extra.guidance)) input.guidance = extra.guidance; - if (Number.isFinite(extra.output_quality)) - input.output_quality = Math.round(extra.output_quality as number); - if ( - typeof extra.output_megapixels === 'string' && - selectedModel.resolutionInputKey - ) { - input[selectedModel.resolutionInputKey] = - extra.output_megapixels + - (selectedModel.resolutionSuffix ?? ''); - } else if (typeof extra.output_megapixels === 'string') { - input.megapixels = extra.output_megapixels; - } - if (Number.isFinite(extra.prompt_strength)) - input.prompt_strength = extra.prompt_strength; - if (typeof extra.negative_prompt === 'string') - input.negative_prompt = extra.negative_prompt; - if (typeof extra.response_format === 'string') - input.output_format = extra.response_format; + ratio, + transformed, + inputImages, + singleImage, + }); const output = await this.#client.run( selectedModel.replicateId as `${string}/${string}`, @@ -191,7 +187,14 @@ export class ReplicateImageGenerationProvider implements IImageProvider { ); } - this.#recordUsage(actor, selectedModel, outputMp, goFast, inputMp); + this.#recordUsage( + actor, + selectedModel, + outputMp, + goFast, + inputMp, + generationMode, + ); return url; } @@ -204,6 +207,130 @@ export class ReplicateImageGenerationProvider implements IImageProvider { return found ?? models.find((m) => m.id === DEFAULT_MODEL)!; } + /** + * Builds the Replicate API input payload from already-aliased+transformed + * params. Image inputs and prompt/ratio are placed explicitly; everything + * else is spread verbatim so newly-allowed keys flow through without + * needing a code change here. + */ + #buildRequest( + model: ReplicateImageModel, + ctx: { + prompt: string; + ratio: { w: number; h: number }; + transformed: Record; + inputImages: string[]; + singleImage?: string; + }, + ): Record { + const { prompt, ratio, transformed, inputImages, singleImage } = ctx; + + const input: Record = { + prompt, + aspect_ratio: this.#toAspectRatio(ratio), + }; + + const handled = new Set( + ReplicateImageGenerationProvider.#CORE_PARAMS, + ); + if (model.imageInputKey) handled.add(model.imageInputKey); + if (model.singleImageInputKey) handled.add(model.singleImageInputKey); + + if (inputImages.length && model.imageInputKey) { + input[model.imageInputKey] = inputImages; + } else if (singleImage && model.singleImageInputKey) { + input[model.singleImageInputKey] = singleImage; + } + + for (const [key, value] of Object.entries(transformed)) { + if (handled.has(key)) continue; + if (value === undefined || value === null) continue; + input[key] = value; + } + + return input; + } + + /** + * Drops params not in `model.allowed_params` (plus `#CORE_PARAMS` and any + * alias targets, so the native key is also accepted). + */ + #filterAllowedParams( + params: IGenerateParams, + model: ReplicateImageModel, + ): IGenerateParams { + const allowedSet = model.allowed_params; + if (!allowedSet) return params; + + const aliasTargets = model.param_aliases + ? Object.values(model.param_aliases) + : []; + const nativeImageKeys: string[] = []; + if (model.imageInputKey) nativeImageKeys.push(model.imageInputKey); + if (model.singleImageInputKey) + nativeImageKeys.push(model.singleImageInputKey); + + const filtered: Record = {}; + for (const key of Object.keys(params)) { + if ( + ReplicateImageGenerationProvider.#CORE_PARAMS.includes(key) || + allowedSet.includes(key) || + aliasTargets.includes(key) || + nativeImageKeys.includes(key) + ) { + filtered[key] = params[key]; + } + } + return filtered as IGenerateParams; + } + + /** + * Renames canonical keys to the model's native API names per + * `model.param_aliases` (e.g. `steps` → `num_inference_steps`). + */ + #applyParamAliases( + params: IGenerateParams, + model: ReplicateImageModel, + ): Record { + const aliases = model.param_aliases; + if (!aliases) return params as Record; + + const result: Record = {}; + for (const [key, value] of Object.entries(params)) { + const nativeKey = aliases[key] ?? key; + result[nativeKey] = value; + } + return result; + } + + /** + * Applies `param_transforms` on top of the aliased map: injects defaults + * for missing keys, then appends any configured string suffix to the + * value. Returns the original map unchanged when the model declares no + * transforms. + */ + #applyTransforms( + aliased: Record, + model: ReplicateImageModel, + ): Record { + const transforms = model.param_transforms; + if (!transforms) return aliased; + + const result = { ...aliased }; + for (const [key, cfg] of Object.entries(transforms)) { + let value = result[key]; + if (value === undefined && cfg.default !== undefined) { + value = cfg.default; + } + if (value === undefined) continue; + if (cfg.suffix !== undefined && typeof value === 'string') { + value = value + cfg.suffix; + } + result[key] = value; + } + return result; + } + #normalizeRatio(ratio?: { w: number; h: number }) { const w = Number(ratio?.w); const h = Number(ratio?.h); @@ -253,10 +380,16 @@ export class ReplicateImageGenerationProvider implements IImageProvider { #resolveCosts( model: ReplicateImageModel, goFast: boolean, + generationMode?: string, ): Record { - return goFast && model.costs_go_fast - ? model.costs_go_fast - : model.costs; + if (goFast && model.costs_go_fast) return model.costs_go_fast; + if ( + generationMode && + model.costs_by_generation_mode?.[generationMode] + ) { + return model.costs_by_generation_mode[generationMode]; + } + return model.costs; } #estimateCost( @@ -264,8 +397,9 @@ export class ReplicateImageGenerationProvider implements IImageProvider { outputMp: number, goFast: boolean, inputMp: number, + generationMode?: string, ): number { - const costs = this.#resolveCosts(model, goFast); + const costs = this.#resolveCosts(model, goFast, generationMode); if (model.billingScheme === 'per-image') { const cents = costs.output; @@ -300,9 +434,10 @@ export class ReplicateImageGenerationProvider implements IImageProvider { outputMp: number, goFast: boolean, inputMp: number, + generationMode?: string, ) { const prefix = `replicate:${model.id}`; - const costs = this.#resolveCosts(model, goFast); + const costs = this.#resolveCosts(model, goFast, generationMode); if (model.billingScheme === 'per-image') { const cents = costs.output; diff --git a/src/backend/drivers/ai-image/providers/replicate/models.ts b/src/backend/drivers/ai-image/providers/replicate/models.ts index 5a63e559a..b9976afc1 100644 --- a/src/backend/drivers/ai-image/providers/replicate/models.ts +++ b/src/backend/drivers/ai-image/providers/replicate/models.ts @@ -24,15 +24,23 @@ export type ReplicateBillingScheme = 'per-image' | 'megapixel'; export type ReplicateImageModel = IImageModel & { replicateId: string; billingScheme: ReplicateBillingScheme; - imageInputKey?: string; - singleImageInputKey?: string; - supportsGoFast?: boolean; - goFastDefault?: boolean; + imageInputKey?: string; // our `input_images` + singleImageInputKey?: string; // our `input_image` + /** Cost map used when `go_fast: true` (e.g. flux-2-dev fast mode). */ costs_go_fast?: Record; - resolutionInputKey?: string; - resolutionSuffix?: string; + /** Cost maps keyed by Leonardo `generation_mode`; falls back to `costs`. */ + costs_by_generation_mode?: Record>; + /** Whitelist of caller-supplied params (canonical names) that are forwarded to Replicate. */ + allowed_params?: string[]; + /** Renames canonical param keys to the model's native API names (e.g. `steps` → `num_inference_steps`). */ + param_aliases?: Record; + /** Per-key value transforms (default + suffix) applied after `param_aliases`. */ + param_transforms?: Record; }; +const ALIAS_FORMAT = { response_format: 'output_format' }; +const ALIAS_FORMAT_STEPS = { ...ALIAS_FORMAT, steps: 'num_inference_steps' }; + // Costs are in USD cents. // Megapixel models: `output_mp` = cost per output megapixel, `input_mp` = // cost per input megapixel (img2img). Some also carry a flat `run` cost. @@ -50,8 +58,15 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ costs: { run: 1.5, input_mp: 1.5, output_mp: 1.5 }, billingScheme: 'megapixel', imageInputKey: 'input_images', - resolutionInputKey: 'resolution', - resolutionSuffix: ' MP', + allowed_params: [ + 'seed', + 'response_format', + 'output_quality', + 'output_megapixels', + 'safety_tolerance', + ], + param_aliases: { ...ALIAS_FORMAT, output_megapixels: 'resolution' }, + param_transforms: { resolution: { suffix: ' MP' } }, }, { id: 'black-forest-labs/flux-2-dev', @@ -65,9 +80,15 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ costs_go_fast: { input_mp: 1.2, output_mp: 1.2 }, billingScheme: 'megapixel', imageInputKey: 'input_images', - supportsGoFast: true, - goFastDefault: true, - resolutionInputKey: 'output_megapixels', + allowed_params: [ + 'seed', + 'response_format', + 'output_quality', + 'disable_safety_checker', + 'go_fast', + ], + param_aliases: ALIAS_FORMAT, + param_transforms: { go_fast: { default: true } }, }, { id: 'black-forest-labs/flux-2-klein-9b-base', @@ -80,7 +101,15 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ costs: { input_mp: 1.1, output_mp: 1.1 }, billingScheme: 'megapixel', imageInputKey: 'images', - resolutionInputKey: 'output_megapixels', + allowed_params: [ + 'seed', + 'guidance', + 'response_format', + 'output_quality', + 'disable_safety_checker', + 'output_megapixels', + ], + param_aliases: ALIAS_FORMAT, }, { id: 'black-forest-labs/flux-2-klein-4b', @@ -93,7 +122,14 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ costs: { input_mp: 0.1, output_mp: 0.1 }, billingScheme: 'megapixel', imageInputKey: 'images', - resolutionInputKey: 'output_megapixels', + allowed_params: [ + 'seed', + 'response_format', + 'output_quality', + 'disable_safety_checker', + 'output_megapixels', + ], + param_aliases: ALIAS_FORMAT, }, // Black Forest Labs FLUX.1 @@ -107,6 +143,18 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ index_cost_key: 'output', costs: { output: 0.3 }, billingScheme: 'per-image', + allowed_params: [ + 'seed', + 'steps', + 'response_format', + 'output_quality', + 'disable_safety_checker', + 'output_megapixels', + ], + param_aliases: { + ...ALIAS_FORMAT_STEPS, + output_megapixels: 'megapixels', + }, }, { id: 'black-forest-labs/flux-1.1-pro', @@ -119,6 +167,14 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ costs: { output: 4 }, billingScheme: 'per-image', singleImageInputKey: 'image_prompt', + allowed_params: [ + 'seed', + 'response_format', + 'output_quality', + 'safety_tolerance', + 'prompt_upsampling', + ], + param_aliases: ALIAS_FORMAT, }, // Leonardo AI @@ -130,8 +186,20 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'Lucid Origin', costs_currency: 'usd-cents', index_cost_key: 'output', - costs: { output: 1.65 }, + costs: { + output: 1.65, // standard: 11 units * $0.0015/unit = $0.0165 + }, + costs_by_generation_mode: { + standard: { output: 1.65 }, + ultra: { output: 7.65 }, // 51 units * $0.0015/unit = $0.0765 + }, billingScheme: 'per-image', + allowed_params: [ + 'style', + 'contrast', + 'prompt_enhance', + 'generation_mode', + ], }, { id: 'leonardoai/phoenix-1.0', @@ -141,7 +209,20 @@ export const REPLICATE_IMAGE_GENERATION_MODELS: ReplicateImageModel[] = [ name: 'Phoenix 1.0', costs_currency: 'usd-cents', index_cost_key: 'output', - costs: { output: 3.75 }, + costs: { + output: 3.75, // quality default: 25 units * $0.0015/unit = $0.0375 + }, + costs_by_generation_mode: { + fast: { output: 1.8 }, // 12 units * $0.0015/unit = $0.018 + quality: { output: 3.75 }, // 25 units * $0.0015/unit = $0.0375 + ultra: { output: 7.5 }, // 50 units * $0.0015/unit = $0.075 + }, billingScheme: 'per-image', + allowed_params: [ + 'style', + 'contrast', + 'prompt_enhance', + 'generation_mode', + ], }, ]; diff --git a/src/backend/drivers/ai-image/types.ts b/src/backend/drivers/ai-image/types.ts index f08403968..e7fc0cd26 100644 --- a/src/backend/drivers/ai-image/types.ts +++ b/src/backend/drivers/ai-image/types.ts @@ -63,6 +63,7 @@ export interface IGenerateParams { input_image?: string; input_image_mime_type?: string; input_images?: string[]; + [key: string]: unknown; } export interface IImageProvider { diff --git a/src/docs/src/AI/txt2img.md b/src/docs/src/AI/txt2img.md index adb7b4e10..704141a8e 100755 --- a/src/docs/src/AI/txt2img.md +++ b/src/docs/src/AI/txt2img.md @@ -95,24 +95,37 @@ For more details, see the [Together AI API reference](https://docs.together.ai/r Available when `provider: 'replicate-image-generation'` or inferred from model: +##### Common options + | Option | Type | Description | |--------|------|-------------| -| `model` | `String` | Image model to use. | -| `ratio` | `Object` | Aspect ratio as `{ w, h }` (e.g., `{ w: 16, h: 9 }`) | -| `seed` | `Number` | Random seed for reproducible generation | -| `steps` | `Number` | Number of inference steps | -| `guidance` | `Number` | Guidance scale for generation | -| `go_fast` | `Boolean` | Use optimized fast mode. Defaults to `true` for `flux-2-dev`. Affects pricing on supported models | -| `output_quality` | `Number` | Output quality (0-100). | -| `output_megapixels` | `String` | Approximate output megapixels (`'0.25'`, `'0.5'`, `'1'`, `'2'`, `'4'`) | -| `input_image` | `String` | URL of an input image for image-to-image generation | -| `input_images` | `Array` | Array of input image URLs for multi-image generation | -| `negative_prompt` | `String` | Text to guide what to avoid in the image | -| `prompt_strength` | `Number` | How strongly the prompt influences the output | -| `disable_safety_checker` | `Boolean` | If `true`, disables the safety checker | -| `response_format` | `String` | Output format: `'webp'`, `'jpg'`, `'png'` | +| `model` | `String` | Model id (e.g. `'black-forest-labs/flux-schnell'`, `'leonardoai/lucid-origin'`). | +| `ratio` | `Object` | Aspect ratio as `{ w, h }` (e.g., `{ w: 16, h: 9 }`). | +| `input_image` | `String` | URL of an input image for image-to-image generation. | +| `input_images` | `Array` | Array of input image URLs for multi-image generation. | -For more details, see the [Replicate API reference](https://replicate.com/docs). +##### Per-model options + +These keys are only forwarded for models that whitelist them (see per-model `allowed_params`): + +| Option | Type | Models | Description | +|--------|------|--------|-------------| +| `seed` | `Number` | most models | Random seed for reproducible generation. | +| `steps` | `Number` | `flux-schnell` | Number of inference steps. | +| `guidance` | `Number` | `flux-2-klein-9b-base` | Guidance scale. | +| `go_fast` | `Boolean` | `flux-2-dev` | Use optimized fast mode. Defaults to `true` for `flux-2-dev`; affects pricing. | +| `output_quality` | `Number` | flux family | Output quality (0–100). | +| `output_megapixels` | `String` | flux family | Approximate output megapixels (e.g. `'0.25'`, `'0.5'`, `'1'`, `'2'`). | +| `disable_safety_checker` | `Boolean` | flux-2-dev / klein / flux-schnell | If `true`, disables the safety checker. | +| `safety_tolerance` | `Number` | `flux-2-pro`, `flux-1.1-pro` | Safety tolerance level. | +| `prompt_upsampling` | `Boolean` | `flux-1.1-pro` | Enable prompt upsampling. | +| `response_format` | `String` | most models | Output format (e.g. `'webp'`, `'jpg'`, `'png'`). | +| `generation_mode` | `String` | Leonardo (`lucid-origin`, `phoenix-1.0`) | Generation tier — affects pricing. e.g. `'standard'`/`'ultra'` (lucid-origin), `'fast'`/`'quality'`/`'ultra'` (phoenix-1.0). | +| `style` | `String` | Leonardo | Stylistic preset. | +| `contrast` | `String` | Leonardo | Contrast preset. | +| `prompt_enhance` | `Boolean` | Leonardo | Server-side prompt enhancement. | + +For more details, see the [Replicate API reference](https://replicate.com/docs) and each model's schema page on Replicate. Any properties not set fall back to provider defaults.