From 83eab0d6acc524bdbdfeed803cffa24f9ffcb3c0 Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Thu, 11 Dec 2025 17:03:20 -0800 Subject: [PATCH] Revert "Revert: single ai image entry point (#2131)" (#2143) This reverts commit 907d0db32897323a07721c87c6f1b27e60b44b22. --- package-lock.json | 6 +- src/backend/package.json | 2 +- .../src/modules/ai/PuterAIChatModule.js | 32 +- src/backend/src/services/ChatAPIService.js | 48 ++- .../src/services/ChatAPIService.test.js | 12 +- src/backend/src/services/PuterSiteService.js | 1 - .../TogetherAiProvider/TogetherAIProvider.ts | 4 +- src/backend/src/services/ai/image/.gitignore | 2 + .../ai/image/AIImageGenerationService.ts | 263 +++++++++++++ .../ai/image/GeminiImageGenerationService.js | 257 ------------- .../ai/image/OpenAIImageGenerationService.js | 359 ------------------ .../image/TogetherImageGenerationService.js | 287 -------------- .../GeminiImageGenerationProvider.ts | 171 +++++++++ .../GeminiImageGenerationProvider/models.ts | 111 ++++++ .../OpenAiImageGenerationProvider.ts | 176 +++++++++ .../OpenAiImageGenerationProvider/models.ts | 71 ++++ .../TogetherImageGenerationProvider.ts | 242 ++++++++++++ .../TogetherImageGenerationProvider/models.ts | 319 ++++++++++++++++ .../src/services/ai/image/providers/types.ts | 29 ++ .../src/services/ai/ocr/MistralOCRService.js | 6 - 20 files changed, 1455 insertions(+), 943 deletions(-) create mode 100644 src/backend/src/services/ai/image/.gitignore create mode 100644 src/backend/src/services/ai/image/AIImageGenerationService.ts delete mode 100644 src/backend/src/services/ai/image/GeminiImageGenerationService.js delete mode 100644 src/backend/src/services/ai/image/OpenAIImageGenerationService.js delete mode 100644 src/backend/src/services/ai/image/TogetherImageGenerationService.js create mode 100644 src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.ts create mode 100644 src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/models.ts create mode 100644 src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.ts create mode 100644 src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/models.ts create mode 100644 src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.ts create mode 100644 src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/models.ts create mode 100644 src/backend/src/services/ai/image/providers/types.ts diff --git a/package-lock.json b/package-lock.json index d4b8917b8..3f3be4208 100644 --- a/package-lock.json +++ b/package-lock.json @@ -14597,7 +14597,9 @@ } }, "node_modules/together-ai": { - "version": "0.32.0", + "version": "0.33.0", + "resolved": "https://registry.npmjs.org/together-ai/-/together-ai-0.33.0.tgz", + "integrity": "sha512-2JdxYwbw+Xw2bW2PHBGqbMTtYsQHoWO9UXvdwIfQkde/swoKp2x/hpxEjtTERzrMP4O5SdDPGxsjfcPXewDJ9A==", "license": "Apache-2.0", "bin": { "together-ai": "bin/cli" @@ -15830,7 +15832,7 @@ "svg-captcha": "^1.4.0", "svgo": "^3.0.2", "tiktoken": "^1.0.16", - "together-ai": "^0.32.0", + "together-ai": "^0.33.0", "tweetnacl": "^1.0.3", "ua-parser-js": "^1.0.38", "uglify-js": "^3.17.4", diff --git a/src/backend/package.json b/src/backend/package.json index 7b49e15be..352a1c1e0 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -82,7 +82,7 @@ "svg-captcha": "^1.4.0", "svgo": "^3.0.2", "tiktoken": "^1.0.16", - "together-ai": "^0.32.0", + "together-ai": "^0.33.0", "tweetnacl": "^1.0.3", "ua-parser-js": "^1.0.38", "uglify-js": "^3.17.4", diff --git a/src/backend/src/modules/ai/PuterAIChatModule.js b/src/backend/src/modules/ai/PuterAIChatModule.js index 77ca70db4..9bdbb4bd9 100644 --- a/src/backend/src/modules/ai/PuterAIChatModule.js +++ b/src/backend/src/modules/ai/PuterAIChatModule.js @@ -22,17 +22,14 @@ import { AdvancedBase } from '@heyputer/putility'; import config from '../../config.js'; import { AIInterfaceService } from '../../services/ai/AIInterfaceService.js'; import { AIChatService } from '../../services/ai/chat/AIChatService.js'; -import { GeminiImageGenerationService } from '../../services/ai/image/GeminiImageGenerationService.js'; -import { OpenAIImageGenerationService } from '../../services/ai/image/OpenAIImageGenerationService.js'; -import { TogetherImageGenerationService } from '../../services/ai/image/TogetherImageGenerationService.js'; +import { AIImageGenerationService } from '../../services/ai/image/AIImageGenerationService.js'; import { AWSTextractService } from '../../services/ai/ocr/AWSTextractService.js'; import { ElevenLabsVoiceChangerService } from '../../services/ai/sts/ElevenLabsVoiceChangerService.js'; import { OpenAISpeechToTextService } from '../../services/ai/stt/OpenAISpeechToTextService.js'; import { AWSPollyService } from '../../services/ai/tts/AWSPollyService.js'; import { ElevenLabsTTSService } from '../../services/ai/tts/ElevenLabsTTSService.js'; import { OpenAITTSService } from '../../services/ai/tts/OpenAITTSService.js'; -import { OpenAIVideoGenerationService } from '../../services/ai/video/OpenAIVideoGenerationService.js'; -import { TogetherVideoGenerationService } from '../../services/ai/video/TogetherVideoGenerationService.js'; +// import { AIVideoGenerationService } from '../../services/ai/video/AIVideoGenerationService.js'; /** * PuterAIModule class extends AdvancedBase to manage and register various AI services. @@ -57,8 +54,13 @@ export class PuterAIModule extends AdvancedBase { // completion ai service services.registerService('ai-chat', AIChatService); - // TODO DS: centralize other service types too + // image generation ai service + services.registerService('ai-image', AIImageGenerationService); + // video generation ai service + // services.registerService('ai-video', AIVideoGenerationService); + + // TODO DS: centralize other service types too // TODO: services should govern their own availability instead of the module deciding what to register if ( config?.services?.['aws-textract']?.aws ) { @@ -78,25 +80,9 @@ export class PuterAIModule extends AdvancedBase { if ( config?.services?.openai || config?.openai ) { - services.registerService('openai-image-generation', OpenAIImageGenerationService); - - services.registerService('openai-video-generation', OpenAIVideoGenerationService); - services.registerService('openai-tts', OpenAITTSService); services.registerService('openai-speech2txt', OpenAISpeechToTextService); } - - if ( config?.services?.['together-ai'] ) { - - services.registerService('together-image-generation', TogetherImageGenerationService); - - services.registerService('together-video-generation', TogetherVideoGenerationService); - } - - if ( config?.services?.['gemini'] ) { - - services.registerService('gemini-image-generation', GeminiImageGenerationService); - } } -} \ No newline at end of file +} diff --git a/src/backend/src/services/ChatAPIService.js b/src/backend/src/services/ChatAPIService.js index 71564037c..1cadc48a9 100644 --- a/src/backend/src/services/ChatAPIService.js +++ b/src/backend/src/services/ChatAPIService.js @@ -50,7 +50,7 @@ class ChatAPIService extends BaseService { })(); // Register the router with the Express app - app.use('/puterai/chat', router); + app.use('/puterai', router); // Install endpoints this.install_chat_endpoints_({ router }); @@ -66,7 +66,7 @@ class ChatAPIService extends BaseService { const Endpoint = this.require('Endpoint'); // Endpoint to list available AI chat models Endpoint({ - route: '/models', + route: '/chat/models', methods: ['GET'], handler: async (req, res) => { try { @@ -89,7 +89,7 @@ class ChatAPIService extends BaseService { // Endpoint to get detailed information about available AI chat models Endpoint({ - route: '/models/details', + route: '/chat/models/details', methods: ['GET'], handler: async (req, res) => { try { @@ -109,6 +109,48 @@ class ChatAPIService extends BaseService { } }, }).attach(router); + + Endpoint({ + route: '/image/models', + methods: ['GET'], + handler: async (req, res) => { + try { + // Use SUService to access AIImageGenerationService as system user + const svc_su = this.services.get('su'); + const models = await svc_su.sudo(async () => { + const svc_imageGen = this.services.get('ai-image'); + // Return the simple model list which contains basic model information + return svc_imageGen.list(); + }); + // Return the list of models + res.json({ models }); + } catch ( error ) { + this.log.error('Error fetching image models:', error); + throw APIError.create('internal_server_error'); + } + }, + }).attach(router); + + Endpoint({ + route: '/image/models/details', + methods: ['GET'], + handler: async (req, res) => { + try { + // Use SUService to access AIImageGenerationService as system user + const svc_su = this.services.get('su'); + const models = await svc_su.sudo(async () => { + const svc_imageGen = this.services.get('ai-image'); + // Return the detailed model list which includes cost and capability information + return svc_imageGen.models(); + }); + // Return the detailed list of models + res.json({ models }); + } catch ( error ) { + this.log.error('Error fetching image model details:', error); + throw APIError.create('internal_server_error'); + } + }, + }).attach(router); } } diff --git a/src/backend/src/services/ChatAPIService.test.js b/src/backend/src/services/ChatAPIService.test.js index 520de1955..222d81a4c 100644 --- a/src/backend/src/services/ChatAPIService.test.js +++ b/src/backend/src/services/ChatAPIService.test.js @@ -124,7 +124,11 @@ describe('ChatAPIService', () => { // Verify expect(mockEndpoint).toHaveBeenCalledWith(expect.objectContaining({ - route: '/models', + route: '/chat/models', + methods: ['GET'], + })); + expect(mockEndpoint).toHaveBeenCalledWith(expect.objectContaining({ + route: '/image/models', methods: ['GET'], })); }); @@ -138,7 +142,11 @@ describe('ChatAPIService', () => { // Verify expect(mockEndpoint).toHaveBeenCalledWith(expect.objectContaining({ - route: '/models/details', + route: '/chat/models/details', + methods: ['GET'], + })); + expect(mockEndpoint).toHaveBeenCalledWith(expect.objectContaining({ + route: '/image/models/details', methods: ['GET'], })); }); diff --git a/src/backend/src/services/PuterSiteService.js b/src/backend/src/services/PuterSiteService.js index 9d194933a..68e27ef6f 100644 --- a/src/backend/src/services/PuterSiteService.js +++ b/src/backend/src/services/PuterSiteService.js @@ -112,7 +112,6 @@ class PuterSiteService extends BaseService { root_dir_id: this.config.devtest_directory, }; } - console.log('???', subdomain, options); const rows = await this.db.read(`SELECT * FROM subdomains WHERE ${ options.is_custom_domain ? 'domain' : 'subdomain' } = ? LIMIT 1`, diff --git a/src/backend/src/services/ai/chat/providers/TogetherAiProvider/TogetherAIProvider.ts b/src/backend/src/services/ai/chat/providers/TogetherAiProvider/TogetherAIProvider.ts index f6fc0eaea..8d0165f67 100644 --- a/src/backend/src/services/ai/chat/providers/TogetherAiProvider/TogetherAIProvider.ts +++ b/src/backend/src/services/ai/chat/providers/TogetherAiProvider/TogetherAIProvider.ts @@ -46,9 +46,9 @@ export class TogetherAIProvider implements IChatProvider { let models: IChatModel[] | undefined = kv.get(this.#kvKey); if ( models ) return models; - const api_models = await this.#together.models.list(); + const apiModels = await this.#together.models.list(); models = []; - for ( const model of api_models ) { + for ( const model of apiModels ) { if ( model.type === 'chat' || model.type === 'code' || model.type === 'language' || model.type === 'moderation' ) { models.push({ id: `togetherai:${model.id}`, diff --git a/src/backend/src/services/ai/image/.gitignore b/src/backend/src/services/ai/image/.gitignore new file mode 100644 index 000000000..aa4a6da26 --- /dev/null +++ b/src/backend/src/services/ai/image/.gitignore @@ -0,0 +1,2 @@ +*.js +*.js.map \ No newline at end of file diff --git a/src/backend/src/services/ai/image/AIImageGenerationService.ts b/src/backend/src/services/ai/image/AIImageGenerationService.ts new file mode 100644 index 000000000..11247615a --- /dev/null +++ b/src/backend/src/services/ai/image/AIImageGenerationService.ts @@ -0,0 +1,263 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// METADATA // {"ai-commented":{"service":"claude"}} +import { APIError } from '../../../api/APIError.js'; +import { ErrorService } from '../../../modules/core/ErrorService.js'; +import { Context } from '../../../util/context.js'; +import BaseService from '../../BaseService.js'; +import { BaseDatabaseAccessService } from '../../database/BaseDatabaseAccessService.js'; +import { DB_WRITE } from '../../database/consts.js'; +import { DriverService } from '../../drivers/DriverService.js'; +import { TypedValue } from '../../drivers/meta/Runtime.js'; +import { EventService } from '../../EventService.js'; +import { MeteringService } from '../../MeteringService/MeteringService.js'; +import { GeminiImageGenerationProvider } from './providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.js'; +import { OpenAiImageGenerationProvider } from './providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.js'; +import { TogetherImageGenerationProvider } from './providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.js'; +import { IGenerateParams, IImageModel, IImageProvider } from './providers/types.js'; + +export class AIImageGenerationService extends BaseService { + + static SERVICE_NAME = 'ai-image'; + + static DEFAULT_PROVIDER = 'openai-image-generation'; + + get meteringService (): MeteringService { + return this.services.get('meteringService').meteringService; + } + + get db (): BaseDatabaseAccessService { + return this.services.get('database').get(DB_WRITE, 'ai-service'); + } + + get errorService (): ErrorService { + return this.services.get('error-service'); + } + + get eventService (): EventService { + return this.services.get('event'); + } + + get driverService (): DriverService { + return this.services.get('driver'); + } + + getProvider (name: string): IImageProvider | undefined { + return this.#providers[name]; + } + + #providers: Record = {}; + #modelIdMap: Record = {}; + + /** Driver interfaces */ + static IMPLEMENTS = { + ['driver-capabilities']: { + supports_test_mode (iface: string, method_name: string) { + return iface === 'puter-image-generation' && + method_name === 'generate'; + }, + }, + ['puter-image-generation']: { + + async generate (...parameters: Parameters) { + return (this as unknown as AIImageGenerationService).generate(...parameters); + }, + }, + }; + + getModel ({ modelId, provider}: { modelId: string, provider?: string }) { + const models = this.#modelIdMap[modelId]; + + if ( ! provider ) { + return models[0]; + } + const model = models.find(m => m.provider === provider); + return model ?? models[0]; + } + + private async registerProviders () { + + const openAiConfig = this.config.providers?.['openai-image-generation'] || this.global_config?.services?.['openai'] || this.global_config?.openai; + if ( openAiConfig && (openAiConfig.apiKey || openAiConfig.secret_key) ) { + this.#providers['openai-image-generation'] = new OpenAiImageGenerationProvider({ apiKey: openAiConfig.apiKey || openAiConfig.secret_key }, this.meteringService, this.errorService); + } + + const geminiConfig = this.config.providers?.['gemini-image-generation'] || this.global_config?.services?.gemini; + if ( geminiConfig && (geminiConfig.apiKey || geminiConfig.secret_key) ) { + this.#providers['gemini-image-generation'] = new GeminiImageGenerationProvider({ apiKey: geminiConfig.apiKey || geminiConfig.secret_key }, this.meteringService, this.errorService); + } + + const togetherConfig = this.config.providers?.['together-image-generation'] || this.global_config?.services?.['together-ai']; + if ( togetherConfig && (togetherConfig.apiKey || togetherConfig.secret_key) ) { + this.#providers['together-image-generation'] = new TogetherImageGenerationProvider({ apiKey: togetherConfig.apiKey || togetherConfig.secret_key }, this.meteringService, this.errorService, this.eventService); + } + + // emit event for extensions to add providers + const extensionProviders = {} as Record; + await this.eventService.emit('ai.image.registerProviders', extensionProviders); + for ( const providerName in extensionProviders ) { + if ( this.#providers[providerName] ) { + console.warn('AIChatService: provider name conflict for ', providerName, ' registering with -extension suffix'); + this.#providers[`${providerName}-extension`] = extensionProviders[providerName]; + continue; + } + this.#providers[providerName] = extensionProviders[providerName]; + } + } + + protected async '__on_boot.consolidation' () { + // register chat providers here + await this.registerProviders(); + + // build model id map + for ( const providerName in this.#providers ) { + const provider = this.#providers[providerName]; + + // alias all driver requests to go here to support legacy routing + this.driverService.register_service_alias(AIImageGenerationService.SERVICE_NAME, + providerName, + { iface: 'puter-image-generation' }); + + // build model id map + for ( const model of await provider.models() ) { + if ( ! this.#modelIdMap[model.id] ) { + this.#modelIdMap[model.id] = []; + } + this.#modelIdMap[model.id].push({ ...model, provider: providerName }); + if ( model.aliases ) { + for ( const alias of model.aliases ) { + // join arrays which are aliased the same + if ( ! this.#modelIdMap[alias] ) { + this.#modelIdMap[alias] = this.#modelIdMap[model.id]; + continue; + } + if ( this.#modelIdMap[alias] !== this.#modelIdMap[model.id] ) { + this.#modelIdMap[alias].push({ ...model, provider: providerName }); + this.#modelIdMap[model.id] = this.#modelIdMap[alias]; + continue; + } + } + } + this.#modelIdMap[model.id].sort((a, b) => a.costs[a.index_cost_key || Object.keys(a.costs)[0]] - b.costs[b.index_cost_key || Object.keys(b.costs)[0]]); + } + } + } + + models () { + return Object.entries(this.#modelIdMap) + .map(([_, models]) => models) + .flat() + .sort((a, b) => { + if ( a.provider === b.provider ) { + return a.id.localeCompare(b.id); + } + return a.provider!.localeCompare(b.provider!); + }); + } + + list () { + return Object.keys(this.#modelIdMap).sort(); + } + + async generate (parameters: IGenerateParams) { + const clientDriverCall = Context.get('client_driver_call'); + let { test_mode: testMode, intended_service: legacyProviderName } = clientDriverCall as { test_mode?: boolean; response_metadata: Record; intended_service?: string }; + + const configuredProviders = Object.keys(this.#providers); + if ( configuredProviders.length === 0 ) { + throw new Error('no image generation providers configured'); + } + + let intendedProvider = (parameters.provider || (legacyProviderName === AIImageGenerationService.SERVICE_NAME ? '' : legacyProviderName)) ?? ''; + + if ( !parameters.model && !intendedProvider ) { + intendedProvider = configuredProviders.includes(AIImageGenerationService.DEFAULT_PROVIDER) + ? AIImageGenerationService.DEFAULT_PROVIDER + : configuredProviders[0]; + } + + if ( intendedProvider && !this.#providers[intendedProvider] ) { + intendedProvider = configuredProviders[0]; + } + + if ( !parameters.model && intendedProvider ) { + parameters.model = this.#providers[intendedProvider].getDefaultModel(); + } + + const model = parameters.model ? this.getModel({ modelId: parameters.model, provider: intendedProvider }) : undefined; + + if ( ! model ) { + const availableModelsUrl = `${this.global_config.origin }/puterai/image/models`; + + throw APIError.create('field_invalid', undefined, { + key: 'model', + expected: `a valid model name from ${availableModelsUrl}`, + got: model, + }); + } + + // call model provider; + const provider = this.#providers[model.provider!]; + if ( ! provider ) { + throw new Error(`no provider found for model ${model.id}`); + } + + if ( model.allowedRatios?.length ) { + if ( parameters.ratio ) { + const isValidRatio = model.allowedRatios.some(r => r.w === parameters.ratio!.w && r.h === parameters.ratio!.h); + if ( ! isValidRatio ) { + parameters.ratio = model.allowedRatios[0]; + } + } else { + parameters.ratio = model.allowedRatios[0]; + } + } + + if ( ! parameters.ratio ) { + parameters.ratio = { w: 1024, h: 1024 }; + } + + if ( model.allowedQualityLevels?.length ) { + if ( parameters.quality ) { + if ( ! model.allowedQualityLevels.includes(parameters.quality) ) { + parameters.quality = model.allowedQualityLevels[0]; + } + } else { + parameters.quality = model.allowedQualityLevels[0]; + } + } + + const url = await provider.generate({ + ...parameters, + model: model.id, + provider: model.provider, + test_mode: testMode, + }); + + const isDataUrl = url.startsWith('data:'); + const image = new TypedValue({ + $: isDataUrl ? 'string:url:data' : 'string:url:web', + content_type: 'image', + }, url); + + return image; + + } +} diff --git a/src/backend/src/services/ai/image/GeminiImageGenerationService.js b/src/backend/src/services/ai/image/GeminiImageGenerationService.js deleted file mode 100644 index 05fafb30a..000000000 --- a/src/backend/src/services/ai/image/GeminiImageGenerationService.js +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// METADATA // {"ai-commented":{"service":"claude"}} -const APIError = require('../../../api/APIError'); -const BaseService = require('../../BaseService'); -const { TypedValue } = require('../../drivers/meta/Runtime'); -const { Context } = require('../../../util/context'); -const { GoogleGenAI } = require('@google/genai'); - -/** -* Service class for generating images using Gemini's API -* Extends BaseService to provide image generation capabilities through -* the puter-image-generation interface. -*/ -class GeminiImageGenerationService extends BaseService { - /** @type {import('../../MeteringService/MeteringService').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - static MODULES = { - }; - - _construct () { - this.models_ = { - 'gemini-2.5-flash-image-preview': { - '1024x1024': 0.039, - }, - 'gemini-3-pro-image-preview': { - '1024x1024': 0.156, - }, - }; - } - - /** - * Initializes the Gemini client with API credentials from config - * @private - * @async - * @returns {Promise} - */ - async _init () { - this.genAI = new GoogleGenAI({ apiKey: this.global_config.services.gemini.apiKey }); - } - - static IMPLEMENTS = { - ['driver-capabilities']: { - supports_test_mode (iface, method_name) { - return iface === 'puter-image-generation' && - method_name === 'generate'; - }, - }, - ['puter-image-generation']: { - /** - * Generates an image using Gemini's gemini-2.5-flash-image-preview - * @param {string} prompt - The text description of the image to generate - * @param {Object} options - Generation options - * @param {Object} options.ratio - Image dimensions ratio object with w/h properties - * @param {string} [options.model='gemini-2.5-flash-image-preview'] - The model to use for generation - * @param {string} [options.input_image] - Base64 encoded input image for image-to-image generation - * @param {string} [options.input_image_mime_type] - MIME type of the input image - * @returns {Promise} URL of the generated image - * @throws {Error} If prompt is not a string or ratio is invalid - */ - async generate (params) { - const { prompt, quality, test_mode, model, ratio, input_image, input_image_mime_type } = params; - - if ( test_mode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'image', - }, 'https://puter-sample-data.puter.site/image_example.png'); - } - - const url = await this.generate(prompt, { - quality, - ratio: ratio || this.constructor.RATIO_SQUARE, - model, - input_image, - input_image_mime_type, - }); - - // Determine if this is a data URL or web URL - const isDataUrl = url.startsWith('data:'); - const image = new TypedValue({ - $: isDataUrl ? 'string:url:data' : 'string:url:web', - content_type: 'image', - }, url); - - return image; - }, - }, - }; - - static RATIO_SQUARE = { w: 1024, h: 1024 }; - - async generate (prompt, { - ratio, - model, - input_image, - input_image_mime_type, - }) { - if ( typeof prompt !== 'string' ) { - throw new Error('`prompt` must be a string'); - } - - if ( !ratio || !this._validate_ratio(ratio, model) ) { - throw new Error(`\`ratio\` must be a valid ratio for model ${ model}`); - } - - // Validate input image if provided - if ( input_image && !input_image_mime_type ) { - throw new Error('`input_image_mime_type` is required when `input_image` is provided'); - } - - if ( input_image_mime_type && !input_image ) { - throw new Error('`input_image` is required when `input_image_mime_type` is provided'); - } - - if ( input_image_mime_type && !this._validate_image_mime_type(input_image_mime_type) ) { - throw new Error('`input_image_mime_type` must be a valid image MIME type (image/png, image/jpeg, image/webp)'); - } - - // Somewhat sane defaults - model = model ?? 'gemini-2.5-flash-image-preview'; - - if ( ! this.models_[model] ) { - throw APIError.create('field_invalid', null, { - key: 'model', - expected: `one of: ${ - Object.keys(this.models_).join(', ')}`, - got: model, - }); - } - - const price_key = `${ratio.w}x${ratio.h}`; - if ( ! this.models_[model][price_key] ) { - const availableSizes = Object.keys(this.models_[model]); - throw APIError.create('field_invalid', null, { - key: 'size/quality combination', - expected: `one of: ${ availableSizes.join(', ')}`, - got: price_key, - }); - } - - const actor = Context.get('actor'); - const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; - if ( user_private_uid === 'UNKNOWN' ) { - this.errors.report('chat-completion-service:unknown-user', { - message: 'failed to get a user ID for a Gemini request', - alarm: true, - trace: true, - }); - } - - const usageType = `gemini:${model}:${price_key}`; - - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, 1); - - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - // Construct the prompt based on whether we have an input image - let contents; - if ( input_image && input_image_mime_type ) { - // Image-to-image generation - contents = [ - { text: `Generate a picture of dimensions ${parseInt(ratio.w)}x${parseInt(ratio.h)} with the prompt: ${prompt}` }, - { - inlineData: { - mimeType: input_image_mime_type, - data: input_image, - }, - }, - ]; - } else { - // Text-to-image generation - contents = `Generate a picture of dimensions ${parseInt(ratio.w)}x${parseInt(ratio.h)} with the prompt: ${prompt}`; - } - - const response = await this.genAI.models.generateContent({ - model, - contents, - }); - - // Metering usage tracking - // Gemini usage: always 1 image, resolution, cost, model - this.meteringService.incrementUsage(actor, usageType, 1); - let url = undefined; - for ( const part of response.candidates[0].content.parts ) { - if ( part.text ) { - // do nothing here - } else if ( part.inlineData ) { - const imageData = part.inlineData.data; - url = `data:image/png;base64,${ imageData}`; - } - } - - if ( ! url ) { - throw new Error('Failed to extract image URL from Gemini response'); - } - - return url; - } - - /** - * Get valid ratios for a specific model - * @param {string} model - The model name - * @returns {Array} Array of valid ratio objects - * @private - */ - _getValidRatios (model) { - if ( - model === 'gemini-2.5-flash-image-preview' || - model === 'gemini-3-pro-image-preview' - ) { - return [this.constructor.RATIO_SQUARE]; - } - return []; - } - - _validate_ratio (ratio, model) { - const validRatios = this._getValidRatios(model); - return validRatios.includes(ratio); - } - - /** - * Validates if the provided MIME type is supported for input images - * @param {string} mimeType - The MIME type to validate - * @returns {boolean} True if the MIME type is supported - * @private - */ - _validate_image_mime_type (mimeType) { - const supportedTypes = ['image/png', 'image/jpeg', 'image/jpg', 'image/webp']; - return supportedTypes.includes(mimeType.toLowerCase()); - } -} - -module.exports = { - GeminiImageGenerationService, -}; diff --git a/src/backend/src/services/ai/image/OpenAIImageGenerationService.js b/src/backend/src/services/ai/image/OpenAIImageGenerationService.js deleted file mode 100644 index fed487306..000000000 --- a/src/backend/src/services/ai/image/OpenAIImageGenerationService.js +++ /dev/null @@ -1,359 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// METADATA // {"ai-commented":{"service":"claude"}} -const APIError = require('../../../api/APIError'); -const BaseService = require('../../BaseService'); -const { TypedValue } = require('../../drivers/meta/Runtime'); -const { Context } = require('../../../util/context'); - -/** -* Service class for generating images using OpenAI's DALL-E API. -* Extends BaseService to provide image generation capabilities through -* the puter-image-generation interface. Supports different aspect ratios -* (square, portrait, landscape) and handles API authentication, request -* validation, and spending tracking. -*/ -class OpenAIImageGenerationService extends BaseService { - /** @type {import('../../MeteringService/MeteringService').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - - static MODULES = { - openai: require('openai'), - }; - - _construct () { - this.models_ = { - 'gpt-image-1-mini': { - 'low:1024x1024': 0.005, - 'low:1024x1536': 0.006, - 'low:1536x1024': 0.006, - 'medium:1024x1024': 0.011, - 'medium:1024x1536': 0.015, - 'medium:1536x1024': 0.015, - 'high:1024x1024': 0.036, - 'high:1024x1536': 0.052, - 'high:1536x1024': 0.052, - }, - 'gpt-image-1': { - 'low:1024x1024': 0.011, - 'low:1024x1536': 0.016, - 'low:1536x1024': 0.016, - 'medium:1024x1024': 0.042, - 'medium:1024x1536': 0.063, - 'medium:1536x1024': 0.063, - 'high:1024x1024': 0.167, - 'high:1024x1536': 0.25, - 'high:1536x1024': 0.25, - }, - 'dall-e-3': { - '1024x1024': 0.04, - '1024x1792': 0.08, - '1792x1024': 0.08, - 'hd:1024x1024': 0.08, - 'hd:1024x1792': 0.12, - 'hd:1792x1024': 0.12, - }, - 'dall-e-2': { - '1024x1024': 0.02, - '512x512': 0.018, - '256x256': 0.016, - }, - }; - } - - /** - * Initializes the OpenAI client with API credentials from config - * @private - * @async - * @returns {Promise} - */ - async _init () { - let apiKey = - this.config?.services?.openai?.apiKey ?? - this.global_config?.services?.openai?.apiKey; - - if ( ! apiKey ) { - apiKey = - this.config?.openai?.secret_key ?? - this.global_config.openai?.secret_key; - - // Log a warning to inform users about the deprecated format - console.warn('The `openai.secret_key` configuration format is deprecated. ' + - 'Please use `services.openai.apiKey` instead.'); - } - - this.openai = new this.modules.openai.OpenAI({ - apiKey, - }); - } - - static IMPLEMENTS = { - ['driver-capabilities']: { - supports_test_mode (iface, method_name) { - return iface === 'puter-image-generation' && - method_name === 'generate'; - }, - }, - ['puter-image-generation']: { - /** - * Generates an image using OpenAI's DALL-E API - * @param {string} prompt - The text description of the image to generate - * @param {Object} options - Generation options - * @param {Object} options.ratio - Image dimensions ratio object with w/h properties - * @param {string} [options.model='dall-e-3'] - The model to use for generation - * @returns {Promise} URL of the generated image - * @throws {Error} If prompt is not a string or ratio is invalid - */ - async generate (params) { - const { prompt, quality, test_mode, model, ratio } = params; - - if ( test_mode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'image', - }, 'https://puter-sample-data.puter.site/image_example.png'); - } - const url = await this.generate(prompt, { - quality, - ratio: ratio || this.constructor.RATIO_SQUARE, - model, - }); - - const image = new TypedValue({ - $: 'string:url:web', - content_type: 'image', - }, url); - - return image; - }, - }, - }; - - static RATIO_SQUARE = { w: 1024, h: 1024 }; - static RATIO_PORTRAIT = { w: 1024, h: 1792 }; - static RATIO_LANDSCAPE = { w: 1792, h: 1024 }; - - // GPT-Image-1 specific ratios - static RATIO_GPT_PORTRAIT = { w: 1024, h: 1536 }; - static RATIO_GPT_LANDSCAPE = { w: 1536, h: 1024 }; - - async generate (prompt, { - ratio, - model, - quality, - }) { - if ( typeof prompt !== 'string' ) { - throw new Error('`prompt` must be a string'); - } - - if ( !ratio || !this._validate_ratio(ratio, model) ) { - throw new Error(`\`ratio\` must be a valid ratio for model ${ model}`); - } - - // Somewhat sane defaults - model = model ?? 'gpt-image-1-mini'; - quality = quality ?? 'low'; - - if ( ! this.models_[model] ) { - throw APIError.create('field_invalid', null, { - key: 'model', - expected: `one of: ${ - Object.keys(this.models_).join(', ')}`, - got: model, - }); - } - - // Validate quality based on the model - const validQualities = this._getValidQualities(model); - if ( quality !== undefined && !validQualities.includes(quality) ) { - throw APIError.create('field_invalid', null, { - key: 'quality', - expected: `one of: ${ validQualities.join(', ').replace(/^$/, 'none (no quality)')}`, - got: quality, - }); - } - - const size = `${ratio.w}x${ratio.h}`; - const price_key = this._buildPriceKey(model, quality, size); - if ( ! this.models_[model][price_key] ) { - const availableSizes = Object.keys(this.models_[model]); - throw APIError.create('field_invalid', null, { - key: 'size/quality combination', - expected: `one of: ${ availableSizes.join(', ')}`, - got: price_key, - }); - } - - const actor = Context.get('actor'); - const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; - if ( user_private_uid === 'UNKNOWN' ) { - this.errors.report('chat-completion-service:unknown-user', { - message: 'failed to get a user ID for an OpenAI request', - alarm: true, - trace: true, - }); - } - - const usageType = `openai:${model}:${price_key}`; - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, 1); - - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - // Build API parameters based on model - const apiParams = this._buildApiParams(model, { - user: user_private_uid, - prompt, - size, - quality, - }); - - const result = await this.openai.images.generate(apiParams); - - // For image generation, usage is typically image count and resolution - this.meteringService.incrementUsage(actor, usageType, 1); - - const spending_meta = { - model, - size: `${ratio.w}x${ratio.h}`, - }; - - if ( quality ) { - spending_meta.size = `${quality }:${ spending_meta.size}`; - } - - const url = result.data?.[0]?.url || (result.data?.[0]?.b64_json ? `data:image/png;base64,${ result.data[0].b64_json}` : null); - - if ( ! url ) { - throw new Error('Failed to extract image URL from OpenAI response'); - } - - return url; - } - - /** - * Get valid quality levels for a specific model - * @param {string} model - The model name - * @returns {Array} Array of valid quality levels - * @private - */ - _getValidQualities (model) { - if ( model === 'gpt-image-1-mini' ) { - return ['low', 'medium', 'high']; - } - if ( model === 'gpt-image-1' ) { - return ['low', 'medium', 'high']; - } - if ( model === 'dall-e-2' ) { - return ['']; - } - if ( model === 'dall-e-3' ) { - return ['', 'hd']; - } - // Fallback for unknown models - assume no quality tiers - return ['']; - } - - /** - * Build the price key for a model based on quality and size - * @param {string} model - The model name - * @param {string} quality - The quality level - * @param {string} size - The image size (e.g., "1024x1024") - * @returns {string} The price key - * @private - */ - _buildPriceKey (model, quality, size) { - if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) { - // gpt-image-1 and gpt-image-1-mini use format: "quality:size" - default to low if not specified - const qualityLevel = quality || 'low'; - return `${qualityLevel}:${size}`; - } else { - // dall-e models use format: "hd:size" or just "size" - return (quality === 'hd' ? 'hd:' : '') + size; - } - } - - /** - * Build API parameters based on the model - * @param {string} model - The model name - * @param {Object} baseParams - Base parameters for the API call - * @returns {Object} API parameters object - * @private - */ - _buildApiParams (model, baseParams) { - const apiParams = { - user: baseParams.user, - prompt: baseParams.prompt, - size: baseParams.size, - }; - - if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) { - // gpt-image-1 requires the model parameter and uses different quality mapping - apiParams.model = model; - // Default to low quality if not specified, consistent with _buildPriceKey - apiParams.quality = baseParams.quality || 'low'; - } else { - // dall-e models - apiParams.model = model; - if ( baseParams.quality === 'hd' ) { - apiParams.quality = 'hd'; - } - } - - return apiParams; - } - - /** - * Get valid ratios for a specific model - * @param {string} model - The model name - * @returns {Array} Array of valid ratio objects - * @private - */ - _getValidRatios (model) { - const commonRatios = [this.constructor.RATIO_SQUARE]; - - if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) { - return [ - ...commonRatios, - this.constructor.RATIO_GPT_PORTRAIT, - this.constructor.RATIO_GPT_LANDSCAPE, - ]; - } else { - // DALL-E models - return [ - ...commonRatios, - this.constructor.RATIO_PORTRAIT, - this.constructor.RATIO_LANDSCAPE, - ]; - } - } - - _validate_ratio (ratio, model) { - const validRatios = this._getValidRatios(model); - return validRatios.includes(ratio); - } -} - -module.exports = { - OpenAIImageGenerationService, -}; diff --git a/src/backend/src/services/ai/image/TogetherImageGenerationService.js b/src/backend/src/services/ai/image/TogetherImageGenerationService.js deleted file mode 100644 index 7908b4333..000000000 --- a/src/backend/src/services/ai/image/TogetherImageGenerationService.js +++ /dev/null @@ -1,287 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -// METADATA // {"ai-commented":{"service":"claude"}} -import { APIError } from 'openai'; -import { Together } from 'together-ai'; -import { Context } from '../../../util/context.js'; -import BaseService from '../../BaseService.js'; -import { TypedValue } from '../../drivers/meta/Runtime.js'; - -/** -* Service class for generating images using Together AI models. -* Extends BaseService to provide image generation capabilities through the -* puter-image-generation interface. Handles authentication, request validation, -* and metering integration. -*/ - -export class TogetherImageGenerationService extends BaseService { - DEFAULT_MODEL = 'black-forest-labs/FLUX.1-schnell'; - DEFAULT_RATIO = { w: 1024, h: 1024 }; - CONDITION_IMAGE_MODELS = [ - 'black-forest-labs/flux.1-kontext-dev', - 'black-forest-labs/flux.1-kontext-pro', - 'black-forest-labs/flux.1-kontext-max', - ]; - - /** @type {import('../../MeteringService/MeteringService.js').MeteringService} */ - get meteringService () { - return this.services.get('meteringService').meteringService; - } - - async _init () { - const apiKey = - this.config?.apiKey ?? - this.global_config?.services?.['together-ai']?.apiKey; - - if ( ! apiKey ) { - throw new Error('Together AI image generation requires an API key'); - } - - this.client = new Together({ apiKey }); - } - - static IMPLEMENTS = { - ['driver-capabilities']: { - supports_test_mode (iface, method_name) { - return iface === 'puter-image-generation' && - method_name === 'generate'; - }, - }, - ['puter-image-generation']: { - async generate (...args) { - return this.generate(...args); - }, - }, - ['models']: { - - }, - }; - - /** - * Generates an image using Together AI image models - * @param {object} params - Generation parameters - * @param {string} params.prompt - Prompt describing the desired image - * @param {string} [params.model] - Together AI model identifier - * @param {object} [params.ratio] - Width/height ratio object (e.g., { w: 1024, h: 1024 }) - * @param {number} [params.width] - Explicit width override - * @param {number} [params.height] - Explicit height override - * @param {string} [params.aspect_ratio] - Aspect ratio string (e.g., "16:9") - * @param {number} [params.steps] - Diffusion step count - * @param {number} [params.seed] - Seed for reproducibility - * @param {string} [params.negative_prompt] - Negative prompt text - * @param {number} [params.n] - Number of images to generate (default 1) - * @param {string} [params.image_url] - Reference image URL for image-to-image - * @param {string} [params.image_base64] - Base64 encoded reference image - * @param {boolean} [params.disable_safety_checker] - Disable Together AI safety checker - * @param {boolean} [params.test_mode] - Enable Puter test mode shortcut - * @returns {Promise} TypedValue containing the generated image URL or data URI - */ - async generate (params) { - const { - prompt, - test_mode, - ratio, - model, - width, - height, - aspect_ratio, - steps, - seed, - negative_prompt, - n, - image_url, - image_base64, - mask_image_url, - mask_image_base64, - prompt_strength, - disable_safety_checker, - response_format, - } = params; - - const svc_event = this.services.get('event'); - svc_event.emit('ai.log.image', { actor: Context.get('actor'), parameters: params, completionId: '0', intended_service: params.model }); - - if ( test_mode ) { - return new TypedValue({ - $: 'string:url:web', - content_type: 'image', - }, 'https://puter-sample-data.puter.site/image_example.png'); - } - - const url = await this.#generate(prompt, { - ratio, - model, - width, - height, - aspect_ratio, - steps, - seed, - negative_prompt, - n, - image_url, - image_base64, - mask_image_url, - mask_image_base64, - prompt_strength, - disable_safety_checker, - response_format, - }); - - const isDataUrl = url.startsWith('data:'); - return new TypedValue({ - $: isDataUrl ? 'string:url:data' : 'string:url:web', - content_type: 'image', - }, url); - } - - async #generate (prompt, options) { - if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { - throw new Error('`prompt` must be a non-empty string'); - } - - const request = this.#buildRequest(prompt, options); - - const actor = Context.get('actor'); - if ( ! actor ) { - throw new Error('actor not found in context'); - } - - const usageType = `together-image:${request.model}`; - const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, 1); - if ( ! usageAllowed ) { - throw APIError.create('insufficient_funds'); - } - - const response = await this.client.images.generate(request); - if ( ! response?.data?.length ) { - throw new Error('Together AI response did not include image data'); - } - - this.meteringService.incrementUsage(actor, usageType, 1); - - const first = response.data[0]; - if ( first.url ) { - return first.url; - } - if ( first.b64_json ) { - return `data:image/png;base64,${ first.b64_json}`; - } - - throw new Error('Together AI response did not include an image URL'); - } - - /** - * Normalizes Together AI image generation request parameters - */ - #buildRequest (prompt, options = {}) { - const { - ratio, - model, - width, - height, - aspect_ratio, - steps, - seed, - negative_prompt, - n, - image_url, - image_base64, - mask_image_url, - mask_image_base64, - prompt_strength, - disable_safety_checker, - response_format, - input_image, - } = options; - - const request = { - prompt, - model: model ?? this.constructor.DEFAULT_MODEL, - }; - const requiresConditionImage = - this.#modelRequiresConditionImage(request.model); - - const ratioWidth = (ratio && ratio.w !== undefined) ? Number(ratio.w) : undefined; - const ratioHeight = (ratio && ratio.h !== undefined) ? Number(ratio.h) : undefined; - const normalizedWidth = this.#normalizeDimension(width !== undefined ? Number(width) : (ratioWidth ?? this.DEFAULT_RATIO.w)); - const normalizedHeight = this.#normalizeDimension(height !== undefined ? Number(height) : (ratioHeight ?? this.DEFAULT_RATIO.h)); - - if ( aspect_ratio ) { - request.aspect_ratio = aspect_ratio; - } else { - if ( normalizedWidth ) request.width = normalizedWidth; - if ( normalizedHeight ) request.height = normalizedHeight; - } - - if ( typeof steps === 'number' && Number.isFinite(steps) ) { - request.steps = Math.max(1, Math.min(50, Math.round(steps))); - } - if ( typeof seed === 'number' && Number.isFinite(seed) ) request.seed = Math.round(seed); - if ( typeof negative_prompt === 'string' ) request.negative_prompt = negative_prompt; - if ( typeof n === 'number' && Number.isFinite(n) ) { - request.n = Math.max(1, Math.min(4, Math.round(n))); - } - if ( typeof disable_safety_checker === 'boolean' ) { - request.disable_safety_checker = disable_safety_checker; - } - if ( typeof response_format === 'string' ) request.response_format = response_format; - - const resolvedImageBase64 = typeof image_base64 === 'string' - ? image_base64 - : (typeof input_image === 'string' ? input_image : undefined); - - if ( typeof image_url === 'string' ) request.image_url = image_url; - if ( resolvedImageBase64 ) request.image_base64 = resolvedImageBase64; - if ( typeof mask_image_url === 'string' ) request.mask_image_url = mask_image_url; - if ( typeof mask_image_base64 === 'string' ) request.mask_image_base64 = mask_image_base64; - if ( typeof prompt_strength === 'number' && Number.isFinite(prompt_strength) ) { - request.prompt_strength = Math.max(0, Math.min(1, prompt_strength)); - } - if ( requiresConditionImage ) { - const conditionSource = resolvedImageBase64 - ? resolvedImageBase64 - : (typeof image_url === 'string' ? image_url : undefined); - - if ( ! conditionSource ) { - throw new Error(`Model ${request.model} requires an image_url or image_base64 input`); - } - - request.condition_image = conditionSource; - } - - return request; - } - - #normalizeDimension (value) { - if ( typeof value !== 'number' ) return undefined; - const rounded = Math.max(64, Math.round(value)); - // Flux models expect multiples of 8. Snap to the nearest multiple without going below 64. - return Math.max(64, Math.round(rounded / 8) * 8); - } - - #modelRequiresConditionImage (model) { - if ( typeof model !== 'string' || model.trim() === '' ) { - return false; - } - - const normalized = model.toLowerCase(); - return this.CONDITION_IMAGE_MODELS.some(required => normalized === required); - } -} \ No newline at end of file diff --git a/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.ts b/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.ts new file mode 100644 index 000000000..facada7ca --- /dev/null +++ b/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/GeminiImageGenerationProvider.ts @@ -0,0 +1,171 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// METADATA // {"ai-commented":{"service":"claude"}} +import { GenerateContentResponse, GoogleGenAI } from '@google/genai'; +import APIError from '../../../../../api/APIError.js'; +import { ErrorService } from '../../../../../modules/core/ErrorService.js'; +import { Context } from '../../../../../util/context.js'; +import { MeteringService } from '../../../../MeteringService/MeteringService.js'; +import { GEMINI_DEFAULT_RATIO, GEMINI_IMAGE_GENERATION_MODELS } from './models.js'; +import { IGenerateParams, IImageModel, IImageProvider } from '../types.js'; + +type GeminiGenerateParams = IGenerateParams & { + input_image?: string; + input_image_mime_type?: string; +}; + +export class GeminiImageGenerationProvider implements IImageProvider { + #meteringService: MeteringService; + #client: GoogleGenAI; + #errors: ErrorService; + + constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService) { + if ( ! config.apiKey ) { + throw new Error('Gemini image generation requires an API key'); + } + this.#meteringService = meteringService; + this.#client = new GoogleGenAI({ apiKey: config.apiKey }); + this.#errors = errorService; + } + + models (): IImageModel[] { + return GEMINI_IMAGE_GENERATION_MODELS; + } + + getDefaultModel (): string { + return GEMINI_IMAGE_GENERATION_MODELS[0].id; + } + + async generate (params: IGenerateParams): Promise { + const { prompt, test_mode } = params; + let { model, ratio, quality } = params; + const { input_image, input_image_mime_type } = params as GeminiGenerateParams; + + const selectedModel = this.models().find(m => m.id === model) || this.models().find(m => m.id === this.getDefaultModel())!; + + if ( test_mode ) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { + throw new Error('`prompt` must be a non-empty string'); + } + + const allowedRatios = selectedModel.allowedRatios ?? [GEMINI_DEFAULT_RATIO]; + ratio = ratio && this.#isValidRatio(ratio, allowedRatios) ? ratio : allowedRatios[0]; + + if ( input_image && !input_image_mime_type ) { + throw new Error('`input_image_mime_type` is required when `input_image` is provided'); + } + + if ( input_image_mime_type && !input_image ) { + throw new Error('`input_image` is required when `input_image_mime_type` is provided'); + } + + if ( input_image_mime_type && !this.#isValidImageMimeType(input_image_mime_type) ) { + throw new Error('`input_image_mime_type` must be a valid image MIME type (image/png, image/jpeg, image/webp)'); + } + + const priceKey = `${quality ? `${quality}:` : ''}${ratio.w}x${ratio.h}`; + const priceInCents = selectedModel.costs[priceKey]; + if ( priceInCents === undefined ) { + const availableSizes = Object.keys(selectedModel.costs); + throw APIError.create('field_invalid', undefined, { + key: 'size/quality combination', + expected: `one of: ${ availableSizes.join(', ')}`, + got: priceKey, + }); + } + + const actor = Context.get('actor'); + const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; + if ( user_private_uid === 'UNKNOWN' ) { + this.#errors.report('gemini-image-generation:unknown-user', { + message: 'failed to get a user ID for a Gemini request', + alarm: true, + trace: true, + }); + } + + const costInMicroCents = priceInCents * 1_000_000; + const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents); + + if ( ! usageAllowed ) { + throw APIError.create('insufficient_funds'); + } + + const contents = this.#buildContents(prompt, ratio, input_image, input_image_mime_type); + const response = await this.#client.models.generateContent({ + model: selectedModel.id, + contents, + }); + + const usageType = `gemini:${selectedModel.id}:${priceKey}`; + this.#meteringService.incrementUsage(actor, usageType, 1, costInMicroCents); + + const url = this.#extractImageUrl(response); + + if ( ! url ) { + throw new Error('Failed to extract image URL from Gemini response'); + } + + return url; + } + + #buildContents (prompt: string, ratio: { w: number; h: number }, input_image?: string, input_image_mime_type?: string) { + if ( input_image && input_image_mime_type ) { + return [ + { text: `Generate a picture of dimensions ${parseInt(`${ratio.w}`)}x${parseInt(`${ratio.h}`)} with the prompt: ${prompt}` }, + { + inlineData: { + mimeType: input_image_mime_type, + data: input_image, + }, + }, + ]; + } + + return `Generate a picture of dimensions ${parseInt(`${ratio.w}`)}x${parseInt(`${ratio.h}`)} with the prompt: ${prompt}`; + } + + #extractImageUrl (response: GenerateContentResponse): string | undefined { + const parts = response?.candidates?.[0]?.content?.parts; + if ( ! Array.isArray(parts) ) { + return undefined; + } + + for ( const part of parts ) { + if ( part?.inlineData?.data ) { + return `data:image/png;base64,${ part.inlineData.data}`; + } + } + return undefined; + } + + #isValidRatio (ratio: { w: number; h: number }, allowedRatios: { w: number; h: number }[]) { + return allowedRatios.some(r => r.w === ratio.w && r.h === ratio.h); + } + + #isValidImageMimeType (mimeType?: string) { + if ( ! mimeType ) return false; + const supportedTypes = ['image/png', 'image/jpeg', 'image/jpg', 'image/webp']; + return supportedTypes.includes(mimeType.toLowerCase()); + } +} diff --git a/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/models.ts b/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/models.ts new file mode 100644 index 000000000..7269937b5 --- /dev/null +++ b/src/backend/src/services/ai/image/providers/GeminiImageGenerationProvider/models.ts @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { IImageModel } from '../types'; + +export const GEMINI_DEFAULT_RATIO = { w: 1024, h: 1024 }; + +export const GEMINI_IMAGE_GENERATION_MODELS: IImageModel[] = [ + { + id: 'gemini-2.5-flash-image-preview', + aliases: ['gemini-2.5-flash-image'], + name: 'Gemini 2.5 Flash Image Preview', + version: '1.0', + costs_currency: 'usd-cents', + index_cost_key: '1024x1024', + allowedQualityLevels: [''], + costs: { + '1x1': 3.9, // $0.039 per image + '2x3': 3.9, // $0.039 per image + '3x2': 3.9, // $0.039 per image + '3x4': 3.9, // $0.039 per image + '4x3': 3.9, // $0.039 per image + '4x5': 3.9, // $0.039 per image + '5x4': 3.9, // $0.039 per image + '9x16': 3.9, // $0.039 per image + '16x9': 3.9, // $0.039 per image + '21x9': 3.9, // $0.039 per image + }, + allowedRatios: [ + { w: 1, h: 1 }, + { w: 2, h: 3 }, + { w: 3, h: 2 }, + { w: 3, h: 4 }, + { w: 4, h: 3 }, + { w: 4, h: 5 }, + { w: 5, h: 4 }, + { w: 9, h: 16 }, + { w: 16, h: 9 }, + { w: 21, h: 9 }, + ], + }, + { + id: 'gemini-3-pro-image-preview', + name: 'Gemini 3 Pro Image Preview', + version: '1.0', + costs_currency: 'usd-cents', + index_cost_key: '1024x1024', + aliases: ['gemini-3-pro-image'], + allowedQualityLevels: ['1K', '2K', '4K'], + allowedRatios: [ + { w: 1, h: 1 }, + { w: 2, h: 3 }, + { w: 3, h: 2 }, + { w: 3, h: 4 }, + { w: 4, h: 3 }, + { w: 4, h: 5 }, + { w: 5, h: 4 }, + { w: 9, h: 16 }, + { w: 16, h: 9 }, + { w: 21, h: 9 }, + ], + costs: { + '1K:1x1': 13.51, // $0.1351 per image + '1K:2x3': 13.51, // $0.1351 per image + '1K:3x2': 13.51, // $0.1351 per image + '1K:3x4': 13.51, // $0.1351 per image + '1K:4x3': 13.51, // $0.1351 per image + '1K:4x5': 13.51, // $0.1351 per image + '1K:5x4': 13.51, // $0.1351 per image + '1K:9x16': 13.51, // $0.1351 per image + '1K:16x9': 13.51, // $0.1351 per image + '1K:21x9': 13.51, // $0.1351 per image + '2K:1x1': 13.51, // $0.1351 per image + '2K:2x3': 13.51, // $0.1351 per image + '2K:3x2': 13.51, // $0.1351 per image + '2K:3x4': 13.51, // $0.1351 per image + '2K:4x3': 13.51, // $0.1351 per image + '2K:4x5': 13.51, // $0.1351 per image + '2K:5x4': 13.51, // $0.1351 per image + '2K:9x16': 13.51, // $0.1351 per image + '2K:16x9': 13.51, // $0.1351 per image + '2K:21x9': 13.51, // $0.1351 per image + '4K:1x1': 24.1, // $0.24 per image + '4K:2x3': 24.1, // $0.24 per image + '4K:3x2': 24.1, // $0.24 per image + '4K:3x4': 24.1, // $0.24 per image + '4K:4x3': 24.1, // $0.24 per image + '4K:4x5': 24.1, // $0.24 per image + '4K:5x4': 24.1, // $0.24 per image + '4K:9x16': 24.1, // $0.24 per image + '4K:16x9': 24.1, // $0.24 per image + '4K:21x9': 24.1, // $0.24 per image + }, + }, +]; diff --git a/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.ts b/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.ts new file mode 100644 index 000000000..84d6ceec4 --- /dev/null +++ b/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/OpenAiImageGenerationProvider.ts @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// METADATA // {"ai-commented":{"service":"claude"}} +import openai, { OpenAI } from 'openai'; +import { ImageGenerateParamsNonStreaming } from 'openai/resources/images.js'; +import APIError from '../../../../../api/APIError.js'; +import { ErrorService } from '../../../../../modules/core/ErrorService.js'; +import { Context } from '../../../../../util/context.js'; +import { MeteringService } from '../../../../MeteringService/MeteringService.js'; +import { IGenerateParams, IImageProvider } from '../types.js'; +import { OPEN_AI_IMAGE_GENERATION_MODELS } from './models.js'; +/** +* Service class for generating images using OpenAI's DALL-E API. +* Extends BaseService to provide image generation capabilities through +* the puter-image-generation interface. Supports different aspect ratios +* (square, portrait, landscape) and handles API authentication, request +* validation, and spending tracking. +*/ +export class OpenAiImageGenerationProvider implements IImageProvider { + #meteringService: MeteringService; + #openai: OpenAI; + #errors: ErrorService; + + constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService) { + this.#meteringService = meteringService; + this.#openai = new openai.OpenAI({ + apiKey: config.apiKey, + }); + this.#errors = errorService; + } + + models () { + return OPEN_AI_IMAGE_GENERATION_MODELS; + } + + getDefaultModel (): string { + return 'dall-e-2'; + } + + async generate ({ prompt, quality, test_mode, model, ratio }: IGenerateParams) { + + const selectedModel = this.models().find(m => m.id === model) || this.models().find(m => m.id === this.getDefaultModel())!; + + if ( test_mode ) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if ( typeof prompt !== 'string' ) { + throw new Error('`prompt` must be a string'); + } + + const validRations = selectedModel?.allowedRatios; + if ( validRations && (!ratio || !validRations.some(r => r.w === ratio.w && r.h === ratio.h)) ) { + ratio = validRations[0]; // Default to the first allowed ratio + } + + if ( ! ratio ) { + ratio = { w: 1024, h: 1024 }; // Fallback ratio + } + + const validQualities = selectedModel?.allowedQualityLevels; + if ( validQualities && (!quality || !validQualities.includes(quality)) ) { + quality = validQualities[0]; // Default to the first allowed quality + } + + const size = `${ratio.w}x${ratio.h}`; + const price_key = this.#buildPriceKey(selectedModel.id, quality!, size); + if ( ! selectedModel?.costs[price_key] ) { + const availableSizes = Object.keys(selectedModel?.costs); + throw APIError.create('field_invalid', undefined, { + key: 'size/quality combination', + expected: `one of: ${ availableSizes.join(', ')}`, + got: price_key, + }); + } + + const actor = Context.get('actor'); + const user_private_uid = actor?.private_uid ?? 'UNKNOWN'; + if ( user_private_uid === 'UNKNOWN' ) { + this.#errors.report('chat-completion-service:unknown-user', { + message: 'failed to get a user ID for an OpenAI request', + alarm: true, + trace: true, + }); + } + + const costInMicroCents = selectedModel.costs[price_key] * 1_000_000; + const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents); + + if ( ! usageAllowed ) { + throw APIError.create('insufficient_funds'); + } + + // Build API parameters based on model + const apiParams = this.#buildApiParams(selectedModel.id, { + user: user_private_uid, + prompt, + size, + quality, + } as Partial); + + const result = await this.#openai.images.generate(apiParams); + + // For image generation, usage is typically image count and resolution + const usageType = `openai:${selectedModel.id}:${price_key}`; + this.#meteringService.incrementUsage(actor, usageType, 1, costInMicroCents); + + const spending_meta = { + model, + size: `${ratio.w}x${ratio.h}`, + }; + + if ( quality ) { + spending_meta.size = `${quality}:${ spending_meta.size}`; + } + + const url = result.data?.[0]?.url || (result.data?.[0]?.b64_json ? `data:image/png;base64,${ result.data[0].b64_json}` : null); + + if ( ! url ) { + throw new Error('Failed to extract image URL from OpenAI response'); + } + + return url; + } + + #buildPriceKey (model: string, quality: string, size: string) { + if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) { + // gpt-image-1 and gpt-image-1-mini use format: "quality:size" - default to low if not specified + const qualityLevel = quality || 'low'; + return `${qualityLevel}:${size}`; + } else { + // dall-e models use format: "hd:size" or just "size" + return (quality === 'hd' ? 'hd:' : '') + size; + } + } + + #buildApiParams (model: string, baseParams: Partial): ImageGenerateParamsNonStreaming { + const apiParams = { + user: baseParams.user, + prompt: baseParams.prompt, + size: baseParams.size, + } as ImageGenerateParamsNonStreaming; + + if ( model === 'gpt-image-1' || model === 'gpt-image-1-mini' ) { + // gpt-image-1 requires the model parameter and uses different quality mapping + apiParams.model = model; + // Default to low quality if not specified, consistent with _buildPriceKey + apiParams.quality = baseParams.quality || 'low'; + } else { + // dall-e models + apiParams.model = model; + if ( baseParams.quality === 'hd' ) { + apiParams.quality = 'hd'; + } + } + + return apiParams; + } +} diff --git a/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/models.ts b/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/models.ts new file mode 100644 index 000000000..5b4217bdf --- /dev/null +++ b/src/backend/src/services/ai/image/providers/OpenAiImageGenerationProvider/models.ts @@ -0,0 +1,71 @@ +import { IImageModel } from '../types'; + +export const OPEN_AI_IMAGE_GENERATION_MODELS: IImageModel[] = [ + + { id: 'gpt-image-1-mini', + name: 'GPT Image 1 Mini', + version: '1.0', + costs_currency: 'usd-cents', + index_cost_key: 'low:1024x1024', + costs: { + 'low:1024x1024': 0.5, + 'low:1024x1536': 0.6, + 'low:1536x1024': 0.6, + 'medium:1024x1024': 1.1, + 'medium:1024x1536': 1.5, + 'medium:1536x1024': 1.5, + 'high:1024x1024': 3.6, + 'high:1024x1536': 5.2, + 'high:1536x1024': 5.2, + }, + allowedQualityLevels: ['low', 'medium', 'high'], + allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1536 }, { w: 1536, h: 1024 }], + }, + { id: 'gpt-image-1', + name: 'GPT Image 1', + version: '1.0', + costs_currency: 'usd-cents', + index_cost_key: 'low:1024x1024', + costs: { + 'low:1024x1024': 1.1, + 'low:1024x1536': 1.6, + 'low:1536x1024': 1.6, + 'medium:1024x1024': 4.2, + 'medium:1024x1536': 6.3, + 'medium:1536x1024': 6.3, + 'high:1024x1024': 16.7, + 'high:1024x1536': 25, + 'high:1536x1024': 25, + }, + allowedQualityLevels: ['low', 'medium', 'high'], + allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1536 }, { w: 1536, h: 1024 }], + }, + { id: 'dall-e-3', + name: 'DALL·E 3', + version: '1.0', + costs_currency: 'usd-cents', + index_cost_key: '1024x1024', + costs: { + '1024x1024': 4, + '1024x1792': 8, + '1792x1024': 8, + 'hd:1024x1024': 8, + 'hd:1024x1792': 12, + 'hd:1792x1024': 12, + }, + allowedQualityLevels: ['', 'hd'], + allowedRatios: [{ w: 1024, h: 1024 }, { w: 1024, h: 1792 }, { w: 1792, h: 1024 }], + }, + { id: 'dall-e-2', + name: 'DALL·E 2', + version: '1.0', + costs_currency: 'usd-cents', + index_cost_key: '1024x1024', + costs: { + '256x256': 1.6, + '512x512': 1.8, + '1024x1024': 2, + }, + allowedRatios: [ { w: 256, h: 256 }, { w: 512, h: 512 }, { w: 1024, h: 1024 }], + }, +]; diff --git a/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.ts b/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.ts new file mode 100644 index 000000000..4ea454ae6 --- /dev/null +++ b/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/TogetherImageGenerationProvider.ts @@ -0,0 +1,242 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// METADATA // {"ai-commented":{"service":"claude"}} +import { Together } from 'together-ai'; +import APIError from '../../../../../api/APIError.js'; +import { ErrorService } from '../../../../../modules/core/ErrorService.js'; +import { Context } from '../../../../../util/context.js'; +import { EventService } from '../../../../EventService.js'; +import { MeteringService } from '../../../../MeteringService/MeteringService.js'; +import { IGenerateParams, IImageModel, IImageProvider } from '../types.js'; +import { TOGETHER_IMAGE_GENERATION_MODELS } from './models.js'; + +const TOGETHER_DEFAULT_RATIO = { w: 1024, h: 1024 }; +type TogetherGenerateParams = IGenerateParams & { + steps?: number; + seed?: number; + negative_prompt?: string; + n?: number; + image_url?: string; + image_base64?: string; + mask_image_url?: string; + mask_image_base64?: string; + prompt_strength?: number; + disable_safety_checker?: boolean; + response_format?: string; + input_image?: string; +}; + +const DEFAULT_MODEL = 'togetherai:black-forest-labs/FLUX.1-schnell'; +const CONDITION_IMAGE_MODELS = [ + 'togetherai:black-forest-labs/flux.1-kontext-dev', + 'togetherai:black-forest-labs/flux.1-kontext-pro', + 'togetherai:black-forest-labs/flux.1-kontext-max', +]; + +export class TogetherImageGenerationProvider implements IImageProvider { + #client: Together; + #meteringService: MeteringService; + #errors: ErrorService; + #eventService: EventService; + + constructor (config: { apiKey: string }, meteringService: MeteringService, errorService: ErrorService, eventService: EventService) { + if ( ! config.apiKey ) { + throw new Error('Together AI image generation requires an API key'); + } + this.#meteringService = meteringService; + this.#errors = errorService; + this.#eventService = eventService; + this.#client = new Together({ apiKey: config.apiKey }); + } + + models (): IImageModel[] { + return TOGETHER_IMAGE_GENERATION_MODELS; + } + + getDefaultModel (): string { + return DEFAULT_MODEL; + } + + async generate (params: IGenerateParams): Promise { + const { prompt, test_mode } = params; + let { model, ratio, quality } = params; + const options = params as TogetherGenerateParams; + + const selectedModel = this.#getModel(model); + + await this.#eventService.emit('ai.log.image', { actor: Context.get('actor'), parameters: params, completionId: '0', intended_service: selectedModel.id }); + + if ( test_mode ) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if ( typeof prompt !== 'string' || prompt.trim().length === 0 ) { + throw new Error('`prompt` must be a non-empty string'); + } + + ratio = ratio || TOGETHER_DEFAULT_RATIO; + + const actor = Context.get('actor'); + if ( ! actor ) { + this.#errors.report('together-image-generation:unknown-actor', { + message: 'failed to resolve actor for Together image generation', + trace: true, + }); + throw new Error('actor not found in context'); + } + + const priceKey = '1MP'; + const centsPerMP = selectedModel.costs[priceKey]; // hardcoded for now since all together ai models use this type of pricing + if ( centsPerMP === undefined ) { + throw new Error(`No pricing configured for model ${selectedModel.id}`); + } + + const usageType = `together-image:${selectedModel.id}:${priceKey}`; + + let MP = (ratio.h * ratio.w) / 1_000_000; + if ( quality ) { + // if quality its gemini 3 image, so price based on those K sizes as MP + MP = parseInt(quality[0]) ; // convert to microcents + } + + const costInMicroCents = centsPerMP * MP * 1_000_000; // cost in microcents + + const usageAllowed = await this.#meteringService.hasEnoughCredits(actor, costInMicroCents); + + if ( ! usageAllowed ) { + throw APIError.create('insufficient_funds'); + } + + const request = this.#buildRequest(prompt, { ...options, ratio, model: selectedModel.id.replace('togetherai:', '') }) as unknown as Together.Images.ImageGenerateParams; + + try { + const response = await this.#client.images.generate(request); + if ( ! response?.data?.length ) { + throw new Error('Together AI response did not include image data'); + } + + this.#meteringService.incrementUsage(actor, usageType, MP, costInMicroCents); + + const first = response.data[0] as { url?: string; b64_json?: string }; + const url = first.url || (first.b64_json ? `data:image/png;base64,${ first.b64_json}` : undefined); + + if ( ! url ) { + throw new Error('Together AI response did not include an image URL'); + } + + return url; + } catch ( error ) { + throw new Error(`Together AI image generation error: ${(error as Error).message}`); + } + } + + #getModel (model?: string) { + return this.models().find(m => m.id === model) || this.models().find(m => m.id === DEFAULT_MODEL)!; + } + + #buildRequest (prompt: string, options: TogetherGenerateParams) { + const { + ratio, + model, + steps, + seed, + negative_prompt, + n, + image_url, + image_base64, + mask_image_url, + mask_image_base64, + prompt_strength, + disable_safety_checker, + response_format, + input_image, + } = options; + + const request: Record = { + prompt, + model: model ?? DEFAULT_MODEL, + }; + + const requiresConditionImage = this.#modelRequiresConditionImage(request.model as string); + + const ratioWidth = ratio?.w !== undefined ? Number(ratio.w) : undefined; + const ratioHeight = ratio?.h !== undefined ? Number(ratio.h) : undefined; + + const normalizedWidth = this.#normalizeDimension((ratioWidth ?? TOGETHER_DEFAULT_RATIO.w)); + const normalizedHeight = this.#normalizeDimension((ratioHeight ?? TOGETHER_DEFAULT_RATIO.h)); + + if ( normalizedWidth ) request.width = normalizedWidth; + if ( normalizedHeight ) request.height = normalizedHeight; + + if ( typeof steps === 'number' && Number.isFinite(steps) ) { + request.steps = Math.max(1, Math.min(50, Math.round(steps))); + } + if ( typeof seed === 'number' && Number.isFinite(seed) ) request.seed = Math.round(seed); + if ( typeof negative_prompt === 'string' ) request.negative_prompt = negative_prompt; + if ( typeof n === 'number' && Number.isFinite(n) ) { + request.n = Math.max(1, Math.min(4, Math.round(n))); + } + if ( disable_safety_checker ) { + request.disable_safety_checker = true; + } + if ( typeof response_format === 'string' ) request.response_format = response_format; + + const resolvedImageBase64 = typeof image_base64 === 'string' + ? image_base64 + : (typeof input_image === 'string' ? input_image : undefined); + + if ( typeof image_url === 'string' ) request.image_url = image_url; + if ( resolvedImageBase64 ) request.image_base64 = resolvedImageBase64; + if ( typeof mask_image_url === 'string' ) request.mask_image_url = mask_image_url; + if ( typeof mask_image_base64 === 'string' ) request.mask_image_base64 = mask_image_base64; + if ( typeof prompt_strength === 'number' && Number.isFinite(prompt_strength) ) { + request.prompt_strength = Math.max(0, Math.min(1, prompt_strength)); + } + if ( requiresConditionImage ) { + const conditionSource = resolvedImageBase64 + ? resolvedImageBase64 + : (typeof image_url === 'string' ? image_url : undefined); + + if ( ! conditionSource ) { + throw new Error(`Model ${request.model} requires an image_url or image_base64 input`); + } + + request.condition_image = conditionSource; + } + + return request; + } + + #normalizeDimension (value?: number) { + if ( typeof value !== 'number' || Number.isNaN(value) ) return undefined; + const rounded = Math.max(64, Math.round(value)); + // Flux models expect multiples of 8. Snap to the nearest multiple without going below 64. + return Math.max(64, Math.round(rounded / 8) * 8); + } + + #modelRequiresConditionImage (modelId?: string) { + if ( typeof modelId !== 'string' || modelId.trim() === '' ) { + return false; + } + + const normalized = modelId.toLowerCase(); + return CONDITION_IMAGE_MODELS.some(required => normalized === required); + } +} diff --git a/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/models.ts b/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/models.ts new file mode 100644 index 000000000..36fad16a2 --- /dev/null +++ b/src/backend/src/services/ai/image/providers/TogetherImageGenerationProvider/models.ts @@ -0,0 +1,319 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { IImageModel } from '../types'; + +export const TOGETHER_IMAGE_GENERATION_MODELS: IImageModel[] = [ + { + id: 'togetherai:ByteDance-Seed/Seedream-3.0', + aliases: ['ByteDance-Seed/Seedream-3.0'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'ByteDance-Seed/Seedream-3.0', + allowedQualityLevels: [''], + costs: { '1MP': 1.8 }, + }, + { + id: 'togetherai:ByteDance-Seed/Seedream-4.0', + aliases: ['ByteDance-Seed/Seedream-4.0'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'ByteDance-Seed/Seedream-4.0', + allowedQualityLevels: [''], + costs: { '1MP': 3 }, + }, + { + id: 'togetherai:HiDream-ai/HiDream-I1-Dev', + aliases: ['HiDream-ai/HiDream-I1-Dev'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'HiDream-ai/HiDream-I1-Dev', + allowedQualityLevels: [''], + costs: { '1MP': 0.45 }, + }, + { + id: 'togetherai:HiDream-ai/HiDream-I1-Fast', + aliases: ['HiDream-ai/HiDream-I1-Fast'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'HiDream-ai/HiDream-I1-Fast', + allowedQualityLevels: [''], + costs: { '1MP': 0.32 }, + }, + { + id: 'togetherai:HiDream-ai/HiDream-I1-Full', + aliases: ['HiDream-ai/HiDream-I1-Full'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'HiDream-ai/HiDream-I1-Full', + allowedQualityLevels: [''], + costs: { '1MP': 0.9 }, + }, + { + id: 'togetherai:Lykon/DreamShaper', + aliases: ['Lykon/DreamShaper'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'Lykon/DreamShaper', + allowedQualityLevels: [''], + costs: { '1MP': 0.06 }, + }, + { + id: 'togetherai:Qwen/Qwen-Image', + aliases: ['Qwen/Qwen-Image'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'Qwen/Qwen-Image', + allowedQualityLevels: [''], + costs: { '1MP': 0.58 }, + }, + { + id: 'togetherai:RunDiffusion/Juggernaut-pro-flux', + aliases: ['RunDiffusion/Juggernaut-pro-flux'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'RunDiffusion/Juggernaut-pro-flux', + allowedQualityLevels: [''], + costs: { '1MP': 0.49 }, + }, + { + id: 'togetherai:Rundiffusion/Juggernaut-Lightning-Flux', + aliases: ['Rundiffusion/Juggernaut-Lightning-Flux'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'Rundiffusion/Juggernaut-Lightning-Flux', + allowedQualityLevels: [''], + costs: { '1MP': 0.17 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-dev', + aliases: ['black-forest-labs/FLUX.1-dev'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-dev', + allowedQualityLevels: [''], + costs: { '1MP': 2.5 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-dev-lora', + aliases: ['black-forest-labs/FLUX.1-dev-lora'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-dev-lora', + allowedQualityLevels: [''], + costs: { '1MP': 2.5 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-kontext-dev', + aliases: ['black-forest-labs/FLUX.1-kontext-dev'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-kontext-dev', + allowedQualityLevels: [''], + costs: { '1MP': 2.5 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-kontext-max', + aliases: ['black-forest-labs/FLUX.1-kontext-max'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-kontext-max', + allowedQualityLevels: [''], + costs: { '1MP': 8 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-kontext-pro', + aliases: ['black-forest-labs/FLUX.1-kontext-pro'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-kontext-pro', + allowedQualityLevels: [''], + costs: { '1MP': 4 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-krea-dev', + aliases: ['black-forest-labs/FLUX.1-krea-dev'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-krea-dev', + allowedQualityLevels: [''], + costs: { '1MP': 2.5 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-pro', + aliases: ['black-forest-labs/FLUX.1-pro'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-pro', + allowedQualityLevels: [''], + costs: { '1MP': 5 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-schnell', + aliases: ['black-forest-labs/FLUX.1-schnell'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-schnell', + allowedQualityLevels: [''], + costs: { '1MP': 0.27 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1.1-pro', + aliases: ['black-forest-labs/FLUX.1.1-pro'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1.1-pro', + allowedQualityLevels: [''], + costs: { '1MP': 4 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.2-pro', + aliases: ['black-forest-labs/FLUX.2-pro'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.2-pro', + allowedQualityLevels: [''], + costs: { '1MP': 3 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.2-flex', + aliases: ['black-forest-labs/FLUX.2-flex'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.2-flex', + allowedQualityLevels: [''], + costs: { '1MP': 3 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.2-dev', + aliases: ['black-forest-labs/FLUX.2-dev'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.2-dev', + allowedQualityLevels: [''], + costs: { '1MP': 3 }, + }, + { + id: 'togetherai:google/flash-image-2.5', + aliases: ['google/flash-image-2.5'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'google/flash-image-2.5', + allowedQualityLevels: ['1K'], + allowedRatios: [ + { w: 1024, h: 1024 }, + { w: 1248, h: 832 }, + { w: 832, h: 1248 }, + { w: 1184, h: 864 }, + { w: 864, h: 1184 }, + { w: 896, h: 1152 }, + { w: 1152, h: 896 }, + { w: 768, h: 1344 }, + { w: 1344, h: 768 }, + { w: 1536, h: 672 }, + { w: 672, h: 1536 }, + + ], + costs: { '1MP': 3.91 }, + }, + { + id: 'togetherai:google/gemini-3-pro-image', + aliases: ['gemini-3-pro-image', 'google/gemini-3-pro-image'], + name: 'gemini-3-pro-image (Together AI)', + costs_currency: 'usd-cents', + index_cost_key: '1MP', + allowedQualityLevels: ['1K', '2K', '4K'], + allowedRatios: [ + { w: 1, h: 1 }, + { w: 2, h: 3 }, + { w: 3, h: 2 }, + { w: 3, h: 4 }, + { w: 4, h: 3 }, + { w: 4, h: 5 }, + { w: 5, h: 4 }, + { w: 9, h: 16 }, + { w: 16, h: 9 }, + { w: 21, h: 9 }, + ], + costs: { '1MP': 13.51 }, + }, + { + id: 'togetherai:google/imagen-4.0-fast', + aliases: ['google/imagen-4.0-fast'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'google/imagen-4.0-fast', + allowedQualityLevels: [''], + costs: { '1MP': 2 }, + }, + { + id: 'togetherai:google/imagen-4.0-preview', + aliases: ['google/imagen-4.0-preview'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'google/imagen-4.0-preview', + allowedQualityLevels: [''], + costs: { '1MP': 4 }, + }, + { + id: 'togetherai:google/imagen-4.0-ultra', + aliases: ['google/imagen-4.0-ultra'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'google/imagen-4.0-ultra', + allowedQualityLevels: [''], + costs: { '1MP': 6.02 }, + }, + { + id: 'togetherai:ideogram/ideogram-3.0', + aliases: ['ideogram/ideogram-3.0'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'ideogram/ideogram-3.0', + allowedQualityLevels: [''], + costs: { '1MP': 6.02 }, + }, + { + id: 'togetherai:stabilityai/stable-diffusion-3-medium', + aliases: ['stabilityai/stable-diffusion-3-medium'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'stabilityai/stable-diffusion-3-medium', + allowedQualityLevels: [''], + costs: { '1MP': 0.19 }, + }, + { + id: 'togetherai:stabilityai/stable-diffusion-xl-base-1.0', + aliases: ['stabilityai/stable-diffusion-xl-base-1.0'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'stabilityai/stable-diffusion-xl-base-1.0', + allowedQualityLevels: [''], + costs: { '1MP': 0.19 }, + }, + { + id: 'togetherai:black-forest-labs/FLUX.1-schnell-Free', + aliases: ['black-forest-labs/FLUX.1-schnell-Free', 'FLUX.1-schnell-Free'], + costs_currency: 'usd-cents', + index_cost_key: '1MP', + name: 'black-forest-labs/FLUX.1-schnell-Free', + allowedQualityLevels: [''], + costs: { '1MP': 0 }, + }, +]; diff --git a/src/backend/src/services/ai/image/providers/types.ts b/src/backend/src/services/ai/image/providers/types.ts new file mode 100644 index 000000000..724b90e75 --- /dev/null +++ b/src/backend/src/services/ai/image/providers/types.ts @@ -0,0 +1,29 @@ +export interface IImageModel { + id: string; + name: string; + description?: string; + version?: string; + costs_currency: string; + index_cost_key?: string; + costs: Record; + allowedQualityLevels?: string[]; + allowedRatios?: { w: number, h: number }[]; + provider?: string; + aliases?: string[]; +} + +export interface IGenerateParams { + prompt: string, + ratio: { w: number, h: number } + model: string, + provider?: string, + test_mode?: boolean + quality?: string, +}; +export interface IImageProvider { + + generate (params: IGenerateParams): Promise; + models (): Promise | IImageModel[]; + getDefaultModel (): string; + +} \ No newline at end of file diff --git a/src/backend/src/services/ai/ocr/MistralOCRService.js b/src/backend/src/services/ai/ocr/MistralOCRService.js index be45b3aa8..aad45f14f 100644 --- a/src/backend/src/services/ai/ocr/MistralOCRService.js +++ b/src/backend/src/services/ai/ocr/MistralOCRService.js @@ -79,12 +79,6 @@ export class MistralOCRService extends BaseService { apiKey: this.config.apiKey, }); - const svc_aiChat = this.services.get('ai-chat'); - svc_aiChat.register_provider({ - service_name: this.service_name, - alias: true, - }); - this.meteringService = this.services.get('meteringService').meteringService; }