From 861aeddf3ec1c90b6ba4fdef3df71ef8bf34507d Mon Sep 17 00:00:00 2001 From: Nariman Jelveh Date: Tue, 28 Oct 2025 18:51:04 -0700 Subject: [PATCH] feat: add OpenAI support to text-to-speech (#1853) - added support for OpenAI as a tts provider in `AIInterfaceService` and `PuterAIModule`. - Updated cost mapping for OpenAI tts models in `openAiCostMap.ts`. - improved `Txt2SpeechOptions` interface to include new parameters for provider, model, and response format. - added tests for OpenAI provider in `txt2speech.test.js` to ensure functionality. --- .../src/modules/puterai/AIInterfaceService.js | 9 +- .../src/modules/puterai/OpenAITTSService.js | 227 ++++++++++++++++++ .../src/modules/puterai/PuterAIModule.js | 3 + .../MeteringService/costMaps/openAiCostMap.ts | 5 + src/puter-js/index.d.ts | 6 +- src/puter-js/src/modules/AI.js | 128 ++++++++-- src/puter-js/test/txt2speech.test.js | 37 ++- 7 files changed, 386 insertions(+), 29 deletions(-) create mode 100644 src/backend/src/modules/puterai/OpenAITTSService.js diff --git a/src/backend/src/modules/puterai/AIInterfaceService.js b/src/backend/src/modules/puterai/AIInterfaceService.js index b7c73b291..bf635a2e4 100644 --- a/src/backend/src/modules/puterai/AIInterfaceService.js +++ b/src/backend/src/modules/puterai/AIInterfaceService.js @@ -170,11 +170,14 @@ class AIInterfaceService extends BaseService { description: 'List available voices.', parameters: { engine: { type: 'string', optional: true }, + provider: { type: 'string', optional: true }, }, }, list_engines: { description: 'List available TTS engines with pricing information.', - parameters: {}, + parameters: { + provider: { type: 'string', optional: true }, + }, result: { type: 'json' }, }, synthesize: { @@ -185,6 +188,10 @@ class AIInterfaceService extends BaseService { language: { type: 'string' }, ssml: { type: 'flag' }, engine: { type: 'string', optional: true }, + model: { type: 'string', optional: true }, + response_format: { type: 'string', optional: true }, + instructions: { type: 'string', optional: true }, + provider: { type: 'string', optional: true }, }, result_choices: [ { diff --git a/src/backend/src/modules/puterai/OpenAITTSService.js b/src/backend/src/modules/puterai/OpenAITTSService.js new file mode 100644 index 000000000..40b27271d --- /dev/null +++ b/src/backend/src/modules/puterai/OpenAITTSService.js @@ -0,0 +1,227 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +const { Readable } = require('stream'); +const APIError = require('../../api/APIError'); +const BaseService = require('../../services/BaseService'); +const { TypedValue } = require('../../services/drivers/meta/Runtime'); +const { Context } = require('../../util/context'); + +const DEFAULT_MODEL = 'gpt-4o-mini-tts'; +const DEFAULT_VOICE = 'alloy'; +const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3'; + +const RESPONSE_CONTENT_TYPES = { + mp3: 'audio/mpeg', + opus: 'audio/opus', + aac: 'audio/aac', + flac: 'audio/flac', + wav: 'audio/wav', + pcm: 'audio/pcm', +}; + +const OPENAI_TTS_VOICES = [ + { id: 'alloy', name: 'Alloy' }, + { id: 'ash', name: 'Ash' }, + { id: 'ballad', name: 'Ballad' }, + { id: 'coral', name: 'Coral' }, + { id: 'echo', name: 'Echo' }, + { id: 'fable', name: 'Fable' }, + { id: 'nova', name: 'Nova' }, + { id: 'onyx', name: 'Onyx' }, + { id: 'sage', name: 'Sage' }, + { id: 'shimmer', name: 'Shimmer' }, +]; + +const OPENAI_TTS_MODELS = [ + { + id: DEFAULT_MODEL, + name: 'GPT-4o mini TTS', + pricing_per_million_chars: 15, + }, + { + id: 'tts-1', + name: 'TTS 1', + pricing_per_million_chars: 15, + }, + { + id: 'tts-1-hd', + name: 'TTS 1 HD', + pricing_per_million_chars: 30, + }, +]; + +/** + * Service that connects the puter-tts driver interface with OpenAI Text-to-Speech API. + * Provides voice synthesis, engine discovery, and test-mode behaviour consistent with + * the AWS Polly implementation. + */ +class OpenAITTSService extends BaseService { + /** @type {import('../../services/MeteringService/MeteringService').MeteringService} */ + get meteringService() { + return this.services.get('meteringService').meteringService; + } + + static MODULES = { + openai: require('openai'), + }; + + async _init() { + let apiKey = + this.config?.services?.openai?.apiKey ?? + this.global_config?.services?.openai?.apiKey; + + if ( !apiKey ) { + apiKey = + this.config?.openai?.secret_key ?? + this.global_config.openai?.secret_key; + + if ( apiKey ) { + console.warn('The `openai.secret_key` configuration format is deprecated. ' + + 'Please use `services.openai.apiKey` instead.'); + } + } + + if ( !apiKey ) { + throw new Error('OpenAI API key not configured'); + } + + this.openai = new this.modules.openai.OpenAI({ apiKey }); + } + + static IMPLEMENTS = { + ['driver-capabilities']: { + supports_test_mode(iface, method_name) { + return iface === 'puter-tts' && method_name === 'synthesize'; + }, + }, + ['puter-tts']: { + async list_voices({ provider } = {}) { + if ( provider && provider !== 'openai' ) { + return []; + } + + return OPENAI_TTS_VOICES.map((voice) => ({ + id: voice.id, + name: voice.name, + language: { + name: 'English', + code: 'en', + }, + provider: 'openai', + supported_models: OPENAI_TTS_MODELS.map(model => model.id), + })); + }, + async list_engines({ provider } = {}) { + if ( provider && provider !== 'openai' ) { + return []; + } + + return OPENAI_TTS_MODELS.map(model => ({ + id: model.id, + name: model.name, + pricing_per_million_chars: model.pricing_per_million_chars, + provider: 'openai', + })); + }, + async synthesize(params) { + return this.synthesize(params); + }, + }, + }; + + async synthesize({ + text, + voice, + model, + response_format, + instructions, + test_mode, + }) { + if ( test_mode ) { + return new TypedValue({ + $: 'string:url:web', + content_type: 'audio', + }, SAMPLE_AUDIO_URL); + } + + if ( typeof text !== 'string' || text.trim() === '' ) { + throw APIError.create('field_required', null, { key: 'text' }); + } + + model = model || DEFAULT_MODEL; + if ( !OPENAI_TTS_MODELS.find(({ id }) => id === model) ) { + throw APIError.create('field_invalid', null, { + key: 'model', + expected: OPENAI_TTS_MODELS.map(({ id }) => id).join(', '), + got: model, + }); + } + + voice = voice || DEFAULT_VOICE; + if ( !OPENAI_TTS_VOICES.find(({ id }) => id === voice) ) { + throw APIError.create('field_invalid', null, { + key: 'voice', + expected: OPENAI_TTS_VOICES.map(({ id }) => id).join(', '), + got: voice, + }); + } + + const format = response_format || 'mp3'; + const contentType = RESPONSE_CONTENT_TYPES[format] || RESPONSE_CONTENT_TYPES.mp3; + + const actor = Context.get('actor'); + const usageType = `openai:${model}:character`; + + const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, text.length); + if ( !usageAllowed ) { + throw APIError.create('insufficient_funds'); + } + + const payload = { + model, + voice, + input: text, + }; + + if ( instructions ) { + payload.instructions = instructions; + } + + if ( response_format ) { + payload.response_format = response_format; + } + + const response = await this.openai.audio.speech.create(payload); + const arrayBuffer = await response.arrayBuffer(); + const buffer = Buffer.from(arrayBuffer); + const stream = Readable.from(buffer); + + this.meteringService.incrementUsage(actor, usageType, text.length); + + return new TypedValue({ + $: 'stream', + content_type: contentType, + }, stream); + } +} + +module.exports = { + OpenAITTSService, +}; diff --git a/src/backend/src/modules/puterai/PuterAIModule.js b/src/backend/src/modules/puterai/PuterAIModule.js index af5a96ff6..812ad3d26 100644 --- a/src/backend/src/modules/puterai/PuterAIModule.js +++ b/src/backend/src/modules/puterai/PuterAIModule.js @@ -64,6 +64,9 @@ class PuterAIModule extends AdvancedBase { const { OpenAIVideoGenerationService } = require('./OpenAIVideoGenerationService'); services.registerService('openai-video-generation', OpenAIVideoGenerationService); + + const { OpenAITTSService } = require('./OpenAITTSService'); + services.registerService('openai-tts', OpenAITTSService); } if ( config?.services?.claude ) { diff --git a/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts b/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts index a2db853f1..60635c903 100644 --- a/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts +++ b/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts @@ -74,4 +74,9 @@ export const OPENAI_COST_MAP = { // GPT-4.5 preview 'openai:gpt-4.5-preview:prompt_tokens': 7500, 'openai:gpt-4.5-preview:completion_tokens': 15000, + + // Text-to-speech models (per character, microcents) + 'openai:gpt-4o-mini-tts:character': 1500, + 'openai:tts-1:character': 1500, + 'openai:tts-1-hd:character': 3000, }; diff --git a/src/puter-js/index.d.ts b/src/puter-js/index.d.ts index aaac57afa..f7cc572bb 100644 --- a/src/puter-js/index.d.ts +++ b/src/puter-js/index.d.ts @@ -129,7 +129,11 @@ interface Txt2VidOptions { interface Txt2SpeechOptions { language?: string; voice?: string; - engine?: 'standard' | 'neural' | 'generative'; + engine?: 'standard' | 'neural' | 'long-form' | 'generative' | string; + provider?: 'aws-polly' | 'openai' | string; + model?: 'gpt-4o-mini-tts' | 'tts-1' | 'tts-1-hd' | string; + response_format?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm' | string; + instructions?: string; } interface ChatResponseChunk { diff --git a/src/puter-js/src/modules/AI.js b/src/puter-js/src/modules/AI.js index ce37534a3..b06820ad6 100644 --- a/src/puter-js/src/modules/AI.js +++ b/src/puter-js/src/modules/AI.js @@ -1,5 +1,15 @@ import * as utils from '../lib/utils.js'; +const normalizeTTSProvider = (value) => { + if (typeof value !== 'string') { + return 'aws-polly'; + } + const lower = value.toLowerCase(); + if (lower === 'openai') return 'openai'; + if (lower === 'aws' || lower === 'polly' || lower === 'aws-polly') return 'aws-polly'; + return value; +}; + class AI{ /** * Creates a new instance with the given authentication token, API origin, and app ID, @@ -183,23 +193,43 @@ class AI{ throw { message: 'Text parameter is required', code: 'text_required' }; } - // Validate engine if provided - if (options.engine) { - const validEngines = ['standard', 'neural', 'long-form', 'generative']; - if (!validEngines.includes(options.engine)) { - throw { message: 'Invalid engine. Must be one of: ' + validEngines.join(', '), code: 'invalid_engine' }; - } + const validEngines = ['standard', 'neural', 'long-form', 'generative']; + let provider = normalizeTTSProvider(options.provider); + + if (options.engine && normalizeTTSProvider(options.engine) === 'openai' && !options.provider) { + provider = 'openai'; } - // Set default values if not provided - if (!options.voice) { - options.voice = 'Joanna'; - } - if (!options.engine) { - options.engine = 'standard'; - } - if (!options.language) { - options.language = 'en-US'; + if (provider === 'openai') { + if (!options.model && typeof options.engine === 'string') { + options.model = options.engine; + } + if (!options.voice) { + options.voice = 'alloy'; + } + if (!options.model) { + options.model = 'gpt-4o-mini-tts'; + } + if (!options.response_format) { + options.response_format = 'mp3'; + } + delete options.engine; + } else { + provider = 'aws-polly'; + + if (options.engine && !validEngines.includes(options.engine)) { + throw { message: 'Invalid engine. Must be one of: ' + validEngines.join(', '), code: 'invalid_engine' }; + } + + if (!options.voice) { + options.voice = 'Joanna'; + } + if (!options.engine) { + options.engine = 'standard'; + } + if (!options.language) { + options.language = 'en-US'; + } } // check input size @@ -214,12 +244,28 @@ class AI{ break; } } - - return await utils.make_driver_method(['source'], 'puter-tts', 'aws-polly', 'synthesize', { + + const driverName = provider === 'openai' ? 'openai-tts' : 'aws-polly'; + + return await utils.make_driver_method(['source'], 'puter-tts', driverName, 'synthesize', { responseType: 'blob', test_mode: testMode ?? false, transform: async (result) => { - const url = await utils.blob_to_url(result); + let url; + if (typeof result === 'string') { + url = result; + } else if (result instanceof Blob) { + url = await utils.blob_to_url(result); + } else if (result instanceof ArrayBuffer) { + const blob = new Blob([result]); + url = await utils.blob_to_url(blob); + } else if (result && typeof result === 'object' && typeof result.arrayBuffer === 'function') { + const arrayBuffer = await result.arrayBuffer(); + const blob = new Blob([arrayBuffer], { type: result.type || undefined }); + url = await utils.blob_to_url(blob); + } else { + throw { message: 'Unexpected audio response format', code: 'invalid_audio_response' }; + } const audio = new Audio(url); audio.toString = () => url; audio.valueOf = () => url; @@ -234,10 +280,27 @@ class AI{ * List available TTS engines with pricing information * @returns {Promise} Array of available engines */ - listEngines: async () => { - return await utils.make_driver_method(['source'], 'puter-tts', 'aws-polly', 'list_engines', { + listEngines: async (options = {}) => { + let provider = 'aws-polly'; + let params = {}; + + if (typeof options === 'string') { + provider = normalizeTTSProvider(options); + } else if (options && typeof options === 'object') { + provider = normalizeTTSProvider(options.provider) || provider; + params = { ...options }; + delete params.provider; + } + + if (provider === 'openai') { + params.provider = 'openai'; + } + + const driverName = provider === 'openai' ? 'openai-tts' : 'aws-polly'; + + return await utils.make_driver_method(['source'], 'puter-tts', driverName, 'list_engines', { responseType: 'text', - }).call(this, {}); + }).call(this, params); }, /** @@ -245,13 +308,26 @@ class AI{ * @param {string} [engine] - Optional engine filter * @returns {Promise} Array of available voices */ - listVoices: async (engine) => { - const params = {}; - if (engine) { - params.engine = engine; + listVoices: async (options) => { + let provider = 'aws-polly'; + let params = {}; + + if (typeof options === 'string') { + params.engine = options; + } else if (options && typeof options === 'object') { + provider = normalizeTTSProvider(options.provider) || provider; + params = { ...options }; + delete params.provider; } - return utils.make_driver_method(['source'], 'puter-tts', 'aws-polly', 'list_voices', { + if (provider === 'openai') { + params.provider = 'openai'; + delete params.engine; + } + + const driverName = provider === 'openai' ? 'openai-tts' : 'aws-polly'; + + return utils.make_driver_method(['source'], 'puter-tts', driverName, 'list_voices', { responseType: 'text', }).call(this, params); } diff --git a/src/puter-js/test/txt2speech.test.js b/src/puter-js/test/txt2speech.test.js index ac047948d..3dde1f950 100644 --- a/src/puter-js/test/txt2speech.test.js +++ b/src/puter-js/test/txt2speech.test.js @@ -135,6 +135,28 @@ const testTxt2SpeechWithTestModeCore = async function() { assert(valueOfValue === srcValue, `valueOf() should return the same as src in test mode. valueOf(): ${valueOfValue}, src: ${srcValue}`); }; +const testTxt2SpeechWithOpenAIProviderCore = async function() { + // Test OpenAI-based text-to-speech using test mode to avoid live requests + const result = await puter.ai.txt2speech("Hello, this is an OpenAI provider test.", { provider: "openai", voice: "alloy" }, true); + + assert(result instanceof Audio, "txt2speech should return an Audio object for OpenAI provider"); + assert(result !== null, "txt2speech should not return null for OpenAI provider"); + + const toStringValue = result.toString(); + const valueOfValue = result.valueOf(); + const srcValue = result.src; + + assert(typeof toStringValue === 'string', "toString() should return a string for OpenAI provider"); + assert(typeof valueOfValue === 'string', "valueOf() should return a string for OpenAI provider"); + assert(typeof srcValue === 'string', "src should be a string for OpenAI provider"); + assert(toStringValue.length > 0, "toString() should not be empty for OpenAI provider"); + assert(valueOfValue.length > 0, "valueOf() should not be empty for OpenAI provider"); + assert(srcValue.length > 0, "src should not be empty for OpenAI provider"); + + assert(toStringValue === srcValue, "toString() should match src for OpenAI provider"); + assert(valueOfValue === srcValue, "valueOf() should match src for OpenAI provider"); +}; + // Export test functions window.txt2speechTests = [ { @@ -174,5 +196,18 @@ window.txt2speechTests = [ fail("testTxt2SpeechWithTestMode failed:", error); } } + }, + + { + name: "testTxt2SpeechWithOpenAIProvider", + description: "Test text-to-speech using the OpenAI provider in test mode", + test: async function() { + try { + await testTxt2SpeechWithOpenAIProviderCore(); + pass("testTxt2SpeechWithOpenAIProvider passed"); + } catch (error) { + fail("testTxt2SpeechWithOpenAIProvider failed:", error); + } + } } -]; \ No newline at end of file +];