feat: add OpenAI support to text-to-speech (#1853)

- added support for OpenAI as a tts provider in `AIInterfaceService` and `PuterAIModule`.
- Updated cost mapping for OpenAI tts models in `openAiCostMap.ts`.
- improved `Txt2SpeechOptions` interface to include new parameters for provider, model, and response format.
- added tests for OpenAI provider in `txt2speech.test.js` to ensure functionality.

src/backend/src/modules/puterai/AIInterfaceService.js

@@ -170,11 +170,14 @@ class AIInterfaceService extends BaseService {
                     description: 'List available voices.',
                     parameters: {
                         engine: { type: 'string', optional: true },
+                        provider: { type: 'string', optional: true },
                     },
                 },
                 list_engines: {
                     description: 'List available TTS engines with pricing information.',
-                    parameters: {},
+                    parameters: {
+                        provider: { type: 'string', optional: true },
+                    },
                     result: { type: 'json' },
                 },
                 synthesize: {
@@ -185,6 +188,10 @@ class AIInterfaceService extends BaseService {
                         language: { type: 'string' },
                         ssml: { type: 'flag' },
                         engine: { type: 'string', optional: true },
+                        model: { type: 'string', optional: true },
+                        response_format: { type: 'string', optional: true },
+                        instructions: { type: 'string', optional: true },
+                        provider: { type: 'string', optional: true },
                     },
                     result_choices: [
                         {

src/backend/src/modules/puterai/OpenAITTSService.js

@@ -0,0 +1,227 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+const { Readable } = require('stream');
+const APIError = require('../../api/APIError');
+const BaseService = require('../../services/BaseService');
+const { TypedValue } = require('../../services/drivers/meta/Runtime');
+const { Context } = require('../../util/context');
+
+const DEFAULT_MODEL = 'gpt-4o-mini-tts';
+const DEFAULT_VOICE = 'alloy';
+const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
+
+const RESPONSE_CONTENT_TYPES = {
+    mp3: 'audio/mpeg',
+    opus: 'audio/opus',
+    aac: 'audio/aac',
+    flac: 'audio/flac',
+    wav: 'audio/wav',
+    pcm: 'audio/pcm',
+};
+
+const OPENAI_TTS_VOICES = [
+    { id: 'alloy', name: 'Alloy' },
+    { id: 'ash', name: 'Ash' },
+    { id: 'ballad', name: 'Ballad' },
+    { id: 'coral', name: 'Coral' },
+    { id: 'echo', name: 'Echo' },
+    { id: 'fable', name: 'Fable' },
+    { id: 'nova', name: 'Nova' },
+    { id: 'onyx', name: 'Onyx' },
+    { id: 'sage', name: 'Sage' },
+    { id: 'shimmer', name: 'Shimmer' },
+];
+
+const OPENAI_TTS_MODELS = [
+    {
+        id: DEFAULT_MODEL,
+        name: 'GPT-4o mini TTS',
+        pricing_per_million_chars: 15,
+    },
+    {
+        id: 'tts-1',
+        name: 'TTS 1',
+        pricing_per_million_chars: 15,
+    },
+    {
+        id: 'tts-1-hd',
+        name: 'TTS 1 HD',
+        pricing_per_million_chars: 30,
+    },
+];
+
+/**
+ * Service that connects the puter-tts driver interface with OpenAI Text-to-Speech API.
+ * Provides voice synthesis, engine discovery, and test-mode behaviour consistent with
+ * the AWS Polly implementation.
+ */
+class OpenAITTSService extends BaseService {
+    /** @type {import('../../services/MeteringService/MeteringService').MeteringService} */
+    get meteringService() {
+        return this.services.get('meteringService').meteringService;
+    }
+
+    static MODULES = {
+        openai: require('openai'),
+    };
+
+    async _init() {
+        let apiKey =
+            this.config?.services?.openai?.apiKey ??
+            this.global_config?.services?.openai?.apiKey;
+
+        if ( !apiKey ) {
+            apiKey =
+                this.config?.openai?.secret_key ??
+                this.global_config.openai?.secret_key;
+
+            if ( apiKey ) {
+                console.warn('The `openai.secret_key` configuration format is deprecated. ' +
+                    'Please use `services.openai.apiKey` instead.');
+            }
+        }
+
+        if ( !apiKey ) {
+            throw new Error('OpenAI API key not configured');
+        }
+
+        this.openai = new this.modules.openai.OpenAI({ apiKey });
+    }
+
+    static IMPLEMENTS = {
+        ['driver-capabilities']: {
+            supports_test_mode(iface, method_name) {
+                return iface === 'puter-tts' && method_name === 'synthesize';
+            },
+        },
+        ['puter-tts']: {
+            async list_voices({ provider } = {}) {
+                if ( provider && provider !== 'openai' ) {
+                    return [];
+                }
+
+                return OPENAI_TTS_VOICES.map((voice) => ({
+                    id: voice.id,
+                    name: voice.name,
+                    language: {
+                        name: 'English',
+                        code: 'en',
+                    },
+                    provider: 'openai',
+                    supported_models: OPENAI_TTS_MODELS.map(model => model.id),
+                }));
+            },
+            async list_engines({ provider } = {}) {
+                if ( provider && provider !== 'openai' ) {
+                    return [];
+                }
+
+                return OPENAI_TTS_MODELS.map(model => ({
+                    id: model.id,
+                    name: model.name,
+                    pricing_per_million_chars: model.pricing_per_million_chars,
+                    provider: 'openai',
+                }));
+            },
+            async synthesize(params) {
+                return this.synthesize(params);
+            },
+        },
+    };
+
+    async synthesize({
+        text,
+        voice,
+        model,
+        response_format,
+        instructions,
+        test_mode,
+    }) {
+        if ( test_mode ) {
+            return new TypedValue({
+                $: 'string:url:web',
+                content_type: 'audio',
+            }, SAMPLE_AUDIO_URL);
+        }
+
+        if ( typeof text !== 'string' || text.trim() === '' ) {
+            throw APIError.create('field_required', null, { key: 'text' });
+        }
+
+        model = model || DEFAULT_MODEL;
+        if ( !OPENAI_TTS_MODELS.find(({ id }) => id === model) ) {
+            throw APIError.create('field_invalid', null, {
+                key: 'model',
+                expected: OPENAI_TTS_MODELS.map(({ id }) => id).join(', '),
+                got: model,
+            });
+        }
+
+        voice = voice || DEFAULT_VOICE;
+        if ( !OPENAI_TTS_VOICES.find(({ id }) => id === voice) ) {
+            throw APIError.create('field_invalid', null, {
+                key: 'voice',
+                expected: OPENAI_TTS_VOICES.map(({ id }) => id).join(', '),
+                got: voice,
+            });
+        }
+
+        const format = response_format || 'mp3';
+        const contentType = RESPONSE_CONTENT_TYPES[format] || RESPONSE_CONTENT_TYPES.mp3;
+
+        const actor = Context.get('actor');
+        const usageType = `openai:${model}:character`;
+
+        const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, text.length);
+        if ( !usageAllowed ) {
+            throw APIError.create('insufficient_funds');
+        }
+
+        const payload = {
+            model,
+            voice,
+            input: text,
+        };
+
+        if ( instructions ) {
+            payload.instructions = instructions;
+        }
+
+        if ( response_format ) {
+            payload.response_format = response_format;
+        }
+
+        const response = await this.openai.audio.speech.create(payload);
+        const arrayBuffer = await response.arrayBuffer();
+        const buffer = Buffer.from(arrayBuffer);
+        const stream = Readable.from(buffer);
+
+        this.meteringService.incrementUsage(actor, usageType, text.length);
+
+        return new TypedValue({
+            $: 'stream',
+            content_type: contentType,
+        }, stream);
+    }
+}
+
+module.exports = {
+    OpenAITTSService,
+};

src/backend/src/modules/puterai/PuterAIModule.js

@@ -64,6 +64,9 @@ class PuterAIModule extends AdvancedBase {
 
             const { OpenAIVideoGenerationService } = require('./OpenAIVideoGenerationService');
             services.registerService('openai-video-generation', OpenAIVideoGenerationService);
+
+            const { OpenAITTSService } = require('./OpenAITTSService');
+            services.registerService('openai-tts', OpenAITTSService);
         }
 
         if ( config?.services?.claude ) {

src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts

@@ -74,4 +74,9 @@ export const OPENAI_COST_MAP = {
     // GPT-4.5 preview
     'openai:gpt-4.5-preview:prompt_tokens': 7500,
     'openai:gpt-4.5-preview:completion_tokens': 15000,
+
+    // Text-to-speech models (per character, microcents)
+    'openai:gpt-4o-mini-tts:character': 1500,
+    'openai:tts-1:character': 1500,
+    'openai:tts-1-hd:character': 3000,
 };

src/puter-js/index.d.ts

@@ -129,7 +129,11 @@ interface Txt2VidOptions {
 interface Txt2SpeechOptions {
     language?: string;
     voice?: string;
-    engine?: 'standard' | 'neural' | 'generative';
+    engine?: 'standard' | 'neural' | 'long-form' | 'generative' | string;
+    provider?: 'aws-polly' | 'openai' | string;
+    model?: 'gpt-4o-mini-tts' | 'tts-1' | 'tts-1-hd' | string;
+    response_format?: 'mp3' | 'opus' | 'aac' | 'flac' | 'wav' | 'pcm' | string;
+    instructions?: string;
 }
 
 interface ChatResponseChunk {

src/puter-js/src/modules/AI.js

@@ -1,5 +1,15 @@
 import * as utils from '../lib/utils.js';
 
+const normalizeTTSProvider = (value) => {
+    if (typeof value !== 'string') {
+        return 'aws-polly';
+    }
+    const lower = value.toLowerCase();
+    if (lower === 'openai') return 'openai';
+    if (lower === 'aws' || lower === 'polly' || lower === 'aws-polly') return 'aws-polly';
+    return value;
+};
+
 class AI{
     /**
      * Creates a new instance with the given authentication token, API origin, and app ID,
@@ -183,23 +193,43 @@ class AI{
             throw { message: 'Text parameter is required', code: 'text_required' };
         }
 
-        // Validate engine if provided
-        if (options.engine) {
-            const validEngines = ['standard', 'neural', 'long-form', 'generative'];
-            if (!validEngines.includes(options.engine)) {
-                throw { message: 'Invalid engine. Must be one of: ' + validEngines.join(', '), code: 'invalid_engine' };
-            }
+        const validEngines = ['standard', 'neural', 'long-form', 'generative'];
+        let provider = normalizeTTSProvider(options.provider);
+
+        if (options.engine && normalizeTTSProvider(options.engine) === 'openai' && !options.provider) {
+            provider = 'openai';
         }
 
-        // Set default values if not provided
-        if (!options.voice) {
-            options.voice = 'Joanna';
-        }
-        if (!options.engine) {
-            options.engine = 'standard';
-        }
-        if (!options.language) {
-            options.language = 'en-US';
+        if (provider === 'openai') {
+            if (!options.model && typeof options.engine === 'string') {
+                options.model = options.engine;
+            }
+            if (!options.voice) {
+                options.voice = 'alloy';
+            }
+            if (!options.model) {
+                options.model = 'gpt-4o-mini-tts';
+            }
+            if (!options.response_format) {
+                options.response_format = 'mp3';
+            }
+            delete options.engine;
+        } else {
+            provider = 'aws-polly';
+
+            if (options.engine && !validEngines.includes(options.engine)) {
+                throw { message: 'Invalid engine. Must be one of: ' + validEngines.join(', '), code: 'invalid_engine' };
+            }
+
+            if (!options.voice) {
+                options.voice = 'Joanna';
+            }
+            if (!options.engine) {
+                options.engine = 'standard';
+            }
+            if (!options.language) {
+                options.language = 'en-US';
+            }
         }
 
         // check input size
@@ -214,12 +244,28 @@ class AI{
                 break;
             }
         }
-    
-        return await utils.make_driver_method(['source'], 'puter-tts', 'aws-polly', 'synthesize', {
+
+        const driverName = provider === 'openai' ? 'openai-tts' : 'aws-polly';
+
+        return await utils.make_driver_method(['source'], 'puter-tts', driverName, 'synthesize', {
             responseType: 'blob',
             test_mode: testMode ?? false,
             transform: async (result) => {
-                const url = await utils.blob_to_url(result);
+                let url;
+                if (typeof result === 'string') {
+                    url = result;
+                } else if (result instanceof Blob) {
+                    url = await utils.blob_to_url(result);
+                } else if (result instanceof ArrayBuffer) {
+                    const blob = new Blob([result]);
+                    url = await utils.blob_to_url(blob);
+                } else if (result && typeof result === 'object' && typeof result.arrayBuffer === 'function') {
+                    const arrayBuffer = await result.arrayBuffer();
+                    const blob = new Blob([arrayBuffer], { type: result.type || undefined });
+                    url = await utils.blob_to_url(blob);
+                } else {
+                    throw { message: 'Unexpected audio response format', code: 'invalid_audio_response' };
+                }
                 const audio = new Audio(url);
                 audio.toString = () => url;
                 audio.valueOf = () => url;
@@ -234,10 +280,27 @@ class AI{
          * List available TTS engines with pricing information
          * @returns {Promise<Array>} Array of available engines
          */
-        listEngines: async () => {
-            return await utils.make_driver_method(['source'], 'puter-tts', 'aws-polly', 'list_engines', {
+        listEngines: async (options = {}) => {
+            let provider = 'aws-polly';
+            let params = {};
+
+            if (typeof options === 'string') {
+                provider = normalizeTTSProvider(options);
+            } else if (options && typeof options === 'object') {
+                provider = normalizeTTSProvider(options.provider) || provider;
+                params = { ...options };
+                delete params.provider;
+            }
+
+            if (provider === 'openai') {
+                params.provider = 'openai';
+            }
+
+            const driverName = provider === 'openai' ? 'openai-tts' : 'aws-polly';
+
+            return await utils.make_driver_method(['source'], 'puter-tts', driverName, 'list_engines', {
                 responseType: 'text',
-            }).call(this, {});
+            }).call(this, params);
         },
 
         /**
@@ -245,13 +308,26 @@ class AI{
          * @param {string} [engine] - Optional engine filter
          * @returns {Promise<Array>} Array of available voices
          */
-        listVoices: async (engine) => {
-            const params = {};
-            if (engine) {
-                params.engine = engine;
+        listVoices: async (options) => {
+            let provider = 'aws-polly';
+            let params = {};
+
+            if (typeof options === 'string') {
+                params.engine = options;
+            } else if (options && typeof options === 'object') {
+                provider = normalizeTTSProvider(options.provider) || provider;
+                params = { ...options };
+                delete params.provider;
             }
 
-            return utils.make_driver_method(['source'], 'puter-tts', 'aws-polly', 'list_voices', {
+            if (provider === 'openai') {
+                params.provider = 'openai';
+                delete params.engine;
+            }
+
+            const driverName = provider === 'openai' ? 'openai-tts' : 'aws-polly';
+
+            return utils.make_driver_method(['source'], 'puter-tts', driverName, 'list_voices', {
                 responseType: 'text',
             }).call(this, params);
         }

src/puter-js/test/txt2speech.test.js

@@ -135,6 +135,28 @@ const testTxt2SpeechWithTestModeCore = async function() {
     assert(valueOfValue === srcValue, `valueOf() should return the same as src in test mode. valueOf(): ${valueOfValue}, src: ${srcValue}`);
 };
 
+const testTxt2SpeechWithOpenAIProviderCore = async function() {
+    // Test OpenAI-based text-to-speech using test mode to avoid live requests
+    const result = await puter.ai.txt2speech("Hello, this is an OpenAI provider test.", { provider: "openai", voice: "alloy" }, true);
+
+    assert(result instanceof Audio, "txt2speech should return an Audio object for OpenAI provider");
+    assert(result !== null, "txt2speech should not return null for OpenAI provider");
+
+    const toStringValue = result.toString();
+    const valueOfValue = result.valueOf();
+    const srcValue = result.src;
+
+    assert(typeof toStringValue === 'string', "toString() should return a string for OpenAI provider");
+    assert(typeof valueOfValue === 'string', "valueOf() should return a string for OpenAI provider");
+    assert(typeof srcValue === 'string', "src should be a string for OpenAI provider");
+    assert(toStringValue.length > 0, "toString() should not be empty for OpenAI provider");
+    assert(valueOfValue.length > 0, "valueOf() should not be empty for OpenAI provider");
+    assert(srcValue.length > 0, "src should not be empty for OpenAI provider");
+
+    assert(toStringValue === srcValue, "toString() should match src for OpenAI provider");
+    assert(valueOfValue === srcValue, "valueOf() should match src for OpenAI provider");
+};
+
 // Export test functions
 window.txt2speechTests = [
     {
@@ -174,5 +196,18 @@ window.txt2speechTests = [
                 fail("testTxt2SpeechWithTestMode failed:", error);
             }
         }
+    },
+
+    {
+        name: "testTxt2SpeechWithOpenAIProvider",
+        description: "Test text-to-speech using the OpenAI provider in test mode",
+        test: async function() {
+            try {
+                await testTxt2SpeechWithOpenAIProviderCore();
+                pass("testTxt2SpeechWithOpenAIProvider passed");
+            } catch (error) {
+                fail("testTxt2SpeechWithOpenAIProvider failed:", error);
+            }
+        }
     }
-]; 
+];