diff --git a/package-lock.json b/package-lock.json
index 2eeef9af7..6b07030a5 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -2316,10 +2316,6 @@
"resolved": "src/phoenix",
"link": true
},
- "node_modules/@heyputer/puter-api-test": {
- "resolved": "tools/api-tester",
- "link": true
- },
"node_modules/@heyputer/puter-wisp": {
"resolved": "src/puter-wisp",
"link": true
@@ -14198,6 +14194,101 @@
"integrity": "sha512-5vQEh3y+DG/lMPM0mCGPDnyV8chYg/g7rl6v3Gd8WMF9S429ox3Xk8qrk174kWhG767KQMqqxLD1WnGd77hiew==",
"license": "MIT"
},
+ "node_modules/music-metadata": {
+ "version": "7.14.0",
+ "resolved": "https://registry.npmjs.org/music-metadata/-/music-metadata-7.14.0.tgz",
+ "integrity": "sha512-xrm3w7SV0Wk+OythZcSbaI8mcr/KHd0knJieu8bVpaPfMv/Agz5EooCAPz3OR5hbYMiUG6dgAPKZKnMzV+3amA==",
+ "license": "MIT",
+ "dependencies": {
+ "@tokenizer/token": "^0.3.0",
+ "content-type": "^1.0.5",
+ "debug": "^4.3.4",
+ "file-type": "^16.5.4",
+ "media-typer": "^1.1.0",
+ "strtok3": "^6.3.0",
+ "token-types": "^4.2.1"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Borewit"
+ }
+ },
+ "node_modules/music-metadata/node_modules/file-type": {
+ "version": "16.5.4",
+ "resolved": "https://registry.npmjs.org/file-type/-/file-type-16.5.4.tgz",
+ "integrity": "sha512-/yFHK0aGjFEgDJjEKP0pWCplsPFPhwyfwevf/pVxiN0tmE4L9LmwWxWukdJSHdoCli4VgQLehjJtwQBnqmsKcw==",
+ "license": "MIT",
+ "dependencies": {
+ "readable-web-to-node-stream": "^3.0.0",
+ "strtok3": "^6.2.4",
+ "token-types": "^4.1.1"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "url": "https://github.com/sindresorhus/file-type?sponsor=1"
+ }
+ },
+ "node_modules/music-metadata/node_modules/media-typer": {
+ "version": "1.1.0",
+ "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-1.1.0.tgz",
+ "integrity": "sha512-aisnrDP4GNe06UcKFnV5bfMNPBUw4jsLGaWwWfnH3v02GnBuXX2MCVn5RbrWo0j3pczUilYblq7fQ7Nw2t5XKw==",
+ "license": "MIT",
+ "engines": {
+ "node": ">= 0.8"
+ }
+ },
+ "node_modules/music-metadata/node_modules/peek-readable": {
+ "version": "4.1.0",
+ "resolved": "https://registry.npmjs.org/peek-readable/-/peek-readable-4.1.0.tgz",
+ "integrity": "sha512-ZI3LnwUv5nOGbQzD9c2iDG6toheuXSZP5esSHBjopsXH4dg19soufvpUGA3uohi5anFtGb2lhAVdHzH6R/Evvg==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=8"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Borewit"
+ }
+ },
+ "node_modules/music-metadata/node_modules/strtok3": {
+ "version": "6.3.0",
+ "resolved": "https://registry.npmjs.org/strtok3/-/strtok3-6.3.0.tgz",
+ "integrity": "sha512-fZtbhtvI9I48xDSywd/somNqgUHl2L2cstmXCCif0itOf96jeW18MBSyrLuNicYQVkvpOxkZtkzujiTJ9LW5Jw==",
+ "license": "MIT",
+ "dependencies": {
+ "@tokenizer/token": "^0.3.0",
+ "peek-readable": "^4.1.0"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Borewit"
+ }
+ },
+ "node_modules/music-metadata/node_modules/token-types": {
+ "version": "4.2.1",
+ "resolved": "https://registry.npmjs.org/token-types/-/token-types-4.2.1.tgz",
+ "integrity": "sha512-6udB24Q737UD/SDsKAHI9FCRP7Bqc9D/MQUV02ORQg5iskjtLJlZJNdN4kKtcdtwCeWIwIHDGaUsTsCCAa8sFQ==",
+ "license": "MIT",
+ "dependencies": {
+ "@tokenizer/token": "^0.3.0",
+ "ieee754": "^1.2.1"
+ },
+ "engines": {
+ "node": ">=10"
+ },
+ "funding": {
+ "type": "github",
+ "url": "https://github.com/sponsors/Borewit"
+ }
+ },
"node_modules/nan": {
"version": "2.23.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.23.0.tgz",
@@ -19240,6 +19331,7 @@
"multer": "^2.0.2",
"multi-progress": "^4.0.0",
"murmurhash": "^2.0.1",
+ "music-metadata": "^7.14.0",
"nodemailer": "^6.9.3",
"on-finished": "^2.4.1",
"openai": "^6.7.0",
@@ -20286,6 +20378,7 @@
"tools/api-tester": {
"name": "@heyputer/puter-api-test",
"version": "0.1.0",
+ "extraneous": true,
"license": "UNLICENSED",
"dependencies": {
"axios": "^1.12.0",
@@ -20294,84 +20387,6 @@
"yaml": "^2.3.1"
}
},
- "tools/api-tester/node_modules/assertion-error": {
- "version": "1.1.0",
- "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz",
- "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==",
- "license": "MIT",
- "engines": {
- "node": "*"
- }
- },
- "tools/api-tester/node_modules/chai": {
- "version": "4.5.0",
- "resolved": "https://registry.npmjs.org/chai/-/chai-4.5.0.tgz",
- "integrity": "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw==",
- "license": "MIT",
- "dependencies": {
- "assertion-error": "^1.1.0",
- "check-error": "^1.0.3",
- "deep-eql": "^4.1.3",
- "get-func-name": "^2.0.2",
- "loupe": "^2.3.6",
- "pathval": "^1.1.1",
- "type-detect": "^4.1.0"
- },
- "engines": {
- "node": ">=4"
- }
- },
- "tools/api-tester/node_modules/check-error": {
- "version": "1.0.3",
- "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.3.tgz",
- "integrity": "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg==",
- "license": "MIT",
- "dependencies": {
- "get-func-name": "^2.0.2"
- },
- "engines": {
- "node": "*"
- }
- },
- "tools/api-tester/node_modules/deep-eql": {
- "version": "4.1.4",
- "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-4.1.4.tgz",
- "integrity": "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg==",
- "license": "MIT",
- "dependencies": {
- "type-detect": "^4.0.0"
- },
- "engines": {
- "node": ">=6"
- }
- },
- "tools/api-tester/node_modules/loupe": {
- "version": "2.3.7",
- "resolved": "https://registry.npmjs.org/loupe/-/loupe-2.3.7.tgz",
- "integrity": "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA==",
- "license": "MIT",
- "dependencies": {
- "get-func-name": "^2.0.1"
- }
- },
- "tools/api-tester/node_modules/pathval": {
- "version": "1.1.1",
- "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.1.tgz",
- "integrity": "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ==",
- "license": "MIT",
- "engines": {
- "node": "*"
- }
- },
- "tools/api-tester/node_modules/type-detect": {
- "version": "4.1.0",
- "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.1.0.tgz",
- "integrity": "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw==",
- "license": "MIT",
- "engines": {
- "node": ">=4"
- }
- },
"tools/comment-parser": {
"version": "1.0.0",
"license": "AGPL-3.0-only",
diff --git a/src/backend/package.json b/src/backend/package.json
index f9670853a..eae43a922 100644
--- a/src/backend/package.json
+++ b/src/backend/package.json
@@ -61,6 +61,7 @@
"multer": "^2.0.2",
"multi-progress": "^4.0.0",
"murmurhash": "^2.0.1",
+ "music-metadata": "^7.14.0",
"nodemailer": "^6.9.3",
"on-finished": "^2.4.1",
"openai": "^6.7.0",
diff --git a/src/backend/src/data/hardcoded-permissions.js b/src/backend/src/data/hardcoded-permissions.js
index 171800911..984c6456d 100644
--- a/src/backend/src/data/hardcoded-permissions.js
+++ b/src/backend/src/data/hardcoded-permissions.js
@@ -25,6 +25,7 @@ const default_implicit_user_app_permissions = {
'driver:puter-image-generation': {},
'driver:puter-video-generation': {},
'driver:puter-tts': {},
+ 'driver:puter-speech2txt': {},
'driver:puter-apps': {},
'driver:puter-subdomains': {},
'driver:temp-email': {},
@@ -60,6 +61,8 @@ const implicit_user_app_permissions = [
'driver:puter-chat-completion:complete': {},
'driver:puter-image-generation:generate': {},
'driver:puter-video-generation:generate': {},
+ 'driver:puter-speech2txt:transcribe': {},
+ 'driver:puter-speech2txt:translate': {},
'driver:puter-analytics:create_trace': {},
'driver:puter-analytics:record': {},
},
diff --git a/src/backend/src/modules/puterai/AIInterfaceService.js b/src/backend/src/modules/puterai/AIInterfaceService.js
index bf635a2e4..41abc73db 100644
--- a/src/backend/src/modules/puterai/AIInterfaceService.js
+++ b/src/backend/src/modules/puterai/AIInterfaceService.js
@@ -205,6 +205,50 @@ class AIInterfaceService extends BaseService {
},
}
})
+
+ col_interfaces.set('puter-speech2txt', {
+ description: 'Speech to text transcription and translation.',
+ methods: {
+ list_models: {
+ description: 'List available speech-to-text models.',
+ result: { type: 'json' },
+ },
+ transcribe: {
+ description: 'Transcribe audio into text.',
+ parameters: {
+ file: { type: 'file' },
+ model: { type: 'string', optional: true },
+ response_format: { type: 'string', optional: true },
+ language: { type: 'string', optional: true },
+ prompt: { type: 'string', optional: true },
+ temperature: { type: 'number', optional: true },
+ logprobs: { type: 'flag', optional: true },
+ timestamp_granularities: { type: 'json', optional: true },
+ stream: { type: 'flag', optional: true },
+ chunking_strategy: { type: 'string', optional: true },
+ known_speaker_names: { type: 'json', optional: true },
+ known_speaker_references: { type: 'json', optional: true },
+ extra_body: { type: 'json', optional: true },
+ },
+ result: { type: 'json' },
+ },
+ translate: {
+ description: 'Translate audio into English text.',
+ parameters: {
+ file: { type: 'file' },
+ model: { type: 'string', optional: true },
+ response_format: { type: 'string', optional: true },
+ prompt: { type: 'string', optional: true },
+ temperature: { type: 'number', optional: true },
+ logprobs: { type: 'flag', optional: true },
+ timestamp_granularities: { type: 'json', optional: true },
+ stream: { type: 'flag', optional: true },
+ extra_body: { type: 'json', optional: true },
+ },
+ result: { type: 'json' },
+ },
+ },
+ });
}
}
diff --git a/src/backend/src/modules/puterai/OpenAISpeechToTextService.js b/src/backend/src/modules/puterai/OpenAISpeechToTextService.js
new file mode 100644
index 000000000..f53d50b2c
--- /dev/null
+++ b/src/backend/src/modules/puterai/OpenAISpeechToTextService.js
@@ -0,0 +1,403 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
+
+const BaseService = require('../../services/BaseService');
+const APIError = require('../../api/APIError');
+const { Context } = require('../../util/context');
+const { FileFacade } = require('../../services/drivers/FileFacade');
+
+const MAX_AUDIO_FILE_SIZE = 25 * 1024 * 1024; // 25 MB per OpenAI limits
+const DEFAULT_TRANSCRIBE_MODEL = 'gpt-4o-mini-transcribe';
+const DEFAULT_TRANSLATE_MODEL = 'whisper-1';
+const SAMPLE_TRANSCRIPT = {
+ text: 'Hello! This is a sample transcription returned while test mode is enabled.',
+ language: 'en',
+ duration_seconds: 2,
+ words: [
+ { start: 0.0, end: 0.5, text: 'Hello' },
+ { start: 0.5, end: 0.9, text: '!' },
+ { start: 1.1, end: 2.0, text: 'This is a sample transcription.' },
+ ],
+};
+
+const TRANSCRIPTION_MODEL_CAPABILITIES = {
+ 'gpt-4o-mini-transcribe': {
+ canPrompt: true,
+ canLogprobs: true,
+ responseFormats: ['json', 'text'],
+ },
+ 'gpt-4o-transcribe': {
+ canPrompt: true,
+ canLogprobs: true,
+ responseFormats: ['json', 'text'],
+ },
+ 'gpt-4o-transcribe-diarize': {
+ canPrompt: false,
+ canLogprobs: false,
+ responseFormats: ['json', 'text', 'diarized_json'],
+ requiresChunkingOverThirtySeconds: true,
+ diarization: true,
+ },
+ 'whisper-1': {
+ canPrompt: true,
+ canLogprobs: false,
+ responseFormats: ['json', 'text', 'srt', 'verbose_json', 'vtt'],
+ timestampGranularities: true,
+ },
+};
+
+class OpenAISpeechToTextService extends BaseService {
+ /** @type {import('../../services/MeteringService/MeteringService').MeteringService} */
+ get meteringService() {
+ return this.services.get('meteringService').meteringService;
+ }
+
+ static MODULES = {
+ openai: require('openai'),
+ musicMetadata: require('music-metadata'),
+ mime: require('mime-types'),
+ path: require('path'),
+ };
+
+ async _init() {
+ let apiKey =
+ this.config?.services?.openai?.apiKey ??
+ this.global_config?.services?.openai?.apiKey;
+
+ if ( !apiKey ) {
+ apiKey =
+ this.config?.openai?.secret_key ??
+ this.global_config.openai?.secret_key;
+
+ if ( apiKey ) {
+ console.warn('The `openai.secret_key` configuration format is deprecated. ' +
+ 'Please use `services.openai.apiKey` instead.');
+ }
+ }
+
+ if ( !apiKey ) {
+ throw new Error('OpenAI API key not configured');
+ }
+
+ this.openai = new this.modules.openai.OpenAI({ apiKey });
+ }
+
+ static IMPLEMENTS = {
+ ['driver-capabilities']: {
+ supports_test_mode(iface, method_name) {
+ return iface === 'puter-speech2txt' &&
+ (method_name === 'transcribe' || method_name === 'translate');
+ },
+ },
+ ['puter-speech2txt']: {
+ async list_models() {
+ return this.listModels();
+ },
+ async transcribe(params) {
+ return this._handleTranscription({ ...params, translate: false });
+ },
+ async translate(params) {
+ return this._handleTranscription({ ...params, translate: true });
+ },
+ },
+ };
+
+ listModels() {
+ return [
+ {
+ id: 'gpt-4o-mini-transcribe',
+ name: 'GPT-4o mini (Transcribe)',
+ type: 'transcription',
+ response_formats: TRANSCRIPTION_MODEL_CAPABILITIES['gpt-4o-mini-transcribe'].responseFormats,
+ supports_prompt: true,
+ supports_logprobs: true,
+ },
+ {
+ id: 'gpt-4o-transcribe',
+ name: 'GPT-4o (Transcribe)',
+ type: 'transcription',
+ response_formats: TRANSCRIPTION_MODEL_CAPABILITIES['gpt-4o-transcribe'].responseFormats,
+ supports_prompt: true,
+ supports_logprobs: true,
+ },
+ {
+ id: 'gpt-4o-transcribe-diarize',
+ name: 'GPT-4o (Transcribe + Diarization)',
+ type: 'transcription',
+ response_formats: TRANSCRIPTION_MODEL_CAPABILITIES['gpt-4o-transcribe-diarize'].responseFormats,
+ supports_prompt: false,
+ supports_logprobs: false,
+ supports_diarization: true,
+ },
+ {
+ id: 'whisper-1',
+ name: 'Whisper 1',
+ type: 'translation',
+ response_formats: TRANSCRIPTION_MODEL_CAPABILITIES['whisper-1'].responseFormats,
+ supports_prompt: true,
+ supports_logprobs: false,
+ supports_timestamp_granularities: true,
+ },
+ ];
+ }
+
+ async _handleTranscription({
+ file,
+ translate = false,
+ model,
+ response_format,
+ language,
+ prompt,
+ temperature,
+ logprobs,
+ timestamp_granularities,
+ chunking_strategy,
+ known_speaker_names,
+ known_speaker_references,
+ extra_body,
+ stream,
+ test_mode,
+ }) {
+ if ( test_mode ) {
+ return {
+ ...SAMPLE_TRANSCRIPT,
+ model: model || (translate ? DEFAULT_TRANSLATE_MODEL : DEFAULT_TRANSCRIBE_MODEL),
+ };
+ }
+
+ if ( stream ) {
+ throw APIError.create('not_yet_supported', null, {
+ message: 'Streaming transcription is not yet supported.',
+ });
+ }
+
+ if ( !file ) {
+ throw APIError.create('field_missing', null, { key: 'file' });
+ }
+
+ if ( ! (file instanceof FileFacade) ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'file',
+ expected: 'file reference',
+ });
+ }
+
+ const {
+ buffer,
+ filename,
+ mimeType,
+ estimatedSeconds,
+ } = await this._prepareAudioBuffer(file);
+
+ const selectedModel = model || (translate ? DEFAULT_TRANSLATE_MODEL : DEFAULT_TRANSCRIBE_MODEL);
+ const capabilities = TRANSCRIPTION_MODEL_CAPABILITIES[selectedModel];
+
+ if ( !capabilities ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'model',
+ expected: Object.keys(TRANSCRIPTION_MODEL_CAPABILITIES).join(', '),
+ got: selectedModel,
+ });
+ }
+
+ if ( response_format && !capabilities.responseFormats.includes(response_format) ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'response_format',
+ expected: capabilities.responseFormats.join(', '),
+ got: response_format,
+ });
+ }
+
+ if ( prompt && !capabilities.canPrompt ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'prompt',
+ expected: `Not supported for model ${selectedModel}`,
+ });
+ }
+
+ if ( logprobs && !capabilities.canLogprobs ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'logprobs',
+ expected: `Not supported for model ${selectedModel}`,
+ });
+ }
+
+ if ( timestamp_granularities && !capabilities.timestampGranularities ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'timestamp_granularities',
+ expected: `Only supported on models that provide timestamp granularity (such as whisper-1).`,
+ });
+ }
+
+ let diarizationChunkingStrategy = chunking_strategy;
+ if ( capabilities.diarization ) {
+ if ( !response_format ) {
+ response_format = 'diarized_json';
+ }
+ if ( !diarizationChunkingStrategy && capabilities.requiresChunkingOverThirtySeconds && estimatedSeconds > 30 ) {
+ diarizationChunkingStrategy = 'auto';
+ }
+ }
+
+ const actor = Context.get('actor');
+ const usageType = `openai:${selectedModel}:second`;
+ const usageAllowed = await this.meteringService.hasEnoughCreditsFor(actor, usageType, estimatedSeconds);
+
+ if ( !usageAllowed ) {
+ throw APIError.create('insufficient_funds');
+ }
+
+ const openaiFile = await this.modules.openai.toFile(
+ buffer,
+ filename,
+ mimeType ? { type: mimeType } : undefined,
+ );
+
+ const payload = {
+ file: openaiFile,
+ model: selectedModel,
+ };
+
+ if ( response_format ) payload.response_format = response_format;
+ if ( language ) payload.language = language;
+ if ( typeof temperature === 'number' ) payload.temperature = temperature;
+ if ( prompt && capabilities.canPrompt ) payload.prompt = prompt;
+ if ( logprobs && capabilities.canLogprobs ) payload.logprobs = logprobs;
+ if ( timestamp_granularities && capabilities.timestampGranularities ) payload.timestamp_granularities = timestamp_granularities;
+ if ( diarizationChunkingStrategy ) payload.chunking_strategy = diarizationChunkingStrategy;
+
+ if ( capabilities.diarization && (known_speaker_names || known_speaker_references) ) {
+ payload.extra_body = {
+ ...(extra_body || {}),
+ ...(known_speaker_names ? { known_speaker_names } : {}),
+ ...(known_speaker_references ? { known_speaker_references } : {}),
+ };
+ } else if ( extra_body ) {
+ payload.extra_body = extra_body;
+ }
+
+ let transcription;
+ if ( translate ) {
+ transcription = await this.openai.audio.translations.create(payload);
+ } else {
+ transcription = await this.openai.audio.transcriptions.create(payload);
+ }
+
+ this.meteringService.incrementUsage(actor, usageType, estimatedSeconds);
+
+ return this._formatResponse(transcription, response_format);
+ }
+
+ async _prepareAudioBuffer(file) {
+ const buffer = await file.get('buffer');
+ if ( !buffer || !buffer.length ) {
+ throw APIError.create('field_invalid', null, {
+ key: 'file',
+ expected: 'non-empty audio file',
+ });
+ }
+
+ if ( buffer.length > MAX_AUDIO_FILE_SIZE ) {
+ throw APIError.create('file_too_large', null, {
+ max_size: MAX_AUDIO_FILE_SIZE,
+ });
+ }
+
+ let filename = 'audio';
+ let mimeType;
+
+ const pathValue = await file.get('path');
+ if ( pathValue ) {
+ filename = this.modules.path.basename(pathValue);
+ } else {
+ const url = await file.get('web_url');
+ if ( url ) {
+ try {
+ const parsed = new URL(url);
+ const candidate = this.modules.path.basename(parsed.pathname);
+ if ( candidate ) filename = candidate;
+ } catch (_) {
+ // Ignore URL parsing errors; we'll fall back to defaults.
+ }
+ }
+ }
+
+ const dataUrl = await file.get('data_url');
+ if ( dataUrl ) {
+ const match = /^data:([^;,]+)[;,]/.exec(dataUrl);
+ if ( match ) {
+ mimeType = match[1];
+ }
+ }
+
+ if ( !mimeType ) {
+ const guessedMime = this.modules.mime.lookup(filename);
+ if ( guessedMime ) {
+ mimeType = guessedMime;
+ }
+ }
+
+ if ( !filename.includes('.') ) {
+ const extension = mimeType ? this.modules.mime.extension(mimeType) : 'mp3';
+ filename = `${filename}.${extension || 'mp3'}`;
+ }
+
+ let estimatedSeconds = Math.ceil(buffer.length / 16000);
+ try {
+ const metadata = await this.modules.musicMetadata.parseBuffer(buffer, {
+ mimeType,
+ size: buffer.length,
+ });
+ if ( metadata?.format?.duration ) {
+ estimatedSeconds = Math.ceil(metadata.format.duration);
+ }
+ } catch (e) {
+ // When metadata parsing fails we fall back to the byte-size estimate.
+ if ( process.env.DEBUG_AUDIO_METADATA === '1' ) {
+ console.warn('Failed to parse audio metadata for duration estimation:', e.message);
+ }
+ }
+
+ estimatedSeconds = Math.max(1, estimatedSeconds);
+
+ return {
+ buffer,
+ filename,
+ mimeType,
+ estimatedSeconds,
+ };
+ }
+
+ _formatResponse(result, response_format) {
+ if ( response_format === 'text' && typeof result === 'string' ) {
+ return result;
+ }
+ if ( typeof result === 'string' ) {
+ return result;
+ }
+ if ( response_format === 'text' && result && typeof result.text === 'string' ) {
+ return result.text;
+ }
+ return result;
+ }
+}
+
+module.exports = {
+ OpenAISpeechToTextService,
+};
diff --git a/src/backend/src/modules/puterai/PuterAIModule.js b/src/backend/src/modules/puterai/PuterAIModule.js
index 812ad3d26..1b1f18c25 100644
--- a/src/backend/src/modules/puterai/PuterAIModule.js
+++ b/src/backend/src/modules/puterai/PuterAIModule.js
@@ -67,6 +67,9 @@ class PuterAIModule extends AdvancedBase {
const { OpenAITTSService } = require('./OpenAITTSService');
services.registerService('openai-tts', OpenAITTSService);
+
+ const { OpenAISpeechToTextService } = require('./OpenAISpeechToTextService');
+ services.registerService('openai-speech2txt', OpenAISpeechToTextService);
}
if ( config?.services?.claude ) {
diff --git a/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts b/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts
index 60635c903..fa942b6a7 100644
--- a/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts
+++ b/src/backend/src/services/MeteringService/costMaps/openAiCostMap.ts
@@ -79,4 +79,10 @@ export const OPENAI_COST_MAP = {
'openai:gpt-4o-mini-tts:character': 1500,
'openai:tts-1:character': 1500,
'openai:tts-1-hd:character': 3000,
+
+ // Speech-to-text models (per second, microcents)
+ 'openai:gpt-4o-transcribe:second': 10000,
+ 'openai:gpt-4o-mini-transcribe:second': 5000,
+ 'openai:gpt-4o-transcribe-diarize:second': 10000,
+ 'openai:whisper-1:second': 10000,
};
diff --git a/src/backend/src/services/drivers/DriverService.js b/src/backend/src/services/drivers/DriverService.js
index 1edc9b922..c23c5c3c6 100644
--- a/src/backend/src/services/drivers/DriverService.js
+++ b/src/backend/src/services/drivers/DriverService.js
@@ -289,6 +289,7 @@ class DriverService extends BaseService {
const iface_to_driver = {
['puter-ocr']: 'aws-textract',
['puter-tts']: 'aws-polly',
+ ['puter-speech2txt']: 'openai-speech2txt',
['puter-chat-completion']: 'openai-completion',
['puter-image-generation']: 'openai-image-generation',
['puter-video-generation']: 'openai-video-generation',
diff --git a/src/puter-js/index.d.ts b/src/puter-js/index.d.ts
index f7cc572bb..08f7f198d 100644
--- a/src/puter-js/index.d.ts
+++ b/src/puter-js/index.d.ts
@@ -59,6 +59,12 @@ interface AI {
txt2speech(text: string, language?: string): Promise;
txt2speech(text: string, language?: string, voice?: string): Promise;
txt2speech(text: string, language?: string, voice?: string, engine?: string): Promise;
+
+ speech2txt(source: string | File | Blob): Promise;
+ speech2txt(source: string | File | Blob, options?: Speech2TxtOptions): Promise;
+ speech2txt(options: Speech2TxtOptions): Promise;
+ speech2txt(source: string | File | Blob, testMode?: boolean): Promise;
+ speech2txt(source: Speech2TxtOptions, testMode?: boolean): Promise;
}
type StreamingChatOptions = Omit & { stream: true };
@@ -136,6 +142,31 @@ interface Txt2SpeechOptions {
instructions?: string;
}
+interface Speech2TxtOptions {
+ file?: string | File | Blob;
+ audio?: string | File | Blob;
+ model?: 'gpt-4o-mini-transcribe' | 'gpt-4o-transcribe' | 'gpt-4o-transcribe-diarize' | 'whisper-1' | string;
+ response_format?: 'json' | 'text' | 'diarized_json' | 'srt' | 'verbose_json' | 'vtt' | string;
+ language?: string;
+ prompt?: string;
+ temperature?: number;
+ logprobs?: boolean;
+ timestamp_granularities?: string[];
+ translate?: boolean;
+ stream?: boolean;
+ chunking_strategy?: string;
+ known_speaker_names?: string[];
+ known_speaker_references?: string[];
+ extra_body?: Record;
+}
+
+interface Speech2TxtResult {
+ text?: string;
+ language?: string;
+ segments?: Array>;
+ [key: string]: any;
+}
+
interface ChatResponseChunk {
text?: string;
[key: string]: any;
diff --git a/src/puter-js/src/modules/AI.js b/src/puter-js/src/modules/AI.js
index b06820ad6..c708ea892 100644
--- a/src/puter-js/src/modules/AI.js
+++ b/src/puter-js/src/modules/AI.js
@@ -274,6 +274,78 @@ class AI{
}).call(this, options);
}
+ speech2txt = async (...args) => {
+ const MAX_INPUT_SIZE = 25 * 1024 * 1024;
+ if ( !args || !args.length ) {
+ throw ({ message: 'Arguments are required', code: 'arguments_required' });
+ }
+
+ const normalizeSource = async (value) => {
+ if ( value instanceof Blob ) {
+ return await utils.blobToDataUri(value);
+ }
+ return value;
+ };
+
+ let options = {};
+ let testMode = false;
+
+ const primary = args[0];
+ if ( primary && typeof primary === 'object' && !Array.isArray(primary) && !(primary instanceof Blob) ) {
+ options = { ...primary };
+ } else {
+ options.file = await normalizeSource(primary);
+ }
+
+ if ( args[1] && typeof args[1] === 'object' && !Array.isArray(args[1]) && !(args[1] instanceof Blob) ) {
+ options = { ...options, ...args[1] };
+ } else if ( typeof args[1] === 'boolean' ) {
+ testMode = args[1];
+ }
+
+ if ( typeof args[2] === 'boolean' ) {
+ testMode = args[2];
+ }
+
+ if ( options.audio ) {
+ options.file = await normalizeSource(options.audio);
+ delete options.audio;
+ }
+
+ if ( options.file instanceof Blob ) {
+ options.file = await normalizeSource(options.file);
+ }
+
+ if ( !options.file ) {
+ throw { message: 'Audio input is required', code: 'audio_required' };
+ }
+
+ if ( typeof options.file === 'string' && options.file.startsWith('data:') ) {
+ const base64 = options.file.split(',')[1] || '';
+ const padding = base64.endsWith('==') ? 2 : (base64.endsWith('=') ? 1 : 0);
+ const byteLength = Math.floor((base64.length * 3) / 4) - padding;
+ if ( byteLength > MAX_INPUT_SIZE ) {
+ throw { message: 'Input size cannot be larger than 25 MB', code: 'input_too_large' };
+ }
+ }
+
+ const driverMethod = options.translate ? 'translate' : 'transcribe';
+ const driverArgs = { ...options };
+ delete driverArgs.translate;
+
+ const responseFormat = driverArgs.response_format;
+
+ return await utils.make_driver_method([], 'puter-speech2txt', 'openai-speech2txt', driverMethod, {
+ test_mode: testMode,
+ transform: async (result) => {
+ if ( responseFormat === 'text' && result && typeof result === 'object' && typeof result.text === 'string' ) {
+ return result.text;
+ }
+ return result;
+ },
+ }).call(this, driverArgs);
+ }
+
// Add new methods for TTS engine management
txt2speech = Object.assign(this.txt2speech, {
/**