refactor(ai-tts): one per-character metering path for TTS providers

meterPerCharacter holds text length x price, runs the synthesis and meters the
same amount once it succeeds; characterCostReport builds the per-character
cost lines. Both sit on a new withAiCreditHold (reserve, run, release) that
Gemini TTS uses directly. SAMPLE_AUDIO_URL is defined once. The TTSProvider
base class is gone: its methods were stubs every provider overrode, and only
Polly read its untyped providerConfig.
This commit is contained in:
Daniel Salazar committed 2026-10-10 19:37:53 -04:00
1 parent 3a8df64c8d
commit 013e1fa687
15 files changed
+673 -585

No files matched your search

@@ -29,6 +29,7 @@ import {
} from '../util/aiCostFactor.js';
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js';
import { loadFileInput } from '../util/fileInput.js';
import { SAMPLE_AUDIO_URL } from '../util/testMode.js';
import { upstreamFetch } from '../util/upstreamErrors.js';
import { VOICE_CHANGER_COSTS } from './costs.js';
@@ -40,7 +41,6 @@ import { VOICE_CHANGER_COSTS } from './costs.js';
const DEFAULT_MODEL = 'eleven_multilingual_sts_v2';
const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM';
const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128';
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
const MAX_AUDIO_FILE_SIZE = 25 * 1024 * 1024;
// Covers the upload as well as the conversion.
const CONVERT_TIMEOUT_MS = 10 * 60 * 1000;
+3 -17
View File
@@ -138,23 +138,9 @@ export class TTSDriver extends PuterDriver {
}
override getReportedCosts(): Record<string, unknown>[] {
const all: Record<string, unknown>[] = [];
for (const p of Object.values(this.#providers)) {
const fn = (
p as unknown as {
getReportedCosts?: () => Record<string, unknown>[];
}
).getReportedCosts;
if (typeof fn === 'function') {
try {
const entries = fn.call(p);
if (Array.isArray(entries)) all.push(...entries);
} catch {
// ignore — cost reporting is best-effort
}
}
}
return all;
return Object.values(this.#providers).flatMap((p) =>
p.getReportedCosts(),
);
}
/**
@@ -1,67 +0,0 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* Abstract base for TTS providers. Each provider wraps a single upstream API
* (OpenAI, ElevenLabs, AWS Polly) and exposes the unified `ITTSProvider`
* contract.
*/
import type { AiMeteringService } from '../../util/aiCostFactor.js';
import type {
ITTSProvider,
ITTSVoice,
ITTSEngine,
ISynthesizeArgs,
} from '../types.js';
export abstract class TTSProvider implements ITTSProvider {
abstract readonly providerName: string;
protected meteringService: AiMeteringService;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
protected providerConfig: any;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
constructor(meteringService: AiMeteringService, config: any) {
this.meteringService = meteringService;
this.providerConfig = config;
}
async listVoices(_args?: Record<string, unknown>): Promise<ITTSVoice[]> {
return [];
}
async listEngines(): Promise<ITTSEngine[]> {
return [];
}
async synthesize(_args: ISynthesizeArgs): Promise<unknown> {
throw new Error('Method not implemented.');
}
/**
* Provider-specific cost catalogue used by the TTSDriver's aggregated
* `getReportedCosts()`. Subclasses override to expose their per-unit
* metering costs. Shape matches the `WithCostsReporting` contract.
*/
getReportedCosts(): Record<string, unknown>[] {
return [];
}
}
@@ -26,16 +26,19 @@ import {
type VoiceId,
} from '@aws-sdk/client-polly';
import { HttpError } from '../../../../core/http/HttpError.js';
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
import { Context } from '../../../../core/context.js';
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
import type { DriverStreamResult } from '../../../meta.js';
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
import { TTSProvider } from '../TTSProvider.js';
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
import type {
ITTSVoice,
ITTSEngine,
ISynthesizeArgs,
ITTSProvider,
} from '../../types.js';
import { characterCostReport, meterPerCharacter } from '../common.js';
import { AWS_POLLY_COSTS } from './costs.js';
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
const VALID_ENGINES = ['standard', 'neural', 'long-form', 'generative'];
// Voices tried, in order, when the caller names none. The head of each list
@@ -56,7 +59,7 @@ interface PollyVoicesResponse {
* audio as a DriverStreamResult. Includes voice caching and engine-aware voice
* selection.
*/
export class AWSPollyTTSProvider extends TTSProvider {
export class AWSPollyTTSProvider implements ITTSProvider {
readonly providerName = 'aws-polly';
private clients: Record<string, PollyClient> = {};
@@ -64,22 +67,16 @@ export class AWSPollyTTSProvider extends TTSProvider {
null;
constructor(
meteringService: AiMeteringService,
config: {
private readonly meteringService: AiMeteringService,
private readonly config: {
access_key: string;
secret_key: string;
region?: string;
},
) {
super(meteringService, config);
}
) {}
private getClient(region?: string): PollyClient {
const cfg = this.providerConfig as {
access_key: string;
secret_key: string;
region?: string;
};
const cfg = this.config;
const resolvedRegion = region ?? cfg.region ?? 'us-west-2';
if (this.clients[resolvedRegion]) {
@@ -207,15 +204,8 @@ export class AWSPollyTTSProvider extends TTSProvider {
}));
}
override getReportedCosts(): Record<string, unknown>[] {
return Object.entries(AWS_POLLY_COSTS).map(
([engine, ucentsPerUnit]) => ({
usageType: `aws-polly:${engine}:character`,
ucentsPerUnit,
unit: 'character',
source: 'driver:aiTts/aws-polly',
}),
);
getReportedCosts(): Record<string, unknown>[] {
return characterCostReport('aws-polly', AWS_POLLY_COSTS);
}
async synthesize(
@@ -252,65 +242,49 @@ export class AWSPollyTTSProvider extends TTSProvider {
});
}
const actor = Context.get('actor')!;
const usageType = `aws-polly:${engine}:character`;
const ucentsPerChar = AWS_POLLY_COSTS[engine] ?? 0;
const totalCost = ucentsPerChar * text.length;
return meterPerCharacter(
this.meteringService,
Context.get('actor')!,
`aws-polly:${engine}:character`,
AWS_POLLY_COSTS[engine] ?? 0,
text,
async () => {
// Resolve voice
let voice = voiceArg ?? undefined;
const hold = await this.meteringService.reserveAiCredits(
actor,
usageType,
totalCost,
if (!voice && language) {
voice =
(await this.getLanguageAppropriateVoice(
language,
engine,
)) ?? undefined;
}
if (!voice) {
voice = await this.getDefaultVoiceForEngine(engine);
}
const client = this.getClient();
const params = {
Engine: engine as Engine,
OutputFormat: 'mp3' as const,
Text: text,
VoiceId: voice as VoiceId,
LanguageCode: (language ?? 'en-US') as LanguageCode,
TextType: (ssml ? 'ssml' : 'text') as 'ssml' | 'text',
};
const command = new SynthesizeSpeechCommand(params);
const response = await client.send(command);
return {
dataType: 'stream',
content_type: 'audio/mpeg',
chunked: true,
stream: response.AudioStream as unknown as import('node:stream').Readable,
};
},
);
if (!hold) {
throw insufficientCreditsError();
}
try {
// Resolve voice
let voice = voiceArg ?? undefined;
if (!voice && language) {
voice =
(await this.getLanguageAppropriateVoice(
language,
engine,
)) ?? undefined;
}
if (!voice) {
voice = await this.getDefaultVoiceForEngine(engine);
}
const client = this.getClient();
const params = {
Engine: engine as Engine,
OutputFormat: 'mp3' as const,
Text: text,
VoiceId: voice as VoiceId,
LanguageCode: (language ?? 'en-US') as LanguageCode,
TextType: (ssml ? 'ssml' : 'text') as 'ssml' | 'text',
};
const command = new SynthesizeSpeechCommand(params);
const response = await client.send(command);
this.meteringService.incrementUsage(
actor,
usageType,
text.length,
totalCost,
);
return {
dataType: 'stream',
content_type: 'audio/mpeg',
chunked: true,
stream: response.AudioStream as unknown as import('node:stream').Readable,
};
} finally {
await hold.release();
}
}
}
@@ -0,0 +1,154 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { Readable } from 'node:stream';
import {
afterAll,
afterEach,
beforeAll,
describe,
expect,
it,
vi,
} from 'vitest';
import { makeActor } from '../../../core/actor.js';
import type { DriverStreamResult } from '../../meta.js';
import { PuterServer } from '../../../server.js';
import { setupTestServer } from '../../../testUtil.js';
import { withAiCostFactor } from '../../util/aiCostFactor.js';
import { characterCostReport, meterPerCharacter } from './common.js';
let server: PuterServer;
beforeAll(async () => {
server = await setupTestServer();
});
afterAll(async () => {
await server?.shutdown();
});
afterEach(() => {
vi.restoreAllMocks();
});
const audio = (): Promise<DriverStreamResult> =>
Promise.resolve({
dataType: 'stream',
content_type: 'audio/mpeg',
stream: Readable.from([]),
});
describe('meterPerCharacter', () => {
const actor = makeActor({
user: { id: 7, uuid: 'tts-meter-user', username: 'tts-meter' },
});
const metering = () =>
withAiCostFactor(
server.services.metering,
server.clients.event,
'ai-tts',
);
it('holds and then meters exactly text length × price', async () => {
const check = vi
.spyOn(server.services.metering, 'hasEnoughCredits')
.mockResolvedValue(true);
const record = vi.spyOn(server.services.metering, 'incrementUsage');
await meterPerCharacter(
metering(),
actor,
'openai:tts-1:character',
1500,
'hello',
audio,
);
expect(check).toHaveBeenCalledWith(actor, 7500);
expect(record).toHaveBeenCalledWith(
actor,
'openai:tts-1:character',
5,
7500,
);
expect(await server.services.metering.getOutstandingHolds(actor)).toBe(
0,
);
});
it('meters nothing when synthesis fails', async () => {
vi.spyOn(
server.services.metering,
'hasEnoughCredits',
).mockResolvedValue(true);
const record = vi.spyOn(server.services.metering, 'incrementUsage');
const boom = new Error('vendor down');
await expect(
meterPerCharacter(
metering(),
actor,
'openai:tts-1:character',
1500,
'hello',
() => Promise.reject(boom),
),
).rejects.toBe(boom);
expect(record).not.toHaveBeenCalled();
});
it('refuses with a 402 before synthesizing when unaffordable', async () => {
vi.spyOn(
server.services.metering,
'hasEnoughCredits',
).mockResolvedValue(false);
const synthesize = vi.fn(audio);
await expect(
meterPerCharacter(
metering(),
actor,
'openai:tts-1:character',
1500,
'hello',
synthesize,
),
).rejects.toMatchObject({ statusCode: 402 });
expect(synthesize).not.toHaveBeenCalled();
});
});
describe('characterCostReport', () => {
it('emits one per-character line per cost-table entry', () => {
expect(
characterCostReport('aws-polly', { standard: 400, neural: 1600 }),
).toEqual([
{
usageType: 'aws-polly:standard:character',
ucentsPerUnit: 400,
unit: 'character',
source: 'driver:aiTts/aws-polly',
},
{
usageType: 'aws-polly:neural:character',
ucentsPerUnit: 1600,
unit: 'character',
source: 'driver:aiTts/aws-polly',
},
]);
});
});
@@ -17,5 +17,45 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import type { Actor } from '../../../core/actor.js';
import {
type AiMeteringService,
withAiCreditHold,
} from '../../util/aiCostFactor.js';
import type { DriverStreamResult } from '../../meta.js';
/** Bounds a synthesis call, audio download included. */
export const TTS_UPSTREAM_TIMEOUT_MS = 120_000;
/**
* Runs `synthesize` under a hold on `text.length × ucentsPerChar`, then meters
* exactly that once it succeeds. Per-character prices are known up front, so
* the hold and the charge are the same amount.
*/
export function meterPerCharacter(
metering: AiMeteringService,
actor: Actor,
usageType: string,
ucentsPerChar: number,
text: string,
synthesize: () => Promise<DriverStreamResult>,
): Promise<DriverStreamResult> {
const cost = ucentsPerChar * text.length;
return withAiCreditHold(metering, actor, usageType, cost, async () => {
const result = await synthesize();
metering.incrementUsage(actor, usageType, text.length, cost);
return result;
});
}
/** The cost-report lines for a `<model or engine> → ucents/char` table. */
export const characterCostReport = (
provider: string,
costs: Record<string, number>,
): Record<string, unknown>[] =>
Object.entries(costs).map(([key, ucentsPerUnit]) => ({
usageType: `${provider}:${key}:character`,
ucentsPerUnit,
unit: 'character',
source: `driver:aiTts/${provider}`,
}));
@@ -19,20 +19,27 @@
import { Readable } from 'node:stream';
import { HttpError } from '../../../../core/http/HttpError.js';
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
import { Context } from '../../../../core/context.js';
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
import type { DriverStreamResult } from '../../../meta.js';
import { upstreamFetch } from '../../../util/upstreamErrors.js';
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
import { TTSProvider } from '../TTSProvider.js';
import { TTS_UPSTREAM_TIMEOUT_MS } from '../common.js';
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
import type {
ITTSVoice,
ITTSEngine,
ISynthesizeArgs,
ITTSProvider,
} from '../../types.js';
import {
characterCostReport,
meterPerCharacter,
TTS_UPSTREAM_TIMEOUT_MS,
} from '../common.js';
import { ELEVENLABS_TTS_COSTS } from './costs.js';
const DEFAULT_MODEL = 'eleven_multilingual_v2';
const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM'; // "Rachel" sample voice
const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128';
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
const ELEVENLABS_TTS_MODELS = [
{ id: DEFAULT_MODEL, name: 'Eleven Multilingual v2' },
@@ -48,7 +55,7 @@ const ELEVENLABS_TTS_MODELS = [
* ElevenLabs TTS provider. Uses the ElevenLabs REST API to synthesize speech
* and returns audio as a DriverStreamResult.
*/
export class ElevenLabsTTSProvider extends TTSProvider {
export class ElevenLabsTTSProvider implements ITTSProvider {
readonly providerName = 'elevenlabs';
private apiKey: string;
@@ -56,15 +63,13 @@ export class ElevenLabsTTSProvider extends TTSProvider {
private defaultVoiceId: string;
constructor(
meteringService: AiMeteringService,
private readonly meteringService: AiMeteringService,
config: {
apiKey: string;
apiBaseUrl?: string;
defaultVoiceId?: string;
},
) {
super(meteringService, config);
this.apiKey = config.apiKey;
this.baseUrl = config.apiBaseUrl ?? 'https://api.elevenlabs.io';
this.defaultVoiceId = config.defaultVoiceId ?? DEFAULT_VOICE_ID;
@@ -131,15 +136,8 @@ export class ElevenLabsTTSProvider extends TTSProvider {
}));
}
override getReportedCosts(): Record<string, unknown>[] {
return Object.entries(ELEVENLABS_TTS_COSTS).map(
([model, ucentsPerUnit]) => ({
usageType: `elevenlabs:${model}:character`,
ucentsPerUnit,
unit: 'character',
source: 'driver:aiTts/elevenlabs',
}),
);
getReportedCosts(): Record<string, unknown>[] {
return characterCostReport('elevenlabs', ELEVENLABS_TTS_COSTS);
}
async synthesize(
@@ -185,66 +183,35 @@ export class ElevenLabsTTSProvider extends TTSProvider {
);
}
const desiredFormat =
output_format || response_format || DEFAULT_OUTPUT_FORMAT;
const payload: Record<string, unknown> = {
text,
model_id: modelId,
output_format:
output_format || response_format || DEFAULT_OUTPUT_FORMAT,
};
const finalVoiceSettings = voice_settings ?? voiceSettings;
if (finalVoiceSettings) payload.voice_settings = finalVoiceSettings;
const actor = Context.get('actor')!;
const usageKey = `elevenlabs:${modelId}:character`;
const ucentsPerChar = ELEVENLABS_TTS_COSTS[modelId];
const totalCost = ucentsPerChar * text.length;
const hold = await this.meteringService.reserveAiCredits(
actor,
usageKey,
totalCost,
return meterPerCharacter(
this.meteringService,
Context.get('actor')!,
`elevenlabs:${modelId}:character`,
ELEVENLABS_TTS_COSTS[modelId],
text,
async () => {
const response = await this.request(
`/v1/text-to-speech/${voiceId}`,
{ method: 'POST', body: payload },
);
const buffer = Buffer.from(await response.arrayBuffer());
return {
dataType: 'stream',
content_type:
response.headers.get('content-type') || 'audio/mpeg',
chunked: true,
stream: Readable.from(buffer),
};
},
);
if (!hold) {
throw insufficientCreditsError();
}
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const payload: any = {
text,
model_id: modelId,
output_format: desiredFormat,
};
const finalVoiceSettings = voice_settings ?? voiceSettings;
if (finalVoiceSettings) {
payload.voice_settings = finalVoiceSettings;
}
const response = await this.request(
`/v1/text-to-speech/${voiceId}`,
{
method: 'POST',
body: payload,
},
);
const arrayBuffer = await response.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
const stream = Readable.from(buffer);
this.meteringService.incrementUsage(
actor,
usageKey,
text.length,
totalCost,
);
const contentType =
response.headers.get('content-type') || 'audio/mpeg';
return {
dataType: 'stream',
content_type: contentType,
chunked: true,
stream,
};
} finally {
await hold.release();
}
}
}
@@ -20,17 +20,23 @@
import { GoogleGenAI } from '@google/genai';
import { Readable } from 'node:stream';
import { HttpError } from '../../../../core/http/HttpError.js';
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
import { Context } from '../../../../core/context.js';
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
import {
type AiMeteringService,
withAiCreditHold,
} from '../../../util/aiCostFactor.js';
import type { DriverStreamResult } from '../../../meta.js';
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
import { TTSProvider } from '../TTSProvider.js';
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
import type {
ITTSVoice,
ITTSEngine,
ISynthesizeArgs,
ITTSProvider,
} from '../../types.js';
import { GEMINI_TTS_COSTS } from './costs.js';
const DEFAULT_MODEL = 'gemini-3.8-flash-tts';
const DEFAULT_VOICE = 'Kore';
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
const GEMINI_TTS_MODELS = [
{
@@ -81,16 +87,15 @@ const GEMINI_TTS_VOICES = [
* `responseModalities: ["AUDIO"]` and `speechConfig` to synthesize speech.
* Returns raw PCM audio wrapped in a WAV container.
*/
export class GeminiTTSProvider extends TTSProvider {
export class GeminiTTSProvider implements ITTSProvider {
readonly providerName = 'gemini';
#client: GoogleGenAI;
constructor(
meteringService: AiMeteringService,
private readonly meteringService: AiMeteringService,
config: { apiKey: string },
) {
super(meteringService, config);
if (!config.apiKey) {
throw new Error('Gemini TTS requires an API key');
}
@@ -115,7 +120,7 @@ export class GeminiTTSProvider extends TTSProvider {
}));
}
override getReportedCosts(): Record<string, unknown>[] {
getReportedCosts(): Record<string, unknown>[] {
return Object.entries(GEMINI_TTS_COSTS).map(([model, costs]) => ({
usageType: `gemini:${model}:tts`,
ucentsInputPerToken: this.#toMicroCents(costs.input / 1_000_000),
@@ -212,114 +217,114 @@ export class GeminiTTSProvider extends TTSProvider {
estimatedInputCostCents + estimatedOutputCostCents,
);
const hold = await this.meteringService.reserveAiCredits(
return withAiCreditHold(
this.meteringService,
actor,
`gemini:${model}`,
estimatedTotalMicroCents,
);
if (!hold) {
throw insufficientCreditsError();
}
async (): Promise<DriverStreamResult> => {
// The TTS models require the text to be framed as a transcript
// to read aloud. Prefixing with "Say:" prevents the model from
// trying to generate conversational text instead of audio.
const inputText = instructions
? `${instructions}\n\nSay the following text aloud:\n${text}`
: `Say the following text aloud:\n${text}`;
try {
// The TTS models require the text to be framed as a transcript
// to read aloud. Prefixing with "Say:" prevents the model from
// trying to generate conversational text instead of audio.
const inputText = instructions
? `${instructions}\n\nSay the following text aloud:\n${text}`
: `Say the following text aloud:\n${text}`;
// Let Google GenAI `ApiError`s bubble — they carry `.status` and
// are mapped to `upstream_*` HttpErrors by the driver-boundary
// translator. Catching here and wrapping as 502 hid the upstream
// status and caused 4xx validation errors to page.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const response: any = await this.#client.models.generateContent({
model,
contents: [{ parts: [{ text: inputText }] }],
config: {
responseModalities: ['AUDIO'],
speechConfig: {
voiceConfig: {
prebuiltVoiceConfig: { voiceName: voice },
// Let Google GenAI `ApiError`s bubble — they carry `.status` and
// are mapped to `upstream_*` HttpErrors by the driver-boundary
// translator. Catching here and wrapping as 502 hid the upstream
// status and caused 4xx validation errors to page.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const response: any = await this.#client.models.generateContent(
{
model,
contents: [{ parts: [{ text: inputText }] }],
config: {
responseModalities: ['AUDIO'],
speechConfig: {
voiceConfig: {
prebuiltVoiceConfig: { voiceName: voice },
},
},
},
},
},
});
// Extract audio data from response
const part = response?.candidates?.[0]?.content?.parts?.[0];
if (!part?.inlineData?.data) {
throw new HttpError(
400,
'Gemini TTS did not return audio data',
{
legacyCode: 'upstream_bad_request',
fields: { provider: 'gemini' },
},
);
}
const audioBase64: string = part.inlineData.data;
const mimeType: string =
part.inlineData.mimeType || 'audio/L16;rate=24000';
// Extract audio data from response
const part = response?.candidates?.[0]?.content?.parts?.[0];
if (!part?.inlineData?.data) {
throw new HttpError(
400,
'Gemini TTS did not return audio data',
{
legacyCode: 'upstream_bad_request',
fields: { provider: 'gemini' },
},
);
}
// Convert base64 PCM to a WAV buffer for broad client compatibility
const pcmBuffer = Buffer.from(audioBase64, 'base64');
let outputBuffer: Buffer;
let contentType: string;
const audioBase64: string = part.inlineData.data;
const mimeType: string =
part.inlineData.mimeType || 'audio/L16;rate=24000';
if (mimeType.startsWith('audio/L16') || mimeType === 'audio/pcm') {
// Wrap raw PCM (16-bit LE, 24kHz, mono) in a WAV container
outputBuffer = this.#wrapPcmInWav(pcmBuffer, 24000, 1, 16);
contentType = 'audio/wav';
} else {
// If the API returns encoded audio (unlikely today), pass through
outputBuffer = pcmBuffer;
contentType = mimeType;
}
// Convert base64 PCM to a WAV buffer for broad client compatibility
const pcmBuffer = Buffer.from(audioBase64, 'base64');
let outputBuffer: Buffer;
let contentType: string;
// Meter actual usage from response metadata
const usage = response.usageMetadata;
const actualInputTokens =
typeof usage?.promptTokenCount === 'number'
? usage.promptTokenCount
: estimatedInputTokens;
const actualOutputTokens =
typeof usage?.candidatesTokenCount === 'number'
? usage.candidatesTokenCount
: estimatedOutputTokens;
if (
mimeType.startsWith('audio/L16') ||
mimeType === 'audio/pcm'
) {
// Wrap raw PCM (16-bit LE, 24kHz, mono) in a WAV container
outputBuffer = this.#wrapPcmInWav(pcmBuffer, 24000, 1, 16);
contentType = 'audio/wav';
} else {
// If the API returns encoded audio (unlikely today), pass through
outputBuffer = pcmBuffer;
contentType = mimeType;
}
const inputCostCents =
(actualInputTokens / 1_000_000) * costs.input;
const outputCostCents =
(actualOutputTokens / 1_000_000) * costs.output_audio;
// Meter actual usage from response metadata
const usage = response.usageMetadata;
const actualInputTokens =
typeof usage?.promptTokenCount === 'number'
? usage.promptTokenCount
: estimatedInputTokens;
const actualOutputTokens =
typeof usage?.candidatesTokenCount === 'number'
? usage.candidatesTokenCount
: estimatedOutputTokens;
const usagePrefix = `gemini:${model}`;
this.meteringService.batchIncrementUsages(actor, [
{
usageType: `${usagePrefix}:input`,
usageAmount: Math.max(actualInputTokens, 1),
costOverride: this.#toMicroCents(inputCostCents),
},
{
usageType: `${usagePrefix}:output:audio`,
usageAmount: Math.max(actualOutputTokens, 1),
costOverride: this.#toMicroCents(outputCostCents),
},
]);
const inputCostCents =
(actualInputTokens / 1_000_000) * costs.input;
const outputCostCents =
(actualOutputTokens / 1_000_000) * costs.output_audio;
const stream = Readable.from(outputBuffer);
const usagePrefix = `gemini:${model}`;
this.meteringService.batchIncrementUsages(actor, [
{
usageType: `${usagePrefix}:input`,
usageAmount: Math.max(actualInputTokens, 1),
costOverride: this.#toMicroCents(inputCostCents),
},
{
usageType: `${usagePrefix}:output:audio`,
usageAmount: Math.max(actualOutputTokens, 1),
costOverride: this.#toMicroCents(outputCostCents),
},
]);
return {
dataType: 'stream',
content_type: contentType,
chunked: true,
stream,
};
} finally {
await hold.release();
}
const stream = Readable.from(outputBuffer);
return {
dataType: 'stream',
content_type: contentType,
chunked: true,
stream,
};
},
);
}
/** Wrap raw PCM samples in a WAV container so browsers can play it. */
@@ -20,17 +20,21 @@
import OpenAI from 'openai';
import { Readable } from 'node:stream';
import { HttpError } from '../../../../core/http/HttpError.js';
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
import { Context } from '../../../../core/context.js';
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
import type { DriverStreamResult } from '../../../meta.js';
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
import { TTSProvider } from '../TTSProvider.js';
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
import type {
ITTSVoice,
ITTSEngine,
ISynthesizeArgs,
ITTSProvider,
} from '../../types.js';
import { characterCostReport, meterPerCharacter } from '../common.js';
import { OPENAI_TTS_COSTS } from './costs.js';
const DEFAULT_MODEL = 'gpt-4o-mini-tts';
const DEFAULT_VOICE = 'alloy';
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
const RESPONSE_CONTENT_TYPES: Record<string, string> = {
mp3: 'audio/mpeg',
@@ -80,16 +84,15 @@ const OPENAI_TTS_MODELS = [
* OpenAI TTS provider. Wraps the OpenAI speech synthesis API and returns audio
* as a DriverStreamResult.
*/
export class OpenAITTSProvider extends TTSProvider {
export class OpenAITTSProvider implements ITTSProvider {
readonly providerName = 'openai';
private openai: OpenAI;
constructor(
meteringService: AiMeteringService,
private readonly meteringService: AiMeteringService,
config: { apiKey: string },
) {
super(meteringService, config);
this.openai = new OpenAI({ apiKey: config.apiKey });
}
@@ -116,15 +119,8 @@ export class OpenAITTSProvider extends TTSProvider {
}));
}
override getReportedCosts(): Record<string, unknown>[] {
return Object.entries(OPENAI_TTS_COSTS).map(
([model, ucentsPerUnit]) => ({
usageType: `openai:${model}:character`,
ucentsPerUnit,
unit: 'character',
source: 'driver:aiTts/openai',
}),
);
getReportedCosts(): Record<string, unknown>[] {
return characterCostReport('openai', OPENAI_TTS_COSTS);
}
async synthesize(
@@ -206,56 +202,27 @@ export class OpenAITTSProvider extends TTSProvider {
const contentType =
RESPONSE_CONTENT_TYPES[format] || RESPONSE_CONTENT_TYPES.mp3;
const actor = Context.get('actor')!;
const usageType = `openai:${model}:character`;
const ucentsPerChar = OPENAI_TTS_COSTS[model] ?? 0;
const totalCost = ucentsPerChar * text.length;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const payload: any = { model, voice, input: text };
if (instructions) payload.instructions = instructions;
if (response_format) payload.response_format = response_format;
const hold = await this.meteringService.reserveAiCredits(
actor,
usageType,
totalCost,
return meterPerCharacter(
this.meteringService,
Context.get('actor')!,
`openai:${model}:character`,
OPENAI_TTS_COSTS[model] ?? 0,
text,
async () => {
const response = await this.openai.audio.speech.create(payload);
const buffer = Buffer.from(await response.arrayBuffer());
return {
dataType: 'stream',
content_type: contentType,
chunked: true,
stream: Readable.from(buffer),
};
},
);
if (!hold) {
throw insufficientCreditsError();
}
try {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const payload: any = {
model,
voice,
input: text,
};
if (instructions) {
payload.instructions = instructions;
}
if (response_format) {
payload.response_format = response_format;
}
const response = await this.openai.audio.speech.create(payload);
const arrayBuffer = await response.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
const stream = Readable.from(buffer);
this.meteringService.incrementUsage(
actor,
usageType,
text.length,
totalCost,
);
return {
dataType: 'stream',
content_type: contentType,
chunked: true,
stream,
};
} finally {
await hold.release();
}
}
}
@@ -19,21 +19,28 @@
import { Readable } from 'node:stream';
import { HttpError } from '../../../../core/http/HttpError.js';
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
import { Context } from '../../../../core/context.js';
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
import type { DriverStreamResult } from '../../../meta.js';
import { upstreamFetch } from '../../../util/upstreamErrors.js';
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
import { TTSProvider } from '../TTSProvider.js';
import { TTS_UPSTREAM_TIMEOUT_MS } from '../common.js';
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
import type {
ITTSVoice,
ITTSEngine,
ISynthesizeArgs,
ITTSProvider,
} from '../../types.js';
import {
characterCostReport,
meterPerCharacter,
TTS_UPSTREAM_TIMEOUT_MS,
} from '../common.js';
import { SPEECHIFY_TTS_COSTS } from './costs.js';
// Public API base only — never an internal/consumer Speechify endpoint.
const API_BASE = 'https://api.speechify.ai';
const CALLER_HEADER = 'Speechify-Caller';
const CALLER_VALUE = 'puter';
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
const DEFAULT_MODEL = 'simba-3.2';
const DEFAULT_VOICE = 'geffen_32';
@@ -62,16 +69,15 @@ const CONTENT_TYPES: Record<string, string> = {
* and returns audio as a DriverStreamResult. Every outbound request carries
* `Speechify-Caller: puter` for integration attribution.
*/
export class SpeechifyTTSProvider extends TTSProvider {
export class SpeechifyTTSProvider implements ITTSProvider {
readonly providerName = 'speechify';
#apiKey: string;
constructor(
meteringService: AiMeteringService,
private readonly meteringService: AiMeteringService,
config: { apiKey: string },
) {
super(meteringService, config);
if (!config.apiKey) {
throw new Error('Speechify TTS requires an API key');
}
@@ -96,15 +102,8 @@ export class SpeechifyTTSProvider extends TTSProvider {
}));
}
override getReportedCosts(): Record<string, unknown>[] {
return Object.entries(SPEECHIFY_TTS_COSTS).map(
([model, ucentsPerUnit]) => ({
usageType: `speechify:${model}:character`,
ucentsPerUnit,
unit: 'character',
source: 'driver:aiTts/speechify',
}),
);
getReportedCosts(): Record<string, unknown>[] {
return characterCostReport('speechify', SPEECHIFY_TTS_COSTS);
}
async synthesize(
@@ -148,83 +147,64 @@ export class SpeechifyTTSProvider extends TTSProvider {
);
}
const voice = voiceArg || DEFAULT_VOICE;
const format = output_format || response_format || 'mp3';
// Speechify's synthesis endpoint expects SSML; wrap plain text so
// callers can keep passing bare strings like every other provider.
const input = /<speak[\s>]/i.test(text)
? text
: `<speak>${text}</speak>`;
const actor = Context.get('actor')!;
const usageType = `speechify:${model}:character`;
const ucentsPerChar = SPEECHIFY_TTS_COSTS[model] ?? 0;
const totalCost = ucentsPerChar * text.length;
const hold = await this.meteringService.reserveAiCredits(
actor,
usageType,
totalCost,
);
if (!hold) {
throw insufficientCreditsError();
}
try {
// Speechify's synthesis endpoint expects SSML; wrap plain text so
// callers can keep passing bare strings like every other provider.
const input = /<speak[\s>]/i.test(text)
? text
: `<speak>${text}</speak>`;
const response = await upstreamFetch(
'Speechify TTS',
`${API_BASE}/v1/audio/speech`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${this.#apiKey}`,
'Content-Type': 'application/json',
[CALLER_HEADER]: CALLER_VALUE,
},
body: JSON.stringify({
input,
voice_id: voice,
model,
audio_format: format,
}),
},
{ timeoutMs: TTS_UPSTREAM_TIMEOUT_MS },
);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const data: any = await response.json();
if (!data?.audio_data) {
throw new HttpError(
400,
'Speechify TTS did not return audio data',
return meterPerCharacter(
this.meteringService,
Context.get('actor')!,
`speechify:${model}:character`,
SPEECHIFY_TTS_COSTS[model] ?? 0,
text,
async () => {
const response = await upstreamFetch(
'Speechify TTS',
`${API_BASE}/v1/audio/speech`,
{
legacyCode: 'upstream_bad_request',
fields: { provider: 'speechify' },
method: 'POST',
headers: {
Authorization: `Bearer ${this.#apiKey}`,
'Content-Type': 'application/json',
[CALLER_HEADER]: CALLER_VALUE,
},
body: JSON.stringify({
input,
voice_id: voiceArg || DEFAULT_VOICE,
model,
audio_format: format,
}),
},
{ timeoutMs: TTS_UPSTREAM_TIMEOUT_MS },
);
}
const buffer = Buffer.from(data.audio_data, 'base64');
const stream = Readable.from(buffer);
const contentType =
CONTENT_TYPES[data.audio_format ?? format] || 'audio/mpeg';
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const data: any = await response.json();
if (!data?.audio_data) {
throw new HttpError(
400,
'Speechify TTS did not return audio data',
{
legacyCode: 'upstream_bad_request',
fields: { provider: 'speechify' },
},
);
}
this.meteringService.incrementUsage(
actor,
usageType,
text.length,
totalCost,
);
return {
dataType: 'stream',
content_type: contentType,
chunked: true,
stream,
};
} finally {
await hold.release();
}
return {
dataType: 'stream',
content_type:
CONTENT_TYPES[data.audio_format ?? format] ||
'audio/mpeg',
chunked: true,
stream: Readable.from(
Buffer.from(data.audio_data, 'base64'),
),
};
},
);
}
}
@@ -19,18 +19,25 @@
import { Readable } from 'node:stream';
import { HttpError } from '../../../../core/http/HttpError.js';
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
import { Context } from '../../../../core/context.js';
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
import type { DriverStreamResult } from '../../../meta.js';
import { upstreamFetch } from '../../../util/upstreamErrors.js';
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
import { TTSProvider } from '../TTSProvider.js';
import { TTS_UPSTREAM_TIMEOUT_MS } from '../common.js';
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
import type {
ITTSVoice,
ITTSEngine,
ISynthesizeArgs,
ITTSProvider,
} from '../../types.js';
import {
characterCostReport,
meterPerCharacter,
TTS_UPSTREAM_TIMEOUT_MS,
} from '../common.js';
import { XAI_TTS_COSTS } from './costs.js';
const API_BASE = 'https://api.x.ai/v1';
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
const XAI_TTS_VOICES = [
{ id: 'eve', name: 'Eve', description: 'Energetic, upbeat' },
@@ -54,16 +61,15 @@ const CODEC_CONTENT_TYPES: Record<string, string> = {
* XAI (Grok) TTS provider. Calls the xAI /v1/tts REST endpoint. Returns audio
* as a DriverStreamResult.
*/
export class XAITTSProvider extends TTSProvider {
export class XAITTSProvider implements ITTSProvider {
readonly providerName = 'xai';
#apiKey: string;
constructor(
meteringService: AiMeteringService,
private readonly meteringService: AiMeteringService,
config: { apiKey: string },
) {
super(meteringService, config);
if (!config.apiKey) {
throw new Error('xAI TTS requires an API key');
}
@@ -90,13 +96,8 @@ export class XAITTSProvider extends TTSProvider {
];
}
override getReportedCosts(): Record<string, unknown>[] {
return Object.entries(XAI_TTS_COSTS).map(([model, ucentsPerUnit]) => ({
usageType: `xai:${model}:character`,
ucentsPerUnit,
unit: 'character',
source: 'driver:aiTts/xai',
}));
getReportedCosts(): Record<string, unknown>[] {
return characterCostReport('xai', XAI_TTS_COSTS);
}
async synthesize(
@@ -130,78 +131,46 @@ export class XAITTSProvider extends TTSProvider {
);
}
const voice = voiceArg || DEFAULT_VOICE;
const body: Record<string, unknown> = {
text,
voice_id: voiceArg || DEFAULT_VOICE,
language: language || 'en',
};
const formatStr = output_format || response_format;
const codec = typeof formatStr === 'string' ? formatStr : 'mp3';
if (formatStr) body.output_format = { codec };
const actor = Context.get('actor')!;
const ucentsPerChar = XAI_TTS_COSTS['xai-tts'] ?? 0;
const totalCost = ucentsPerChar * text.length;
const hold = await this.meteringService.reserveAiCredits(
actor,
return meterPerCharacter(
this.meteringService,
Context.get('actor')!,
'xai:xai-tts:character',
totalCost,
);
if (!hold) {
throw insufficientCreditsError();
}
try {
// Build request body
const body: Record<string, unknown> = {
text,
voice_id: voice,
language: language || 'en',
};
// Handle output format
const formatStr = output_format || response_format;
if (formatStr) {
const codec = typeof formatStr === 'string' ? formatStr : 'mp3';
body.output_format = { codec };
}
const response = await upstreamFetch(
'xAI TTS',
`${API_BASE}/tts`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${this.#apiKey}`,
'Content-Type': 'application/json',
XAI_TTS_COSTS['xai-tts'] ?? 0,
text,
async () => {
const response = await upstreamFetch(
'xAI TTS',
`${API_BASE}/tts`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${this.#apiKey}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(body),
},
body: JSON.stringify(body),
},
{ timeoutMs: TTS_UPSTREAM_TIMEOUT_MS },
);
const arrayBuffer = await response.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
const stream = Readable.from(buffer);
// Determine content type from response or codec
const respContentType =
response.headers.get('content-type') || 'audio/mpeg';
const codec =
(body.output_format as { codec?: string } | undefined)?.codec ??
'mp3';
const contentType = CODEC_CONTENT_TYPES[codec] || respContentType;
// Meter usage
this.meteringService.incrementUsage(
actor,
'xai:xai-tts:character',
text.length,
totalCost,
);
return {
dataType: 'stream',
content_type: contentType,
chunked: true,
stream,
};
} finally {
await hold.release();
}
{ timeoutMs: TTS_UPSTREAM_TIMEOUT_MS },
);
const buffer = Buffer.from(await response.arrayBuffer());
return {
dataType: 'stream',
content_type:
CODEC_CONTENT_TYPES[codec] ||
response.headers.get('content-type') ||
'audio/mpeg',
chunked: true,
stream: Readable.from(buffer),
};
},
);
}
}
+3
View File
@@ -68,4 +68,7 @@ export interface ITTSProvider {
/** Synthesize speech from text. Returns a DriverStreamResult. */
synthesize(args: ISynthesizeArgs): Promise<unknown>;
/** Per-unit metering costs, aggregated by the driver. */
getReportedCosts(): Record<string, unknown>[];
}
@@ -16,6 +16,7 @@ import {
type AiMeteringService,
aiModelKey,
withAiCostFactor,
withAiCreditHold,
} from './aiCostFactor.ts';
type Listener = (
@@ -315,4 +316,69 @@ describe('AI cost factor', () => {
expect(boom).toHaveBeenCalled();
expect(result.total).toBe(1000);
});
describe('withAiCreditHold', () => {
const usageType = 'openai:tts-1:character';
it('holds the factored cost while run executes, then releases it', async () => {
listen(1.3);
const check = vi
.spyOn(metering, 'hasEnoughCredits')
.mockResolvedValue(true);
try {
let heldDuringRun = 0;
const result = await withAiCreditHold(
scoped,
actor,
usageType,
1000,
async () => {
heldDuringRun =
await metering.getOutstandingHolds(actor);
return 'done';
},
);
expect(result).toBe('done');
expect(heldDuringRun).toBe(1300);
expect(await metering.getOutstandingHolds(actor)).toBe(0);
} finally {
check.mockRestore();
}
});
it('releases the hold when run throws', async () => {
const check = vi
.spyOn(metering, 'hasEnoughCredits')
.mockResolvedValue(true);
const boom = new Error('upstream down');
try {
await expect(
withAiCreditHold(scoped, actor, usageType, 1000, () =>
Promise.reject(boom),
),
).rejects.toBe(boom);
expect(await metering.getOutstandingHolds(actor)).toBe(0);
} finally {
check.mockRestore();
}
});
it('refuses with a 402 and never runs when the cost is unaffordable', async () => {
const check = vi
.spyOn(metering, 'hasEnoughCredits')
.mockResolvedValue(false);
const run = vi.fn();
try {
await expect(
withAiCreditHold(scoped, actor, usageType, 1000, run),
).rejects.toMatchObject({
statusCode: 402,
legacyCode: 'insufficient_funds',
});
expect(run).not.toHaveBeenCalled();
} finally {
check.mockRestore();
}
});
});
});
+22
View File
@@ -19,6 +19,7 @@
import type { EventClient } from '../../clients/event/EventClient.js';
import type { Actor } from '../../core/actor';
import { insufficientCreditsError } from '../../services/metering/enforcement.js';
import type { MeteringService } from '../../services/metering/MeteringService.js';
import type {
CreditHold,
@@ -280,3 +281,24 @@ export function withAiCostFactor(
forService.set(driver, facade);
return facade;
}
/**
* Holds `amount` (priced as {@link AiMeteringService.reserveAiCredits} prices
* it) while `run` executes, refusing with a 402 when the actor can't afford it.
* The hold is released however `run` ends; `run` records the usage.
*/
export async function withAiCreditHold<T>(
metering: AiMeteringService,
actor: Actor,
usageType: string,
amount: number,
run: () => Promise<T>,
): Promise<T> {
const hold = await metering.reserveAiCredits(actor, usageType, amount);
if (!hold) throw insufficientCreditsError();
try {
return await run();
} finally {
await hold.release();
}
}
+22
View File
@@ -0,0 +1,22 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/** Audio the speech drivers return in `test_mode`, without calling a vendor. */
export const SAMPLE_AUDIO_URL =
'https://puter-sample-data.puter.site/tts_example.mp3';