mirror of
https://github.com/HeyPuter/puter.git
synced 2026-10-17 09:09:43 +00:00
refactor(ai-tts): one per-character metering path for TTS providers
meterPerCharacter holds text length x price, runs the synthesis and meters the same amount once it succeeds; characterCostReport builds the per-character cost lines. Both sit on a new withAiCreditHold (reserve, run, release) that Gemini TTS uses directly. SAMPLE_AUDIO_URL is defined once. The TTSProvider base class is gone: its methods were stubs every provider overrode, and only Polly read its untyped providerConfig.
This commit is contained in:
1 parent
3a8df64c8d
commit
013e1fa687
15 files changed
+673
-585
No files matched your search
@@ -29,6 +29,7 @@ import {
|
||||
} from '../util/aiCostFactor.js';
|
||||
import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js';
|
||||
import { loadFileInput } from '../util/fileInput.js';
|
||||
import { SAMPLE_AUDIO_URL } from '../util/testMode.js';
|
||||
import { upstreamFetch } from '../util/upstreamErrors.js';
|
||||
import { VOICE_CHANGER_COSTS } from './costs.js';
|
||||
|
||||
@@ -40,7 +41,6 @@ import { VOICE_CHANGER_COSTS } from './costs.js';
|
||||
const DEFAULT_MODEL = 'eleven_multilingual_sts_v2';
|
||||
const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM';
|
||||
const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128';
|
||||
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
|
||||
const MAX_AUDIO_FILE_SIZE = 25 * 1024 * 1024;
|
||||
// Covers the upload as well as the conversion.
|
||||
const CONVERT_TIMEOUT_MS = 10 * 60 * 1000;
|
||||
|
||||
@@ -138,23 +138,9 @@ export class TTSDriver extends PuterDriver {
|
||||
}
|
||||
|
||||
override getReportedCosts(): Record<string, unknown>[] {
|
||||
const all: Record<string, unknown>[] = [];
|
||||
for (const p of Object.values(this.#providers)) {
|
||||
const fn = (
|
||||
p as unknown as {
|
||||
getReportedCosts?: () => Record<string, unknown>[];
|
||||
}
|
||||
).getReportedCosts;
|
||||
if (typeof fn === 'function') {
|
||||
try {
|
||||
const entries = fn.call(p);
|
||||
if (Array.isArray(entries)) all.push(...entries);
|
||||
} catch {
|
||||
// ignore — cost reporting is best-effort
|
||||
}
|
||||
}
|
||||
}
|
||||
return all;
|
||||
return Object.values(this.#providers).flatMap((p) =>
|
||||
p.getReportedCosts(),
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -1,67 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Abstract base for TTS providers. Each provider wraps a single upstream API
|
||||
* (OpenAI, ElevenLabs, AWS Polly) and exposes the unified `ITTSProvider`
|
||||
* contract.
|
||||
*/
|
||||
|
||||
import type { AiMeteringService } from '../../util/aiCostFactor.js';
|
||||
import type {
|
||||
ITTSProvider,
|
||||
ITTSVoice,
|
||||
ITTSEngine,
|
||||
ISynthesizeArgs,
|
||||
} from '../types.js';
|
||||
|
||||
export abstract class TTSProvider implements ITTSProvider {
|
||||
abstract readonly providerName: string;
|
||||
|
||||
protected meteringService: AiMeteringService;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
protected providerConfig: any;
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
constructor(meteringService: AiMeteringService, config: any) {
|
||||
this.meteringService = meteringService;
|
||||
this.providerConfig = config;
|
||||
}
|
||||
|
||||
async listVoices(_args?: Record<string, unknown>): Promise<ITTSVoice[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async listEngines(): Promise<ITTSEngine[]> {
|
||||
return [];
|
||||
}
|
||||
|
||||
async synthesize(_args: ISynthesizeArgs): Promise<unknown> {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
|
||||
/**
|
||||
* Provider-specific cost catalogue used by the TTSDriver's aggregated
|
||||
* `getReportedCosts()`. Subclasses override to expose their per-unit
|
||||
* metering costs. Shape matches the `WithCostsReporting` contract.
|
||||
*/
|
||||
getReportedCosts(): Record<string, unknown>[] {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
@@ -26,16 +26,19 @@ import {
|
||||
type VoiceId,
|
||||
} from '@aws-sdk/client-polly';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import type { DriverStreamResult } from '../../../meta.js';
|
||||
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
|
||||
import { TTSProvider } from '../TTSProvider.js';
|
||||
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
|
||||
import type {
|
||||
ITTSVoice,
|
||||
ITTSEngine,
|
||||
ISynthesizeArgs,
|
||||
ITTSProvider,
|
||||
} from '../../types.js';
|
||||
import { characterCostReport, meterPerCharacter } from '../common.js';
|
||||
import { AWS_POLLY_COSTS } from './costs.js';
|
||||
|
||||
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
|
||||
|
||||
const VALID_ENGINES = ['standard', 'neural', 'long-form', 'generative'];
|
||||
|
||||
// Voices tried, in order, when the caller names none. The head of each list
|
||||
@@ -56,7 +59,7 @@ interface PollyVoicesResponse {
|
||||
* audio as a DriverStreamResult. Includes voice caching and engine-aware voice
|
||||
* selection.
|
||||
*/
|
||||
export class AWSPollyTTSProvider extends TTSProvider {
|
||||
export class AWSPollyTTSProvider implements ITTSProvider {
|
||||
readonly providerName = 'aws-polly';
|
||||
|
||||
private clients: Record<string, PollyClient> = {};
|
||||
@@ -64,22 +67,16 @@ export class AWSPollyTTSProvider extends TTSProvider {
|
||||
null;
|
||||
|
||||
constructor(
|
||||
meteringService: AiMeteringService,
|
||||
config: {
|
||||
private readonly meteringService: AiMeteringService,
|
||||
private readonly config: {
|
||||
access_key: string;
|
||||
secret_key: string;
|
||||
region?: string;
|
||||
},
|
||||
) {
|
||||
super(meteringService, config);
|
||||
}
|
||||
) {}
|
||||
|
||||
private getClient(region?: string): PollyClient {
|
||||
const cfg = this.providerConfig as {
|
||||
access_key: string;
|
||||
secret_key: string;
|
||||
region?: string;
|
||||
};
|
||||
const cfg = this.config;
|
||||
const resolvedRegion = region ?? cfg.region ?? 'us-west-2';
|
||||
|
||||
if (this.clients[resolvedRegion]) {
|
||||
@@ -207,15 +204,8 @@ export class AWSPollyTTSProvider extends TTSProvider {
|
||||
}));
|
||||
}
|
||||
|
||||
override getReportedCosts(): Record<string, unknown>[] {
|
||||
return Object.entries(AWS_POLLY_COSTS).map(
|
||||
([engine, ucentsPerUnit]) => ({
|
||||
usageType: `aws-polly:${engine}:character`,
|
||||
ucentsPerUnit,
|
||||
unit: 'character',
|
||||
source: 'driver:aiTts/aws-polly',
|
||||
}),
|
||||
);
|
||||
getReportedCosts(): Record<string, unknown>[] {
|
||||
return characterCostReport('aws-polly', AWS_POLLY_COSTS);
|
||||
}
|
||||
|
||||
async synthesize(
|
||||
@@ -252,65 +242,49 @@ export class AWSPollyTTSProvider extends TTSProvider {
|
||||
});
|
||||
}
|
||||
|
||||
const actor = Context.get('actor')!;
|
||||
const usageType = `aws-polly:${engine}:character`;
|
||||
const ucentsPerChar = AWS_POLLY_COSTS[engine] ?? 0;
|
||||
const totalCost = ucentsPerChar * text.length;
|
||||
return meterPerCharacter(
|
||||
this.meteringService,
|
||||
Context.get('actor')!,
|
||||
`aws-polly:${engine}:character`,
|
||||
AWS_POLLY_COSTS[engine] ?? 0,
|
||||
text,
|
||||
async () => {
|
||||
// Resolve voice
|
||||
let voice = voiceArg ?? undefined;
|
||||
|
||||
const hold = await this.meteringService.reserveAiCredits(
|
||||
actor,
|
||||
usageType,
|
||||
totalCost,
|
||||
if (!voice && language) {
|
||||
voice =
|
||||
(await this.getLanguageAppropriateVoice(
|
||||
language,
|
||||
engine,
|
||||
)) ?? undefined;
|
||||
}
|
||||
|
||||
if (!voice) {
|
||||
voice = await this.getDefaultVoiceForEngine(engine);
|
||||
}
|
||||
|
||||
const client = this.getClient();
|
||||
|
||||
const params = {
|
||||
Engine: engine as Engine,
|
||||
OutputFormat: 'mp3' as const,
|
||||
Text: text,
|
||||
VoiceId: voice as VoiceId,
|
||||
LanguageCode: (language ?? 'en-US') as LanguageCode,
|
||||
TextType: (ssml ? 'ssml' : 'text') as 'ssml' | 'text',
|
||||
};
|
||||
|
||||
const command = new SynthesizeSpeechCommand(params);
|
||||
const response = await client.send(command);
|
||||
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: 'audio/mpeg',
|
||||
chunked: true,
|
||||
stream: response.AudioStream as unknown as import('node:stream').Readable,
|
||||
};
|
||||
},
|
||||
);
|
||||
if (!hold) {
|
||||
throw insufficientCreditsError();
|
||||
}
|
||||
|
||||
try {
|
||||
// Resolve voice
|
||||
let voice = voiceArg ?? undefined;
|
||||
|
||||
if (!voice && language) {
|
||||
voice =
|
||||
(await this.getLanguageAppropriateVoice(
|
||||
language,
|
||||
engine,
|
||||
)) ?? undefined;
|
||||
}
|
||||
|
||||
if (!voice) {
|
||||
voice = await this.getDefaultVoiceForEngine(engine);
|
||||
}
|
||||
|
||||
const client = this.getClient();
|
||||
|
||||
const params = {
|
||||
Engine: engine as Engine,
|
||||
OutputFormat: 'mp3' as const,
|
||||
Text: text,
|
||||
VoiceId: voice as VoiceId,
|
||||
LanguageCode: (language ?? 'en-US') as LanguageCode,
|
||||
TextType: (ssml ? 'ssml' : 'text') as 'ssml' | 'text',
|
||||
};
|
||||
|
||||
const command = new SynthesizeSpeechCommand(params);
|
||||
const response = await client.send(command);
|
||||
|
||||
this.meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
text.length,
|
||||
totalCost,
|
||||
);
|
||||
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: 'audio/mpeg',
|
||||
chunked: true,
|
||||
stream: response.AudioStream as unknown as import('node:stream').Readable,
|
||||
};
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { Readable } from 'node:stream';
|
||||
import {
|
||||
afterAll,
|
||||
afterEach,
|
||||
beforeAll,
|
||||
describe,
|
||||
expect,
|
||||
it,
|
||||
vi,
|
||||
} from 'vitest';
|
||||
import { makeActor } from '../../../core/actor.js';
|
||||
import type { DriverStreamResult } from '../../meta.js';
|
||||
import { PuterServer } from '../../../server.js';
|
||||
import { setupTestServer } from '../../../testUtil.js';
|
||||
import { withAiCostFactor } from '../../util/aiCostFactor.js';
|
||||
import { characterCostReport, meterPerCharacter } from './common.js';
|
||||
|
||||
let server: PuterServer;
|
||||
beforeAll(async () => {
|
||||
server = await setupTestServer();
|
||||
});
|
||||
afterAll(async () => {
|
||||
await server?.shutdown();
|
||||
});
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
const audio = (): Promise<DriverStreamResult> =>
|
||||
Promise.resolve({
|
||||
dataType: 'stream',
|
||||
content_type: 'audio/mpeg',
|
||||
stream: Readable.from([]),
|
||||
});
|
||||
|
||||
describe('meterPerCharacter', () => {
|
||||
const actor = makeActor({
|
||||
user: { id: 7, uuid: 'tts-meter-user', username: 'tts-meter' },
|
||||
});
|
||||
const metering = () =>
|
||||
withAiCostFactor(
|
||||
server.services.metering,
|
||||
server.clients.event,
|
||||
'ai-tts',
|
||||
);
|
||||
|
||||
it('holds and then meters exactly text length × price', async () => {
|
||||
const check = vi
|
||||
.spyOn(server.services.metering, 'hasEnoughCredits')
|
||||
.mockResolvedValue(true);
|
||||
const record = vi.spyOn(server.services.metering, 'incrementUsage');
|
||||
|
||||
await meterPerCharacter(
|
||||
metering(),
|
||||
actor,
|
||||
'openai:tts-1:character',
|
||||
1500,
|
||||
'hello',
|
||||
audio,
|
||||
);
|
||||
|
||||
expect(check).toHaveBeenCalledWith(actor, 7500);
|
||||
expect(record).toHaveBeenCalledWith(
|
||||
actor,
|
||||
'openai:tts-1:character',
|
||||
5,
|
||||
7500,
|
||||
);
|
||||
expect(await server.services.metering.getOutstandingHolds(actor)).toBe(
|
||||
0,
|
||||
);
|
||||
});
|
||||
|
||||
it('meters nothing when synthesis fails', async () => {
|
||||
vi.spyOn(
|
||||
server.services.metering,
|
||||
'hasEnoughCredits',
|
||||
).mockResolvedValue(true);
|
||||
const record = vi.spyOn(server.services.metering, 'incrementUsage');
|
||||
const boom = new Error('vendor down');
|
||||
|
||||
await expect(
|
||||
meterPerCharacter(
|
||||
metering(),
|
||||
actor,
|
||||
'openai:tts-1:character',
|
||||
1500,
|
||||
'hello',
|
||||
() => Promise.reject(boom),
|
||||
),
|
||||
).rejects.toBe(boom);
|
||||
expect(record).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('refuses with a 402 before synthesizing when unaffordable', async () => {
|
||||
vi.spyOn(
|
||||
server.services.metering,
|
||||
'hasEnoughCredits',
|
||||
).mockResolvedValue(false);
|
||||
const synthesize = vi.fn(audio);
|
||||
|
||||
await expect(
|
||||
meterPerCharacter(
|
||||
metering(),
|
||||
actor,
|
||||
'openai:tts-1:character',
|
||||
1500,
|
||||
'hello',
|
||||
synthesize,
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 402 });
|
||||
expect(synthesize).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('characterCostReport', () => {
|
||||
it('emits one per-character line per cost-table entry', () => {
|
||||
expect(
|
||||
characterCostReport('aws-polly', { standard: 400, neural: 1600 }),
|
||||
).toEqual([
|
||||
{
|
||||
usageType: 'aws-polly:standard:character',
|
||||
ucentsPerUnit: 400,
|
||||
unit: 'character',
|
||||
source: 'driver:aiTts/aws-polly',
|
||||
},
|
||||
{
|
||||
usageType: 'aws-polly:neural:character',
|
||||
ucentsPerUnit: 1600,
|
||||
unit: 'character',
|
||||
source: 'driver:aiTts/aws-polly',
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -17,5 +17,45 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import type { Actor } from '../../../core/actor.js';
|
||||
import {
|
||||
type AiMeteringService,
|
||||
withAiCreditHold,
|
||||
} from '../../util/aiCostFactor.js';
|
||||
import type { DriverStreamResult } from '../../meta.js';
|
||||
|
||||
/** Bounds a synthesis call, audio download included. */
|
||||
export const TTS_UPSTREAM_TIMEOUT_MS = 120_000;
|
||||
|
||||
/**
|
||||
* Runs `synthesize` under a hold on `text.length × ucentsPerChar`, then meters
|
||||
* exactly that once it succeeds. Per-character prices are known up front, so
|
||||
* the hold and the charge are the same amount.
|
||||
*/
|
||||
export function meterPerCharacter(
|
||||
metering: AiMeteringService,
|
||||
actor: Actor,
|
||||
usageType: string,
|
||||
ucentsPerChar: number,
|
||||
text: string,
|
||||
synthesize: () => Promise<DriverStreamResult>,
|
||||
): Promise<DriverStreamResult> {
|
||||
const cost = ucentsPerChar * text.length;
|
||||
return withAiCreditHold(metering, actor, usageType, cost, async () => {
|
||||
const result = await synthesize();
|
||||
metering.incrementUsage(actor, usageType, text.length, cost);
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
/** The cost-report lines for a `<model or engine> → ucents/char` table. */
|
||||
export const characterCostReport = (
|
||||
provider: string,
|
||||
costs: Record<string, number>,
|
||||
): Record<string, unknown>[] =>
|
||||
Object.entries(costs).map(([key, ucentsPerUnit]) => ({
|
||||
usageType: `${provider}:${key}:character`,
|
||||
ucentsPerUnit,
|
||||
unit: 'character',
|
||||
source: `driver:aiTts/${provider}`,
|
||||
}));
|
||||
@@ -19,20 +19,27 @@
|
||||
|
||||
import { Readable } from 'node:stream';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import type { DriverStreamResult } from '../../../meta.js';
|
||||
import { upstreamFetch } from '../../../util/upstreamErrors.js';
|
||||
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
|
||||
import { TTSProvider } from '../TTSProvider.js';
|
||||
import { TTS_UPSTREAM_TIMEOUT_MS } from '../common.js';
|
||||
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
|
||||
import type {
|
||||
ITTSVoice,
|
||||
ITTSEngine,
|
||||
ISynthesizeArgs,
|
||||
ITTSProvider,
|
||||
} from '../../types.js';
|
||||
import {
|
||||
characterCostReport,
|
||||
meterPerCharacter,
|
||||
TTS_UPSTREAM_TIMEOUT_MS,
|
||||
} from '../common.js';
|
||||
import { ELEVENLABS_TTS_COSTS } from './costs.js';
|
||||
|
||||
const DEFAULT_MODEL = 'eleven_multilingual_v2';
|
||||
const DEFAULT_VOICE_ID = '21m00Tcm4TlvDq8ikWAM'; // "Rachel" sample voice
|
||||
const DEFAULT_OUTPUT_FORMAT = 'mp3_44100_128';
|
||||
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
|
||||
|
||||
const ELEVENLABS_TTS_MODELS = [
|
||||
{ id: DEFAULT_MODEL, name: 'Eleven Multilingual v2' },
|
||||
@@ -48,7 +55,7 @@ const ELEVENLABS_TTS_MODELS = [
|
||||
* ElevenLabs TTS provider. Uses the ElevenLabs REST API to synthesize speech
|
||||
* and returns audio as a DriverStreamResult.
|
||||
*/
|
||||
export class ElevenLabsTTSProvider extends TTSProvider {
|
||||
export class ElevenLabsTTSProvider implements ITTSProvider {
|
||||
readonly providerName = 'elevenlabs';
|
||||
|
||||
private apiKey: string;
|
||||
@@ -56,15 +63,13 @@ export class ElevenLabsTTSProvider extends TTSProvider {
|
||||
private defaultVoiceId: string;
|
||||
|
||||
constructor(
|
||||
meteringService: AiMeteringService,
|
||||
private readonly meteringService: AiMeteringService,
|
||||
config: {
|
||||
apiKey: string;
|
||||
apiBaseUrl?: string;
|
||||
defaultVoiceId?: string;
|
||||
},
|
||||
) {
|
||||
super(meteringService, config);
|
||||
|
||||
this.apiKey = config.apiKey;
|
||||
this.baseUrl = config.apiBaseUrl ?? 'https://api.elevenlabs.io';
|
||||
this.defaultVoiceId = config.defaultVoiceId ?? DEFAULT_VOICE_ID;
|
||||
@@ -131,15 +136,8 @@ export class ElevenLabsTTSProvider extends TTSProvider {
|
||||
}));
|
||||
}
|
||||
|
||||
override getReportedCosts(): Record<string, unknown>[] {
|
||||
return Object.entries(ELEVENLABS_TTS_COSTS).map(
|
||||
([model, ucentsPerUnit]) => ({
|
||||
usageType: `elevenlabs:${model}:character`,
|
||||
ucentsPerUnit,
|
||||
unit: 'character',
|
||||
source: 'driver:aiTts/elevenlabs',
|
||||
}),
|
||||
);
|
||||
getReportedCosts(): Record<string, unknown>[] {
|
||||
return characterCostReport('elevenlabs', ELEVENLABS_TTS_COSTS);
|
||||
}
|
||||
|
||||
async synthesize(
|
||||
@@ -185,66 +183,35 @@ export class ElevenLabsTTSProvider extends TTSProvider {
|
||||
);
|
||||
}
|
||||
|
||||
const desiredFormat =
|
||||
output_format || response_format || DEFAULT_OUTPUT_FORMAT;
|
||||
const payload: Record<string, unknown> = {
|
||||
text,
|
||||
model_id: modelId,
|
||||
output_format:
|
||||
output_format || response_format || DEFAULT_OUTPUT_FORMAT,
|
||||
};
|
||||
const finalVoiceSettings = voice_settings ?? voiceSettings;
|
||||
if (finalVoiceSettings) payload.voice_settings = finalVoiceSettings;
|
||||
|
||||
const actor = Context.get('actor')!;
|
||||
const usageKey = `elevenlabs:${modelId}:character`;
|
||||
const ucentsPerChar = ELEVENLABS_TTS_COSTS[modelId];
|
||||
const totalCost = ucentsPerChar * text.length;
|
||||
|
||||
const hold = await this.meteringService.reserveAiCredits(
|
||||
actor,
|
||||
usageKey,
|
||||
totalCost,
|
||||
return meterPerCharacter(
|
||||
this.meteringService,
|
||||
Context.get('actor')!,
|
||||
`elevenlabs:${modelId}:character`,
|
||||
ELEVENLABS_TTS_COSTS[modelId],
|
||||
text,
|
||||
async () => {
|
||||
const response = await this.request(
|
||||
`/v1/text-to-speech/${voiceId}`,
|
||||
{ method: 'POST', body: payload },
|
||||
);
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type:
|
||||
response.headers.get('content-type') || 'audio/mpeg',
|
||||
chunked: true,
|
||||
stream: Readable.from(buffer),
|
||||
};
|
||||
},
|
||||
);
|
||||
if (!hold) {
|
||||
throw insufficientCreditsError();
|
||||
}
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const payload: any = {
|
||||
text,
|
||||
model_id: modelId,
|
||||
output_format: desiredFormat,
|
||||
};
|
||||
|
||||
const finalVoiceSettings = voice_settings ?? voiceSettings;
|
||||
if (finalVoiceSettings) {
|
||||
payload.voice_settings = finalVoiceSettings;
|
||||
}
|
||||
|
||||
const response = await this.request(
|
||||
`/v1/text-to-speech/${voiceId}`,
|
||||
{
|
||||
method: 'POST',
|
||||
body: payload,
|
||||
},
|
||||
);
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const stream = Readable.from(buffer);
|
||||
|
||||
this.meteringService.incrementUsage(
|
||||
actor,
|
||||
usageKey,
|
||||
text.length,
|
||||
totalCost,
|
||||
);
|
||||
|
||||
const contentType =
|
||||
response.headers.get('content-type') || 'audio/mpeg';
|
||||
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: contentType,
|
||||
chunked: true,
|
||||
stream,
|
||||
};
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -20,17 +20,23 @@
|
||||
import { GoogleGenAI } from '@google/genai';
|
||||
import { Readable } from 'node:stream';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import {
|
||||
type AiMeteringService,
|
||||
withAiCreditHold,
|
||||
} from '../../../util/aiCostFactor.js';
|
||||
import type { DriverStreamResult } from '../../../meta.js';
|
||||
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
|
||||
import { TTSProvider } from '../TTSProvider.js';
|
||||
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
|
||||
import type {
|
||||
ITTSVoice,
|
||||
ITTSEngine,
|
||||
ISynthesizeArgs,
|
||||
ITTSProvider,
|
||||
} from '../../types.js';
|
||||
import { GEMINI_TTS_COSTS } from './costs.js';
|
||||
|
||||
const DEFAULT_MODEL = 'gemini-3.8-flash-tts';
|
||||
const DEFAULT_VOICE = 'Kore';
|
||||
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
|
||||
|
||||
const GEMINI_TTS_MODELS = [
|
||||
{
|
||||
@@ -81,16 +87,15 @@ const GEMINI_TTS_VOICES = [
|
||||
* `responseModalities: ["AUDIO"]` and `speechConfig` to synthesize speech.
|
||||
* Returns raw PCM audio wrapped in a WAV container.
|
||||
*/
|
||||
export class GeminiTTSProvider extends TTSProvider {
|
||||
export class GeminiTTSProvider implements ITTSProvider {
|
||||
readonly providerName = 'gemini';
|
||||
|
||||
#client: GoogleGenAI;
|
||||
|
||||
constructor(
|
||||
meteringService: AiMeteringService,
|
||||
private readonly meteringService: AiMeteringService,
|
||||
config: { apiKey: string },
|
||||
) {
|
||||
super(meteringService, config);
|
||||
if (!config.apiKey) {
|
||||
throw new Error('Gemini TTS requires an API key');
|
||||
}
|
||||
@@ -115,7 +120,7 @@ export class GeminiTTSProvider extends TTSProvider {
|
||||
}));
|
||||
}
|
||||
|
||||
override getReportedCosts(): Record<string, unknown>[] {
|
||||
getReportedCosts(): Record<string, unknown>[] {
|
||||
return Object.entries(GEMINI_TTS_COSTS).map(([model, costs]) => ({
|
||||
usageType: `gemini:${model}:tts`,
|
||||
ucentsInputPerToken: this.#toMicroCents(costs.input / 1_000_000),
|
||||
@@ -212,114 +217,114 @@ export class GeminiTTSProvider extends TTSProvider {
|
||||
estimatedInputCostCents + estimatedOutputCostCents,
|
||||
);
|
||||
|
||||
const hold = await this.meteringService.reserveAiCredits(
|
||||
return withAiCreditHold(
|
||||
this.meteringService,
|
||||
actor,
|
||||
`gemini:${model}`,
|
||||
estimatedTotalMicroCents,
|
||||
);
|
||||
if (!hold) {
|
||||
throw insufficientCreditsError();
|
||||
}
|
||||
async (): Promise<DriverStreamResult> => {
|
||||
// The TTS models require the text to be framed as a transcript
|
||||
// to read aloud. Prefixing with "Say:" prevents the model from
|
||||
// trying to generate conversational text instead of audio.
|
||||
const inputText = instructions
|
||||
? `${instructions}\n\nSay the following text aloud:\n${text}`
|
||||
: `Say the following text aloud:\n${text}`;
|
||||
|
||||
try {
|
||||
// The TTS models require the text to be framed as a transcript
|
||||
// to read aloud. Prefixing with "Say:" prevents the model from
|
||||
// trying to generate conversational text instead of audio.
|
||||
const inputText = instructions
|
||||
? `${instructions}\n\nSay the following text aloud:\n${text}`
|
||||
: `Say the following text aloud:\n${text}`;
|
||||
|
||||
// Let Google GenAI `ApiError`s bubble — they carry `.status` and
|
||||
// are mapped to `upstream_*` HttpErrors by the driver-boundary
|
||||
// translator. Catching here and wrapping as 502 hid the upstream
|
||||
// status and caused 4xx validation errors to page.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const response: any = await this.#client.models.generateContent({
|
||||
model,
|
||||
contents: [{ parts: [{ text: inputText }] }],
|
||||
config: {
|
||||
responseModalities: ['AUDIO'],
|
||||
speechConfig: {
|
||||
voiceConfig: {
|
||||
prebuiltVoiceConfig: { voiceName: voice },
|
||||
// Let Google GenAI `ApiError`s bubble — they carry `.status` and
|
||||
// are mapped to `upstream_*` HttpErrors by the driver-boundary
|
||||
// translator. Catching here and wrapping as 502 hid the upstream
|
||||
// status and caused 4xx validation errors to page.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const response: any = await this.#client.models.generateContent(
|
||||
{
|
||||
model,
|
||||
contents: [{ parts: [{ text: inputText }] }],
|
||||
config: {
|
||||
responseModalities: ['AUDIO'],
|
||||
speechConfig: {
|
||||
voiceConfig: {
|
||||
prebuiltVoiceConfig: { voiceName: voice },
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
// Extract audio data from response
|
||||
const part = response?.candidates?.[0]?.content?.parts?.[0];
|
||||
if (!part?.inlineData?.data) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Gemini TTS did not return audio data',
|
||||
{
|
||||
legacyCode: 'upstream_bad_request',
|
||||
fields: { provider: 'gemini' },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
const audioBase64: string = part.inlineData.data;
|
||||
const mimeType: string =
|
||||
part.inlineData.mimeType || 'audio/L16;rate=24000';
|
||||
// Extract audio data from response
|
||||
const part = response?.candidates?.[0]?.content?.parts?.[0];
|
||||
if (!part?.inlineData?.data) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Gemini TTS did not return audio data',
|
||||
{
|
||||
legacyCode: 'upstream_bad_request',
|
||||
fields: { provider: 'gemini' },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Convert base64 PCM to a WAV buffer for broad client compatibility
|
||||
const pcmBuffer = Buffer.from(audioBase64, 'base64');
|
||||
let outputBuffer: Buffer;
|
||||
let contentType: string;
|
||||
const audioBase64: string = part.inlineData.data;
|
||||
const mimeType: string =
|
||||
part.inlineData.mimeType || 'audio/L16;rate=24000';
|
||||
|
||||
if (mimeType.startsWith('audio/L16') || mimeType === 'audio/pcm') {
|
||||
// Wrap raw PCM (16-bit LE, 24kHz, mono) in a WAV container
|
||||
outputBuffer = this.#wrapPcmInWav(pcmBuffer, 24000, 1, 16);
|
||||
contentType = 'audio/wav';
|
||||
} else {
|
||||
// If the API returns encoded audio (unlikely today), pass through
|
||||
outputBuffer = pcmBuffer;
|
||||
contentType = mimeType;
|
||||
}
|
||||
// Convert base64 PCM to a WAV buffer for broad client compatibility
|
||||
const pcmBuffer = Buffer.from(audioBase64, 'base64');
|
||||
let outputBuffer: Buffer;
|
||||
let contentType: string;
|
||||
|
||||
// Meter actual usage from response metadata
|
||||
const usage = response.usageMetadata;
|
||||
const actualInputTokens =
|
||||
typeof usage?.promptTokenCount === 'number'
|
||||
? usage.promptTokenCount
|
||||
: estimatedInputTokens;
|
||||
const actualOutputTokens =
|
||||
typeof usage?.candidatesTokenCount === 'number'
|
||||
? usage.candidatesTokenCount
|
||||
: estimatedOutputTokens;
|
||||
if (
|
||||
mimeType.startsWith('audio/L16') ||
|
||||
mimeType === 'audio/pcm'
|
||||
) {
|
||||
// Wrap raw PCM (16-bit LE, 24kHz, mono) in a WAV container
|
||||
outputBuffer = this.#wrapPcmInWav(pcmBuffer, 24000, 1, 16);
|
||||
contentType = 'audio/wav';
|
||||
} else {
|
||||
// If the API returns encoded audio (unlikely today), pass through
|
||||
outputBuffer = pcmBuffer;
|
||||
contentType = mimeType;
|
||||
}
|
||||
|
||||
const inputCostCents =
|
||||
(actualInputTokens / 1_000_000) * costs.input;
|
||||
const outputCostCents =
|
||||
(actualOutputTokens / 1_000_000) * costs.output_audio;
|
||||
// Meter actual usage from response metadata
|
||||
const usage = response.usageMetadata;
|
||||
const actualInputTokens =
|
||||
typeof usage?.promptTokenCount === 'number'
|
||||
? usage.promptTokenCount
|
||||
: estimatedInputTokens;
|
||||
const actualOutputTokens =
|
||||
typeof usage?.candidatesTokenCount === 'number'
|
||||
? usage.candidatesTokenCount
|
||||
: estimatedOutputTokens;
|
||||
|
||||
const usagePrefix = `gemini:${model}`;
|
||||
this.meteringService.batchIncrementUsages(actor, [
|
||||
{
|
||||
usageType: `${usagePrefix}:input`,
|
||||
usageAmount: Math.max(actualInputTokens, 1),
|
||||
costOverride: this.#toMicroCents(inputCostCents),
|
||||
},
|
||||
{
|
||||
usageType: `${usagePrefix}:output:audio`,
|
||||
usageAmount: Math.max(actualOutputTokens, 1),
|
||||
costOverride: this.#toMicroCents(outputCostCents),
|
||||
},
|
||||
]);
|
||||
const inputCostCents =
|
||||
(actualInputTokens / 1_000_000) * costs.input;
|
||||
const outputCostCents =
|
||||
(actualOutputTokens / 1_000_000) * costs.output_audio;
|
||||
|
||||
const stream = Readable.from(outputBuffer);
|
||||
const usagePrefix = `gemini:${model}`;
|
||||
this.meteringService.batchIncrementUsages(actor, [
|
||||
{
|
||||
usageType: `${usagePrefix}:input`,
|
||||
usageAmount: Math.max(actualInputTokens, 1),
|
||||
costOverride: this.#toMicroCents(inputCostCents),
|
||||
},
|
||||
{
|
||||
usageType: `${usagePrefix}:output:audio`,
|
||||
usageAmount: Math.max(actualOutputTokens, 1),
|
||||
costOverride: this.#toMicroCents(outputCostCents),
|
||||
},
|
||||
]);
|
||||
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: contentType,
|
||||
chunked: true,
|
||||
stream,
|
||||
};
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
const stream = Readable.from(outputBuffer);
|
||||
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: contentType,
|
||||
chunked: true,
|
||||
stream,
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
/** Wrap raw PCM samples in a WAV container so browsers can play it. */
|
||||
|
||||
@@ -20,17 +20,21 @@
|
||||
import OpenAI from 'openai';
|
||||
import { Readable } from 'node:stream';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import type { DriverStreamResult } from '../../../meta.js';
|
||||
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
|
||||
import { TTSProvider } from '../TTSProvider.js';
|
||||
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
|
||||
import type {
|
||||
ITTSVoice,
|
||||
ITTSEngine,
|
||||
ISynthesizeArgs,
|
||||
ITTSProvider,
|
||||
} from '../../types.js';
|
||||
import { characterCostReport, meterPerCharacter } from '../common.js';
|
||||
import { OPENAI_TTS_COSTS } from './costs.js';
|
||||
|
||||
const DEFAULT_MODEL = 'gpt-4o-mini-tts';
|
||||
const DEFAULT_VOICE = 'alloy';
|
||||
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
|
||||
|
||||
const RESPONSE_CONTENT_TYPES: Record<string, string> = {
|
||||
mp3: 'audio/mpeg',
|
||||
@@ -80,16 +84,15 @@ const OPENAI_TTS_MODELS = [
|
||||
* OpenAI TTS provider. Wraps the OpenAI speech synthesis API and returns audio
|
||||
* as a DriverStreamResult.
|
||||
*/
|
||||
export class OpenAITTSProvider extends TTSProvider {
|
||||
export class OpenAITTSProvider implements ITTSProvider {
|
||||
readonly providerName = 'openai';
|
||||
|
||||
private openai: OpenAI;
|
||||
|
||||
constructor(
|
||||
meteringService: AiMeteringService,
|
||||
private readonly meteringService: AiMeteringService,
|
||||
config: { apiKey: string },
|
||||
) {
|
||||
super(meteringService, config);
|
||||
this.openai = new OpenAI({ apiKey: config.apiKey });
|
||||
}
|
||||
|
||||
@@ -116,15 +119,8 @@ export class OpenAITTSProvider extends TTSProvider {
|
||||
}));
|
||||
}
|
||||
|
||||
override getReportedCosts(): Record<string, unknown>[] {
|
||||
return Object.entries(OPENAI_TTS_COSTS).map(
|
||||
([model, ucentsPerUnit]) => ({
|
||||
usageType: `openai:${model}:character`,
|
||||
ucentsPerUnit,
|
||||
unit: 'character',
|
||||
source: 'driver:aiTts/openai',
|
||||
}),
|
||||
);
|
||||
getReportedCosts(): Record<string, unknown>[] {
|
||||
return characterCostReport('openai', OPENAI_TTS_COSTS);
|
||||
}
|
||||
|
||||
async synthesize(
|
||||
@@ -206,56 +202,27 @@ export class OpenAITTSProvider extends TTSProvider {
|
||||
const contentType =
|
||||
RESPONSE_CONTENT_TYPES[format] || RESPONSE_CONTENT_TYPES.mp3;
|
||||
|
||||
const actor = Context.get('actor')!;
|
||||
const usageType = `openai:${model}:character`;
|
||||
const ucentsPerChar = OPENAI_TTS_COSTS[model] ?? 0;
|
||||
const totalCost = ucentsPerChar * text.length;
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const payload: any = { model, voice, input: text };
|
||||
if (instructions) payload.instructions = instructions;
|
||||
if (response_format) payload.response_format = response_format;
|
||||
|
||||
const hold = await this.meteringService.reserveAiCredits(
|
||||
actor,
|
||||
usageType,
|
||||
totalCost,
|
||||
return meterPerCharacter(
|
||||
this.meteringService,
|
||||
Context.get('actor')!,
|
||||
`openai:${model}:character`,
|
||||
OPENAI_TTS_COSTS[model] ?? 0,
|
||||
text,
|
||||
async () => {
|
||||
const response = await this.openai.audio.speech.create(payload);
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: contentType,
|
||||
chunked: true,
|
||||
stream: Readable.from(buffer),
|
||||
};
|
||||
},
|
||||
);
|
||||
if (!hold) {
|
||||
throw insufficientCreditsError();
|
||||
}
|
||||
|
||||
try {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const payload: any = {
|
||||
model,
|
||||
voice,
|
||||
input: text,
|
||||
};
|
||||
|
||||
if (instructions) {
|
||||
payload.instructions = instructions;
|
||||
}
|
||||
|
||||
if (response_format) {
|
||||
payload.response_format = response_format;
|
||||
}
|
||||
|
||||
const response = await this.openai.audio.speech.create(payload);
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const stream = Readable.from(buffer);
|
||||
|
||||
this.meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
text.length,
|
||||
totalCost,
|
||||
);
|
||||
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: contentType,
|
||||
chunked: true,
|
||||
stream,
|
||||
};
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,21 +19,28 @@
|
||||
|
||||
import { Readable } from 'node:stream';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import type { DriverStreamResult } from '../../../meta.js';
|
||||
import { upstreamFetch } from '../../../util/upstreamErrors.js';
|
||||
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
|
||||
import { TTSProvider } from '../TTSProvider.js';
|
||||
import { TTS_UPSTREAM_TIMEOUT_MS } from '../common.js';
|
||||
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
|
||||
import type {
|
||||
ITTSVoice,
|
||||
ITTSEngine,
|
||||
ISynthesizeArgs,
|
||||
ITTSProvider,
|
||||
} from '../../types.js';
|
||||
import {
|
||||
characterCostReport,
|
||||
meterPerCharacter,
|
||||
TTS_UPSTREAM_TIMEOUT_MS,
|
||||
} from '../common.js';
|
||||
import { SPEECHIFY_TTS_COSTS } from './costs.js';
|
||||
|
||||
// Public API base only — never an internal/consumer Speechify endpoint.
|
||||
const API_BASE = 'https://api.speechify.ai';
|
||||
const CALLER_HEADER = 'Speechify-Caller';
|
||||
const CALLER_VALUE = 'puter';
|
||||
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
|
||||
|
||||
const DEFAULT_MODEL = 'simba-3.2';
|
||||
const DEFAULT_VOICE = 'geffen_32';
|
||||
@@ -62,16 +69,15 @@ const CONTENT_TYPES: Record<string, string> = {
|
||||
* and returns audio as a DriverStreamResult. Every outbound request carries
|
||||
* `Speechify-Caller: puter` for integration attribution.
|
||||
*/
|
||||
export class SpeechifyTTSProvider extends TTSProvider {
|
||||
export class SpeechifyTTSProvider implements ITTSProvider {
|
||||
readonly providerName = 'speechify';
|
||||
|
||||
#apiKey: string;
|
||||
|
||||
constructor(
|
||||
meteringService: AiMeteringService,
|
||||
private readonly meteringService: AiMeteringService,
|
||||
config: { apiKey: string },
|
||||
) {
|
||||
super(meteringService, config);
|
||||
if (!config.apiKey) {
|
||||
throw new Error('Speechify TTS requires an API key');
|
||||
}
|
||||
@@ -96,15 +102,8 @@ export class SpeechifyTTSProvider extends TTSProvider {
|
||||
}));
|
||||
}
|
||||
|
||||
override getReportedCosts(): Record<string, unknown>[] {
|
||||
return Object.entries(SPEECHIFY_TTS_COSTS).map(
|
||||
([model, ucentsPerUnit]) => ({
|
||||
usageType: `speechify:${model}:character`,
|
||||
ucentsPerUnit,
|
||||
unit: 'character',
|
||||
source: 'driver:aiTts/speechify',
|
||||
}),
|
||||
);
|
||||
getReportedCosts(): Record<string, unknown>[] {
|
||||
return characterCostReport('speechify', SPEECHIFY_TTS_COSTS);
|
||||
}
|
||||
|
||||
async synthesize(
|
||||
@@ -148,83 +147,64 @@ export class SpeechifyTTSProvider extends TTSProvider {
|
||||
);
|
||||
}
|
||||
|
||||
const voice = voiceArg || DEFAULT_VOICE;
|
||||
const format = output_format || response_format || 'mp3';
|
||||
// Speechify's synthesis endpoint expects SSML; wrap plain text so
|
||||
// callers can keep passing bare strings like every other provider.
|
||||
const input = /<speak[\s>]/i.test(text)
|
||||
? text
|
||||
: `<speak>${text}</speak>`;
|
||||
|
||||
const actor = Context.get('actor')!;
|
||||
const usageType = `speechify:${model}:character`;
|
||||
const ucentsPerChar = SPEECHIFY_TTS_COSTS[model] ?? 0;
|
||||
const totalCost = ucentsPerChar * text.length;
|
||||
|
||||
const hold = await this.meteringService.reserveAiCredits(
|
||||
actor,
|
||||
usageType,
|
||||
totalCost,
|
||||
);
|
||||
if (!hold) {
|
||||
throw insufficientCreditsError();
|
||||
}
|
||||
|
||||
try {
|
||||
// Speechify's synthesis endpoint expects SSML; wrap plain text so
|
||||
// callers can keep passing bare strings like every other provider.
|
||||
const input = /<speak[\s>]/i.test(text)
|
||||
? text
|
||||
: `<speak>${text}</speak>`;
|
||||
|
||||
const response = await upstreamFetch(
|
||||
'Speechify TTS',
|
||||
`${API_BASE}/v1/audio/speech`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.#apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
[CALLER_HEADER]: CALLER_VALUE,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
input,
|
||||
voice_id: voice,
|
||||
model,
|
||||
audio_format: format,
|
||||
}),
|
||||
},
|
||||
{ timeoutMs: TTS_UPSTREAM_TIMEOUT_MS },
|
||||
);
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const data: any = await response.json();
|
||||
if (!data?.audio_data) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Speechify TTS did not return audio data',
|
||||
return meterPerCharacter(
|
||||
this.meteringService,
|
||||
Context.get('actor')!,
|
||||
`speechify:${model}:character`,
|
||||
SPEECHIFY_TTS_COSTS[model] ?? 0,
|
||||
text,
|
||||
async () => {
|
||||
const response = await upstreamFetch(
|
||||
'Speechify TTS',
|
||||
`${API_BASE}/v1/audio/speech`,
|
||||
{
|
||||
legacyCode: 'upstream_bad_request',
|
||||
fields: { provider: 'speechify' },
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.#apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
[CALLER_HEADER]: CALLER_VALUE,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
input,
|
||||
voice_id: voiceArg || DEFAULT_VOICE,
|
||||
model,
|
||||
audio_format: format,
|
||||
}),
|
||||
},
|
||||
{ timeoutMs: TTS_UPSTREAM_TIMEOUT_MS },
|
||||
);
|
||||
}
|
||||
|
||||
const buffer = Buffer.from(data.audio_data, 'base64');
|
||||
const stream = Readable.from(buffer);
|
||||
const contentType =
|
||||
CONTENT_TYPES[data.audio_format ?? format] || 'audio/mpeg';
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
const data: any = await response.json();
|
||||
if (!data?.audio_data) {
|
||||
throw new HttpError(
|
||||
400,
|
||||
'Speechify TTS did not return audio data',
|
||||
{
|
||||
legacyCode: 'upstream_bad_request',
|
||||
fields: { provider: 'speechify' },
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
this.meteringService.incrementUsage(
|
||||
actor,
|
||||
usageType,
|
||||
text.length,
|
||||
totalCost,
|
||||
);
|
||||
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: contentType,
|
||||
chunked: true,
|
||||
stream,
|
||||
};
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type:
|
||||
CONTENT_TYPES[data.audio_format ?? format] ||
|
||||
'audio/mpeg',
|
||||
chunked: true,
|
||||
stream: Readable.from(
|
||||
Buffer.from(data.audio_data, 'base64'),
|
||||
),
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -19,18 +19,25 @@
|
||||
|
||||
import { Readable } from 'node:stream';
|
||||
import { HttpError } from '../../../../core/http/HttpError.js';
|
||||
import { insufficientCreditsError } from '../../../../services/metering/enforcement.js';
|
||||
import { Context } from '../../../../core/context.js';
|
||||
import type { AiMeteringService } from '../../../util/aiCostFactor.js';
|
||||
import type { DriverStreamResult } from '../../../meta.js';
|
||||
import { upstreamFetch } from '../../../util/upstreamErrors.js';
|
||||
import type { ITTSVoice, ITTSEngine, ISynthesizeArgs } from '../../types.js';
|
||||
import { TTSProvider } from '../TTSProvider.js';
|
||||
import { TTS_UPSTREAM_TIMEOUT_MS } from '../common.js';
|
||||
import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js';
|
||||
import type {
|
||||
ITTSVoice,
|
||||
ITTSEngine,
|
||||
ISynthesizeArgs,
|
||||
ITTSProvider,
|
||||
} from '../../types.js';
|
||||
import {
|
||||
characterCostReport,
|
||||
meterPerCharacter,
|
||||
TTS_UPSTREAM_TIMEOUT_MS,
|
||||
} from '../common.js';
|
||||
import { XAI_TTS_COSTS } from './costs.js';
|
||||
|
||||
const API_BASE = 'https://api.x.ai/v1';
|
||||
const SAMPLE_AUDIO_URL = 'https://puter-sample-data.puter.site/tts_example.mp3';
|
||||
|
||||
const XAI_TTS_VOICES = [
|
||||
{ id: 'eve', name: 'Eve', description: 'Energetic, upbeat' },
|
||||
@@ -54,16 +61,15 @@ const CODEC_CONTENT_TYPES: Record<string, string> = {
|
||||
* XAI (Grok) TTS provider. Calls the xAI /v1/tts REST endpoint. Returns audio
|
||||
* as a DriverStreamResult.
|
||||
*/
|
||||
export class XAITTSProvider extends TTSProvider {
|
||||
export class XAITTSProvider implements ITTSProvider {
|
||||
readonly providerName = 'xai';
|
||||
|
||||
#apiKey: string;
|
||||
|
||||
constructor(
|
||||
meteringService: AiMeteringService,
|
||||
private readonly meteringService: AiMeteringService,
|
||||
config: { apiKey: string },
|
||||
) {
|
||||
super(meteringService, config);
|
||||
if (!config.apiKey) {
|
||||
throw new Error('xAI TTS requires an API key');
|
||||
}
|
||||
@@ -90,13 +96,8 @@ export class XAITTSProvider extends TTSProvider {
|
||||
];
|
||||
}
|
||||
|
||||
override getReportedCosts(): Record<string, unknown>[] {
|
||||
return Object.entries(XAI_TTS_COSTS).map(([model, ucentsPerUnit]) => ({
|
||||
usageType: `xai:${model}:character`,
|
||||
ucentsPerUnit,
|
||||
unit: 'character',
|
||||
source: 'driver:aiTts/xai',
|
||||
}));
|
||||
getReportedCosts(): Record<string, unknown>[] {
|
||||
return characterCostReport('xai', XAI_TTS_COSTS);
|
||||
}
|
||||
|
||||
async synthesize(
|
||||
@@ -130,78 +131,46 @@ export class XAITTSProvider extends TTSProvider {
|
||||
);
|
||||
}
|
||||
|
||||
const voice = voiceArg || DEFAULT_VOICE;
|
||||
const body: Record<string, unknown> = {
|
||||
text,
|
||||
voice_id: voiceArg || DEFAULT_VOICE,
|
||||
language: language || 'en',
|
||||
};
|
||||
const formatStr = output_format || response_format;
|
||||
const codec = typeof formatStr === 'string' ? formatStr : 'mp3';
|
||||
if (formatStr) body.output_format = { codec };
|
||||
|
||||
const actor = Context.get('actor')!;
|
||||
const ucentsPerChar = XAI_TTS_COSTS['xai-tts'] ?? 0;
|
||||
const totalCost = ucentsPerChar * text.length;
|
||||
|
||||
const hold = await this.meteringService.reserveAiCredits(
|
||||
actor,
|
||||
return meterPerCharacter(
|
||||
this.meteringService,
|
||||
Context.get('actor')!,
|
||||
'xai:xai-tts:character',
|
||||
totalCost,
|
||||
);
|
||||
if (!hold) {
|
||||
throw insufficientCreditsError();
|
||||
}
|
||||
|
||||
try {
|
||||
// Build request body
|
||||
const body: Record<string, unknown> = {
|
||||
text,
|
||||
voice_id: voice,
|
||||
language: language || 'en',
|
||||
};
|
||||
|
||||
// Handle output format
|
||||
const formatStr = output_format || response_format;
|
||||
if (formatStr) {
|
||||
const codec = typeof formatStr === 'string' ? formatStr : 'mp3';
|
||||
body.output_format = { codec };
|
||||
}
|
||||
|
||||
const response = await upstreamFetch(
|
||||
'xAI TTS',
|
||||
`${API_BASE}/tts`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.#apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
XAI_TTS_COSTS['xai-tts'] ?? 0,
|
||||
text,
|
||||
async () => {
|
||||
const response = await upstreamFetch(
|
||||
'xAI TTS',
|
||||
`${API_BASE}/tts`,
|
||||
{
|
||||
method: 'POST',
|
||||
headers: {
|
||||
Authorization: `Bearer ${this.#apiKey}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
},
|
||||
{ timeoutMs: TTS_UPSTREAM_TIMEOUT_MS },
|
||||
);
|
||||
|
||||
const arrayBuffer = await response.arrayBuffer();
|
||||
const buffer = Buffer.from(arrayBuffer);
|
||||
const stream = Readable.from(buffer);
|
||||
|
||||
// Determine content type from response or codec
|
||||
const respContentType =
|
||||
response.headers.get('content-type') || 'audio/mpeg';
|
||||
const codec =
|
||||
(body.output_format as { codec?: string } | undefined)?.codec ??
|
||||
'mp3';
|
||||
const contentType = CODEC_CONTENT_TYPES[codec] || respContentType;
|
||||
|
||||
// Meter usage
|
||||
this.meteringService.incrementUsage(
|
||||
actor,
|
||||
'xai:xai-tts:character',
|
||||
text.length,
|
||||
totalCost,
|
||||
);
|
||||
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type: contentType,
|
||||
chunked: true,
|
||||
stream,
|
||||
};
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
{ timeoutMs: TTS_UPSTREAM_TIMEOUT_MS },
|
||||
);
|
||||
const buffer = Buffer.from(await response.arrayBuffer());
|
||||
return {
|
||||
dataType: 'stream',
|
||||
content_type:
|
||||
CODEC_CONTENT_TYPES[codec] ||
|
||||
response.headers.get('content-type') ||
|
||||
'audio/mpeg',
|
||||
chunked: true,
|
||||
stream: Readable.from(buffer),
|
||||
};
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -68,4 +68,7 @@ export interface ITTSProvider {
|
||||
|
||||
/** Synthesize speech from text. Returns a DriverStreamResult. */
|
||||
synthesize(args: ISynthesizeArgs): Promise<unknown>;
|
||||
|
||||
/** Per-unit metering costs, aggregated by the driver. */
|
||||
getReportedCosts(): Record<string, unknown>[];
|
||||
}
|
||||
@@ -16,6 +16,7 @@ import {
|
||||
type AiMeteringService,
|
||||
aiModelKey,
|
||||
withAiCostFactor,
|
||||
withAiCreditHold,
|
||||
} from './aiCostFactor.ts';
|
||||
|
||||
type Listener = (
|
||||
@@ -315,4 +316,69 @@ describe('AI cost factor', () => {
|
||||
expect(boom).toHaveBeenCalled();
|
||||
expect(result.total).toBe(1000);
|
||||
});
|
||||
|
||||
describe('withAiCreditHold', () => {
|
||||
const usageType = 'openai:tts-1:character';
|
||||
|
||||
it('holds the factored cost while run executes, then releases it', async () => {
|
||||
listen(1.3);
|
||||
const check = vi
|
||||
.spyOn(metering, 'hasEnoughCredits')
|
||||
.mockResolvedValue(true);
|
||||
try {
|
||||
let heldDuringRun = 0;
|
||||
const result = await withAiCreditHold(
|
||||
scoped,
|
||||
actor,
|
||||
usageType,
|
||||
1000,
|
||||
async () => {
|
||||
heldDuringRun =
|
||||
await metering.getOutstandingHolds(actor);
|
||||
return 'done';
|
||||
},
|
||||
);
|
||||
expect(result).toBe('done');
|
||||
expect(heldDuringRun).toBe(1300);
|
||||
expect(await metering.getOutstandingHolds(actor)).toBe(0);
|
||||
} finally {
|
||||
check.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('releases the hold when run throws', async () => {
|
||||
const check = vi
|
||||
.spyOn(metering, 'hasEnoughCredits')
|
||||
.mockResolvedValue(true);
|
||||
const boom = new Error('upstream down');
|
||||
try {
|
||||
await expect(
|
||||
withAiCreditHold(scoped, actor, usageType, 1000, () =>
|
||||
Promise.reject(boom),
|
||||
),
|
||||
).rejects.toBe(boom);
|
||||
expect(await metering.getOutstandingHolds(actor)).toBe(0);
|
||||
} finally {
|
||||
check.mockRestore();
|
||||
}
|
||||
});
|
||||
|
||||
it('refuses with a 402 and never runs when the cost is unaffordable', async () => {
|
||||
const check = vi
|
||||
.spyOn(metering, 'hasEnoughCredits')
|
||||
.mockResolvedValue(false);
|
||||
const run = vi.fn();
|
||||
try {
|
||||
await expect(
|
||||
withAiCreditHold(scoped, actor, usageType, 1000, run),
|
||||
).rejects.toMatchObject({
|
||||
statusCode: 402,
|
||||
legacyCode: 'insufficient_funds',
|
||||
});
|
||||
expect(run).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
check.mockRestore();
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
import type { EventClient } from '../../clients/event/EventClient.js';
|
||||
import type { Actor } from '../../core/actor';
|
||||
import { insufficientCreditsError } from '../../services/metering/enforcement.js';
|
||||
import type { MeteringService } from '../../services/metering/MeteringService.js';
|
||||
import type {
|
||||
CreditHold,
|
||||
@@ -280,3 +281,24 @@ export function withAiCostFactor(
|
||||
forService.set(driver, facade);
|
||||
return facade;
|
||||
}
|
||||
|
||||
/**
|
||||
* Holds `amount` (priced as {@link AiMeteringService.reserveAiCredits} prices
|
||||
* it) while `run` executes, refusing with a 402 when the actor can't afford it.
|
||||
* The hold is released however `run` ends; `run` records the usage.
|
||||
*/
|
||||
export async function withAiCreditHold<T>(
|
||||
metering: AiMeteringService,
|
||||
actor: Actor,
|
||||
usageType: string,
|
||||
amount: number,
|
||||
run: () => Promise<T>,
|
||||
): Promise<T> {
|
||||
const hold = await metering.reserveAiCredits(actor, usageType, amount);
|
||||
if (!hold) throw insufficientCreditsError();
|
||||
try {
|
||||
return await run();
|
||||
} finally {
|
||||
await hold.release();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
/** Audio the speech drivers return in `test_mode`, without calling a vendor. */
|
||||
export const SAMPLE_AUDIO_URL =
|
||||
'https://puter-sample-data.puter.site/tts_example.mp3';
|
||||
Reference in new issue
Block a user