From be16bca1d9f2d421be66f58f745f2a235ec8a61b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Kujund=C5=BEi=C4=87?= <51266541+404oops@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:21:47 +0200 Subject: [PATCH] Keep Gemini speech instructions separate and validate speech requests (#4233) * fix(ai): keep Gemini speech instructions separate from transcript * Clarify Gemini speech instructions * style(ai-tts): keep Gemini cost comment prettier-clean Co-Authored-By: Claude Opus 5.5 * Validate speech requests and settle reported Gemini usage * Clarify speech settlement for unreadable responses * Reserve the published Gemini speech token budget * Clarify speech accounting for bounded responses * Limit only Gemini TTS text to 3000 characters; charge overrun replies The 3000-character text cap moves from the TTS driver into the Gemini provider, so other providers keep their own limits. The Gemini reply cap rises to 64 MiB, which fits the full 16,384-token audio budget. A reply that still overruns it or times out reports no usage, so it is charged the held budget and the error tells the caller to split the text. --------- Co-authored-by: 404oops Co-authored-by: Claude Opus 5.5 Co-authored-by: Daniel Salazar --- .../providers/gemini/GeminiChatProvider.ts | 9 +- src/backend/drivers/ai-tts/TTSDriver.test.ts | 196 ++- src/backend/drivers/ai-tts/TTSDriver.ts | 8 +- .../drivers/ai-tts/providers/common.test.ts | 28 +- .../drivers/ai-tts/providers/common.ts | 11 + .../GeminiTTSProvider.integration.test.ts | 149 ++ .../gemini/GeminiTTSProvider.test.ts | 1491 +++++++++++++---- .../providers/gemini/GeminiTTSProvider.ts | 539 ++++-- .../drivers/ai-tts/providers/gemini/costs.ts | 33 +- src/backend/drivers/util/aiLimits.ts | 18 + src/backend/drivers/util/geminiRest.ts | 31 + src/docs/src/AI/txt2speech.md | 10 +- src/docs/src/rate-limits-and-quotas.md | 7 +- src/puter-js/src/modules/ai/types.js | 4 +- 14 files changed, 2034 insertions(+), 500 deletions(-) create mode 100644 src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.integration.test.ts create mode 100644 src/backend/drivers/util/geminiRest.ts diff --git a/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts index 9c5138ef2..965172755 100644 --- a/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts +++ b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts @@ -26,6 +26,7 @@ import { import { OpenAI } from 'openai'; import { HttpError } from '../../../../core/http/HttpError.js'; import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { usesGoogleGeminiEndpoint } from '../../../util/geminiRest.js'; import type { IChatModel, ICompleteArguments, @@ -76,11 +77,9 @@ export class GeminiChatProvider extends OpenAICompatProvider { // A custom base URL may be a proxy that only speaks the // OpenAI-compatible API, so native models are left out rather than // sent straight to Google. - const native = - config.apiBaseUrl === undefined || - config.apiBaseUrl.replace(/\/*$/, '/') === COMPAT_BASE_URL - ? geminiNativeClient(new GoogleGenAI({ apiKey: config.apiKey })) - : undefined; + const native = usesGoogleGeminiEndpoint(config.apiBaseUrl) + ? geminiNativeClient(new GoogleGenAI({ apiKey: config.apiKey })) + : undefined; const client: ChatCompletionsClient = { chat: { completions: { diff --git a/src/backend/drivers/ai-tts/TTSDriver.test.ts b/src/backend/drivers/ai-tts/TTSDriver.test.ts index 824e5e15b..294c2b7d4 100644 --- a/src/backend/drivers/ai-tts/TTSDriver.test.ts +++ b/src/backend/drivers/ai-tts/TTSDriver.test.ts @@ -20,13 +20,12 @@ /** * Offline unit tests for TTSDriver. * - * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock - * redis) with API credentials for every TTS provider so the driver - * registers and indexes them all. Then drives `server.drivers.aiTts` - * directly. Provider SDKs and global `fetch` are mocked at their - * network boundaries so the driver's routing and dispatch logic runs - * without real egress. Aligns with AGENTS.md: "Prefer test server - * over mocking deps." + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock redis) with + * API credentials for every TTS provider so the driver registers and indexes + * them all. Then drives `server.drivers.aiTts` directly. Provider SDKs and + * global `fetch` are mocked at their network boundaries so the driver's routing + * and dispatch logic runs without real egress. Aligns with AGENTS.md: "Prefer + * test server over mocking deps." */ import { @@ -95,10 +94,9 @@ const { pollySendMock } = vi.hoisted(() => ({ })); vi.mock('@aws-sdk/client-polly', async () => { - const actual = - await vi.importActual( - '@aws-sdk/client-polly', - ); + const actual = await vi.importActual< + typeof import('@aws-sdk/client-polly') + >('@aws-sdk/client-polly'); return { ...actual, PollyClient: vi.fn().mockImplementation(function ( @@ -184,12 +182,14 @@ const pollyDescribeVoices = { }; const pollyDispatch = () => - pollySendMock.mockImplementation((cmd: { constructor: { name: string } }) => { - if (cmd.constructor.name === 'DescribeVoicesCommand') { - return Promise.resolve(pollyDescribeVoices); - } - return Promise.resolve({ AudioStream: 'audio-bytes' }); - }); + pollySendMock.mockImplementation( + (cmd: { constructor: { name: string } }) => { + if (cmd.constructor.name === 'DescribeVoicesCommand') { + return Promise.resolve(pollyDescribeVoices); + } + return Promise.resolve({ AudioStream: 'audio-bytes' }); + }, + ); // ── Provider registration & list ──────────────────────────────────── @@ -261,7 +261,10 @@ describe('TTSDriver.synthesize provider routing', () => { audio_data: Buffer.from('audio').toString('base64'), audio_format: 'mp3', }), - { status: 200, headers: { 'content-type': 'application/json' } }, + { + status: 200, + headers: { 'content-type': 'application/json' }, + }, ), ); @@ -348,7 +351,9 @@ describe('TTSDriver.synthesize provider routing', () => { it('treats an engine that names a provider as the provider', async () => { openaiSpeechCreateMock.mockResolvedValueOnce(openaiAudioResponse()); - await withActor(() => driver.synthesize({ text: 'hi', engine: 'openai' })); + await withActor(() => + driver.synthesize({ text: 'hi', engine: 'openai' }), + ); expect(openaiSpeechCreateMock).toHaveBeenCalledTimes(1); // The alias selected the provider; it is not carried on as a model. @@ -361,7 +366,11 @@ describe('TTSDriver.synthesize provider routing', () => { openaiSpeechCreateMock.mockResolvedValueOnce(openaiAudioResponse()); await withActor(() => - driver.synthesize({ text: 'hi', provider: 'openai', engine: 'tts-1' }), + driver.synthesize({ + text: 'hi', + provider: 'openai', + engine: 'tts-1', + }), ); expect(openaiSpeechCreateMock.mock.calls[0][0].model).toBe('tts-1'); @@ -379,6 +388,143 @@ describe('TTSDriver.synthesize provider routing', () => { }); }); +// -- Text length cap -- + +describe('TTSDriver.synthesize text length cap', () => { + it('leaves text length to the provider', async () => { + openaiSpeechCreateMock.mockResolvedValueOnce(openaiAudioResponse()); + const text = 'x'.repeat(3001); + + await withActor(() => driver.synthesize({ text, provider: 'openai' })); + + expect(openaiSpeechCreateMock.mock.calls[0][0].input).toBe(text); + }); +}); + +// -- Gemini behind a custom apiBaseUrl -- + +describe('TTSDriver Gemini registration with a custom apiBaseUrl', () => { + const PROXY = 'https://gemini-proxy.test/v1beta/openai/'; + const GOOGLE = 'https://generativelanguage.googleapis.com/'; + + const wav = Buffer.alloc(64, 7); + wav.write('RIFF', 0, 'latin1'); + wav.write('WAVE', 8, 'latin1'); + const geminiReply = { + status: 'completed', + steps: [ + { + type: 'model_output', + content: [ + { + type: 'audio', + mime_type: 'audio/wav', + data: wav.toString('base64'), + }, + ], + }, + ], + usage: { total_input_tokens: 1, total_output_tokens: 25 }, + }; + + /** Boots a server with `providers`, answering Gemini calls with audio. */ + const withServer = async ( + providers: Record, + fn: (tts: TTSDriver) => Promise, + ) => { + const booted = await setupTestServer({ providers } as never); + try { + const passthrough = fetchSpy.getMockImplementation()!; + fetchSpy.mockImplementation((input, init) => + String(input).startsWith(GOOGLE) + ? Promise.resolve(Response.json(geminiReply)) + : passthrough(input, init), + ); + await fn(booted.drivers.aiTts as unknown as TTSDriver); + } finally { + await booted.shutdown(); + } + }; + + const googleCalls = () => + fetchSpy.mock.calls.filter(([url]) => String(url).startsWith(GOOGLE)); + + it('leaves Gemini out and answers an explicit request with a config error, without calling Google', async () => { + await withServer( + { gemini: { apiKey: 'proxy-key', apiBaseUrl: PROXY } }, + async (tts) => { + expect(await tts.list()).not.toContain('gemini'); + + const notConfigured = { + statusCode: 400, + legacyCode: 'bad_request', + message: expect.stringContaining( + 'TTS provider not configured: gemini', + ), + }; + await expect( + withActor(() => + tts.synthesize({ text: 'hi', provider: 'gemini' }), + ), + ).rejects.toMatchObject(notConfigured); + await expect( + withActor(() => + tts.synthesize({ text: 'hi', engine: 'google' }), + ), + ).rejects.toMatchObject(notConfigured); + await expect( + withDriverName('gemini-tts', () => + tts.synthesize({ text: 'hi' }), + ), + ).rejects.toMatchObject(notConfigured); + expect(googleCalls()).toHaveLength(0); + }, + ); + }); + + it('sends Google a gemini-tts key, never the proxied gemini key', async () => { + await withServer( + { + gemini: { apiKey: 'proxy-key', apiBaseUrl: PROXY }, + 'gemini-tts': { apiKey: 'tts-key' }, + }, + async (tts) => { + await withActor(() => + tts.synthesize({ text: 'hi', provider: 'gemini' }), + ); + const calls = googleCalls(); + expect(calls).toHaveLength(1); + const headers = calls[0]![1]!.headers as Record; + expect(headers['x-goog-api-key']).toBe('tts-key'); + }, + ); + }); + + it.each([ + 'https://generativelanguage.googleapis.com/v1beta/openai/', + 'https://generativelanguage.googleapis.com/v1beta/openai', + ])( + 'keeps Gemini on the shared key when apiBaseUrl is Google’s own %s', + async (apiBaseUrl) => { + await withServer( + { gemini: { apiKey: 'gem-key', apiBaseUrl } }, + async (tts) => { + await withActor(() => + tts.synthesize({ text: 'hi', provider: 'gemini' }), + ); + const calls = googleCalls(); + expect(calls).toHaveLength(1); + const headers = calls[0]![1]!.headers as Record< + string, + string + >; + expect(headers['x-goog-api-key']).toBe('gem-key'); + }, + ); + }, + ); +}); + // ── list_voices / list_engines aggregation ───────────────────────── describe('TTSDriver list_voices / list_engines', () => { @@ -410,9 +556,9 @@ describe('TTSDriver list_voices / list_engines', () => { }); it('list_voices({ provider }) throws 400 when the provider is unknown', async () => { - await expect(driver.list_voices({ provider: 'nope' })).rejects.toMatchObject( - { statusCode: 400 }, - ); + await expect( + driver.list_voices({ provider: 'nope' }), + ).rejects.toMatchObject({ statusCode: 400 }); }); it('list_voices() defaults to aws-polly', async () => { @@ -490,7 +636,9 @@ describe('TTSDriver.synthesize error mapping', () => { server.services.metering, 'incrementUsage', ); - openaiSpeechCreateMock.mockRejectedValueOnce(new Error('upstream blew up')); + openaiSpeechCreateMock.mockRejectedValueOnce( + new Error('upstream blew up'), + ); await expect( withActor(() => diff --git a/src/backend/drivers/ai-tts/TTSDriver.ts b/src/backend/drivers/ai-tts/TTSDriver.ts index 281c8f4de..9ae33a15e 100644 --- a/src/backend/drivers/ai-tts/TTSDriver.ts +++ b/src/backend/drivers/ai-tts/TTSDriver.ts @@ -26,6 +26,7 @@ import { withAiCostFactor, } from '../util/aiCostFactor.js'; import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js'; +import { usesGoogleGeminiEndpoint } from '../util/geminiRest.js'; import { ProviderRegistry, readProviderKey } from '../util/providerRegistry.js'; import { TTS_CATALOG, TTS_DRIVER_ALIASES } from './providerAliases.js'; import { AWSPollyTTSProvider } from './providers/awsPolly/AWSPollyTTSProvider.js'; @@ -254,11 +255,14 @@ export class TTSDriver extends PuterDriver { ); } + // Gemini TTS only calls Google, so it skips a block whose key belongs + // to a custom `apiBaseUrl`. register('gemini', () => keyed( (apiKey) => new GeminiTTSProvider(m, { apiKey }), - providers['gemini'], - providers['gemini-tts'], + ...[providers['gemini'], providers['gemini-tts']].filter( + (cfg) => usesGoogleGeminiEndpoint(cfg?.apiBaseUrl), + ), ), ); register('xai', () => diff --git a/src/backend/drivers/ai-tts/providers/common.test.ts b/src/backend/drivers/ai-tts/providers/common.test.ts index 5e63b4fb8..f40914eba 100644 --- a/src/backend/drivers/ai-tts/providers/common.test.ts +++ b/src/backend/drivers/ai-tts/providers/common.test.ts @@ -32,7 +32,11 @@ import type { DriverStreamResult } from '../../meta.js'; import { PuterServer } from '../../../server.js'; import { setupTestServer } from '../../../testUtil.js'; import { withAiCostFactor } from '../../util/aiCostFactor.js'; -import { characterCostReport, meterPerCharacter } from './common.js'; +import { + assertMaxChars, + characterCostReport, + meterPerCharacter, +} from './common.js'; let server: PuterServer; beforeAll(async () => { @@ -227,6 +231,28 @@ describe('meterPerCharacter', () => { }); }); +describe('assertMaxChars', () => { + it('accepts a string at the cap and values that are not strings', () => { + expect(() => assertMaxChars('text', 'x'.repeat(5), 5)).not.toThrow(); + expect(() => assertMaxChars('text', undefined, 5)).not.toThrow(); + expect(() => assertMaxChars('text', ['a', 'b'], 1)).not.toThrow(); + }); + + it('rejects a string over the cap with a 400 input_too_large naming the key', () => { + let err: unknown; + try { + assertMaxChars('instructions', 'x'.repeat(6), 5); + } catch (e) { + err = e; + } + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'input_too_large', + fields: { key: 'instructions', max: 5 }, + }); + }); +}); + describe('characterCostReport', () => { it('emits one per-character line per cost-table entry', () => { expect( diff --git a/src/backend/drivers/ai-tts/providers/common.ts b/src/backend/drivers/ai-tts/providers/common.ts index 66ecd134b..e18b29d3a 100644 --- a/src/backend/drivers/ai-tts/providers/common.ts +++ b/src/backend/drivers/ai-tts/providers/common.ts @@ -18,6 +18,7 @@ */ import type { Actor } from '../../../core/actor.js'; +import { HttpError } from '../../../core/http/HttpError.js'; import { type AiMeteringService, withStreamedAiCharge, @@ -30,6 +31,16 @@ import type { DriverStreamResult } from '../../meta.js'; */ export const TTS_UPSTREAM_TIMEOUT_MS = 120_000; +/** Rejects a string argument longer than its published cap with a 400. */ +export function assertMaxChars(key: string, value: unknown, max: number): void { + if (typeof value === 'string' && value.length > max) { + throw new HttpError(400, `${key} must be at most ${max} characters`, { + legacyCode: 'input_too_large', + fields: { key, max }, + }); + } +} + /** * Runs `synthesize` under a hold on `text.length × ucentsPerChar` and meters * exactly that once its audio starts arriving. Per-character prices are known diff --git a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.integration.test.ts b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.integration.test.ts new file mode 100644 index 000000000..b5a1f927a --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.integration.test.ts @@ -0,0 +1,149 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Gemini TTS provider against the live + * Interactions API, one short sentence per model. Skipped when + * `PUTER_TEST_AI_GEMINI_API_KEY` is unset. With + * `PUTER_TEST_AI_GEMINI_TTS_AUDIO_DIR` set, each WAV is written there so + * the spoken words can be checked against `TEXT` by ear or transcription. + */ + +import axios from 'axios'; +import { setupPuterTestEnv } from '../../../../testUtil.js'; +import { mkdir, writeFile } from 'node:fs/promises'; +import { join } from 'node:path'; +import { describe, expect, it } from 'vitest'; +import type { UsageInput } from '../../../../services/metering/types.js'; +import { + INTEGRATION_TEST_TIMEOUT_MS, + makeMeteringStub, + optionalEnv, + readStreamResult, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { GeminiTTSProvider } from './GeminiTTSProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_GEMINI_API_KEY'; +const AUDIO_DIR_ENV_VAR = 'PUTER_TEST_AI_GEMINI_TTS_AUDIO_DIR'; + +const TEXT = 'The orange kite drifted over seven quiet hills.'; +const INSTRUCTIONS = 'Read this slowly in a warm, calm voice.'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'GeminiTTSProvider (integration)', + () => { + it.each(['gemini-3.8-flash-tts', 'gemini-3.8-flash-lite-tts'])( + 'returns WAV audio and token usage from %s', + async (model) => { + const usages: UsageInput[] = []; + const metering = makeMeteringStub(); + metering.batchIncrementUsages = async (_actor, entries) => { + usages.push(...entries); + return {} as never; + }; + const provider = new GeminiTTSProvider(metering, { + apiKey: optionalEnv(ENV_VAR)!, + }); + + const result = await withTestActor(() => + provider.synthesize({ + text: TEXT, + model, + voice: 'kore', + instructions: INSTRUCTIONS, + }), + ); + + expect(result).toMatchObject({ content_type: 'audio/wav' }); + const wav = await readStreamResult(result); + expect(wav.length).toBeGreaterThan(44); + expect(wav.toString('latin1', 0, 4)).toBe('RIFF'); + expect(wav.toString('latin1', 8, 12)).toBe('WAVE'); + + const input = usages.find( + (u) => u.usageType === `gemini:${model}:input`, + ); + const output = usages.find( + (u) => u.usageType === `gemini:${model}:output:audio`, + ); + // Text and style together are a few dozen tokens; anything + // near the reply size would mean the JSON was counted. + expect(input?.usageAmount).toBeGreaterThan(0); + expect(input?.usageAmount).toBeLessThan(200); + expect(output?.usageAmount).toBeGreaterThan(0); + + const audioDir = optionalEnv(AUDIO_DIR_ENV_VAR); + if (audioDir) { + await mkdir(audioDir, { recursive: true }); + await writeFile(join(audioDir, `${model}.wav`), wav); + } + }, + INTEGRATION_TEST_TIMEOUT_MS, + ); + }, +); + + +describe.skipIf(skipUnlessEnv(ENV_VAR))('Gemini TTS authenticated HTTP (integration)', () => { + it('returns WAV through the public driver route', async () => { + const env = await setupPuterTestEnv({ + providers: { + gemini: { apiKey: optionalEnv(ENV_VAR)! }, + ollama: { enabled: false }, + }, + unlimitedMetering: true, + extensions: [], + }); + try { + const endpoint = new URL('/drivers/call', env.apiOrigin); + const host = endpoint.host; + endpoint.hostname = '127.0.0.1'; + const response = await axios.request({ + url: endpoint.toString(), + method: 'POST', + responseType: 'arraybuffer', + validateStatus: () => true, + headers: { + Authorization: `Bearer ${env.users.user.apiToken}`, + Origin: env.apiOrigin, + Host: host, + }, + data: { + interface: 'puter-tts', + method: 'synthesize', + args: { + provider: 'gemini', + model: 'gemini-3.8-flash-lite-tts', + text: TEXT, + instructions: INSTRUCTIONS, + }, + }, + }); + expect(response.status).toBe(200); + expect(response.headers['content-type']).toContain('audio/wav'); + const wav = Buffer.from(response.data); + expect(wav.toString('latin1', 0, 4)).toBe('RIFF'); + expect(wav.toString('latin1', 8, 12)).toBe('WAVE'); + } finally { + await env.shutdown(); + } + }, INTEGRATION_TEST_TIMEOUT_MS); +}); diff --git a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.test.ts b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.test.ts index 11cf10254..c3c6857e8 100644 --- a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.test.ts +++ b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.test.ts @@ -20,11 +20,11 @@ /** * Offline unit tests for GeminiTTSProvider. * - * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock - * redis) and constructs GeminiTTSProvider directly against the live - * wired `MeteringService` so the recording side runs end-to-end. The - * Google GenAI SDK is mocked at the module boundary — that's the real - * network egress point. + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock redis) and + * constructs GeminiTTSProvider directly against the live wired + * `MeteringService` so holds and recording run end-to-end. Global `fetch` is + * stubbed with canned Interactions API responses, the real network egress + * point. */ import { Readable } from 'node:stream'; @@ -40,35 +40,26 @@ import { type MockInstance, } from 'vitest'; +import { Context } from '../../../../core/context.js'; import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { UsageInput } from '../../../../services/metering/types.js'; import { withAiCostFactor } from '../../../util/aiCostFactor.js'; import { PuterServer } from '../../../../server.js'; import { setupTestServer } from '../../../../testUtil.js'; -import { withTestActor } from '../../../integrationTestUtil.js'; -import { GeminiTTSProvider } from './GeminiTTSProvider.js'; -import { GEMINI_TTS_COSTS } from './costs.js'; +import { callerError, withTestActor } from '../../../integrationTestUtil.js'; +import { + GEMINI_TTS_MAX_RESPONSE_BYTES, + GeminiTTSProvider, +} from './GeminiTTSProvider.js'; +import { GEMINI_TTS_COSTS, GEMINI_TTS_LAUNCH_PRICING_ENDS } from './costs.js'; -// ── Google GenAI SDK mock ─────────────────────────────────────────── +const INTERACTIONS_URL = + 'https://generativelanguage.googleapis.com/v1beta/interactions'; -const { generateContentMock, googleAICtor } = vi.hoisted(() => ({ - generateContentMock: vi.fn(), - googleAICtor: vi.fn(), -})); - -vi.mock('@google/genai', () => { - const GoogleGenAI = vi.fn().mockImplementation(function ( - this: Record, - opts: unknown, - ) { - googleAICtor(opts); - this.models = { generateContent: generateContentMock }; - }); - return { GoogleGenAI }; -}); - -// ── Test harness ──────────────────────────────────────────────────── +// -- Test harness -- let server: PuterServer; +let fetchSpy: MockInstance; let hasCreditsSpy: MockInstance; let batchIncrementUsagesSpy: MockInstance< MeteringService['batchIncrementUsages'] @@ -89,33 +80,139 @@ const aiMetering = () => const makeProvider = () => new GeminiTTSProvider(aiMetering(), { apiKey: 'test-key' }); -// Build a canned generateContent response with PCM audio data and -// usageMetadata that the provider can meter against. -const audioResponse = ( - base64Pcm = Buffer.from('PCMPCMPCM').toString('base64'), +/** A RIFF/WAVE container as the Interactions API returns it. */ +const WAV_BYTES = (() => { + const wav = Buffer.alloc(64, 7); + wav.write('RIFF', 0, 'latin1'); + wav.writeUInt32LE(56, 4); + wav.write('WAVE', 8, 'latin1'); + wav.write('fmt ', 12, 'latin1'); + return wav; +})(); + +const audioBlock = ( + bytes: Buffer = WAV_BYTES, + mimeType: string = 'audio/wav', +) => ({ type: 'audio', mime_type: mimeType, data: bytes.toString('base64') }); + +const USAGE = { + total_input_tokens: 7, + input_tokens_by_modality: [{ modality: 'text', tokens: 7 }], + total_output_tokens: 68, + output_tokens_by_modality: [{ modality: 'audio', tokens: 68 }], + total_thought_tokens: 0, + total_tokens: 75, +}; + +/** What `USAGE` settles to on the default model at launch rates. */ +const SETTLED_USAGE = [ { - mimeType = 'audio/L16;rate=24000', - promptTokenCount = 10, - candidatesTokenCount = 250, - }: { - mimeType?: string; - promptTokenCount?: number; - candidatesTokenCount?: number; + usageType: 'gemini:gemini-3.8-flash-tts:input', + usageAmount: 7, + costOverride: 7 * 50, + }, + { + usageType: 'gemini:gemini-3.8-flash-tts:output:audio', + usageAmount: 68, + costOverride: 68 * 900, + }, +]; + +/** µ¢ held for the full 8,192 input and 16,384 audio token budget. */ +const reservation = (inputRate: number, audioRate: number) => + 8192 * inputRate + 16_384 * audioRate; + +/** The default model's hold at launch rates. */ +const FLASH_RESERVATION = reservation(50, 900); + +/** A completed reply; `usage` given as undefined leaves the field out. */ +const interaction = ( + opts: { + content?: unknown[]; + status?: string; + usage?: unknown; + steps?: unknown[]; } = {}, ) => ({ - candidates: [ - { - content: { - parts: [{ inlineData: { mimeType, data: base64Pcm } }], - }, - }, + model: 'gemini-3.8-flash-tts', + status: opts.status ?? 'completed', + steps: opts.steps ?? [ + { type: 'model_output', content: opts.content ?? [audioBlock()] }, ], - usageMetadata: { promptTokenCount, candidatesTokenCount }, + ...('usage' in opts ? { usage: opts.usage } : { usage: USAGE }), }); +const replyWith = (body: unknown, init?: ResponseInit) => + fetchSpy.mockResolvedValueOnce(Response.json(body, init)); + +/** What the provider sent to the Interactions API. */ +const sentRequest = (call = 0) => { + const [url, init] = fetchSpy.mock.calls[call]!; + return { + url: String(url), + method: init?.method, + headers: init?.headers as Record, + body: JSON.parse(String(init?.body)), + signal: init?.signal, + }; +}; + +const readAll = async (stream: Readable): Promise => { + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +}; + +const meteredUsages = (): UsageInput[] => + batchIncrementUsagesSpy.mock.calls.flatMap(([, usages]) => usages); + +const metered = (suffix: string) => + meteredUsages().find((u) => u.usageType.endsWith(suffix))!; + +/** A user the gate really holds credit for, unlike the system actor. */ +const payingUser = () => + ({ + user: { + uuid: `gemini-tts-${Math.random().toString(36).slice(2)}`, + username: 'gemini-tts-user', + email: 'gemini-tts@test.com', + }, + }) as never; + +/** Spies on the credit holds taken, recording the amount and each release. */ +const trackHolds = () => { + const metering = server.services.metering; + const realReserve = metering.reserveCredits.bind(metering); + const held: number[] = []; + const released = vi.fn(); + vi.spyOn(metering, 'reserveCredits').mockImplementation( + async (actor, amount, opts) => { + held.push(amount); + const hold = await realReserve(actor, amount, opts); + return { + ...hold, + release: async () => { + released(); + await hold.release(); + }, + }; + }, + ); + return { held, released }; +}; + beforeEach(() => { - generateContentMock.mockReset(); - googleAICtor.mockReset(); + // The full flash hold at later rates exceeds the system actor's allowance; + // tests of later rates set the clock themselves. + vi.useFakeTimers({ + toFake: ['Date'], + now: GEMINI_TTS_LAUNCH_PRICING_ENDS - 1, + }); + fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockRejectedValue(new Error('unexpected fetch')) as MockInstance< + typeof fetch + >; hasCreditsSpy = vi.spyOn(server.services.metering, 'hasEnoughCredits'); batchIncrementUsagesSpy = vi.spyOn( server.services.metering, @@ -124,18 +221,13 @@ beforeEach(() => { }); afterEach(() => { + vi.useRealTimers(); vi.restoreAllMocks(); }); -// ── Construction ──────────────────────────────────────────────────── +// -- Construction -- describe('GeminiTTSProvider construction', () => { - it('constructs the GoogleGenAI SDK with the configured api key', () => { - makeProvider(); - expect(googleAICtor).toHaveBeenCalledTimes(1); - expect(googleAICtor).toHaveBeenCalledWith({ apiKey: 'test-key' }); - }); - it('throws when no apiKey is supplied', () => { expect( () => @@ -146,97 +238,119 @@ describe('GeminiTTSProvider construction', () => { }); }); -// ── Voice / engine catalog ────────────────────────────────────────── +// -- Voice / engine catalog -- describe('GeminiTTSProvider catalog', () => { it('listVoices returns the documented Gemini voices with provider=gemini', async () => { - const provider = makeProvider(); - const voices = await provider.listVoices(); + const voices = await makeProvider().listVoices(); expect(voices.length).toBeGreaterThan(0); for (const voice of voices) { expect(voice.provider).toBe('gemini'); } - // Default voice (Kore) is present. expect(voices.find((v) => v.id === 'Kore')).toBeDefined(); - // supported_models matches the documented engine list. - expect(voices[0].supported_models).toEqual( - expect.arrayContaining([ - 'gemini-3.8-flash-tts', - 'gemini-3.8-flash-lite-tts', - ]), - ); + expect(voices[0].supported_models).toEqual([ + 'gemini-3.8-flash-tts', + 'gemini-3.8-flash-lite-tts', + ]); }); - it('listEngines returns the documented model list with provider=gemini', async () => { - const provider = makeProvider(); - const engines = await provider.listEngines(); - const ids = engines.map((e) => e.id); - expect(ids).toEqual( - expect.arrayContaining([ - 'gemini-3.8-flash-tts', - 'gemini-3.8-flash-lite-tts', - ]), - ); + it('listEngines returns both models with provider=gemini', async () => { + const engines = await makeProvider().listEngines(); + expect(engines.map((e) => e.id)).toEqual([ + 'gemini-3.8-flash-tts', + 'gemini-3.8-flash-lite-tts', + ]); }); }); -// ── Reported costs ────────────────────────────────────────────────── +// -- Reported costs -- describe('GeminiTTSProvider.getReportedCosts', () => { - it('emits per-model line items with ucents converted from cents/1M tokens', () => { + const reportedRates = () => + Object.fromEntries( + makeProvider() + .getReportedCosts() + .map((r) => [ + r.usageType, + [r.ucentsInputPerToken, r.ucentsOutputAudioPerToken], + ]), + ); + + it('reports the launch rates through 2026-12-31 UTC', () => { + vi.useFakeTimers({ + toFake: ['Date'], + now: GEMINI_TTS_LAUNCH_PRICING_ENDS - 1, + }); + expect(reportedRates()).toEqual({ + 'gemini:gemini-3.8-flash-tts:tts': [50, 900], + 'gemini:gemini-3.8-flash-lite-tts:tts': [50, 600], + }); + }); + + it('reports doubled rates from 2027-01-01 UTC, read at call time', () => { + vi.useFakeTimers({ + toFake: ['Date'], + now: GEMINI_TTS_LAUNCH_PRICING_ENDS - 1, + }); const provider = makeProvider(); - const reported = provider.getReportedCosts() as Array<{ - usageType: string; - ucentsInputPerToken: number; - ucentsOutputAudioPerToken: number; - unit: string; - source: string; - }>; - expect(reported).toHaveLength(Object.keys(GEMINI_TTS_COSTS).length); - for (const [model] of Object.entries(GEMINI_TTS_COSTS)) { - const entry = reported.find( - (r) => r.usageType === `gemini:${model}:tts`, - ); - expect(entry).toBeDefined(); - expect(entry?.unit).toBe('token'); - expect(entry?.source).toBe('driver:aiTts/gemini'); - // ucents are integer microcents — must be >= 1 by construction. - expect(entry?.ucentsInputPerToken).toBeGreaterThanOrEqual(1); - expect(entry?.ucentsOutputAudioPerToken).toBeGreaterThanOrEqual(1); + provider.getReportedCosts(); + vi.setSystemTime(GEMINI_TTS_LAUNCH_PRICING_ENDS); + expect( + Object.fromEntries( + provider + .getReportedCosts() + .map((r) => [ + r.usageType, + [r.ucentsInputPerToken, r.ucentsOutputAudioPerToken], + ]), + ), + ).toEqual({ + 'gemini:gemini-3.8-flash-tts:tts': [100, 1800], + 'gemini:gemini-3.8-flash-lite-tts:tts': [100, 1200], + }); + }); + + it('labels each line as a token rate from this driver', () => { + for (const entry of makeProvider().getReportedCosts()) { + expect(entry).toMatchObject({ + unit: 'token', + source: 'driver:aiTts/gemini', + }); } }); }); -// ── test_mode bypass ──────────────────────────────────────────────── +// -- test_mode bypass -- describe('GeminiTTSProvider.synthesize test_mode', () => { - it('returns the canned sample URL without hitting credits or the SDK', async () => { - const provider = makeProvider(); + it('returns the canned sample URL without credits or network', async () => { const result = await withTestActor(() => - provider.synthesize({ text: 'hi', test_mode: true }), + makeProvider().synthesize({ text: 'hi', test_mode: true }), ); expect(result).toEqual({ url: 'https://puter-sample-data.puter.site/tts_example.mp3', content_type: 'audio', }); expect(hasCreditsSpy).not.toHaveBeenCalled(); - expect(generateContentMock).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); }); }); -// ── Argument validation ───────────────────────────────────────────── +// -- Argument validation -- describe('GeminiTTSProvider.synthesize argument validation', () => { it.each([ 'gemini-2.5-flash-preview-tts', 'gemini-2.5-pro-preview-tts', 'gemini-3.1-flash-tts-preview', - ])('rejects the deprecated %s with 400', async (model) => { - const provider = makeProvider(); + 'gemini-fake', + ])('rejects the unlisted model %s with 400', async (model) => { await expect( - withTestActor(() => provider.synthesize({ text: 'hi', model })), + withTestActor(() => + makeProvider().synthesize({ text: 'hi', model }), + ), ).rejects.toMatchObject({ statusCode: 400 }); - expect(generateContentMock).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); }); it('throws 400 when text is missing or blank', async () => { @@ -247,277 +361,1026 @@ describe('GeminiTTSProvider.synthesize argument validation', () => { await expect( withTestActor(() => provider.synthesize({ text: ' ' })), ).rejects.toMatchObject({ statusCode: 400 }); - expect(generateContentMock).not.toHaveBeenCalled(); - }); - - it('throws 400 when the model is not in the catalog', async () => { - const provider = makeProvider(); - await expect( - withTestActor(() => - provider.synthesize({ text: 'hi', model: 'gemini-fake' }), - ), - ).rejects.toMatchObject({ statusCode: 400 }); - expect(generateContentMock).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); }); it('throws 400 when the voice is not in the catalog', async () => { - const provider = makeProvider(); await expect( withTestActor(() => - provider.synthesize({ text: 'hi', voice: 'NotAVoice' }), + makeProvider().synthesize({ text: 'hi', voice: 'NotAVoice' }), ), ).rejects.toMatchObject({ statusCode: 400 }); - expect(generateContentMock).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('throws 400 when instructions is not a string', async () => { + await expect( + withTestActor(() => + makeProvider().synthesize({ + text: 'hi', + instructions: { style: 'calm' } as never, + }), + ), + ).rejects.toMatchObject({ + statusCode: 400, + legacyCode: 'field_invalid', + }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('accepts text of exactly 3000 characters', async () => { + const text = 'x'.repeat(3000); + replyWith(interaction()); + await withTestActor(() => makeProvider().synthesize({ text })); + expect(sentRequest().body.input[0].content[0].text).toBe(text); + }); + + it('rejects text over 3000 characters with 400 input_too_large before holding credit', async () => { + await expect( + withTestActor(() => + makeProvider().synthesize({ text: 'x'.repeat(3001) }), + ), + ).rejects.toMatchObject({ + statusCode: 400, + legacyCode: 'input_too_large', + fields: { key: 'text', max: 3000 }, + }); + expect(hasCreditsSpy).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('accepts instructions of exactly 3000 characters', async () => { + const instructions = 'x'.repeat(3000); + replyWith(interaction()); + await withTestActor(() => + makeProvider().synthesize({ text: 'hi', instructions }), + ); + expect( + sentRequest().body.input[0].content[0].annotations[0].style, + ).toBe(instructions); + }); + + it('rejects instructions over 3000 characters with 400 input_too_large before holding credit', async () => { + await expect( + withTestActor(() => + makeProvider().synthesize({ + text: 'hi', + instructions: 'x'.repeat(3001), + }), + ), + ).rejects.toMatchObject({ + statusCode: 400, + legacyCode: 'input_too_large', + fields: { key: 'instructions', max: 3000 }, + }); + expect(hasCreditsSpy).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); }); }); -// ── Credit gate ───────────────────────────────────────────────────── +// -- Credit gate -- describe('GeminiTTSProvider.synthesize credit gate', () => { - it('throws 402 BEFORE hitting Gemini when actor lacks credits', async () => { - const provider = makeProvider(); + it('throws 402 before calling Gemini when the actor lacks credits', async () => { hasCreditsSpy.mockResolvedValueOnce(false); - await expect( - withTestActor(() => provider.synthesize({ text: 'hi' })), + withTestActor(() => makeProvider().synthesize({ text: 'hi' })), ).rejects.toMatchObject({ statusCode: 402 }); - expect(generateContentMock).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + /** Leaves `actor` exactly `available` µ¢ to commit by holding the rest. */ + const leaveAvailable = async (actor: never, available: number) => { + const metering = server.services.metering; + const remaining = await metering.getRemainingUsage(actor); + expect(remaining).toBeGreaterThan(available); + return metering.reserveCredits(actor, remaining - available); + }; + + it.each([ + ['covers the usage of a short text', 1_000_000], + ['is one µ¢ short of the full budget', FLASH_RESERVATION - 1], + ])( + 'refuses with 402 before calling Gemini when available credit %s', + async (_label, available) => { + const actor = payingUser(); + const blocking = await leaveAvailable(actor, available); + try { + await expect( + withTestActor( + () => makeProvider().synthesize({ text: 'hi' }), + actor, + ), + ).rejects.toMatchObject({ statusCode: 402 }); + expect(hasCreditsSpy).toHaveBeenCalledWith( + actor, + FLASH_RESERVATION, + ); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + expect( + await server.services.metering.getRemainingUsage(actor), + ).toBe(available); + } finally { + await blocking.release(); + } + }, + ); + + it('runs with exactly the full budget available, then releases all but the reported usage', async () => { + const actor = payingUser(); + const blocking = await leaveAvailable(actor, FLASH_RESERVATION); + try { + replyWith(interaction()); + await withTestActor( + () => makeProvider().synthesize({ text: 'hi' }), + actor, + ); + + expect(meteredUsages()).toEqual(SETTLED_USAGE); + await expect + .poll(() => server.services.metering.getRemainingUsage(actor)) + .toBe(FLASH_RESERVATION - 7 * 50 - 68 * 900); + } finally { + await blocking.release(); + } }); }); -// ── Request shape ─────────────────────────────────────────────────── +// -- Request shape -- describe('GeminiTTSProvider.synthesize request shape', () => { - it('frames text as a "Say the following text aloud:" transcript, defaults to flash/Kore', async () => { + it.each(['gemini-3.8-flash-tts', 'gemini-3.8-flash-lite-tts'])( + 'sends %s the text verbatim and the instructions as speech metadata', + async (model) => { + const text = ' Hello,\nworld. Read me exactly. '; + replyWith(interaction()); + + await withTestActor(() => + makeProvider().synthesize({ + text, + model, + voice: 'Zephyr', + instructions: 'Speak softly, like a librarian.', + }), + ); + + const sent = sentRequest(); + expect(sent.url).toBe(INTERACTIONS_URL); + expect(sent.method).toBe('POST'); + expect(sent.headers).toEqual({ + 'x-goog-api-key': 'test-key', + 'Content-Type': 'application/json', + }); + expect(sent.body).toEqual({ + model, + input: [ + { + type: 'user_input', + content: [ + { + type: 'text', + text, + annotations: [ + { + type: 'speech_metadata', + style: 'Speak softly, like a librarian.', + }, + ], + }, + ], + }, + ], + response_format: { type: 'audio' }, + generation_config: { + speech_config: [{ voice: 'Zephyr' }], + max_output_tokens: 16_384, + }, + store: false, + background: false, + stream: false, + }); + }, + ); + + it.each(['gemini-3.8-flash-tts', 'gemini-3.8-flash-lite-tts'])( + 'caps %s output at the 16,384 audio tokens it holds credit for', + async (model) => { + replyWith(interaction()); + await withTestActor(() => + makeProvider().synthesize({ text: 'hi', model }), + ); + expect( + sentRequest().body.generation_config.max_output_tokens, + ).toBe(16_384); + }, + ); + + it('omits annotations without instructions and defaults to flash/Kore', async () => { + replyWith(interaction()); + replyWith(interaction()); const provider = makeProvider(); - generateContentMock.mockResolvedValueOnce(audioResponse()); await withTestActor(() => provider.synthesize({ text: 'hello' })); - - const sent = generateContentMock.mock.calls[0]![0]; - expect(sent.model).toBe('gemini-3.8-flash-tts'); - expect(sent.contents[0].parts[0].text).toBe( - 'Say the following text aloud:\nhello', + await withTestActor(() => + provider.synthesize({ text: 'hello', instructions: ' ' }), ); - expect(sent.config.responseModalities).toEqual(['AUDIO']); - expect( - sent.config.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName, - ).toBe('Kore'); + + for (const call of [0, 1]) { + const { body } = sentRequest(call); + expect(body.model).toBe('gemini-3.8-flash-tts'); + expect(body.input[0].content).toEqual([ + { type: 'text', text: 'hello' }, + ]); + expect(body.generation_config.speech_config).toEqual([ + { voice: 'Kore' }, + ]); + } }); - it('prepends instructions to the transcript when supplied', async () => { - const provider = makeProvider(); - generateContentMock.mockResolvedValueOnce(audioResponse()); + it.each([ + ['kore', 'Kore'], + ['ZEPHYR', 'Zephyr'], + ['sadAlTager', 'Sadaltager'], + ])( + 'accepts voice %s case-insensitively and sends canonical %s', + async (requested, canonical) => { + replyWith(interaction()); + await withTestActor(() => + makeProvider().synthesize({ text: 'hi', voice: requested }), + ); + expect(sentRequest().body.generation_config.speech_config).toEqual([ + { voice: canonical }, + ]); + }, + ); - await withTestActor(() => - provider.synthesize({ - text: 'hi', - instructions: 'Speak softly', - voice: 'Zephyr', - model: 'gemini-3.8-flash-lite-tts', - }), - ); - - const sent = generateContentMock.mock.calls[0]![0]; - expect(sent.model).toBe('gemini-3.8-flash-lite-tts'); - expect(sent.contents[0].parts[0].text).toBe( - 'Speak softly\n\nSay the following text aloud:\nhi', - ); - expect( - sent.config.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName, - ).toBe('Zephyr'); - }); - - it('accepts voice names case-insensitively but forwards the caller spelling', async () => { - const provider = makeProvider(); - generateContentMock.mockResolvedValueOnce(audioResponse()); - - await withTestActor(() => - provider.synthesize({ text: 'hi', voice: 'kore' }), - ); - - const sent = generateContentMock.mock.calls[0]![0]; - expect( - sent.config.speechConfig.voiceConfig.prebuiltVoiceConfig.voiceName, - ).toBe('kore'); + it('bounds the call with the 2-minute TTS deadline', async () => { + const timeoutSpy = vi.spyOn(AbortSignal, 'timeout'); + replyWith(interaction()); + await withTestActor(() => makeProvider().synthesize({ text: 'hi' })); + expect(timeoutSpy).toHaveBeenCalledWith(120_000); }); }); -// ── Streaming output & WAV wrapping ──────────────────────────────── +// -- Audio output -- -describe('GeminiTTSProvider.synthesize streaming output', () => { - it('wraps PCM payloads into a WAV container and returns a readable stream', async () => { - const provider = makeProvider(); - const pcm = Buffer.from('PCM-PAYLOAD'); - generateContentMock.mockResolvedValueOnce( - audioResponse(pcm.toString('base64')), - ); +describe('GeminiTTSProvider.synthesize audio output', () => { + it('streams WAV audio through byte for byte', async () => { + replyWith(interaction()); const result = (await withTestActor(() => - provider.synthesize({ text: 'hi' }), - )) as { - stream: Readable; - content_type: string; - chunked: boolean; - }; + makeProvider().synthesize({ text: 'hi' }), + )) as { stream: Readable; content_type: string; chunked: boolean }; expect(result.content_type).toBe('audio/wav'); expect(result.chunked).toBe(true); expect(result.stream).toBeInstanceOf(Readable); - - const chunks: Buffer[] = []; - for await (const chunk of result.stream) { - chunks.push(chunk as Buffer); - } - const buffer = Buffer.concat(chunks); - // 44-byte WAV header + PCM data length. - expect(buffer.length).toBe(44 + pcm.length); - expect(buffer.subarray(0, 4).toString()).toBe('RIFF'); - expect(buffer.subarray(8, 12).toString()).toBe('WAVE'); - // Trailing bytes match the original PCM verbatim. - expect(buffer.subarray(44)).toEqual(pcm); + expect(await readAll(result.stream)).toEqual(WAV_BYTES); }); - it('passes encoded (non-PCM) audio through with the upstream mime type', async () => { - const provider = makeProvider(); - const encoded = Buffer.from('ALREADY-MP3'); - generateContentMock.mockResolvedValueOnce( - audioResponse(encoded.toString('base64'), { - mimeType: 'audio/mpeg', + it('wraps raw L16 PCM in a WAV header at the declared rate', async () => { + const pcm = Buffer.from('PCM-PAYLOAD'); + replyWith( + interaction({ + content: [audioBlock(pcm, 'audio/L16;codec=pcm;rate=16000')], }), ); const result = (await withTestActor(() => - provider.synthesize({ text: 'hi' }), + makeProvider().synthesize({ text: 'hi' }), )) as { stream: Readable; content_type: string }; - expect(result.content_type).toBe('audio/mpeg'); - const chunks: Buffer[] = []; - for await (const chunk of result.stream) { - chunks.push(chunk as Buffer); - } - expect(Buffer.concat(chunks)).toEqual(encoded); - }); -}); - -// ── Cost reporting & metering ─────────────────────────────────────── - -describe('GeminiTTSProvider.synthesize metering', () => { - it('meters input + output:audio as batched line items at the model rates', async () => { - const provider = makeProvider(); - generateContentMock.mockResolvedValueOnce( - audioResponse(undefined, { - promptTokenCount: 8, - candidatesTokenCount: 200, - }), - ); - - await withTestActor(() => - provider.synthesize({ - text: 'hi', - model: 'gemini-3.8-flash-tts', - }), - ); - - expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); - const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; - const types = (entries as Array<{ usageType: string }>).map( - (e) => e.usageType, - ); - expect(types).toEqual([ - 'gemini:gemini-3.8-flash-tts:input', - 'gemini:gemini-3.8-flash-tts:output:audio', - ]); - const inputEntry = ( - entries as Array<{ usageType: string; usageAmount: number }> - ).find((e) => e.usageType.endsWith(':input'))!; - const outputEntry = ( - entries as Array<{ usageType: string; usageAmount: number }> - ).find((e) => e.usageType.endsWith('output:audio'))!; - expect(inputEntry.usageAmount).toBe(8); - expect(outputEntry.usageAmount).toBe(200); + expect(result.content_type).toBe('audio/wav'); + const wav = await readAll(result.stream); + expect(wav.length).toBe(44 + pcm.length); + expect(wav.subarray(0, 4).toString()).toBe('RIFF'); + expect(wav.subarray(8, 12).toString()).toBe('WAVE'); + expect(wav.readUInt32LE(24)).toBe(16_000); + expect(wav.subarray(44)).toEqual(pcm); }); - it('falls back to estimated token counts when usageMetadata is missing', async () => { - const provider = makeProvider(); - generateContentMock.mockResolvedValueOnce({ - candidates: [ - { - content: { - parts: [ - { - inlineData: { - mimeType: 'audio/L16;rate=24000', - data: Buffer.from('p').toString('base64'), - }, - }, + it('wraps PCM without a declared rate at 24 kHz', async () => { + replyWith( + interaction({ + content: [audioBlock(Buffer.from('pcm'), 'audio/pcm')], + }), + ); + const result = (await withTestActor(() => + makeProvider().synthesize({ text: 'hi' }), + )) as { stream: Readable }; + expect((await readAll(result.stream)).readUInt32LE(24)).toBe(24_000); + }); + + it('returns the last audio block across steps and parts', async () => { + const first = Buffer.from(WAV_BYTES); + first.write('first', 50, 'latin1'); + const last = Buffer.from(WAV_BYTES); + last.write('last!', 50, 'latin1'); + replyWith( + interaction({ + steps: [ + { + type: 'thought', + content: [audioBlock(Buffer.from('x'))], + }, + { + type: 'model_output', + content: [ + { type: 'text', text: 'note' }, + audioBlock(first), ], }, - }, - ], - // No usageMetadata. - }); + { + type: 'model_output', + content: [ + audioBlock(last), + { type: 'text', text: 'end' }, + ], + }, + ], + }), + ); - await withTestActor(() => provider.synthesize({ text: 'hello there' })); - - const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; - const inputEntry = ( - entries as Array<{ usageType: string; usageAmount: number }> - ).find((e) => e.usageType.endsWith(':input'))!; - // estimatedInputTokens = ceil(len/4) for 'hello there' (11 chars) = 3. - expect(inputEntry.usageAmount).toBe(3); + const result = (await withTestActor(() => + makeProvider().synthesize({ text: 'hi' }), + )) as { stream: Readable }; + expect(await readAll(result.stream)).toEqual(last); }); }); -// ── Error paths ───────────────────────────────────────────────────── +// -- Upstream failures -- -describe('GeminiTTSProvider.synthesize error paths', () => { - it('throws 500 when no cost data exists for the resolved model', async () => { - const provider = makeProvider(); - // Surgically delete the cost entry for the flash model so the - // provider can't find pricing during the request. - const stash = GEMINI_TTS_COSTS['gemini-3.8-flash-tts']; - delete (GEMINI_TTS_COSTS as Record)[ - 'gemini-3.8-flash-tts' - ]; - try { - await expect( - withTestActor(() => provider.synthesize({ text: 'hi' })), - ).rejects.toMatchObject({ statusCode: 500 }); - } finally { - (GEMINI_TTS_COSTS as Record)[ - 'gemini-3.8-flash-tts' - ] = stash; - } - }); +describe('GeminiTTSProvider.synthesize upstream failures', () => { + it.each([['failed'], ['cancelled'], ['incomplete'], ['in_progress']])( + 'charges the reported usage once, then fails a %s interaction as 502 upstream_failed', + async (status) => { + const { released } = trackHolds(); + replyWith(interaction({ status })); - it('lets SDK errors bubble untouched so the driver boundary can classify them', async () => { - const provider = makeProvider(); - // Google GenAI `ApiError`s carry `.status` — surfacing them - // through the driver-boundary translator yields a proper - // `upstream_*` HttpError instead of a 500/502 page. - const apiError = Object.assign(new Error('bad voice'), { status: 400 }); - generateContentMock.mockRejectedValueOnce(apiError); + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); - await expect( - withTestActor(() => provider.synthesize({ text: 'hi' })), - ).rejects.toMatchObject({ status: 400, message: 'bad voice' }); - expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); - }); + expect(err).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + }); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual(SETTLED_USAGE); + expect(released).toHaveBeenCalledTimes(1); + }, + ); - it('throws 400 upstream_bad_request when Gemini response has no inline audio data', async () => { - const provider = makeProvider(); - generateContentMock.mockResolvedValueOnce({ - candidates: [{ content: { parts: [{ text: 'no audio here' }] } }], - usageMetadata: { promptTokenCount: 1, candidatesTokenCount: 1 }, - }); + it('charges the reported usage once, then fails a refusal (text only) as 400 upstream_bad_request', async () => { + const { released } = trackHolds(); + replyWith( + interaction({ + content: [{ type: 'text', text: 'I cannot read that.' }], + }), + ); - await expect( - withTestActor(() => provider.synthesize({ text: 'hi' })), - ).rejects.toMatchObject({ + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode: 400, legacyCode: 'upstream_bad_request', }); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual(SETTLED_USAGE); + expect(released).toHaveBeenCalledTimes(1); + }); + + const FAILED_REPLIES: [ + string, + { status?: string; content?: unknown[] }, + number, + ][] = [ + ['a failed interaction', { status: 'failed' }, 502], + [ + 'a refusal', + { content: [{ type: 'text', text: 'I cannot read that.' }] }, + 400, + ], + [ + 'malformed audio', + { content: [{ type: 'audio', mime_type: 'audio/wav', data: '' }] }, + 502, + ], + ]; + + it.each(FAILED_REPLIES)( + 'charges nothing for %s without usage, and releases the hold', + async (_label, opts, statusCode) => { + const { released } = trackHolds(); + replyWith(interaction({ ...opts, usage: undefined })); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode }); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it.each(FAILED_REPLIES)( + 'charges only the input for %s whose output total has no audio breakdown', + async (_label, opts, statusCode) => { + const { released } = trackHolds(); + replyWith( + interaction({ + ...opts, + usage: { total_input_tokens: 7, total_output_tokens: 68 }, + }), + ); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode }); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual([SETTLED_USAGE[0]]); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it.each(FAILED_REPLIES)( + 'charges only the input for %s when only the input count is usable', + async (_label, opts, statusCode) => { + const { released } = trackHolds(); + replyWith( + interaction({ + ...opts, + usage: { + input_tokens_by_modality: [ + { modality: 'text', tokens: 7 }, + ], + output_tokens_by_modality: [ + { modality: 'audio', tokens: -1 }, + ], + }, + }), + ); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode }); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual([SETTLED_USAGE[0]]); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it.each([ + ['no data', { type: 'audio', mime_type: 'audio/wav' }], + [ + 'a URI instead of inline data', + { + type: 'audio', + mime_type: 'audio/wav', + uri: 'https://generativelanguage.googleapis.com/files/abc', + }, + ], + ['empty data', { type: 'audio', mime_type: 'audio/wav', data: '' }], + [ + 'invalid base64', + { type: 'audio', mime_type: 'audio/wav', data: '***not base64***' }, + ], + ['no MIME type', { type: 'audio', data: WAV_BYTES.toString('base64') }], + ['an unsupported MIME type', audioBlock(WAV_BYTES, 'audio/mpeg')], + [ + 'WAV bytes without a RIFF header', + audioBlock(Buffer.alloc(64, 1), 'audio/wav'), + ], + ['empty PCM', audioBlock(Buffer.alloc(0), 'audio/L16;rate=24000')], + ])( + 'charges the reported usage once, then fails an audio block with %s as 502 upstream_failed', + async (_label, block) => { + replyWith(interaction({ content: [block] })); + await expect( + withTestActor(() => makeProvider().synthesize({ text: 'hi' })), + ).rejects.toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + }); + // Never follows a provider URI. + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual(SETTLED_USAGE); + }, + ); + + it('fails a non-JSON reply as 502 upstream_failed', async () => { + fetchSpy.mockResolvedValueOnce(new Response('oops')); + await expect( + withTestActor(() => makeProvider().synthesize({ text: 'hi' })), + ).rejects.toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + }); + }); + + it.each([ + [400, 400, 'upstream_bad_request'], + [404, 400, 'upstream_bad_request'], + [401, 500, 'upstream_auth_failed'], + [429, 429, 'upstream_rate_limited'], + [500, 400, 'upstream_provider_unavailable'], + [503, 400, 'upstream_provider_unavailable'], + ])( + 'maps an upstream %i to %i %s and releases the hold', + async (upstreamStatus, statusCode, legacyCode) => { + const { released } = trackHolds(); + replyWith( + { + error: { + code: upstreamStatus, + message: 'Request failed', + status: 'SOME_STATUS', + }, + }, + { status: upstreamStatus }, + ); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ + statusCode, + legacyCode, + fields: { provider: 'gemini', upstreamStatus }, + }); + expect(released).toHaveBeenCalledTimes(1); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + }, + ); + + it('charges the held budget for a transport timeout and says how to avoid it', async () => { + const { held, released } = trackHolds(); + fetchSpy.mockRejectedValueOnce( + new DOMException( + 'The operation was aborted due to timeout', + 'TimeoutError', + ), + ); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ + statusCode: 504, + legacyCode: 'upstream_timeout', + message: expect.stringContaining('Split the text'), + }); + expect(released).toHaveBeenCalledTimes(1); + expect(metered(':input').usageAmount).toBe(8192); + expect(metered(':output:audio').usageAmount).toBe(16_384); + expect( + meteredUsages().reduce((sum, u) => sum + u.costOverride!, 0), + ).toBe(held[0]); + }); + + it('surfaces a body that times out mid-read as 504 upstream_timeout', async () => { + fetchSpy.mockResolvedValueOnce( + new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(new TextEncoder().encode('{"st')); + controller.error( + new DOMException( + 'The operation was aborted due to timeout', + 'TimeoutError', + ), + ); + }, + }), + ), + ); + + const err = await callerError(() => + makeProvider().synthesize({ text: 'hi' }), + ); + + expect(err).toMatchObject({ + statusCode: 504, + legacyCode: 'upstream_timeout', + }); + }); + + it('stops reading a reply over the size cap, charges the held budget and says how to avoid it', async () => { + const { released } = trackHolds(); + const chunk = new Uint8Array(1024 * 1024).fill(0x20); + let sent = 0; + fetchSpy.mockResolvedValueOnce( + new Response( + new ReadableStream({ + pull(controller) { + if (sent > GEMINI_TTS_MAX_RESPONSE_BYTES) { + controller.close(); + return; + } + sent += chunk.byteLength; + controller.enqueue(chunk); + }, + }), + ), + ); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'input_too_large', + message: expect.stringContaining('Split the text'), + }); + expect(sent).toBeLessThanOrEqual( + GEMINI_TTS_MAX_RESPONSE_BYTES + 2 ** 20, + ); + expect(released).toHaveBeenCalledTimes(1); + expect(metered(':output:audio').usageAmount).toBe(16_384); + }); + + it('reads a reply big enough for the whole audio budget', () => { + // ~25 tokens/s of 24 kHz 16-bit PCM, base64-encoded. + const seconds = 16_384 / 25; + const base64Bytes = Math.ceil((seconds * 24_000 * 2) / 3) * 4; + expect(GEMINI_TTS_MAX_RESPONSE_BYTES).toBeGreaterThan(base64Bytes); + }); + + it('aborts the upstream call when the caller disconnects, and releases the hold', async () => { + const { released } = trackHolds(); + const abort = new AbortController(); + let upstreamSignal: AbortSignal | undefined; + fetchSpy.mockImplementationOnce( + (_url, init) => + new Promise((_resolve, reject) => { + upstreamSignal = init!.signal!; + upstreamSignal.addEventListener( + 'abort', + () => reject(upstreamSignal!.reason), + { once: true }, + ); + abort.abort(); + }), + ); + + await expect( + withTestActor(() => { + Context.set('abortSignal', abort.signal); + return makeProvider().synthesize({ text: 'hi' }); + }, payingUser()), + ).rejects.toMatchObject({ name: 'AbortError' }); + + expect(upstreamSignal?.aborted).toBe(true); + expect(released).toHaveBeenCalledTimes(1); expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); }); }); + +// -- Metering -- + +describe('GeminiTTSProvider.synthesize metering', () => { + it.each([ + ['gemini-3.8-flash-tts', 900], + ['gemini-3.8-flash-lite-tts', 600], + ])( + 'meters %s text input and audio output from the usage modalities', + async (model, audioRate) => { + replyWith(interaction()); + + await withTestActor(() => + makeProvider().synthesize({ + text: 'hi', + model, + instructions: 'Calm', + }), + ); + + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual([ + { + usageType: `gemini:${model}:input`, + usageAmount: 7, + costOverride: 7 * 50, + }, + { + usageType: `gemini:${model}:output:audio`, + usageAmount: 68, + costOverride: 68 * audioRate, + }, + ]); + }, + ); + + it('sums repeated modality entries', async () => { + replyWith( + interaction({ + usage: { + input_tokens_by_modality: [ + { modality: 'TEXT', tokens: 3 }, + { modality: 'text', tokens: 4 }, + ], + output_tokens_by_modality: [ + { modality: 'audio', tokens: 40 }, + { modality: 'audio', tokens: 28 }, + ], + }, + }), + ); + await withTestActor(() => makeProvider().synthesize({ text: 'hi' })); + expect(metered(':input').usageAmount).toBe(7); + expect(metered(':output:audio').usageAmount).toBe(68); + }); + + it('uses the aggregate totals when audio comes back without a modality breakdown', async () => { + replyWith( + interaction({ + usage: { total_input_tokens: 9, total_output_tokens: 80 }, + }), + ); + await withTestActor(() => makeProvider().synthesize({ text: 'hi' })); + expect(metered(':input').usageAmount).toBe(9); + expect(metered(':output:audio').usageAmount).toBe(80); + }); + + it.each([ + ['missing usage', undefined], + ['null usage', null], + [ + 'unsafe aggregate counts', + { + input_tokens_by_modality: [ + { modality: 'text', tokens: Number.MAX_SAFE_INTEGER }, + { modality: 'text', tokens: Number.MAX_SAFE_INTEGER }, + ], + output_tokens_by_modality: [ + { modality: 'audio', tokens: Number.MAX_SAFE_INTEGER }, + { modality: 'audio', tokens: Number.MAX_SAFE_INTEGER }, + ], + }, + ], + [ + 'unsafe token costs', + { + total_input_tokens: Number.MAX_SAFE_INTEGER, + total_output_tokens: Number.MAX_SAFE_INTEGER, + }, + ], + [ + 'negative counts', + { + input_tokens_by_modality: [{ modality: 'text', tokens: -1 }], + output_tokens_by_modality: [{ modality: 'audio', tokens: -5 }], + }, + ], + [ + 'fractional and string counts', + { + input_tokens_by_modality: [{ modality: 'text', tokens: 1.5 }], + output_tokens_by_modality: [ + { modality: 'audio', tokens: '68' }, + ], + }, + ], + [ + 'non-finite totals', + { total_input_tokens: Infinity, total_output_tokens: NaN }, + ], + [ + 'a breakdown that lacks the billed modality', + { + total_input_tokens: 7, + input_tokens_by_modality: [{ modality: 'image', tokens: 7 }], + total_output_tokens: 68, + output_tokens_by_modality: [{ modality: 'text', tokens: 68 }], + }, + ], + ['reported zeros', { total_input_tokens: 0, total_output_tokens: 0 }], + ])( + 'charges nothing on %s and fails the audio as 502 upstream_failed', + async (_label, usage) => { + const { released } = trackHolds(); + replyWith(interaction({ usage })); + + const err = await callerError( + () => + makeProvider().synthesize({ + text: 'hello there', + instructions: 'Speak softly', + }), + payingUser(), + ); + + expect(err).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + }); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it.each([ + [ + 'input', + { + total_input_tokens: 7, + output_tokens_by_modality: [{ modality: 'audio', tokens: -1 }], + }, + [SETTLED_USAGE[0]], + ], + [ + 'audio', + { + total_input_tokens: 'seven', + output_tokens_by_modality: [{ modality: 'audio', tokens: 68 }], + }, + [SETTLED_USAGE[1]], + ], + ])( + 'charges only the readable %s count once, then fails the audio as 502 upstream_failed', + async (_label, usage, settled) => { + const { released } = trackHolds(); + replyWith(interaction({ usage })); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + }); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual(settled); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it('charges a reported zero as nothing, with no one-token minimum', async () => { + replyWith( + interaction({ + status: 'failed', + usage: { + total_input_tokens: 7, + output_tokens_by_modality: [ + { modality: 'audio', tokens: 0 }, + ], + }, + }), + ); + + await callerError(() => makeProvider().synthesize({ text: 'hi' })); + expect(meteredUsages()).toEqual([SETTLED_USAGE[0]]); + }); + + const TEXTS: [string, string][] = [ + ['short spaced text', 'hi'], + ['long spaced English', 'Read me. '.repeat(333)], + ['long unspaced Latin', 'a'.repeat(3000)], + ['unspaced Chinese', '你好世界'.repeat(750)], + ['unspaced Japanese', 'こんにちは世界'.repeat(428)], + ['unspaced Thai', 'สวัสดีครับ'.repeat(300)], + ['emoji', '😀'.repeat(1500)], + ]; + const RESERVATION_CASES: [string, string, string | undefined][] = + TEXTS.flatMap(([label, text]) => [ + [`${label} without instructions`, text, undefined], + [`${label} with maximum instructions`, text, '慢'.repeat(3000)], + ]); + + it.each(RESERVATION_CASES)( + 'holds the full token budget for %s and settles only the reported usage', + async (_label, text, instructions) => { + const { held, released } = trackHolds(); + replyWith(interaction()); + + await withTestActor( + () => makeProvider().synthesize({ text, instructions }), + payingUser(), + ); + + expect(held).toEqual([FLASH_RESERVATION]); + expect(meteredUsages()).toEqual(SETTLED_USAGE); + await expect.poll(() => released.mock.calls.length).toBe(1); + }, + ); +}); + +// -- Scheduled rates -- + +describe('GeminiTTSProvider.synthesize scheduled rates', () => { + const BEFORE = GEMINI_TTS_LAUNCH_PRICING_ENDS - 1; + const AT = GEMINI_TTS_LAUNCH_PRICING_ENDS; + + beforeEach(() => { + vi.useFakeTimers({ toFake: ['Date'], now: BEFORE }); + }); + + const SCHEDULE: [string, number, number, number, number][] = [ + ['gemini-3.8-flash-tts', BEFORE, 50, 900, 15_155_200], + ['gemini-3.8-flash-tts', AT, 100, 1800, 30_310_400], + ['gemini-3.8-flash-lite-tts', BEFORE, 50, 600, 10_240_000], + ['gemini-3.8-flash-lite-tts', AT, 100, 1200, 20_480_000], + ]; + + it.each(SCHEDULE)( + 'holds and meters %s started at %i at %i/%i µ¢ per token', + async (model, startedAt, inputRate, audioRate, reserved) => { + vi.setSystemTime(startedAt); + const { held } = trackHolds(); + replyWith(interaction()); + + await withTestActor( + () => makeProvider().synthesize({ text: 'hi', model }), + payingUser(), + ); + + expect(held).toEqual([reserved]); + expect(reservation(inputRate, audioRate)).toBe(reserved); + expect(metered(':input').costOverride).toBe(7 * inputRate); + expect(metered(':output:audio').costOverride).toBe(68 * audioRate); + }, + ); + + it.each(SCHEDULE)( + 'holds and meters %s started at %i at its AI cost factor', + async (model, startedAt, inputRate, audioRate, reserved) => { + vi.setSystemTime(startedAt); + const { held } = trackHolds(); + const factor = (_key: string, event: { factor: number }) => { + event.factor = 1.5; + }; + server.clients.event.on('ai.cost.factor.*', factor); + replyWith(interaction()); + try { + await withTestActor( + () => makeProvider().synthesize({ text: 'hi', model }), + payingUser(), + ); + } finally { + server.clients.event.off('ai.cost.factor.*', factor); + } + + expect(hasCreditsSpy).toHaveBeenCalledWith( + expect.anything(), + reserved * 1.5, + ); + expect(held).toEqual([reserved * 1.5]); + expect(metered(':input').costOverride).toBe(7 * inputRate * 1.5); + expect(metered(':output:audio').costOverride).toBe( + 68 * audioRate * 1.5, + ); + }, + ); + + it('settles a request that crosses the change at the rates it was held at', async () => { + const { held } = trackHolds(); + fetchSpy.mockImplementationOnce(async () => { + vi.setSystemTime(AT); + return Response.json(interaction()); + }); + + await withTestActor( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(held).toEqual([FLASH_RESERVATION]); + expect(metered(':input').costOverride).toBe(7 * 50); + expect(metered(':output:audio').costOverride).toBe(68 * 900); + }); + + it('throws 500 when no cost data exists for the resolved model', async () => { + const stash = GEMINI_TTS_COSTS['gemini-3.8-flash-tts']; + delete GEMINI_TTS_COSTS['gemini-3.8-flash-tts']; + try { + await expect( + withTestActor(() => makeProvider().synthesize({ text: 'hi' })), + ).rejects.toMatchObject({ statusCode: 500 }); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + GEMINI_TTS_COSTS['gemini-3.8-flash-tts'] = stash; + } + }); +}); diff --git a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts index 6f95f0816..3f8353ee6 100644 --- a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts +++ b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts @@ -17,15 +17,26 @@ * along with this program. If not, see . */ -import { GoogleGenAI } from '@google/genai'; import { Readable } from 'node:stream'; +import type { Actor } from '../../../../core/actor.js'; import { HttpError } from '../../../../core/http/HttpError.js'; import { Context } from '../../../../core/context.js'; import { type AiMeteringService, withAiCreditHold, } from '../../../util/aiCostFactor.js'; +import type { UsageInput } from '../../../../services/metering/types.js'; import type { DriverStreamResult } from '../../../meta.js'; +import { + AI_TTS_GEMINI_RESERVED_TOKENS, + AI_TTS_MAX_GEMINI_INSTRUCTIONS_CHARS, + AI_TTS_MAX_GEMINI_TEXT_CHARS, +} from '../../../util/aiLimits.js'; +import { readBoundedBody } from '../../../util/fileInput.js'; +import { + isUpstreamTimeoutError, + upstreamFetch, +} from '../../../util/upstreamErrors.js'; import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js'; import type { ITTSVoice, @@ -33,10 +44,25 @@ import type { ISynthesizeArgs, ITTSProvider, } from '../../types.js'; -import { GEMINI_TTS_COSTS } from './costs.js'; +import { assertMaxChars, TTS_UPSTREAM_TIMEOUT_MS } from '../common.js'; +import { + GEMINI_TTS_COSTS, + type GeminiTTSCosts, + geminiTTSCostsAt, +} from './costs.js'; + +const INTERACTIONS_URL = + 'https://generativelanguage.googleapis.com/v1beta/interactions'; + +/** + * Largest Interactions reply read; audio arrives base64-encoded inside JSON. + * The full audio budget (~11 min of 24 kHz PCM at ~25 tokens/s) is ~42 MB. + */ +export const GEMINI_TTS_MAX_RESPONSE_BYTES = 64 * 1024 * 1024; const DEFAULT_MODEL = 'gemini-3.8-flash-tts'; const DEFAULT_VOICE = 'Kore'; +const DEFAULT_PCM_SAMPLE_RATE = 24_000; const GEMINI_TTS_MODELS = [ { @@ -82,15 +108,138 @@ const GEMINI_TTS_VOICES = [ { id: 'Sulafat', name: 'Sulafat', description: 'Warm' }, ]; +// -- Interactions API wire shapes -- + +interface InteractionTextInput { + type: 'text'; + text: string; + annotations?: { type: 'speech_metadata'; style: string }[]; +} + +interface InteractionRequest { + model: string; + input: { type: 'user_input'; content: InteractionTextInput[] }[]; + response_format: { type: 'audio' }; + generation_config: { + speech_config: { voice: string }[]; + max_output_tokens: number; + }; + store: false; + background: false; + stream: false; +} + +interface InteractionContent { + type?: string; + mime_type?: string; + data?: string; +} + +interface InteractionModalityTokens { + modality?: string; + tokens?: number; +} + +interface InteractionUsage { + total_input_tokens?: number; + input_tokens_by_modality?: InteractionModalityTokens[]; + total_output_tokens?: number; + output_tokens_by_modality?: InteractionModalityTokens[]; +} + +/** The reply as documented; every field is checked before use. */ +interface Interaction { + status?: string; + steps?: { type?: string; content?: InteractionContent[] }[]; + usage?: InteractionUsage; +} + +const WAV_MIME_TYPES = new Set(['audio/wav', 'audio/x-wav', 'audio/wave']); +const PCM_MIME_TYPES = new Set(['audio/l16', 'audio/pcm']); +const BASE64_PATTERN = /^[A-Za-z0-9+/_-]+={0,2}$/; + +const upstreamFailure = (message: string): HttpError => + new HttpError(502, message, { + legacyCode: 'upstream_failed', + fields: { provider: 'gemini' }, + }); + +const SPLIT_TEXT_HINT = 'Split the text into shorter requests.'; + +const audioTooLong = (): HttpError => + new HttpError( + 400, + `Gemini TTS generated more audio than one request can return. ${SPLIT_TEXT_HINT}`, + { legacyCode: 'input_too_large', fields: { provider: 'gemini' } }, + ); + +const audioTimedOut = (cause: unknown): HttpError => + new HttpError( + 504, + `Gemini TTS timed out generating the audio. ${SPLIT_TEXT_HINT}`, + { + legacyCode: 'upstream_timeout', + fields: { provider: 'gemini' }, + cause, + }, + ); + +/** A usable token count: a finite, non-negative integer. */ +const tokenCount = (value: unknown): number | undefined => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 + ? value + : undefined; + /** - * Gemini TTS provider. Calls the Gemini generateContent API with - * `responseModalities: ["AUDIO"]` and `speechConfig` to synthesize speech. - * Returns raw PCM audio wrapped in a WAV container. + * `modality`'s tokens from a by-modality breakdown, or `total` when there is no + * breakdown to read. Undefined when the counts are unusable or ambiguous. + */ +const usageTokens = ( + byModality: unknown, + total: unknown, + modality: string, + rate: number, +): number | undefined => { + if (!Array.isArray(byModality) || byModality.length === 0) { + const count = tokenCount(total); + return count !== undefined && Number.isSafeInteger(count * rate) + ? count + : undefined; + } + let sum: number | undefined; + for (const entry of byModality as InteractionModalityTokens[]) { + if (String(entry?.modality).toLowerCase() !== modality) continue; + const tokens = tokenCount(entry.tokens); + if (tokens === undefined) return undefined; + sum = (sum ?? 0) + tokens; + if (!Number.isSafeInteger(sum * rate)) return undefined; + } + return sum; +}; + +/** The last audio block of the model's output, as the API documents it. */ +const lastAudioContent = ( + interaction: Interaction, +): InteractionContent | undefined => { + const steps = Array.isArray(interaction.steps) ? interaction.steps : []; + return steps + .flatMap((step) => + step?.type === 'model_output' && Array.isArray(step.content) + ? step.content + : [], + ) + .filter((content) => content?.type === 'audio') + .at(-1); +}; + +/** + * Gemini TTS provider. Calls the Gemini Interactions API, which reads `text` + * verbatim and takes `instructions` as a separate speech-style annotation. */ export class GeminiTTSProvider implements ITTSProvider { readonly providerName = 'gemini'; - #client: GoogleGenAI; + #apiKey: string; constructor( private readonly meteringService: AiMeteringService, @@ -99,7 +248,7 @@ export class GeminiTTSProvider implements ITTSProvider { if (!config.apiKey) { throw new Error('Gemini TTS requires an API key'); } - this.#client = new GoogleGenAI({ apiKey: config.apiKey }); + this.#apiKey = config.apiKey; } async listVoices(): Promise { @@ -121,15 +270,20 @@ export class GeminiTTSProvider implements ITTSProvider { } getReportedCosts(): Record[] { - return Object.entries(GEMINI_TTS_COSTS).map(([model, costs]) => ({ - usageType: `gemini:${model}:tts`, - ucentsInputPerToken: this.#toMicroCents(costs.input / 1_000_000), - ucentsOutputAudioPerToken: this.#toMicroCents( - costs.output_audio / 1_000_000, - ), - unit: 'token', - source: 'driver:aiTts/gemini', - })); + const now = Date.now(); + return Object.keys(GEMINI_TTS_COSTS).flatMap((model) => { + const costs = geminiTTSCostsAt(model, now); + if (!costs) return []; + return [ + { + usageType: `gemini:${model}:tts`, + ucentsInputPerToken: costs.input, + ucentsOutputAudioPerToken: costs.output_audio, + unit: 'token', + source: 'driver:aiTts/gemini', + }, + ]; + }); } async synthesize( @@ -153,6 +307,20 @@ export class GeminiTTSProvider implements ITTSProvider { fields: { key: 'text' }, }); } + assertMaxChars('text', text, AI_TTS_MAX_GEMINI_TEXT_CHARS); + + if (instructions != null && typeof instructions !== 'string') { + throw new HttpError(400, 'instructions must be a string', { + legacyCode: 'field_invalid', + fields: { key: 'instructions', expected: 'string' }, + }); + } + assertMaxChars( + 'instructions', + instructions, + AI_TTS_MAX_GEMINI_INSTRUCTIONS_CHARS, + ); + const style = instructions?.trim() ? instructions : undefined; const model = modelArg || DEFAULT_MODEL; if (!GEMINI_TTS_MODELS.find(({ id }) => id === model)) { @@ -172,15 +340,14 @@ export class GeminiTTSProvider implements ITTSProvider { ); } - const voice = voiceArg || DEFAULT_VOICE; - if ( - !GEMINI_TTS_VOICES.find( - ({ id }) => id.toLowerCase() === voice.toLowerCase(), - ) - ) { + const requestedVoice = String(voiceArg || DEFAULT_VOICE); + const voice = GEMINI_TTS_VOICES.find( + ({ id }) => id.toLowerCase() === requestedVoice.toLowerCase(), + )?.id; + if (!voice) { throw new HttpError( 400, - `Invalid voice: ${voice}. Expected: ${GEMINI_TTS_VOICES.map(({ id }) => id).join(', ')}`, + `Invalid voice: ${requestedVoice}. Expected: ${GEMINI_TTS_VOICES.map(({ id }) => id).join(', ')}`, { legacyCode: 'field_invalid', fields: { @@ -188,145 +355,250 @@ export class GeminiTTSProvider implements ITTSProvider { expected: GEMINI_TTS_VOICES.map(({ id }) => id).join( ', ', ), - got: voice, + got: requestedVoice, }, }, ); } const actor = Context.get('actor')!; - const costs = GEMINI_TTS_COSTS[model]; + // Priced once, so a request that crosses a rate change settles at the + // rates it was held at. + const costs = geminiTTSCostsAt(model, Date.now()); if (!costs) { throw new HttpError(500, `No cost data for model: ${model}`, { legacyCode: 'internal_error', }); } - // Estimate input tokens (~4 chars per token) and a rough output - // audio duration (~150 words/min, 25 tokens/sec). - const estimatedInputTokens = Math.max(1, Math.ceil(text.length / 4)); - const wordCount = text.split(/\s+/).length; - const estimatedDurationSec = Math.max(1, (wordCount / 150) * 60); - const estimatedOutputTokens = Math.ceil(estimatedDurationSec * 25); + // The full token budget, whatever the text; settlement charges only + // the reported usage. + const reservedMicroCents = + AI_TTS_GEMINI_RESERVED_TOKENS.input * costs.input + + AI_TTS_GEMINI_RESERVED_TOKENS.audio * costs.output_audio; - const estimatedInputCostCents = - (estimatedInputTokens / 1_000_000) * costs.input; - const estimatedOutputCostCents = - (estimatedOutputTokens / 1_000_000) * costs.output_audio; - const estimatedTotalMicroCents = this.#toMicroCents( - estimatedInputCostCents + estimatedOutputCostCents, - ); + const content: InteractionTextInput = { type: 'text', text }; + if (style) { + content.annotations = [{ type: 'speech_metadata', style }]; + } + const request: InteractionRequest = { + model, + input: [{ type: 'user_input', content: [content] }], + response_format: { type: 'audio' }, + generation_config: { + speech_config: [{ voice }], + // Request the same audio budget used for the credit hold. + max_output_tokens: AI_TTS_GEMINI_RESERVED_TOKENS.audio, + }, + store: false, + background: false, + stream: false, + }; return withAiCreditHold( this.meteringService, actor, `gemini:${model}`, - estimatedTotalMicroCents, + reservedMicroCents, async (): Promise => { - // The TTS models require the text to be framed as a transcript - // to read aloud. Prefixing with "Say:" prevents the model from - // trying to generate conversational text instead of audio. - const inputText = instructions - ? `${instructions}\n\nSay the following text aloud:\n${text}` - : `Say the following text aloud:\n${text}`; - - // Let Google GenAI `ApiError`s bubble — they carry `.status` and - // are mapped to `upstream_*` HttpErrors by the driver-boundary - // translator. Catching here and wrapping as 502 hid the upstream - // status and caused 4xx validation errors to page. - // eslint-disable-next-line @typescript-eslint/no-explicit-any - const response: any = await this.#client.models.generateContent( - { - model, - contents: [{ parts: [{ text: inputText }] }], - config: { - responseModalities: ['AUDIO'], - speechConfig: { - voiceConfig: { - prebuiltVoiceConfig: { voiceName: voice }, - }, - }, - }, - }, - ); - - // Extract audio data from response - const part = response?.candidates?.[0]?.content?.parts?.[0]; - if (!part?.inlineData?.data) { - throw new HttpError( - 400, - 'Gemini TTS did not return audio data', + const signal = Context.get('abortSignal'); + let overran = false; + let raw: Buffer; + try { + // Non-2xx answers are classified at the driver boundary. + const response = await upstreamFetch( + 'gemini', + INTERACTIONS_URL, { - legacyCode: 'upstream_bad_request', - fields: { provider: 'gemini' }, + method: 'POST', + headers: { + 'x-goog-api-key': this.#apiKey, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(request), + signal, }, + { + providerLabel: 'Gemini', + timeoutMs: TTS_UPSTREAM_TIMEOUT_MS, + }, + ); + raw = await readBoundedBody( + response, + GEMINI_TTS_MAX_RESPONSE_BYTES, + () => { + overran = true; + return audioTooLong(); + }, + ); + } catch (err) { + const timedOut = + isUpstreamTimeoutError(err) && !signal?.aborted; + if (!overran && !timedOut) throw err; + // Google bills audio we never got to read, and there is + // no usage to settle from, so the held budget is charged. + await this.#meter( + actor, + model, + costs, + AI_TTS_GEMINI_RESERVED_TOKENS.input, + AI_TTS_GEMINI_RESERVED_TOKENS.audio, + ); + throw overran ? err : audioTimedOut(err); + } + + let interaction: Interaction | null = null; + try { + interaction = JSON.parse(raw.toString('utf8')); + } catch { + // Handled with the non-object case below. + } + if (!interaction || typeof interaction !== 'object') { + throw upstreamFailure( + 'Gemini TTS returned an unreadable response', ); } - const audioBase64: string = part.inlineData.data; - const mimeType: string = - part.inlineData.mimeType || 'audio/L16;rate=24000'; + // A count Gemini doesn't report readably is left out, not + // estimated. Output without a modality breakdown may be refusal + // text, so it counts as audio only once audio came back. + const usage = interaction.usage; + const inputTokens = usageTokens( + usage?.input_tokens_by_modality, + usage?.total_input_tokens, + 'text', + costs.input, + ); + let outputTokens = usageTokens( + usage?.output_tokens_by_modality, + undefined, + 'audio', + costs.output_audio, + ); + try { + if (interaction.status !== 'completed') { + throw upstreamFailure( + 'Gemini TTS did not complete the request', + ); + } - // Convert base64 PCM to a WAV buffer for broad client compatibility - const pcmBuffer = Buffer.from(audioBase64, 'base64'); - let outputBuffer: Buffer; - let contentType: string; + const audio = lastAudioContent(interaction); + if (!audio) { + throw new HttpError( + 400, + 'Gemini TTS did not return audio data', + { + legacyCode: 'upstream_bad_request', + fields: { provider: 'gemini' }, + }, + ); + } + const { body, contentType } = this.#decodeAudio(audio); - if ( - mimeType.startsWith('audio/L16') || - mimeType === 'audio/pcm' - ) { - // Wrap raw PCM (16-bit LE, 24kHz, mono) in a WAV container - outputBuffer = this.#wrapPcmInWav(pcmBuffer, 24000, 1, 16); - contentType = 'audio/wav'; - } else { - // If the API returns encoded audio (unlikely today), pass through - outputBuffer = pcmBuffer; - contentType = mimeType; + outputTokens = usageTokens( + usage?.output_tokens_by_modality, + usage?.total_output_tokens, + 'audio', + costs.output_audio, + ); + if (!inputTokens || !outputTokens) { + throw upstreamFailure( + 'Gemini TTS did not report usage for the audio', + ); + } + + return { + dataType: 'stream', + content_type: contentType, + chunked: true, + stream: Readable.from(body), + }; + } finally { + // Every outcome pays once for the counts it reports. + await this.#meter( + actor, + model, + costs, + inputTokens, + outputTokens, + ); } - - // Meter actual usage from response metadata - const usage = response.usageMetadata; - const actualInputTokens = - typeof usage?.promptTokenCount === 'number' - ? usage.promptTokenCount - : estimatedInputTokens; - const actualOutputTokens = - typeof usage?.candidatesTokenCount === 'number' - ? usage.candidatesTokenCount - : estimatedOutputTokens; - - const inputCostCents = - (actualInputTokens / 1_000_000) * costs.input; - const outputCostCents = - (actualOutputTokens / 1_000_000) * costs.output_audio; - - const usagePrefix = `gemini:${model}`; - this.meteringService.batchIncrementUsages(actor, [ - { - usageType: `${usagePrefix}:input`, - usageAmount: Math.max(actualInputTokens, 1), - costOverride: this.#toMicroCents(inputCostCents), - }, - { - usageType: `${usagePrefix}:output:audio`, - usageAmount: Math.max(actualOutputTokens, 1), - costOverride: this.#toMicroCents(outputCostCents), - }, - ]); - - const stream = Readable.from(outputBuffer); - - return { - dataType: 'stream', - content_type: contentType, - chunked: true, - stream, - }; }, ); } + async #meter( + actor: Actor, + model: string, + costs: GeminiTTSCosts, + inputTokens: number | undefined, + outputTokens: number | undefined, + ): Promise { + const usagePrefix = `gemini:${model}`; + const usages: UsageInput[] = []; + if (inputTokens) { + usages.push({ + usageType: `${usagePrefix}:input`, + usageAmount: inputTokens, + costOverride: inputTokens * costs.input, + }); + } + if (outputTokens) { + usages.push({ + usageType: `${usagePrefix}:output:audio`, + usageAmount: outputTokens, + costOverride: outputTokens * costs.output_audio, + }); + } + if (usages.length > 0) { + await this.meteringService.batchIncrementUsages(actor, usages); + } + } + + /** WAV passes through byte for byte; raw PCM is wrapped in a WAV header. */ + #decodeAudio(audio: InteractionContent): { + body: Buffer; + contentType: string; + } { + const { data, mime_type: mimeType } = audio; + if ( + typeof data !== 'string' || + !BASE64_PATTERN.test(data) || + typeof mimeType !== 'string' + ) { + throw upstreamFailure('Gemini TTS returned invalid audio data'); + } + const bytes = Buffer.from(data, 'base64'); + const [essence, ...params] = mimeType + .split(';') + .map((part) => part.trim().toLowerCase()); + + if (WAV_MIME_TYPES.has(essence)) { + if ( + bytes.length <= 44 || + bytes.toString('latin1', 0, 4) !== 'RIFF' || + bytes.toString('latin1', 8, 12) !== 'WAVE' + ) { + throw upstreamFailure('Gemini TTS returned invalid audio data'); + } + return { body: bytes, contentType: 'audio/wav' }; + } + if (PCM_MIME_TYPES.has(essence) && bytes.length > 0) { + const rate = Number( + params.find((p) => p.startsWith('rate='))?.slice(5), + ); + const sampleRate = + Number.isSafeInteger(rate) && rate > 0 + ? rate + : DEFAULT_PCM_SAMPLE_RATE; + return { + body: this.#wrapPcmInWav(bytes, sampleRate, 1, 16), + contentType: 'audio/wav', + }; + } + throw upstreamFailure('Gemini TTS returned invalid audio data'); + } + /** Wrap raw PCM samples in a WAV container so browsers can play it. */ #wrapPcmInWav( pcm: Buffer, @@ -362,9 +634,4 @@ export class GeminiTTSProvider implements ITTSProvider { return buffer; } - - #toMicroCents(cents: number): number { - if (!Number.isFinite(cents) || cents <= 0) return 1; - return Math.ceil(cents * 1_000_000); - } } diff --git a/src/backend/drivers/ai-tts/providers/gemini/costs.ts b/src/backend/drivers/ai-tts/providers/gemini/costs.ts index 080259365..6ef5410e5 100644 --- a/src/backend/drivers/ai-tts/providers/gemini/costs.ts +++ b/src/backend/drivers/ai-tts/providers/gemini/costs.ts @@ -17,18 +17,17 @@ * along with this program. If not, see . */ -// Gemini TTS pricing in USD per 1M tokens (launch pricing through -// 2026-12-31; both rates double on 2027-01-01): -// gemini-3.8-flash-tts: input $0.50, output (audio) $9.00 -// gemini-3.8-flash-lite-tts: input $0.50, output (audio) $6.00 -// -// Audio output tokens = ~25 tokens/second of audio. -// -// Costs here are in USD-cents per 1M tokens for input and output. -export const GEMINI_TTS_COSTS: Record< - string, - { input: number; output_audio: number } -> = { +/** USD-cents per 1M tokens: text input, and audio output (~25 tokens/s). */ +export interface GeminiTTSCosts { + input: number; + output_audio: number; +} + +/** The instant launch pricing ends and both rates double. */ +export const GEMINI_TTS_LAUNCH_PRICING_ENDS = Date.UTC(2027, 0, 1); + +/** Launch pricing, through 2026-12-31 UTC. */ +export const GEMINI_TTS_COSTS: Record = { 'gemini-3.8-flash-tts': { input: 50, // $0.50 per 1M tokens output_audio: 900, // $9.00 per 1M tokens @@ -38,3 +37,13 @@ export const GEMINI_TTS_COSTS: Record< output_audio: 600, // $6.00 per 1M tokens }, }; + +/** `model`'s rates in force at `at` (ms since epoch); undefined when unpriced. */ +export const geminiTTSCostsAt = ( + model: string, + at: number, +): GeminiTTSCosts | undefined => { + const launch = GEMINI_TTS_COSTS[model]; + if (!launch || at < GEMINI_TTS_LAUNCH_PRICING_ENDS) return launch; + return { input: launch.input * 2, output_audio: launch.output_audio * 2 }; +}; diff --git a/src/backend/drivers/util/aiLimits.ts b/src/backend/drivers/util/aiLimits.ts index 5b8e77826..fdfa1c33c 100644 --- a/src/backend/drivers/util/aiLimits.ts +++ b/src/backend/drivers/util/aiLimits.ts @@ -61,6 +61,24 @@ export const AI_CONCURRENT: DriverConcurrentConfig = { }, }; +// -- Text-to-speech input caps ---------------------------------------- +// +// Counted in UTF-16 code units, as puter.js counts them before sending. + +/** Longest Gemini TTS `text`. Other providers enforce their own limits. */ +export const AI_TTS_MAX_GEMINI_TEXT_CHARS = 3000; + +/** Longest Gemini TTS `instructions`, which are billed as input with `text`. */ +export const AI_TTS_MAX_GEMINI_INSTRUCTIONS_CHARS = 3000; + +/** + * Tokens a Gemini TTS request holds credit for: both models' published input + * and output limits; `audio` is also sent as `max_output_tokens`. Characters + * don't predict tokens or audio length across scripts and delivery, so the + * hold covers the published token budget. + */ +export const AI_TTS_GEMINI_RESERVED_TOKENS = { input: 8192, audio: 16_384 }; + // -- Claude server-tool clamps ---------------------------------------- // // Applied both when ClaudeProvider builds the request (so Anthropic never diff --git a/src/backend/drivers/util/geminiRest.ts b/src/backend/drivers/util/geminiRest.ts new file mode 100644 index 000000000..9dc0bb98f --- /dev/null +++ b/src/backend/drivers/util/geminiRest.ts @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +const GOOGLE_COMPAT_BASE_URL = + 'https://generativelanguage.googleapis.com/v1beta/openai/'; + +/** + * Whether a `gemini` config block's `apiBaseUrl` is unset or Google's own. Any + * other value may be a proxy holding its own key, which must not be sent to + * Google's native APIs. + */ +export const usesGoogleGeminiEndpoint = (apiBaseUrl: unknown): boolean => + apiBaseUrl === undefined || + (typeof apiBaseUrl === 'string' && + apiBaseUrl.replace(/\/*$/, '/') === GOOGLE_COMPAT_BASE_URL); diff --git a/src/docs/src/AI/txt2speech.md b/src/docs/src/AI/txt2speech.md index 80a57214f..51bba674e 100755 --- a/src/docs/src/AI/txt2speech.md +++ b/src/docs/src/AI/txt2speech.md @@ -20,7 +20,7 @@ puter.ai.txt2speech(text, language, voice, engine, testMode = false) #### `text` (String) (required) -A string containing the text you want to convert to speech. The text must be less than 3000 characters long. Defaults to AWS Polly provider when no options are provided. +A string containing the text you want to convert to speech. The text can be up to 3,000 characters long. Longer text is rejected with an `input_too_large` error. Defaults to AWS Polly provider when no options are provided. #### `testMode` (Boolean) (optional) @@ -84,9 +84,13 @@ Available when `provider: 'gemini'`: |--------|------|-------------| | `model` | `String` | TTS model. Available: `'gemini-3.8-flash-tts'` (default), `'gemini-3.8-flash-lite-tts'` | | `voice` | `String` | Voice name. Defaults to `'Kore'`. Available: `'Zephyr'`, `'Puck'`, `'Charon'`, `'Kore'`, `'Fenrir'`, `'Leda'`, `'Orus'`, `'Aoede'`, `'Callirrhoe'`, `'Autonoe'`, `'Enceladus'`, `'Iapetus'`, `'Umbriel'`, `'Algieba'`, `'Despina'`, `'Erinome'`, `'Algenib'`, `'Rasalgethi'`, `'Laomedeia'`, `'Achernar'`, `'Alnilam'`, `'Schedar'`, `'Gacrux'`, `'Pulcherrima'`, `'Achird'`, `'Zubenelgenubi'`, `'Vindemiatrix'`, `'Sadachbia'`, `'Sadaltager'`, `'Sulafat'` | -| `instructions` | `String` | Natural language instructions to control speaking style (tone, speed, mood, etc.) | +| `instructions` | `String` | Natural language instructions to control speaking style (tone, speed, mood, etc.), up to 3,000 characters | -For more details about Gemini TTS, see the [Google Gemini TTS documentation](https://ai.google.dev/gemini-api/docs/text-to-speech). +Gemini generates speech from the `text` field. The `instructions` string isn't included in the generated audio. It guides delivery (tone, pacing, accent, emotion). Instructions count toward input tokens. Longer instructions are rejected with an `input_too_large` error. Audio is returned as WAV. + +A Gemini request is charged for the tokens Gemini reports, even when it fails or returns no audio. If Gemini returns audio without reporting its token usage, the request fails with an `upstream_failed` error. + +For more details about Gemini TTS, see the [Google Gemini speech generation documentation](https://ai.google.dev/gemini-api/docs/speech-generation). #### xAI (Grok) Options diff --git a/src/docs/src/rate-limits-and-quotas.md b/src/docs/src/rate-limits-and-quotas.md index 46055e720..1e888d21f 100644 --- a/src/docs/src/rate-limits-and-quotas.md +++ b/src/docs/src/rate-limits-and-quotas.md @@ -65,6 +65,7 @@ Shared by chat, image generation, video, TTS, speech and OCR. Each interface and | Input | Limit | | ---------------------------------------------- | ---------------- | | Text for `txt2speech()` | 3,000 characters | +| Gemini `instructions` for `txt2speech()` | 3,000 characters | | Audio for `speech2speech()` | 25 MiB | | Image inlined in a chat message | 5 MB | | File attached to a chat message, Claude models | 30 MB | @@ -76,7 +77,11 @@ The standard chat timeout is 10 minutes for a response to start, or to finish if A `gemini-4-argon` response that isn't streamed is capped at 32,768 output tokens, and a larger `max_tokens` is lowered to that cap. Streaming uses the model-directory limit. -Text-to-speech calls to ElevenLabs, xAI and Speechify time out after 2 minutes; xAI speech-to-text and voice-changer calls after 10 minutes, upload included. A call that times out before any audio or transcript comes back fails with `504 upstream_timeout`. +Text-to-speech calls to ElevenLabs, xAI, Speechify and Gemini time out after 2 minutes; xAI speech-to-text and voice-changer calls after 10 minutes, upload included. A call that times out before any audio or transcript comes back fails with `504 upstream_timeout`. For Gemini, the 2 minutes cover the whole reply, audio included. + +A Gemini text-to-speech reply is capped at 64 MiB, enough for the full audio budget. A readable Gemini text-to-speech reply is charged for the token usage it reports, even when it fails or returns no audio. A reply that times out or exceeds the cap reports no usage, so it is charged the full reservation and fails with `504 upstream_timeout` or `400 input_too_large`; split long text into shorter requests. Requests with an unreadable reply are not charged. + +Every Gemini text-to-speech request reserves credit for the models' full published budget of 8,192 input tokens and 16,384 audio tokens, at the model's current rate. Each request also limits Gemini's output to 16,384 tokens. An incomplete reply fails with `502 upstream_failed`. If the reply is readable within the size and time limits, it is charged for the usage it reports. The reservation is the same for any `text` and `instructions`, in any language. A request fails with `402 insufficient_funds` unless your available credit covers the whole reservation, even when the text is short. When the request ends, the unused part is released and you're charged only for the usage Gemini reports. The reservation isn't a cap on the charge. If Gemini reports more tokens than its published limits, the charge follows the report. The OpenAI- and Anthropic-compatible endpoints (`/puterai/openai/v1/*`, `/puterai/anthropic/v1/messages`) require a paid plan; a free account gets `402 subscription_required`. The same models are available to every account through `puter.ai.*` and `/drivers/call`, and the model catalogue endpoints are open to everyone. diff --git a/src/puter-js/src/modules/ai/types.js b/src/puter-js/src/modules/ai/types.js index ebe312a2f..7c89abbeb 100644 --- a/src/puter-js/src/modules/ai/types.js +++ b/src/puter-js/src/modules/ai/types.js @@ -379,7 +379,7 @@ * Options for `txt2speech()`. * * @typedef {Object} Txt2SpeechOptions - * @property {string} [text] Text to synthesize. Must be less than 3000 characters. + * @property {string} [text] Text to synthesize. At most 3000 characters. * @property {string} [language] Language code. For AWS Polly defaults to `'en-US'`; for xAI a BCP-47 * code defaulting to `'en'` (supports `'auto'`). * @property {string} [voice] Voice ID used for synthesis (provider-specific). Defaults to `'Joanna'` @@ -396,7 +396,7 @@ * @property {string} [output_format] Output format for ElevenLabs (defaults to `'mp3_44100_128'`) and * xAI (`'mp3'` default, `'wav'`, `'pcm'`, `'mulaw'`, `'alaw'`). * @property {string} [instructions] Natural-language guidance for voice style such as tone, speed, and - * mood (OpenAI and Gemini). + * mood (OpenAI and Gemini). At most 3000 characters for Gemini. * @property {Record} [voice_settings] ElevenLabs voice tuning options (e.g. stability, * similarity boost, speed). * @property {boolean} [ssml] When `true`, AWS Polly treats `text` as SSML markup.