From 33c84b4f1bd080400e17cabc82ec411de6801fcf Mon Sep 17 00:00:00 2001 From: 404oops Date: Fri, 16 Oct 2026 03:26:46 +0200 Subject: [PATCH] Validate speech requests and settle reported Gemini usage --- .../providers/gemini/GeminiChatProvider.ts | 9 +- src/backend/drivers/ai-tts/TTSDriver.test.ts | 226 ++++++++++-- src/backend/drivers/ai-tts/TTSDriver.ts | 17 +- .../drivers/ai-tts/providers/common.test.ts | 28 +- .../drivers/ai-tts/providers/common.ts | 11 + .../gemini/GeminiTTSProvider.test.ts | 322 ++++++++++++++---- .../providers/gemini/GeminiTTSProvider.ts | 138 +++++--- src/backend/drivers/util/aiLimits.ts | 10 + src/backend/drivers/util/geminiRest.ts | 31 ++ src/docs/src/AI/txt2speech.md | 8 +- src/docs/src/rate-limits-and-quotas.md | 3 +- src/puter-js/src/modules/ai/types.js | 4 +- 12 files changed, 657 insertions(+), 150 deletions(-) create mode 100644 src/backend/drivers/util/geminiRest.ts diff --git a/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts index 9c5138ef2..965172755 100644 --- a/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts +++ b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts @@ -26,6 +26,7 @@ import { import { OpenAI } from 'openai'; import { HttpError } from '../../../../core/http/HttpError.js'; import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { usesGoogleGeminiEndpoint } from '../../../util/geminiRest.js'; import type { IChatModel, ICompleteArguments, @@ -76,11 +77,9 @@ export class GeminiChatProvider extends OpenAICompatProvider { // A custom base URL may be a proxy that only speaks the // OpenAI-compatible API, so native models are left out rather than // sent straight to Google. - const native = - config.apiBaseUrl === undefined || - config.apiBaseUrl.replace(/\/*$/, '/') === COMPAT_BASE_URL - ? geminiNativeClient(new GoogleGenAI({ apiKey: config.apiKey })) - : undefined; + const native = usesGoogleGeminiEndpoint(config.apiBaseUrl) + ? geminiNativeClient(new GoogleGenAI({ apiKey: config.apiKey })) + : undefined; const client: ChatCompletionsClient = { chat: { completions: { diff --git a/src/backend/drivers/ai-tts/TTSDriver.test.ts b/src/backend/drivers/ai-tts/TTSDriver.test.ts index 824e5e15b..39a886236 100644 --- a/src/backend/drivers/ai-tts/TTSDriver.test.ts +++ b/src/backend/drivers/ai-tts/TTSDriver.test.ts @@ -20,13 +20,12 @@ /** * Offline unit tests for TTSDriver. * - * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock - * redis) with API credentials for every TTS provider so the driver - * registers and indexes them all. Then drives `server.drivers.aiTts` - * directly. Provider SDKs and global `fetch` are mocked at their - * network boundaries so the driver's routing and dispatch logic runs - * without real egress. Aligns with AGENTS.md: "Prefer test server - * over mocking deps." + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock redis) with + * API credentials for every TTS provider so the driver registers and indexes + * them all. Then drives `server.drivers.aiTts` directly. Provider SDKs and + * global `fetch` are mocked at their network boundaries so the driver's routing + * and dispatch logic runs without real egress. Aligns with AGENTS.md: "Prefer + * test server over mocking deps." */ import { @@ -95,10 +94,9 @@ const { pollySendMock } = vi.hoisted(() => ({ })); vi.mock('@aws-sdk/client-polly', async () => { - const actual = - await vi.importActual( - '@aws-sdk/client-polly', - ); + const actual = await vi.importActual< + typeof import('@aws-sdk/client-polly') + >('@aws-sdk/client-polly'); return { ...actual, PollyClient: vi.fn().mockImplementation(function ( @@ -184,12 +182,14 @@ const pollyDescribeVoices = { }; const pollyDispatch = () => - pollySendMock.mockImplementation((cmd: { constructor: { name: string } }) => { - if (cmd.constructor.name === 'DescribeVoicesCommand') { - return Promise.resolve(pollyDescribeVoices); - } - return Promise.resolve({ AudioStream: 'audio-bytes' }); - }); + pollySendMock.mockImplementation( + (cmd: { constructor: { name: string } }) => { + if (cmd.constructor.name === 'DescribeVoicesCommand') { + return Promise.resolve(pollyDescribeVoices); + } + return Promise.resolve({ AudioStream: 'audio-bytes' }); + }, + ); // ── Provider registration & list ──────────────────────────────────── @@ -261,7 +261,10 @@ describe('TTSDriver.synthesize provider routing', () => { audio_data: Buffer.from('audio').toString('base64'), audio_format: 'mp3', }), - { status: 200, headers: { 'content-type': 'application/json' } }, + { + status: 200, + headers: { 'content-type': 'application/json' }, + }, ), ); @@ -348,7 +351,9 @@ describe('TTSDriver.synthesize provider routing', () => { it('treats an engine that names a provider as the provider', async () => { openaiSpeechCreateMock.mockResolvedValueOnce(openaiAudioResponse()); - await withActor(() => driver.synthesize({ text: 'hi', engine: 'openai' })); + await withActor(() => + driver.synthesize({ text: 'hi', engine: 'openai' }), + ); expect(openaiSpeechCreateMock).toHaveBeenCalledTimes(1); // The alias selected the provider; it is not carried on as a model. @@ -361,7 +366,11 @@ describe('TTSDriver.synthesize provider routing', () => { openaiSpeechCreateMock.mockResolvedValueOnce(openaiAudioResponse()); await withActor(() => - driver.synthesize({ text: 'hi', provider: 'openai', engine: 'tts-1' }), + driver.synthesize({ + text: 'hi', + provider: 'openai', + engine: 'tts-1', + }), ); expect(openaiSpeechCreateMock.mock.calls[0][0].model).toBe('tts-1'); @@ -379,6 +388,173 @@ describe('TTSDriver.synthesize provider routing', () => { }); }); +// -- Text length cap -- + +describe('TTSDriver.synthesize text length cap', () => { + it('passes text of exactly 3000 characters to the provider', async () => { + openaiSpeechCreateMock.mockResolvedValueOnce(openaiAudioResponse()); + const text = 'x'.repeat(3000); + + await withActor(() => driver.synthesize({ text, provider: 'openai' })); + + expect(openaiSpeechCreateMock.mock.calls[0][0].input).toBe(text); + }); + + it.each([undefined, 'openai', 'gemini', 'xai', 'elevenlabs'])( + 'rejects 3001 characters with 400 input_too_large before provider %s is called', + async (provider) => { + await expect( + withActor(() => + driver.synthesize({ text: 'x'.repeat(3001), provider }), + ), + ).rejects.toMatchObject({ + statusCode: 400, + legacyCode: 'input_too_large', + fields: { key: 'text', max: 3000 }, + }); + expect(openaiSpeechCreateMock).not.toHaveBeenCalled(); + expect(pollySendMock).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); + }, + ); + + it('rejects over-long text in test mode too, as puter.js does', async () => { + await expect( + withActor(() => + driver.synthesize({ + text: 'x'.repeat(3001), + provider: 'openai', + test_mode: true, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + }); +}); + +// -- Gemini behind a custom apiBaseUrl -- + +describe('TTSDriver Gemini registration with a custom apiBaseUrl', () => { + const PROXY = 'https://gemini-proxy.test/v1beta/openai/'; + const GOOGLE = 'https://generativelanguage.googleapis.com/'; + + const wav = Buffer.alloc(64, 7); + wav.write('RIFF', 0, 'latin1'); + wav.write('WAVE', 8, 'latin1'); + const geminiReply = { + status: 'completed', + steps: [ + { + type: 'model_output', + content: [ + { + type: 'audio', + mime_type: 'audio/wav', + data: wav.toString('base64'), + }, + ], + }, + ], + usage: { total_input_tokens: 1, total_output_tokens: 25 }, + }; + + /** Boots a server with `providers`, answering Gemini calls with audio. */ + const withServer = async ( + providers: Record, + fn: (tts: TTSDriver) => Promise, + ) => { + const booted = await setupTestServer({ providers } as never); + try { + const passthrough = fetchSpy.getMockImplementation()!; + fetchSpy.mockImplementation((input, init) => + String(input).startsWith(GOOGLE) + ? Promise.resolve(Response.json(geminiReply)) + : passthrough(input, init), + ); + await fn(booted.drivers.aiTts as unknown as TTSDriver); + } finally { + await booted.shutdown(); + } + }; + + const googleCalls = () => + fetchSpy.mock.calls.filter(([url]) => String(url).startsWith(GOOGLE)); + + it('leaves Gemini out and answers an explicit request with a config error, without calling Google', async () => { + await withServer( + { gemini: { apiKey: 'proxy-key', apiBaseUrl: PROXY } }, + async (tts) => { + expect(await tts.list()).not.toContain('gemini'); + + const notConfigured = { + statusCode: 400, + legacyCode: 'bad_request', + message: expect.stringContaining( + 'TTS provider not configured: gemini', + ), + }; + await expect( + withActor(() => + tts.synthesize({ text: 'hi', provider: 'gemini' }), + ), + ).rejects.toMatchObject(notConfigured); + await expect( + withActor(() => + tts.synthesize({ text: 'hi', engine: 'google' }), + ), + ).rejects.toMatchObject(notConfigured); + await expect( + withDriverName('gemini-tts', () => + tts.synthesize({ text: 'hi' }), + ), + ).rejects.toMatchObject(notConfigured); + expect(googleCalls()).toHaveLength(0); + }, + ); + }); + + it('sends Google a gemini-tts key, never the proxied gemini key', async () => { + await withServer( + { + gemini: { apiKey: 'proxy-key', apiBaseUrl: PROXY }, + 'gemini-tts': { apiKey: 'tts-key' }, + }, + async (tts) => { + await withActor(() => + tts.synthesize({ text: 'hi', provider: 'gemini' }), + ); + const calls = googleCalls(); + expect(calls).toHaveLength(1); + const headers = calls[0]![1]!.headers as Record; + expect(headers['x-goog-api-key']).toBe('tts-key'); + }, + ); + }); + + it.each([ + 'https://generativelanguage.googleapis.com/v1beta/openai/', + 'https://generativelanguage.googleapis.com/v1beta/openai', + ])( + 'keeps Gemini on the shared key when apiBaseUrl is Google’s own %s', + async (apiBaseUrl) => { + await withServer( + { gemini: { apiKey: 'gem-key', apiBaseUrl } }, + async (tts) => { + await withActor(() => + tts.synthesize({ text: 'hi', provider: 'gemini' }), + ); + const calls = googleCalls(); + expect(calls).toHaveLength(1); + const headers = calls[0]![1]!.headers as Record< + string, + string + >; + expect(headers['x-goog-api-key']).toBe('gem-key'); + }, + ); + }, + ); +}); + // ── list_voices / list_engines aggregation ───────────────────────── describe('TTSDriver list_voices / list_engines', () => { @@ -410,9 +586,9 @@ describe('TTSDriver list_voices / list_engines', () => { }); it('list_voices({ provider }) throws 400 when the provider is unknown', async () => { - await expect(driver.list_voices({ provider: 'nope' })).rejects.toMatchObject( - { statusCode: 400 }, - ); + await expect( + driver.list_voices({ provider: 'nope' }), + ).rejects.toMatchObject({ statusCode: 400 }); }); it('list_voices() defaults to aws-polly', async () => { @@ -490,7 +666,9 @@ describe('TTSDriver.synthesize error mapping', () => { server.services.metering, 'incrementUsage', ); - openaiSpeechCreateMock.mockRejectedValueOnce(new Error('upstream blew up')); + openaiSpeechCreateMock.mockRejectedValueOnce( + new Error('upstream blew up'), + ); await expect( withActor(() => diff --git a/src/backend/drivers/ai-tts/TTSDriver.ts b/src/backend/drivers/ai-tts/TTSDriver.ts index 281c8f4de..64b719df7 100644 --- a/src/backend/drivers/ai-tts/TTSDriver.ts +++ b/src/backend/drivers/ai-tts/TTSDriver.ts @@ -25,10 +25,16 @@ import { type AiMeteringService, withAiCostFactor, } from '../util/aiCostFactor.js'; -import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js'; +import { + AI_CONCURRENT, + AI_RATE_LIMIT, + AI_TTS_MAX_TEXT_CHARS, +} from '../util/aiLimits.js'; +import { usesGoogleGeminiEndpoint } from '../util/geminiRest.js'; import { ProviderRegistry, readProviderKey } from '../util/providerRegistry.js'; import { TTS_CATALOG, TTS_DRIVER_ALIASES } from './providerAliases.js'; import { AWSPollyTTSProvider } from './providers/awsPolly/AWSPollyTTSProvider.js'; +import { assertMaxChars } from './providers/common.js'; import { ElevenLabsTTSProvider } from './providers/elevenlabs/ElevenLabsTTSProvider.js'; import { GeminiTTSProvider } from './providers/gemini/GeminiTTSProvider.js'; import { OpenAITTSProvider } from './providers/openai/OpenAITTSProvider.js'; @@ -135,6 +141,8 @@ export class TTSDriver extends PuterDriver { legacyCode: 'unauthorized', }); + assertMaxChars('text', args.text, AI_TTS_MAX_TEXT_CHARS); + const providerName = this.#resolveProvider(args); const provider = this.#providers.get(providerName); if (!provider) { @@ -254,11 +262,14 @@ export class TTSDriver extends PuterDriver { ); } + // Gemini TTS only calls Google, so it skips a block whose key belongs + // to a custom `apiBaseUrl`. register('gemini', () => keyed( (apiKey) => new GeminiTTSProvider(m, { apiKey }), - providers['gemini'], - providers['gemini-tts'], + ...[providers['gemini'], providers['gemini-tts']].filter( + (cfg) => usesGoogleGeminiEndpoint(cfg?.apiBaseUrl), + ), ), ); register('xai', () => diff --git a/src/backend/drivers/ai-tts/providers/common.test.ts b/src/backend/drivers/ai-tts/providers/common.test.ts index 5e63b4fb8..f40914eba 100644 --- a/src/backend/drivers/ai-tts/providers/common.test.ts +++ b/src/backend/drivers/ai-tts/providers/common.test.ts @@ -32,7 +32,11 @@ import type { DriverStreamResult } from '../../meta.js'; import { PuterServer } from '../../../server.js'; import { setupTestServer } from '../../../testUtil.js'; import { withAiCostFactor } from '../../util/aiCostFactor.js'; -import { characterCostReport, meterPerCharacter } from './common.js'; +import { + assertMaxChars, + characterCostReport, + meterPerCharacter, +} from './common.js'; let server: PuterServer; beforeAll(async () => { @@ -227,6 +231,28 @@ describe('meterPerCharacter', () => { }); }); +describe('assertMaxChars', () => { + it('accepts a string at the cap and values that are not strings', () => { + expect(() => assertMaxChars('text', 'x'.repeat(5), 5)).not.toThrow(); + expect(() => assertMaxChars('text', undefined, 5)).not.toThrow(); + expect(() => assertMaxChars('text', ['a', 'b'], 1)).not.toThrow(); + }); + + it('rejects a string over the cap with a 400 input_too_large naming the key', () => { + let err: unknown; + try { + assertMaxChars('instructions', 'x'.repeat(6), 5); + } catch (e) { + err = e; + } + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'input_too_large', + fields: { key: 'instructions', max: 5 }, + }); + }); +}); + describe('characterCostReport', () => { it('emits one per-character line per cost-table entry', () => { expect( diff --git a/src/backend/drivers/ai-tts/providers/common.ts b/src/backend/drivers/ai-tts/providers/common.ts index 66ecd134b..e18b29d3a 100644 --- a/src/backend/drivers/ai-tts/providers/common.ts +++ b/src/backend/drivers/ai-tts/providers/common.ts @@ -18,6 +18,7 @@ */ import type { Actor } from '../../../core/actor.js'; +import { HttpError } from '../../../core/http/HttpError.js'; import { type AiMeteringService, withStreamedAiCharge, @@ -30,6 +31,16 @@ import type { DriverStreamResult } from '../../meta.js'; */ export const TTS_UPSTREAM_TIMEOUT_MS = 120_000; +/** Rejects a string argument longer than its published cap with a 400. */ +export function assertMaxChars(key: string, value: unknown, max: number): void { + if (typeof value === 'string' && value.length > max) { + throw new HttpError(400, `${key} must be at most ${max} characters`, { + legacyCode: 'input_too_large', + fields: { key, max }, + }); + } +} + /** * Runs `synthesize` under a hold on `text.length × ucentsPerChar` and meters * exactly that once its audio starts arriving. Per-character prices are known diff --git a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.test.ts b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.test.ts index 39279d06d..1a6910569 100644 --- a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.test.ts +++ b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.test.ts @@ -20,11 +20,11 @@ /** * Offline unit tests for GeminiTTSProvider. * - * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock - * redis) and constructs GeminiTTSProvider directly against the live - * wired `MeteringService` so holds and recording run end-to-end. Global - * `fetch` is stubbed with canned Interactions API responses, the real - * network egress point. + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock redis) and + * constructs GeminiTTSProvider directly against the live wired + * `MeteringService` so holds and recording run end-to-end. Global `fetch` is + * stubbed with canned Interactions API responses, the real network egress + * point. */ import { Readable } from 'node:stream'; @@ -104,6 +104,20 @@ const USAGE = { total_tokens: 75, }; +/** What `USAGE` settles to on the default model at launch rates. */ +const SETTLED_USAGE = [ + { + usageType: 'gemini:gemini-3.8-flash-tts:input', + usageAmount: 7, + costOverride: 7 * 50, + }, + { + usageType: 'gemini:gemini-3.8-flash-tts:output:audio', + usageAmount: 68, + costOverride: 68 * 900, + }, +]; + /** A completed reply; `usage` given as undefined leaves the field out. */ const interaction = ( opts: { @@ -183,9 +197,9 @@ const trackHolds = () => { beforeEach(() => { fetchSpy = vi .spyOn(globalThis, 'fetch') - .mockRejectedValue( - new Error('unexpected fetch'), - ) as MockInstance; + .mockRejectedValue(new Error('unexpected fetch')) as MockInstance< + typeof fetch + >; hasCreditsSpy = vi.spyOn(server.services.metering, 'hasEnoughCredits'); batchIncrementUsagesSpy = vi.spyOn( server.services.metering, @@ -360,6 +374,34 @@ describe('GeminiTTSProvider.synthesize argument validation', () => { }); expect(fetchSpy).not.toHaveBeenCalled(); }); + + it('accepts instructions of exactly 3000 characters', async () => { + const instructions = 'x'.repeat(3000); + replyWith(interaction()); + await withTestActor(() => + makeProvider().synthesize({ text: 'hi', instructions }), + ); + expect( + sentRequest().body.input[0].content[0].annotations[0].style, + ).toBe(instructions); + }); + + it('rejects instructions over 3000 characters with 400 input_too_large before holding credit', async () => { + await expect( + withTestActor(() => + makeProvider().synthesize({ + text: 'hi', + instructions: 'x'.repeat(3001), + }), + ), + ).rejects.toMatchObject({ + statusCode: 400, + legacyCode: 'input_too_large', + fields: { key: 'instructions', max: 3000 }, + }); + expect(hasCreditsSpy).not.toHaveBeenCalled(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); }); // -- Credit gate -- @@ -460,9 +502,9 @@ describe('GeminiTTSProvider.synthesize request shape', () => { await withTestActor(() => makeProvider().synthesize({ text: 'hi', voice: requested }), ); - expect(sentRequest().body.generation_config.speech_config).toEqual( - [{ voice: canonical }], - ); + expect(sentRequest().body.generation_config.speech_config).toEqual([ + { voice: canonical }, + ]); }, ); @@ -531,14 +573,23 @@ describe('GeminiTTSProvider.synthesize audio output', () => { replyWith( interaction({ steps: [ - { type: 'thought', content: [audioBlock(Buffer.from('x'))] }, { - type: 'model_output', - content: [{ type: 'text', text: 'note' }, audioBlock(first)], + type: 'thought', + content: [audioBlock(Buffer.from('x'))], }, { type: 'model_output', - content: [audioBlock(last), { type: 'text', text: 'end' }], + content: [ + { type: 'text', text: 'note' }, + audioBlock(first), + ], + }, + { + type: 'model_output', + content: [ + audioBlock(last), + { type: 'text', text: 'end' }, + ], }, ], }), @@ -554,40 +605,137 @@ describe('GeminiTTSProvider.synthesize audio output', () => { // -- Upstream failures -- describe('GeminiTTSProvider.synthesize upstream failures', () => { - it.each([ - ['failed'], - ['cancelled'], - ['incomplete'], - ['in_progress'], - ])( - 'fails a %s interaction as 502 upstream_failed without charging', + it.each([['failed'], ['cancelled'], ['incomplete'], ['in_progress']])( + 'charges the reported usage once, then fails a %s interaction as 502 upstream_failed', async (status) => { + const { released } = trackHolds(); replyWith(interaction({ status })); - await expect( - withTestActor(() => makeProvider().synthesize({ text: 'hi' })), - ).rejects.toMatchObject({ + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode: 502, legacyCode: 'upstream_failed', }); - expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual(SETTLED_USAGE); + expect(released).toHaveBeenCalledTimes(1); }, ); - it('fails a completed reply with only text (a refusal) as 400 upstream_bad_request', async () => { + it('charges the reported usage once, then fails a refusal (text only) as 400 upstream_bad_request', async () => { + const { released } = trackHolds(); replyWith( interaction({ content: [{ type: 'text', text: 'I cannot read that.' }], }), ); - await expect( - withTestActor(() => makeProvider().synthesize({ text: 'hi' })), - ).rejects.toMatchObject({ + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode: 400, legacyCode: 'upstream_bad_request', }); - expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual(SETTLED_USAGE); + expect(released).toHaveBeenCalledTimes(1); }); + const FAILED_REPLIES: [ + string, + { status?: string; content?: unknown[] }, + number, + ][] = [ + ['a failed interaction', { status: 'failed' }, 502], + [ + 'a refusal', + { content: [{ type: 'text', text: 'I cannot read that.' }] }, + 400, + ], + [ + 'malformed audio', + { content: [{ type: 'audio', mime_type: 'audio/wav', data: '' }] }, + 502, + ], + ]; + + it.each(FAILED_REPLIES)( + 'charges nothing for %s without usage, and releases the hold', + async (_label, opts, statusCode) => { + const { released } = trackHolds(); + replyWith(interaction({ ...opts, usage: undefined })); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode }); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it.each(FAILED_REPLIES)( + 'charges only the input for %s whose output total has no audio breakdown', + async (_label, opts, statusCode) => { + const { released } = trackHolds(); + replyWith( + interaction({ + ...opts, + usage: { total_input_tokens: 7, total_output_tokens: 68 }, + }), + ); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode }); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual([SETTLED_USAGE[0]]); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it.each(FAILED_REPLIES)( + 'charges only the input for %s when only the input count is usable', + async (_label, opts, statusCode) => { + const { released } = trackHolds(); + replyWith( + interaction({ + ...opts, + usage: { + input_tokens_by_modality: [ + { modality: 'text', tokens: 7 }, + ], + output_tokens_by_modality: [ + { modality: 'audio', tokens: -1 }, + ], + }, + }), + ); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ statusCode }); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual([SETTLED_USAGE[0]]); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + it.each([ ['no data', { type: 'audio', mime_type: 'audio/wav' }], [ @@ -603,10 +751,7 @@ describe('GeminiTTSProvider.synthesize upstream failures', () => { 'invalid base64', { type: 'audio', mime_type: 'audio/wav', data: '***not base64***' }, ], - [ - 'no MIME type', - { type: 'audio', data: WAV_BYTES.toString('base64') }, - ], + ['no MIME type', { type: 'audio', data: WAV_BYTES.toString('base64') }], ['an unsupported MIME type', audioBlock(WAV_BYTES, 'audio/mpeg')], [ 'WAV bytes without a RIFF header', @@ -614,7 +759,7 @@ describe('GeminiTTSProvider.synthesize upstream failures', () => { ], ['empty PCM', audioBlock(Buffer.alloc(0), 'audio/L16;rate=24000')], ])( - 'fails an audio block with %s as 502 upstream_failed', + 'charges the reported usage once, then fails an audio block with %s as 502 upstream_failed', async (_label, block) => { replyWith(interaction({ content: [block] })); await expect( @@ -625,7 +770,8 @@ describe('GeminiTTSProvider.synthesize upstream failures', () => { }); // Never follows a provider URI. expect(fetchSpy).toHaveBeenCalledTimes(1); - expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual(SETTLED_USAGE); }, ); @@ -843,7 +989,7 @@ describe('GeminiTTSProvider.synthesize metering', () => { expect(metered(':output:audio').usageAmount).toBe(68); }); - it('uses the aggregate totals when there is no modality breakdown', async () => { + it('uses the aggregate totals when audio comes back without a modality breakdown', async () => { replyWith( interaction({ usage: { total_input_tokens: 9, total_output_tokens: 80 }, @@ -854,8 +1000,6 @@ describe('GeminiTTSProvider.synthesize metering', () => { expect(metered(':output:audio').usageAmount).toBe(80); }); - // 'hello there' with 'Speak softly' is 23 characters, so 6 input tokens; - // two words round up to one second of audio, so 25 output tokens. it.each([ ['missing usage', undefined], ['null usage', null], @@ -890,7 +1034,9 @@ describe('GeminiTTSProvider.synthesize metering', () => { 'fractional and string counts', { input_tokens_by_modality: [{ modality: 'text', tokens: 1.5 }], - output_tokens_by_modality: [{ modality: 'audio', tokens: '68' }], + output_tokens_by_modality: [ + { modality: 'audio', tokens: '68' }, + ], }, ], [ @@ -906,22 +1052,84 @@ describe('GeminiTTSProvider.synthesize metering', () => { output_tokens_by_modality: [{ modality: 'text', tokens: 68 }], }, ], - ])('falls back to the estimate on %s', async (_label, usage) => { - replyWith(interaction({ usage })); - await withTestActor(() => - makeProvider().synthesize({ - text: 'hello there', - instructions: 'Speak softly', + ['reported zeros', { total_input_tokens: 0, total_output_tokens: 0 }], + ])( + 'charges nothing on %s and fails the audio as 502 upstream_failed', + async (_label, usage) => { + const { released } = trackHolds(); + replyWith(interaction({ usage })); + + const err = await callerError( + () => + makeProvider().synthesize({ + text: 'hello there', + instructions: 'Speak softly', + }), + payingUser(), + ); + + expect(err).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + }); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it.each([ + [ + 'input', + { + total_input_tokens: 7, + output_tokens_by_modality: [{ modality: 'audio', tokens: -1 }], + }, + [SETTLED_USAGE[0]], + ], + [ + 'audio', + { + total_input_tokens: 'seven', + output_tokens_by_modality: [{ modality: 'audio', tokens: 68 }], + }, + [SETTLED_USAGE[1]], + ], + ])( + 'charges only the readable %s count once, then fails the audio as 502 upstream_failed', + async (_label, usage, settled) => { + const { released } = trackHolds(); + replyWith(interaction({ usage })); + + const err = await callerError( + () => makeProvider().synthesize({ text: 'hi' }), + payingUser(), + ); + + expect(err).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + }); + expect(batchIncrementUsagesSpy).toHaveBeenCalledTimes(1); + expect(meteredUsages()).toEqual(settled); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it('charges a reported zero as nothing, with no one-token minimum', async () => { + replyWith( + interaction({ + status: 'failed', + usage: { + total_input_tokens: 7, + output_tokens_by_modality: [ + { modality: 'audio', tokens: 0 }, + ], + }, }), ); - expect(metered(':input')).toMatchObject({ - usageAmount: 6, - costOverride: 6 * 50, - }); - expect(metered(':output:audio')).toMatchObject({ - usageAmount: 25, - costOverride: 25 * 900, - }); + + await callerError(() => makeProvider().synthesize({ text: 'hi' })); + expect(meteredUsages()).toEqual([SETTLED_USAGE[0]]); }); it('holds an estimate covering the instructions and releases it once metered', async () => { @@ -978,9 +1186,7 @@ describe('GeminiTTSProvider.synthesize scheduled rates', () => { expect(held).toEqual([1 * inputRate + 25 * audioRate]); expect(metered(':input').costOverride).toBe(7 * inputRate); - expect(metered(':output:audio').costOverride).toBe( - 68 * audioRate, - ); + expect(metered(':output:audio').costOverride).toBe(68 * audioRate); }, ); diff --git a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts index f276c2774..b7013c75c 100644 --- a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts +++ b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts @@ -24,7 +24,9 @@ import { type AiMeteringService, withAiCreditHold, } from '../../../util/aiCostFactor.js'; +import type { UsageInput } from '../../../../services/metering/types.js'; import type { DriverStreamResult } from '../../../meta.js'; +import { AI_TTS_MAX_GEMINI_INSTRUCTIONS_CHARS } from '../../../util/aiLimits.js'; import { readBoundedBody } from '../../../util/fileInput.js'; import { upstreamFetch } from '../../../util/upstreamErrors.js'; import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js'; @@ -34,7 +36,7 @@ import type { ISynthesizeArgs, ITTSProvider, } from '../../types.js'; -import { TTS_UPSTREAM_TIMEOUT_MS } from '../common.js'; +import { assertMaxChars, TTS_UPSTREAM_TIMEOUT_MS } from '../common.js'; import { GEMINI_TTS_COSTS, geminiTTSCostsAt } from './costs.js'; const INTERACTIONS_URL = @@ -147,7 +149,10 @@ const upstreamFailure = (message: string): HttpError => const responseTooLarge = (maxBytes: number): HttpError => upstreamFailure(`Gemini TTS response exceeds ${maxBytes} bytes`); -/** The µ¢ cost of `tokens` at a cents-per-1M rate, which is µ¢ per token. */ +/** + * The µ¢ held for an estimated `tokens` at a cents-per-1M rate (µ¢ per token). + * Never below 1; reported usage is charged at its exact cost instead. + */ const tokenCost = (tokens: number, centsPerMillion: number): number => Math.max(1, Math.ceil(tokens * centsPerMillion)); @@ -281,6 +286,11 @@ export class GeminiTTSProvider implements ITTSProvider { fields: { key: 'instructions', expected: 'string' }, }); } + assertMaxChars( + 'instructions', + instructions, + AI_TTS_MAX_GEMINI_INSTRUCTIONS_CHARS, + ); const style = instructions?.trim() ? instructions : undefined; const model = modelArg || DEFAULT_MODEL; @@ -402,64 +412,86 @@ export class GeminiTTSProvider implements ITTSProvider { 'Gemini TTS returned an unreadable response', ); } - if (interaction.status !== 'completed') { - throw upstreamFailure( - 'Gemini TTS did not complete the request', - ); - } - - const audio = lastAudioContent(interaction); - if (!audio) { - throw new HttpError( - 400, - 'Gemini TTS did not return audio data', - { - legacyCode: 'upstream_bad_request', - fields: { provider: 'gemini' }, - }, - ); - } - const { body, contentType } = this.#decodeAudio(audio); + // A count Gemini doesn't report readably is left out, not + // estimated. Output without a modality breakdown may be refusal + // text, so it counts as audio only once audio came back. const usage = interaction.usage; - const inputTokens = - usageTokens( - usage?.input_tokens_by_modality, - usage?.total_input_tokens, - 'text', - costs.input, - ) ?? estimatedInputTokens; - const outputTokens = - usageTokens( + const inputTokens = usageTokens( + usage?.input_tokens_by_modality, + usage?.total_input_tokens, + 'text', + costs.input, + ); + let outputTokens = usageTokens( + usage?.output_tokens_by_modality, + undefined, + 'audio', + costs.output_audio, + ); + try { + if (interaction.status !== 'completed') { + throw upstreamFailure( + 'Gemini TTS did not complete the request', + ); + } + + const audio = lastAudioContent(interaction); + if (!audio) { + throw new HttpError( + 400, + 'Gemini TTS did not return audio data', + { + legacyCode: 'upstream_bad_request', + fields: { provider: 'gemini' }, + }, + ); + } + const { body, contentType } = this.#decodeAudio(audio); + + outputTokens = usageTokens( usage?.output_tokens_by_modality, usage?.total_output_tokens, 'audio', costs.output_audio, - ) ?? estimatedOutputTokens; + ); + if (!inputTokens || !outputTokens) { + throw upstreamFailure( + 'Gemini TTS did not report usage for the audio', + ); + } - const usagePrefix = `gemini:${model}`; - this.meteringService.batchIncrementUsages(actor, [ - { - usageType: `${usagePrefix}:input`, - usageAmount: Math.max(inputTokens, 1), - costOverride: tokenCost(inputTokens, costs.input), - }, - { - usageType: `${usagePrefix}:output:audio`, - usageAmount: Math.max(outputTokens, 1), - costOverride: tokenCost( - outputTokens, - costs.output_audio, - ), - }, - ]); - - return { - dataType: 'stream', - content_type: contentType, - chunked: true, - stream: Readable.from(body), - }; + return { + dataType: 'stream', + content_type: contentType, + chunked: true, + stream: Readable.from(body), + }; + } finally { + // Every outcome pays once for the counts it reports. + const usagePrefix = `gemini:${model}`; + const usages: UsageInput[] = []; + if (inputTokens) { + usages.push({ + usageType: `${usagePrefix}:input`, + usageAmount: inputTokens, + costOverride: inputTokens * costs.input, + }); + } + if (outputTokens) { + usages.push({ + usageType: `${usagePrefix}:output:audio`, + usageAmount: outputTokens, + costOverride: outputTokens * costs.output_audio, + }); + } + if (usages.length > 0) { + await this.meteringService.batchIncrementUsages( + actor, + usages, + ); + } + } }, ); } diff --git a/src/backend/drivers/util/aiLimits.ts b/src/backend/drivers/util/aiLimits.ts index 5b8e77826..f36e576db 100644 --- a/src/backend/drivers/util/aiLimits.ts +++ b/src/backend/drivers/util/aiLimits.ts @@ -61,6 +61,16 @@ export const AI_CONCURRENT: DriverConcurrentConfig = { }, }; +// -- Text-to-speech input caps ---------------------------------------- +// +// Counted in UTF-16 code units, as puter.js counts them before sending. + +/** Longest `text` any TTS provider is sent. */ +export const AI_TTS_MAX_TEXT_CHARS = 3000; + +/** Longest Gemini TTS `instructions`, which are billed as input with `text`. */ +export const AI_TTS_MAX_GEMINI_INSTRUCTIONS_CHARS = 3000; + // -- Claude server-tool clamps ---------------------------------------- // // Applied both when ClaudeProvider builds the request (so Anthropic never diff --git a/src/backend/drivers/util/geminiRest.ts b/src/backend/drivers/util/geminiRest.ts new file mode 100644 index 000000000..9dc0bb98f --- /dev/null +++ b/src/backend/drivers/util/geminiRest.ts @@ -0,0 +1,31 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +const GOOGLE_COMPAT_BASE_URL = + 'https://generativelanguage.googleapis.com/v1beta/openai/'; + +/** + * Whether a `gemini` config block's `apiBaseUrl` is unset or Google's own. Any + * other value may be a proxy holding its own key, which must not be sent to + * Google's native APIs. + */ +export const usesGoogleGeminiEndpoint = (apiBaseUrl: unknown): boolean => + apiBaseUrl === undefined || + (typeof apiBaseUrl === 'string' && + apiBaseUrl.replace(/\/*$/, '/') === GOOGLE_COMPAT_BASE_URL); diff --git a/src/docs/src/AI/txt2speech.md b/src/docs/src/AI/txt2speech.md index 866be78c0..51bba674e 100755 --- a/src/docs/src/AI/txt2speech.md +++ b/src/docs/src/AI/txt2speech.md @@ -20,7 +20,7 @@ puter.ai.txt2speech(text, language, voice, engine, testMode = false) #### `text` (String) (required) -A string containing the text you want to convert to speech. The text must be less than 3000 characters long. Defaults to AWS Polly provider when no options are provided. +A string containing the text you want to convert to speech. The text can be up to 3,000 characters long. Longer text is rejected with an `input_too_large` error. Defaults to AWS Polly provider when no options are provided. #### `testMode` (Boolean) (optional) @@ -84,9 +84,11 @@ Available when `provider: 'gemini'`: |--------|------|-------------| | `model` | `String` | TTS model. Available: `'gemini-3.8-flash-tts'` (default), `'gemini-3.8-flash-lite-tts'` | | `voice` | `String` | Voice name. Defaults to `'Kore'`. Available: `'Zephyr'`, `'Puck'`, `'Charon'`, `'Kore'`, `'Fenrir'`, `'Leda'`, `'Orus'`, `'Aoede'`, `'Callirrhoe'`, `'Autonoe'`, `'Enceladus'`, `'Iapetus'`, `'Umbriel'`, `'Algieba'`, `'Despina'`, `'Erinome'`, `'Algenib'`, `'Rasalgethi'`, `'Laomedeia'`, `'Achernar'`, `'Alnilam'`, `'Schedar'`, `'Gacrux'`, `'Pulcherrima'`, `'Achird'`, `'Zubenelgenubi'`, `'Vindemiatrix'`, `'Sadachbia'`, `'Sadaltager'`, `'Sulafat'` | -| `instructions` | `String` | Natural language instructions to control speaking style (tone, speed, mood, etc.) | +| `instructions` | `String` | Natural language instructions to control speaking style (tone, speed, mood, etc.), up to 3,000 characters | -Gemini generates speech from the `text` field. The `instructions` string isn't included in the generated audio. It guides delivery (tone, pacing, accent, emotion). Instructions count toward input tokens. Audio is returned as WAV. +Gemini generates speech from the `text` field. The `instructions` string isn't included in the generated audio. It guides delivery (tone, pacing, accent, emotion). Instructions count toward input tokens. Longer instructions are rejected with an `input_too_large` error. Audio is returned as WAV. + +A Gemini request is charged for the tokens Gemini reports, even when it fails or returns no audio. If Gemini returns audio without reporting its token usage, the request fails with an `upstream_failed` error. For more details about Gemini TTS, see the [Google Gemini speech generation documentation](https://ai.google.dev/gemini-api/docs/speech-generation). diff --git a/src/docs/src/rate-limits-and-quotas.md b/src/docs/src/rate-limits-and-quotas.md index 5359420a1..0e307123d 100644 --- a/src/docs/src/rate-limits-and-quotas.md +++ b/src/docs/src/rate-limits-and-quotas.md @@ -65,6 +65,7 @@ Shared by chat, image generation, video, TTS, speech and OCR. Each interface and | Input | Limit | | ---------------------------------------------- | ---------------- | | Text for `txt2speech()` | 3,000 characters | +| Gemini `instructions` for `txt2speech()` | 3,000 characters | | Audio for `speech2speech()` | 25 MiB | | Image inlined in a chat message | 5 MB | | File attached to a chat message, Claude models | 30 MB | @@ -78,7 +79,7 @@ A `gemini-4-argon` response that isn't streamed is capped at 32,768 output token Text-to-speech calls to ElevenLabs, xAI, Speechify and Gemini time out after 2 minutes; xAI speech-to-text and voice-changer calls after 10 minutes, upload included. A call that times out before any audio or transcript comes back fails with `504 upstream_timeout`. For Gemini, the 2 minutes cover the whole reply, audio included. -A Gemini text-to-speech reply is capped at 32 MiB, about 8 minutes of audio. A larger reply fails with `502 upstream_failed`. +A Gemini text-to-speech reply is capped at 32 MiB, about 8 minutes of audio. A larger reply fails with `502 upstream_failed`. A Gemini text-to-speech request is charged for the tokens Gemini reports, even when it fails or returns no audio. The OpenAI- and Anthropic-compatible endpoints (`/puterai/openai/v1/*`, `/puterai/anthropic/v1/messages`) require a paid plan; a free account gets `402 subscription_required`. The same models are available to every account through `puter.ai.*` and `/drivers/call`, and the model catalogue endpoints are open to everyone. diff --git a/src/puter-js/src/modules/ai/types.js b/src/puter-js/src/modules/ai/types.js index ebe312a2f..7c89abbeb 100644 --- a/src/puter-js/src/modules/ai/types.js +++ b/src/puter-js/src/modules/ai/types.js @@ -379,7 +379,7 @@ * Options for `txt2speech()`. * * @typedef {Object} Txt2SpeechOptions - * @property {string} [text] Text to synthesize. Must be less than 3000 characters. + * @property {string} [text] Text to synthesize. At most 3000 characters. * @property {string} [language] Language code. For AWS Polly defaults to `'en-US'`; for xAI a BCP-47 * code defaulting to `'en'` (supports `'auto'`). * @property {string} [voice] Voice ID used for synthesis (provider-specific). Defaults to `'Joanna'` @@ -396,7 +396,7 @@ * @property {string} [output_format] Output format for ElevenLabs (defaults to `'mp3_44100_128'`) and * xAI (`'mp3'` default, `'wav'`, `'pcm'`, `'mulaw'`, `'alaw'`). * @property {string} [instructions] Natural-language guidance for voice style such as tone, speed, and - * mood (OpenAI and Gemini). + * mood (OpenAI and Gemini). At most 3000 characters for Gemini. * @property {Record} [voice_settings] ElevenLabs voice tuning options (e.g. stability, * similarity boost, speed). * @property {boolean} [ssml] When `true`, AWS Polly treats `text` as SSML markup.