From e950a87f2977607b0c8d0a60a930e0805836222b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20Kujund=C5=BEi=C4=87?= <51266541+404oops@users.noreply.github.com> Date: Sat, 17 Oct 2026 08:21:48 +0200 Subject: [PATCH] Add Gemini Omni video generation with token metering (#4235) * Add Gemini Omni video generation with token metering * Keep explicit Gemini video model selection within provider * Settle reported Gemini video usage before validating output --------- Co-authored-by: 404oops --- .../gemini/GeminiSpeechToTextProvider.ts | 7 +- .../providers/gemini/GeminiTTSProvider.ts | 7 +- .../ai-video/VideoGenerationDriver.test.ts | 7 +- .../drivers/ai-video/VideoGenerationDriver.ts | 29 +- .../GeminiVideoProvider.integration.test.ts | 152 ++ .../gemini/GeminiVideoProvider.test.ts | 1597 +++++++++++++++++ .../providers/gemini/GeminiVideoProvider.ts | 621 +++++++ .../ai-video/providers/gemini/models.ts | 58 + src/backend/drivers/util/geminiRest.ts | 5 + src/docs/src/AI/txt2vid.md | 57 +- src/docs/src/examples.js | 6 + .../examples/ai-txt2vid-gemini.html | 11 + src/docs/src/rate-limits-and-quotas.md | 20 +- src/puter-js/src/modules/ai/types.js | 28 +- src/puter-js/tests/api/suites/ai.suite.ts | 121 ++ 15 files changed, 2686 insertions(+), 40 deletions(-) create mode 100644 src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.integration.test.ts create mode 100644 src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.test.ts create mode 100644 src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.ts create mode 100644 src/backend/drivers/ai-video/providers/gemini/models.ts create mode 100644 src/docs/src/playground/examples/ai-txt2vid-gemini.html diff --git a/src/backend/drivers/ai-speech2txt/providers/gemini/GeminiSpeechToTextProvider.ts b/src/backend/drivers/ai-speech2txt/providers/gemini/GeminiSpeechToTextProvider.ts index cfa17f7a9..8b3163b8a 100644 --- a/src/backend/drivers/ai-speech2txt/providers/gemini/GeminiSpeechToTextProvider.ts +++ b/src/backend/drivers/ai-speech2txt/providers/gemini/GeminiSpeechToTextProvider.ts @@ -25,6 +25,7 @@ import { loadFileInput } from '../../../util/fileInput.js'; import { geminiUpstreamFailure, postGeminiJson, + geminiTokenCount as tokenCount, usesGoogleGeminiEndpoint, } from '../../../util/geminiRest.js'; import type { @@ -213,12 +214,6 @@ const notConfigured = (): HttpError => noAlarm: true, }); -/** A usable token count: a safe, non-negative integer. */ -const tokenCount = (value: unknown): number | undefined => - typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 - ? value - : undefined; - /** Seconds from a Duration string such as `0.300s`. */ const offsetSeconds = (value: unknown): number | undefined => { if (typeof value !== 'string') return undefined; diff --git a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts index c99d7b41a..1d7295847 100644 --- a/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts +++ b/src/backend/drivers/ai-tts/providers/gemini/GeminiTTSProvider.ts @@ -35,6 +35,7 @@ import { import { geminiUpstreamFailure as upstreamFailure, postGeminiJson, + geminiTokenCount as tokenCount, } from '../../../util/geminiRest.js'; import { isUpstreamTimeoutError } from '../../../util/upstreamErrors.js'; import { SAMPLE_AUDIO_URL } from '../../../util/testMode.js'; @@ -175,12 +176,6 @@ const audioTimedOut = (cause: unknown): HttpError => }, ); -/** A usable token count: a finite, non-negative integer. */ -const tokenCount = (value: unknown): number | undefined => - typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 - ? value - : undefined; - /** * `modality`'s tokens from a by-modality breakdown, or `total` when there is no * breakdown to read. Undefined when the counts are unusable or ambiguous. diff --git a/src/backend/drivers/ai-video/VideoGenerationDriver.test.ts b/src/backend/drivers/ai-video/VideoGenerationDriver.test.ts index b4ef8e0fa..ede8ecb6e 100644 --- a/src/backend/drivers/ai-video/VideoGenerationDriver.test.ts +++ b/src/backend/drivers/ai-video/VideoGenerationDriver.test.ts @@ -96,8 +96,8 @@ beforeAll(async () => { server = await setupTestServer({ providers: { 'together-video-generation': { apiKey: 'tg-key' }, - // Configured on purpose: a Gemini key must no longer register a - // video provider now that its Veo previews are gone. + // Configured on purpose: Gemini video reads the shared `gemini` + // block, so this one alone must not register a provider. 'gemini-video-generation': { apiKey: 'gem-key' }, }, } as never); @@ -267,12 +267,11 @@ describe('VideoGenerationDriver.generate provider routing', () => { expect(togetherVideosCreateMock).not.toHaveBeenCalled(); }); - it('registers no Gemini video provider even when a Gemini key is configured', async () => { + it('registers no Gemini video provider from a gemini-video-generation key alone', async () => { const all = await driver.models(); expect( all.some((m) => m.provider === 'gemini-video-generation'), ).toBe(false); - expect(driver.driverAliases).not.toContain('gemini-video-generation'); }); it.each([ diff --git a/src/backend/drivers/ai-video/VideoGenerationDriver.ts b/src/backend/drivers/ai-video/VideoGenerationDriver.ts index 9454d78e7..2df3179bc 100644 --- a/src/backend/drivers/ai-video/VideoGenerationDriver.ts +++ b/src/backend/drivers/ai-video/VideoGenerationDriver.ts @@ -30,9 +30,11 @@ import { resolveOutputPath, saveGeneratedMediaToFS, } from '../util/generatedMedia.js'; +import { usesGoogleGeminiEndpoint } from '../util/geminiRest.js'; import { ModelCatalog } from '../util/modelCatalog.js'; import { readProviderKey } from '../util/providerRegistry.js'; import { BytePlusVideoProvider } from './providers/byteplus/BytePlusVideoProvider.js'; +import { GeminiVideoProvider } from './providers/gemini/GeminiVideoProvider.js'; import { TogetherVideoProvider } from './providers/together/TogetherVideoProvider.js'; import type { IGenerateVideoParams, @@ -72,8 +74,8 @@ const tierForPixels = ({ /** * Driver implementing the `puter-video-generation` interface. * - * Manages multiple upstream providers (Together, BytePlus/Seedance) and handles - * model resolution, provider routing, and parameter normalisation. Each + * Manages multiple upstream providers (Together, BytePlus/Seedance, Gemini) + * and handles model resolution, provider routing, and parameter normalisation. Each * provider is a plain `IVideoProvider` -- the driver instantiates them from * config on boot. * @@ -88,6 +90,7 @@ export class VideoGenerationDriver extends PuterDriver { readonly driverAliases = [ 'together-video-generation', 'byteplus-video-generation', + 'gemini-video-generation', ]; readonly isDefault = true; @@ -190,6 +193,11 @@ export class VideoGenerationDriver extends PuterDriver { } if (intendedProvider && !this.#providers[intendedProvider]) { + if (intendedProvider === 'gemini-video-generation') { + throw new HttpError(400, 'Gemini video generation is not configured', { + legacyCode: 'bad_request', + }); + } intendedProvider = fallbackProvider; } @@ -341,6 +349,20 @@ export class VideoGenerationDriver extends PuterDriver { m, ); } + + // Shares the chat provider's `gemini` key. A custom `apiBaseUrl` is a + // proxy for chat only, so its key is never sent to Google directly. + const geminiKey = readProviderKey(providers['gemini']); + if ( + geminiKey && + usesGoogleGeminiEndpoint(providers['gemini']?.apiBaseUrl) + ) { + this.#providers['gemini-video-generation'] = + new GeminiVideoProvider( + { apiKey: geminiKey }, + { stores: this.stores, fs: this.services.fs, metering: m }, + ); + } } // -- Model map ----------------------------------------------------------- @@ -455,6 +477,7 @@ export class VideoGenerationDriver extends PuterDriver { ); if (exactPuterIdMatch) return exactPuterIdMatch; - return models.find((m) => m.provider === provider) ?? models[0]; + return models.find((m) => m.provider === provider) ?? + (provider === 'gemini-video-generation' ? null : models[0]); } } diff --git a/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.integration.test.ts b/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.integration.test.ts new file mode 100644 index 000000000..2955feb6b --- /dev/null +++ b/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.integration.test.ts @@ -0,0 +1,152 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for Gemini Omni video generation against the live API, + * through the authenticated `/drivers/call` route. Generates two 360p clips + * (about $0.35 each): one from text, one from a 1x1 PNG. Skipped when + * `PUTER_TEST_AI_GEMINI_API_KEY` is unset. + */ + +import axios from 'axios'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; +import type { UsageInput } from '../../../../services/metering/types.js'; +import { setupPuterTestEnv, type PuterTestEnv } from '../../../../testUtil.js'; +import { sniffVideoMime } from '../../../../util/mediaSniff.js'; +import { + INTEGRATION_TEST_TIMEOUT_MS, + optionalEnv, + skipUnlessEnv, +} from '../../../integrationTestUtil.js'; + +const ENV_VAR = 'PUTER_TEST_AI_GEMINI_API_KEY'; +const MODEL = 'gemini-omni-1.1-flash'; +const USAGE_PREFIX = `gemini-video-generation:${MODEL}`; +// One unary call renders the clip, which routinely takes a few minutes. +const VIDEO_TIMEOUT_MS = 5 * 60 * 1000; +const PNG = + 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=='; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'Gemini Omni video generation authenticated HTTP (integration)', + () => { + let env: PuterTestEnv; + + const generate = async (args: Record) => { + const endpoint = new URL('/drivers/call', env.apiOrigin); + const host = endpoint.host; + endpoint.hostname = '127.0.0.1'; + return axios.request({ + url: endpoint.toString(), + method: 'POST', + validateStatus: () => true, + maxContentLength: Infinity, + headers: { + Authorization: `Bearer ${env.users.user.apiToken}`, + Origin: env.apiOrigin, + Host: host, + }, + data: { + interface: 'puter-video-generation', + method: 'generate', + args: { + model: MODEL, + prompt: 'A stationary red ball on a white table.', + ...args, + }, + }, + }); + }; + + /** Generates once and checks the clip and what was metered for it. */ + const expectClipAndUsage = async (args: Record) => { + const batch = vi.spyOn( + env.server.services.metering, + 'batchIncrementUsages', + ); + try { + const response = await generate(args); + expect(response.status).toBe(200); + const result = response.data.result as string; + expect(result.startsWith('data:video/mp4;base64,')).toBe(true); + const bytes = Buffer.from( + result.slice('data:video/mp4;base64,'.length), + 'base64', + ); + expect(bytes.byteLength).toBeGreaterThan(10_000); + expect(sniffVideoMime(bytes)).toBe('video/mp4'); + + const usages: UsageInput[] = batch.mock.calls + .flatMap(([, entries]) => entries) + .filter((u) => u.usageType.startsWith(`${USAGE_PREFIX}:`)); + const byType = (suffix: string) => + usages.find((u) => u.usageType === `${USAGE_PREFIX}:${suffix}`); + const input = byType('input'); + const video = byType('output:video'); + expect(input?.usageAmount).toBeGreaterThan(0); + expect(input?.costOverride).toBe(input!.usageAmount * 150); + expect(video?.usageAmount).toBeGreaterThan(1000); + expect(video?.costOverride).toBe(video!.usageAmount * 1750); + const text = byType('output:text'); + if (text) { + expect(text.costOverride).toBe(text.usageAmount * 900); + } + return { input: input!.usageAmount }; + } finally { + batch.mockRestore(); + } + }; + + beforeAll(async () => { + env = await setupPuterTestEnv({ + providers: { + gemini: { apiKey: optionalEnv(ENV_VAR)! }, + ollama: { enabled: false }, + }, + unlimitedMetering: true, + extensions: [], + }); + }, INTEGRATION_TEST_TIMEOUT_MS); + + afterAll(async () => { + await env?.shutdown(); + }); + + it( + 'generates a 360p clip from text, metering input and video tokens', + { timeout: VIDEO_TIMEOUT_MS }, + async () => { + await expectClipAndUsage({ size: '360p' }); + }, + ); + + it( + 'generates a portrait 360p clip from an image', + { timeout: VIDEO_TIMEOUT_MS }, + async () => { + const { input } = await expectClipAndUsage({ + size: '360x640', + input_reference: PNG, + }); + // The image is counted on top of the prompt. + expect(input).toBeGreaterThan(20); + }, + ); + }, +); diff --git a/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.test.ts b/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.test.ts new file mode 100644 index 000000000..580f5e636 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.test.ts @@ -0,0 +1,1597 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for the Gemini Omni video provider. + * + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock redis) with + * Gemini and Together keys and drives the unified `puter-video-generation` + * driver, so routing, file loading, credit holds, usage recording and + * `puter_output_path` run end to end. Global `fetch` is stubbed with canned + * countTokens and Interactions replies, and `secureFetch` with a canned image, + * the real network egress points. + */ + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; +import { v4 as uuidv4 } from 'uuid'; + +import type { Actor } from '../../../../core/actor.js'; +import { Context, runWithContext } from '../../../../core/context.js'; +import { PuterServer } from '../../../../server.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { UsageInput } from '../../../../services/metering/types.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { generateDefaultFsentries } from '../../../../util/userProvisioning.js'; +import { callerError } from '../../../integrationTestUtil.js'; +import { loadFileInput } from '../../../util/fileInput.js'; +import type { IGenerateVideoParams } from '../../types.js'; +import type { VideoGenerationDriver } from '../../VideoGenerationDriver.js'; +import { + GEMINI_OMNI_COUNT_TOKENS_MAX_RESPONSE_BYTES, + GEMINI_OMNI_MAX_IMAGE_BYTES, + GEMINI_OMNI_MAX_RESPONSE_BYTES, +} from './GeminiVideoProvider.js'; + +const { secureFetchMock } = vi.hoisted(() => ({ secureFetchMock: vi.fn() })); + +vi.mock('../../../../util/secureHttp.js', async (importOriginal) => ({ + ...(await importOriginal< + typeof import('../../../../util/secureHttp.js') + >()), + secureFetch: secureFetchMock, +})); + +const MODEL = 'gemini-omni-1.1-flash'; +const PROVIDER = 'gemini-video-generation'; +const API_BASE = 'https://generativelanguage.googleapis.com/v1beta'; +const INTERACTIONS_URL = `${API_BASE}/interactions`; +const COUNT_TOKENS_URL = `${API_BASE}/models/${MODEL}:countTokens`; +const SAMPLE_URL = 'https://assets.puter.site/txt2vid.mp4'; +const PROMPT = 'A stationary red ball.'; + +// Mirrors the published rates: $1.50/M input, $9/M text output and +// thinking, $17.50/M video output. +const INPUT_RATE = 150; +const TEXT_RATE = 900; +const VIDEO_RATE = 1750; +const OUTPUT_ALLOWANCE = 65536; +const COUNTED_TOKENS = 18; +const holdFor = (inputTokens: number) => + inputTokens * INPUT_RATE + + 173760 * VIDEO_RATE + + OUTPUT_ALLOWANCE * TEXT_RATE; +const HOLD = holdFor(COUNTED_TOKENS); + +// A 1x1 PNG, and the head of an MP4 (an `ftyp` box with an `isom` brand). +const PNG_BASE64 = + 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=='; +const PNG = Buffer.from(PNG_BASE64, 'base64'); +const MP4 = Buffer.concat([ + Buffer.from([0, 0, 0, 0x18]), + Buffer.from('ftypisom', 'latin1'), + Buffer.from([0, 0, 2, 0]), + Buffer.from('isomiso2', 'latin1'), + Buffer.from('mdat-frames', 'latin1'), +]); + +// -- Test harness -- + +let server: PuterServer; +let videoDriver: VideoGenerationDriver; +let fetchSpy: MockInstance; +let hasCreditsSpy: MockInstance; +let batchSpy: MockInstance; + +const generate = (args: Record = {}) => + videoDriver.generate({ + prompt: PROMPT, + model: MODEL, + ...args, + } as IGenerateVideoParams); + +beforeAll(async () => { + server = await setupTestServer({ + providers: { + gemini: { apiKey: 'gemini-test-key' }, + 'together-video-generation': { apiKey: 'tg-key' }, + }, + } as never); + videoDriver = server.drivers.aiVideo as unknown as VideoGenerationDriver; +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +beforeEach(() => { + fetchSpy = vi + .spyOn(globalThis, 'fetch') + .mockRejectedValue(new Error('unexpected fetch')) as MockInstance< + typeof fetch + >; + secureFetchMock.mockReset(); + secureFetchMock.mockRejectedValue(new Error('unexpected secureFetch')); + // The full hold is more than a new account's allowance. + hasCreditsSpy = vi + .spyOn(server.services.metering, 'hasEnoughCredits') + .mockResolvedValue(true); + batchSpy = vi.spyOn(server.services.metering, 'batchIncrementUsages'); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +const makeUser = async (): Promise<{ actor: Actor; userId: number }> => { + const username = `gemini-video-${Math.random().toString(36).slice(2, 10)}`; + const created = await server.stores.user.create({ + username, + uuid: uuidv4(), + password: null, + email: `${username}@test.local`, + free_storage: 100 * 1024 * 1024, + requires_email_confirmation: false, + }); + await generateDefaultFsentries( + server.clients.db, + server.stores.user, + created, + ); + const refreshed = (await server.stores.user.getById(created.id))!; + return { + userId: refreshed.id, + actor: { + user: { + id: refreshed.id, + uuid: refreshed.uuid, + username: refreshed.username, + email: refreshed.email ?? null, + email_confirmed: true, + } as Actor['user'], + }, + }; +}; + +const withActor = (actor: Actor, fn: () => T | Promise): Promise => + Promise.resolve(runWithContext({ actor }, fn)); + +/** A caller with no user id, enough for test mode and routing. */ +const TEST_MODE_ACTOR: Actor = { + user: { uuid: uuidv4(), username: 'gemini-video-test-mode' }, +} as Actor; + +const testMode = (args: Record = {}) => + withActor(TEST_MODE_ACTOR, () => generate({ test_mode: true, ...args })); + +const USAGE_720P = { + total_input_tokens: 18, + input_tokens_by_modality: [{ modality: 'text', tokens: 18 }], + total_output_tokens: 58539, + output_tokens_by_modality: [{ modality: 'video', tokens: 57920 }], + total_thought_tokens: 418, + total_tokens: 58975, +}; + +const USAGE_360P_IMAGE = { + total_input_tokens: 1107, + input_tokens_by_modality: [ + { modality: 'image', tokens: 1100 }, + { modality: 'text', tokens: 7 }, + ], + total_output_tokens: 19960, + output_tokens_by_modality: [{ modality: 'video', tokens: 19310 }], + total_thought_tokens: 220, + total_tokens: 21287, +}; + +/** A reply that produced text and thinking but no video. */ +const USAGE_TEXT_ONLY = { + total_input_tokens: 18, + total_output_tokens: 12, + output_tokens_by_modality: [{ modality: 'text', tokens: 12 }], + total_thought_tokens: 240, +}; + +const videoBlock = (bytes: Buffer = MP4) => ({ + type: 'video', + mime_type: 'video/mp4', + data: bytes.toString('base64'), +}); + +/** A completed reply; `usage` given as undefined leaves the field out. */ +const interaction = ( + opts: { + status?: string; + content?: unknown[]; + steps?: unknown[]; + usage?: unknown; + } = {}, +) => ({ + status: opts.status ?? 'completed', + steps: opts.steps ?? [ + { type: 'model_output', content: opts.content ?? [videoBlock()] }, + ], + ...('usage' in opts ? { usage: opts.usage } : { usage: USAGE_720P }), +}); + +/** Queues the countTokens reply, then the Interactions reply. */ +const reply = ( + body: unknown = interaction(), + count: unknown = { totalTokens: COUNTED_TOKENS }, +) => { + fetchSpy + .mockResolvedValueOnce(Response.json(count)) + .mockResolvedValueOnce(Response.json(body)); +}; + +const sent = (call: number) => { + const [url, init] = fetchSpy.mock.calls[call]!; + return { + url: String(url), + method: init?.method, + headers: init?.headers as Record, + body: JSON.parse(String(init?.body)), + }; +}; + +const metered = (): UsageInput[] => + batchSpy.mock.calls.flatMap(([, usages]) => usages); + +/** Records each credit hold taken, its options, and its release. */ +const trackHolds = (order: string[] = []) => { + const metering = server.services.metering; + const realReserve = metering.reserveCredits.bind(metering); + const held: { amount: number; ttlMs?: number }[] = []; + const released = vi.fn(); + vi.spyOn(metering, 'reserveCredits').mockImplementation( + async (actor, amount, opts) => { + held.push({ amount, ttlMs: opts?.ttlMs }); + const hold = await realReserve(actor, amount, opts); + return { + ...hold, + release: async () => { + order.push('release'); + released(); + await hold.release(); + }, + }; + }, + ); + return { held, released }; +}; + +/** A body that streams past `maxBytes` without declaring its length. */ +const oversizedBody = (maxBytes: number) => { + const chunk = new Uint8Array(1024 * 1024).fill(0x20); + let sentBytes = 0; + return new Response( + new ReadableStream({ + pull(controller) { + if (sentBytes > maxBytes) { + controller.close(); + return; + } + sentBytes += chunk.byteLength; + controller.enqueue(chunk); + }, + }), + ); +}; + +const pngDataUri = `data:image/png;base64,${PNG_BASE64}`; + +// -- Catalog and routing -- + +describe('GeminiVideoProvider catalog', () => { + it('lists Gemini Omni with per-token cent rates and no duration or size ladder', async () => { + const model = (await videoDriver.models()).find( + (m) => m.provider === PROVIDER, + ); + expect(model).toMatchObject({ + id: MODEL, + puterId: `google:google/${MODEL}`, + name: 'Gemini Omni 1.1 Flash', + costs_currency: 'usd-cents', + output_cost_key: 'output:video', + costs: { + input: 0.00015, + 'output:text': 0.0009, + 'output:video': 0.00175, + }, + promptLength: { min: 1, max: 4000 }, + }); + expect(model?.durationSeconds).toBeUndefined(); + expect(model?.dimensions).toBeUndefined(); + expect(await videoDriver.list()).toContain(`google:google/${MODEL}`); + }); + + it('reports one cost line per usage type it records', () => { + expect( + videoDriver + .getReportedCosts() + .filter((c) => c.source === `driver:aiVideo/${PROVIDER}`), + ).toEqual([ + { + usageType: `${PROVIDER}:${MODEL}:input`, + costValue: 0.00015, + source: `driver:aiVideo/${PROVIDER}`, + }, + { + usageType: `${PROVIDER}:${MODEL}:output:text`, + costValue: 0.0009, + source: `driver:aiVideo/${PROVIDER}`, + }, + { + usageType: `${PROVIDER}:${MODEL}:output:video`, + costValue: 0.00175, + source: `driver:aiVideo/${PROVIDER}`, + }, + ]); + }); + + it('registers the gemini-video-generation driver alias', () => { + expect(videoDriver.driverAliases).toContain(PROVIDER); + }); +}); + +describe('GeminiVideoProvider routing', () => { + // Gemini refuses `seconds` even in test mode, so that rejection proves a + // call reached it rather than Together. + const reachesGemini = (call: Promise) => + expect(call).rejects.toMatchObject({ + statusCode: 400, + message: 'seconds is not supported by Gemini Omni', + }); + + it.each([ + MODEL, + `google:google/${MODEL}`, + `google/${MODEL}`, + ` ${MODEL.toUpperCase()} `, + ])('routes model %s to Gemini', async (model) => { + await reachesGemini(testMode({ model, seconds: 5 })); + }); + + it.each(['provider', 'driver'] as const)( + 'rejects a Together model when Gemini is pinned by %s', + async (selection) => { + const model = (await videoDriver.models()).find( + (entry) => entry.provider === 'together-video-generation', + )!.id; + await expect( + Promise.resolve( + runWithContext( + { + actor: TEST_MODE_ACTOR, + ...(selection === 'driver' + ? { driverName: PROVIDER } + : {}), + }, + () => + videoDriver.generate({ + prompt: PROMPT, + model, + test_mode: true, + ...(selection === 'provider' + ? { provider: PROVIDER } + : {}), + }), + ), + ), + ).rejects.toMatchObject({ + statusCode: 400, + message: `Model not found: ${model}`, + }); + expect(fetchSpy).not.toHaveBeenCalled(); + }, + ); + + it('allows an unpinned Together model when Gemini is configured', async () => { + const model = (await videoDriver.models()).find( + (entry) => entry.provider === 'together-video-generation', + )!.id; + await expect( + withActor(TEST_MODE_ACTOR, () => + videoDriver.generate({ + prompt: PROMPT, + model, + test_mode: true, + }), + ), + ).resolves.toBe(SAMPLE_URL); + }); + + it('keeps Together as the default when both are configured', async () => { + await expect( + Promise.resolve( + runWithContext({ actor: TEST_MODE_ACTOR }, () => + videoDriver.generate({ + prompt: PROMPT, + seconds: 5, + test_mode: true, + }), + ), + ), + ).resolves.toBe(SAMPLE_URL); + }); + + it('routes the gemini-video-generation driver name and provider to Gemini Omni', async () => { + const call = (args: Record, driverName?: string) => + Promise.resolve( + runWithContext( + { + actor: TEST_MODE_ACTOR, + ...(driverName ? { driverName } : {}), + }, + () => + videoDriver.generate({ + prompt: PROMPT, + test_mode: true, + ...args, + } as IGenerateVideoParams), + ), + ); + await reachesGemini(call({ seconds: 5 }, PROVIDER)); + await reachesGemini(call({ seconds: 5, provider: PROVIDER })); + expect(await call({}, PROVIDER)).toBe(SAMPLE_URL); + }); +}); + +describe('GeminiVideoProvider registration', () => { + it('is not registered without a gemini key', async () => { + const bare = await setupTestServer({ + providers: { 'together-video-generation': { apiKey: 'tg-key' } }, + } as never); + try { + const driver = bare.drivers + .aiVideo as unknown as VideoGenerationDriver; + expect( + (await driver.models()).some((m) => m.provider === PROVIDER), + ).toBe(false); + await expect( + withActor(TEST_MODE_ACTOR, () => + driver.generate({ + prompt: PROMPT, + provider: PROVIDER, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + await expect( + Promise.resolve( + runWithContext( + { + actor: TEST_MODE_ACTOR, + driverName: PROVIDER, + }, + () => driver.generate({ prompt: PROMPT }), + ), + ), + ).rejects.toMatchObject({ + statusCode: 400, + }); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + await bare.shutdown(); + } + }); + + it('is not registered for a custom Gemini endpoint, so its key never reaches Google', async () => { + const custom = await setupTestServer({ + providers: { + gemini: { + apiKey: 'proxy-test-key', + apiBaseUrl: 'https://proxy.example/v1', + }, + 'together-video-generation': { apiKey: 'tg-key' }, + }, + } as never); + try { + const driver = custom.drivers + .aiVideo as unknown as VideoGenerationDriver; + expect( + (await driver.models()).some((m) => m.provider === PROVIDER), + ).toBe(false); + await expect( + withActor(TEST_MODE_ACTOR, () => + driver.generate({ + prompt: PROMPT, + provider: PROVIDER, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + await expect( + Promise.resolve( + runWithContext( + { + actor: TEST_MODE_ACTOR, + driverName: PROVIDER, + }, + () => driver.generate({ prompt: PROMPT }), + ), + ), + ).rejects.toMatchObject({ + statusCode: 400, + }); + expect(fetchSpy).not.toHaveBeenCalled(); + await expect( + withActor(TEST_MODE_ACTOR, () => + driver.generate({ prompt: PROMPT, model: MODEL }), + ), + ).rejects.toMatchObject({ + statusCode: 400, + message: `Model not found: ${MODEL}`, + }); + await expect( + Promise.resolve( + runWithContext( + { + actor: TEST_MODE_ACTOR, + driverName: PROVIDER, + }, + () => driver.generate({ prompt: PROMPT }), + ), + ), + ).rejects.toMatchObject({ + statusCode: 400, + }); + expect(fetchSpy).not.toHaveBeenCalled(); + } finally { + await custom.shutdown(); + } + }); +}); + +// -- test_mode -- + +describe('GeminiVideoProvider test_mode', () => { + it('returns the sample clip without network, credits or metering', async () => { + expect(await testMode()).toBe(SAMPLE_URL); + expect(fetchSpy).not.toHaveBeenCalled(); + expect(hasCreditsSpy).not.toHaveBeenCalled(); + expect(batchSpy).not.toHaveBeenCalled(); + }); + + it('validates an image input shape without loading it', async () => { + expect(await testMode({ input_reference: pngDataUri })).toBe( + SAMPLE_URL, + ); + expect(secureFetchMock).not.toHaveBeenCalled(); + await expect(testMode({ input_reference: 42 })).rejects.toMatchObject({ + statusCode: 400, + }); + }); +}); + +// -- Option validation -- + +describe('GeminiVideoProvider option validation', () => { + it.each([ + ['seconds', 8], + ['seconds', '8'], + ['duration', 8], + ['fps', 24], + ['steps', 20], + ['guidance_scale', 7], + ['seed', 1], + ['negative_prompt', 'blur'], + ['output_quality', 90], + ['generate_audio', false], + ['generate_audio', true], + ['frame_images', [{ input_image: pngDataUri, frame: 0 }]], + ['last_frame', pngDataUri], + ['reference_images', [pngDataUri]], + ['reference_images', []], + ['metadata', { a: 1 }], + ])('refuses %s=%j, test mode included', async (key, value) => { + const { actor } = await makeUser(); + for (const test_mode of [true, false]) { + await expect( + withActor(actor, () => generate({ [key]: value, test_mode })), + ).rejects.toMatchObject({ + statusCode: 400, + legacyCode: 'bad_request', + message: `${key} is not supported by Gemini Omni`, + }); + } + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('accepts output_format video/mp4 and refuses any other', async () => { + expect(await testMode({ output_format: 'video/mp4' })).toBe(SAMPLE_URL); + for (const output_format of ['mp4', 'MP4', 'video/webm']) { + await expect(testMode({ output_format })).rejects.toMatchObject({ + statusCode: 400, + message: 'output_format must be video/mp4', + }); + } + }); + + it('accepts a 4,000-character prompt and refuses a longer one', async () => { + expect(await testMode({ prompt: 'a'.repeat(4000) })).toBe(SAMPLE_URL); + await expect( + testMode({ prompt: 'a'.repeat(4001) }), + ).rejects.toMatchObject({ + statusCode: 400, + message: 'prompt must be at most 4000 characters', + }); + }); + + it.each([ + ['an unpublished resolution', { size: '480p' }], + ['an unpublished pixel size', { size: '1024x576' }], + ['a square size', { size: '720x720' }], + ['a size with spaces', { size: '1280 x 720' }], + ['a non-string size', { size: 720 }], + ['an empty resolution', { resolution: ' ' }], + [ + 'conflicting size and resolution', + { size: '720p', resolution: '1080p' }, + ], + ['width without height', { width: 1280 }], + ['height without width', { height: 720 }], + ['an unpublished width and height', { width: 1000, height: 500 }], + ['string width and height', { width: '1280', height: '720' }], + [ + 'width and height against a pixel size', + { size: '1280x720', width: 1920, height: 1080 }, + ], + [ + 'width and height against a resolution', + { size: '720p', width: 640, height: 360 }, + ], + [ + 'a flipped width and height', + { size: '1280x720', width: 720, height: 1280 }, + ], + ])('refuses %s, test mode included', async (_label, args) => { + await expect(testMode(args)).rejects.toMatchObject({ + statusCode: 400, + legacyCode: 'bad_request', + }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); +}); + +// -- Request shape -- + +describe('GeminiVideoProvider request shape', () => { + it('counts tokens, then sends an unstored unary 720p 16:9 interaction', async () => { + const { actor } = await makeUser(); + reply(); + await withActor(actor, () => generate()); + + expect(fetchSpy).toHaveBeenCalledTimes(2); + const count = sent(0); + expect(count.url).toBe(COUNT_TOKENS_URL); + expect(count.method).toBe('POST'); + expect(count.headers).toEqual({ + 'x-goog-api-key': 'gemini-test-key', + 'Content-Type': 'application/json', + }); + expect(count.body).toEqual({ + contents: [{ parts: [{ text: PROMPT }] }], + }); + + const request = sent(1); + expect(request.url).toBe(INTERACTIONS_URL); + expect(request.headers['x-goog-api-key']).toBe('gemini-test-key'); + expect(request.body).toEqual({ + model: MODEL, + input: PROMPT, + response_format: { + type: 'video', + resolution: '720p', + aspect_ratio: '16:9', + }, + generation_config: { max_output_tokens: 65536 }, + store: false, + background: false, + stream: false, + }); + }); + + it.each([ + [{ size: '360x640' }, '360p', '9:16'], + [{ size: '640x360' }, '360p', '16:9'], + [{ resolution: '360p' }, '360p', '16:9'], + [{ size: '720x1280' }, '720p', '9:16'], + [{ size: '1080P' }, '1080p', '16:9'], + [{ size: '1080x1920' }, '1080p', '9:16'], + [{ size: '4K' }, '4k', '16:9'], + [{ size: '2160x3840' }, '4k', '9:16'], + [{ width: 360, height: 640 }, '360p', '9:16'], + [{ size: '720p', width: 720, height: 1280 }, '720p', '9:16'], + [{ size: '1920x1080', resolution: '1920X1080' }, '1080p', '16:9'], + ])( + 'sends %j as %s at %s, untouched by the driver', + async (args, resolution, aspectRatio) => { + const { actor } = await makeUser(); + reply(); + await withActor(actor, () => generate(args)); + expect(sent(1).body.response_format).toEqual({ + type: 'video', + resolution, + aspect_ratio: aspectRatio, + }); + }, + ); + + it('sends a data URI image first, with the image-to-video task', async () => { + const { actor } = await makeUser(); + reply(); + await withActor(actor, () => + generate({ input_reference: pngDataUri, size: '360x640' }), + ); + expect(sent(0).body).toEqual({ + contents: [ + { + parts: [ + { + inlineData: { + mimeType: 'image/png', + data: PNG_BASE64, + }, + }, + { text: PROMPT }, + ], + }, + ], + }); + expect(sent(1).body).toMatchObject({ + input: [ + { type: 'image', mime_type: 'image/png', data: PNG_BASE64 }, + { type: 'text', text: PROMPT }, + ], + generation_config: { + max_output_tokens: 65536, + video_config: { task: 'image_to_video' }, + }, + }); + }); + + it('takes raw base64 and an image URL', async () => { + const { actor } = await makeUser(); + reply(); + await withActor(actor, () => generate({ input_reference: PNG_BASE64 })); + expect(sent(1).body.input[0]).toEqual({ + type: 'image', + mime_type: 'image/png', + data: PNG_BASE64, + }); + + secureFetchMock.mockResolvedValueOnce( + new Response(PNG, { headers: { 'content-type': 'image/png' } }), + ); + reply(); + await withActor(actor, () => + generate({ input_reference: 'https://example.com/frame.png' }), + ); + expect(secureFetchMock).toHaveBeenCalledWith( + 'https://example.com/frame.png', + expect.objectContaining({ signal: expect.any(AbortSignal) }), + ); + expect(sent(3).body.input[0].data).toBe(PNG_BASE64); + }); + + it("reads the caller's own Puter file", async () => { + const { actor, userId } = await makeUser(); + const path = `/${actor.user.username}/frame.png`; + await server.services.fs.write(userId, { + fileMetadata: { + path, + size: PNG.byteLength, + contentType: 'image/png', + }, + fileContent: PNG, + }); + reply(); + await withActor(actor, () => generate({ input_reference: path })); + expect(sent(1).body.input[0].data).toBe(PNG_BASE64); + }); + + it("refuses another user's Puter file before calling Gemini", async () => { + const owner = await makeUser(); + const intruder = await makeUser(); + const path = `/${owner.actor.user.username}/private.png`; + await server.services.fs.write(owner.userId, { + fileMetadata: { + path, + size: PNG.byteLength, + contentType: 'image/png', + }, + fileContent: PNG, + }); + const err = await withActor(intruder.actor, () => + generate({ input_reference: path }), + ).then( + () => null, + (e: unknown) => e, + ); + expect([403, 404]).toContain( + (err as { statusCode?: number } | null)?.statusCode, + ); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('refuses an image over 5 MiB with 413 before any call', async () => { + const { actor } = await makeUser(); + const big = Buffer.alloc(GEMINI_OMNI_MAX_IMAGE_BYTES + 1); + PNG.copy(big); + await expect( + withActor(actor, () => + generate({ + input_reference: `data:image/png;base64,${big.toString('base64')}`, + }), + ), + ).rejects.toMatchObject({ + statusCode: 413, + legacyCode: 'input_too_large', + }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it.each([ + [ + 'text bytes', + `data:image/png;base64,${Buffer.from('not an image').toString('base64')}`, + ], + ['a GIF', 'data:image/gif;base64,R0lGODlhAQABAAAAACw='], + ])('refuses %s as the image', async (_label, input_reference) => { + const { actor } = await makeUser(); + await expect( + withActor(actor, () => generate({ input_reference })), + ).rejects.toMatchObject({ + statusCode: 400, + message: 'input_reference must be a PNG, JPEG or WebP image', + }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('bounds countTokens at 30 seconds and the generation at 10 minutes', async () => { + const timeoutSpy = vi.spyOn(AbortSignal, 'timeout'); + const { actor } = await makeUser(); + reply(); + await withActor(actor, () => generate()); + expect(timeoutSpy).toHaveBeenCalledWith(30_000); + expect(timeoutSpy).toHaveBeenCalledWith(600_000); + }); +}); + +// -- Response -- + +describe('GeminiVideoProvider response', () => { + it('returns the MP4 as a data URI', async () => { + const { actor } = await makeUser(); + reply(); + expect(await withActor(actor, () => generate())).toBe( + `data:video/mp4;base64,${MP4.toString('base64')}`, + ); + }); + + it('returns the last video and ignores thought and text blocks', async () => { + const { actor } = await makeUser(); + const last = Buffer.concat([MP4, Buffer.from('-last')]); + reply( + interaction({ + steps: [ + { + type: 'thought', + content: [videoBlock(Buffer.from('x'))], + }, + { + type: 'model_output', + content: [ + videoBlock(), + { type: 'text', text: 'Here is your clip.' }, + videoBlock(last), + ], + }, + ], + }), + ); + expect(await withActor(actor, () => generate())).toBe( + `data:video/mp4;base64,${last.toString('base64')}`, + ); + }); + + it('saves the clip to puter_output_path', async () => { + const { actor } = await makeUser(); + const path = `/${actor.user.username}/omni.mp4`; + reply(); + await withActor(actor, () => generate({ puter_output_path: path })); + const saved = await withActor(actor, () => + loadFileInput(server.stores, server.services.fs, actor, path), + ); + expect(saved.buffer.equals(MP4)).toBe(true); + expect(saved.mimeType).toBe('video/mp4'); + }); + + it('charges the reported usage of a reply with no video, then fails it as 400 upstream_bad_request', async () => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + reply(interaction({ content: [{ type: 'text', text: 'No.' }] })); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 400, + legacyCode: 'upstream_bad_request', + message: 'Gemini did not return a video', + }); + expect(batchSpy).toHaveBeenCalledTimes(1); + expect(metered().map((u) => u.usageAmount)).toEqual([ + 18, + 57920, + 58539 - 57920 + 418, + ]); + expect(released).toHaveBeenCalledTimes(1); + expect(batchSpy.mock.invocationCallOrder[0]).toBeLessThan( + released.mock.invocationCallOrder[0]!, + ); + }); + + it('charges text-only usage of a reply with no video as input and text', async () => { + const { actor } = await makeUser(); + reply( + interaction({ + content: [{ type: 'text', text: 'No.' }], + usage: USAGE_TEXT_ONLY, + }), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 400, + legacyCode: 'upstream_bad_request', + }); + expect(metered()).toEqual([ + { + usageType: `${PROVIDER}:${MODEL}:input`, + usageAmount: 18, + costOverride: 18 * INPUT_RATE, + }, + { + usageType: `${PROVIDER}:${MODEL}:output:text`, + usageAmount: 12 + 240, + costOverride: (12 + 240) * TEXT_RATE, + }, + ]); + }); + + it.each([ + ['a non-MP4 MIME type', { ...videoBlock(), mime_type: 'video/webm' }], + [ + 'a URI instead of bytes', + { + type: 'video', + mime_type: 'video/mp4', + uri: 'https://example.com/v.mp4', + }, + ], + [ + 'invalid base64', + { type: 'video', mime_type: 'video/mp4', data: 'not base64!' }, + ], + ['bytes that are not an MP4', videoBlock(Buffer.from('plain bytes!'))], + ])( + 'charges the reported usage once and fails a video with %s as 502', + async (_label, block) => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + reply(interaction({ content: [block] })); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + message: 'Gemini video generation returned a malformed video', + }); + expect(batchSpy).toHaveBeenCalledTimes(1); + expect(metered().map((u) => u.usageAmount)).toEqual([ + 18, + 57920, + 58539 - 57920 + 418, + ]); + expect(released).toHaveBeenCalledTimes(1); + expect(secureFetchMock).not.toHaveBeenCalled(); + }, + ); + + it('fails a malformed video with unusable usage without charging', async () => { + const { actor } = await makeUser(); + reply( + interaction({ + content: [videoBlock(Buffer.from('plain bytes!'))], + usage: null, + }), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + message: 'Gemini video generation returned a malformed video', + }); + expect(batchSpy).not.toHaveBeenCalled(); + }); + + it.each(['incomplete', 'failed', 'cancelled', 'in_progress'])( + 'charges the reported usage of a %s interaction once, then fails it as 502', + async (status) => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + reply(interaction({ status })); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + message: 'Gemini video generation did not complete the request', + }); + expect(batchSpy).toHaveBeenCalledTimes(1); + expect(metered().map((u) => u.usageAmount)).toEqual([ + 18, + 57920, + 58539 - 57920 + 418, + ]); + expect(released).toHaveBeenCalledTimes(1); + expect(batchSpy.mock.invocationCallOrder[0]).toBeLessThan( + released.mock.invocationCallOrder[0]!, + ); + }, + ); + + it('charges text-only usage of a failed interaction without inventing video', async () => { + const { actor } = await makeUser(); + reply( + interaction({ + status: 'failed', + content: [], + usage: USAGE_TEXT_ONLY, + }), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + message: 'Gemini video generation did not complete the request', + }); + expect(metered().map((u) => [u.usageType, u.usageAmount])).toEqual([ + [`${PROVIDER}:${MODEL}:input`, 18], + [`${PROVIDER}:${MODEL}:output:text`, 12 + 240], + ]); + }); + + it('charges input and thinking of a failed interaction that reports no output and no breakdown', async () => { + const { actor } = await makeUser(); + reply( + interaction({ + status: 'failed', + content: [], + usage: { + total_input_tokens: 18, + total_output_tokens: 0, + total_thought_tokens: 240, + }, + }), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + message: 'Gemini video generation did not complete the request', + }); + expect(metered().map((u) => [u.usageType, u.usageAmount])).toEqual([ + [`${PROVIDER}:${MODEL}:input`, 18], + [`${PROVIDER}:${MODEL}:output:text`, 240], + ]); + }); + + it.each([ + ['missing usage', undefined], + [ + 'output without a modality breakdown', + { ...USAGE_TEXT_ONLY, output_tokens_by_modality: undefined }, + ], + ])( + 'fails a failed interaction with %s as not completed, without charging', + async (_label, usage) => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + reply(interaction({ status: 'failed', content: [], usage })); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + message: 'Gemini video generation did not complete the request', + }); + expect(batchSpy).not.toHaveBeenCalled(); + expect(released).toHaveBeenCalledTimes(1); + }, + ); + + it('releases the hold when settling a failed interaction throws', async () => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + batchSpy.mockRejectedValueOnce(new Error('metering down')); + reply(interaction({ status: 'failed' })); + await expect(withActor(actor, () => generate())).rejects.toThrow( + 'metering down', + ); + expect(batchSpy).toHaveBeenCalledTimes(1); + expect(released).toHaveBeenCalledTimes(1); + }); + + it('stops reading a reply over 64 MiB', async () => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + fetchSpy + .mockResolvedValueOnce(Response.json({ totalTokens: 18 })) + .mockResolvedValueOnce( + oversizedBody(GEMINI_OMNI_MAX_RESPONSE_BYTES), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + message: `Gemini video generation response exceeds ${64 * 1024 * 1024} bytes`, + }); + expect(released).toHaveBeenCalledTimes(1); + expect(batchSpy).not.toHaveBeenCalled(); + }); + + it.each([ + [400, 400, 'upstream_bad_request'], + [401, 500, 'upstream_auth_failed'], + [429, 429, 'upstream_rate_limited'], + [503, 400, 'upstream_provider_unavailable'], + ])( + 'maps an upstream %i to %i %s and releases the hold', + async (upstreamStatus, statusCode, legacyCode) => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + fetchSpy + .mockResolvedValueOnce(Response.json({ totalTokens: 18 })) + .mockResolvedValueOnce( + Response.json( + { error: { code: upstreamStatus, message: 'Failed' } }, + { status: upstreamStatus }, + ), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode, + legacyCode, + fields: { provider: 'gemini', upstreamStatus }, + }); + expect(released).toHaveBeenCalledTimes(1); + expect(batchSpy).not.toHaveBeenCalled(); + }, + ); + + it('surfaces a generation timeout as 504 and releases the hold', async () => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + fetchSpy + .mockResolvedValueOnce(Response.json({ totalTokens: 18 })) + .mockRejectedValueOnce( + new DOMException( + 'The operation was aborted due to timeout', + 'TimeoutError', + ), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 504, + legacyCode: 'upstream_timeout', + }); + expect(released).toHaveBeenCalledTimes(1); + expect(batchSpy).not.toHaveBeenCalled(); + }); + + it('aborts the generation when the caller disconnects, and releases the hold', async () => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + const abort = new AbortController(); + let upstreamSignal: AbortSignal | undefined; + fetchSpy + .mockResolvedValueOnce(Response.json({ totalTokens: 18 })) + .mockImplementationOnce( + (_url, init) => + new Promise((_resolve, reject) => { + upstreamSignal = init!.signal!; + upstreamSignal.addEventListener( + 'abort', + () => reject(upstreamSignal!.reason), + { once: true }, + ); + abort.abort(); + }), + ); + + await expect( + withActor(actor, () => { + Context.set('abortSignal', abort.signal); + return generate(); + }), + ).rejects.toMatchObject({ name: 'AbortError' }); + expect(upstreamSignal?.aborted).toBe(true); + expect(released).toHaveBeenCalledTimes(1); + expect(batchSpy).not.toHaveBeenCalled(); + }); +}); + +// -- countTokens preflight -- + +describe('GeminiVideoProvider countTokens preflight', () => { + it.each([ + ['a missing count', {}], + ['a zero count', { totalTokens: 0 }], + ['a negative count', { totalTokens: -1 }], + ['a fractional count', { totalTokens: 1.5 }], + ['a string count', { totalTokens: '18' }], + ['an unsafe count', { totalTokens: Number.MAX_SAFE_INTEGER + 1 }], + ])( + 'fails %s as 502 before holding credit or generating', + async (_label, count) => { + const { actor } = await makeUser(); + const { held } = trackHolds(); + fetchSpy.mockResolvedValueOnce(Response.json(count)); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + message: 'Gemini token count returned an unusable count', + }); + expect(held).toEqual([]); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }, + ); + + it('refuses input over 131,072 tokens before holding credit', async () => { + const { actor } = await makeUser(); + const { held } = trackHolds(); + fetchSpy.mockResolvedValueOnce(Response.json({ totalTokens: 131_073 })); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 400, + legacyCode: 'bad_request', + code: 'input_too_large', + }); + expect(held).toEqual([]); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it('accepts input of exactly 131,072 tokens and holds for it', async () => { + const { actor } = await makeUser(); + const { held } = trackHolds(); + reply(interaction(), { totalTokens: 131_072 }); + await withActor(actor, () => generate()); + expect(held[0]?.amount).toBe(holdFor(131_072)); + }); + + it('stops reading a countTokens reply over 64 KiB', async () => { + const { actor } = await makeUser(); + fetchSpy.mockResolvedValueOnce( + oversizedBody(GEMINI_OMNI_COUNT_TOKENS_MAX_RESPONSE_BYTES), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + message: `Gemini token count response exceeds ${64 * 1024} bytes`, + }); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); + + it('surfaces a countTokens timeout as 504', async () => { + const { actor } = await makeUser(); + fetchSpy.mockRejectedValueOnce( + new DOMException( + 'The operation was aborted due to timeout', + 'TimeoutError', + ), + ); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 504, + legacyCode: 'upstream_timeout', + }); + expect(fetchSpy).toHaveBeenCalledTimes(1); + }); +}); + +// -- Credit hold -- + +describe('GeminiVideoProvider credit hold', () => { + it('throws 402 after counting tokens, before generating', async () => { + const { actor } = await makeUser(); + hasCreditsSpy.mockResolvedValueOnce(false); + fetchSpy.mockResolvedValueOnce(Response.json({ totalTokens: 18 })); + await expect(withActor(actor, () => generate())).rejects.toMatchObject({ + statusCode: 402, + }); + expect(hasCreditsSpy.mock.calls[0]![1]).toBe(HOLD); + expect(fetchSpy).toHaveBeenCalledTimes(1); + expect(batchSpy).not.toHaveBeenCalled(); + }); + + it('holds input plus the output and thinking allowances for 11 minutes', async () => { + const { actor } = await makeUser(); + const { held } = trackHolds(); + let heldDuringCall = -1; + fetchSpy + .mockResolvedValueOnce(Response.json({ totalTokens: 18 })) + .mockImplementationOnce(async () => { + heldDuringCall = await server.stores.creditHold.outstanding( + actor.user.uuid, + ); + return Response.json(interaction()); + }); + + await withActor(actor, () => generate()); + + // Input plus verified 4K video usage and a separate text allowance. + expect(HOLD).toBe(363_065_100); + expect(held).toEqual([{ amount: HOLD, ttlMs: 11 * 60 * 1000 }]); + expect(heldDuringCall).toBe(HOLD); + await vi.waitFor(async () => + expect( + await server.stores.creditHold.outstanding(actor.user.uuid), + ).toBe(0), + ); + }); + + it('records usage before releasing the hold', async () => { + const { actor } = await makeUser(); + const order: string[] = []; + trackHolds(order); + batchSpy.mockRestore(); + const metering = server.services.metering; + const realBatch = metering.batchIncrementUsages.bind(metering); + vi.spyOn(metering, 'batchIncrementUsages').mockImplementation( + async (...args) => { + const result = await realBatch(...args); + order.push('recorded'); + return result; + }, + ); + reply(); + await withActor(actor, () => generate()); + expect(order).toEqual(['recorded', 'release']); + }); +}); + +// -- Usage settlement -- + +describe('GeminiVideoProvider usage', () => { + it('charges the observed 720p usage: input, video, and other output plus thinking as text', async () => { + const { actor } = await makeUser(); + reply(); + await withActor(actor, () => generate()); + expect(batchSpy).toHaveBeenCalledTimes(1); + expect(metered()).toEqual([ + { + usageType: `${PROVIDER}:${MODEL}:input`, + usageAmount: 18, + costOverride: 18 * INPUT_RATE, + }, + { + usageType: `${PROVIDER}:${MODEL}:output:video`, + usageAmount: 57920, + costOverride: 57920 * VIDEO_RATE, + }, + { + usageType: `${PROVIDER}:${MODEL}:output:text`, + usageAmount: 58539 - 57920 + 418, + costOverride: (58539 - 57920 + 418) * TEXT_RATE, + }, + ]); + }); + + it('settles verified 4K usage beyond the requested output cap', async () => { + const { actor } = await makeUser(); + reply( + interaction({ + usage: { + total_input_tokens: 14, + total_output_tokens: 174289, + output_tokens_by_modality: [ + { modality: 'video', tokens: 173760 }, + ], + total_thought_tokens: 175, + }, + }), + ); + await withActor(actor, () => generate({ size: '4k' })); + expect(metered().map((u) => [u.usageAmount, u.costOverride])).toEqual([ + [14, 14 * INPUT_RATE], + [173760, 173760 * VIDEO_RATE], + [704, 704 * TEXT_RATE], + ]); + }); + + it('charges image and text input at one rate, and settles above the estimated reservation', async () => { + const { actor } = await makeUser(); + reply( + interaction({ + usage: { + total_input_tokens: 1107, + input_tokens_by_modality: + USAGE_360P_IMAGE.input_tokens_by_modality, + total_output_tokens: 301_000, + output_tokens_by_modality: [ + { modality: 'video', tokens: 300_000 }, + ], + total_thought_tokens: 548, + }, + }), + ); + await withActor(actor, () => + generate({ input_reference: pngDataUri, size: '360x640' }), + ); + expect(metered().map((u) => [u.usageAmount, u.costOverride])).toEqual([ + [1107, 1107 * INPUT_RATE], + [300_000, 300_000 * VIDEO_RATE], + [1548, 1548 * TEXT_RATE], + ]); + }); + + it('charges the observed 360p image-to-video usage', async () => { + const { actor } = await makeUser(); + reply(interaction({ usage: USAGE_360P_IMAGE })); + await withActor(actor, () => + generate({ input_reference: pngDataUri, size: '360x640' }), + ); + expect(metered().map((u) => u.usageAmount)).toEqual([ + 1107, + 19310, + 19960 - 19310 + 220, + ]); + }); + + it('sums video across modality entries and treats missing thinking as none', async () => { + const { actor } = await makeUser(); + reply( + interaction({ + usage: { + total_input_tokens: 18, + total_output_tokens: 1000, + output_tokens_by_modality: [ + { modality: 'VIDEO', tokens: 600 }, + { modality: 'video', tokens: 300 }, + { modality: 'audio', tokens: 40 }, + ], + }, + }), + ); + await withActor(actor, () => generate()); + expect(metered().map((u) => u.usageAmount)).toEqual([18, 900, 100]); + }); + + it('leaves out a text line when all output is video and there is no thinking', async () => { + const { actor } = await makeUser(); + reply( + interaction({ + usage: { + total_input_tokens: 18, + total_output_tokens: 900, + output_tokens_by_modality: [ + { modality: 'video', tokens: 900 }, + ], + total_thought_tokens: 0, + }, + }), + ); + await withActor(actor, () => generate()); + expect(metered().map((u) => u.usageType)).toEqual([ + `${PROVIDER}:${MODEL}:input`, + `${PROVIDER}:${MODEL}:output:video`, + ]); + }); + + it.each([ + ['missing usage', undefined], + ['null usage', null], + [ + 'no output modality breakdown', + { ...USAGE_720P, output_tokens_by_modality: undefined }, + ], + [ + 'no video tokens for the returned video', + { + ...USAGE_720P, + output_tokens_by_modality: [{ modality: 'text', tokens: 10 }], + }, + ], + [ + 'no video tokens and no breakdown', + { + ...USAGE_720P, + total_output_tokens: 0, + output_tokens_by_modality: undefined, + }, + ], + [ + 'a missing input total', + { ...USAGE_720P, total_input_tokens: undefined }, + ], + [ + 'a missing output total', + { ...USAGE_720P, total_output_tokens: undefined }, + ], + [ + 'a negative video count', + { + ...USAGE_720P, + output_tokens_by_modality: [{ modality: 'video', tokens: -1 }], + }, + ], + [ + 'a fractional video count', + { + ...USAGE_720P, + output_tokens_by_modality: [{ modality: 'video', tokens: 1.5 }], + }, + ], + [ + 'a string video count', + { + ...USAGE_720P, + output_tokens_by_modality: [ + { modality: 'video', tokens: '57920' }, + ], + }, + ], + [ + 'video over the output total', + { ...USAGE_720P, total_output_tokens: 57919 }, + ], + [ + 'modalities that sum past the output total', + { + ...USAGE_720P, + output_tokens_by_modality: [ + { modality: 'video', tokens: 57920 }, + { modality: 'text', tokens: 620 }, + ], + }, + ], + [ + 'a negative thinking count', + { ...USAGE_720P, total_thought_tokens: -1 }, + ], + ['a string input count', { ...USAGE_720P, total_input_tokens: '18' }], + [ + 'an unsafe cost', + { + ...USAGE_720P, + total_output_tokens: Number.MAX_SAFE_INTEGER, + output_tokens_by_modality: [ + { modality: 'video', tokens: Number.MAX_SAFE_INTEGER }, + ], + }, + ], + [ + 'an overflowing modality sum', + { + ...USAGE_720P, + total_output_tokens: Number.MAX_SAFE_INTEGER, + output_tokens_by_modality: [ + { modality: 'video', tokens: Number.MAX_SAFE_INTEGER }, + { modality: 'video', tokens: Number.MAX_SAFE_INTEGER }, + ], + }, + ], + ])('fails %s as 502 without fabricating usage', async (_label, usage) => { + const { actor } = await makeUser(); + const { released } = trackHolds(); + reply(interaction({ usage })); + expect(await callerError(() => generate(), actor)).toMatchObject({ + statusCode: 502, + legacyCode: 'upstream_failed', + message: 'Gemini video generation returned unusable usage', + }); + expect(batchSpy).not.toHaveBeenCalled(); + expect(released).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.ts b/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.ts new file mode 100644 index 000000000..0dd30287e --- /dev/null +++ b/src/backend/drivers/ai-video/providers/gemini/GeminiVideoProvider.ts @@ -0,0 +1,621 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { Actor } from '../../../../core/actor.js'; +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import { insufficientCreditsError } from '../../../../services/metering/enforcement.js'; +import type { UsageInput } from '../../../../services/metering/types.js'; +import { normalizeRawBase64ImageString } from '../../../../util/appIcon.js'; +import { + decodeStrictBase64, + sniffImageMime, + sniffVideoMime, +} from '../../../../util/mediaSniff.js'; +import type { AiMeteringService } from '../../../util/aiCostFactor.js'; +import { loadFileInput } from '../../../util/fileInput.js'; +import { + geminiUpstreamFailure, + postGeminiJson, + geminiTokenCount as tokenCount, +} from '../../../util/geminiRest.js'; +import { fetchImageBytes, isHttpUrl } from '../../../util/imageInput.js'; +import type { IGenerateVideoParams, IVideoModel } from '../../types.js'; +import { VideoProvider } from '../VideoProvider.js'; +import { + GEMINI_OMNI_COSTS, + GEMINI_OMNI_MODEL, + GEMINI_VIDEO_GENERATION_MODELS, +} from './models.js'; + +const SAMPLE_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; + +export const GEMINI_OMNI_MAX_PROMPT_CHARS = 4000; +export const GEMINI_OMNI_MAX_IMAGE_BYTES = 5 * 1024 * 1024; +/** Uses the lower input limit returned by the model metadata. */ +export const GEMINI_OMNI_MAX_INPUT_TOKENS = 131_072; +/** Requested output cap; video generation can exceed it. */ +export const GEMINI_OMNI_MAX_OUTPUT_TOKENS = 65_536; +const VIDEO_RESERVATION_ALLOWANCE = 173_760; +const TEXT_RESERVATION_ALLOWANCE = 65_536; +export const GEMINI_OMNI_MAX_RESPONSE_BYTES = 64 * 1024 * 1024; +/** Covers the generation and reading the reply. */ +export const GEMINI_OMNI_TIMEOUT_MS = 10 * 60 * 1000; +export const GEMINI_OMNI_COUNT_TOKENS_MAX_RESPONSE_BYTES = 64 * 1024; +export const GEMINI_OMNI_COUNT_TOKENS_TIMEOUT_MS = 30_000; +/** Outlasts the upstream deadline so the hold stands until usage is recorded. */ +export const GEMINI_OMNI_HOLD_TTL_MS = 11 * 60 * 1000; + +type OmniResolution = '360p' | '720p' | '1080p' | '4k'; +type OmniAspectRatio = '16:9' | '9:16'; + +/** Landscape pixel size of each resolution; portrait swaps the sides. */ +const LANDSCAPE_SIZES: Record = { + '360p': [640, 360], + '720p': [1280, 720], + '1080p': [1920, 1080], + '4k': [3840, 2160], +}; + +const IMAGE_MIME_TYPES = new Set(['image/png', 'image/jpeg', 'image/webp']); + +/** Options other providers take; any value but null is refused. */ +const UNSUPPORTED_OPTIONS = [ + 'seconds', + 'duration', + 'fps', + 'steps', + 'guidance_scale', + 'seed', + 'negative_prompt', + 'output_quality', + 'generate_audio', + 'frame_images', + 'last_frame', + 'reference_images', + 'metadata', +] as const satisfies readonly (keyof IGenerateVideoParams)[]; + +interface OmniSize { + resolution: OmniResolution; + aspectRatio: OmniAspectRatio; +} + +interface OmniRequest extends OmniSize { + prompt: string; + inputReference?: string; +} + +interface InlineImage { + mimeType: string; + data: string; +} + +export interface GeminiVideoDeps { + stores: Parameters[0]; + fs: Parameters[1]; + metering: AiMeteringService; +} + +// -- Gemini wire shapes -- + +type InteractionInput = + | { type: 'image'; mime_type: string; data: string } + | { type: 'text'; text: string }; + +interface InteractionRequest { + model: string; + input: string | InteractionInput[]; + response_format: { + type: 'video'; + resolution: OmniResolution; + aspect_ratio: OmniAspectRatio; + }; + generation_config: { + max_output_tokens: number; + video_config?: { task: 'image_to_video' }; + }; + store: false; + background: false; + stream: false; +} + +interface InteractionModalityTokens { + modality?: string; + tokens?: number; +} + +interface InteractionUsage { + total_input_tokens?: number; + total_output_tokens?: number; + output_tokens_by_modality?: InteractionModalityTokens[]; + total_thought_tokens?: number; +} + +interface InteractionContent { + type?: string; + mime_type?: string; + data?: string; +} + +/** The reply as documented; every field is checked before use. */ +interface Interaction { + status?: string; + steps?: { type?: string; content?: InteractionContent[] }[]; + usage?: InteractionUsage; +} + +type CountTokensPart = + { text: string } | { inlineData: { mimeType: string; data: string } }; + +interface CountTokensRequest { + contents: { parts: CountTokensPart[] }[]; +} + +interface CountTokensReply { + totalTokens?: number; +} + +const badRequest = (message: string): HttpError => + new HttpError(400, message, { legacyCode: 'bad_request' }); + +const isSet = (value: unknown): boolean => + value !== undefined && value !== null; + +/** + * Billable tokens from the reply's usage: input of every modality, video + * output, and the rest of the output plus thinking as text. Undefined when a + * count is missing, malformed, inconsistent or would cost more than a safe + * integer, or when a returned video has no video tokens. + */ +const billedTokens = ( + usage: InteractionUsage | undefined, + returnedVideo: boolean, +): { input: number; video: number; text: number } | undefined => { + if (!usage || typeof usage !== 'object') return undefined; + const input = tokenCount(usage.total_input_tokens); + const output = tokenCount(usage.total_output_tokens); + const thoughts = + usage.total_thought_tokens === undefined + ? 0 + : tokenCount(usage.total_thought_tokens); + // Only an empty output can go without a modality breakdown. + const byModality = + usage.output_tokens_by_modality === undefined && output === 0 + ? [] + : usage.output_tokens_by_modality; + if ( + input === undefined || + output === undefined || + thoughts === undefined || + !Array.isArray(byModality) + ) + return undefined; + + let video = 0; + let listed = 0; + for (const entry of byModality) { + const tokens = tokenCount(entry?.tokens); + if (tokens === undefined) return undefined; + listed += tokens; + if (String(entry.modality).toLowerCase() === 'video') video += tokens; + } + // `total_output_tokens` excludes thinking, so the modalities fit inside it. + if ((returnedVideo && video === 0) || listed > output) return undefined; + + const text = output - video + thoughts; + const costs = GEMINI_OMNI_COSTS; + if ( + !Number.isSafeInteger( + input * costs.input + + video * costs.outputVideo + + text * costs.outputText, + ) + ) + return undefined; + return { input, video, text }; +}; + +/** The pixel size a resolution and aspect ratio produce. */ +const pixelSize = ( + resolution: OmniResolution, + aspectRatio: OmniAspectRatio, +): string => { + const [long, short] = LANDSCAPE_SIZES[resolution]; + return aspectRatio === '16:9' ? `${long}x${short}` : `${short}x${long}`; +}; + +/** Every accepted `WIDTHxHEIGHT`. */ +const SIZES: Record = {}; +for (const resolution of Object.keys(LANDSCAPE_SIZES) as OmniResolution[]) { + for (const aspectRatio of ['16:9', '9:16'] as const) { + SIZES[pixelSize(resolution, aspectRatio)] = { resolution, aspectRatio }; + } +} + +const SIZE_CHOICES = [ + ...Object.keys(LANDSCAPE_SIZES), + ...Object.keys(SIZES), +].join(', '); + +/** + * Gemini Omni video generation through Gemini's Interactions API. One unary + * call returns the clip inline, so the hold is sized from a `countTokens` + * preflight plus the output and thinking allowances, and the reply's usage + * settles it. + */ +export class GeminiVideoProvider extends VideoProvider { + #apiKey: string; + #deps: GeminiVideoDeps; + + constructor(config: { apiKey: string }, deps: GeminiVideoDeps) { + super(); + if (!config.apiKey) { + throw new Error('Gemini video generation requires an API key'); + } + this.#apiKey = config.apiKey; + this.#deps = deps; + } + + getDefaultModel(): string { + return GEMINI_OMNI_MODEL; + } + + async models(): Promise { + return GEMINI_VIDEO_GENERATION_MODELS; + } + + async generate(params: IGenerateVideoParams): Promise { + const request = this.#readRequest(params); + if (params.test_mode) return SAMPLE_VIDEO_URL; + + const actor = Context.get('actor') as Actor | undefined; + if (!actor) { + throw new HttpError(401, 'Authentication required', { + legacyCode: 'unauthorized', + }); + } + const signal = Context.get('abortSignal'); + const image = + request.inputReference === undefined + ? undefined + : await this.#loadImage(actor, request.inputReference, signal); + + const inputTokens = await this.#countTokens( + request.prompt, + image, + signal, + ); + if (inputTokens > GEMINI_OMNI_MAX_INPUT_TOKENS) { + throw new HttpError( + 400, + `Input is ${inputTokens} tokens; Gemini Omni accepts at most ${GEMINI_OMNI_MAX_INPUT_TOKENS}`, + { legacyCode: 'bad_request', code: 'input_too_large' }, + ); + } + + // The video allowance covers verified 4K usage, with a separate text + // and thinking allowance. Neither is a guaranteed spending bound. + const costs = GEMINI_OMNI_COSTS; + const holdAmount = + inputTokens * costs.input + + VIDEO_RESERVATION_ALLOWANCE * costs.outputVideo + + TEXT_RESERVATION_ALLOWANCE * costs.outputText; + const metering = this.#deps.metering; + const usagePrefix = `gemini-video-generation:${GEMINI_OMNI_MODEL}`; + const hold = await metering.reserveAiCredits( + actor, + usagePrefix, + holdAmount, + { ttlMs: GEMINI_OMNI_HOLD_TTL_MS }, + ); + if (!hold) throw insufficientCreditsError(); + + try { + const interaction = await postGeminiJson( + this.#apiKey, + 'interactions', + this.#interactionRequest(request, image), + { + label: 'Gemini video generation', + maxBytes: GEMINI_OMNI_MAX_RESPONSE_BYTES, + timeoutMs: GEMINI_OMNI_TIMEOUT_MS, + signal, + }, + ); + const videoBlock = this.#findVideo(interaction); + + // Usage the reply reports is charged before the reply is checked, + // so a failed or videoless generation still pays for its tokens. + const billed = billedTokens( + interaction.usage, + videoBlock !== undefined, + ); + if (billed) { + const usages: UsageInput[] = [ + { + usageType: `${usagePrefix}:input`, + usageAmount: billed.input, + costOverride: billed.input * costs.input, + }, + { + usageType: `${usagePrefix}:output:video`, + usageAmount: billed.video, + costOverride: billed.video * costs.outputVideo, + }, + { + usageType: `${usagePrefix}:output:text`, + usageAmount: billed.text, + costOverride: billed.text * costs.outputText, + }, + ].filter((usage) => usage.usageAmount > 0); + await metering.batchIncrementUsages(actor, usages); + } + + if (interaction.status !== 'completed') { + throw geminiUpstreamFailure( + 'Gemini video generation did not complete the request', + ); + } + const video = this.#readVideo(videoBlock); + if (!billed) { + throw geminiUpstreamFailure( + 'Gemini video generation returned unusable usage', + ); + } + + return `data:video/mp4;base64,${video}`; + } finally { + await hold.release(); + } + } + + /** Validates every option before any I/O, test mode included. */ + #readRequest(params: IGenerateVideoParams): OmniRequest { + if (params.model !== undefined && params.model !== GEMINI_OMNI_MODEL) { + throw badRequest( + `Unsupported model: ${String(params.model)}. Expected: ${GEMINI_OMNI_MODEL}`, + ); + } + const { prompt } = params; + if (typeof prompt !== 'string' || !prompt.trim()) { + throw badRequest('prompt must be a non-empty string'); + } + if (prompt.length > GEMINI_OMNI_MAX_PROMPT_CHARS) { + throw badRequest( + `prompt must be at most ${GEMINI_OMNI_MAX_PROMPT_CHARS} characters`, + ); + } + for (const key of UNSUPPORTED_OPTIONS) { + if (isSet(params[key])) { + throw badRequest(`${key} is not supported by Gemini Omni`); + } + } + if ( + isSet(params.output_format) && + params.output_format !== 'video/mp4' + ) { + throw badRequest('output_format must be video/mp4'); + } + const inputReference = params.input_reference; + if ( + isSet(inputReference) && + (typeof inputReference !== 'string' || !inputReference.trim()) + ) { + throw badRequest( + 'input_reference must be an image URL, data URI, base64 string or Puter file path', + ); + } + return { + prompt, + ...this.#readSize(params), + ...(isSet(inputReference) + ? { inputReference: inputReference as string } + : {}), + }; + } + + /** + * `size` and `resolution` take a resolution or an exact pixel size; `width` + * with `height` take an exact pixel size. Controls that are given together + * must agree. Defaults to 720p at 16:9. + */ + #readSize(params: IGenerateVideoParams): OmniSize { + const unsupported = () => + badRequest(`size must be one of: ${SIZE_CHOICES}`); + + const named: string[] = []; + for (const value of [params.size, params.resolution]) { + if (!isSet(value)) continue; + if (typeof value !== 'string' || !value.trim()) throw unsupported(); + named.push(value.trim().toLowerCase()); + } + if (named.length === 2 && named[0] !== named[1]) { + throw badRequest('size and resolution must match'); + } + + let resolution: OmniResolution | undefined; + let aspectRatio: OmniAspectRatio | undefined; + if (named.length > 0) { + const wanted = named[0]!; + if (Object.hasOwn(LANDSCAPE_SIZES, wanted)) { + resolution = wanted as OmniResolution; + } else if (Object.hasOwn(SIZES, wanted)) { + ({ resolution, aspectRatio } = SIZES[wanted]!); + } else { + throw unsupported(); + } + } + + const { width, height } = params; + if (isSet(width) || isSet(height)) { + const pixels = `${width}x${height}`; + if ( + !Number.isInteger(width) || + !Number.isInteger(height) || + !Object.hasOwn(SIZES, pixels) + ) { + throw badRequest( + `width and height must together be one of: ${Object.keys(SIZES).join(', ')}`, + ); + } + const exact = SIZES[pixels]!; + if ( + (resolution && resolution !== exact.resolution) || + (aspectRatio && aspectRatio !== exact.aspectRatio) + ) { + throw badRequest('width and height must match size'); + } + ({ resolution, aspectRatio } = exact); + } + + return { + resolution: resolution ?? '720p', + aspectRatio: aspectRatio ?? '16:9', + }; + } + + /** The first-frame image, read within the caller's access and size cap. */ + async #loadImage( + actor: Actor, + input: string, + signal: AbortSignal | undefined, + ): Promise { + const bytes = isHttpUrl(input) + ? ( + await fetchImageBytes(input, { + maxBytes: GEMINI_OMNI_MAX_IMAGE_BYTES, + signal, + }) + ).bytes + : ( + await loadFileInput( + this.#deps.stores, + this.#deps.fs, + actor, + normalizeRawBase64ImageString(input), + { maxBytes: GEMINI_OMNI_MAX_IMAGE_BYTES }, + ) + ).buffer; + const mimeType = sniffImageMime(bytes); + if (!mimeType || !IMAGE_MIME_TYPES.has(mimeType)) { + throw badRequest( + 'input_reference must be a PNG, JPEG or WebP image', + ); + } + return { mimeType, data: bytes.toString('base64') }; + } + + /** Input tokens for the prompt and image, counted as they will be sent. */ + async #countTokens( + prompt: string, + image: InlineImage | undefined, + signal: AbortSignal | undefined, + ): Promise { + const parts: CountTokensPart[] = image + ? [{ inlineData: image }, { text: prompt }] + : [{ text: prompt }]; + const request: CountTokensRequest = { contents: [{ parts }] }; + const reply = await postGeminiJson( + this.#apiKey, + `models/${GEMINI_OMNI_MODEL}:countTokens`, + request, + { + label: 'Gemini token count', + maxBytes: GEMINI_OMNI_COUNT_TOKENS_MAX_RESPONSE_BYTES, + timeoutMs: GEMINI_OMNI_COUNT_TOKENS_TIMEOUT_MS, + signal, + }, + ); + const tokens = tokenCount(reply.totalTokens); + if (!tokens) { + throw geminiUpstreamFailure( + 'Gemini token count returned an unusable count', + ); + } + return tokens; + } + + #interactionRequest( + request: OmniRequest, + image: InlineImage | undefined, + ): InteractionRequest { + const body: InteractionRequest = { + model: GEMINI_OMNI_MODEL, + input: image + ? [ + { + type: 'image', + mime_type: image.mimeType, + data: image.data, + }, + { type: 'text', text: request.prompt }, + ] + : request.prompt, + response_format: { + type: 'video', + resolution: request.resolution, + aspect_ratio: request.aspectRatio, + }, + generation_config: { + max_output_tokens: GEMINI_OMNI_MAX_OUTPUT_TOKENS, + }, + store: false, + background: false, + stream: false, + }; + if (image) { + body.generation_config.video_config = { task: 'image_to_video' }; + } + return body; + } + + /** The last video block the model returned, not yet checked. */ + #findVideo(interaction: Interaction): InteractionContent | undefined { + const steps = Array.isArray(interaction.steps) ? interaction.steps : []; + return steps + .flatMap((step) => + step?.type === 'model_output' && Array.isArray(step.content) + ? step.content + : [], + ) + .filter((content) => content?.type === 'video') + .at(-1); + } + + /** The base64 MP4 held by the video block. */ + #readVideo(video: InteractionContent | undefined): string { + if (!video) { + throw new HttpError(400, 'Gemini did not return a video', { + legacyCode: 'upstream_bad_request', + fields: { provider: 'gemini' }, + }); + } + const bytes = + video.mime_type === 'video/mp4' && typeof video.data === 'string' + ? decodeStrictBase64(video.data) + : null; + if (!bytes || sniffVideoMime(bytes) !== 'video/mp4') { + throw geminiUpstreamFailure( + 'Gemini video generation returned a malformed video', + ); + } + return video.data!; + } +} diff --git a/src/backend/drivers/ai-video/providers/gemini/models.ts b/src/backend/drivers/ai-video/providers/gemini/models.ts new file mode 100644 index 000000000..f26fb2450 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/gemini/models.ts @@ -0,0 +1,58 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IVideoModel } from '../../types.js'; + +export const GEMINI_OMNI_MODEL = 'gemini-omni-1.1-flash'; + +/** + * µ¢ per token (USD-cents per 1M tokens). Input of every modality shares one + * rate; thinking and non-video output are billed as text. + */ +export interface GeminiOmniCosts { + input: number; + outputText: number; + outputVideo: number; +} + +export const GEMINI_OMNI_COSTS: GeminiOmniCosts = { + input: 150, // $1.50 per 1M tokens + outputText: 900, // $9.00 per 1M tokens + outputVideo: 1750, // $17.50 per 1M tokens +}; + +// Billed per token, so no `durationSeconds` or `dimensions`: the driver would +// otherwise inject a duration and rewrite sizes this model does not take. +export const GEMINI_VIDEO_GENERATION_MODELS: IVideoModel[] = [ + { + id: GEMINI_OMNI_MODEL, + puterId: `google:google/${GEMINI_OMNI_MODEL}`, + name: 'Gemini Omni 1.1 Flash', + costs_currency: 'usd-cents', + output_cost_key: 'output:video', + // Cents per token, matching the usage types the provider records. + costs: { + input: GEMINI_OMNI_COSTS.input / 1_000_000, + 'output:text': GEMINI_OMNI_COSTS.outputText / 1_000_000, + 'output:video': GEMINI_OMNI_COSTS.outputVideo / 1_000_000, + }, + promptLength: { min: 1, max: 4000 }, + defaultUsageKey: `gemini-video-generation:${GEMINI_OMNI_MODEL}:output:video`, + }, +]; diff --git a/src/backend/drivers/util/geminiRest.ts b/src/backend/drivers/util/geminiRest.ts index e27526e8b..6d7c493e8 100644 --- a/src/backend/drivers/util/geminiRest.ts +++ b/src/backend/drivers/util/geminiRest.ts @@ -88,3 +88,8 @@ export async function postGeminiJson( } return parsed as T; } + +export const geminiTokenCount = (value: unknown): number | undefined => + typeof value === 'number' && Number.isSafeInteger(value) && value >= 0 + ? value + : undefined; diff --git a/src/docs/src/AI/txt2vid.md b/src/docs/src/AI/txt2vid.md index 96b46f999..a295da777 100644 --- a/src/docs/src/AI/txt2vid.md +++ b/src/docs/src/AI/txt2vid.md @@ -1,10 +1,10 @@ --- title: puter.ai.txt2vid() -description: Generate short video clips from text or reference images with Wan, Seedance, Veo and other models through Puter.js. +description: Generate short video clips from text or reference images with Wan, Seedance, Veo, Gemini Omni and other models through Puter.js. platforms: [websites, apps, nodejs, workers] --- -Given a prompt, generate a short video clip using AI. Puter routes the request to one of two upstream providers (Together AI, BytePlus Seedance) based on the model you pick, waits for the clip to render, and resolves with a ready-to-play video. Every model can also start from an image you supply (image-to-video), and the same option names work across providers. +Given a prompt, generate a short video clip using AI. Puter routes the request to one of three upstream providers (Together AI, BytePlus Seedance, Google Gemini) based on the model you pick, waits for the clip to render, and resolves with a ready-to-play video. Every model can also start from an image you supply (image-to-video), and the same option names work across providers. ## Syntax @@ -34,9 +34,9 @@ Additional settings for the generation request. |--------|------|-------------| | `prompt` | `String` | Text description for the video generation | | `model` | `String` | Video model to use. Defaults to `'togetherai:wan-ai/wan2.7-t2v'` (Wan 2.7 text-to-video). See [Choosing a model](#choosing-a-model) | -| `provider` | `String` | Pin the request to one provider: `'together-video-generation'` or `'byteplus-video-generation'`. Only needed when a model id exists on more than one provider | -| `seconds` | `Number` | Target clip length in seconds. Each model supports a fixed set of durations; an unsupported value falls back to the model default instead of failing. `duration` is an alias | -| `size` | `String` | Output size as `'WIDTHxHEIGHT'` (e.g. `'1280x720'`) on every provider. Models that work in resolution tiers take the tier of the shorter side plus the aspect ratio, and also accept the tier directly (`'720p'`). An unsupported value falls back to the model default. `resolution` is an alias | +| `provider` | `String` | Pin the request to one provider: `'together-video-generation'`, `'byteplus-video-generation'` or `'gemini-video-generation'`. Only needed when a model id exists on more than one provider | +| `seconds` | `Number` | Target clip length in seconds. Each model supports a fixed set of durations; an unsupported value falls back to the model default instead of failing. Gemini Omni has no length control and rejects this option. `duration` is an alias | +| `size` | `String` | Output size as `'WIDTHxHEIGHT'` (e.g. `'1280x720'`) on every provider. Models that work in resolution tiers take the tier of the shorter side plus the aspect ratio, and also accept the tier directly (`'720p'`). An unsupported value falls back to the model default, except on Gemini Omni, which rejects it. `resolution` is an alias | | `width`, `height` | `Number` | Output size in pixels on Together AI models sized in pixels; the aspect ratio on Seedance and Wan 2.7. Filled in from `size` when you leave them out | | `input_reference` | `String` | Image the clip starts from (image-to-video): a public URL, a `data:` URI or raw base64, on every provider | | `last_frame` | `String` | Image the clip ends on, same formats. Requires `input_reference` on Seedance | @@ -55,7 +55,7 @@ When you pass no model, Puter uses Wan 2.7 text-to-video on Together AI (`wan2.7 Google's Veo 3.1 preview models (`veo-3.1-generate-preview`, `veo-3.1-fast-generate-preview`, `veo-3.1-lite-generate-preview`) are no longer offered: Google deprecated them with a shutdown on October 22, 2026, and requests for those ids fail with `Model not found`. Veo 3.1 and Veo 3.1 Lite remain available through Together AI (see below). -`seconds` and `size` are normalized rather than rejected: a value the model does not offer is replaced by the model default, which is the first value listed in the tables below and also what you get when you leave the option out. +On Together AI and Seedance, `seconds` and `size` are normalized rather than rejected: a value the model does not offer is replaced by the model default, which is the first value listed in the tables below and also what you get when you leave the option out. Gemini Omni rejects a value it doesn't offer with `bad_request` instead (see [Gemini Omni options](#gemini-omni-options)). #### Image inputs @@ -65,6 +65,7 @@ Google's Veo 3.1 preview models (`veo-3.1-generate-preview`, `veo-3.1-fast-gener |----------|-------------|------------|------------------| | Together AI | Models with a `first` keyframe | Models with a `last` keyframe | Model-dependent (Seedance 2.5: up to 30) | | BytePlus Seedance | Yes | Yes, with `input_reference` | Seedance 2.0: up to 9; Seedance 2.5: up to 30; cannot be combined with frames | +| Google Gemini | Yes, one PNG, JPEG or WebP image up to 5 MiB | No | No | #### Together AI options @@ -157,6 +158,41 @@ An invalid combination (a last frame without a first frame, reference images tog For more details, see the [BytePlus ModelArk video generation reference](https://docs.byteplus.com/en/docs/ModelArk/1520757). +#### Gemini Omni options + +Available when using `gemini-omni-1.1-flash` (provider `'gemini-video-generation'`), which also answers to `google/gemini-omni-1.1-flash`. It's offered when the Puter deployment has Gemini configured. It generates a clip from a prompt, or from a prompt and one starting image. The model picks the clip length and renders at 24 frames per second. + +| `size` | 16:9 (default) | 9:16 | +|--------|----------------|------| +| `'360p'` | `640x360` | `360x640` | +| `'720p'` (default) | `1280x720` | `720x1280` | +| `'1080p'` | `1920x1080` | `1080x1920` | +| `'4k'` | `3840x2160` | `2160x3840` | + +Google produces 1080p and 4K by upscaling. Pass the resolution on its own for 16:9, or the exact size for either orientation. + +| Option | Type | Description | +|--------|------|-------------| +| `prompt` | `String` | Up to 4,000 characters | +| `size` | `String` | A resolution or exact size from the table above, case-insensitive. Defaults to `'720p'` at 16:9. `resolution` is an alias, and must match `size` when you pass both | +| `width`, `height` | `Number` | Together, an exact size from the table above. They must match `size` when you pass it | +| `input_reference` | `String` | The image the clip starts from: a public URL, a `data:` URI, raw base64, or the path of a file in the user's Puter filesystem. PNG, JPEG or WebP, up to 5 MiB | +| `output_format` | `String` | Only `'video/mp4'` | + +Gemini Omni rejects `seconds`, `duration`, `fps`, `steps`, `guidance_scale`, `seed`, `negative_prompt`, `output_quality`, `generate_audio`, `frame_images`, `last_frame`, `reference_images` and `metadata` with `bad_request`, test mode included, since it has no matching control. + +Gemini Omni is priced per token. Input costs $1.50 per million tokens. Video output costs $17.50 per million tokens. Other output and the model's thinking cost $9 per million tokens. The prompt and image together are limited to 131,072 input tokens. Other limits are listed in [Rate limits and quotas](/rate-limits-and-quotas/#video-generation). + +```html;ai-txt2vid-gemini + + +``` + Any properties not set fall back to provider defaults. #### Saving to Puter filesystem @@ -179,8 +215,9 @@ Every successful generation is charged to the user's AI credits according to the - **Seedance** is priced per second. If the balance cannot cover the requested length, Puter shortens the clip to the longest duration the model supports that the balance does cover, and rejects with `insufficient_funds` only when even the shortest clip is unaffordable. The response does not flag a shortened clip, so read `video.duration` once metadata has loaded if the exact length matters. - **Together AI** prices its older models per clip, so those requests are either accepted at the model's duration or rejected with `insufficient_funds`. Its per-second models (marked in the Together AI table), including the default Wan 2.7, are shortened like Seedance, and the final charge is the amount Together AI reports for the job. +- **Gemini Omni** is priced per token and has no length control, so it's never shortened. Before generating, Puter counts the input tokens and holds an estimate of about $3.63 plus the input. The estimate covers 173,760 video tokens and 65,536 text and thinking tokens. A balance that can't cover it fails with `insufficient_funds` before generation starts. The charge is for the tokens Google reports for the clip. It can be higher than the estimate, because Google can go past the requested output limit. When Google's reply reports token usage but fails or has no usable clip, those tokens are charged and the request still fails. -A request that fails or times out is not charged. The request and concurrency limits that apply to every AI call are listed in [Rate limits and quotas](/rate-limits-and-quotas/). +Apart from that Gemini Omni case, a request that fails or times out is not charged. The request and concurrency limits that apply to every AI call are listed in [Rate limits and quotas](/rate-limits-and-quotas/). #### How long it takes @@ -190,7 +227,7 @@ Video generation is slow: expect anywhere from tens of seconds to several minute A `Promise` that resolves to an `HTMLVideoElement` (in browsers) that you can append to the DOM straight away: -- `src` is the clip's URL: an `https:` URL on the provider's storage (Together AI or Seedance). Provider URLs are temporary; keep a copy with `puter_output_path` if you need the clip later. +- `src` is the clip's URL: an `https:` URL on the provider's storage (Together AI or Seedance), or a `data:video/mp4;base64,…` URI holding the whole clip (Gemini Omni). Provider URLs are temporary; keep a copy with `puter_output_path` if you need the clip later. - `controls` is enabled and `preload` is `"metadata"`. - The `data-source` attribute carries the same URL, and `data-mime-type` the MIME type when it is known (for example `video/mp4`). - `String(video)` returns the URL, so the element can be dropped into a template or passed to `fetch()`. @@ -210,14 +247,14 @@ A rejection carries the error body exactly as the backend sent it, or `{ message | `message` | Human-readable reason. `error` carries the same text for older clients. | | `code` | Stable error code; see the table below. | | `errorCode` | A more specific code alongside a general `code`. Today the only value is `moderation_flagged`. | -| `provider` | Which upstream handled the request: `together` or `byteplus`. Present on errors raised while a job was running. | +| `provider` | Which upstream handled the request: `together`, `byteplus` or `gemini`. Present on errors raised while a job was running. | | `upstreamCode` | The provider's own error code, when it gave one. | | `upstreamStatus` | The HTTP status the provider returned, when it rejected the request before a job started. | | Code | Meaning | | --- | --- | | `prompt_required` | Raised by the SDK before any request is made: the call had no prompt. | -| `bad_request` (without `errorCode`) | Puter rejected the request before contacting a provider: an unknown `model` (`Model not found: …`), an invalid combination of image inputs, or an image URL that could not be fetched. `message` says which. Arrives as HTTP 400. | +| `bad_request` (without `errorCode`) | Puter rejected the request before contacting a provider: an unknown `model` (`Model not found: …`), an invalid combination of image inputs, an option Gemini Omni doesn't support, or an image URL that could not be fetched. `message` says which. Arrives as HTTP 400. | | `access_denied` | `puter_output_path` points somewhere the caller may not write. Arrives as HTTP 403, before any credits are spent. `cannot_write_to_root` (HTTP 400) is the same check for a path directly under `/`. | | `upstream_timeout` | The provider did not finish the clip within the ten minutes Puter waits for it, or stopped answering. Arrives as HTTP 504. The request itself was fine; retry it, ideally with a shorter clip or a faster model. | | `errorCode: moderation_flagged` | The provider's content filter refused the prompt or removed the generated video. Arrives as HTTP 400, with `code: bad_request`. Change the prompt rather than retrying it as-is. | diff --git a/src/docs/src/examples.js b/src/docs/src/examples.js index 06a7929d2..d94ae8e44 100644 --- a/src/docs/src/examples.js +++ b/src/docs/src/examples.js @@ -277,6 +277,12 @@ const examples = [ slug: 'ai-txt2vid', source: '/playground/examples/ai-txt2vid.html', }, + { + title: 'Text to Video with Gemini Omni', + description: 'Generate a video clip with Gemini Omni through Puter.js.', + slug: 'ai-txt2vid-gemini', + source: '/playground/examples/ai-txt2vid-gemini.html', + }, { title: 'Text to Video with options', description: 'Generate videos with custom options using Puter.js AI API. Run and experiment with advanced text-to-video parameters in the playground.', diff --git a/src/docs/src/playground/examples/ai-txt2vid-gemini.html b/src/docs/src/playground/examples/ai-txt2vid-gemini.html new file mode 100644 index 000000000..8f5df9394 --- /dev/null +++ b/src/docs/src/playground/examples/ai-txt2vid-gemini.html @@ -0,0 +1,11 @@ + + + + + + diff --git a/src/docs/src/rate-limits-and-quotas.md b/src/docs/src/rate-limits-and-quotas.md index 545e67dd8..510b7fa61 100644 --- a/src/docs/src/rate-limits-and-quotas.md +++ b/src/docs/src/rate-limits-and-quotas.md @@ -47,7 +47,7 @@ For AI: - A streamed response that stops before the model reports its token counts (an upstream error mid-stream, for example) is charged on an estimate of what was streamed. A request that produced no output is free. - AI model rates can change. Check the [AI model directory](https://developer.puter.com/ai/models/) for current pricing. -- In-flight chat, image generation, video generation, text-to-speech, speech-to-text and voice-changer requests reserve the most they could cost until they finish, then settle at their real cost. If several expensive requests start at once and the balance can't cover all of them, the later ones fail with `402 insufficient_funds`; a per-second video is first shortened to what the balance still covers. +- In-flight chat, image generation, video generation, text-to-speech, speech-to-text and voice-changer requests reserve the most they could cost until they finish, then settle at their real cost. If several expensive requests start at once and the balance can't cover all of them, the later ones fail with `402 insufficient_funds`; a per-second video is first shortened to what the balance still covers. Gemini Omni video is the exception. Its reservation is an estimate, because Google can return more output tokens than requested, and it settles at the tokens actually used even when they cost more (see [Video generation](#video-generation)). ## Rate limits @@ -116,6 +116,24 @@ A web search is metered at a flat per-request rate in addition to the tokens it See [`txt2img()`](/AI/txt2img) for provider-specific options and supported models. +### Video generation + +[`puter.ai.txt2vid()`](/AI/txt2vid) waits up to 10 minutes for a clip. These limits apply to Gemini Omni (`gemini-omni-1.1-flash`) on top of the AI limits above: + +| Limit | Value | +|-------|-------| +| Prompt | 4,000 characters | +| Starting image | One PNG, JPEG or WebP image, 5 MiB. A URL is fetched with a 30-second timeout | +| Input tokens, prompt and image together | 131,072 | +| Requested output tokens | 65,536 | +| Input token count | 30-second timeout, 64 KiB reply | +| Generation | 10-minute timeout, reply and clip included. 64 MiB reply | +| Reservation | Counted input tokens, plus 173,760 video tokens and 65,536 text and thinking tokens, about $3.63 plus input. Held for up to 11 minutes | + +A longer prompt, a larger image, or input over the token limit fails with `bad_request` (`413 input_too_large` for an image file or `data:` URI over 5 MiB) before generation starts. A reply over 64 MiB fails with `502 upstream_failed`, and a generation that runs past 10 minutes fails with `504 upstream_timeout`. Neither is charged. + +The reservation is a conservative estimate based on verified generation usage, not a maximum charge. Google can return more output than the requested 65,536 tokens, and the charge is for the input, video, output and thinking tokens Google reports, even when that's more than the reservation. A reply that reports token usage is charged for it even when the reply failed or has no usable clip. + ### Remote chat image inlining For chat providers that require remote image URLs to be inlined (currently Moonshot): diff --git a/src/puter-js/src/modules/ai/types.js b/src/puter-js/src/modules/ai/types.js index 3624af25c..765d4b211 100644 --- a/src/puter-js/src/modules/ai/types.js +++ b/src/puter-js/src/modules/ai/types.js @@ -336,22 +336,28 @@ * * @typedef {Object} Txt2VidOptions * @property {string} [prompt] Text description of the clip. - * @property {string} [provider] Pin the request to one provider: `'together-video-generation'` or - * `'byteplus-video-generation'`. Defaults to the provider that owns `model`, or Together AI when - * neither is given. + * @property {string} [provider] Pin the request to one provider: `'together-video-generation'`, + * `'byteplus-video-generation'` or `'gemini-video-generation'`. Defaults to the provider that owns + * `model`, or Together AI when neither is given. * @property {string} [driver] Same effect as `provider`. * @property {string} [model] Video model id (provider-specific). Defaults to - * `'togetherai:wan-ai/wan2.7-t2v'` (Wan 2.7 text-to-video). + * `'togetherai:wan-ai/wan2.7-t2v'` (Wan 2.7 text-to-video). `'gemini-omni-1.1-flash'` selects + * Gemini Omni, which rejects `seconds`, `duration`, `fps`, `steps`, `guidance_scale`, `seed`, + * `negative_prompt`, `output_quality`, `generate_audio`, `frame_images`, `last_frame`, + * `reference_images` and `metadata`, and takes a prompt of at most 4,000 characters. * @property {number} [seconds] Clip length in seconds. A value the model does not offer falls back - * to the model default. + * to the model default. Gemini Omni has no length control and rejects it. * @property {number} [duration] Alias of `seconds`. * @property {boolean} [test_mode] When `true`, returns a sample clip without using credits. * @property {string} [size] Output size as `'WIDTHxHEIGHT'` on every provider (tier-based models map - * it to the tier of the shorter side plus an aspect ratio), or a tier such as `'720p'` for Seedance - * and Wan 2.7. + * it to the tier of the shorter side plus an aspect ratio), or a tier such as `'720p'` for Seedance, + * Wan 2.7 and Gemini Omni. Gemini Omni takes `'360p'`, `'720p'` (default), `'1080p'` or `'4k'`, or + * the exact 16:9 or 9:16 size of one of them (`'640x360'`, `'360x640'`, `'1280x720'`, …), and + * rejects any other value. * @property {string} [resolution] Alias of `size`. * @property {string} [input_reference] First-frame image for image-to-video: a URL, data URI or raw - * base64, on every provider. + * base64, on every provider. Gemini Omni also takes a Puter file path, and accepts one PNG, JPEG or + * WebP image up to 5 MiB. * @property {string} [last_frame] Last-frame image, same formats as `input_reference`. * @property {string[]} [reference_images] Subject/style reference images (URL, data URI or base64). * Seedance 2.0: up to 9; Seedance 2.5: up to 30; Together: model-dependent. @@ -360,12 +366,14 @@ * and Together models with audio). Defaults to `true` on Seedance. * @property {number} [seed] Random seed (Together, Seedance 1.x). * @property {number} [width] Output width in pixels on Together models sized in pixels; with `height`, - * selects the aspect ratio on Seedance and Wan 2.7. Filled in from `size` when omitted. + * selects the aspect ratio on Seedance and Wan 2.7. Filled in from `size` when omitted. On Gemini + * Omni, `width` and `height` together must be one of its exact sizes and match `size` when both are + * given. * @property {number} [height] Output height in pixels; see `width`. * @property {number} [fps] TogetherAI. * @property {number} [steps] TogetherAI. * @property {number} [guidance_scale] TogetherAI. - * @property {string} [output_format] TogetherAI. + * @property {string} [output_format] TogetherAI. Gemini Omni accepts only `'video/mp4'`. * @property {number} [output_quality] TogetherAI. * @property {Array<{ input_image: string, frame: number }>} [frame_images] TogetherAI: keyframe images * for image-to-video. diff --git a/src/puter-js/tests/api/suites/ai.suite.ts b/src/puter-js/tests/api/suites/ai.suite.ts index cd5c0ba28..3ee68586c 100644 --- a/src/puter-js/tests/api/suites/ai.suite.ts +++ b/src/puter-js/tests/api/suites/ai.suite.ts @@ -29,6 +29,8 @@ const TINY_AUDIO = 'data:audio/mp3;base64,QUJD'; const TINY_PNG = 'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=='; const TINY_MP4 = 'data:video/mp4;base64,QUJD'; +/** What every video provider returns in test mode. */ +const SAMPLE_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; /** The `{ message, code }` shape both client-side and driver rejections carry. */ type SdkError = { message?: string; code?: string }; @@ -1228,6 +1230,125 @@ export default suite('ai', { t.assert.equal(foreign.code, 'access_denied'); }, + // Gemini Omni registers only with a Gemini key; test mode never calls it. + + 'txt2vid on gemini omni returns the sample clip in test mode': { + requires: ['ai.gemini'], + fn: async (t) => { + useApiToken(t); + const video = await t.puter.ai.txt2vid( + { + prompt: 'A stationary red ball.', + model: 'gemini-omni-1.1-flash', + size: '360x640', + input_reference: TINY_PNG, + }, + true, + ); + t.assert.equal(String(video), SAMPLE_VIDEO_URL); + t.assert.equal(video.src, SAMPLE_VIDEO_URL); + }, + }, + + 'txt2vid reaches gemini omni through its model aliases and driver name': { + requires: ['ai.gemini'], + fn: async (t) => { + useApiToken(t); + // Gemini Omni refuses `seconds` even in test mode, which proves + // where the call landed. + for (const options of [ + { model: 'google/gemini-omni-1.1-flash' }, + { model: 'google:google/gemini-omni-1.1-flash' }, + { driver: 'gemini-video-generation' }, + { provider: 'gemini-video-generation' }, + ]) { + const error = await errorOf(t, () => + t.puter.ai.txt2vid({ + prompt: 'a cat', + seconds: 4, + test_mode: true, + ...options, + }), + ); + t.assert.equal(error.code, 'bad_request'); + t.assert.equal( + error.message, + 'seconds is not supported by Gemini Omni', + `${JSON.stringify(options)} should reach Gemini Omni`, + ); + } + const video = await t.puter.ai.txt2vid({ + prompt: 'a cat', + driver: 'gemini-video-generation', + test_mode: true, + }); + t.assert.equal(String(video), SAMPLE_VIDEO_URL); + }, + }, + + 'txt2vid on gemini omni rejects duration and unsupported sizes': { + requires: ['ai.gemini'], + fn: async (t) => { + useApiToken(t); + const duration = await errorOf(t, () => + t.puter.ai.txt2vid({ + prompt: 'a cat', + model: 'gemini-omni-1.1-flash', + duration: 4, + test_mode: true, + }), + ); + t.assert.equal( + duration.message, + 'seconds is not supported by Gemini Omni', + ); + for (const size of ['480p', '1024x576']) { + const error = await errorOf(t, () => + t.puter.ai.txt2vid({ + prompt: 'a cat', + model: 'gemini-omni-1.1-flash', + size, + test_mode: true, + }), + ); + t.assert.equal(error.code, 'bad_request'); + t.assert.ok( + (error.message ?? '').startsWith('size must be one of'), + `size ${size} should be refused, got ${error.message}`, + ); + } + }, + }, + + 'the video model catalog lists gemini omni with per-token rates': { + requires: ['ai.gemini'], + fn: async (t) => { + const res = await fetch( + `${t.env.apiOrigin}/puterai/video/models/details?provider=gemini-video-generation`, + ); + t.assert.equal(res.status, 200); + const { models } = (await res.json()) as { + models: Array<{ + id?: string; + costs?: Record; + output_cost_key?: string; + durationSeconds?: unknown; + dimensions?: unknown; + }>; + }; + const omni = models.find((m) => m.id === 'gemini-omni-1.1-flash'); + t.assert.ok(omni, `catalog should list Gemini Omni, got ${JSON.stringify(models)}`); + t.assert.deepEqual(omni!.costs, { + input: 0.00015, + 'output:text': 0.0009, + 'output:video': 0.00175, + }); + t.assert.equal(omni!.output_cost_key, 'output:video'); + t.assert.equal(omni!.durationSeconds, undefined); + t.assert.equal(omni!.dimensions, undefined); + }, + }, + // -- txt2speech -------------------------------------------------- 'txt2speech rejects a call with no text': async (t) => {