Add Gemini Omni video generation with token metering (#4235)

* Add Gemini Omni video generation with token metering

* Keep explicit Gemini video model selection within provider

* Settle reported Gemini video usage before validating output

---------

Co-authored-by: 404oops <oops404@util.404oops.com>

src/puter-js/src/modules/ai/types.js

@@ -336,22 +336,28 @@
  *
  * @typedef {Object} Txt2VidOptions
  * @property {string} [prompt] Text description of the clip.
- * @property {string} [provider] Pin the request to one provider: `'together-video-generation'` or
- * `'byteplus-video-generation'`. Defaults to the provider that owns `model`, or Together AI when
- * neither is given.
+ * @property {string} [provider] Pin the request to one provider: `'together-video-generation'`,
+ * `'byteplus-video-generation'` or `'gemini-video-generation'`. Defaults to the provider that owns
+ * `model`, or Together AI when neither is given.
  * @property {string} [driver] Same effect as `provider`.
  * @property {string} [model] Video model id (provider-specific). Defaults to
- * `'togetherai:wan-ai/wan2.7-t2v'` (Wan 2.7 text-to-video).
+ * `'togetherai:wan-ai/wan2.7-t2v'` (Wan 2.7 text-to-video). `'gemini-omni-1.1-flash'` selects
+ * Gemini Omni, which rejects `seconds`, `duration`, `fps`, `steps`, `guidance_scale`, `seed`,
+ * `negative_prompt`, `output_quality`, `generate_audio`, `frame_images`, `last_frame`,
+ * `reference_images` and `metadata`, and takes a prompt of at most 4,000 characters.
  * @property {number} [seconds] Clip length in seconds. A value the model does not offer falls back
- * to the model default.
+ * to the model default. Gemini Omni has no length control and rejects it.
  * @property {number} [duration] Alias of `seconds`.
  * @property {boolean} [test_mode] When `true`, returns a sample clip without using credits.
  * @property {string} [size] Output size as `'WIDTHxHEIGHT'` on every provider (tier-based models map
- * it to the tier of the shorter side plus an aspect ratio), or a tier such as `'720p'` for Seedance
- * and Wan 2.7.
+ * it to the tier of the shorter side plus an aspect ratio), or a tier such as `'720p'` for Seedance,
+ * Wan 2.7 and Gemini Omni. Gemini Omni takes `'360p'`, `'720p'` (default), `'1080p'` or `'4k'`, or
+ * the exact 16:9 or 9:16 size of one of them (`'640x360'`, `'360x640'`, `'1280x720'`, …), and
+ * rejects any other value.
  * @property {string} [resolution] Alias of `size`.
  * @property {string} [input_reference] First-frame image for image-to-video: a URL, data URI or raw
- * base64, on every provider.
+ * base64, on every provider. Gemini Omni also takes a Puter file path, and accepts one PNG, JPEG or
+ * WebP image up to 5 MiB.
  * @property {string} [last_frame] Last-frame image, same formats as `input_reference`.
  * @property {string[]} [reference_images] Subject/style reference images (URL, data URI or base64).
  * Seedance 2.0: up to 9; Seedance 2.5: up to 30; Together: model-dependent.
@@ -360,12 +366,14 @@
  * and Together models with audio). Defaults to `true` on Seedance.
  * @property {number} [seed] Random seed (Together, Seedance 1.x).
  * @property {number} [width] Output width in pixels on Together models sized in pixels; with `height`,
- * selects the aspect ratio on Seedance and Wan 2.7. Filled in from `size` when omitted.
+ * selects the aspect ratio on Seedance and Wan 2.7. Filled in from `size` when omitted. On Gemini
+ * Omni, `width` and `height` together must be one of its exact sizes and match `size` when both are
+ * given.
  * @property {number} [height] Output height in pixels; see `width`.
  * @property {number} [fps] TogetherAI.
  * @property {number} [steps] TogetherAI.
  * @property {number} [guidance_scale] TogetherAI.
- * @property {string} [output_format] TogetherAI.
+ * @property {string} [output_format] TogetherAI. Gemini Omni accepts only `'video/mp4'`.
  * @property {number} [output_quality] TogetherAI.
  * @property {Array<{ input_image: string, frame: number }>} [frame_images] TogetherAI: keyframe images
  * for image-to-video.

src/puter-js/tests/api/suites/ai.suite.ts

@@ -29,6 +29,8 @@ const TINY_AUDIO = 'data:audio/mp3;base64,QUJD';
 const TINY_PNG =
     'data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==';
 const TINY_MP4 = 'data:video/mp4;base64,QUJD';
+/** What every video provider returns in test mode. */
+const SAMPLE_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4';
 
 /** The `{ message, code }` shape both client-side and driver rejections carry. */
 type SdkError = { message?: string; code?: string };
@@ -1228,6 +1230,125 @@ export default suite('ai', {
         t.assert.equal(foreign.code, 'access_denied');
     },
 
+    // Gemini Omni registers only with a Gemini key; test mode never calls it.
+
+    'txt2vid on gemini omni returns the sample clip in test mode': {
+        requires: ['ai.gemini'],
+        fn: async (t) => {
+            useApiToken(t);
+            const video = await t.puter.ai.txt2vid(
+                {
+                    prompt: 'A stationary red ball.',
+                    model: 'gemini-omni-1.1-flash',
+                    size: '360x640',
+                    input_reference: TINY_PNG,
+                },
+                true,
+            );
+            t.assert.equal(String(video), SAMPLE_VIDEO_URL);
+            t.assert.equal(video.src, SAMPLE_VIDEO_URL);
+        },
+    },
+
+    'txt2vid reaches gemini omni through its model aliases and driver name': {
+        requires: ['ai.gemini'],
+        fn: async (t) => {
+            useApiToken(t);
+            // Gemini Omni refuses `seconds` even in test mode, which proves
+            // where the call landed.
+            for (const options of [
+                { model: 'google/gemini-omni-1.1-flash' },
+                { model: 'google:google/gemini-omni-1.1-flash' },
+                { driver: 'gemini-video-generation' },
+                { provider: 'gemini-video-generation' },
+            ]) {
+                const error = await errorOf(t, () =>
+                    t.puter.ai.txt2vid({
+                        prompt: 'a cat',
+                        seconds: 4,
+                        test_mode: true,
+                        ...options,
+                    }),
+                );
+                t.assert.equal(error.code, 'bad_request');
+                t.assert.equal(
+                    error.message,
+                    'seconds is not supported by Gemini Omni',
+                    `${JSON.stringify(options)} should reach Gemini Omni`,
+                );
+            }
+            const video = await t.puter.ai.txt2vid({
+                prompt: 'a cat',
+                driver: 'gemini-video-generation',
+                test_mode: true,
+            });
+            t.assert.equal(String(video), SAMPLE_VIDEO_URL);
+        },
+    },
+
+    'txt2vid on gemini omni rejects duration and unsupported sizes': {
+        requires: ['ai.gemini'],
+        fn: async (t) => {
+            useApiToken(t);
+            const duration = await errorOf(t, () =>
+                t.puter.ai.txt2vid({
+                    prompt: 'a cat',
+                    model: 'gemini-omni-1.1-flash',
+                    duration: 4,
+                    test_mode: true,
+                }),
+            );
+            t.assert.equal(
+                duration.message,
+                'seconds is not supported by Gemini Omni',
+            );
+            for (const size of ['480p', '1024x576']) {
+                const error = await errorOf(t, () =>
+                    t.puter.ai.txt2vid({
+                        prompt: 'a cat',
+                        model: 'gemini-omni-1.1-flash',
+                        size,
+                        test_mode: true,
+                    }),
+                );
+                t.assert.equal(error.code, 'bad_request');
+                t.assert.ok(
+                    (error.message ?? '').startsWith('size must be one of'),
+                    `size ${size} should be refused, got ${error.message}`,
+                );
+            }
+        },
+    },
+
+    'the video model catalog lists gemini omni with per-token rates': {
+        requires: ['ai.gemini'],
+        fn: async (t) => {
+            const res = await fetch(
+                `${t.env.apiOrigin}/puterai/video/models/details?provider=gemini-video-generation`,
+            );
+            t.assert.equal(res.status, 200);
+            const { models } = (await res.json()) as {
+                models: Array<{
+                    id?: string;
+                    costs?: Record<string, number>;
+                    output_cost_key?: string;
+                    durationSeconds?: unknown;
+                    dimensions?: unknown;
+                }>;
+            };
+            const omni = models.find((m) => m.id === 'gemini-omni-1.1-flash');
+            t.assert.ok(omni, `catalog should list Gemini Omni, got ${JSON.stringify(models)}`);
+            t.assert.deepEqual(omni!.costs, {
+                input: 0.00015,
+                'output:text': 0.0009,
+                'output:video': 0.00175,
+            });
+            t.assert.equal(omni!.output_cost_key, 'output:video');
+            t.assert.equal(omni!.durationSeconds, undefined);
+            t.assert.equal(omni!.dimensions, undefined);
+        },
+    },
+
     // -- txt2speech --------------------------------------------------
 
     'txt2speech rejects a call with no text': async (t) => {