diff --git a/config.template.jsonc b/config.template.jsonc index ca0329d96..5f99853ff 100644 --- a/config.template.jsonc +++ b/config.template.jsonc @@ -341,7 +341,9 @@ "apiKey": "", "apiBaseUrl": "https://llm.onerouter.pro/v1" }, - // BytePlus ModelArk. `apiBaseUrl` selects the region; see + // BytePlus ModelArk. One key powers chat (Seed/GLM/DeepSeek), image + // generation (Seedream) and video generation (Seedance); `apiBaseUrl` + // selects the region; see // https://docs.byteplus.com/en/docs/ModelArk/1330310 for options. "byteplus": { "apiKey": "", diff --git a/src/backend/drivers/ai-image/ImageGenerationDriver.ts b/src/backend/drivers/ai-image/ImageGenerationDriver.ts index 3bac30081..f63d2eb10 100644 --- a/src/backend/drivers/ai-image/ImageGenerationDriver.ts +++ b/src/backend/drivers/ai-image/ImageGenerationDriver.ts @@ -26,6 +26,7 @@ import { HttpError } from '../../core/http/HttpError.js'; import type { Actor } from '../../core/actor.js'; import { PuterDriver } from '../types.js'; import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js'; +import { BytePlusImageProvider } from './providers/byteplus/BytePlusImageProvider.js'; import { CloudflareImageProvider } from './providers/cloudflare/CloudflareImageProvider.js'; import { GeminiImageProvider } from './providers/gemini/GeminiImageProvider.js'; import { OpenAiImageProvider } from './providers/openai/OpenAiImageProvider.js'; @@ -59,6 +60,7 @@ export class ImageGenerationDriver extends PuterDriver { 'cloudflare-image-generation', 'xai-image-generation', 'replicate-image-generation', + 'byteplus-image-generation', ]; readonly isDefault = true; @@ -290,8 +292,7 @@ export class ImageGenerationDriver extends PuterDriver { const cloudflare = (providers['cloudflare-image-generation'] ?? providers['cloudflare-workers-ai-image'] ?? providers['cloudflare-workers-ai']) as - | Record - | undefined; + Record | undefined; const cfToken = (cloudflare?.apiToken as string | undefined) ?? (cloudflare?.apiKey as string | undefined) ?? @@ -306,8 +307,7 @@ export class ImageGenerationDriver extends PuterDriver { apiToken: cfToken, accountId: cfAccount, apiBaseUrl: cloudflare?.apiBaseUrl as - | string - | undefined, + string | undefined, }, m, ); @@ -332,6 +332,26 @@ export class ImageGenerationDriver extends PuterDriver { m, ); } + + // Falls back to the shared `byteplus` (ai-chat) key; `apiBaseUrl` + // selects the ModelArk region, same as the chat provider. + const byteplusCfg = (providers['byteplus-image-generation'] ?? + providers['byteplus']) as Record | undefined; + const byteplusKey = readKey( + providers['byteplus-image-generation'], + providers['byteplus'], + ); + if (byteplusKey) { + this.#providers['byteplus-image-generation'] = + new BytePlusImageProvider( + { + apiKey: byteplusKey, + apiBaseUrl: byteplusCfg?.apiBaseUrl as + string | undefined, + }, + m, + ); + } } async #buildModelMap() { diff --git a/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.integration.test.ts b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.integration.test.ts new file mode 100644 index 000000000..939b2a522 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.integration.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the BytePlus ModelArk image provider. + * + * Generates one 1K image on seedream-4-0 (the cheapest catalog entry, + * $0.03/image). Skipped when `PUTER_TEST_AI_BYTEPLUS_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + INTEGRATION_TEST_TIMEOUT_MS, + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { BytePlusImageProvider } from './BytePlusImageProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_BYTEPLUS_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'BytePlusImageProvider (integration)', + () => { + it( + 'generates an image URL from seedream-4-0', + { timeout: INTEGRATION_TEST_TIMEOUT_MS }, + async () => { + const provider = new BytePlusImageProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const url = await withTestActor(() => + provider.generate({ + model: 'seedream-4-0', + prompt: 'a single red dot on a white background', + quality: '1k', + }), + ); + + expect(typeof url).toBe('string'); + expect( + url.startsWith('https://') || url.startsWith('data:'), + ).toBe(true); + }, + ); + }, +); diff --git a/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.test.ts b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.test.ts new file mode 100644 index 000000000..3b1a37eeb --- /dev/null +++ b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.test.ts @@ -0,0 +1,504 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for BytePlusImageProvider. + * + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock + * redis) and constructs BytePlusImageProvider directly against the + * live wired `MeteringService` so the recording side is exercised + * end-to-end. Ark's image API is OpenAI-compatible so the OpenAI SDK + * is mocked at the module boundary; that's the real network egress + * point. + */ + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { BYTEPLUS_IMAGE_GENERATION_MODELS } from './models.js'; +import { BytePlusImageProvider } from './BytePlusImageProvider.js'; + +// ── OpenAI SDK mock ───────────────────────────────────────────────── + +const { generateMock, openAICtor } = vi.hoisted(() => ({ + generateMock: vi.fn(), + openAICtor: vi.fn(), +})); + +vi.mock('openai', () => { + const OpenAICtor = vi.fn().mockImplementation(function ( + this: Record, + opts: unknown, + ) { + openAICtor(opts); + this.images = { generate: generateMock }; + // Some sibling providers boot through the same SDK module. + this.chat = { completions: { create: vi.fn() } }; + this.post = vi.fn(); + }); + return { OpenAI: OpenAICtor, default: { OpenAI: OpenAICtor } }; +}); + +// ── Test harness ──────────────────────────────────────────────────── + +let server: PuterServer; +let hasCreditsSpy: MockInstance; +let batchIncrementUsagesSpy: MockInstance< + MeteringService['batchIncrementUsages'] +>; + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = ( + config: { apiKey?: string; apiBaseUrl?: string } = {}, +) => + new BytePlusImageProvider( + { + apiKey: config.apiKey ?? 'test-key', + ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}), + }, + server.services.metering, + ); + +beforeEach(() => { + generateMock.mockReset(); + openAICtor.mockReset(); + hasCreditsSpy = vi.spyOn(server.services.metering, 'hasEnoughCredits'); + batchIncrementUsagesSpy = vi.spyOn( + server.services.metering, + 'batchIncrementUsages', + ); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +const sampleResponse = { data: [{ url: 'https://ark.example/img/1' }] }; + +const findModel = (id: string) => + BYTEPLUS_IMAGE_GENERATION_MODELS.find((m) => m.id === id)!; + +// ── Construction ──────────────────────────────────────────────────── + +describe('BytePlusImageProvider construction', () => { + it('defaults the OpenAI SDK to the ap-southeast ModelArk base URL', () => { + makeProvider(); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://ark.ap-southeast.bytepluses.com/api/v3', + }); + }); + + it('honors a configured apiBaseUrl (region selection)', () => { + makeProvider({ + apiBaseUrl: 'https://ark.eu-west.bytepluses.com/api/v3', + }); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://ark.eu-west.bytepluses.com/api/v3', + }); + }); + + it('throws when no apiKey is supplied', () => { + expect( + () => + new BytePlusImageProvider( + { apiKey: '' }, + server.services.metering, + ), + ).toThrow(/API key/i); + }); +}); + +// ── Model catalog ─────────────────────────────────────────────────── + +describe('BytePlusImageProvider model catalog', () => { + it('returns seedream-5-0-lite as the default', () => { + expect(makeProvider().getDefaultModel()).toBe( + 'seedream-5-0-lite-260128', + ); + }); + + it('exposes the static catalog verbatim', () => { + expect(makeProvider().models()).toBe( + BYTEPLUS_IMAGE_GENERATION_MODELS, + ); + }); +}); + +// ── test_mode / validation / credit gate ──────────────────────────── + +describe('BytePlusImageProvider.generate gates', () => { + it('returns the canned sample URL in test_mode without side effects', async () => { + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'x', test_mode: true }), + ); + expect(result).toBe( + 'https://puter-sample-data.puter.site/image_example.png', + ); + expect(hasCreditsSpy).not.toHaveBeenCalled(); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it('throws 400 on a missing or blank prompt', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: undefined as unknown as string, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + await expect( + withTestActor(() => makeProvider().generate({ prompt: ' ' })), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it('throws 402 BEFORE hitting ModelArk when the actor lacks credits', async () => { + hasCreditsSpy.mockResolvedValueOnce(false); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 402 }); + expect(generateMock).not.toHaveBeenCalled(); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + }); +}); + +// ── Model resolution ──────────────────────────────────────────────── + +describe('BytePlusImageProvider.generate model resolution', () => { + it('falls back to the default model for unknown ids', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ model: 'nope', prompt: 'hi' }), + ); + expect(generateMock.mock.calls[0]![0].model).toBe( + 'seedream-5-0-lite-260128', + ); + }); + + it('resolves series and byteplus-prefixed aliases to the dated id', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'byteplus/seedream-4-0', + prompt: 'hi', + }), + ); + expect(generateMock.mock.calls[0]![0].model).toBe( + 'seedream-4-0-250828', + ); + }); + + it('resolves the undated seedream-5-0 alias to the lite snapshot', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ model: 'seedream-5-0', prompt: 'hi' }), + ); + expect(generateMock.mock.calls[0]![0].model).toBe( + 'seedream-5-0-lite-260128', + ); + }); +}); + +// ── Request shape ─────────────────────────────────────────────────── + +describe('BytePlusImageProvider.generate request shape', () => { + it('sends the tier keyword size (Ark default 2K), url format and no watermark', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ prompt: 'a red dot' }), + ); + const sent = generateMock.mock.calls[0]![0]; + expect(sent.size).toBe('2K'); + expect(sent.response_format).toBe('url'); + expect(sent.watermark).toBe(false); + expect(sent.image).toBeUndefined(); + }); + + it('maps quality to the tier keyword case-insensitively', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ prompt: 'hi', quality: '1.5K' }), + ); + expect(generateMock.mock.calls[0]![0].size).toBe('1.5K'); + }); + + it('resolves an aspect ratio + tier to the documented pixel size', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + quality: '1k', + ratio: { w: 16, h: 9 }, + }), + ); + expect(generateMock.mock.calls[0]![0].size).toBe('1424x800'); + }); + + it('passes explicit pixel dimensions straight through', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + ratio: { w: 2048, h: 1024 }, + }), + ); + expect(generateMock.mock.calls[0]![0].size).toBe('2048x1024'); + }); + + it('rejects explicit pixel dimensions outside Ark limits', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + ratio: { w: 6000, h: 6000 }, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it('omits size for the i2i-only seededit model', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'seededit-3-0-i2i', + prompt: 'add a hat', + input_image: 'https://example.com/cat.png', + }), + ); + const sent = generateMock.mock.calls[0]![0]; + expect(sent.model).toBe('seededit-3-0-i2i-250628'); + expect(sent.size).toBeUndefined(); + expect(sent.image).toBe('https://example.com/cat.png'); + }); +}); + +// ── Input images ──────────────────────────────────────────────────── + +describe('BytePlusImageProvider.generate input images', () => { + const PNG = 'data:image/png;base64,iVBORw0KGgo='; + + it('sends a single input image as a string and multiple as an array', async () => { + generateMock.mockResolvedValue(sampleResponse); + + await withTestActor(() => + makeProvider().generate({ prompt: 'hi', input_images: [PNG] }), + ); + expect(generateMock.mock.calls[0]![0].image).toBe(PNG); + + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + input_images: [PNG, PNG], + }), + ); + expect(generateMock.mock.calls[1]![0].image).toEqual([PNG, PNG]); + }); + + it('wraps raw base64 into a data URI using the mime hint', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + input_image: 'AAAA', + input_image_mime_type: 'image/webp', + }), + ); + expect(generateMock.mock.calls[0]![0].image).toBe( + 'data:image/webp;base64,AAAA', + ); + }); + + it('rejects more input images than the model accepts', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + model: 'seededit-3-0-i2i', + prompt: 'hi', + input_images: [PNG, PNG], + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it('requires an input image for seededit', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + model: 'seededit-3-0-i2i', + prompt: 'hi', + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(generateMock).not.toHaveBeenCalled(); + }); +}); + +// ── Metering ──────────────────────────────────────────────────────── + +describe('BytePlusImageProvider.generate metering', () => { + it('meters flat per-image models at their catalog rate', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ model: 'seedream-4-0', prompt: 'hi' }), + ); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + expect(entries).toEqual([ + { + usageType: + 'byteplus-image-generation:seedream-4-0-250828:per-image', + usageAmount: 1, + costOverride: + findModel('seedream-4-0-250828').costs['per-image'] * + 1_000_000, + }, + ]); + }); + + it('bills the pro model at the high tier for 2K output', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'dola-seedream-5-0-pro', + prompt: 'hi', + }), + ); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + const pro = findModel('dola-seedream-5-0-pro-260628'); + expect(entries).toEqual([ + { + usageType: + 'byteplus-image-generation:dola-seedream-5-0-pro-260628:output:2k', + usageAmount: 1, + costOverride: pro.costs['output:2k'] * 1_000_000, + }, + ]); + }); + + it('bills the pro model at the low tier for 1k/1.5k output', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'dola-seedream-5-0-pro', + prompt: 'hi', + quality: '1.5k', + }), + ); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + expect( + (entries as Array<{ usageType: string }>)[0].usageType, + ).toBe( + 'byteplus-image-generation:dola-seedream-5-0-pro-260628:output:1.5k', + ); + }); + + it('bills pro input images from the second one on (first is free)', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + const PNG = 'data:image/png;base64,iVBORw0KGgo='; + await withTestActor(() => + makeProvider().generate({ + model: 'dola-seedream-5-0-pro', + prompt: 'hi', + input_images: [PNG, PNG, PNG], + }), + ); + const pro = findModel('dola-seedream-5-0-pro-260628'); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + const inputEntry = ( + entries as Array<{ + usageType: string; + usageAmount: number; + costOverride: number; + }> + ).find((e) => e.usageType.endsWith(':input_image'))!; + expect(inputEntry.usageAmount).toBe(2); + expect(inputEntry.costOverride).toBe( + 2 * pro.costs.input_image * 1_000_000, + ); + }); + + it('does not bill input images on flat-rate models', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + const PNG = 'data:image/png;base64,iVBORw0KGgo='; + await withTestActor(() => + makeProvider().generate({ + model: 'seedream-4-5', + prompt: 'hi', + input_images: [PNG, PNG, PNG], + }), + ); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + expect(entries).toHaveLength(1); + }); +}); + +// ── Response handling ─────────────────────────────────────────────── + +describe('BytePlusImageProvider.generate response handling', () => { + it('falls back to a data URI when the response carries b64_json', async () => { + generateMock.mockResolvedValueOnce({ + data: [{ b64_json: 'AAAA', output_format: 'png' }], + }); + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'hi' }), + ); + expect(result).toBe('data:image/png;base64,AAAA'); + }); + + it('surfaces a per-image upstream error as 400 without metering', async () => { + generateMock.mockResolvedValueOnce({ + data: [{ error: { code: 'x', message: 'moderated' } }], + }); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + }); + + it('throws when the response has no usable image data', async () => { + generateMock.mockResolvedValueOnce({ data: [{}] }); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toThrow(/Failed to extract image URL/); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.ts b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.ts new file mode 100644 index 000000000..997facb06 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.ts @@ -0,0 +1,310 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { OpenAI } from 'openai'; +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { + IGenerateParams, + IImageModel, + IImageProvider, +} from '../../types.js'; +import { isHttpUrl } from '../../inputImage.js'; +import { + BYTEPLUS_IMAGE_GENERATION_MODELS, + SEEDREAM_RESOLUTION_MAP, +} from './models.js'; + +const DEFAULT_MODEL = 'seedream-5-0-lite-260128'; + +// Ark's explicit-pixel `size` bounds ("method 2"): total pixels within +// [1280x720, 2048x2048x1.1025] and aspect ratio within [1/16, 16]. +const MIN_TOTAL_PIXELS = 921_600; +const MAX_TOTAL_PIXELS = 4_624_220; +// dola-seedream-5-0-pro's price break: ≤ 2.61MP bills the "1.5K or lower" +// rate, above it the higher rate. +const PRO_TIER_BREAK_PIXELS = 2_610_000; + +// Per-request reference image caps, per the API reference. +const MAX_INPUT_IMAGES_PRO = 10; +const MAX_INPUT_IMAGES_SEEDREAM = 14; + +type BytePlusImageConfig = { + apiKey: string; + apiBaseUrl?: string; +}; + +interface ArkImageResponse { + data?: Array<{ + url?: string; + b64_json?: string; + output_format?: string; + error?: { code?: string; message?: string }; + }>; + usage?: { generated_images?: number }; +} + +/** + * BytePlus ModelArk image generation provider (Seedream / SeedEdit). + * + * Ark's `POST /images/generations` is OpenAI-compatible enough to reuse the + * OpenAI SDK (same client/auth as BytePlusProvider in ai-chat); Ark-specific + * params (`image`, `watermark`, tier-style `size`) pass through the SDK + * untouched. https://docs.byteplus.com/en/docs/ModelArk/1541523 + */ +export class BytePlusImageProvider implements IImageProvider { + #client: OpenAI; + #meteringService: MeteringService; + + constructor(config: BytePlusImageConfig, meteringService: MeteringService) { + if (!config.apiKey) { + throw new Error('BytePlus image generation requires an API key'); + } + this.#meteringService = meteringService; + this.#client = new OpenAI({ + apiKey: config.apiKey, + baseURL: + config.apiBaseUrl ?? + 'https://ark.ap-southeast.bytepluses.com/api/v3', + }); + } + + models(): IImageModel[] { + return BYTEPLUS_IMAGE_GENERATION_MODELS; + } + + getDefaultModel(): string { + return DEFAULT_MODEL; + } + + async generate(params: IGenerateParams): Promise { + const { prompt, test_mode, model, ratio, quality } = params; + let { input_images } = params; + const { input_image, input_image_mime_type } = params; + + const selectedModel = this.#getModel(model); + const isSeedEdit = selectedModel.id.startsWith('seededit-'); + const isPro = selectedModel.pricing_unit === 'per-tier'; + + if (test_mode) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if (typeof prompt !== 'string' || prompt.trim().length === 0) { + throw new HttpError(400, '`prompt` must be a non-empty string', { + legacyCode: 'bad_request', + }); + } + + // Backwards compat: fold singular `input_image` into `input_images`. + if (input_image && (!input_images || input_images.length === 0)) { + input_images = [input_image]; + } + const maxInputImages = isSeedEdit + ? 1 + : isPro + ? MAX_INPUT_IMAGES_PRO + : MAX_INPUT_IMAGES_SEEDREAM; + if (input_images && input_images.length > maxInputImages) { + throw new HttpError( + 400, + `${selectedModel.id} accepts at most ${maxInputImages} input image(s)`, + { legacyCode: 'bad_request' }, + ); + } + if (isSeedEdit && (!input_images || input_images.length === 0)) { + throw new HttpError( + 400, + `${selectedModel.id} is image-to-image only; pass an input image via input_image`, + { legacyCode: 'bad_request' }, + ); + } + const inputImageCount = input_images?.length ?? 0; + + const tier = this.#normalizeTier(quality); + const size = isSeedEdit ? undefined : this.#resolveSize(tier, ratio); + + // The pro model bills by output pixel count; everything else is a + // flat per-image rate. + let outputCostKey: string; + if (isPro) { + const pixels = this.#sizePixels(size!); + outputCostKey = + pixels !== undefined + ? pixels > PRO_TIER_BREAK_PIXELS + ? 'output:2k' + : 'output:1k' + : `output:${tier}`; + } else { + outputCostKey = 'per-image'; + } + const outputCents = selectedModel.costs[outputCostKey]; + if (outputCents === undefined) { + throw new Error( + `Model ${selectedModel.id} missing '${outputCostKey}' cost`, + ); + } + // First input image is free on the pro model; the rest are billed. + const inputImageCents = selectedModel.costs.input_image ?? 0; + const billableInputs = + inputImageCents > 0 ? Math.max(0, inputImageCount - 1) : 0; + const estimatedCents = outputCents + billableInputs * inputImageCents; + + const actor = Context.get('actor'); + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + estimatedCents * 1_000_000, + ); + if (!usageAllowed) { + throw new HttpError( + 402, + 'Insufficient credits for image generation', + { legacyCode: 'insufficient_funds' }, + ); + } + + const image = + inputImageCount > 0 + ? input_images!.map((img) => + this.#toImageRef(img, input_image_mime_type), + ) + : undefined; + + const response = (await this.#client.images.generate({ + model: selectedModel.id, + prompt, + // Ark-specific params not in the OpenAI type; passed through. + ...(size ? { size } : {}), + ...(image ? { image: image.length === 1 ? image[0] : image } : {}), + response_format: 'url', + watermark: false, + } as Parameters[0])) as ArkImageResponse; + + const first = response.data?.[0]; + if (first?.error) { + throw new HttpError( + 400, + first.error.message ?? 'Image generation failed', + { + legacyCode: 'upstream_failed', + fields: { provider: 'byteplus' }, + }, + ); + } + const url = + first?.url || + (first?.b64_json + ? `data:image/${first.output_format ?? 'jpeg'};base64,${first.b64_json}` + : undefined); + if (!url) { + throw new Error( + 'Failed to extract image URL from BytePlus response', + ); + } + + const usageEntries = [ + { + usageType: `byteplus-image-generation:${selectedModel.id}:${outputCostKey}`, + usageAmount: 1, + costOverride: outputCents * 1_000_000, + }, + ]; + if (billableInputs > 0) { + usageEntries.push({ + usageType: `byteplus-image-generation:${selectedModel.id}:input_image`, + usageAmount: billableInputs, + costOverride: billableInputs * inputImageCents * 1_000_000, + }); + } + this.#meteringService.batchIncrementUsages(actor, usageEntries); + + return url; + } + + // Ark accepts a public URL or a `data:image/...;base64,` URI; wrap raw + // base64 payloads so they're valid. + #toImageRef(img: string, mimeHint?: string): string { + return isHttpUrl(img) || img.startsWith('data:') + ? img + : `data:${mimeHint ?? 'image/png'};base64,${img}`; + } + + #normalizeTier(quality?: string): '1k' | '1.5k' | '2k' { + const q = (quality ?? '').toLowerCase(); + // Ark's default size is 2K when unspecified. + return q === '1k' || q === '1.5k' || q === '2k' ? q : '2k'; + } + + /** + * Resolve the `size` request param: + * + * - `ratio` holding real pixel dimensions → explicit `WxH` (method 2) + * - `ratio` holding an aspect ratio with a known tier mapping → the + * documented `WxH` for (aspect, tier) + * - Otherwise → the tier keyword (`1K`/`1.5K`/`2K`, method 1) + */ + #resolveSize( + tier: '1k' | '1.5k' | '2k', + ratio?: { w: number; h: number }, + ): string { + if (ratio?.w && ratio?.h) { + const pixels = ratio.w * ratio.h; + if (pixels >= MIN_TOTAL_PIXELS) { + const aspect = ratio.w / ratio.h; + if ( + pixels > MAX_TOTAL_PIXELS || + aspect < 1 / 16 || + aspect > 16 + ) { + throw new HttpError( + 400, + `Requested size ${ratio.w}x${ratio.h} is outside BytePlus limits ` + + `(total pixels ≤ ${MAX_TOTAL_PIXELS}, aspect ratio within [1/16, 16])`, + { legacyCode: 'bad_request' }, + ); + } + return `${ratio.w}x${ratio.h}`; + } + const mapped = + SEEDREAM_RESOLUTION_MAP[`${ratio.w}:${ratio.h}`]?.[tier]; + if (mapped) return `${mapped.w}x${mapped.h}`; + } + return { '1k': '1K', '1.5k': '1.5K', '2k': '2K' }[tier]; + } + + /** Pixel count of an explicit `WxH` size; undefined for tier keywords. */ + #sizePixels(size: string): number | undefined { + const m = /^(\d+)x(\d+)$/.exec(size); + if (!m) return undefined; + return Number(m[1]) * Number(m[2]); + } + + #getModel(model?: string) { + const models = this.models(); + const wanted = (model ?? '').trim().toLowerCase(); + const found = models.find( + (m) => + m.id === wanted || + m.puterId === wanted || + m.aliases?.some((a) => a.toLowerCase() === wanted), + ); + return found || models.find((m) => m.id === DEFAULT_MODEL)!; + } +} diff --git a/src/backend/drivers/ai-image/providers/byteplus/models.ts b/src/backend/drivers/ai-image/providers/byteplus/models.ts new file mode 100644 index 000000000..1d4d08745 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/byteplus/models.ts @@ -0,0 +1,176 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IImageModel } from '../../types.js'; + +// Ark's `size` "method 1" tiers and the pixel dimensions each (tier, aspect +// ratio) pair resolves to, from the image generation API reference: +// https://docs.byteplus.com/en/docs/ModelArk/1541523 +// The table is identical for dola-seedream-5-0-pro, seedream-5-0-lite, +// seedream-4-5 and seedream-4-0. +export const SEEDREAM_RESOLUTION_MAP: Record< + string, + Record +> = { + '1:1': { + '1k': { w: 1024, h: 1024 }, + '1.5k': { w: 1536, h: 1536 }, + '2k': { w: 2048, h: 2048 }, + }, + '4:3': { + '1k': { w: 1152, h: 864 }, + '1.5k': { w: 1792, h: 1344 }, + '2k': { w: 2368, h: 1776 }, + }, + '3:4': { + '1k': { w: 864, h: 1152 }, + '1.5k': { w: 1344, h: 1792 }, + '2k': { w: 1776, h: 2368 }, + }, + '16:9': { + '1k': { w: 1424, h: 800 }, + '1.5k': { w: 2048, h: 1152 }, + '2k': { w: 2816, h: 1584 }, + }, + '9:16': { + '1k': { w: 800, h: 1424 }, + '1.5k': { w: 1152, h: 2048 }, + '2k': { w: 1584, h: 2816 }, + }, + '3:2': { + '1k': { w: 1248, h: 832 }, + '1.5k': { w: 1872, h: 1248 }, + '2k': { w: 2496, h: 1664 }, + }, + '2:3': { + '1k': { w: 832, h: 1248 }, + '1.5k': { w: 1248, h: 1872 }, + '2k': { w: 1664, h: 2496 }, + }, + '21:9': { + '1k': { w: 1568, h: 672 }, + '1.5k': { w: 2352, h: 1008 }, + '2k': { w: 3136, h: 1344 }, + }, +}; + +const SEEDREAM_QUALITY_LEVELS = ['1k', '1.5k', '2k']; + +// Costs are in usd-cents per image, hardcoded from +// https://docs.byteplus.com/en/docs/ModelArk/1544106 (pricing) and +// https://docs.byteplus.com/en/docs/ModelArk/1330310 (catalog). +// +// dola-seedream-5-0-pro bills by output pixel count — ≤ 2.61MP ("1.5K or +// lower") vs above — plus a per-input-image rate from the second reference +// image onward (the first is free). Every other model is a flat per-image +// rate with free image input. +export const BYTEPLUS_IMAGE_GENERATION_MODELS: IImageModel[] = [ + { + puterId: 'byteplus:byteplus/dola-seedream-5-0-pro-260628', + id: 'dola-seedream-5-0-pro-260628', + aliases: [ + 'byteplus/dola-seedream-5-0-pro-260628', + 'dola-seedream-5-0-pro', + 'byteplus/dola-seedream-5-0-pro', + 'seedream-5-0-pro', + ], + name: 'Dola Seedream 5.0 Pro', + costs_currency: 'usd-cents', + pricing_unit: 'per-tier', + index_cost_key: 'output:1k', + costs: { + 'output:1k': 4.5, // $0.045 per image ≤ 2.61MP + 'output:1.5k': 4.5, // same price as 1K, better quality + 'output:2k': 9, // $0.09 per image > 2.61MP + input_image: 0.3, // $0.003 per input image from the 2nd on + }, + allowedQualityLevels: SEEDREAM_QUALITY_LEVELS, + resolution_map: SEEDREAM_RESOLUTION_MAP, + }, + { + // The catalog lists `seedream-5-0-260128` as the same service + // ("also supports: seedream-5-0-lite-260128"); billing is published + // under the -lite id, so that's the primary id here. + puterId: 'byteplus:byteplus/seedream-5-0-lite-260128', + id: 'seedream-5-0-lite-260128', + aliases: [ + 'byteplus/seedream-5-0-lite-260128', + 'seedream-5-0-lite', + 'byteplus/seedream-5-0-lite', + 'seedream-5-0-260128', + 'seedream-5-0', + ], + name: 'Seedream 5.0 Lite', + costs_currency: 'usd-cents', + pricing_unit: 'per-image', + index_cost_key: 'per-image', + costs: { 'per-image': 3.5 }, + allowedQualityLevels: SEEDREAM_QUALITY_LEVELS, + resolution_map: SEEDREAM_RESOLUTION_MAP, + }, + { + puterId: 'byteplus:byteplus/seedream-4-5-251128', + id: 'seedream-4-5-251128', + aliases: [ + 'byteplus/seedream-4-5-251128', + 'seedream-4-5', + 'byteplus/seedream-4-5', + ], + name: 'Seedream 4.5', + costs_currency: 'usd-cents', + pricing_unit: 'per-image', + index_cost_key: 'per-image', + costs: { 'per-image': 4 }, + allowedQualityLevels: SEEDREAM_QUALITY_LEVELS, + resolution_map: SEEDREAM_RESOLUTION_MAP, + }, + { + puterId: 'byteplus:byteplus/seedream-4-0-250828', + id: 'seedream-4-0-250828', + aliases: [ + 'byteplus/seedream-4-0-250828', + 'seedream-4-0', + 'byteplus/seedream-4-0', + ], + name: 'Seedream 4.0', + costs_currency: 'usd-cents', + pricing_unit: 'per-image', + index_cost_key: 'per-image', + costs: { 'per-image': 3 }, + allowedQualityLevels: SEEDREAM_QUALITY_LEVELS, + resolution_map: SEEDREAM_RESOLUTION_MAP, + }, + { + // Image-to-image only; output keeps the source image's size, so the + // Seedream size/tier machinery doesn't apply. + puterId: 'byteplus:byteplus/seededit-3-0-i2i-250628', + id: 'seededit-3-0-i2i-250628', + aliases: [ + 'byteplus/seededit-3-0-i2i-250628', + 'seededit-3-0-i2i', + 'byteplus/seededit-3-0-i2i', + 'seededit-3-0', + ], + name: 'SeedEdit 3.0 (image-to-image)', + costs_currency: 'usd-cents', + pricing_unit: 'per-image', + index_cost_key: 'per-image', + costs: { 'per-image': 3 }, + }, +]; diff --git a/src/backend/drivers/ai-video/VideoGenerationDriver.ts b/src/backend/drivers/ai-video/VideoGenerationDriver.ts index 967917eb1..697822bcd 100644 --- a/src/backend/drivers/ai-video/VideoGenerationDriver.ts +++ b/src/backend/drivers/ai-video/VideoGenerationDriver.ts @@ -25,6 +25,7 @@ import { HttpError } from '../../core/http/HttpError.js'; import type { Actor } from '../../core/actor.js'; import { PuterDriver } from '../types.js'; import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js'; +import { BytePlusVideoProvider } from './providers/byteplus/BytePlusVideoProvider.js'; import { GeminiVideoProvider } from './providers/gemini/GeminiVideoProvider.js'; import { OpenAIVideoProvider } from './providers/openai/OpenAIVideoProvider.js'; import { TogetherVideoProvider } from './providers/together/TogetherVideoProvider.js'; @@ -56,6 +57,7 @@ export class VideoGenerationDriver extends PuterDriver { 'openai-video-generation', 'together-video-generation', 'gemini-video-generation', + 'byteplus-video-generation', ]; readonly isDefault = true; @@ -290,6 +292,26 @@ export class VideoGenerationDriver extends PuterDriver { this.#providers['gemini-video-generation'] = new GeminiVideoProvider({ apiKey: geminiKey }, m); } + + // Falls back to the shared `byteplus` (ai-chat) key; `apiBaseUrl` + // selects the ModelArk region, same as the chat provider. + const byteplusCfg = (providers['byteplus-video-generation'] ?? + providers['byteplus']) as Record | undefined; + const byteplusKey = readKey( + providers['byteplus-video-generation'], + providers['byteplus'], + ); + if (byteplusKey) { + this.#providers['byteplus-video-generation'] = + new BytePlusVideoProvider( + { + apiKey: byteplusKey, + apiBaseUrl: byteplusCfg?.apiBaseUrl as + string | undefined, + }, + m, + ); + } } // -- Model map ----------------------------------------------------------- diff --git a/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.integration.test.ts b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.integration.test.ts new file mode 100644 index 000000000..b2a20cfee --- /dev/null +++ b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.integration.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the BytePlus ModelArk video provider. + * + * Generates a 2s 480p clip on seedance-1-0-pro-fast (~$0.02). Video tasks + * poll for completion, so the timeout is generous. Skipped when + * `PUTER_TEST_AI_BYTEPLUS_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { BytePlusVideoProvider } from './BytePlusVideoProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_BYTEPLUS_API_KEY'; +// Task-based generation routinely takes a couple of minutes. +const VIDEO_TIMEOUT_MS = 5 * 60 * 1000; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'BytePlusVideoProvider (integration)', + () => { + it( + 'generates a video URL from seedance-1-0-pro-fast', + { timeout: VIDEO_TIMEOUT_MS }, + async () => { + const provider = new BytePlusVideoProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const url = await withTestActor(() => + provider.generate({ + model: 'seedance-1-0-pro-fast', + prompt: 'a red ball rolls to the right', + resolution: '480p', + seconds: 2, + }), + ); + + expect(typeof url).toBe('string'); + expect((url as string).startsWith('https://')).toBe(true); + }, + ); + }, +); diff --git a/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.test.ts b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.test.ts new file mode 100644 index 000000000..cc027abd0 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.test.ts @@ -0,0 +1,435 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for BytePlusVideoProvider. + * + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock + * redis) and constructs BytePlusVideoProvider directly against the + * live wired `MeteringService`. Ark's task-based video API has no + * SDK — the provider hits it via global `fetch`, which we stub; + * that's the real network egress point. + */ + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { BYTEPLUS_VIDEO_GENERATION_MODELS } from './models.js'; +import { BytePlusVideoProvider } from './BytePlusVideoProvider.js'; + +// ── Test harness ──────────────────────────────────────────────────── + +let server: PuterServer; +let fetchSpy: MockInstance; +let remainingUsageSpy: MockInstance; +let incrementUsageSpy: MockInstance; + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = ( + config: { apiKey?: string; apiBaseUrl?: string } = {}, +) => + new BytePlusVideoProvider( + { + apiKey: config.apiKey ?? 'test-key', + ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}), + pollIntervalMs: 1, + }, + server.services.metering, + ); + +beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, 'fetch') as MockInstance; + remainingUsageSpy = vi.spyOn( + server.services.metering, + 'getRemainingUsage', + ); + // Plenty of credit unless a test says otherwise. + remainingUsageSpy.mockResolvedValue(Number.MAX_SAFE_INTEGER); + incrementUsageSpy = vi.spyOn(server.services.metering, 'incrementUsage'); + incrementUsageSpy.mockResolvedValue({} as never); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +const jsonResponse = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + +const succeededTask = (overrides: Record = {}) => ({ + id: 'cgt-test-1', + status: 'succeeded', + content: { video_url: 'https://ark.example/video.mp4' }, + usage: { completion_tokens: 108_000, total_tokens: 108_000 }, + resolution: '720p', + duration: 5, + ...overrides, +}); + +/** Queue up the POST-create response followed by GET-poll responses. */ +const mockTaskFlow = (...pollBodies: unknown[]) => { + fetchSpy.mockResolvedValueOnce(jsonResponse({ id: 'cgt-test-1' })); + for (const body of pollBodies) { + fetchSpy.mockResolvedValueOnce(jsonResponse(body)); + } +}; + +const sentBody = (callIndex = 0): Record => + JSON.parse( + (fetchSpy.mock.calls[callIndex]![1] as RequestInit).body as string, + ); + +const findModel = (id: string) => + BYTEPLUS_VIDEO_GENERATION_MODELS.find((m) => m.id === id)!; + +// ── Construction / catalog ────────────────────────────────────────── + +describe('BytePlusVideoProvider construction and catalog', () => { + it('throws when no apiKey is supplied', () => { + expect( + () => + new BytePlusVideoProvider( + { apiKey: '' }, + server.services.metering, + ), + ).toThrow(/API key/i); + }); + + it('does not call out at construction (lazy fetch)', () => { + makeProvider(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('defaults to seedance 2.0 mini', () => { + expect(makeProvider().getDefaultModel()).toBe( + 'dreamina-seedance-2-0-mini-260615', + ); + }); + + it('exposes the static catalog', async () => { + expect(await makeProvider().models()).toBe( + BYTEPLUS_VIDEO_GENERATION_MODELS, + ); + }); +}); + +// ── Gates ─────────────────────────────────────────────────────────── + +describe('BytePlusVideoProvider.generate gates', () => { + it('returns the canned sample URL in test_mode without network calls', async () => { + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'x', test_mode: true }), + ); + expect(result).toBe('https://assets.puter.site/txt2vid.mp4'); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('throws 400 on a missing or blank prompt', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: undefined as unknown as string, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('throws 402 BEFORE creating a task when even the shortest clip is unaffordable', async () => { + remainingUsageSpy.mockResolvedValue(0); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 402 }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); +}); + +// ── Request shape ─────────────────────────────────────────────────── + +describe('BytePlusVideoProvider.generate request shape', () => { + it('POSTs a create-task request with text content and defaults', async () => { + mockTaskFlow(succeededTask()); + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'a kitten yawns' }), + ); + + expect(result).toBe('https://ark.example/video.mp4'); + const [url, init] = fetchSpy.mock.calls[0]!; + expect(String(url)).toBe( + 'https://ark.ap-southeast.bytepluses.com/api/v3/contents/generations/tasks', + ); + expect((init as RequestInit).method).toBe('POST'); + expect( + (init as RequestInit & { headers: Record }) + .headers.Authorization, + ).toBe('Bearer test-key'); + + const body = sentBody(); + expect(body.model).toBe('dreamina-seedance-2-0-mini-260615'); + expect(body.content).toEqual([ + { type: 'text', text: 'a kitten yawns' }, + ]); + expect(body.resolution).toBe('720p'); + expect(body.duration).toBe(5); + expect(body.watermark).toBe(false); + expect(body.generate_audio).toBe(true); + + // Poll goes to GET tasks/{id}. + const [pollUrl, pollInit] = fetchSpy.mock.calls[1]!; + expect(String(pollUrl)).toBe( + 'https://ark.ap-southeast.bytepluses.com/api/v3/contents/generations/tasks/cgt-test-1', + ); + expect((pollInit as RequestInit).method).toBe('GET'); + }); + + it('resolves aliases, uppercases 4k, and clamps duration to the model range', async () => { + mockTaskFlow(succeededTask({ resolution: '4k' })); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-2-0', + prompt: 'hi', + resolution: '4k', + seconds: 99, + }), + ); + const body = sentBody(); + expect(body.model).toBe('dreamina-seedance-2-0-260128'); + expect(body.resolution).toBe('4K'); + expect(body.duration).toBe(15); + }); + + it('omits generate_audio for models without audio and passes seed for 1.x', async () => { + mockTaskFlow(succeededTask({ resolution: '1080p' })); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-1-0-pro', + prompt: 'hi', + seed: 11, + }), + ); + const body = sentBody(); + expect(body.generate_audio).toBeUndefined(); + expect(body.seed).toBe(11); + expect(body.resolution).toBe('1080p'); + }); + + it('derives a supported ratio from width/height and omits unsupported ones', async () => { + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + width: 1280, + height: 720, + }), + ); + expect(sentBody().ratio).toBe('16:9'); + + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ prompt: 'hi', width: 999, height: 100 }), + ); + expect(sentBody(2).ratio).toBeUndefined(); + }); + + it('maps input_reference/last_frame to first/last frame roles', async () => { + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + input_reference: 'https://example.com/first.png', + last_frame: 'https://example.com/last.png', + }), + ); + expect(sentBody().content).toEqual([ + { type: 'text', text: 'hi' }, + { + type: 'image_url', + image_url: { url: 'https://example.com/first.png' }, + role: 'first_frame', + }, + { + type: 'image_url', + image_url: { url: 'https://example.com/last.png' }, + role: 'last_frame', + }, + ]); + }); + + it('maps reference_images for the 2.0 series and rejects them elsewhere', async () => { + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-2-0', + prompt: 'hi', + reference_images: ['https://example.com/ref.png'], + }), + ); + expect(sentBody().content).toContainEqual({ + type: 'image_url', + image_url: { url: 'https://example.com/ref.png' }, + role: 'reference_image', + }); + + await expect( + withTestActor(() => + makeProvider().generate({ + model: 'seedance-1-0-pro', + prompt: 'hi', + reference_images: ['https://example.com/ref.png'], + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + }); + + it('rejects last_frame without a first frame', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + last_frame: 'https://example.com/last.png', + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); +}); + +// ── Polling / outcomes ────────────────────────────────────────────── + +describe('BytePlusVideoProvider.generate polling and outcomes', () => { + it('keeps polling while the task is queued/running', async () => { + mockTaskFlow( + { id: 'cgt-test-1', status: 'queued' }, + { id: 'cgt-test-1', status: 'running' }, + succeededTask(), + ); + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'hi' }), + ); + expect(result).toBe('https://ark.example/video.mp4'); + expect(fetchSpy).toHaveBeenCalledTimes(4); // 1 create + 3 polls + }); + + it('throws 400 upstream_failed when the task fails, without metering', async () => { + mockTaskFlow({ + id: 'cgt-test-1', + status: 'failed', + error: { code: 'moderation', message: 'blocked' }, + }); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 400, message: 'blocked' }); + expect(incrementUsageSpy).not.toHaveBeenCalled(); + }); + + it('maps upstream 5xx on create to a 502', async () => { + fetchSpy.mockResolvedValueOnce( + jsonResponse({ error: { message: 'boom' } }, 500), + ); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 502 }); + }); +}); + +// ── Metering ──────────────────────────────────────────────────────── + +describe('BytePlusVideoProvider.generate metering', () => { + it('bills the tokens the task reports at the resolution rate', async () => { + mockTaskFlow(succeededTask({ resolution: '1080p' })); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-2-0', + prompt: 'hi', + resolution: '1080p', + }), + ); + const model = findModel('dreamina-seedance-2-0-260128'); + const rate = model.costs!['video_tokens:1080p']; + expect(incrementUsageSpy).toHaveBeenCalledWith( + expect.anything(), + 'byteplus-video-generation:dreamina-seedance-2-0-260128:video_tokens:1080p', + 108_000, + 108_000 * rate * 1_000_000, + ); + }); + + it('bills seedance 1.5 pro at the silent rate when generate_audio is false', async () => { + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-1-5-pro', + prompt: 'hi', + generate_audio: false, + }), + ); + expect(sentBody().generate_audio).toBe(false); + const model = findModel('seedance-1-5-pro-251215'); + const rate = model.costs!['video_tokens:silent']; + expect(incrementUsageSpy).toHaveBeenCalledWith( + expect.anything(), + 'byteplus-video-generation:seedance-1-5-pro-251215:video_tokens:silent', + 108_000, + 108_000 * rate * 1_000_000, + ); + }); + + it('clamps the clip to what remaining credit buys', async () => { + // seedance 2.0 mini @720p: 1280×720×24/1024 = 21600 tokens/s at + // 0.00035¢/token → 7.56¢/s. Grant ~15¢ ≈ 2s... below the 4s + // minimum → 402. Grant ~40¢ → 5s requested, affordable. + const perSecondMicroCents = 21_600 * 0.00035 * 1_000_000; + remainingUsageSpy.mockResolvedValue(4.5 * perSecondMicroCents); + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ prompt: 'hi', seconds: 10 }), + ); + expect(sentBody().duration).toBe(4); + + remainingUsageSpy.mockResolvedValue(2 * perSecondMicroCents); + await expect( + withTestActor(() => + makeProvider().generate({ prompt: 'hi', seconds: 10 }), + ), + ).rejects.toMatchObject({ statusCode: 402 }); + }); +}); diff --git a/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.ts b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.ts new file mode 100644 index 000000000..496636ce4 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.ts @@ -0,0 +1,463 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IGenerateVideoParams, IVideoModel } from '../../types.js'; +import { capSecondsToRemainingCredits } from '../../creditCap.js'; +import { VideoProvider } from '../VideoProvider.js'; +import { + BYTEPLUS_VIDEO_GENERATION_MODELS, + BYTEPLUS_VIDEO_SPECS, + type BytePlusVideoSpec, +} from './models.js'; + +const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; +const DEFAULT_BASE_URL = 'https://ark.ap-southeast.bytepluses.com/api/v3'; +const DEFAULT_POLL_INTERVAL_MS = 5_000; +const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; +const DEFAULT_MODEL = 'dreamina-seedance-2-0-mini-260615'; +// Seedance 2.0 multimodal reference accepts up to 9 reference images. +const MAX_REFERENCE_IMAGES = 9; + +const ARK_RATIOS = ['16:9', '4:3', '1:1', '3:4', '9:16', '21:9']; + +type BytePlusVideoConfig = { + apiKey: string; + apiBaseUrl?: string; + /** Test hook — polling cadence for task status checks. */ + pollIntervalMs?: number; +}; + +interface ArkVideoTask { + id: string; + status: + 'queued' | 'running' | 'cancelled' | 'succeeded' | 'failed' | 'expired'; + content?: { video_url?: string }; + usage?: { completion_tokens?: number; total_tokens?: number }; + resolution?: string; + duration?: number; + error?: { code?: string; message?: string } | null; +} + +/** + * BytePlus ModelArk video generation provider (Seedance). + * + * Ark's video API is task-based rather than OpenAI-shaped: POST + * `/contents/generations/tasks` returns a task id, which is then polled via GET + * until it leaves queued/running. Billing is per video token (≈ duration × + * width × height × fps / 1024) with the authoritative count in the final task's + * `usage.completion_tokens`. + * https://docs.byteplus.com/en/docs/ModelArk/1520757 + */ +export class BytePlusVideoProvider extends VideoProvider { + #apiKey: string; + #baseUrl: string; + #pollIntervalMs: number; + #meteringService: MeteringService; + + constructor(config: BytePlusVideoConfig, meteringService: MeteringService) { + super(); + if (!config.apiKey) { + throw new Error('BytePlus video generation requires an API key'); + } + this.#apiKey = config.apiKey; + this.#baseUrl = (config.apiBaseUrl ?? DEFAULT_BASE_URL).replace( + /\/$/, + '', + ); + this.#pollIntervalMs = + config.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + this.#meteringService = meteringService; + } + + getDefaultModel(): string { + return DEFAULT_MODEL; + } + + async models(): Promise { + return BYTEPLUS_VIDEO_GENERATION_MODELS; + } + + async generate(params: IGenerateVideoParams): Promise { + const { + prompt, + model: requestedModel, + seconds, + duration, + size, + resolution, + width, + height, + seed, + generate_audio: generateAudio, + input_reference: inputReference, + last_frame: lastFrame, + reference_images: referenceImages, + test_mode: testMode, + } = params ?? {}; + + if (typeof prompt !== 'string' || !prompt.trim()) { + throw new HttpError(400, 'prompt must be a non-empty string', { + legacyCode: 'bad_request', + }); + } + + const model = this.#getModel(requestedModel); + const spec = BYTEPLUS_VIDEO_SPECS[model.id]; + + if (testMode) { + return DEFAULT_TEST_VIDEO_URL; + } + + const actor = Context.get('actor'); + if (!actor) { + throw new HttpError(401, 'Authentication required', { + legacyCode: 'unauthorized', + }); + } + + const resolutionKey = this.#normalizeResolution( + size ?? resolution, + model, + ); + // Audio is priced in for 1.5 Pro, so always send an explicit value to + // keep billing deterministic; Ark's own default is true. + const audioOn = spec.supportsAudio && generateAudio !== false; + const costKey = this.#costKey(model, resolutionKey, audioOn); + const centsPerToken = model.costs?.[costKey]; + if (!centsPerToken) { + throw new Error( + `No pricing configured for video model ${model.id} at ${resolutionKey}`, + ); + } + + const dims = spec.dims[resolutionKey]; + const tokensPerSecond = (dims.w * dims.h * 24) / 1024; + const perSecondMicroCents = tokensPerSecond * centsPerToken * 1_000_000; + + const requestedSeconds = Math.min( + this.#coerceSeconds(seconds ?? duration) ?? spec.duration.default, + spec.duration.max, + ); + // Ark durations are a contiguous integer range, so no ladder — + // clamp to whatever whole seconds the actor can still afford. + const cappedSeconds = await capSecondsToRemainingCredits({ + metering: this.#meteringService, + actor, + perSecondMicroCents, + requestedSeconds, + allowedSeconds: null, + minSeconds: spec.duration.min, + modelId: model.id, + }); + + const body: Record = { + model: model.id, + content: this.#buildContent(prompt, spec, model.id, { + inputReference, + lastFrame, + referenceImages, + }), + resolution: resolutionKey === '4k' ? '4K' : resolutionKey, + duration: cappedSeconds, + watermark: false, + }; + if (spec.supportsAudio) { + body.generate_audio = audioOn; + } + const ratio = this.#deriveRatio(width, height); + if (ratio) { + body.ratio = ratio; + } + if ( + spec.supportsSeed && + typeof seed === 'number' && + Number.isFinite(seed) + ) { + body.seed = Math.round(seed); + } + + const task = await this.#createTask(body); + const finalTask = await this.#pollUntilComplete(task.id); + + if (finalTask.status !== 'succeeded') { + const errorMessage = + finalTask.error?.message ?? + `Video generation ${finalTask.status}`; + // Ark's `failed` covers both user-input issues (content + // moderation) and upstream outages — same ambiguity as the + // Together provider, so expose it the same way: a 400 with + // `upstream_failed` that the alarm gate skips. + throw new HttpError(400, errorMessage, { + legacyCode: 'upstream_failed', + fields: { provider: 'byteplus' }, + }); + } + + const videoUrl = finalTask.content?.video_url; + if (typeof videoUrl !== 'string' || !videoUrl.trim()) { + throw new Error('BytePlus response did not include a video URL'); + } + + // Bill the tokens the task actually reports; fall back to the + // pre-flight estimate if usage is missing. + const finalResolutionKey = this.#normalizeResolution( + finalTask.resolution, + model, + resolutionKey, + ); + const finalCostKey = this.#costKey(model, finalResolutionKey, audioOn); + const finalCentsPerToken = model.costs?.[finalCostKey] ?? centsPerToken; + const tokens = + finalTask.usage?.completion_tokens ?? + Math.round(tokensPerSecond * cappedSeconds); + await this.#meteringService.incrementUsage( + actor, + `byteplus-video-generation:${model.id}:${finalCostKey}`, + tokens, + tokens * finalCentsPerToken * 1_000_000, + ); + + return videoUrl; + } + + #buildContent( + prompt: string, + spec: BytePlusVideoSpec, + modelId: string, + images: { + inputReference?: unknown; + lastFrame?: string; + referenceImages?: string[]; + }, + ): Array> { + const { inputReference, lastFrame, referenceImages } = images; + const content: Array> = [ + { type: 'text', text: prompt }, + ]; + + const firstFrame = + typeof inputReference === 'string' && inputReference.trim() + ? inputReference + : undefined; + const hasReferenceImages = + Array.isArray(referenceImages) && referenceImages.length > 0; + + // Ark treats first/last-frame and reference-image generation as + // mutually exclusive scenarios. + if (hasReferenceImages && (firstFrame || lastFrame)) { + throw new HttpError( + 400, + 'reference_images cannot be combined with input_reference/last_frame', + { legacyCode: 'bad_request' }, + ); + } + + if (hasReferenceImages) { + if (!spec.supportsReferenceImages) { + throw new HttpError( + 400, + `${modelId} does not support reference_images`, + { legacyCode: 'bad_request' }, + ); + } + for (const img of referenceImages!.slice(0, MAX_REFERENCE_IMAGES)) { + if (typeof img !== 'string' || !img.trim()) continue; + content.push({ + type: 'image_url', + image_url: { url: img }, + role: 'reference_image', + }); + } + return content; + } + + if (lastFrame && !firstFrame) { + throw new HttpError( + 400, + 'last_frame requires a first-frame image via input_reference', + { legacyCode: 'bad_request' }, + ); + } + if (firstFrame) { + content.push({ + type: 'image_url', + image_url: { url: firstFrame }, + ...(lastFrame ? { role: 'first_frame' } : {}), + }); + } + if (lastFrame) { + if (!spec.supportsLastFrame) { + throw new HttpError( + 400, + `${modelId} does not support last_frame`, + { legacyCode: 'bad_request' }, + ); + } + content.push({ + type: 'image_url', + image_url: { url: lastFrame }, + role: 'last_frame', + }); + } + + return content; + } + + async #createTask(body: Record): Promise { + return (await this.#request('POST', '/contents/generations/tasks', { + body, + })) as ArkVideoTask; + } + + async #pollUntilComplete(taskId: string): Promise { + const start = Date.now(); + for (;;) { + const task = (await this.#request( + 'GET', + `/contents/generations/tasks/${taskId}`, + )) as ArkVideoTask; + if (task.status !== 'queued' && task.status !== 'running') { + return task; + } + if (Date.now() - start > DEFAULT_TIMEOUT_MS) { + throw new Error( + 'Timed out waiting for BytePlus video generation to complete', + ); + } + await this.#delay(this.#pollIntervalMs); + } + } + + async #request( + method: string, + path: string, + opts: { body?: Record } = {}, + ): Promise { + const response = await fetch(`${this.#baseUrl}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.#apiKey}`, + ...(opts.body ? { 'Content-Type': 'application/json' } : {}), + }, + ...(opts.body ? { body: JSON.stringify(opts.body) } : {}), + }); + const payload = (await response.json().catch(() => ({}))) as Record< + string, + unknown + >; + if (!response.ok) { + const message = + ((payload.error as Record) + ?.message as string) ?? + `BytePlus video API error (status ${response.status})`; + throw new HttpError(response.status >= 500 ? 502 : 400, message, { + legacyCode: 'upstream_failed', + fields: { provider: 'byteplus' }, + }); + } + return payload; + } + + async #delay(ms: number): Promise { + return await new Promise((resolve) => setTimeout(resolve, ms)); + } + + #getModel(requestedModel?: string): IVideoModel { + const wanted = (requestedModel ?? '').trim().toLowerCase(); + const found = BYTEPLUS_VIDEO_GENERATION_MODELS.find( + (m) => + m.id === wanted || + m.puterId === wanted || + m.aliases?.some((a) => a.toLowerCase() === wanted), + ); + return ( + found ?? + BYTEPLUS_VIDEO_GENERATION_MODELS.find( + (m) => m.id === DEFAULT_MODEL, + )! + ); + } + + /** '480p' | '720p' | '1080p' | '4k', falling back to the model default. */ + #normalizeResolution( + candidate: unknown, + model: IVideoModel, + fallback?: string, + ): string { + const spec = BYTEPLUS_VIDEO_SPECS[model.id]; + if (typeof candidate === 'string') { + const normalized = candidate.trim().toLowerCase(); + if (spec.dims[normalized]) return normalized; + } + return fallback ?? model.dimensions![0].toLowerCase(); + } + + #costKey( + model: IVideoModel, + resolutionKey: string, + audioOn: boolean, + ): string { + const costs = model.costs ?? {}; + if (costs[`video_tokens:${resolutionKey}`] !== undefined) { + return `video_tokens:${resolutionKey}`; + } + const audioKey = audioOn ? 'video_tokens:audio' : 'video_tokens:silent'; + if (costs[audioKey] !== undefined) { + return audioKey; + } + return 'video_tokens'; + } + + #coerceSeconds(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) { + const rounded = Math.round(value); + return rounded > 0 ? rounded : undefined; + } + if (typeof value === 'string') { + const numeric = Number.parseInt(value, 10); + return Number.isFinite(numeric) && numeric > 0 + ? numeric + : undefined; + } + return undefined; + } + + /** Snap width/height to one of Ark's supported aspect-ratio strings. */ + #deriveRatio(width?: number, height?: number): string | undefined { + if ( + typeof width !== 'number' || + typeof height !== 'number' || + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { + return undefined; + } + const gcd = (a: number, b: number): number => + b === 0 ? a : gcd(b, a % b); + const d = gcd(Math.round(width), Math.round(height)) || 1; + const candidate = `${Math.round(width) / d}:${Math.round(height) / d}`; + // Unsupported ratios are omitted so Ark's `adaptive` default applies. + return ARK_RATIOS.includes(candidate) ? candidate : undefined; + } +} diff --git a/src/backend/drivers/ai-video/providers/byteplus/models.ts b/src/backend/drivers/ai-video/providers/byteplus/models.ts new file mode 100644 index 000000000..ecb6c9e52 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/byteplus/models.ts @@ -0,0 +1,258 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IVideoModel } from '../../types.js'; + +/** + * Ark-specific model behavior the shared IVideoModel shape can't carry. Keyed + * by model id. + */ +export interface BytePlusVideoSpec { + /** Valid `duration` range (integer seconds) and Ark's default. */ + duration: { min: number; max: number; default: number }; + /** + * 16:9 output dimensions per resolution key, from the create-task API's + * ratio table. Used only for pre-flight cost estimates — actual billing + * uses the `usage.completion_tokens` the task reports. + */ + dims: Record; + /** Supports `generate_audio` (Seedance 2.0 series + 1.5 Pro). */ + supportsAudio: boolean; + /** Supports first+last frame image-to-video. */ + supportsLastFrame: boolean; + /** Supports multimodal `reference_image` inputs (Seedance 2.0 series). */ + supportsReferenceImages: boolean; + /** Supports the `seed` param (not the Seedance 2.0 series). */ + supportsSeed: boolean; +} + +// Duration ladder for the driver's normalization: element 0 is the fallback +// default, the rest enumerate the model's contiguous valid range. +const seconds = (def: number, min: number, max: number): number[] => [ + def, + ...Array.from({ length: max - min + 1 }, (_, i) => min + i).filter( + (s) => s !== def, + ), +]; + +const FPS = [24]; + +// USD per million tokens → usd-cents per token. +const perMToken = (usd: number): number => (usd * 100) / 1_000_000; + +// Hardcoded from https://docs.byteplus.com/en/docs/ModelArk/1544106 (pricing, +// "online inference / input without video" rates — reference-video input is +// not exposed through this driver) and the create-task API reference +// https://docs.byteplus.com/en/docs/ModelArk/1520757 (capabilities). +// +// Video is billed per token: tokens ≈ duration × width × height × fps / 1024, +// with the authoritative count returned as `usage.completion_tokens`. +// `default-duration-per-video` is the estimated cents for a 5s clip at the +// model's default resolution — it exists for cross-provider cost sorting and +// display, not billing. +// +// dreamina-seedance-2-5-260628 is priced on the pricing page but the API +// reference still lists its API access as "available soon", so it's +// deliberately absent here. +export const BYTEPLUS_VIDEO_GENERATION_MODELS: IVideoModel[] = [ + { + id: 'dreamina-seedance-2-0-260128', + puterId: 'byteplus:byteplus/dreamina-seedance-2-0-260128', + aliases: [ + 'dreamina-seedance-2-0', + 'byteplus/dreamina-seedance-2-0', + 'seedance-2-0', + ], + name: 'Dreamina Seedance 2.0', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + 'video_tokens:480p': perMToken(7.0), + 'video_tokens:720p': perMToken(7.0), + 'video_tokens:1080p': perMToken(7.7), + 'video_tokens:4k': perMToken(4.0), + 'default-duration-per-video': 76, + }, + durationSeconds: seconds(5, 4, 15), + // '4K' repeats Ark's documented casing so the driver's strict + // `includes` match accepts either spelling. + dimensions: ['720p', '480p', '1080p', '4k', '4K'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:dreamina-seedance-2-0-260128:video_tokens:720p', + }, + { + id: 'dreamina-seedance-2-0-fast-260128', + puterId: 'byteplus:byteplus/dreamina-seedance-2-0-fast-260128', + aliases: [ + 'dreamina-seedance-2-0-fast', + 'byteplus/dreamina-seedance-2-0-fast', + 'seedance-2-0-fast', + ], + name: 'Dreamina Seedance 2.0 Fast', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + video_tokens: perMToken(5.6), + 'default-duration-per-video': 60, + }, + durationSeconds: seconds(5, 4, 15), + dimensions: ['720p', '480p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:dreamina-seedance-2-0-fast-260128:video_tokens', + }, + { + id: 'dreamina-seedance-2-0-mini-260615', + puterId: 'byteplus:byteplus/dreamina-seedance-2-0-mini-260615', + aliases: [ + 'dreamina-seedance-2-0-mini', + 'byteplus/dreamina-seedance-2-0-mini', + 'seedance-2-0-mini', + ], + name: 'Dreamina Seedance 2.0 Mini', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + video_tokens: perMToken(3.5), + 'default-duration-per-video': 38, + }, + durationSeconds: seconds(5, 4, 15), + dimensions: ['720p', '480p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:dreamina-seedance-2-0-mini-260615:video_tokens', + }, + { + id: 'seedance-1-5-pro-251215', + puterId: 'byteplus:byteplus/seedance-1-5-pro-251215', + aliases: ['seedance-1-5-pro', 'byteplus/seedance-1-5-pro'], + name: 'Seedance 1.5 Pro', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + 'video_tokens:audio': perMToken(2.4), + 'video_tokens:silent': perMToken(1.2), + 'default-duration-per-video': 26, + }, + durationSeconds: seconds(5, 4, 12), + dimensions: ['720p', '480p', '1080p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:seedance-1-5-pro-251215:video_tokens:audio', + }, + { + id: 'seedance-1-0-pro-250528', + puterId: 'byteplus:byteplus/seedance-1-0-pro-250528', + aliases: ['seedance-1-0-pro', 'byteplus/seedance-1-0-pro'], + name: 'Seedance 1.0 Pro', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + video_tokens: perMToken(2.5), + 'default-duration-per-video': 61, + }, + durationSeconds: seconds(5, 2, 12), + dimensions: ['1080p', '480p', '720p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:seedance-1-0-pro-250528:video_tokens', + }, + { + id: 'seedance-1-0-pro-fast-251015', + puterId: 'byteplus:byteplus/seedance-1-0-pro-fast-251015', + aliases: ['seedance-1-0-pro-fast', 'byteplus/seedance-1-0-pro-fast'], + name: 'Seedance 1.0 Pro Fast', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + video_tokens: perMToken(1.0), + 'default-duration-per-video': 24, + }, + durationSeconds: seconds(5, 2, 12), + dimensions: ['1080p', '480p', '720p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:seedance-1-0-pro-fast-251015:video_tokens', + }, +]; + +const SEEDANCE_2_0_DIMS = { + '480p': { w: 864, h: 496 }, + '720p': { w: 1280, h: 720 }, + '1080p': { w: 1920, h: 1080 }, + '4k': { w: 3840, h: 2160 }, +}; + +const SEEDANCE_1_0_DIMS = { + '480p': { w: 864, h: 480 }, + '720p': { w: 1248, h: 704 }, + '1080p': { w: 1920, h: 1088 }, +}; + +export const BYTEPLUS_VIDEO_SPECS: Record = { + 'dreamina-seedance-2-0-260128': { + duration: { min: 4, max: 15, default: 5 }, + dims: SEEDANCE_2_0_DIMS, + supportsAudio: true, + supportsLastFrame: true, + supportsReferenceImages: true, + supportsSeed: false, + }, + 'dreamina-seedance-2-0-fast-260128': { + duration: { min: 4, max: 15, default: 5 }, + dims: SEEDANCE_2_0_DIMS, + supportsAudio: true, + supportsLastFrame: true, + supportsReferenceImages: true, + supportsSeed: false, + }, + 'dreamina-seedance-2-0-mini-260615': { + duration: { min: 4, max: 15, default: 5 }, + dims: SEEDANCE_2_0_DIMS, + supportsAudio: true, + supportsLastFrame: true, + supportsReferenceImages: true, + supportsSeed: false, + }, + 'seedance-1-5-pro-251215': { + duration: { min: 4, max: 12, default: 5 }, + dims: SEEDANCE_2_0_DIMS, + supportsAudio: true, + supportsLastFrame: true, + supportsReferenceImages: false, + supportsSeed: true, + }, + 'seedance-1-0-pro-250528': { + duration: { min: 2, max: 12, default: 5 }, + dims: SEEDANCE_1_0_DIMS, + supportsAudio: false, + supportsLastFrame: true, + supportsReferenceImages: false, + supportsSeed: true, + }, + 'seedance-1-0-pro-fast-251015': { + duration: { min: 2, max: 12, default: 5 }, + dims: SEEDANCE_1_0_DIMS, + supportsAudio: false, + supportsLastFrame: false, + supportsReferenceImages: false, + supportsSeed: true, + }, +}; diff --git a/src/backend/drivers/ai-video/types.ts b/src/backend/drivers/ai-video/types.ts index d1624dace..0bc6df384 100644 --- a/src/backend/drivers/ai-video/types.ts +++ b/src/backend/drivers/ai-video/types.ts @@ -60,6 +60,7 @@ export interface IGenerateVideoParams { output_format?: string; output_quality?: number; negative_prompt?: string; + generate_audio?: boolean; reference_images?: string[]; frame_images?: object[]; last_frame?: string;