From 19cd5f4476e3c20ea4403945b401fbf4ba5c4279 Mon Sep 17 00:00:00 2001 From: Nariman Jelveh Date: Sat, 15 Aug 2026 21:08:17 -0700 Subject: [PATCH 1/2] feat: add BytePlus ModelArk providers (chat, image, video) (#3498) * feat: add BytePlus ModelArk chat provider Adds BytePlus ModelArk as a provider for the puter-chat-completion driver, following the MiniMax/ZAI providers as reference per doc/contributing-apis.md. - OpenAI-compatible endpoint at ark.ap-southeast.bytepluses.com/api/v3 (apiBaseUrl config selects the region) - Static catalog of 16 chat models (Seed 2.x/1.x incl. vision, GLM, DeepSeek, GPT-OSS) with limits and per-token pricing from the official docs - Passes Ark's thinking/response_format/stop params through custom; normalizes reasoning_content to reasoning - Bare deepseek-v4-* names stay with the first-party DeepSeek provider; BytePlus only claims prefixed aliases - Offline unit tests (mocked SDK against a real test server) plus an env-gated integration test * feat: add BytePlus image and video providers Extends the BytePlus ModelArk integration to the puter-image-generation and puter-video-generation drivers, reusing the same services.byteplus API key and regional apiBaseUrl as the chat provider. Image (Seedream/SeedEdit via OpenAI-compatible /images/generations): - dola-seedream-5-0-pro (pixel-tier pricing + billed input images from the 2nd on), seedream-5-0-lite, 4-5, 4-0, and seededit-3-0-i2i - quality tiers 1K/1.5K/2K; aspect ratios resolve to Ark's documented pixel sizes; explicit WxH passes through with Ark's bounds enforced Video (Seedance via Ark's async /contents/generations/tasks + polling): - Seedance 2.0 / 2.0 Fast / 2.0 Mini / 1.5 Pro / 1.0 Pro / 1.0 Pro Fast (2.5 is priced but its API isn't live yet, so it's excluded) - per-video-token billing from usage.completion_tokens, with per-second estimates feeding the credit cap; audio vs silent rates for 1.5 Pro - first/last frame and reference-image inputs; generate_audio param added to IGenerateVideoParams Pricing and capabilities hardcoded from the official docs (ModelArk pages 1544106, 1330310, 1520757, 1521309, 1541523). Offline unit tests mock the SDK / global fetch; integration tests are env-gated on PUTER_TEST_AI_BYTEPLUS_API_KEY. * fix: correct BytePlus catalogs and validation against the live API Verified the three BytePlus providers against ModelArk with a real key; these are the mismatches that surfaced. - Drop seededit-3-0-i2i-250628. Ark reports it as Shutdown and every request 404s. Its now-unreachable image-to-image branches in the provider go with it. - seedream-4-5 and the 5.0 series enforce a 3,686,400 pixel minimum, so they only accept the 2K tier. Mark them 2k-only and snap an unsupported tier up to the nearest allowed one, which also keeps the aspect-ratio table from mapping to a sub-minimum size. - glm-4-7 has a 204,800 token context, not 256K. - Guard the actor in the image provider like the video provider does. - Round a sub-minimum video duration up to the shortest supported clip instead of reporting it as insufficient funds. - Gate video resolution on the model's own dimensions; the dims table is shared across a family and accepts more than any one model does. * Tighten BytePlus AI provider handling Extract shared reasoning-content normalization for OpenAI-style chat providers, and harden BytePlus image/video behavior. This updates image tier and size validation, normalizes aspect ratios and input image refs, prevents mismatched BytePlus key/base URL fallback config, makes video resolution matching case-insensitive, and rejects excess reference images instead of silently truncating them. Tests were expanded to cover the new BytePlus request and validation paths. --- config.template.jsonc | 10 + src/backend/clients/event/types.ts | 9 +- .../controllers/auth/AuthController.ts | 3 +- .../controllers/drivers/DriverController.ts | 3 +- .../controllers/puterai/PuterAIController.ts | 3 +- src/backend/core/http/middleware/gates.ts | 3 +- src/backend/core/http/types.ts | 11 +- .../ChatCompletionDriver.edges.test.ts | 2 + .../drivers/ai-chat/ChatCompletionDriver.ts | 13 + .../BytePlusProvider.integration.test.ts | 63 ++ .../byteplus/BytePlusProvider.test.ts | 815 ++++++++++++++++++ .../providers/byteplus/BytePlusProvider.ts | 180 ++++ .../ai-chat/providers/byteplus/models.ts | 196 +++++ .../neuralwatt/NeuralwattProvider.ts | 22 +- .../ai-chat/providers/neuralwatt/models.ts | 18 +- .../ai-chat/providers/zai/ZAIProvider.ts | 30 +- .../drivers/ai-chat/utils/OpenAIUtil.js | 30 + .../drivers/ai-image/ImageGenerationDriver.ts | 31 +- src/backend/drivers/ai-image/inputImage.ts | 28 +- .../BytePlusImageProvider.integration.test.ts | 66 ++ .../byteplus/BytePlusImageProvider.test.ts | 550 ++++++++++++ .../byteplus/BytePlusImageProvider.ts | 345 ++++++++ .../ai-image/providers/byteplus/models.ts | 164 ++++ .../drivers/ai-video/VideoGenerationDriver.ts | 38 +- .../BytePlusVideoProvider.integration.test.ts | 67 ++ .../byteplus/BytePlusVideoProvider.test.ts | 464 ++++++++++ .../byteplus/BytePlusVideoProvider.ts | 476 ++++++++++ .../ai-video/providers/byteplus/models.ts | 256 ++++++ src/backend/drivers/ai-video/types.ts | 1 + src/backend/drivers/meta.ts | 9 +- src/backend/drivers/types.ts | 3 +- src/backend/types.ts | 10 +- 32 files changed, 3814 insertions(+), 105 deletions(-) create mode 100644 src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.integration.test.ts create mode 100644 src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts create mode 100644 src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts create mode 100644 src/backend/drivers/ai-chat/providers/byteplus/models.ts create mode 100644 src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.integration.test.ts create mode 100644 src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.test.ts create mode 100644 src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.ts create mode 100644 src/backend/drivers/ai-image/providers/byteplus/models.ts create mode 100644 src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.integration.test.ts create mode 100644 src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.test.ts create mode 100644 src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.ts create mode 100644 src/backend/drivers/ai-video/providers/byteplus/models.ts diff --git a/config.template.jsonc b/config.template.jsonc index d11ba607a..22a3271bf 100644 --- a/config.template.jsonc +++ b/config.template.jsonc @@ -341,6 +341,16 @@ "apiKey": "", "apiBaseUrl": "https://llm.onerouter.pro/v1" }, + // BytePlus ModelArk. One key powers chat (Seed/GLM/DeepSeek), image + // generation (Seedream) and video generation (Seedance); `apiBaseUrl` + // selects the region; see + // https://docs.byteplus.com/en/docs/ModelArk/1330310 for options. + "byteplus": { + "apiKey": "", + "apiBaseUrl": "https://ark.ap-southeast.bytepluses.com/api/v3" + }, + "zai": { "apiKey": "" }, + "alibaba": { "apiKey": "" }, "together-ai": { "apiKey": "" }, // Local Ollama. `enabled: false` skips the auto-probe at startup // (otherwise Puter logs ECONNREFUSED on every boot when no Ollama diff --git a/src/backend/clients/event/types.ts b/src/backend/clients/event/types.ts index b8a57511c..7e137d3fc 100644 --- a/src/backend/clients/event/types.ts +++ b/src/backend/clients/event/types.ts @@ -542,11 +542,10 @@ export type EventKey = keyof EventMap & string; // Generates a wildcard for every non-final dot-separated prefix of K. export type WildcardPrefixes = K extends `${infer Head}.${infer Tail}` - ? - | `${Head}.*` - | (Tail extends `${string}.${string}` - ? `${Head}.${WildcardPrefixes}` - : never) + ? | `${Head}.*` + | (Tail extends `${string}.${string}` + ? `${Head}.${WildcardPrefixes}` + : never) : never; export type ListenKey = EventKey | WildcardPrefixes; diff --git a/src/backend/controllers/auth/AuthController.ts b/src/backend/controllers/auth/AuthController.ts index 13180b12f..46fc9d50b 100644 --- a/src/backend/controllers/auth/AuthController.ts +++ b/src/backend/controllers/auth/AuthController.ts @@ -1150,8 +1150,7 @@ export class AuthController extends PuterController { is_temp: user!.password === null && user!.email === null, ip: (req?.headers?.['x-forwarded-for'] as - | string - | undefined) || + string | undefined) || ( req as unknown as { connection?: { remoteAddress?: string }; diff --git a/src/backend/controllers/drivers/DriverController.ts b/src/backend/controllers/drivers/DriverController.ts index 1e93313e5..fb0d169c7 100644 --- a/src/backend/controllers/drivers/DriverController.ts +++ b/src/backend/controllers/drivers/DriverController.ts @@ -299,8 +299,7 @@ export class DriverController extends PuterController { if (req.actor) { const permService = this.services.permission as unknown as - | PermissionService - | undefined; + PermissionService | undefined; if (permService) { // Build via PermissionUtil.join so any `:` in a driver or // interface name is escaped — raw interpolation would let a diff --git a/src/backend/controllers/puterai/PuterAIController.ts b/src/backend/controllers/puterai/PuterAIController.ts index 9ccf9be01..87d8a541b 100644 --- a/src/backend/controllers/puterai/PuterAIController.ts +++ b/src/backend/controllers/puterai/PuterAIController.ts @@ -587,8 +587,7 @@ export class PuterAIController extends PuterController { text: extractTextContent( ( messageResult.message as - | Record - | undefined + Record | undefined )?.content, ), index: 0, diff --git a/src/backend/core/http/middleware/gates.ts b/src/backend/core/http/middleware/gates.ts index de2ce8dbb..c67013d1e 100644 --- a/src/backend/core/http/middleware/gates.ts +++ b/src/backend/core/http/middleware/gates.ts @@ -413,8 +413,7 @@ export const assertVerifiedAccount = ( */ export const assertPhoneVerified = ( user: - | { phone?: unknown; requires_phone_verification?: unknown } - | undefined, + { phone?: unknown; requires_phone_verification?: unknown } | undefined, ): void => { if (user?.phone && !user?.requires_phone_verification) return; throw new HttpError(403, 'Please verify your phone number to continue', { diff --git a/src/backend/core/http/types.ts b/src/backend/core/http/types.ts index 98809b8f2..3df062d8f 100644 --- a/src/backend/core/http/types.ts +++ b/src/backend/core/http/types.ts @@ -26,12 +26,7 @@ import type { Actor } from '../actor'; * it instead of accepting any token that authenticates. */ export type TokenSource = - | 'body' - | 'header' - | 'x-api-key' - | 'cookie' - | 'query' - | 'handshake'; + 'body' | 'header' | 'x-api-key' | 'cookie' | 'query' | 'handshake'; /** * Every route method PuterRouter exposes. Mirrors the express router surface @@ -441,9 +436,7 @@ export type AuthRequired = O extends { ? true : O extends { requireSubscription: - | true - | readonly string[] - | string[]; + true | readonly string[] | string[]; } ? true : false; diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts index 7096a683d..d0eff6479 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts @@ -73,6 +73,7 @@ const FULL_PROVIDER_CONFIG = { 'together-ai': { apiKey: 'k' }, openrouter: { apiKey: 'k', apiBaseUrl: 'https://openrouter.test' }, infron: { apiKey: 'k' }, + byteplus: { apiKey: 'k' }, neuralwatt: { apiKey: 'k' }, // Suppress auto-discovery of a developer's local Ollama. ollama: { enabled: false }, @@ -184,6 +185,7 @@ describe('ChatCompletionDriver provider registration', () => { 'together-ai', 'openrouter', 'infron', + 'byteplus', 'neuralwatt', 'fake-chat', ]) { diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts index 8ca8347a5..b848d63f2 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -35,6 +35,7 @@ import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js'; import { AlibabaProvider } from './providers/alibaba/AlibabaProvider.js'; import { AzureChatProvider } from './providers/azure/AzureChatProvider.js'; import { AzureResponsesProvider } from './providers/azure/AzureResponsesProvider.js'; +import { BytePlusProvider } from './providers/byteplus/BytePlusProvider.js'; import { ClaudeProvider } from './providers/claude/ClaudeProvider.js'; import { DeepSeekProvider } from './providers/deepseek/DeepSeekProvider.js'; import { FakeChatProvider } from './providers/FakeChatProvider.js'; @@ -1251,6 +1252,18 @@ export class ChatCompletionDriver extends PuterDriver { ); } + const byteplus = providers['byteplus']; + const byteplusKey = readKey(byteplus); + if (byteplusKey) { + this.#providers['byteplus'] = new BytePlusProvider( + { + apiKey: byteplusKey, + apiBaseUrl: byteplus?.apiBaseUrl as string | undefined, + }, + metering, + ); + } + const neuralwatt = providers['neuralwatt']; const neuralwattKey = readKey(neuralwatt); if (neuralwattKey) { diff --git a/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.integration.test.ts new file mode 100644 index 000000000..3adcb602e --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.integration.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the BytePlus ModelArk provider. + * + * Uses `seed-1-6-flash-250715` with `thinking: disabled` passed through + * `custom`. Ark's seed models default to deep reasoning and route those + * tokens to a `reasoning_content` field, leaving `content` empty under + * tight budgets. Disabling thinking forces a plain text response so the + * usual `message.content` assertion works. Skipped when + * `PUTER_TEST_AI_BYTEPLUS_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + INTEGRATION_TEST_TIMEOUT_MS, + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { BytePlusProvider } from './BytePlusProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_BYTEPLUS_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))('BytePlusProvider (integration)', () => { + it('returns a non-empty completion from seed-1-6-flash-250715', { timeout: INTEGRATION_TEST_TIMEOUT_MS }, async () => { + const provider = new BytePlusProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'seed-1-6-flash-250715', + messages: [{ role: 'user', content: 'Say hi in one word.' }], + max_tokens: 16, + custom: { thinking: { type: 'disabled' } }, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts new file mode 100644 index 000000000..478794a59 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts @@ -0,0 +1,815 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for BytePlusProvider. + * + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock + * redis) and constructs BytePlusProvider directly against the live + * wired `MeteringService` so the recording side is exercised + * end-to-end. The OpenAI SDK is mocked at the module boundary — + * ModelArk is OpenAI-compatible so the provider talks to it through + * the same client — so the provider never reaches the network. The + * companion integration test (BytePlusProvider.integration.test.ts) + * exercises the real ModelArk endpoint. + */ + +import { Writable } from 'node:stream'; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import { SYSTEM_ACTOR } from '../../../../core/actor.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { AIChatStream } from '../../utils/Streaming.js'; +import { BytePlusProvider } from './BytePlusProvider.js'; +import { BYTEPLUS_MODELS } from './models.js'; + +// -- OpenAI SDK mock ------------------------------------------------- +// +// `vi.hoisted` lets us share spies between the (hoisted) factory and +// the test body so each test can stub `chat.completions.create` with +// the response shape it cares about. ModelArk uses the OpenAI wire +// shape so the provider talks to it via the OpenAI SDK. + +const { createMock, openAICtor } = vi.hoisted(() => { + const createMock = vi.fn(); + const openAICtor = vi.fn(); + return { createMock, openAICtor }; +}); + +vi.mock('openai', () => { + const OpenAICtor = vi.fn().mockImplementation(function ( + this: Record, + opts: unknown, + ) { + openAICtor(opts); + this.chat = { completions: { create: createMock } }; + }); + // Some providers (e.g. OllamaChatProvider) import the default export + // and access `.OpenAI` on it, so expose the same constructor under + // both shapes — the test server boots every provider, not just + // BytePlus. + return { OpenAI: OpenAICtor, default: { OpenAI: OpenAICtor } }; +}); + +// -- Test harness ---------------------------------------------------- + +let server: PuterServer; +let recordSpy: MockInstance; + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = ( + config: { apiKey?: string; apiBaseUrl?: string } = {}, +) => { + const provider = new BytePlusProvider( + { + apiKey: config.apiKey ?? 'test-key', + ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}), + }, + server.services.metering, + ); + return { provider }; +}; + +const asAsyncIterable = (items: T[]): AsyncIterable => ({ + async *[Symbol.asyncIterator]() { + for (const item of items) { + yield item; + } + }, +}); + +const makeCapturingChatStream = () => { + const chunks: string[] = []; + const sink = new Writable({ + write(chunk, _enc, cb) { + chunks.push(chunk.toString('utf8')); + cb(); + }, + }); + const chatStream = new AIChatStream({ stream: sink }); + return { + chatStream, + events: () => + chunks + .join('') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)), + }; +}; + +beforeEach(() => { + createMock.mockReset(); + openAICtor.mockReset(); + // Spy on the live MeteringService — we don't replace the impl + // (that would skip the recording side we want covered) but we + // capture the calls the provider makes so per-test assertions + // can verify metering shape. + recordSpy = vi.spyOn(server.services.metering, 'utilRecordUsageObject'); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +// -- Construction ---------------------------------------------------- + +describe('BytePlusProvider construction', () => { + it('points the OpenAI SDK at the ModelArk base URL with the configured key', () => { + makeProvider(); + expect(openAICtor).toHaveBeenCalledTimes(1); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://ark.ap-southeast.bytepluses.com/api/v3', + }); + }); + + it('honours a custom apiBaseUrl override (e.g. the eu-west region)', () => { + makeProvider({ + apiBaseUrl: 'https://ark.eu-west.bytepluses.com/api/v3', + }); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://ark.eu-west.bytepluses.com/api/v3', + }); + }); +}); + +// -- Model catalog --------------------------------------------------- + +describe('BytePlusProvider model catalog', () => { + it('returns seed-2-0-lite-260428 as the default', () => { + const { provider } = makeProvider(); + expect(provider.getDefaultModel()).toBe('seed-2-0-lite-260428'); + }); + + it('exposes the static BYTEPLUS_MODELS list verbatim from models()', () => { + const { provider } = makeProvider(); + expect(provider.models()).toBe(BYTEPLUS_MODELS); + }); + + it('list() flattens canonical ids and aliases', () => { + const { provider } = makeProvider(); + const names = provider.list(); + for (const m of BYTEPLUS_MODELS) { + expect(names).toContain(m.id); + for (const a of m.aliases ?? []) { + expect(names).toContain(a); + } + } + // Sanity: undated series aliases resolve alongside canonical ids. + expect(names).toContain('seed-1-6'); + expect(names).toContain('byteplus/seed-1-6'); + expect(names).toContain('byteplus/seed-1-6-250915'); + }); + + it('never claims the bare deepseek-v4 names owned by the DeepSeek provider', () => { + const { provider } = makeProvider(); + const names = provider.list(); + expect(names).not.toContain('deepseek-v4-pro'); + expect(names).not.toContain('deepseek-v4-flash'); + expect(names).toContain('byteplus/deepseek-v4-pro'); + expect(names).toContain('byteplus/deepseek-v4-flash'); + }); +}); + +// -- Request shape --------------------------------------------------- + +describe('BytePlusProvider.complete request shape', () => { + const baseCompletion = { + choices: [ + { + message: { content: 'hi', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }; + + it('forwards model, messages, and bare-bones request without optional knobs', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'hello' }], + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect(args.model).toBe('seed-1-6-250915'); + expect(args.messages).toEqual([{ role: 'user', content: 'hello' }]); + // Optional generation knobs should be absent unless supplied. + expect('max_tokens' in args).toBe(false); + expect('temperature' in args).toBe(false); + expect('top_p' in args).toBe(false); + expect('tools' in args).toBe(false); + expect('tool_choice' in args).toBe(false); + expect('thinking' in args).toBe(false); + }); + + it('forwards max_tokens, temperature, top_p, tools, and tool_choice when supplied', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + const tools = [ + { + type: 'function', + function: { + name: 'lookup', + description: 'find a thing', + parameters: { + type: 'object', + properties: { q: { type: 'string' } }, + required: ['q'], + }, + }, + }, + ]; + + await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'hi' }], + max_tokens: 256, + temperature: 0.4, + top_p: 0.9, + tools, + tool_choice: 'auto', + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect(args.max_tokens).toBe(256); + expect(args.temperature).toBe(0.4); + expect(args.top_p).toBe(0.9); + expect(args.tools).toBe(tools); + expect(args.tool_choice).toBe('auto'); + }); + + it('forwards Ark-specific custom params (thinking, stop, response_format)', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'hi' }], + custom: { + thinking: { type: 'disabled' }, + stop: ['\n\n'], + response_format: { type: 'json_object' }, + }, + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect(args.thinking).toEqual({ type: 'disabled' }); + expect(args.stop).toEqual(['\n\n']); + expect(args.response_format).toEqual({ type: 'json_object' }); + }); + + it('strips Anthropic-style cache_control from messages before sending', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [ + { + role: 'user', + content: 'hi', + cache_control: { type: 'ephemeral' }, + } as unknown as { role: string; content: string }, + ], + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect('cache_control' in args.messages[0]).toBe(false); + }); + + it('only sets stream_options.include_usage when streaming', async () => { + const { provider } = makeProvider(); + // Non-stream path. + createMock.mockResolvedValueOnce(baseCompletion); + await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'hi' }], + stream: false, + }), + ); + const [nonStreamArgs] = createMock.mock.calls[0]!; + expect(nonStreamArgs.stream).toBe(false); + expect('stream_options' in nonStreamArgs).toBe(false); + + // Stream path. + createMock.mockReturnValueOnce(asAsyncIterable([])); + await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'hi' }], + stream: true, + }), + ); + const [streamArgs] = createMock.mock.calls[1]!; + expect(streamArgs.stream).toBe(true); + expect(streamArgs.stream_options).toEqual({ include_usage: true }); + }); +}); + +// -- Model resolution ------------------------------------------------ + +describe('BytePlusProvider model resolution', () => { + const baseCompletion = { + choices: [ + { + message: { content: 'ok', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }; + + it('resolves an exact canonical id', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'glm-4-7-251222', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + expect(createMock.mock.calls[0]![0].model).toBe('glm-4-7-251222'); + expect(recordSpy).toHaveBeenCalledWith( + expect.any(Object), + expect.anything(), + 'byteplus:glm-4-7-251222', + expect.any(Object), + ); + }); + + it('resolves an undated series alias to the newest snapshot (alias rewriting)', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'seed-1-6', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + // The wire model should be the canonical dated id, not the alias. + expect(createMock.mock.calls[0]![0].model).toBe('seed-1-6-250915'); + expect(recordSpy).toHaveBeenCalledWith( + expect.any(Object), + expect.anything(), + 'byteplus:seed-1-6-250915', + expect.any(Object), + ); + }); + + it('falls back to the default model when given an unknown id', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'totally-not-a-real-model', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + expect(createMock.mock.calls[0]![0].model).toBe( + 'seed-2-0-lite-260428', + ); + }); +}); + +// -- Non-stream completion + reasoning_content normalisation --------- + +describe('BytePlusProvider.complete non-stream output', () => { + it('returns the first choice and runs the metered usage calculator', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'hi there', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 10 }, + }, + }); + + const result = await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + expect(result).toMatchObject({ + message: { content: 'hi there', role: 'assistant' }, + finish_reason: 'stop', + }); + expect((result as { usage: unknown }).usage).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 10, + }); + + // Cost overrides scale per-token usage by the per-token cents from + // the model's costs table, so derive expectations from + // BYTEPLUS_MODELS directly to avoid hardcoded float-precision drift. + const seed16 = BYTEPLUS_MODELS.find( + (m) => m.id === 'seed-1-6-250915', + )!; + expect(recordSpy).toHaveBeenCalledTimes(1); + const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!; + expect(usage).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 10, + }); + expect(actor).toBe(SYSTEM_ACTOR); + expect(prefix).toBe('byteplus:seed-1-6-250915'); + expect(overrides.prompt_tokens).toBeCloseTo( + 100 * Number(seed16.costs.prompt_tokens), + 5, + ); + expect(overrides.completion_tokens).toBeCloseTo( + 50 * Number(seed16.costs.completion_tokens), + 5, + ); + expect(overrides.cached_tokens).toBeCloseTo( + 10 * Number(seed16.costs.cached_tokens ?? 0), + 5, + ); + }); + + it('preserves OpenAI-shaped tool_calls on the assistant response', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { + name: 'lookup', + arguments: '{"q":"puter"}', + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + + const result = (await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'do a tool call' }], + tools: [ + { + type: 'function', + function: { name: 'lookup', parameters: {} }, + }, + ], + }), + )) as { message: { tool_calls?: unknown[] }; finish_reason: string }; + + expect(result.finish_reason).toBe('tool_calls'); + expect(result.message.tool_calls).toEqual([ + { + id: 'call_1', + type: 'function', + function: { + name: 'lookup', + arguments: '{"q":"puter"}', + }, + }, + ]); + }); + + it('renames Ark `reasoning_content` to `reasoning` on the message', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { + role: 'assistant', + content: 'final answer', + reasoning_content: 'thinking out loud', + }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + + const result = (await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'hi' }], + }), + )) as { message: Record }; + + expect(result.message.reasoning).toBe('thinking out loud'); + expect('reasoning_content' in result.message).toBe(false); + }); + + it('does not overwrite an existing `reasoning` field if both are present', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { + role: 'assistant', + content: 'final', + reasoning: 'original', + reasoning_content: 'should-be-dropped', + }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + + const result = (await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'hi' }], + }), + )) as { message: Record }; + + expect(result.message.reasoning).toBe('original'); + expect('reasoning_content' in result.message).toBe(false); + }); +}); + +// -- Streaming deltas ------------------------------------------------ + +describe('BytePlusProvider.complete streaming', () => { + it('streams text deltas through to text events and meters final usage', async () => { + const { provider } = makeProvider(); + createMock.mockReturnValueOnce( + asAsyncIterable([ + { choices: [{ delta: { content: 'hel' } }] }, + { choices: [{ delta: { content: 'lo' } }] }, + { + choices: [{ delta: {} }], + usage: { + prompt_tokens: 4, + completion_tokens: 2, + prompt_tokens_details: { cached_tokens: 1 }, + }, + }, + ]), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'say hi' }], + stream: true, + }), + ); + expect((result as { stream: boolean }).stream).toBe(true); + + const harness = makeCapturingChatStream(); + await ( + result as { + init_chat_stream: (p: { chatStream: unknown }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }); + + const events = harness.events(); + const textEvents = events.filter((e) => e.type === 'text'); + expect(textEvents.map((e) => e.text)).toEqual(['hel', 'lo']); + + const usageEvent = events.find((e) => e.type === 'usage'); + expect(usageEvent?.usage).toEqual({ + prompt_tokens: 4, + completion_tokens: 2, + cached_tokens: 1, + }); + + const seed16 = BYTEPLUS_MODELS.find( + (m) => m.id === 'seed-1-6-250915', + )!; + expect(recordSpy).toHaveBeenCalledTimes(1); + const [, , prefix, overrides] = recordSpy.mock.calls[0]!; + expect(prefix).toBe('byteplus:seed-1-6-250915'); + expect(overrides.prompt_tokens).toBeCloseTo( + 4 * Number(seed16.costs.prompt_tokens), + 5, + ); + expect(overrides.completion_tokens).toBeCloseTo( + 2 * Number(seed16.costs.completion_tokens), + 5, + ); + expect(overrides.cached_tokens).toBeCloseTo( + 1 * Number(seed16.costs.cached_tokens ?? 0), + 5, + ); + }); + + it('streams reasoning_content deltas as reasoning events', async () => { + const { provider } = makeProvider(); + createMock.mockReturnValueOnce( + asAsyncIterable([ + { + choices: [ + { delta: { reasoning_content: 'pondering...' } }, + ], + }, + { choices: [{ delta: { content: 'answer' } }] }, + { + choices: [{ delta: {} }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }, + ]), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'think hard' }], + stream: true, + }), + ); + + const harness = makeCapturingChatStream(); + await ( + result as { + init_chat_stream: (p: { chatStream: unknown }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }); + + const events = harness.events(); + const reasoningEvents = events.filter((e) => e.type === 'reasoning'); + expect(reasoningEvents.map((e) => e.reasoning)).toEqual([ + 'pondering...', + ]); + const textEvents = events.filter((e) => e.type === 'text'); + expect(textEvents.map((e) => e.text)).toEqual(['answer']); + }); + + it('builds a tool_use block from streamed function-call deltas', async () => { + const { provider } = makeProvider(); + createMock.mockReturnValueOnce( + asAsyncIterable([ + { + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: 'call_1', + function: { + name: 'lookup', + arguments: '{"q":', + }, + }, + ], + }, + }, + ], + }, + { + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + function: { arguments: '"puter"}' }, + }, + ], + }, + }, + ], + }, + { + choices: [{ delta: {} }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }, + ]), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'do tool call' }], + tools: [ + { + type: 'function', + function: { name: 'lookup', parameters: {} }, + }, + ], + stream: true, + }), + ); + + const harness = makeCapturingChatStream(); + await ( + result as { + init_chat_stream: (p: { chatStream: unknown }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }); + + const events = harness.events(); + const toolEvent = events.find((e) => e.type === 'tool_use'); + expect(toolEvent).toBeDefined(); + expect(toolEvent?.id).toBe('call_1'); + expect(toolEvent?.name).toBe('lookup'); + expect(toolEvent?.input).toEqual({ q: 'puter' }); + }); +}); + +// -- Error mapping --------------------------------------------------- + +describe('BytePlusProvider.complete error mapping', () => { + it('rethrows errors raised by the OpenAI client unchanged', async () => { + const { provider } = makeProvider(); + const apiError = new Error('ModelArk exploded'); + createMock.mockRejectedValueOnce(apiError); + + await expect( + withTestActor(() => + provider.complete({ + model: 'seed-1-6-250915', + messages: [{ role: 'user', content: 'boom' }], + }), + ), + ).rejects.toBe(apiError); + + // No metering should be recorded on a failed call. + expect(recordSpy).not.toHaveBeenCalled(); + }); +}); + +// -- Moderation ------------------------------------------------------ + +describe('BytePlusProvider.checkModeration', () => { + it('throws — BytePlus provider does not implement moderation', () => { + const { provider } = makeProvider(); + expect(() => provider.checkModeration('anything')).toThrow( + /not implemented/i, + ); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts new file mode 100644 index 000000000..b28b64f4b --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts @@ -0,0 +1,180 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { OpenAI } from 'openai'; +import { ChatCompletionCreateParams } from 'openai/resources/index.js'; +import { Context } from '../../../../core/context.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; +import { BYTEPLUS_MODELS } from './models.js'; + +type BytePlusConfig = { + apiKey: string; + apiBaseUrl?: string; +}; + +type BytePlusCustomParams = { + response_format?: unknown; + stop?: string[]; + // Ark-specific toggle for deep reasoning; the seed models reason by + // default and route those tokens to `reasoning_content`. + thinking?: { + type?: 'enabled' | 'disabled' | 'auto'; + }; +}; + +const asRecord = (value: unknown): Record => + value && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : {}; + +/** + * BytePlus ModelArk provider — an OpenAI-compatible endpoint serving + * ByteDance's Seed models plus hosted third-party models (GLM, DeepSeek, + * GPT-OSS). https://docs.byteplus.com/en/docs/ModelArk/1330626 + */ +export class BytePlusProvider implements IChatProvider { + #openai: OpenAI; + + #meteringService: MeteringService; + + #defaultModel = 'seed-2-0-lite-260428'; + + constructor(config: BytePlusConfig, meteringService: MeteringService) { + this.#openai = new OpenAI({ + apiKey: config.apiKey, + baseURL: + config.apiBaseUrl ?? + 'https://ark.ap-southeast.bytepluses.com/api/v3', + }); + this.#meteringService = meteringService; + } + + getDefaultModel() { + return this.#defaultModel; + } + + models() { + return BYTEPLUS_MODELS; + } + + list() { + const modelIds: string[] = []; + for (const model of this.models()) { + modelIds.push(model.id); + if (model.aliases) { + modelIds.push(...model.aliases); + } + } + return modelIds; + } + + async complete( + params: ICompleteArguments, + ): ReturnType { + const { + custom, + max_tokens, + stream, + temperature, + tools, + tool_choice, + top_p, + } = params; + let { messages, model } = params; + const actor = Context.get('actor'); + const availableModels = this.models(); + const modelUsed = + availableModels.find((m) => + [m.id, ...(m.aliases || [])].includes(model), + ) || availableModels.find((m) => m.id === this.getDefaultModel())!; + + messages = await OpenAIUtil.process_input_messages(messages); + messages = messages.map((message) => { + delete message.cache_control; + return message; + }); + + const customParams = asRecord(custom) as BytePlusCustomParams; + + const completionParams: ChatCompletionCreateParams = { + messages, + model: modelUsed.id, + ...(tools ? { tools } : {}), + ...(tool_choice !== undefined ? { tool_choice } : {}), + ...(max_tokens !== undefined ? { max_tokens } : {}), + ...(temperature !== undefined ? { temperature } : {}), + ...(top_p !== undefined ? { top_p } : {}), + ...(customParams.response_format + ? { response_format: customParams.response_format } + : {}), + ...(customParams.stop ? { stop: customParams.stop } : {}), + ...(customParams.thinking + ? { thinking: customParams.thinking } + : {}), + stream: !!stream, + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), + } as ChatCompletionCreateParams; + + const completion = + await this.#openai.chat.completions.create(completionParams); + + const result = await OpenAIUtil.handle_completion_output({ + usage_calculator: ({ usage }) => { + const trackedUsage = usage + ? OpenAIUtil.extractMeteredUsage(usage) + : { + prompt_tokens: 0, + completion_tokens: 0, + cached_tokens: 0, + }; + const costsOverride = Object.fromEntries( + Object.entries(trackedUsage).map(([key, value]) => { + return [key, value * Number(modelUsed.costs[key] ?? 0)]; + }), + ); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor!, + `byteplus:${modelUsed.id}`, + costsOverride, + ); + return trackedUsage; + }, + stream, + completion, + }); + + // Ark's deep-reasoning models return `reasoning_content` (DeepSeek + // wire convention); expose it under `reasoning` like other providers. + OpenAIUtil.normalizeReasoningContent(result); + return result; + } + + checkModeration( + _text: string, + ): ReturnType { + throw new Error('Method not implemented.'); + } +} diff --git a/src/backend/drivers/ai-chat/providers/byteplus/models.ts b/src/backend/drivers/ai-chat/providers/byteplus/models.ts new file mode 100644 index 000000000..83ab9282b --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/byteplus/models.ts @@ -0,0 +1,196 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IChatModel } from '../../types.js'; +import { usdPerMToken } from '../../utils/pricing.js'; + +const K = 1_000; + +const bytePlusModel = ( + id: string, + name: string, + context: number, + maxTokens: number, + costs: IChatModel['costs'], + opts: { + input?: string[]; + extraAliases?: string[]; + openWeights?: boolean; + } = {}, +): IChatModel => ({ + puterId: `byteplus:byteplus/${id}`, + id, + name, + aliases: [`byteplus/${id}`, ...(opts.extraAliases ?? [])], + modalities: { input: opts.input ?? ['text'], output: ['text'] }, + open_weights: opts.openWeights ?? false, + tool_call: true, + context, + max_tokens: maxTokens, + costs_currency: 'usd-cents', + input_cost_key: 'prompt_tokens', + output_cost_key: 'completion_tokens', + costs, +}); + +// An undated series alias (e.g. `seed-1-6`) points at the newest snapshot of +// that series, mirroring how other providers alias their rolling names. +const series = (alias: string) => [alias, `byteplus/${alias}`]; + +const VISION = ['text', 'image', 'video']; + +// Hardcoded from https://docs.byteplus.com/en/docs/ModelArk/1330310 (catalog) +// and https://docs.byteplus.com/en/docs/ModelArk/1544106 (pricing). +// +// Some models bill a higher rate for prompts above 128K tokens; the costs +// below are the base tier. seed-2-0-lite/mini-260428 also accept audio input +// at a much higher per-token rate ModelArk folds into prompt_tokens, so audio +// is deliberately left out of `modalities` to keep metering honest. +export const BYTEPLUS_MODELS: IChatModel[] = [ + bytePlusModel( + 'dola-seed-2-1-turbo-260628', + 'Dola Seed 2.1 Turbo', + 256 * K, + 256 * K, + usdPerMToken(0.5, 2.5, 0.1), + { input: VISION, extraAliases: series('dola-seed-2-1-turbo') }, + ), + bytePlusModel( + 'seed-2-0-lite-260428', + 'Seed 2.0 Lite', + 256 * K, + 128 * K, + usdPerMToken(0.25, 2, 0.05), + { input: VISION, extraAliases: series('seed-2-0-lite') }, + ), + bytePlusModel( + 'seed-2-0-lite-260228', + 'Seed 2.0 Lite', + 256 * K, + 128 * K, + usdPerMToken(0.25, 2, 0.05), + { input: VISION }, + ), + bytePlusModel( + 'seed-2-0-mini-260428', + 'Seed 2.0 Mini', + 256 * K, + 128 * K, + usdPerMToken(0.1, 0.4, 0.02), + { input: VISION, extraAliases: series('seed-2-0-mini') }, + ), + bytePlusModel( + 'seed-2-0-mini-260215', + 'Seed 2.0 Mini', + 256 * K, + 128 * K, + usdPerMToken(0.1, 0.4, 0.02), + { input: VISION }, + ), + bytePlusModel( + 'seed-2-0-pro-260328', + 'Seed 2.0 Pro', + 256 * K, + 128 * K, + usdPerMToken(0.5, 3, 0.1), + { input: VISION, extraAliases: series('seed-2-0-pro') }, + ), + bytePlusModel( + 'seed-2-0-code-preview-260328', + 'Seed 2.0 Code Preview', + 256 * K, + 128 * K, + usdPerMToken(0.5, 3, 0.1), + { input: VISION, extraAliases: series('seed-2-0-code-preview') }, + ), + bytePlusModel( + 'seed-1-8-251228', + 'Seed 1.8', + 256 * K, + 64 * K, + usdPerMToken(0.25, 2, 0.05), + { input: VISION, extraAliases: series('seed-1-8') }, + ), + bytePlusModel( + 'seed-1-6-250915', + 'Seed 1.6', + 256 * K, + 32 * K, + usdPerMToken(0.25, 2, 0.05), + { input: VISION, extraAliases: series('seed-1-6') }, + ), + bytePlusModel( + 'seed-1-6-flash-250715', + 'Seed 1.6 Flash', + 256 * K, + 32 * K, + usdPerMToken(0.075, 0.3, 0.015), + { input: VISION, extraAliases: series('seed-1-6-flash') }, + ), + bytePlusModel( + 'glm-5-2-260617', + 'GLM-5.2', + 1_024 * K, + 128 * K, + usdPerMToken(1.4, 4.4, 0.26), + { extraAliases: series('glm-5-2') }, + ), + bytePlusModel( + 'glm-4-7-251222', + 'GLM-4.7', + 200 * K, + 128 * K, + usdPerMToken(0.6, 2.2, 0.11), + { extraAliases: series('glm-4-7') }, + ), + // The bare `deepseek-v4-*` names belong to the first-party DeepSeek + // provider, so these only carry the byteplus-prefixed aliases. + bytePlusModel( + 'deepseek-v4-pro-260425', + 'DeepSeek V4 Pro', + 1_024 * K, + 384 * K, + usdPerMToken(1.74, 3.48, 0.145), + { extraAliases: ['byteplus/deepseek-v4-pro'] }, + ), + bytePlusModel( + 'deepseek-v4-flash-260425', + 'DeepSeek V4 Flash', + 1_024 * K, + 384 * K, + usdPerMToken(0.14, 0.28, 0.028), + { extraAliases: ['byteplus/deepseek-v4-flash'] }, + ), + bytePlusModel( + 'deepseek-v3-2-251201', + 'DeepSeek V3.2', + 128 * K, + 32 * K, + usdPerMToken(0.28, 0.42, 0.056), + { extraAliases: series('deepseek-v3-2') }, + ), + bytePlusModel( + 'gpt-oss-120b-250805', + 'GPT-OSS 120B', + 128 * K, + 64 * K, + usdPerMToken(0.1, 0.5), + { extraAliases: series('gpt-oss-120b'), openWeights: true }, + ), +]; diff --git a/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts index a1a55e044..589f500be 100644 --- a/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts +++ b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts @@ -95,8 +95,7 @@ export class NeuralwattProvider implements IChatProvider { async models(): Promise { let apiModels = kv.get(KV_MODELS_KEY) as - | NeuralwattApiModel[] - | undefined; + NeuralwattApiModel[] | undefined; if (!apiModels) { try { const resp = await axios.request({ @@ -127,16 +126,15 @@ export class NeuralwattProvider implements IChatProvider { } /** - * Cached account accounting method (`energy` | `token`) from - * `GET /v1/quota`. Used only to annotate returned usage — billing - * always prefers `cost.request_cost_usd` on the completion. + * Cached account accounting method (`energy` | `token`) from `GET + * /v1/quota`. Used only to annotate returned usage — billing always prefers + * `cost.request_cost_usd` on the completion. */ async getAccountingMethod(): Promise< NeuralwattAccountingMethod | undefined > { let method = kv.get(KV_QUOTA_KEY) as - | NeuralwattAccountingMethod - | undefined; + NeuralwattAccountingMethod | undefined; if (method === 'energy' || method === 'token') return method; try { @@ -187,9 +185,7 @@ export class NeuralwattProvider implements IChatProvider { if (!modelUsed) { if (hasImages) { - modelUsed = availableModels.find((m) => - modelSupportsVision(m), - ); + modelUsed = availableModels.find((m) => modelSupportsVision(m)); } modelUsed = modelUsed || @@ -221,8 +217,7 @@ export class NeuralwattProvider implements IChatProvider { messages = await OpenAIUtil.process_input_messages(messages); - const requestedReasoningEffort = - reasoning_effort ?? reasoning?.effort; + const requestedReasoningEffort = reasoning_effort ?? reasoning?.effort; const supportsReasoningEffort = modelUsed.reasoning_effort === true; const completionParams = { @@ -353,8 +348,7 @@ export class NeuralwattProvider implements IChatProvider { ...chunk.usage, ...(typeof chunk.cost?.request_cost_usd === 'number' ? { - request_cost_usd: - chunk.cost.request_cost_usd, + request_cost_usd: chunk.cost.request_cost_usd, } : {}), ...(chunk.energy diff --git a/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts b/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts index 6e35ddc1d..a6be43041 100644 --- a/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts +++ b/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts @@ -24,10 +24,10 @@ export const NEURALWATT_ID_PREFIX = 'neuralwatt:'; export const NEURALWATT_DEFAULT_MODEL = 'neuralwatt:deepseek-v4-flash'; /** - * Shape of one entry in Neuralwatt's `GET /v1/models` catalog. - * Pricing is USD per million tokens under `metadata.pricing` and is used - * only for Puter preflight estimates / token-cost fallback. Live billing - * uses each completion's `cost.request_cost_usd`. + * Shape of one entry in Neuralwatt's `GET /v1/models` catalog. Pricing is USD + * per million tokens under `metadata.pricing` and is used only for Puter + * preflight estimates / token-cost fallback. Live billing uses each + * completion's `cost.request_cost_usd`. */ export type NeuralwattApiModel = { @@ -94,9 +94,9 @@ export const stripNeuralwattPrefix = (modelId: string): string => : modelId; /** - * Map a Neuralwatt catalog entry to Puter's `IChatModel`. Returns `null` - * when pricing is TBD (placeholders) so the model is not offered for - * preflight credit checks until Neuralwatt publishes real rates. + * Map a Neuralwatt catalog entry to Puter's `IChatModel`. Returns `null` when + * pricing is TBD (placeholders) so the model is not offered for preflight + * credit checks until Neuralwatt publishes real rates. */ export const mapNeuralwattApiModel = ( model: NeuralwattApiModel, @@ -166,8 +166,8 @@ export const modelSupportsVision = (model: IChatModel): boolean => model.modalities.input.includes('image'); /** - * Detect image / puter_path parts so we can prefer a vision-capable model - * from the Neuralwatt catalog (or reject a text-only pick). + * Detect image / puter_path parts so we can prefer a vision-capable model from + * the Neuralwatt catalog (or reject a text-only pick). */ export const messagesHaveImageContent = ( messages: Array<{ content?: unknown }>, diff --git a/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts b/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts index 2bc7b776f..714c38bbf 100644 --- a/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts +++ b/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts @@ -180,7 +180,7 @@ export class ZAIProvider implements IChatProvider { completion, }); - this.#normalizeReasoningContent(result); + OpenAIUtil.normalizeReasoningContent(result); return result; } @@ -189,32 +189,4 @@ export class ZAIProvider implements IChatProvider { ): ReturnType { throw new Error('Method not implemented.'); } - - #normalizeReasoningContent( - result: Awaited>, - ) { - if (!('message' in result) || !result.message) return; - - const message = result.message as Record; - if ( - message.reasoning === undefined && - message.reasoning_content !== undefined - ) { - message.reasoning = message.reasoning_content; - } - delete message.reasoning_content; - - if (!Array.isArray(message.content)) return; - - for (const contentPart of message.content) { - const part = asRecord(contentPart); - if ( - part.reasoning === undefined && - part.reasoning_content !== undefined - ) { - part.reasoning = part.reasoning_content; - } - delete part.reasoning_content; - } - } } diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js index 00e7c3569..26acf35dd 100644 --- a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js @@ -265,6 +265,36 @@ export const extractMeteredUsage = (usage) => { }; }; +// Renames one object's DeepSeek-wire `reasoning_content` to the `reasoning` +// key Puter exposes, without clobbering an existing `reasoning`. +const renameReasoningContent = (obj) => { + if (obj.reasoning === undefined && obj.reasoning_content !== undefined) { + obj.reasoning = obj.reasoning_content; + } + delete obj.reasoning_content; +}; + +/** + * Normalize a non-streaming completion result whose provider follows the + * DeepSeek wire convention (`reasoning_content` on the message and content + * parts) to Puter's `reasoning` key. The streaming path already does this in + * create_chat_stream_handler. + */ +export const normalizeReasoningContent = (result) => { + if (!result || typeof result !== 'object') return; + if (!('message' in result) || !result.message) return; + + const message = result.message; + renameReasoningContent(message); + + if (!Array.isArray(message.content)) return; + for (const part of message.content) { + if (part && typeof part === 'object' && !Array.isArray(part)) { + renameReasoningContent(part); + } + } +}; + export const create_chat_stream_handler = ({ deviations, completion, usage_calculator }) => async ({ chatStream }) => { diff --git a/src/backend/drivers/ai-image/ImageGenerationDriver.ts b/src/backend/drivers/ai-image/ImageGenerationDriver.ts index b8ae10d20..6628b8ba3 100644 --- a/src/backend/drivers/ai-image/ImageGenerationDriver.ts +++ b/src/backend/drivers/ai-image/ImageGenerationDriver.ts @@ -27,6 +27,7 @@ import type { Actor } from '../../core/actor.js'; import { PuterDriver } from '../types.js'; import { secureFetch } from '../../util/secureHttp.js'; import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js'; +import { BytePlusImageProvider } from './providers/byteplus/BytePlusImageProvider.js'; import { CloudflareImageProvider } from './providers/cloudflare/CloudflareImageProvider.js'; import { GeminiImageProvider } from './providers/gemini/GeminiImageProvider.js'; import { OpenAiImageProvider } from './providers/openai/OpenAiImageProvider.js'; @@ -60,6 +61,7 @@ export class ImageGenerationDriver extends PuterDriver { 'cloudflare-image-generation', 'xai-image-generation', 'replicate-image-generation', + 'byteplus-image-generation', ]; readonly isDefault = true; @@ -291,8 +293,7 @@ export class ImageGenerationDriver extends PuterDriver { const cloudflare = (providers['cloudflare-image-generation'] ?? providers['cloudflare-workers-ai-image'] ?? providers['cloudflare-workers-ai']) as - | Record - | undefined; + Record | undefined; const cfToken = (cloudflare?.apiToken as string | undefined) ?? (cloudflare?.apiKey as string | undefined) ?? @@ -307,8 +308,7 @@ export class ImageGenerationDriver extends PuterDriver { apiToken: cfToken, accountId: cfAccount, apiBaseUrl: cloudflare?.apiBaseUrl as - | string - | undefined, + string | undefined, }, m, ); @@ -333,6 +333,29 @@ export class ImageGenerationDriver extends PuterDriver { m, ); } + + // Falls back to the shared `byteplus` (ai-chat) key; `apiBaseUrl` + // selects the ModelArk region, same as the chat provider. Each field + // falls through independently so a partial image-specific block can't + // pair its missing apiBaseUrl with the shared block's key (or vice + // versa) and point a region-scoped key at the wrong endpoint. + const byteplusImageCfg = providers['byteplus-image-generation'] as + Record | undefined; + const byteplusSharedCfg = providers['byteplus'] as + Record | undefined; + const byteplusKey = readKey(byteplusImageCfg, byteplusSharedCfg); + if (byteplusKey) { + this.#providers['byteplus-image-generation'] = + new BytePlusImageProvider( + { + apiKey: byteplusKey, + apiBaseUrl: (byteplusImageCfg?.apiBaseUrl ?? + byteplusSharedCfg?.apiBaseUrl) as + string | undefined, + }, + m, + ); + } } async #buildModelMap() { diff --git a/src/backend/drivers/ai-image/inputImage.ts b/src/backend/drivers/ai-image/inputImage.ts index ba97031fc..e31ab7863 100644 --- a/src/backend/drivers/ai-image/inputImage.ts +++ b/src/backend/drivers/ai-image/inputImage.ts @@ -20,8 +20,8 @@ /** * Shared helpers for `input_images` (image-to-image) handling across image * providers. `input_images` is the canonical, cross-provider field; an entry - * may be a public URL, a data-URI, or raw base64. Providers whose upstream - * API needs base64 use these helpers to normalize URLs server-side (via the + * may be a public URL, a data-URI, or raw base64. Providers whose upstream API + * needs base64 use these helpers to normalize URLs server-side (via the * SSRF-guarded `secureFetch`); providers that accept URLs natively (Replicate, * xAI) pass them through untouched. */ @@ -35,9 +35,20 @@ export function isHttpUrl(s: string): boolean { } /** - * Resolve the single input image for providers that only support one. - * Throws 400 if `input_images` carries more than one entry. Returns the - * chosen image string (URL / data-URI / raw base64) or undefined. + * Normalize an input-image string for providers that accept URLs natively: + * http(s) URLs and data-URIs pass through untouched, raw base64 is wrapped with + * `mimeHint` (default image/png). + */ +export function toUrlOrDataUri(img: string, mimeHint?: string): string { + return isHttpUrl(img) || img.startsWith('data:') + ? img + : `data:${mimeHint ?? 'image/png'};base64,${img}`; +} + +/** + * Resolve the single input image for providers that only support one. Throws + * 400 if `input_images` carries more than one entry. Returns the chosen image + * string (URL / data-URI / raw base64) or undefined. */ export function resolveSingleInputImage( params: Pick, @@ -84,10 +95,9 @@ export async function fetchImageAsBase64( } /** - * Normalize any input-image string to a base64 data-URI: - * • http(s) URL → fetched via secureFetch - * • data-URI → returned as-is - * • raw base64 → wrapped with `mimeHint` (default image/png) + * Normalize any input-image string to a base64 data-URI: • http(s) URL → + * fetched via secureFetch • data-URI → returned as-is • raw base64 → wrapped + * with `mimeHint` (default image/png) */ export async function toBase64DataUri( img: string, diff --git a/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.integration.test.ts b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.integration.test.ts new file mode 100644 index 000000000..939b2a522 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.integration.test.ts @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the BytePlus ModelArk image provider. + * + * Generates one 1K image on seedream-4-0 (the cheapest catalog entry, + * $0.03/image). Skipped when `PUTER_TEST_AI_BYTEPLUS_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + INTEGRATION_TEST_TIMEOUT_MS, + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { BytePlusImageProvider } from './BytePlusImageProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_BYTEPLUS_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'BytePlusImageProvider (integration)', + () => { + it( + 'generates an image URL from seedream-4-0', + { timeout: INTEGRATION_TEST_TIMEOUT_MS }, + async () => { + const provider = new BytePlusImageProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const url = await withTestActor(() => + provider.generate({ + model: 'seedream-4-0', + prompt: 'a single red dot on a white background', + quality: '1k', + }), + ); + + expect(typeof url).toBe('string'); + expect( + url.startsWith('https://') || url.startsWith('data:'), + ).toBe(true); + }, + ); + }, +); diff --git a/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.test.ts b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.test.ts new file mode 100644 index 000000000..ebac4f7ef --- /dev/null +++ b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.test.ts @@ -0,0 +1,550 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for BytePlusImageProvider. + * + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock + * redis) and constructs BytePlusImageProvider directly against the + * live wired `MeteringService` so the recording side is exercised + * end-to-end. Ark's image API is OpenAI-compatible so the OpenAI SDK + * is mocked at the module boundary; that's the real network egress + * point. + */ + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { BYTEPLUS_IMAGE_GENERATION_MODELS } from './models.js'; +import { BytePlusImageProvider } from './BytePlusImageProvider.js'; + +// -- OpenAI SDK mock ------------------------------------------------- + +const { generateMock, openAICtor } = vi.hoisted(() => ({ + generateMock: vi.fn(), + openAICtor: vi.fn(), +})); + +vi.mock('openai', () => { + const OpenAICtor = vi.fn().mockImplementation(function ( + this: Record, + opts: unknown, + ) { + openAICtor(opts); + this.images = { generate: generateMock }; + // Some sibling providers boot through the same SDK module. + this.chat = { completions: { create: vi.fn() } }; + this.post = vi.fn(); + }); + return { OpenAI: OpenAICtor, default: { OpenAI: OpenAICtor } }; +}); + +// -- Test harness ---------------------------------------------------- + +let server: PuterServer; +let hasCreditsSpy: MockInstance; +let batchIncrementUsagesSpy: MockInstance< + MeteringService['batchIncrementUsages'] +>; + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = ( + config: { apiKey?: string; apiBaseUrl?: string } = {}, +) => + new BytePlusImageProvider( + { + apiKey: config.apiKey ?? 'test-key', + ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}), + }, + server.services.metering, + ); + +beforeEach(() => { + generateMock.mockReset(); + openAICtor.mockReset(); + hasCreditsSpy = vi.spyOn(server.services.metering, 'hasEnoughCredits'); + batchIncrementUsagesSpy = vi.spyOn( + server.services.metering, + 'batchIncrementUsages', + ); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +const sampleResponse = { data: [{ url: 'https://ark.example/img/1' }] }; + +const findModel = (id: string) => + BYTEPLUS_IMAGE_GENERATION_MODELS.find((m) => m.id === id)!; + +// -- Construction ---------------------------------------------------- + +describe('BytePlusImageProvider construction', () => { + it('defaults the OpenAI SDK to the ap-southeast ModelArk base URL', () => { + makeProvider(); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://ark.ap-southeast.bytepluses.com/api/v3', + }); + }); + + it('honors a configured apiBaseUrl (region selection)', () => { + makeProvider({ + apiBaseUrl: 'https://ark.eu-west.bytepluses.com/api/v3', + }); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://ark.eu-west.bytepluses.com/api/v3', + }); + }); + + it('throws when no apiKey is supplied', () => { + expect( + () => + new BytePlusImageProvider( + { apiKey: '' }, + server.services.metering, + ), + ).toThrow(/API key/i); + }); +}); + +// -- Model catalog --------------------------------------------------- + +describe('BytePlusImageProvider model catalog', () => { + it('returns seedream-5-0-lite as the default', () => { + expect(makeProvider().getDefaultModel()).toBe( + 'seedream-5-0-lite-260128', + ); + }); + + it('exposes the static catalog verbatim', () => { + expect(makeProvider().models()).toBe( + BYTEPLUS_IMAGE_GENERATION_MODELS, + ); + }); +}); + +// -- test_mode / validation / credit gate ---------------------------- + +describe('BytePlusImageProvider.generate gates', () => { + it('returns the canned sample URL in test_mode without side effects', async () => { + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'x', test_mode: true }), + ); + expect(result).toBe( + 'https://puter-sample-data.puter.site/image_example.png', + ); + expect(hasCreditsSpy).not.toHaveBeenCalled(); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it('throws 400 on a missing or blank prompt', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: undefined as unknown as string, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + await expect( + withTestActor(() => makeProvider().generate({ prompt: ' ' })), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it('throws 402 BEFORE hitting ModelArk when the actor lacks credits', async () => { + hasCreditsSpy.mockResolvedValueOnce(false); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 402 }); + expect(generateMock).not.toHaveBeenCalled(); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + }); +}); + +// -- Model resolution ------------------------------------------------ + +describe('BytePlusImageProvider.generate model resolution', () => { + it('falls back to the default model for unknown ids', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ model: 'nope', prompt: 'hi' }), + ); + expect(generateMock.mock.calls[0]![0].model).toBe( + 'seedream-5-0-lite-260128', + ); + }); + + it('resolves series and byteplus-prefixed aliases to the dated id', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'byteplus/seedream-4-0', + prompt: 'hi', + }), + ); + expect(generateMock.mock.calls[0]![0].model).toBe( + 'seedream-4-0-250828', + ); + }); + + it('resolves the undated seedream-5-0 alias to the lite snapshot', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ model: 'seedream-5-0', prompt: 'hi' }), + ); + expect(generateMock.mock.calls[0]![0].model).toBe( + 'seedream-5-0-lite-260128', + ); + }); +}); + +// -- Request shape --------------------------------------------------- + +describe('BytePlusImageProvider.generate request shape', () => { + it('sends the tier keyword size (Ark default 2K), url format and no watermark', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ prompt: 'a red dot' }), + ); + const sent = generateMock.mock.calls[0]![0]; + expect(sent.size).toBe('2K'); + expect(sent.response_format).toBe('url'); + expect(sent.watermark).toBe(false); + expect(sent.image).toBeUndefined(); + }); + + it('maps quality to the tier keyword case-insensitively', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'seedream-4-0', + prompt: 'hi', + quality: '1.5K', + }), + ); + expect(generateMock.mock.calls[0]![0].size).toBe('1.5K'); + }); + + it('resolves an aspect ratio + tier to the documented pixel size', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'seedream-4-0', + prompt: 'hi', + quality: '1k', + ratio: { w: 16, h: 9 }, + }), + ); + expect(generateMock.mock.calls[0]![0].size).toBe('1424x800'); + }); + + // seedream-4-5 and the 5.0 series reject anything under 3,686,400 pixels, + // which rules out every 1K/1.5K size — both the tier keyword and the + // aspect-ratio mapping have to land on 2K. + it('snaps a sub-minimum tier up to the smallest the model accepts', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'seedream-4-5', + prompt: 'hi', + quality: '1k', + }), + ); + expect(generateMock.mock.calls[0]![0].size).toBe('2K'); + + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + quality: '1.5k', + ratio: { w: 16, h: 9 }, + }), + ); + expect(generateMock.mock.calls[1]![0].size).toBe('2816x1584'); + }); + + it('passes explicit pixel dimensions straight through', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'seedream-4-0', + prompt: 'hi', + ratio: { w: 2048, h: 1024 }, + }), + ); + expect(generateMock.mock.calls[0]![0].size).toBe('2048x1024'); + }); + + it('rejects explicit pixel dimensions below a 2K-only model minimum', async () => { + // The default model (seedream-5-0-lite) only accepts >= 3,686,400 px, + // so a size that passes the generic floor still fails pre-flight + // instead of round-tripping to Ark for a 400. + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + ratio: { w: 2048, h: 1024 }, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it('reduces w:h to lowest terms when mapping an aspect ratio', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'seedream-4-0', + prompt: 'hi', + quality: '1k', + ratio: { w: 32, h: 18 }, + }), + ); + expect(generateMock.mock.calls[0]![0].size).toBe('1424x800'); + }); + + it('rejects explicit pixel dimensions outside Ark limits', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + ratio: { w: 6000, h: 6000 }, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(generateMock).not.toHaveBeenCalled(); + }); + + it('sends a URL input image untouched', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'seedream-4-0', + prompt: 'add a hat', + input_image: 'https://example.com/cat.png', + }), + ); + const sent = generateMock.mock.calls[0]![0]; + expect(sent.model).toBe('seedream-4-0-250828'); + expect(sent.image).toBe('https://example.com/cat.png'); + }); +}); + +// -- Input images ---------------------------------------------------- + +describe('BytePlusImageProvider.generate input images', () => { + const PNG = 'data:image/png;base64,iVBORw0KGgo='; + + it('sends a single input image as a string and multiple as an array', async () => { + generateMock.mockResolvedValue(sampleResponse); + + await withTestActor(() => + makeProvider().generate({ prompt: 'hi', input_images: [PNG] }), + ); + expect(generateMock.mock.calls[0]![0].image).toBe(PNG); + + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + input_images: [PNG, PNG], + }), + ); + expect(generateMock.mock.calls[1]![0].image).toEqual([PNG, PNG]); + }); + + it('wraps raw base64 into a data URI using the mime hint', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + input_image: 'AAAA', + input_image_mime_type: 'image/webp', + }), + ); + expect(generateMock.mock.calls[0]![0].image).toBe( + 'data:image/webp;base64,AAAA', + ); + }); + + it('rejects more input images than the model accepts', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + model: 'dola-seedream-5-0-pro', + prompt: 'hi', + input_images: Array(11).fill(PNG), + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(generateMock).not.toHaveBeenCalled(); + }); +}); + +// -- Metering -------------------------------------------------------- + +describe('BytePlusImageProvider.generate metering', () => { + it('meters flat per-image models at their catalog rate', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ model: 'seedream-4-0', prompt: 'hi' }), + ); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + expect(entries).toEqual([ + { + usageType: + 'byteplus-image-generation:seedream-4-0-250828:per-image', + usageAmount: 1, + costOverride: + findModel('seedream-4-0-250828').costs['per-image'] * + 1_000_000, + }, + ]); + }); + + it('bills the pro model at the high tier for 2K output', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'dola-seedream-5-0-pro', + prompt: 'hi', + }), + ); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + const pro = findModel('dola-seedream-5-0-pro-260628'); + expect(entries).toEqual([ + { + usageType: + 'byteplus-image-generation:dola-seedream-5-0-pro-260628:output:2k', + usageAmount: 1, + costOverride: pro.costs['output:2k'] * 1_000_000, + }, + ]); + }); + + it('bills the pro model at the low tier for 1k/1.5k output', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + await withTestActor(() => + makeProvider().generate({ + model: 'dola-seedream-5-0-pro', + prompt: 'hi', + quality: '1.5k', + }), + ); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + expect( + (entries as Array<{ usageType: string }>)[0].usageType, + ).toBe( + 'byteplus-image-generation:dola-seedream-5-0-pro-260628:output:1.5k', + ); + }); + + it('bills pro input images from the second one on (first is free)', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + const PNG = 'data:image/png;base64,iVBORw0KGgo='; + await withTestActor(() => + makeProvider().generate({ + model: 'dola-seedream-5-0-pro', + prompt: 'hi', + input_images: [PNG, PNG, PNG], + }), + ); + const pro = findModel('dola-seedream-5-0-pro-260628'); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + const inputEntry = ( + entries as Array<{ + usageType: string; + usageAmount: number; + costOverride: number; + }> + ).find((e) => e.usageType.endsWith(':input_image'))!; + expect(inputEntry.usageAmount).toBe(2); + expect(inputEntry.costOverride).toBe( + 2 * pro.costs.input_image * 1_000_000, + ); + }); + + it('does not bill input images on flat-rate models', async () => { + generateMock.mockResolvedValueOnce(sampleResponse); + const PNG = 'data:image/png;base64,iVBORw0KGgo='; + await withTestActor(() => + makeProvider().generate({ + model: 'seedream-4-5', + prompt: 'hi', + input_images: [PNG, PNG, PNG], + }), + ); + const [, entries] = batchIncrementUsagesSpy.mock.calls[0]!; + expect(entries).toHaveLength(1); + }); +}); + +// -- Response handling ----------------------------------------------- + +describe('BytePlusImageProvider.generate response handling', () => { + it('falls back to a data URI when the response carries b64_json', async () => { + generateMock.mockResolvedValueOnce({ + data: [{ b64_json: 'AAAA', output_format: 'png' }], + }); + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'hi' }), + ); + expect(result).toBe('data:image/png;base64,AAAA'); + }); + + it('surfaces a per-image upstream error as 400 without metering', async () => { + generateMock.mockResolvedValueOnce({ + data: [{ error: { code: 'x', message: 'moderated' } }], + }); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + }); + + it('throws when the response has no usable image data', async () => { + generateMock.mockResolvedValueOnce({ data: [{}] }); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toThrow(/Failed to extract image URL/); + expect(batchIncrementUsagesSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.ts b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.ts new file mode 100644 index 000000000..d9c8962b6 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/byteplus/BytePlusImageProvider.ts @@ -0,0 +1,345 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { OpenAI } from 'openai'; +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { + IGenerateParams, + IImageModel, + IImageProvider, +} from '../../types.js'; +import { toUrlOrDataUri } from '../../inputImage.js'; +import { + BYTEPLUS_IMAGE_GENERATION_MODELS, + SEEDREAM_RESOLUTION_MAP, +} from './models.js'; + +const DEFAULT_MODEL = 'seedream-5-0-lite-260128'; + +// Ark's explicit-pixel `size` bounds ("method 2"): total pixels within +// [1280x720, 2048x2048x1.1025] and aspect ratio within [1/16, 16]. +const MIN_TOTAL_PIXELS = 921_600; +const MAX_TOTAL_PIXELS = 4_624_220; +// Models restricted to the 2K tier (see SEEDREAM_2K_ONLY in models.ts) +// enforce this higher minimum on explicit sizes too. +const MIN_TOTAL_PIXELS_2K_ONLY = 3_686_400; +// dola-seedream-5-0-pro's price break: ≤ 2.61MP bills the "1.5K or lower" +// rate, above it the higher rate. +const PRO_TIER_BREAK_PIXELS = 2_610_000; + +// Ark's `size` tiers, smallest first. +const TIERS = ['1k', '1.5k', '2k'] as const; +type Tier = (typeof TIERS)[number]; +const isTier = (v: string): v is Tier => + (TIERS as readonly string[]).includes(v); + +// Per-request reference image caps, per the API reference. +const MAX_INPUT_IMAGES_PRO = 10; +const MAX_INPUT_IMAGES_SEEDREAM = 14; + +type BytePlusImageConfig = { + apiKey: string; + apiBaseUrl?: string; +}; + +interface ArkImageResponse { + data?: Array<{ + url?: string; + b64_json?: string; + output_format?: string; + error?: { code?: string; message?: string }; + }>; + usage?: { generated_images?: number }; +} + +/** + * BytePlus ModelArk image generation provider (Seedream). + * + * Ark's `POST /images/generations` is OpenAI-compatible enough to reuse the + * OpenAI SDK (same client/auth as BytePlusProvider in ai-chat); Ark-specific + * params (`image`, `watermark`, tier-style `size`) pass through the SDK + * untouched. https://docs.byteplus.com/en/docs/ModelArk/1541523 + */ +export class BytePlusImageProvider implements IImageProvider { + #client: OpenAI; + #meteringService: MeteringService; + + constructor(config: BytePlusImageConfig, meteringService: MeteringService) { + if (!config.apiKey) { + throw new Error('BytePlus image generation requires an API key'); + } + this.#meteringService = meteringService; + this.#client = new OpenAI({ + apiKey: config.apiKey, + baseURL: + config.apiBaseUrl ?? + 'https://ark.ap-southeast.bytepluses.com/api/v3', + }); + } + + models(): IImageModel[] { + return BYTEPLUS_IMAGE_GENERATION_MODELS; + } + + getDefaultModel(): string { + return DEFAULT_MODEL; + } + + async generate(params: IGenerateParams): Promise { + const { prompt, test_mode, model, ratio, quality } = params; + let { input_images } = params; + const { input_image, input_image_mime_type } = params; + + const selectedModel = this.#getModel(model); + const isPro = selectedModel.pricing_unit === 'per-tier'; + + if (test_mode) { + return 'https://puter-sample-data.puter.site/image_example.png'; + } + + if (typeof prompt !== 'string' || prompt.trim().length === 0) { + throw new HttpError(400, '`prompt` must be a non-empty string', { + legacyCode: 'bad_request', + }); + } + + // Backwards compat: fold singular `input_image` into `input_images`. + if (input_image && (!input_images || input_images.length === 0)) { + input_images = [input_image]; + } + const maxInputImages = isPro + ? MAX_INPUT_IMAGES_PRO + : MAX_INPUT_IMAGES_SEEDREAM; + if (input_images && input_images.length > maxInputImages) { + throw new HttpError( + 400, + `${selectedModel.id} accepts at most ${maxInputImages} input image(s)`, + { legacyCode: 'bad_request' }, + ); + } + const inputImageCount = input_images?.length ?? 0; + + const tier = this.#normalizeTier(quality, selectedModel); + const size = this.#resolveSize(tier, selectedModel, ratio); + + // The pro model bills by output pixel count; everything else is a + // flat per-image rate. + let outputCostKey: string; + if (isPro) { + const pixels = this.#sizePixels(size); + outputCostKey = + pixels !== undefined + ? pixels > PRO_TIER_BREAK_PIXELS + ? 'output:2k' + : 'output:1k' + : `output:${tier}`; + } else { + outputCostKey = 'per-image'; + } + const outputCents = selectedModel.costs[outputCostKey]; + if (outputCents === undefined) { + throw new Error( + `Model ${selectedModel.id} missing '${outputCostKey}' cost`, + ); + } + // First input image is free on the pro model; the rest are billed. + const inputImageCents = selectedModel.costs.input_image ?? 0; + const billableInputs = + inputImageCents > 0 ? Math.max(0, inputImageCount - 1) : 0; + const estimatedCents = outputCents + billableInputs * inputImageCents; + + const actor = Context.get('actor'); + if (!actor) { + throw new HttpError(401, 'Authentication required', { + legacyCode: 'unauthorized', + }); + } + const usageAllowed = await this.#meteringService.hasEnoughCredits( + actor, + estimatedCents * 1_000_000, + ); + if (!usageAllowed) { + throw new HttpError( + 402, + 'Insufficient credits for image generation', + { legacyCode: 'insufficient_funds' }, + ); + } + + const image = + inputImageCount > 0 + ? input_images!.map((img) => + toUrlOrDataUri(img, input_image_mime_type), + ) + : undefined; + + const response = (await this.#client.images.generate({ + model: selectedModel.id, + prompt, + // Ark-specific params not in the OpenAI type; passed through. + ...(size ? { size } : {}), + ...(image ? { image: image.length === 1 ? image[0] : image } : {}), + response_format: 'url', + watermark: false, + } as Parameters[0])) as ArkImageResponse; + + const first = response.data?.[0]; + if (first?.error) { + throw new HttpError( + 400, + first.error.message ?? 'Image generation failed', + { + legacyCode: 'upstream_failed', + fields: { provider: 'byteplus' }, + }, + ); + } + const url = + first?.url || + (first?.b64_json + ? `data:image/${first.output_format ?? 'jpeg'};base64,${first.b64_json}` + : undefined); + if (!url) { + throw new Error( + 'Failed to extract image URL from BytePlus response', + ); + } + + const usageEntries = [ + { + usageType: `byteplus-image-generation:${selectedModel.id}:${outputCostKey}`, + usageAmount: 1, + costOverride: outputCents * 1_000_000, + }, + ]; + if (billableInputs > 0) { + usageEntries.push({ + usageType: `byteplus-image-generation:${selectedModel.id}:input_image`, + usageAmount: billableInputs, + costOverride: billableInputs * inputImageCents * 1_000_000, + }); + } + this.#meteringService.batchIncrementUsages(actor, usageEntries); + + return url; + } + + /** + * Pick the tier to request. Models with a minimum output-pixel count reject + * the smaller tiers outright (and the aspect-ratio table maps them to + * sub-minimum sizes), so snap up to the nearest tier the model allows + * rather than letting Ark 400 the request. + */ + #normalizeTier(quality: string | undefined, model: IImageModel): Tier { + const q = (quality ?? '').toLowerCase(); + // OpenAI-style quality names other Puter providers accept map onto + // the nearest Ark tier so e.g. 'low' isn't silently billed at the + // 2K rate; anything else falls back to Ark's own default of 2K. + const synonym: Record = { + low: '1k', + medium: '1.5k', + high: '2k', + hd: '2k', + }; + const requested: Tier = isTier(q) ? q : (synonym[q] ?? '2k'); + + const allowed = TIERS.filter( + (t) => model.allowedQualityLevels?.includes(t) ?? true, + ); + if (allowed.length === 0 || allowed.includes(requested)) { + return requested; + } + return ( + allowed.find((t) => TIERS.indexOf(t) > TIERS.indexOf(requested)) ?? + allowed[allowed.length - 1] + ); + } + + /** + * Resolve the `size` request param: + * + * - `ratio` holding real pixel dimensions → explicit `WxH` (method 2) + * - `ratio` holding an aspect ratio with a known tier mapping → the + * documented `WxH` for (aspect, tier) + * - Otherwise → the tier keyword (`1K`/`1.5K`/`2K`, method 1) + */ + #resolveSize( + tier: Tier, + model: IImageModel, + ratio?: { w: number; h: number }, + ): string { + if (ratio?.w && ratio?.h) { + const pixels = ratio.w * ratio.h; + if (pixels >= MIN_TOTAL_PIXELS) { + // 2K-only models enforce a higher minimum on explicit sizes + // too — fail fast with the real constraint instead of letting + // Ark 400 the request after the round-trip. + const minPixels = model.allowedQualityLevels?.includes('1k') + ? MIN_TOTAL_PIXELS + : MIN_TOTAL_PIXELS_2K_ONLY; + const aspect = ratio.w / ratio.h; + if ( + pixels < minPixels || + pixels > MAX_TOTAL_PIXELS || + aspect < 1 / 16 || + aspect > 16 + ) { + throw new HttpError( + 400, + `Requested size ${ratio.w}x${ratio.h} is outside BytePlus limits ` + + `for ${model.id} (total pixels within [${minPixels}, ${MAX_TOTAL_PIXELS}], ` + + 'aspect ratio within [1/16, 16])', + { legacyCode: 'bad_request' }, + ); + } + return `${ratio.w}x${ratio.h}`; + } + // Reduce w:h to lowest terms so any spelling of a supported + // aspect (8:6, 32:18, ...) finds its documented tier size. + const gcd = (a: number, b: number): number => + b === 0 ? a : gcd(b, a % b); + const d = gcd(Math.round(ratio.w), Math.round(ratio.h)) || 1; + const key = `${Math.round(ratio.w) / d}:${Math.round(ratio.h) / d}`; + const mapped = SEEDREAM_RESOLUTION_MAP[key]?.[tier]; + if (mapped) return `${mapped.w}x${mapped.h}`; + } + return { '1k': '1K', '1.5k': '1.5K', '2k': '2K' }[tier]; + } + + /** Pixel count of an explicit `WxH` size; undefined for tier keywords. */ + #sizePixels(size: string): number | undefined { + const m = /^(\d+)x(\d+)$/.exec(size); + if (!m) return undefined; + return Number(m[1]) * Number(m[2]); + } + + #getModel(model?: string) { + const models = this.models(); + const wanted = (model ?? '').trim().toLowerCase(); + const found = models.find( + (m) => + m.id === wanted || + m.puterId === wanted || + m.aliases?.some((a) => a.toLowerCase() === wanted), + ); + return found || models.find((m) => m.id === DEFAULT_MODEL)!; + } +} diff --git a/src/backend/drivers/ai-image/providers/byteplus/models.ts b/src/backend/drivers/ai-image/providers/byteplus/models.ts new file mode 100644 index 000000000..28547d406 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/byteplus/models.ts @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IImageModel } from '../../types.js'; + +// Ark's `size` "method 1" tiers and the pixel dimensions each (tier, aspect +// ratio) pair resolves to, from the image generation API reference: +// https://docs.byteplus.com/en/docs/ModelArk/1541523 +// The table is identical for dola-seedream-5-0-pro, seedream-5-0-lite, +// seedream-4-5 and seedream-4-0. +export const SEEDREAM_RESOLUTION_MAP: Record< + string, + Record +> = { + '1:1': { + '1k': { w: 1024, h: 1024 }, + '1.5k': { w: 1536, h: 1536 }, + '2k': { w: 2048, h: 2048 }, + }, + '4:3': { + '1k': { w: 1152, h: 864 }, + '1.5k': { w: 1792, h: 1344 }, + '2k': { w: 2368, h: 1776 }, + }, + '3:4': { + '1k': { w: 864, h: 1152 }, + '1.5k': { w: 1344, h: 1792 }, + '2k': { w: 1776, h: 2368 }, + }, + '16:9': { + '1k': { w: 1424, h: 800 }, + '1.5k': { w: 2048, h: 1152 }, + '2k': { w: 2816, h: 1584 }, + }, + '9:16': { + '1k': { w: 800, h: 1424 }, + '1.5k': { w: 1152, h: 2048 }, + '2k': { w: 1584, h: 2816 }, + }, + '3:2': { + '1k': { w: 1248, h: 832 }, + '1.5k': { w: 1872, h: 1248 }, + '2k': { w: 2496, h: 1664 }, + }, + '2:3': { + '1k': { w: 832, h: 1248 }, + '1.5k': { w: 1248, h: 1872 }, + '2k': { w: 1664, h: 2496 }, + }, + '21:9': { + '1k': { w: 1568, h: 672 }, + '1.5k': { w: 2352, h: 1008 }, + '2k': { w: 3136, h: 1344 }, + }, +}; + +const SEEDREAM_QUALITY_LEVELS = ['1k', '1.5k', '2k']; + +// seedream-4-5 and the 5.0 series enforce a minimum output of 3,686,400 +// pixels, which every 1K and 1.5K entry in the table above falls under — +// they only accept the 2K tier. +const SEEDREAM_2K_ONLY = ['2k']; + +// Costs are in usd-cents per image, hardcoded from +// https://docs.byteplus.com/en/docs/ModelArk/1544106 (pricing) and +// https://docs.byteplus.com/en/docs/ModelArk/1330310 (catalog). +// +// dola-seedream-5-0-pro bills by output pixel count — ≤ 2.61MP ("1.5K or +// lower") vs above — plus a per-input-image rate from the second reference +// image onward (the first is free). Every other model is a flat per-image +// rate with free image input. +export const BYTEPLUS_IMAGE_GENERATION_MODELS: IImageModel[] = [ + { + puterId: 'byteplus:byteplus/dola-seedream-5-0-pro-260628', + id: 'dola-seedream-5-0-pro-260628', + aliases: [ + 'byteplus/dola-seedream-5-0-pro-260628', + 'dola-seedream-5-0-pro', + 'byteplus/dola-seedream-5-0-pro', + 'seedream-5-0-pro', + ], + name: 'Dola Seedream 5.0 Pro', + costs_currency: 'usd-cents', + pricing_unit: 'per-tier', + index_cost_key: 'output:1k', + costs: { + 'output:1k': 4.5, // $0.045 per image ≤ 2.61MP + 'output:1.5k': 4.5, // same price as 1K, better quality + 'output:2k': 9, // $0.09 per image > 2.61MP + input_image: 0.3, // $0.003 per input image from the 2nd on + }, + allowedQualityLevels: SEEDREAM_QUALITY_LEVELS, + resolution_map: SEEDREAM_RESOLUTION_MAP, + }, + { + // The catalog lists `seedream-5-0-260128` as the same service + // ("also supports: seedream-5-0-lite-260128"); billing is published + // under the -lite id, so that's the primary id here. + puterId: 'byteplus:byteplus/seedream-5-0-lite-260128', + id: 'seedream-5-0-lite-260128', + aliases: [ + 'byteplus/seedream-5-0-lite-260128', + 'seedream-5-0-lite', + 'byteplus/seedream-5-0-lite', + 'seedream-5-0-260128', + 'seedream-5-0', + ], + name: 'Seedream 5.0 Lite', + costs_currency: 'usd-cents', + pricing_unit: 'per-image', + index_cost_key: 'per-image', + costs: { 'per-image': 3.5 }, + allowedQualityLevels: SEEDREAM_2K_ONLY, + resolution_map: SEEDREAM_RESOLUTION_MAP, + }, + { + puterId: 'byteplus:byteplus/seedream-4-5-251128', + id: 'seedream-4-5-251128', + aliases: [ + 'byteplus/seedream-4-5-251128', + 'seedream-4-5', + 'byteplus/seedream-4-5', + ], + name: 'Seedream 4.5', + costs_currency: 'usd-cents', + pricing_unit: 'per-image', + index_cost_key: 'per-image', + costs: { 'per-image': 4 }, + allowedQualityLevels: SEEDREAM_2K_ONLY, + resolution_map: SEEDREAM_RESOLUTION_MAP, + }, + { + puterId: 'byteplus:byteplus/seedream-4-0-250828', + id: 'seedream-4-0-250828', + aliases: [ + 'byteplus/seedream-4-0-250828', + 'seedream-4-0', + 'byteplus/seedream-4-0', + ], + name: 'Seedream 4.0', + costs_currency: 'usd-cents', + pricing_unit: 'per-image', + index_cost_key: 'per-image', + costs: { 'per-image': 3 }, + allowedQualityLevels: SEEDREAM_QUALITY_LEVELS, + resolution_map: SEEDREAM_RESOLUTION_MAP, + }, +]; diff --git a/src/backend/drivers/ai-video/VideoGenerationDriver.ts b/src/backend/drivers/ai-video/VideoGenerationDriver.ts index e3d73eb0c..58906717f 100644 --- a/src/backend/drivers/ai-video/VideoGenerationDriver.ts +++ b/src/backend/drivers/ai-video/VideoGenerationDriver.ts @@ -26,6 +26,7 @@ import type { Actor } from '../../core/actor.js'; import { PuterDriver } from '../types.js'; import { secureFetch } from '../../util/secureHttp.js'; import { AI_CONCURRENT, AI_RATE_LIMIT } from '../util/aiLimits.js'; +import { BytePlusVideoProvider } from './providers/byteplus/BytePlusVideoProvider.js'; import { GeminiVideoProvider } from './providers/gemini/GeminiVideoProvider.js'; import { OpenAIVideoProvider } from './providers/openai/OpenAIVideoProvider.js'; import { TogetherVideoProvider } from './providers/together/TogetherVideoProvider.js'; @@ -57,6 +58,7 @@ export class VideoGenerationDriver extends PuterDriver { 'openai-video-generation', 'together-video-generation', 'gemini-video-generation', + 'byteplus-video-generation', ]; readonly isDefault = true; @@ -220,11 +222,16 @@ export class VideoGenerationDriver extends PuterDriver { ? args.resolution : undefined; + // Case-insensitive so '4K' matches a catalog entry spelled '4k'; + // the matched catalog spelling (not the caller's) is forwarded. const normalizedResolution = - requestedResolution && - model.dimensions.includes(requestedResolution) - ? requestedResolution - : model.dimensions[0]; + (requestedResolution && + model.dimensions.find( + (d) => + d.toLowerCase() === + requestedResolution.toLowerCase(), + )) || + model.dimensions[0]; args.size = normalizedResolution; args.resolution = normalizedResolution; } @@ -291,6 +298,29 @@ export class VideoGenerationDriver extends PuterDriver { this.#providers['gemini-video-generation'] = new GeminiVideoProvider({ apiKey: geminiKey }, m); } + + // Falls back to the shared `byteplus` (ai-chat) key; `apiBaseUrl` + // selects the ModelArk region, same as the chat provider. Each field + // falls through independently so a partial video-specific block can't + // pair its missing apiBaseUrl with the shared block's key (or vice + // versa) and point a region-scoped key at the wrong endpoint. + const byteplusVideoCfg = providers['byteplus-video-generation'] as + Record | undefined; + const byteplusSharedCfg = providers['byteplus'] as + Record | undefined; + const byteplusKey = readKey(byteplusVideoCfg, byteplusSharedCfg); + if (byteplusKey) { + this.#providers['byteplus-video-generation'] = + new BytePlusVideoProvider( + { + apiKey: byteplusKey, + apiBaseUrl: (byteplusVideoCfg?.apiBaseUrl ?? + byteplusSharedCfg?.apiBaseUrl) as + string | undefined, + }, + m, + ); + } } // -- Model map ----------------------------------------------------------- diff --git a/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.integration.test.ts b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.integration.test.ts new file mode 100644 index 000000000..b2a20cfee --- /dev/null +++ b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.integration.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the BytePlus ModelArk video provider. + * + * Generates a 2s 480p clip on seedance-1-0-pro-fast (~$0.02). Video tasks + * poll for completion, so the timeout is generous. Skipped when + * `PUTER_TEST_AI_BYTEPLUS_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { BytePlusVideoProvider } from './BytePlusVideoProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_BYTEPLUS_API_KEY'; +// Task-based generation routinely takes a couple of minutes. +const VIDEO_TIMEOUT_MS = 5 * 60 * 1000; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'BytePlusVideoProvider (integration)', + () => { + it( + 'generates a video URL from seedance-1-0-pro-fast', + { timeout: VIDEO_TIMEOUT_MS }, + async () => { + const provider = new BytePlusVideoProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const url = await withTestActor(() => + provider.generate({ + model: 'seedance-1-0-pro-fast', + prompt: 'a red ball rolls to the right', + resolution: '480p', + seconds: 2, + }), + ); + + expect(typeof url).toBe('string'); + expect((url as string).startsWith('https://')).toBe(true); + }, + ); + }, +); diff --git a/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.test.ts b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.test.ts new file mode 100644 index 000000000..a3b2dc38d --- /dev/null +++ b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.test.ts @@ -0,0 +1,464 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for BytePlusVideoProvider. + * + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock + * redis) and constructs BytePlusVideoProvider directly against the + * live wired `MeteringService`. Ark's task-based video API has no + * SDK — the provider hits it via global `fetch`, which we stub; + * that's the real network egress point. + */ + +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { BYTEPLUS_VIDEO_GENERATION_MODELS } from './models.js'; +import { BytePlusVideoProvider } from './BytePlusVideoProvider.js'; + +// -- Test harness ---------------------------------------------------- + +let server: PuterServer; +let fetchSpy: MockInstance; +let remainingUsageSpy: MockInstance; +let incrementUsageSpy: MockInstance; + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = ( + config: { apiKey?: string; apiBaseUrl?: string } = {}, +) => + new BytePlusVideoProvider( + { + apiKey: config.apiKey ?? 'test-key', + ...(config.apiBaseUrl ? { apiBaseUrl: config.apiBaseUrl } : {}), + pollIntervalMs: 1, + }, + server.services.metering, + ); + +beforeEach(() => { + fetchSpy = vi.spyOn(globalThis, 'fetch') as MockInstance; + remainingUsageSpy = vi.spyOn( + server.services.metering, + 'getRemainingUsage', + ); + // Plenty of credit unless a test says otherwise. + remainingUsageSpy.mockResolvedValue(Number.MAX_SAFE_INTEGER); + incrementUsageSpy = vi.spyOn(server.services.metering, 'incrementUsage'); + incrementUsageSpy.mockResolvedValue({} as never); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +const jsonResponse = (body: unknown, status = 200) => + new Response(JSON.stringify(body), { + status, + headers: { 'content-type': 'application/json' }, + }); + +const succeededTask = (overrides: Record = {}) => ({ + id: 'cgt-test-1', + status: 'succeeded', + content: { video_url: 'https://ark.example/video.mp4' }, + usage: { completion_tokens: 108_000, total_tokens: 108_000 }, + resolution: '720p', + duration: 5, + ...overrides, +}); + +/** Queue up the POST-create response followed by GET-poll responses. */ +const mockTaskFlow = (...pollBodies: unknown[]) => { + fetchSpy.mockResolvedValueOnce(jsonResponse({ id: 'cgt-test-1' })); + for (const body of pollBodies) { + fetchSpy.mockResolvedValueOnce(jsonResponse(body)); + } +}; + +const sentBody = (callIndex = 0): Record => + JSON.parse( + (fetchSpy.mock.calls[callIndex]![1] as RequestInit).body as string, + ); + +const findModel = (id: string) => + BYTEPLUS_VIDEO_GENERATION_MODELS.find((m) => m.id === id)!; + +// -- Construction / catalog ------------------------------------------ + +describe('BytePlusVideoProvider construction and catalog', () => { + it('throws when no apiKey is supplied', () => { + expect( + () => + new BytePlusVideoProvider( + { apiKey: '' }, + server.services.metering, + ), + ).toThrow(/API key/i); + }); + + it('does not call out at construction (lazy fetch)', () => { + makeProvider(); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('defaults to seedance 2.0 mini', () => { + expect(makeProvider().getDefaultModel()).toBe( + 'dreamina-seedance-2-0-mini-260615', + ); + }); + + it('exposes the static catalog', async () => { + expect(await makeProvider().models()).toBe( + BYTEPLUS_VIDEO_GENERATION_MODELS, + ); + }); +}); + +// -- Gates ----------------------------------------------------------- + +describe('BytePlusVideoProvider.generate gates', () => { + it('returns the canned sample URL in test_mode without network calls', async () => { + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'x', test_mode: true }), + ); + expect(result).toBe('https://assets.puter.site/txt2vid.mp4'); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('throws 400 on a missing or blank prompt', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: undefined as unknown as string, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('throws 402 BEFORE creating a task when even the shortest clip is unaffordable', async () => { + remainingUsageSpy.mockResolvedValue(0); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 402 }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); +}); + +// -- Request shape --------------------------------------------------- + +describe('BytePlusVideoProvider.generate request shape', () => { + it('POSTs a create-task request with text content and defaults', async () => { + mockTaskFlow(succeededTask()); + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'a kitten yawns' }), + ); + + expect(result).toBe('https://ark.example/video.mp4'); + const [url, init] = fetchSpy.mock.calls[0]!; + expect(String(url)).toBe( + 'https://ark.ap-southeast.bytepluses.com/api/v3/contents/generations/tasks', + ); + expect((init as RequestInit).method).toBe('POST'); + expect( + (init as RequestInit & { headers: Record }) + .headers.Authorization, + ).toBe('Bearer test-key'); + + const body = sentBody(); + expect(body.model).toBe('dreamina-seedance-2-0-mini-260615'); + expect(body.content).toEqual([ + { type: 'text', text: 'a kitten yawns' }, + ]); + expect(body.resolution).toBe('720p'); + expect(body.duration).toBe(5); + expect(body.watermark).toBe(false); + expect(body.generate_audio).toBe(true); + + // Poll goes to GET tasks/{id}. + const [pollUrl, pollInit] = fetchSpy.mock.calls[1]!; + expect(String(pollUrl)).toBe( + 'https://ark.ap-southeast.bytepluses.com/api/v3/contents/generations/tasks/cgt-test-1', + ); + expect((pollInit as RequestInit).method).toBe('GET'); + }); + + it('resolves aliases, uppercases 4k, and clamps duration to the model range', async () => { + mockTaskFlow(succeededTask({ resolution: '4k' })); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-2-0', + prompt: 'hi', + resolution: '4k', + seconds: 99, + }), + ); + const body = sentBody(); + expect(body.model).toBe('dreamina-seedance-2-0-260128'); + expect(body.resolution).toBe('4K'); + expect(body.duration).toBe(15); + }); + + // A clip shorter than the model's minimum is a duration to round up, not + // an affordability problem — it used to surface as "insufficient funds". + it('rounds a sub-minimum duration up to the shortest supported clip', async () => { + mockTaskFlow(succeededTask({ resolution: '480p' })); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-2-0-mini', + prompt: 'hi', + resolution: '480p', + seconds: 2, + }), + ); + expect(sentBody().duration).toBe(4); + }); + + // `dims` is shared across a model family, so it can't be the gate for + // what an individual model accepts. + it('falls back to the default resolution when the model does not list it', async () => { + mockTaskFlow(succeededTask({ resolution: '720p' })); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-2-0-mini', + prompt: 'hi', + resolution: '1080p', + }), + ); + expect(sentBody().resolution).toBe('720p'); + }); + + it('omits generate_audio for models without audio and passes seed for 1.x', async () => { + mockTaskFlow(succeededTask({ resolution: '1080p' })); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-1-0-pro', + prompt: 'hi', + seed: 11, + }), + ); + const body = sentBody(); + expect(body.generate_audio).toBeUndefined(); + expect(body.seed).toBe(11); + expect(body.resolution).toBe('1080p'); + }); + + it('derives a supported ratio from width/height and omits unsupported ones', async () => { + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + width: 1280, + height: 720, + }), + ); + expect(sentBody().ratio).toBe('16:9'); + + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ prompt: 'hi', width: 999, height: 100 }), + ); + expect(sentBody(2).ratio).toBeUndefined(); + }); + + it('maps input_reference/last_frame to first/last frame roles', async () => { + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + input_reference: 'https://example.com/first.png', + last_frame: 'https://example.com/last.png', + }), + ); + expect(sentBody().content).toEqual([ + { type: 'text', text: 'hi' }, + { + type: 'image_url', + image_url: { url: 'https://example.com/first.png' }, + role: 'first_frame', + }, + { + type: 'image_url', + image_url: { url: 'https://example.com/last.png' }, + role: 'last_frame', + }, + ]); + }); + + it('maps reference_images for the 2.0 series and rejects them elsewhere', async () => { + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-2-0', + prompt: 'hi', + reference_images: ['https://example.com/ref.png'], + }), + ); + expect(sentBody().content).toContainEqual({ + type: 'image_url', + image_url: { url: 'https://example.com/ref.png' }, + role: 'reference_image', + }); + + await expect( + withTestActor(() => + makeProvider().generate({ + model: 'seedance-1-0-pro', + prompt: 'hi', + reference_images: ['https://example.com/ref.png'], + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + }); + + it('rejects last_frame without a first frame', async () => { + await expect( + withTestActor(() => + makeProvider().generate({ + prompt: 'hi', + last_frame: 'https://example.com/last.png', + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(fetchSpy).not.toHaveBeenCalled(); + }); +}); + +// -- Polling / outcomes ---------------------------------------------- + +describe('BytePlusVideoProvider.generate polling and outcomes', () => { + it('keeps polling while the task is queued/running', async () => { + mockTaskFlow( + { id: 'cgt-test-1', status: 'queued' }, + { id: 'cgt-test-1', status: 'running' }, + succeededTask(), + ); + const result = await withTestActor(() => + makeProvider().generate({ prompt: 'hi' }), + ); + expect(result).toBe('https://ark.example/video.mp4'); + expect(fetchSpy).toHaveBeenCalledTimes(4); // 1 create + 3 polls + }); + + it('throws 400 upstream_failed when the task fails, without metering', async () => { + mockTaskFlow({ + id: 'cgt-test-1', + status: 'failed', + error: { code: 'moderation', message: 'blocked' }, + }); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 400, message: 'blocked' }); + expect(incrementUsageSpy).not.toHaveBeenCalled(); + }); + + it('maps upstream 5xx on create to a 502', async () => { + fetchSpy.mockResolvedValueOnce( + jsonResponse({ error: { message: 'boom' } }, 500), + ); + await expect( + withTestActor(() => makeProvider().generate({ prompt: 'hi' })), + ).rejects.toMatchObject({ statusCode: 502 }); + }); +}); + +// -- Metering -------------------------------------------------------- + +describe('BytePlusVideoProvider.generate metering', () => { + it('bills the tokens the task reports at the resolution rate', async () => { + mockTaskFlow(succeededTask({ resolution: '1080p' })); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-2-0', + prompt: 'hi', + resolution: '1080p', + }), + ); + const model = findModel('dreamina-seedance-2-0-260128'); + const rate = model.costs!['video_tokens:1080p']; + expect(incrementUsageSpy).toHaveBeenCalledWith( + expect.anything(), + 'byteplus-video-generation:dreamina-seedance-2-0-260128:video_tokens:1080p', + 108_000, + 108_000 * rate * 1_000_000, + ); + }); + + it('bills seedance 1.5 pro at the silent rate when generate_audio is false', async () => { + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ + model: 'seedance-1-5-pro', + prompt: 'hi', + generate_audio: false, + }), + ); + expect(sentBody().generate_audio).toBe(false); + const model = findModel('seedance-1-5-pro-251215'); + const rate = model.costs!['video_tokens:silent']; + expect(incrementUsageSpy).toHaveBeenCalledWith( + expect.anything(), + 'byteplus-video-generation:seedance-1-5-pro-251215:video_tokens:silent', + 108_000, + 108_000 * rate * 1_000_000, + ); + }); + + it('clamps the clip to what remaining credit buys', async () => { + // seedance 2.0 mini @720p: 1280×720×24/1024 = 21600 tokens/s at + // 0.00035¢/token → 7.56¢/s. Grant ~15¢ ≈ 2s... below the 4s + // minimum → 402. Grant ~40¢ → 5s requested, affordable. + const perSecondMicroCents = 21_600 * 0.00035 * 1_000_000; + remainingUsageSpy.mockResolvedValue(4.5 * perSecondMicroCents); + mockTaskFlow(succeededTask()); + await withTestActor(() => + makeProvider().generate({ prompt: 'hi', seconds: 10 }), + ); + expect(sentBody().duration).toBe(4); + + remainingUsageSpy.mockResolvedValue(2 * perSecondMicroCents); + await expect( + withTestActor(() => + makeProvider().generate({ prompt: 'hi', seconds: 10 }), + ), + ).rejects.toMatchObject({ statusCode: 402 }); + }); +}); diff --git a/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.ts b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.ts new file mode 100644 index 000000000..ff0cac016 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/byteplus/BytePlusVideoProvider.ts @@ -0,0 +1,476 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import type { IGenerateVideoParams, IVideoModel } from '../../types.js'; +import { capSecondsToRemainingCredits } from '../../creditCap.js'; +import { VideoProvider } from '../VideoProvider.js'; +import { + BYTEPLUS_VIDEO_GENERATION_MODELS, + BYTEPLUS_VIDEO_SPECS, + type BytePlusVideoSpec, +} from './models.js'; + +const DEFAULT_TEST_VIDEO_URL = 'https://assets.puter.site/txt2vid.mp4'; +const DEFAULT_BASE_URL = 'https://ark.ap-southeast.bytepluses.com/api/v3'; +const DEFAULT_POLL_INTERVAL_MS = 5_000; +const DEFAULT_TIMEOUT_MS = 5 * 60 * 1000; +const DEFAULT_MODEL = 'dreamina-seedance-2-0-mini-260615'; +// Seedance 2.0 multimodal reference accepts up to 9 reference images. +const MAX_REFERENCE_IMAGES = 9; + +const ARK_RATIOS = ['16:9', '4:3', '1:1', '3:4', '9:16', '21:9']; + +type BytePlusVideoConfig = { + apiKey: string; + apiBaseUrl?: string; + /** Test hook — polling cadence for task status checks. */ + pollIntervalMs?: number; +}; + +interface ArkVideoTask { + id: string; + status: + 'queued' | 'running' | 'cancelled' | 'succeeded' | 'failed' | 'expired'; + content?: { video_url?: string }; + usage?: { completion_tokens?: number; total_tokens?: number }; + resolution?: string; + duration?: number; + error?: { code?: string; message?: string } | null; +} + +/** + * BytePlus ModelArk video generation provider (Seedance). + * + * Ark's video API is task-based rather than OpenAI-shaped: POST + * `/contents/generations/tasks` returns a task id, which is then polled via GET + * until it leaves queued/running. Billing is per video token (≈ duration × + * width × height × fps / 1024) with the authoritative count in the final task's + * `usage.completion_tokens`. + * https://docs.byteplus.com/en/docs/ModelArk/1520757 + */ +export class BytePlusVideoProvider extends VideoProvider { + #apiKey: string; + #baseUrl: string; + #pollIntervalMs: number; + #meteringService: MeteringService; + + constructor(config: BytePlusVideoConfig, meteringService: MeteringService) { + super(); + if (!config.apiKey) { + throw new Error('BytePlus video generation requires an API key'); + } + this.#apiKey = config.apiKey; + this.#baseUrl = (config.apiBaseUrl ?? DEFAULT_BASE_URL).replace( + /\/$/, + '', + ); + this.#pollIntervalMs = + config.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS; + this.#meteringService = meteringService; + } + + getDefaultModel(): string { + return DEFAULT_MODEL; + } + + async models(): Promise { + return BYTEPLUS_VIDEO_GENERATION_MODELS; + } + + async generate(params: IGenerateVideoParams): Promise { + const { + prompt, + model: requestedModel, + seconds, + duration, + size, + resolution, + width, + height, + seed, + generate_audio: generateAudio, + input_reference: inputReference, + last_frame: lastFrame, + reference_images: referenceImages, + test_mode: testMode, + } = params ?? {}; + + if (typeof prompt !== 'string' || !prompt.trim()) { + throw new HttpError(400, 'prompt must be a non-empty string', { + legacyCode: 'bad_request', + }); + } + + const model = this.#getModel(requestedModel); + const spec = BYTEPLUS_VIDEO_SPECS[model.id]; + + if (testMode) { + return DEFAULT_TEST_VIDEO_URL; + } + + const actor = Context.get('actor'); + if (!actor) { + throw new HttpError(401, 'Authentication required', { + legacyCode: 'unauthorized', + }); + } + + const resolutionKey = this.#normalizeResolution( + size ?? resolution, + model, + ); + // Audio is priced in for 1.5 Pro, so always send an explicit value to + // keep billing deterministic; Ark's own default is true. + const audioOn = spec.supportsAudio && generateAudio !== false; + const costKey = this.#costKey(model, resolutionKey, audioOn); + const centsPerToken = model.costs?.[costKey]; + if (!centsPerToken) { + throw new Error( + `No pricing configured for video model ${model.id} at ${resolutionKey}`, + ); + } + + const dims = spec.dims[resolutionKey]; + const tokensPerSecond = (dims.w * dims.h * 24) / 1024; + const perSecondMicroCents = tokensPerSecond * centsPerToken * 1_000_000; + + const requestedSeconds = Math.min( + this.#coerceSeconds(seconds ?? duration) ?? spec.duration.default, + spec.duration.max, + ); + // `durationSeconds` enumerates the model's whole-second range, so a + // sub-minimum request rounds up to the shortest supported clip rather + // than being rejected as unaffordable. + const cappedSeconds = await capSecondsToRemainingCredits({ + metering: this.#meteringService, + actor, + perSecondMicroCents, + requestedSeconds, + allowedSeconds: model.durationSeconds, + modelId: model.id, + }); + + const body: Record = { + model: model.id, + content: this.#buildContent(prompt, spec, model.id, { + inputReference, + lastFrame, + referenceImages, + }), + resolution: resolutionKey === '4k' ? '4K' : resolutionKey, + duration: cappedSeconds, + watermark: false, + }; + if (spec.supportsAudio) { + body.generate_audio = audioOn; + } + const ratio = this.#deriveRatio(width, height); + if (ratio) { + body.ratio = ratio; + } + if ( + spec.supportsSeed && + typeof seed === 'number' && + Number.isFinite(seed) + ) { + body.seed = Math.round(seed); + } + + const task = await this.#createTask(body); + const finalTask = await this.#pollUntilComplete(task.id); + + if (finalTask.status !== 'succeeded') { + const errorMessage = + finalTask.error?.message ?? + `Video generation ${finalTask.status}`; + // Ark's `failed` covers both user-input issues (content + // moderation) and upstream outages — same ambiguity as the + // Together provider, so expose it the same way: a 400 with + // `upstream_failed` that the alarm gate skips. + throw new HttpError(400, errorMessage, { + legacyCode: 'upstream_failed', + fields: { provider: 'byteplus' }, + }); + } + + const videoUrl = finalTask.content?.video_url; + if (typeof videoUrl !== 'string' || !videoUrl.trim()) { + throw new Error('BytePlus response did not include a video URL'); + } + + // Bill the tokens the task actually reports; fall back to the + // pre-flight estimate if usage is missing. + const finalResolutionKey = this.#normalizeResolution( + finalTask.resolution, + model, + resolutionKey, + ); + const finalCostKey = this.#costKey(model, finalResolutionKey, audioOn); + const finalCentsPerToken = model.costs?.[finalCostKey] ?? centsPerToken; + const tokens = + finalTask.usage?.completion_tokens ?? + Math.round(tokensPerSecond * cappedSeconds); + await this.#meteringService.incrementUsage( + actor, + `byteplus-video-generation:${model.id}:${finalCostKey}`, + tokens, + tokens * finalCentsPerToken * 1_000_000, + ); + + return videoUrl; + } + + #buildContent( + prompt: string, + spec: BytePlusVideoSpec, + modelId: string, + images: { + inputReference?: unknown; + lastFrame?: string; + referenceImages?: string[]; + }, + ): Array> { + const { inputReference, lastFrame, referenceImages } = images; + const content: Array> = [ + { type: 'text', text: prompt }, + ]; + + const firstFrame = + typeof inputReference === 'string' && inputReference.trim() + ? inputReference + : undefined; + const hasReferenceImages = + Array.isArray(referenceImages) && referenceImages.length > 0; + + // Ark treats first/last-frame and reference-image generation as + // mutually exclusive scenarios. + if (hasReferenceImages && (firstFrame || lastFrame)) { + throw new HttpError( + 400, + 'reference_images cannot be combined with input_reference/last_frame', + { legacyCode: 'bad_request' }, + ); + } + + if (hasReferenceImages) { + if (!spec.supportsReferenceImages) { + throw new HttpError( + 400, + `${modelId} does not support reference_images`, + { legacyCode: 'bad_request' }, + ); + } + if (referenceImages!.length > MAX_REFERENCE_IMAGES) { + throw new HttpError( + 400, + `${modelId} accepts at most ${MAX_REFERENCE_IMAGES} reference image(s)`, + { legacyCode: 'bad_request' }, + ); + } + for (const img of referenceImages!) { + if (typeof img !== 'string' || !img.trim()) continue; + content.push({ + type: 'image_url', + image_url: { url: img }, + role: 'reference_image', + }); + } + return content; + } + + if (lastFrame && !firstFrame) { + throw new HttpError( + 400, + 'last_frame requires a first-frame image via input_reference', + { legacyCode: 'bad_request' }, + ); + } + if (firstFrame) { + content.push({ + type: 'image_url', + image_url: { url: firstFrame }, + ...(lastFrame ? { role: 'first_frame' } : {}), + }); + } + if (lastFrame) { + if (!spec.supportsLastFrame) { + throw new HttpError( + 400, + `${modelId} does not support last_frame`, + { legacyCode: 'bad_request' }, + ); + } + content.push({ + type: 'image_url', + image_url: { url: lastFrame }, + role: 'last_frame', + }); + } + + return content; + } + + async #createTask(body: Record): Promise { + return (await this.#request('POST', '/contents/generations/tasks', { + body, + })) as ArkVideoTask; + } + + async #pollUntilComplete(taskId: string): Promise { + const start = Date.now(); + for (;;) { + const task = (await this.#request( + 'GET', + `/contents/generations/tasks/${taskId}`, + )) as ArkVideoTask; + if (task.status !== 'queued' && task.status !== 'running') { + return task; + } + if (Date.now() - start > DEFAULT_TIMEOUT_MS) { + throw new Error( + 'Timed out waiting for BytePlus video generation to complete', + ); + } + await this.#delay(this.#pollIntervalMs); + } + } + + async #request( + method: string, + path: string, + opts: { body?: Record } = {}, + ): Promise { + const response = await fetch(`${this.#baseUrl}${path}`, { + method, + headers: { + Authorization: `Bearer ${this.#apiKey}`, + ...(opts.body ? { 'Content-Type': 'application/json' } : {}), + }, + ...(opts.body ? { body: JSON.stringify(opts.body) } : {}), + }); + const payload = (await response.json().catch(() => ({}))) as Record< + string, + unknown + >; + if (!response.ok) { + const message = + ((payload.error as Record) + ?.message as string) ?? + `BytePlus video API error (status ${response.status})`; + throw new HttpError(response.status >= 500 ? 502 : 400, message, { + legacyCode: 'upstream_failed', + fields: { provider: 'byteplus' }, + }); + } + return payload; + } + + async #delay(ms: number): Promise { + return await new Promise((resolve) => setTimeout(resolve, ms)); + } + + #getModel(requestedModel?: string): IVideoModel { + const wanted = (requestedModel ?? '').trim().toLowerCase(); + const found = BYTEPLUS_VIDEO_GENERATION_MODELS.find( + (m) => + m.id === wanted || + m.puterId === wanted || + m.aliases?.some((a) => a.toLowerCase() === wanted), + ); + return ( + found ?? + BYTEPLUS_VIDEO_GENERATION_MODELS.find( + (m) => m.id === DEFAULT_MODEL, + )! + ); + } + + /** '480p' | '720p' | '1080p' | '4k', falling back to the model default. */ + #normalizeResolution( + candidate: unknown, + model: IVideoModel, + fallback?: string, + ): string { + const spec = BYTEPLUS_VIDEO_SPECS[model.id]; + if (typeof candidate === 'string') { + const normalized = candidate.trim().toLowerCase(); + // The shared `dims` table covers a whole model family, so gate on + // what this model actually advertises — otherwise e.g. 1080p on a + // 480p/720p-only model reaches Ark just to be rejected there. + const supported = model.dimensions!.some( + (d) => d.toLowerCase() === normalized, + ); + if (supported && spec.dims[normalized]) return normalized; + } + return fallback ?? model.dimensions![0].toLowerCase(); + } + + #costKey( + model: IVideoModel, + resolutionKey: string, + audioOn: boolean, + ): string { + const costs = model.costs ?? {}; + if (costs[`video_tokens:${resolutionKey}`] !== undefined) { + return `video_tokens:${resolutionKey}`; + } + const audioKey = audioOn ? 'video_tokens:audio' : 'video_tokens:silent'; + if (costs[audioKey] !== undefined) { + return audioKey; + } + return 'video_tokens'; + } + + #coerceSeconds(value: unknown): number | undefined { + if (typeof value === 'number' && Number.isFinite(value)) { + const rounded = Math.round(value); + return rounded > 0 ? rounded : undefined; + } + if (typeof value === 'string') { + const numeric = Number.parseInt(value, 10); + return Number.isFinite(numeric) && numeric > 0 + ? numeric + : undefined; + } + return undefined; + } + + /** Snap width/height to one of Ark's supported aspect-ratio strings. */ + #deriveRatio(width?: number, height?: number): string | undefined { + if ( + typeof width !== 'number' || + typeof height !== 'number' || + !Number.isFinite(width) || + !Number.isFinite(height) || + width <= 0 || + height <= 0 + ) { + return undefined; + } + const gcd = (a: number, b: number): number => + b === 0 ? a : gcd(b, a % b); + const d = gcd(Math.round(width), Math.round(height)) || 1; + const candidate = `${Math.round(width) / d}:${Math.round(height) / d}`; + // Unsupported ratios are omitted so Ark's `adaptive` default applies. + return ARK_RATIOS.includes(candidate) ? candidate : undefined; + } +} diff --git a/src/backend/drivers/ai-video/providers/byteplus/models.ts b/src/backend/drivers/ai-video/providers/byteplus/models.ts new file mode 100644 index 000000000..5fad94af2 --- /dev/null +++ b/src/backend/drivers/ai-video/providers/byteplus/models.ts @@ -0,0 +1,256 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IVideoModel } from '../../types.js'; + +/** + * Ark-specific model behavior the shared IVideoModel shape can't carry. Keyed + * by model id. + */ +export interface BytePlusVideoSpec { + /** Valid `duration` range (integer seconds) and Ark's default. */ + duration: { min: number; max: number; default: number }; + /** + * 16:9 output dimensions per resolution key, from the create-task API's + * ratio table. Used only for pre-flight cost estimates — actual billing + * uses the `usage.completion_tokens` the task reports. + */ + dims: Record; + /** Supports `generate_audio` (Seedance 2.0 series + 1.5 Pro). */ + supportsAudio: boolean; + /** Supports first+last frame image-to-video. */ + supportsLastFrame: boolean; + /** Supports multimodal `reference_image` inputs (Seedance 2.0 series). */ + supportsReferenceImages: boolean; + /** Supports the `seed` param (not the Seedance 2.0 series). */ + supportsSeed: boolean; +} + +// Duration ladder for the driver's normalization: element 0 is the fallback +// default, the rest enumerate the model's contiguous valid range. +const seconds = (def: number, min: number, max: number): number[] => [ + def, + ...Array.from({ length: max - min + 1 }, (_, i) => min + i).filter( + (s) => s !== def, + ), +]; + +const FPS = [24]; + +// USD per million tokens → usd-cents per token. +const perMToken = (usd: number): number => (usd * 100) / 1_000_000; + +// Hardcoded from https://docs.byteplus.com/en/docs/ModelArk/1544106 (pricing, +// "online inference / input without video" rates — reference-video input is +// not exposed through this driver) and the create-task API reference +// https://docs.byteplus.com/en/docs/ModelArk/1520757 (capabilities). +// +// Video is billed per token: tokens ≈ duration × width × height × fps / 1024, +// with the authoritative count returned as `usage.completion_tokens`. +// `default-duration-per-video` is the estimated cents for a 5s clip at the +// model's default resolution — it exists for cross-provider cost sorting and +// display, not billing. +// +// dreamina-seedance-2-5-260628 is priced on the pricing page but the API +// reference still lists its API access as "available soon", so it's +// deliberately absent here. +export const BYTEPLUS_VIDEO_GENERATION_MODELS: IVideoModel[] = [ + { + id: 'dreamina-seedance-2-0-260128', + puterId: 'byteplus:byteplus/dreamina-seedance-2-0-260128', + aliases: [ + 'dreamina-seedance-2-0', + 'byteplus/dreamina-seedance-2-0', + 'seedance-2-0', + ], + name: 'Dreamina Seedance 2.0', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + 'video_tokens:480p': perMToken(7.0), + 'video_tokens:720p': perMToken(7.0), + 'video_tokens:1080p': perMToken(7.7), + 'video_tokens:4k': perMToken(4.0), + 'default-duration-per-video': 76, + }, + durationSeconds: seconds(5, 4, 15), + dimensions: ['720p', '480p', '1080p', '4k'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:dreamina-seedance-2-0-260128:video_tokens:720p', + }, + { + id: 'dreamina-seedance-2-0-fast-260128', + puterId: 'byteplus:byteplus/dreamina-seedance-2-0-fast-260128', + aliases: [ + 'dreamina-seedance-2-0-fast', + 'byteplus/dreamina-seedance-2-0-fast', + 'seedance-2-0-fast', + ], + name: 'Dreamina Seedance 2.0 Fast', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + video_tokens: perMToken(5.6), + 'default-duration-per-video': 60, + }, + durationSeconds: seconds(5, 4, 15), + dimensions: ['720p', '480p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:dreamina-seedance-2-0-fast-260128:video_tokens', + }, + { + id: 'dreamina-seedance-2-0-mini-260615', + puterId: 'byteplus:byteplus/dreamina-seedance-2-0-mini-260615', + aliases: [ + 'dreamina-seedance-2-0-mini', + 'byteplus/dreamina-seedance-2-0-mini', + 'seedance-2-0-mini', + ], + name: 'Dreamina Seedance 2.0 Mini', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + video_tokens: perMToken(3.5), + 'default-duration-per-video': 38, + }, + durationSeconds: seconds(5, 4, 15), + dimensions: ['720p', '480p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:dreamina-seedance-2-0-mini-260615:video_tokens', + }, + { + id: 'seedance-1-5-pro-251215', + puterId: 'byteplus:byteplus/seedance-1-5-pro-251215', + aliases: ['seedance-1-5-pro', 'byteplus/seedance-1-5-pro'], + name: 'Seedance 1.5 Pro', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + 'video_tokens:audio': perMToken(2.4), + 'video_tokens:silent': perMToken(1.2), + 'default-duration-per-video': 26, + }, + durationSeconds: seconds(5, 4, 12), + dimensions: ['720p', '480p', '1080p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:seedance-1-5-pro-251215:video_tokens:audio', + }, + { + id: 'seedance-1-0-pro-250528', + puterId: 'byteplus:byteplus/seedance-1-0-pro-250528', + aliases: ['seedance-1-0-pro', 'byteplus/seedance-1-0-pro'], + name: 'Seedance 1.0 Pro', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + video_tokens: perMToken(2.5), + 'default-duration-per-video': 61, + }, + durationSeconds: seconds(5, 2, 12), + dimensions: ['1080p', '480p', '720p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:seedance-1-0-pro-250528:video_tokens', + }, + { + id: 'seedance-1-0-pro-fast-251015', + puterId: 'byteplus:byteplus/seedance-1-0-pro-fast-251015', + aliases: ['seedance-1-0-pro-fast', 'byteplus/seedance-1-0-pro-fast'], + name: 'Seedance 1.0 Pro Fast', + costs_currency: 'usd-cents', + output_cost_key: 'default-duration-per-video', + costs: { + video_tokens: perMToken(1.0), + 'default-duration-per-video': 24, + }, + durationSeconds: seconds(5, 2, 12), + dimensions: ['1080p', '480p', '720p'], + fps: FPS, + defaultUsageKey: + 'byteplus-video-generation:seedance-1-0-pro-fast-251015:video_tokens', + }, +]; + +const SEEDANCE_2_0_DIMS = { + '480p': { w: 864, h: 496 }, + '720p': { w: 1280, h: 720 }, + '1080p': { w: 1920, h: 1080 }, + '4k': { w: 3840, h: 2160 }, +}; + +const SEEDANCE_1_0_DIMS = { + '480p': { w: 864, h: 480 }, + '720p': { w: 1248, h: 704 }, + '1080p': { w: 1920, h: 1088 }, +}; + +export const BYTEPLUS_VIDEO_SPECS: Record = { + 'dreamina-seedance-2-0-260128': { + duration: { min: 4, max: 15, default: 5 }, + dims: SEEDANCE_2_0_DIMS, + supportsAudio: true, + supportsLastFrame: true, + supportsReferenceImages: true, + supportsSeed: false, + }, + 'dreamina-seedance-2-0-fast-260128': { + duration: { min: 4, max: 15, default: 5 }, + dims: SEEDANCE_2_0_DIMS, + supportsAudio: true, + supportsLastFrame: true, + supportsReferenceImages: true, + supportsSeed: false, + }, + 'dreamina-seedance-2-0-mini-260615': { + duration: { min: 4, max: 15, default: 5 }, + dims: SEEDANCE_2_0_DIMS, + supportsAudio: true, + supportsLastFrame: true, + supportsReferenceImages: true, + supportsSeed: false, + }, + 'seedance-1-5-pro-251215': { + duration: { min: 4, max: 12, default: 5 }, + dims: SEEDANCE_2_0_DIMS, + supportsAudio: true, + supportsLastFrame: true, + supportsReferenceImages: false, + supportsSeed: true, + }, + 'seedance-1-0-pro-250528': { + duration: { min: 2, max: 12, default: 5 }, + dims: SEEDANCE_1_0_DIMS, + supportsAudio: false, + supportsLastFrame: true, + supportsReferenceImages: false, + supportsSeed: true, + }, + 'seedance-1-0-pro-fast-251015': { + duration: { min: 2, max: 12, default: 5 }, + dims: SEEDANCE_1_0_DIMS, + supportsAudio: false, + supportsLastFrame: false, + supportsReferenceImages: false, + supportsSeed: true, + }, +}; diff --git a/src/backend/drivers/ai-video/types.ts b/src/backend/drivers/ai-video/types.ts index d1624dace..0bc6df384 100644 --- a/src/backend/drivers/ai-video/types.ts +++ b/src/backend/drivers/ai-video/types.ts @@ -60,6 +60,7 @@ export interface IGenerateVideoParams { output_format?: string; output_quality?: number; negative_prompt?: string; + generate_audio?: boolean; reference_images?: string[]; frame_images?: object[]; last_frame?: string; diff --git a/src/backend/drivers/meta.ts b/src/backend/drivers/meta.ts index 7a8f7c66b..7c97af236 100644 --- a/src/backend/drivers/meta.ts +++ b/src/backend/drivers/meta.ts @@ -446,8 +446,7 @@ export function resolveDriverMeta( // drivers declare a raw object on the instance, which we validate here // so a malformed `rateLimit` field still fails loud at registration. const protoRateLimit = proto[DRIVER_RATE_LIMIT_KEY] as - | DriverRateLimitConfig - | undefined; + DriverRateLimitConfig | undefined; let rateLimit: DriverRateLimitConfig | undefined; if (protoRateLimit) { rateLimit = protoRateLimit; @@ -459,8 +458,7 @@ export function resolveDriverMeta( } const protoConcurrent = proto[DRIVER_CONCURRENT_KEY] as - | DriverConcurrentConfig - | undefined; + DriverConcurrentConfig | undefined; let concurrent: DriverConcurrentConfig | undefined; if (protoConcurrent) { concurrent = protoConcurrent; @@ -472,8 +470,7 @@ export function resolveDriverMeta( } const protoRequireSubscription = proto[DRIVER_REQUIRE_SUBSCRIPTION_KEY] as - | DriverRequireSubscriptionConfig - | undefined; + DriverRequireSubscriptionConfig | undefined; let requireSubscription: DriverRequireSubscriptionConfig | undefined; if (protoRequireSubscription) { requireSubscription = protoRequireSubscription; diff --git a/src/backend/drivers/types.ts b/src/backend/drivers/types.ts index febd74e85..75944180b 100644 --- a/src/backend/drivers/types.ts +++ b/src/backend/drivers/types.ts @@ -139,7 +139,8 @@ export const PuterDriver = class PuterDriver implements WithCostsReporting { public onServerShutdown() { return; } - public getReportedCosts(): // eslint-disable-next-line @typescript-eslint/no-explicit-any + public getReportedCosts(): + // eslint-disable-next-line @typescript-eslint/no-explicit-any | Record[] // eslint-disable-next-line @typescript-eslint/no-explicit-any | Promise[]> { return []; diff --git a/src/backend/types.ts b/src/backend/types.ts index 172460299..62dfece28 100644 --- a/src/backend/types.ts +++ b/src/backend/types.ts @@ -239,12 +239,7 @@ export interface IPreludeConfig { * an RCS agent provisioned in the Prelude account to actually use RCS. */ preferredChannel?: - | 'sms' - | 'rcs' - | 'whatsapp' - | 'viber' - | 'zalo' - | 'telegram'; + 'sms' | 'rcs' | 'whatsapp' | 'viber' | 'zalo' | 'telegram'; } /** @@ -1056,7 +1051,8 @@ export interface WithLifecycle extends Object { } export interface WithCostsReporting extends WithLifecycle { - getReportedCosts?: () => // eslint-disable-next-line @typescript-eslint/no-explicit-any + getReportedCosts?: () => + // eslint-disable-next-line @typescript-eslint/no-explicit-any | Promise[]> // eslint-disable-next-line @typescript-eslint/no-explicit-any | Record[]; From d22b656df8af8c6af8354b26a68f3879b7129a61 Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Sat, 15 Aug 2026 22:06:51 -0700 Subject: [PATCH 2/2] fix: metering issues with addons (#3588) --- extensions/metering.ts | 6 +- .../services/metering/MeteringService.test.ts | 135 +++++++++++++- .../services/metering/MeteringService.ts | 174 ++++++++++++++---- src/backend/services/metering/types.ts | 10 + 4 files changed, 280 insertions(+), 45 deletions(-) diff --git a/extensions/metering.ts b/extensions/metering.ts index 155d568d2..21ceba905 100644 --- a/extensions/metering.ts +++ b/extensions/metering.ts @@ -70,8 +70,10 @@ const scaleUsageByType = ( ): Record => { const out: Record = {}; for (const [key, value] of Object.entries(usage)) { - if (key === 'total') { - out.total = toCredits(Number(value) || 0, multiplier); + // `allowanceUsed` is the month's allowance-charged spend — monetary, + // scalar, scaled like the total. + if (key === 'total' || key === 'allowanceUsed') { + out[key] = toCredits(Number(value) || 0, multiplier); } else if ( value && typeof value === 'object' && diff --git a/src/backend/services/metering/MeteringService.test.ts b/src/backend/services/metering/MeteringService.test.ts index 92c5386ce..b3d410757 100644 --- a/src/backend/services/metering/MeteringService.test.ts +++ b/src/backend/services/metering/MeteringService.test.ts @@ -453,6 +453,30 @@ describe('MeteringService', () => { expect(addons.consumedPurchaseCredits).toBe(1_000_000); }); }); + + it('charges the allowance first and records the split on the month record', async () => { + const overActor: Actor = { user: makeUser() }; + const sub = await target.getActorSubscription(overActor); + await target.updateAddonCredit(overActor.user.uuid!, 5_000_000); + + // One increment that straddles the boundary: the allowance part + // lands in `allowanceUsed`, only the rest draws down credit. + await target.incrementUsage( + overActor, + 'kv:read', + 1, + sub.monthUsageAllowance + 1_000_000, + ); + + await waitFor(async () => { + const { usage } = + await target.getActorCurrentMonthUsageDetails(overActor); + expect(usage.allowanceUsed).toBe(sub.monthUsageAllowance); + expect(usage.total).toBe(sub.monthUsageAllowance + 1_000_000); + const addons = await target.getActorAddons(overActor); + expect(addons.consumedPurchaseCredits).toBe(1_000_000); + }); + }); }); // ── overuse alarm ──────────────────────────────────────────────── @@ -806,7 +830,9 @@ describe('MeteringService', () => { } const incrSpy = vi.spyOn(server.stores.meteringBuffer, 'incr'); await target.flushBufferedUsages(); - expect(incrSpy).toHaveBeenCalledOnce(); + // One usage write for all ten buffered events, plus the settle + // write that records the allowance/credit split. + expect(incrSpy).toHaveBeenCalledTimes(2); incrSpy.mockRestore(); const { usage } = @@ -894,7 +920,8 @@ describe('MeteringService', () => { ]); } await target.flushBufferedUsages(); - expect(spy).toHaveBeenCalledTimes(concurrency * 3); + // Per bucket: the usage write plus the allowance settle. + expect(spy).toHaveBeenCalledTimes(concurrency * 3 * 2); expect(peak).toBeLessThanOrEqual(concurrency); } finally { spy.mockRestore(); @@ -924,7 +951,8 @@ describe('MeteringService', () => { target.flushBufferedUsages(), target.flushBufferedUsages(), ]); - expect(started).toBe(1); + // One cycle ran (usage write + allowance settle), not three. + expect(started).toBe(2); } finally { spy.mockRestore(); } @@ -1191,6 +1219,32 @@ describe('MeteringService', () => { expect(result.total).toBe(100); }); + it('re-anchors allowanceUsed so the adjusted total is what the allowance is billed', async () => { + const sub = await target.getActorSubscription(actor); + await target.updateAddonCredit(actor.user.uuid!, 5_000_000); + + // Overspend so the month holds allowance + credit-charged spend. + await target.incrementUsage( + actor, + 'kv:read', + 1, + sub.monthUsageAllowance + 5_000_000, + ); + expect(await target.getRemainingUsage(actor)).toBe(0); + + // Support sets the month back down: the new total is billed to + // the allowance in full and the rest of it reopens. + const result = await target.setActorCurrentMonthUsageTotal( + actor, + 1_000, + ); + expect(result.total).toBe(1_000); + expect(result.allowanceUsed).toBe(1_000); + expect(await target.getRemainingUsage(actor)).toBe( + sub.monthUsageAllowance - 1_000, + ); + }); + it('rejects a negative total', async () => { await expect( target.setActorCurrentMonthUsageTotal(actor, -1), @@ -1308,6 +1362,76 @@ describe('MeteringService', () => { expect(allowed.remaining).toBe(4_000_000); }); + it('keeps the allowance and credit pools separate across a mid-month upgrade', async () => { + const freeSub = await target.getActorSubscription(actor); + const freeAllowance = freeSub.monthUsageAllowance; + await target.updateAddonCredit(actor.user.uuid!, 5_000_000); + + // Exhaust the free allowance, then draw 2_000_000 from credit. + await target.incrementUsage(actor, 'kv:read', 1, freeAllowance); + await target.incrementUsage(actor, 'kv:read', 1, 2_000_000); + await waitFor(async () => { + const addons = await target.getActorAddons(actor); + expect(addons.consumedPurchaseCredits).toBe(2_000_000); + }); + + // Upgrade mid-month to ten times the allowance. The allowance + // pool reopens (freeAllowance of 10x used); the credit pool is + // exactly where it was (2 of 5 consumed). + const paid = { + id: 'upgrade-paid', + monthUsageAllowance: freeAllowance * 10, + monthlyStorageAllowance: 1024 * 1024 * 1024, + }; + target.registerPolicy(paid); + target.registerSubscriptionResolver(async () => 'upgrade-paid'); + target.invalidateActorSubscription(actor.user.uuid!); + + const allowed = await target.getAllowedUsage(actor); + expect(allowed.remaining).toBe(freeAllowance * 9 + 3_000_000); + expect(allowed.addons.consumedPurchaseCredits).toBe(2_000_000); + + // Further spend consumes the reopened allowance, not credit. + await target.incrementUsage(actor, 'kv:read', 1, freeAllowance * 9); + const addons = await target.getActorAddons(actor); + expect(addons.consumedPurchaseCredits).toBe(2_000_000); + expect((await target.getAllowedUsage(actor)).remaining).toBe( + 3_000_000, + ); + + // Only once the new allowance is full does credit drain again. + await target.incrementUsage(actor, 'kv:read', 1, 1_000_000); + await waitFor(async () => { + const after = await target.getActorAddons(actor); + expect(after.consumedPurchaseCredits).toBe(3_000_000); + }); + }); + + it('falls back to the pre-split reading for month records without allowanceUsed', async () => { + const sub = await target.getActorSubscription(actor); + await target.updateAddonCredit(actor.user.uuid!, 5_000_000); + await server.stores.kv.incr({ + key: `${POLICY_PREFIX}:actor:${actor.user.uuid}:addons`, + pathAndAmountMap: { consumedPurchaseCredits: 5_000_000 }, + }); + + // A legacy month record: total spans allowance + credit overage, + // no allowanceUsed split recorded. + const month = `${new Date().getUTCFullYear()}-${String(new Date().getUTCMonth() + 1).padStart(2, '0')}`; + await server.stores.meteringBuffer.incr({ + key: `${METRICS_PREFIX}:actor:${actor.user.uuid}:${month}`, + pathAndAmountMap: { + total: sub.monthUsageAllowance + 5_000_000, + }, + }); + + // Pre-split behavior: the whole total counts against the + // allowance (capped at it), so nothing changes at the deploy + // that introduced the field. + const allowed = await target.getAllowedUsage(actor); + expect(allowed.remaining).toBe(0); + }); + it('counts consumed credits from prior months against the credit pool only', async () => { // Simulate a prior-month overage: consumed credits exist but the // current month has no usage (monthly usage keys roll over). @@ -1894,8 +2018,9 @@ describe('MeteringService', () => { const incr = vi.spyOn(server.stores.meteringBuffer, 'incr'); const usage = await target.getActorCurrentMonthUsageDetails(actor); - // Four charges across two listeners, settling as a single write. - expect(incr).toHaveBeenCalledTimes(1); + // Four charges across two listeners fold into a single usage + // write; the second call is the allowance settle. + expect(incr).toHaveBeenCalledTimes(2); expect(incr.mock.calls[0]![0].pathAndAmountMap).toEqual({ total: 600, 'workers:monthly.units': 5, diff --git a/src/backend/services/metering/MeteringService.ts b/src/backend/services/metering/MeteringService.ts index ebace255d..2fe763e8b 100644 --- a/src/backend/services/metering/MeteringService.ts +++ b/src/backend/services/metering/MeteringService.ts @@ -497,9 +497,10 @@ export class MeteringService extends PuterService { actorSubscription.monthUsageAllowance, ); - await this.maybeConsumeAddonCredits( + const settledAllowanceUsed = await this.settleIncrementCharges( userId, - actorUsages.total, + actorUsageKey, + actorUsages, actorSubscription.monthUsageAllowance, actorAddons, totalCost, @@ -519,7 +520,7 @@ export class MeteringService extends PuterService { this.rememberRemainingCredits( userId, - actorUsages.total, + settledAllowanceUsed, actorSubscription.monthUsageAllowance, actorAddons, ); @@ -684,9 +685,10 @@ export class MeteringService extends PuterService { actorSubscription.monthUsageAllowance, ); - await this.maybeConsumeAddonCredits( + const settledAllowanceUsed = await this.settleIncrementCharges( userId, - actorUsages.total, + actorUsageKey, + actorUsages, actorSubscription.monthUsageAllowance, actorAddons, totalBatchCost, @@ -704,7 +706,7 @@ export class MeteringService extends PuterService { this.rememberRemainingCredits( userId, - actorUsages.total, + settledAllowanceUsed, actorSubscription.monthUsageAllowance, actorAddons, ); @@ -928,7 +930,21 @@ export class MeteringService extends PuterService { const currentTotal = (current as UsageByType | null)?.total ?? 0; const delta = normalizedTotal - currentTotal; - if (delta === 0) { + // The adjusted total is taken to be allowance-charged in full — the + // knob's job is "this is what the month has cost the plan", and it + // doubles as the repair for records whose split predates + // `allowanceUsed`. Overshoot past the allowance is harmless: readers + // clamp. The credit pool is deliberately untouched; admin moves it + // through `updateAddonCredit`. + const subscription = await this.getActorSubscription(actor); + const allowanceUsedDelta = + normalizedTotal - + MeteringService.allowanceUsedFrom( + current as UsageByType | null, + subscription.monthUsageAllowance, + ); + + if (delta === 0 && allowanceUsedDelta === 0) { return (current as UsageByType) || ({ total: 0 } as UsageByType); } @@ -942,7 +958,12 @@ export class MeteringService extends PuterService { const updated = ( await this.stores.meteringBuffer.incr({ key: actorUsageKey, - pathAndAmountMap, + // `allowanceUsed` belongs to the actor record alone — the aux + // aggregates below reuse `pathAndAmountMap` without it. + pathAndAmountMap: { + ...pathAndAmountMap, + allowanceUsed: allowanceUsedDelta, + }, }) ).res as unknown as UsageByType; @@ -1105,7 +1126,10 @@ export class MeteringService extends PuterService { return { remaining: MeteringService.remainingFrom( - currentMonthUsage.usage.total || 0, + MeteringService.allowanceUsedFrom( + currentMonthUsage.usage, + userSubscription.monthUsageAllowance, + ), userSubscription.monthUsageAllowance, addons, ), @@ -1115,21 +1139,42 @@ export class MeteringService extends PuterService { } /** - * What's left of an actor's budget, from the three numbers it's made of. + * How much of this month's spend was charged to the subscription allowance. + * Records from before `allowanceUsed` was tracked fall back to the + * pre-split reading — everything counted against the allowance, capped at + * it — which is also what keeps balances unchanged across the deploy that + * introduced the field. + */ + private static allowanceUsedFrom( + usage: UsageByType | null | undefined, + monthUsageAllowance: number, + ): number { + if (!usage) return 0; + return ( + usage.allowanceUsed ?? + Math.min(usage.total || 0, Math.max(0, monthUsageAllowance || 0)) + ); + } + + /** + * What's left of an actor's budget: what remains of the monthly allowance + * plus what remains of the lifetime credit pool. * - * Overage past the allowance is already charged to purchased credits via - * `consumedPurchaseCredits`, so the allowance and the credit pool are - * netted separately — subtracting month usage AND consumed credits from one - * combined pool would charge the overage twice. + * The pools are independent and each spend lands in exactly one of them + * (allowance first — see `settleIncrementCharges`), so this is a plain sum. + * `allowanceUsed` rather than the month total is what the allowance is + * netted against: the total also contains credit-charged overage, which + * must not bill the allowance too — visibly so when a plan change raises + * the allowance mid-month. */ private static remainingFrom( - monthUsageTotal: number, + allowanceUsed: number, monthUsageAllowance: number, addons: UsageAddons | null | undefined, ): number { const remainingAllowance = Math.max( 0, - (monthUsageAllowance || 0) - (monthUsageTotal || 0), + (monthUsageAllowance || 0) - (allowanceUsed || 0), ); const remainingPurchasedCredits = Math.max( 0, @@ -1235,7 +1280,10 @@ export class MeteringService extends PuterService { ]); this.rememberRemainingCredits( uuid, - currentMonthUsage.usage.total || 0, + MeteringService.allowanceUsedFrom( + currentMonthUsage.usage, + subscription.monthUsageAllowance, + ), subscription.monthUsageAllowance, addons, ); @@ -1261,7 +1309,7 @@ export class MeteringService extends PuterService { */ private rememberRemainingCredits( userId: string, - monthUsageTotal: number, + allowanceUsed: number, monthUsageAllowance: number, addons: UsageAddons | null | undefined, ): void { @@ -1272,7 +1320,7 @@ export class MeteringService extends PuterService { this.rememberHasCredits( userId, MeteringService.remainingFrom( - monthUsageTotal, + allowanceUsed, monthUsageAllowance, addons, ) > 0, @@ -1688,33 +1736,83 @@ export class MeteringService extends PuterService { return this.settledMonth === month && this.settledActors.has(claimId); } - private async maybeConsumeAddonCredits( + /** + * Split a settled increment between the two budget pools, allowance first: + * whatever fits under the monthly allowance bumps the month record's + * `allowanceUsed`, and only the part that doesn't fit draws down the + * lifetime credit pool. Each spend lands in exactly one pool, which is what + * lets `remainingFrom` sum them independently. + * + * A month record without `allowanceUsed` predates the split; its first + * settled increment folds the fallback baseline into the write, so the + * record answers directly from then on and no balance moves on the deploy + * that introduced the field. + * + * Returns the month's allowance-charged spend as of after this settle — + * `usageRecord` itself predates the write, so callers deciding on the + * balance must use this rather than re-read the record they hold. + */ + private async settleIncrementCharges( userId: string, - totalUsage: number, + actorUsageKey: string, + usageRecord: UsageByType, monthUsageAllowance: number, addons: UsageAddons, incrementCost: number, - ): Promise { - if (totalUsage <= monthUsageAllowance) return; - if (!addons.purchasedCredits) return; - if (addons.purchasedCredits <= (addons.consumedPurchaseCredits || 0)) - return; + ): Promise { + if (incrementCost <= 0) { + return MeteringService.allowanceUsedFrom( + usageRecord, + monthUsageAllowance, + ); + } - const withinBoundsUsage = Math.max( + const totalBefore = Math.max( 0, - monthUsageAllowance - totalUsage + incrementCost, + (usageRecord.total || 0) - incrementCost, ); - const overageUsage = incrementCost - withinBoundsUsage; - if (overageUsage <= 0) return; + // Same fallback as `allowanceUsedFrom`, measured before this + // increment's own cost. + const usedBefore = + usageRecord.allowanceUsed ?? + Math.min(totalBefore, Math.max(0, monthUsageAllowance || 0)); + const baseline = + usageRecord.allowanceUsed === undefined ? usedBefore : 0; - const toConsume = Math.min( - overageUsage, - addons.purchasedCredits - (addons.consumedPurchaseCredits || 0), - ); - await this.stores.kv.incr({ - key: `${POLICY_PREFIX}:actor:${userId}:addons`, - pathAndAmountMap: { consumedPurchaseCredits: toConsume }, - }); + const headroom = Math.max(0, monthUsageAllowance - usedBefore); + const allowanceCharge = Math.min(incrementCost, headroom); + const overage = incrementCost - allowanceCharge; + + const writes: Promise[] = []; + if (baseline + allowanceCharge > 0) { + writes.push( + this.stores.meteringBuffer.incr({ + key: actorUsageKey, + pathAndAmountMap: { + allowanceUsed: baseline + allowanceCharge, + }, + }), + ); + } + + const remainingCredits = + (addons.purchasedCredits || 0) - + (addons.consumedPurchaseCredits || 0); + if (overage > 0 && remainingCredits > 0) { + writes.push( + this.stores.kv.incr({ + key: `${POLICY_PREFIX}:actor:${userId}:addons`, + pathAndAmountMap: { + consumedPurchaseCredits: Math.min( + overage, + remainingCredits, + ), + }, + }), + ); + } + await Promise.all(writes); + return usedBefore + allowanceCharge; } private maybeAlertOveruse(ctx: { diff --git a/src/backend/services/metering/types.ts b/src/backend/services/metering/types.ts index 3d5170a72..6dc136e48 100644 --- a/src/backend/services/metering/types.ts +++ b/src/backend/services/metering/types.ts @@ -41,6 +41,16 @@ export interface UsageInput { export type UsageByType = { total: number; + /** + * The part of `total` that was charged to the monthly subscription + * allowance. The allowance is consumed first; spend past it draws down the + * lifetime credit pool (`consumedPurchaseCredits`) instead, so the two + * pools never bill the same spend. Living on the month record, it resets + * with the month the way the allowance itself does. Absent on records from + * before the split was tracked — readers fall back to counting `total` + * against the allowance, capped at the allowance. + */ + allowanceUsed?: number; /** * Claim counter for the month's recurring charges — see * `MONTHLY_CHARGE_CLAIM`. Absent until the first read or write of the