diff --git a/.github/workflows/ai-provider-integration-tests.yaml b/.github/workflows/ai-provider-integration-tests.yaml new file mode 100644 index 000000000..3b66bf78b --- /dev/null +++ b/.github/workflows/ai-provider-integration-tests.yaml @@ -0,0 +1,62 @@ +name: AI Provider Integration Tests + +# Hits real provider APIs with cheap models, so we only run when the +# files exercised by these tests change. Each provider's test reads its +# credential from PUTER_TEST_AI__API_KEY and skips itself +# silently if the secret is missing — fork PRs see green skips, the +# main repo runs the tests it has secrets for. + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - 'src/backend/drivers/ai-chat/**' + - 'src/backend/drivers/ai-image/**' + - 'src/backend/drivers/ai-tts/**' + - 'src/backend/drivers/integrationTestUtil.ts' + - '.github/workflows/ai-provider-integration-tests.yaml' + - 'package.json' + - 'package-lock.json' + +permissions: + contents: read + +jobs: + test: + runs-on: ubuntu-latest + # Skip on PRs from forks: secrets are not exposed there, so every + # test would skip anyway. Keeps the actions tab clean. + if: github.event.pull_request.head.repo.full_name == github.repository + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Run AI provider integration tests + # Vitest treats positional args as filename substring filters. + # The pattern matches every `*.integration.test.ts` file and + # nothing else — the regular backend test suite stays out. + run: npm run test:backend -- integration.test + env: + CI: 'true' + PUTER_TEST_AI_CLAUDE_API_KEY: ${{ secrets.PUTER_TEST_AI_CLAUDE_API_KEY }} + PUTER_TEST_AI_OPENAI_API_KEY: ${{ secrets.PUTER_TEST_AI_OPENAI_API_KEY }} + PUTER_TEST_AI_GEMINI_API_KEY: ${{ secrets.PUTER_TEST_AI_GEMINI_API_KEY }} + PUTER_TEST_AI_GROQ_API_KEY: ${{ secrets.PUTER_TEST_AI_GROQ_API_KEY }} + PUTER_TEST_AI_MISTRAL_API_KEY: ${{ secrets.PUTER_TEST_AI_MISTRAL_API_KEY }} + PUTER_TEST_AI_DEEPSEEK_API_KEY: ${{ secrets.PUTER_TEST_AI_DEEPSEEK_API_KEY }} + PUTER_TEST_AI_XAI_API_KEY: ${{ secrets.PUTER_TEST_AI_XAI_API_KEY }} + PUTER_TEST_AI_OPENROUTER_API_KEY: ${{ secrets.PUTER_TEST_AI_OPENROUTER_API_KEY }} + PUTER_TEST_AI_TOGETHER_API_KEY: ${{ secrets.PUTER_TEST_AI_TOGETHER_API_KEY }} + PUTER_TEST_AI_MOONSHOT_API_KEY: ${{ secrets.PUTER_TEST_AI_MOONSHOT_API_KEY }} + PUTER_TEST_AI_ZAI_API_KEY: ${{ secrets.PUTER_TEST_AI_ZAI_API_KEY }} + PUTER_TEST_AI_ELEVENLABS_API_KEY: ${{ secrets.PUTER_TEST_AI_ELEVENLABS_API_KEY }} diff --git a/.github/workflows/backend-tests.yaml b/.github/workflows/backend-tests.yaml index a3299295b..9d24cc32c 100644 --- a/.github/workflows/backend-tests.yaml +++ b/.github/workflows/backend-tests.yaml @@ -11,10 +11,21 @@ permissions: jobs: test: runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - ref: ${{ github.base_ref }} + artifact: base + - ref: ${{ github.head_ref }} + artifact: pr steps: - - name: Checkout repository + - name: Checkout ${{ matrix.ref }} uses: actions/checkout@v4 + with: + ref: ${{ matrix.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} - name: Set up Node.js uses: actions/setup-node@v4 @@ -30,17 +41,37 @@ jobs: env: CI: 'true' - - name: Report coverage on PR - if: always() - uses: davelosert/vitest-coverage-report-action@v2 - with: - json-summary-path: src/backend/coverage/coverage-summary.json - json-final-path: src/backend/coverage/coverage-final.json - - name: Upload coverage artifact if: always() uses: actions/upload-artifact@v4 with: - name: backend-coverage + name: backend-coverage-${{ matrix.artifact }} path: src/backend/coverage/ retention-days: 14 + + report-coverage: + needs: test + if: always() + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Download PR coverage + uses: actions/download-artifact@v4 + with: + name: backend-coverage-pr + path: src/backend/coverage + + - name: Download base coverage + uses: actions/download-artifact@v4 + with: + name: backend-coverage-base + path: coverage-base + + - name: Report coverage on PR + uses: davelosert/vitest-coverage-report-action@v2 + with: + json-summary-path: src/backend/coverage/coverage-summary.json + json-final-path: src/backend/coverage/coverage-final.json + json-summary-compare-path: coverage-base/coverage-summary.json diff --git a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.integration.test.ts new file mode 100644 index 000000000..fa641d6d4 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.integration.test.ts @@ -0,0 +1,69 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Claude provider. + * + * Hits the real Anthropic API with a tiny prompt against the cheapest + * model (Haiku) so the smoke check runs fast and doesn't accumulate + * cost. Skipped automatically when `PUTER_TEST_AI_CLAUDE_API_KEY` is + * not set; in CI, only triggered when the Claude provider source + * actually changes (see `.github/workflows/ai-provider-integration-tests.yaml`). + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { ClaudeProvider } from './ClaudeProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_CLAUDE_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))('ClaudeProvider (integration)', () => { + const buildProvider = () => + new ClaudeProvider( + makeMeteringStub(), + // Stores / FS only consulted for `puter_path` uploads — text-only + // prompts never reach those code paths. + { fsEntry: undefined as never, s3Object: undefined as never }, + undefined as never, + { apiKey: optionalEnv(ENV_VAR)! }, + ); + + it('returns a non-empty completion from claude-haiku-4-5', async () => { + const provider = buildProvider(); + const result = await withTestActor(() => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [{ role: 'user', content: 'Say hi in one word.' }], + max_tokens: 16, + }), + ); + + expect(result).toHaveProperty('message'); + const content = (result as { message: { content: unknown } }).message + .content as Array<{ type: string; text?: string }>; + expect(Array.isArray(content)).toBe(true); + const text = content.find((c) => c.type === 'text')?.text ?? ''; + expect(text.length).toBeGreaterThan(0); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.integration.test.ts new file mode 100644 index 000000000..3c4c106f3 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.integration.test.ts @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the DeepSeek provider. + * + * Uses `deepseek-chat` (the cheap V3 chat model, provider default). + * Skipped when `PUTER_TEST_AI_DEEPSEEK_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { DeepSeekProvider } from './DeepSeekProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_DEEPSEEK_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'DeepSeekProvider (integration)', + () => { + it('returns a non-empty completion from deepseek-chat', async () => { + const provider = new DeepSeekProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'deepseek-chat', + messages: [ + { role: 'user', content: 'Say hi in one word.' }, + ], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); + }, +); diff --git a/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.integration.test.ts new file mode 100644 index 000000000..5611c2442 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.integration.test.ts @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Gemini chat provider. + * + * Hits the real Google Gemini API with `gemini-2.0-flash-lite` (one + * of the cheapest variants). Skipped when `PUTER_TEST_AI_GEMINI_API_KEY` + * is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { GeminiChatProvider } from './GeminiChatProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_GEMINI_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'GeminiChatProvider (integration)', + () => { + it('returns a non-empty completion from gemini-2.0-flash-lite', async () => { + const provider = new GeminiChatProvider(makeMeteringStub(), { + apiKey: optionalEnv(ENV_VAR)!, + }); + + const result = await withTestActor(() => + provider.complete({ + model: 'gemini-2.0-flash-lite', + messages: [ + { role: 'user', content: 'Say hi in one word.' }, + ], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); + }, +); diff --git a/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.integration.test.ts new file mode 100644 index 000000000..4689cf644 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.integration.test.ts @@ -0,0 +1,58 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Groq provider. + * + * Uses `llama-3.1-8b-instant` (the provider's default and cheapest + * generally-available model). Skipped when `PUTER_TEST_AI_GROQ_API_KEY` + * is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { GroqAIProvider } from './GroqAIProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_GROQ_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))('GroqAIProvider (integration)', () => { + it('returns a non-empty completion from llama-3.1-8b-instant', async () => { + const provider = new GroqAIProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'llama-3.1-8b-instant', + messages: [{ role: 'user', content: 'Say hi in one word.' }], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.integration.test.ts new file mode 100644 index 000000000..b18bd2481 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.integration.test.ts @@ -0,0 +1,76 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Mistral provider. + * + * Uses `mistral-small-2506` (provider default, cheapest tier). Skipped + * when `PUTER_TEST_AI_MISTRAL_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { MistralAIProvider } from './MistralAiProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_MISTRAL_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'MistralAIProvider (integration)', + () => { + it('returns a non-empty completion from mistral-small', async () => { + const provider = new MistralAIProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'mistral-small-2506', + messages: [ + { role: 'user', content: 'Say hi in one word.' }, + ], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: unknown } }).message + ?.content; + // Mistral SDK may return string or array of content parts. + const asString = + typeof text === 'string' + ? text + : Array.isArray(text) + ? text + .map((p) => + typeof p === 'string' + ? p + : (p as { text?: string })?.text, + ) + .filter(Boolean) + .join('') + : ''; + expect(asString.length).toBeGreaterThan(0); + }); + }, +); diff --git a/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.integration.test.ts new file mode 100644 index 000000000..146df4cf3 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.integration.test.ts @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Moonshot provider. + * + * Uses `moonshot-v1-8k` (the cheapest 8K-context variant). Skipped + * when `PUTER_TEST_AI_MOONSHOT_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { MoonshotProvider } from './MoonshotProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_MOONSHOT_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'MoonshotProvider (integration)', + () => { + it('returns a non-empty completion from moonshot-v1-8k', async () => { + const provider = new MoonshotProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'moonshot-v1-8k', + messages: [ + { role: 'user', content: 'Say hi in one word.' }, + ], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); + }, +); diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.integration.test.ts new file mode 100644 index 000000000..a754f3aff --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.integration.test.ts @@ -0,0 +1,70 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the OpenAI chat-completions provider. + * + * Hits the real OpenAI API with `gpt-4o-mini` — non-reasoning so + * `max_tokens=16` actually returns visible text (reasoning models like + * `gpt-5-nano` would burn the budget on thinking tokens before + * emitting any response). Skipped when `PUTER_TEST_AI_OPENAI_API_KEY` + * is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { OpenAiChatProvider } from './OpenAiChatCompletionsProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_OPENAI_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'OpenAiChatCompletionsProvider (integration)', + () => { + const buildProvider = () => + new OpenAiChatProvider( + makeMeteringStub(), + { fsEntry: undefined as never, s3Object: undefined as never }, + undefined as never, + { apiKey: optionalEnv(ENV_VAR)! }, + ); + + it('returns a non-empty completion from gpt-4o-mini', async () => { + const provider = buildProvider(); + const result = await withTestActor(() => + provider.complete({ + model: 'gpt-4o-mini', + messages: [ + { role: 'user', content: 'Say hi in one word.' }, + ], + max_tokens: 16, + }), + ); + + // OpenAIUtil returns the OpenAI choice object directly. + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); + }, +); diff --git a/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.integration.test.ts new file mode 100644 index 000000000..ef25776c0 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.integration.test.ts @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the OpenRouter aggregator. + * + * Routes through OpenRouter to a tiny upstream model + * (`google/gemini-2.0-flash-lite-001`). Skipped when + * `PUTER_TEST_AI_OPENROUTER_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { OpenRouterProvider } from './OpenRouterProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_OPENROUTER_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'OpenRouterProvider (integration)', + () => { + it('returns a non-empty completion via OpenRouter', async () => { + const provider = new OpenRouterProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'openrouter:google/gemini-2.0-flash-lite-001', + messages: [ + { role: 'user', content: 'Say hi in one word.' }, + ], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); + }, +); diff --git a/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.integration.test.ts new file mode 100644 index 000000000..e5d16da4c --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.integration.test.ts @@ -0,0 +1,62 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Together AI provider. + * + * Uses the `meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo` cheap variant + * via Together. Skipped when `PUTER_TEST_AI_TOGETHER_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { TogetherAIProvider } from './TogetherAIProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_TOGETHER_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'TogetherAIProvider (integration)', + () => { + it('returns a non-empty completion from Llama 3.1 8B', async () => { + const provider = new TogetherAIProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'togetherai:meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo', + messages: [ + { role: 'user', content: 'Say hi in one word.' }, + ], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); + }, +); diff --git a/src/backend/drivers/ai-chat/providers/xai/XAIProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/xai/XAIProvider.integration.test.ts new file mode 100644 index 000000000..5bd2c7b2a --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/xai/XAIProvider.integration.test.ts @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the xAI (Grok) provider. + * + * Uses `grok-3-mini` — the cheapest small variant. Skipped when + * `PUTER_TEST_AI_XAI_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { XAIProvider } from './XAIProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_XAI_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))('XAIProvider (integration)', () => { + it('returns a non-empty completion from grok-3-mini', async () => { + const provider = new XAIProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'grok-3-mini', + messages: [{ role: 'user', content: 'Say hi in one word.' }], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.integration.test.ts new file mode 100644 index 000000000..7ac7e5a5e --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.integration.test.ts @@ -0,0 +1,57 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Z.AI (GLM) provider. + * + * Uses `glm-4.7-flashx` — the cheapest text variant on the price + * sheet. Skipped when `PUTER_TEST_AI_ZAI_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { ZAIProvider } from './ZAIProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_ZAI_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))('ZAIProvider (integration)', () => { + it('returns a non-empty completion from glm-4.7-flashx', async () => { + const provider = new ZAIProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'glm-4.7-flashx', + messages: [{ role: 'user', content: 'Say hi in one word.' }], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); +}); diff --git a/src/backend/drivers/ai-image/providers/gemini/GeminiImageProvider.integration.test.ts b/src/backend/drivers/ai-image/providers/gemini/GeminiImageProvider.integration.test.ts new file mode 100644 index 000000000..6a683cb68 --- /dev/null +++ b/src/backend/drivers/ai-image/providers/gemini/GeminiImageProvider.integration.test.ts @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Gemini image generation provider. + * + * Uses `imagen-4.0-fast-generate-001` ($0.02/image — cheapest + * Gemini imagen variant). Skipped when `PUTER_TEST_AI_GEMINI_API_KEY` + * is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { GeminiImageProvider } from './GeminiImageProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_GEMINI_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'GeminiImageProvider (integration)', + () => { + it('returns image data from imagen-4.0-fast', async () => { + const provider = new GeminiImageProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.generate({ + model: 'imagen-4.0-fast-generate-001', + prompt: 'a tiny red dot on a white background', + ratio: { w: 1, h: 1 }, + }), + ); + + expect(typeof result).toBe('string'); + expect((result as string).length).toBeGreaterThan(0); + }); + }, +); diff --git a/src/backend/drivers/ai-image/providers/openai/OpenAiImageProvider.integration.test.ts b/src/backend/drivers/ai-image/providers/openai/OpenAiImageProvider.integration.test.ts new file mode 100644 index 000000000..b58e3055a --- /dev/null +++ b/src/backend/drivers/ai-image/providers/openai/OpenAiImageProvider.integration.test.ts @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the OpenAI image generation provider. + * + * Uses `dall-e-2` at 256x256 — the cheapest OpenAI image + * configuration ($0.016/image). Skipped when + * `PUTER_TEST_AI_OPENAI_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { OpenAiImageProvider } from './OpenAiImageProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_OPENAI_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'OpenAiImageProvider (integration)', + () => { + it('returns an image url/data from dall-e-2 at 256x256', async () => { + const provider = new OpenAiImageProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.generate({ + model: 'dall-e-2', + prompt: 'a tiny red dot on a white background', + ratio: { w: 256, h: 256 }, + }), + ); + + expect(typeof result).toBe('string'); + expect((result as string).length).toBeGreaterThan(0); + }); + }, +); diff --git a/src/backend/drivers/ai-tts/providers/elevenlabs/ElevenLabsTTSProvider.integration.test.ts b/src/backend/drivers/ai-tts/providers/elevenlabs/ElevenLabsTTSProvider.integration.test.ts new file mode 100644 index 000000000..ed4ecfdd9 --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/elevenlabs/ElevenLabsTTSProvider.integration.test.ts @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the ElevenLabs TTS provider. + * + * Uses `eleven_flash_v2_5` (the cheapest tier) with a tiny input. + * Skipped when `PUTER_TEST_AI_ELEVENLABS_API_KEY` is unset. + */ + +import { Readable } from 'node:stream'; +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { ElevenLabsTTSProvider } from './ElevenLabsTTSProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_ELEVENLABS_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'ElevenLabsTTSProvider (integration)', + () => { + it('returns an audio stream from eleven_flash_v2_5', async () => { + const provider = new ElevenLabsTTSProvider(makeMeteringStub(), { + apiKey: optionalEnv(ENV_VAR)!, + }); + + const result = (await withTestActor(() => + provider.synthesize({ + text: 'hi', + model: 'eleven_flash_v2_5', + }), + )) as { stream: Readable; content_type: string }; + + expect(result.stream).toBeInstanceOf(Readable); + const chunks: Buffer[] = []; + for await (const chunk of result.stream) { + chunks.push(chunk as Buffer); + } + const total = chunks.reduce((n, c) => n + c.length, 0); + expect(total).toBeGreaterThan(0); + }); + }, +); diff --git a/src/backend/drivers/ai-tts/providers/openai/OpenAITTSProvider.integration.test.ts b/src/backend/drivers/ai-tts/providers/openai/OpenAITTSProvider.integration.test.ts new file mode 100644 index 000000000..e7e540413 --- /dev/null +++ b/src/backend/drivers/ai-tts/providers/openai/OpenAITTSProvider.integration.test.ts @@ -0,0 +1,72 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the OpenAI TTS provider. + * + * Uses `tts-1` (the cheapest OpenAI TTS model) with a 2-character + * input to keep cost negligible. Skipped when + * `PUTER_TEST_AI_OPENAI_API_KEY` is unset. + */ + +import { Readable } from 'node:stream'; +import { describe, expect, it } from 'vitest'; +import { + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { OpenAITTSProvider } from './OpenAITTSProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_OPENAI_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'OpenAITTSProvider (integration)', + () => { + it('returns an audio stream from tts-1', async () => { + const provider = new OpenAITTSProvider(makeMeteringStub(), { + apiKey: optionalEnv(ENV_VAR)!, + }); + + const result = (await withTestActor(() => + provider.synthesize({ + text: 'hi', + model: 'tts-1', + voice: 'alloy', + response_format: 'mp3', + }), + )) as { stream: Readable; content_type: string }; + + expect(result).toMatchObject({ + content_type: expect.stringContaining('audio'), + }); + expect(result.stream).toBeInstanceOf(Readable); + + // Drain the stream so we know real bytes came back, not an + // empty placeholder. + const chunks: Buffer[] = []; + for await (const chunk of result.stream) { + chunks.push(chunk as Buffer); + } + const total = chunks.reduce((n, c) => n + c.length, 0); + expect(total).toBeGreaterThan(0); + }); + }, +); diff --git a/src/backend/drivers/integrationTestUtil.ts b/src/backend/drivers/integrationTestUtil.ts new file mode 100644 index 000000000..275576d51 --- /dev/null +++ b/src/backend/drivers/integrationTestUtil.ts @@ -0,0 +1,78 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Shared utilities for AI provider integration tests. + * + * Each test reads its credentials from `PUTER_TEST_AI_*` env vars + * (loaded by the vitest config's `PUTER_` prefix) and skips itself + * when the var is missing — tests run only on developer machines and + * in CI environments that supply the right secrets. + * + * Filename intentionally omits `.test.` so vitest does not treat this + * helper as a test file. + */ + +import type { Actor } from '../core/actor.js'; +import { SYSTEM_ACTOR } from '../core/actor.js'; +import { runWithContext } from '../core/context.js'; +import type { MeteringService } from '../services/metering/MeteringService.js'; + +/** + * Returns the env var value, or `undefined` if missing/empty. + * Used as the gate for `describe.skipIf` blocks. + */ +export const optionalEnv = (name: string): string | undefined => { + const v = process.env[name]; + return v && v.length > 0 ? v : undefined; +}; + +/** + * Returns true when the env var is unset, signaling the test block + * should be skipped. Pair with `describe.skipIf(skipUnlessEnv(...))`. + */ +export const skipUnlessEnv = (name: string): boolean => !optionalEnv(name); + +/** + * Returns a no-op MeteringService stub. Real metering would write to + * DynamoDB / Redis, which integration tests for AI providers don't + * care about — we just need the provider's metering calls to not + * throw and to short-circuit credit checks. + */ +export const makeMeteringStub = (): MeteringService => + ({ + utilRecordUsageObject: () => Promise.resolve([] as never), + incrementUsage: () => Promise.resolve({} as never), + batchIncrementUsages: () => Promise.resolve([] as never), + hasEnoughCredits: () => Promise.resolve(true), + getReportedCosts: () => [], + }) as unknown as MeteringService; + +/** + * Run `fn` inside a request-scoped context with `SYSTEM_ACTOR` set, + * which is what providers expect (`Context.get('actor')`). The system + * actor bypasses metering / quota gates by design. + */ +export const withTestActor = ( + fn: () => T | Promise, + actor: Actor = SYSTEM_ACTOR, +): Promise => + Promise.resolve( + runWithContext({ actor, requestId: 'integration-test' }, fn), + );