From b22ba4cbd82d56bbff8c79b9b4730a48b2d47203 Mon Sep 17 00:00:00 2001 From: ProgrammerIn-wonderland <3838shah@gmail.com> Date: Tue, 12 May 2026 21:57:47 -0400 Subject: [PATCH] add tests --- .../AlibabaProvider.integration.test.ts | 63 ++ .../providers/alibaba/AlibabaProvider.test.ts | 720 ++++++++++++++++++ 2 files changed, 783 insertions(+) create mode 100644 src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.integration.test.ts create mode 100644 src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts diff --git a/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.integration.test.ts b/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.integration.test.ts new file mode 100644 index 000000000..585fd1563 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.integration.test.ts @@ -0,0 +1,63 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Integration test for the Alibaba provider. + * + * Uses `qwen-turbo` (cheapest generally-available model). Skipped + * when `PUTER_TEST_AI_ALIBABA_API_KEY` is unset. + */ + +import { describe, expect, it } from 'vitest'; +import { + INTEGRATION_TEST_TIMEOUT_MS, + makeMeteringStub, + optionalEnv, + skipUnlessEnv, + withTestActor, +} from '../../../integrationTestUtil.js'; +import { AlibabaProvider } from './AlibabaProvider.js'; + +const ENV_VAR = 'PUTER_TEST_AI_ALIBABA_API_KEY'; + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'AlibabaProvider (integration)', + () => { + it('returns a non-empty completion from qwen-turbo', { timeout: INTEGRATION_TEST_TIMEOUT_MS }, async () => { + const provider = new AlibabaProvider( + { apiKey: optionalEnv(ENV_VAR)! }, + makeMeteringStub(), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'qwen-turbo', + messages: [ + { role: 'user', content: 'Say hi in one word.' }, + ], + max_tokens: 16, + }), + ); + + const text = (result as { message?: { content?: string } }).message + ?.content; + expect(typeof text === 'string' && text.length > 0).toBe(true); + }); + }, +); diff --git a/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts b/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts new file mode 100644 index 000000000..5d66e0a2e --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts @@ -0,0 +1,720 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for AlibabaProvider. + * + * Boots a real PuterServer (in-memory sqlite + dynamo + s3 + mock + * redis) and constructs AlibabaProvider directly against the live + * wired `MeteringService` so the recording side is exercised end-to- + * end. Alibaba is OpenAI-compatible so the OpenAI SDK is mocked at + * the module boundary; that's the real network egress point. The + * companion integration test (AlibabaProvider.integration.test.ts) + * exercises the real DashScope endpoint. + */ + +import { Writable } from 'node:stream'; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import { SYSTEM_ACTOR } from '../../../../core/actor.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { AIChatStream } from '../../utils/Streaming.js'; +import { ALIBABA_MODELS } from './models.js'; +import { AlibabaProvider } from './AlibabaProvider.js'; + +// ── OpenAI SDK mock ───────────────────────────────────────────────── + +const { createMock, openAICtor } = vi.hoisted(() => { + const createMock = vi.fn(); + const openAICtor = vi.fn(); + return { createMock, openAICtor }; +}); + +vi.mock('openai', () => { + const OpenAICtor = vi.fn().mockImplementation(function ( + this: Record, + opts: unknown, + ) { + openAICtor(opts); + this.chat = { completions: { create: createMock } }; + }); + return { OpenAI: OpenAICtor, default: { OpenAI: OpenAICtor } }; +}); + +// ── Test harness ──────────────────────────────────────────────────── + +let server: PuterServer; +let recordSpy: MockInstance; + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = (config?: { apiKey?: string; apiBaseUrl?: string }) => { + const provider = new AlibabaProvider( + { apiKey: 'test-key', ...config }, + server.services.metering, + ); + return { provider }; +}; + +const asAsyncIterable = (items: T[]): AsyncIterable => ({ + async *[Symbol.asyncIterator]() { + for (const item of items) { + yield item; + } + }, +}); + +const makeCapturingChatStream = () => { + const chunks: string[] = []; + const sink = new Writable({ + write(chunk, _enc, cb) { + chunks.push(chunk.toString('utf8')); + cb(); + }, + }); + const chatStream = new AIChatStream({ stream: sink }); + return { + chatStream, + events: () => + chunks + .join('') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)), + }; +}; + +beforeEach(() => { + createMock.mockReset(); + openAICtor.mockReset(); + recordSpy = vi.spyOn(server.services.metering, 'utilRecordUsageObject'); +}); + +afterEach(() => { + vi.restoreAllMocks(); +}); + +// ── Construction ──────────────────────────────────────────────────── + +describe('AlibabaProvider construction', () => { + it('points the OpenAI SDK at the DashScope base URL with the configured key', () => { + makeProvider(); + expect(openAICtor).toHaveBeenCalledTimes(1); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: + 'https://dashscope-intl.aliyuncs.com/compatible-mode/v1', + }); + }); + + it('uses a custom base URL when configured', () => { + makeProvider({ apiBaseUrl: 'https://custom.endpoint/v1' }); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://custom.endpoint/v1', + }); + }); +}); + +// ── Model catalog ─────────────────────────────────────────────────── + +describe('AlibabaProvider model catalog', () => { + it('returns qwen-plus-latest as the default', () => { + const { provider } = makeProvider(); + expect(provider.getDefaultModel()).toBe('qwen-plus-latest'); + }); + + it('exposes the static ALIBABA_MODELS list verbatim from models()', () => { + const { provider } = makeProvider(); + expect(provider.models()).toBe(ALIBABA_MODELS); + }); + + it('list() flattens canonical ids and aliases', async () => { + const { provider } = makeProvider(); + const ids = await provider.list(); + for (const m of ALIBABA_MODELS) { + expect(ids).toContain(m.id); + for (const a of m.aliases ?? []) { + expect(ids).toContain(a); + } + } + expect(ids).toContain('qwen-plus'); + expect(ids).toContain('qwen/qwen-plus'); + expect(ids).toContain('qwen-max'); + expect(ids).toContain('qwen/qwen-max'); + }); +}); + +// ── Request shape ─────────────────────────────────────────────────── + +describe('AlibabaProvider.complete request shape', () => { + const baseCompletion = { + choices: [ + { + message: { content: 'hi', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }; + + it('forwards model + messages and defaults max_tokens to 1000 when caller omits it', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hello' }], + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect(args.model).toBe('qwen-plus'); + expect(args.messages).toEqual([{ role: 'user', content: 'hello' }]); + expect(args.max_tokens).toBe(1000); + }); + + it('respects an explicit max_tokens override', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + max_tokens: 256, + }), + ); + + expect(createMock.mock.calls[0]![0].max_tokens).toBe(256); + }); + + it('forwards temperature when supplied', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + temperature: 0.7, + }), + ); + + expect(createMock.mock.calls[0]![0].temperature).toBe(0.7); + }); + + it('omits the `tools` key entirely when no tools are supplied', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect('tools' in args).toBe(false); + }); + + it('passes tool definitions through unchanged when supplied', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + const tools = [ + { + type: 'function', + function: { + name: 'lookup', + parameters: { type: 'object', properties: {} }, + }, + }, + ]; + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + tools, + }), + ); + + expect(createMock.mock.calls[0]![0].tools).toBe(tools); + }); + + it('only sets stream_options.include_usage when streaming', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + stream: false, + }), + ); + const [nonStreamArgs] = createMock.mock.calls[0]!; + expect(nonStreamArgs.stream).toBe(false); + expect('stream_options' in nonStreamArgs).toBe(false); + + createMock.mockReturnValueOnce(asAsyncIterable([])); + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + stream: true, + }), + ); + const [streamArgs] = createMock.mock.calls[1]!; + expect(streamArgs.stream).toBe(true); + expect(streamArgs.stream_options).toEqual({ include_usage: true }); + }); + + it('hoists Puter-style tool_use blocks into OpenAI tool_calls before sending', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [ + { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'call_1', + name: 'lookup', + input: { q: 'puter' }, + }, + ], + }, + ], + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect(args.messages[0].content).toBeNull(); + expect(args.messages[0].tool_calls).toEqual([ + { + id: 'call_1', + type: 'function', + function: { + name: 'lookup', + arguments: JSON.stringify({ q: 'puter' }), + }, + }, + ]); + }); +}); + +// ── Model resolution ──────────────────────────────────────────────── + +describe('AlibabaProvider model resolution', () => { + const baseCompletion = { + choices: [ + { + message: { content: 'ok', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }; + + it('resolves an exact canonical id', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'qwen-max', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + expect(createMock.mock.calls[0]![0].model).toBe('qwen-max'); + expect(recordSpy).toHaveBeenCalledWith( + expect.any(Object), + expect.anything(), + 'alibaba:qwen-max', + expect.any(Object), + ); + }); + + it('resolves an alias to its canonical id (alias rewriting)', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'qwen/qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + expect(createMock.mock.calls[0]![0].model).toBe('qwen-plus'); + expect(recordSpy).toHaveBeenCalledWith( + expect.any(Object), + expect.anything(), + 'alibaba:qwen-plus', + expect.any(Object), + ); + }); +}); + +// ── Non-stream completion ─────────────────────────────────────────── + +describe('AlibabaProvider.complete non-stream output', () => { + it('returns the first choice and runs the metered usage calculator', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'hi there', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 100, completion_tokens: 50 }, + }); + + const result = await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + expect(result).toMatchObject({ + message: { content: 'hi there', role: 'assistant' }, + finish_reason: 'stop', + }); + expect((result as { usage: unknown }).usage).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 0, + }); + + const qwenPlus = ALIBABA_MODELS.find((m) => m.id === 'qwen-plus')!; + expect(recordSpy).toHaveBeenCalledTimes(1); + const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!; + expect(usage).toEqual({ + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 0, + }); + expect(actor).toBe(SYSTEM_ACTOR); + expect(prefix).toBe('alibaba:qwen-plus'); + expect(overrides).toEqual({ + prompt_tokens: 100 * Number(qwenPlus.costs.prompt_tokens), + completion_tokens: 50 * Number(qwenPlus.costs.completion_tokens), + cached_tokens: 0, + }); + }); + + it('preserves OpenAI-shaped tool_calls on the assistant response', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { + role: 'assistant', + content: null, + tool_calls: [ + { + id: 'call_1', + type: 'function', + function: { + name: 'lookup', + arguments: '{"q":"puter"}', + }, + }, + ], + }, + finish_reason: 'tool_calls', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }); + + const result = (await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'do a tool call' }], + tools: [ + { + type: 'function', + function: { name: 'lookup', parameters: {} }, + }, + ], + }), + )) as { message: { tool_calls?: unknown[] }; finish_reason: string }; + + expect(result.finish_reason).toBe('tool_calls'); + expect(result.message.tool_calls).toEqual([ + { + id: 'call_1', + type: 'function', + function: { name: 'lookup', arguments: '{"q":"puter"}' }, + }, + ]); + }); + + it('zeroes cached_tokens when prompt_tokens_details is missing', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'ok', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 7, completion_tokens: 3 }, + }); + + await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + const [usage, , , overrides] = recordSpy.mock.calls[0]!; + expect(usage.cached_tokens).toBe(0); + expect(overrides).toMatchObject({ cached_tokens: 0 }); + }); + + it('accounts for cached_tokens when prompt_tokens_details is present', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'ok', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 50, + completion_tokens: 20, + prompt_tokens_details: { cached_tokens: 15 }, + }, + }); + + await withTestActor(() => + provider.complete({ + model: 'qwen3.6-max-preview', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + const model = ALIBABA_MODELS.find( + (m) => m.id === 'qwen3.6-max-preview', + )!; + const [usage, , prefix, overrides] = recordSpy.mock.calls[0]!; + expect(usage).toEqual({ + prompt_tokens: 50, + completion_tokens: 20, + cached_tokens: 15, + }); + expect(prefix).toBe('alibaba:qwen3.6-max-preview'); + expect(overrides).toEqual({ + prompt_tokens: 50 * Number(model.costs.prompt_tokens), + completion_tokens: 20 * Number(model.costs.completion_tokens), + cached_tokens: 15 * Number(model.costs.cached_tokens ?? 0), + }); + }); +}); + +// ── Streaming deltas ──────────────────────────────────────────────── + +describe('AlibabaProvider.complete streaming', () => { + it('streams text deltas through to text events and meters final usage', async () => { + const { provider } = makeProvider(); + createMock.mockReturnValueOnce( + asAsyncIterable([ + { choices: [{ delta: { content: 'hel' } }] }, + { choices: [{ delta: { content: 'lo' } }] }, + { + choices: [{ delta: {} }], + usage: { prompt_tokens: 4, completion_tokens: 2 }, + }, + ]), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'say hi' }], + stream: true, + }), + ); + expect((result as { stream: boolean }).stream).toBe(true); + + const harness = makeCapturingChatStream(); + await ( + result as { + init_chat_stream: (p: { chatStream: unknown }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }); + + const events = harness.events(); + const textEvents = events.filter((e) => e.type === 'text'); + expect(textEvents.map((e) => e.text)).toEqual(['hel', 'lo']); + + const usageEvent = events.find((e) => e.type === 'usage'); + expect(usageEvent?.usage).toEqual({ + prompt_tokens: 4, + completion_tokens: 2, + cached_tokens: 0, + }); + + const qwenPlus = ALIBABA_MODELS.find((m) => m.id === 'qwen-plus')!; + expect(recordSpy).toHaveBeenCalledTimes(1); + const [, , prefix, overrides] = recordSpy.mock.calls[0]!; + expect(prefix).toBe('alibaba:qwen-plus'); + expect(overrides).toEqual({ + prompt_tokens: 4 * Number(qwenPlus.costs.prompt_tokens), + completion_tokens: 2 * Number(qwenPlus.costs.completion_tokens), + cached_tokens: 0, + }); + }); + + it('builds a tool_use block from streamed function-call deltas', async () => { + const { provider } = makeProvider(); + createMock.mockReturnValueOnce( + asAsyncIterable([ + { + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + id: 'call_1', + function: { + name: 'lookup', + arguments: '{"q":', + }, + }, + ], + }, + }, + ], + }, + { + choices: [ + { + delta: { + tool_calls: [ + { + index: 0, + function: { arguments: '"puter"}' }, + }, + ], + }, + }, + ], + }, + { + choices: [{ delta: {} }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }, + ]), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'do tool call' }], + tools: [ + { + type: 'function', + function: { name: 'lookup', parameters: {} }, + }, + ], + stream: true, + }), + ); + + const harness = makeCapturingChatStream(); + await ( + result as { + init_chat_stream: (p: { chatStream: unknown }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }); + + const events = harness.events(); + const toolEvent = events.find((e) => e.type === 'tool_use'); + expect(toolEvent).toBeDefined(); + expect(toolEvent?.id).toBe('call_1'); + expect(toolEvent?.name).toBe('lookup'); + expect(toolEvent?.input).toEqual({ q: 'puter' }); + }); +}); + +// ── Error mapping ─────────────────────────────────────────────────── + +describe('AlibabaProvider.complete error mapping', () => { + it('rethrows errors raised by the OpenAI client unchanged', async () => { + const { provider } = makeProvider(); + const apiError = new Error('DashScope exploded'); + createMock.mockRejectedValueOnce(apiError); + + await expect( + withTestActor(() => + provider.complete({ + model: 'qwen-plus', + messages: [{ role: 'user', content: 'boom' }], + }), + ), + ).rejects.toBe(apiError); + + expect(recordSpy).not.toHaveBeenCalled(); + }); +}); + +// ── Moderation ────────────────────────────────────────────────────── + +describe('AlibabaProvider.checkModeration', () => { + it('throws — Alibaba provider does not implement moderation', () => { + const { provider } = makeProvider(); + expect(() => provider.checkModeration('anything')).toThrow( + /not implemented/i, + ); + }); +});