From ff16ec890049cd48228f163264cc1684424dc6e7 Mon Sep 17 00:00:00 2001 From: 404oops <51266541+404oops@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:01:33 +0200 Subject: [PATCH] Add Neuralwatt support (#3509) --- .../ChatCompletionDriver.edges.test.ts | 6 + .../drivers/ai-chat/ChatCompletionDriver.ts | 20 +- .../neuralwatt/NeuralwattProvider.test.ts | 699 ++++++++++++++++++ .../neuralwatt/NeuralwattProvider.ts | 384 ++++++++++ .../ai-chat/providers/neuralwatt/models.ts | 186 +++++ 5 files changed, 1294 insertions(+), 1 deletion(-) create mode 100644 src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts create mode 100644 src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts create mode 100644 src/backend/drivers/ai-chat/providers/neuralwatt/models.ts diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts index 529f9ee41..02790c1c6 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts @@ -48,6 +48,7 @@ import { ChatCompletionDriver } from './ChatCompletionDriver.js'; import { AzureChatProvider } from './providers/azure/AzureChatProvider.js'; import { FakeChatProvider } from './providers/FakeChatProvider.js'; import { InfronProvider } from './providers/infron/InfronProvider.js'; +import { NeuralwattProvider } from './providers/neuralwatt/NeuralwattProvider.js'; import { OpenAiChatProvider } from './providers/openai/OpenAiChatCompletionsProvider.js'; import { OpenRouterProvider } from './providers/openrouter/OpenRouterProvider.js'; import { TogetherAIProvider } from './providers/together/TogetherAIProvider.js'; @@ -72,6 +73,7 @@ const FULL_PROVIDER_CONFIG = { 'together-ai': { apiKey: 'k' }, openrouter: { apiKey: 'k', apiBaseUrl: 'https://openrouter.test' }, infron: { apiKey: 'k' }, + neuralwatt: { apiKey: 'k' }, // Suppress auto-discovery of a developer's local Ollama. ollama: { enabled: false }, }, @@ -120,6 +122,9 @@ beforeAll(async () => { vi.spyOn(InfronProvider.prototype, 'models').mockResolvedValue( aggregatorCatalog('infron-only-model') as never, ); + vi.spyOn(NeuralwattProvider.prototype, 'models').mockResolvedValue( + aggregatorCatalog('neuralwatt-only-model') as never, + ); fullDriver = await makeDriver(FULL_PROVIDER_CONFIG); fakeOnlyDriver = await makeDriver({ providers: { ollama: { enabled: false } }, @@ -176,6 +181,7 @@ describe('ChatCompletionDriver provider registration', () => { 'together-ai', 'openrouter', 'infron', + 'neuralwatt', 'fake-chat', ]) { expect(providers).toContain(expected); diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts index 1e009bf04..5d6dca9c3 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -41,6 +41,7 @@ import { InfronProvider } from './providers/infron/InfronProvider.js'; import { MiniMaxProvider } from './providers/minimax/MiniMaxProvider.js'; import { MistralAIProvider } from './providers/mistral/MistralAiProvider.js'; import { MoonshotProvider } from './providers/moonshot/MoonshotProvider.js'; +import { NeuralwattProvider } from './providers/neuralwatt/NeuralwattProvider.js'; import { OllamaChatProvider } from './providers/ollama/OllamaProvider.js'; import { OpenAiChatProvider } from './providers/openai/OpenAiChatCompletionsProvider.js'; import { OpenAiResponsesChatProvider } from './providers/openai/OpenAiChatResponsesProvider.js'; @@ -999,6 +1000,18 @@ export class ChatCompletionDriver extends PuterDriver { ); } + const neuralwatt = providers['neuralwatt']; + const neuralwattKey = readKey(neuralwatt); + if (neuralwattKey) { + this.#providers['neuralwatt'] = new NeuralwattProvider( + { + apiKey: neuralwattKey, + apiBaseUrl: neuralwatt?.apiBaseUrl as string | undefined, + }, + metering, + ); + } + // Fake provider — always available for testing this.#providers['fake-chat'] = new FakeChatProvider(); } @@ -1006,7 +1019,12 @@ export class ChatCompletionDriver extends PuterDriver { // -- Model map --------------------------------------------------- async #buildModelMap() { - const AGGREGATORS = new Set(['together-ai', 'openrouter', 'infron']); + const AGGREGATORS = new Set([ + 'together-ai', + 'openrouter', + 'infron', + 'neuralwatt', + ]); for (const providerName in this.#providers) { const provider = this.#providers[providerName]; diff --git a/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts new file mode 100644 index 000000000..259e63a19 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts @@ -0,0 +1,699 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Offline unit tests for NeuralwattProvider. + * + * Boots a real PuterServer and constructs NeuralwattProvider against the + * live MeteringService. The OpenAI SDK and axios (catalog + quota) are + * mocked at their module boundaries. + */ + +import { Writable } from 'node:stream'; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, + vi, + type MockInstance, +} from 'vitest'; + +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { PuterServer } from '../../../../server.js'; +import { setupTestServer } from '../../../../testUtil.js'; +import { kv } from '../../../../util/kvSingleton.js'; +import { withTestActor } from '../../../integrationTestUtil.js'; +import { AIChatStream } from '../../utils/Streaming.js'; +import { + mapNeuralwattApiModel, + NEURALWATT_DEFAULT_MODEL, + stripNeuralwattPrefix, +} from './models.js'; +import { NeuralwattProvider } from './NeuralwattProvider.js'; + +// ── OpenAI SDK mock ───────────────────────────────────────────────── + +const { createMock, openAICtor } = vi.hoisted(() => ({ + createMock: vi.fn(), + openAICtor: vi.fn(), +})); + +vi.mock('openai', () => { + const OpenAICtor = vi.fn().mockImplementation(function ( + this: Record, + opts: unknown, + ) { + openAICtor(opts); + this.chat = { completions: { create: createMock } }; + }); + return { OpenAI: OpenAICtor, default: { OpenAI: OpenAICtor } }; +}); + +// ── axios mock (models + quota) ───────────────────────────────────── + +const { axiosRequestMock } = vi.hoisted(() => ({ + axiosRequestMock: vi.fn(), +})); + +vi.mock('axios', () => ({ + default: { request: axiosRequestMock }, + request: axiosRequestMock, +})); + +// ── Test harness ──────────────────────────────────────────────────── + +let server: PuterServer; +let recordSpy: MockInstance; + +const KV_MODELS_KEY = 'neuralwattChat:models'; +const KV_QUOTA_KEY = 'neuralwattChat:quota'; + +const SAMPLE_API_MODELS = [ + { + id: 'deepseek-v4-flash', + created: 1_700_000_000, + max_model_len: 1_000_000, + metadata: { + display_name: 'DeepSeek V4 Flash', + description: 'Fast tool-calling model', + pricing: { + input_per_million: 0.14, + output_per_million: 0.28, + cached_input_per_million: 0.014, + pricing_tbd: false, + }, + capabilities: { + tools: true, + vision: false, + streaming: true, + }, + limits: { + max_context_length: 1_000_000, + max_output_tokens: 384_000, + }, + }, + }, + { + id: 'zai-org/GLM-5.1-FP8', + metadata: { + display_name: 'GLM 5.1', + pricing: { + input_per_million: 0.35, + output_per_million: 1.38, + cached_input_per_million: 0.035, + pricing_tbd: false, + }, + capabilities: { + tools: true, + vision: false, + reasoning: true, + }, + limits: { + max_context_length: 202_752, + max_output_tokens: 16_384, + }, + }, + }, + { + id: 'coming-soon-model', + metadata: { + display_name: 'Coming Soon', + pricing: { + input_per_million: 0, + output_per_million: 0, + pricing_tbd: true, + }, + capabilities: { tools: true }, + limits: { max_context_length: 8_000, max_output_tokens: 1_000 }, + }, + }, + { + id: 'deprecated-model', + metadata: { + display_name: 'Old', + deprecated: true, + pricing: { + input_per_million: 1, + output_per_million: 2, + pricing_tbd: false, + }, + capabilities: { tools: true }, + limits: { max_context_length: 8_000, max_output_tokens: 1_000 }, + }, + }, +]; + +const mockCatalogAndQuota = ( + accountingMethod: 'energy' | 'token' = 'energy', +) => { + axiosRequestMock.mockImplementation(async (opts: { url?: string }) => { + if (opts.url?.endsWith('/quota')) { + return { + data: { + balance: { accounting_method: accountingMethod }, + }, + }; + } + return { data: { data: SAMPLE_API_MODELS } }; + }); +}; + +beforeAll(async () => { + server = await setupTestServer(); +}); + +afterAll(async () => { + await server?.shutdown(); +}); + +const makeProvider = (config?: { apiBaseUrl?: string }) => { + const provider = new NeuralwattProvider( + { apiKey: 'test-key', ...config }, + server.services.metering, + ); + return { provider }; +}; + +const asAsyncIterable = (items: T[]): AsyncIterable => ({ + async *[Symbol.asyncIterator]() { + for (const item of items) { + yield item; + } + }, +}); + +const makeCapturingChatStream = () => { + const chunks: string[] = []; + const sink = new Writable({ + write(chunk, _enc, cb) { + chunks.push(chunk.toString('utf8')); + cb(); + }, + }); + const chatStream = new AIChatStream({ stream: sink }); + return { + chatStream, + events: () => + chunks + .join('') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)), + }; +}; + +beforeEach(() => { + createMock.mockReset(); + openAICtor.mockReset(); + axiosRequestMock.mockReset(); + mockCatalogAndQuota('energy'); + kv.del(KV_MODELS_KEY); + kv.del(KV_QUOTA_KEY); + recordSpy = vi.spyOn(server.services.metering, 'utilRecordUsageObject'); +}); + +afterEach(() => { + vi.restoreAllMocks(); + kv.del(KV_MODELS_KEY); + kv.del(KV_QUOTA_KEY); +}); + +// ── Mapping helpers ───────────────────────────────────────────────── + +describe('Neuralwatt model mapping helpers', () => { + it('strips the neuralwatt: prefix for upstream ids', () => { + expect(stripNeuralwattPrefix('neuralwatt:deepseek-v4-flash')).toBe( + 'deepseek-v4-flash', + ); + expect(stripNeuralwattPrefix('deepseek-v4-flash')).toBe( + 'deepseek-v4-flash', + ); + }); + + it('maps catalog pricing into usd-cents cost keys and skips pricing_tbd', () => { + const mapped = mapNeuralwattApiModel(SAMPLE_API_MODELS[0]!); + expect(mapped).toMatchObject({ + id: 'neuralwatt:deepseek-v4-flash', + costs_currency: 'usd-cents', + input_cost_key: 'prompt_tokens', + output_cost_key: 'completion_tokens', + costs: { + tokens: 1_000_000, + prompt_tokens: 0.14 * 100, + completion_tokens: 0.28 * 100, + cached_tokens: 0.014 * 100, + }, + tool_call: true, + max_tokens: 384_000, + modalities: { input: ['text'], output: ['text'] }, + }); + expect(mapNeuralwattApiModel(SAMPLE_API_MODELS[2]!)).toBeNull(); + }); + + it('marks vision models from capabilities.vision', () => { + const mapped = mapNeuralwattApiModel({ + id: 'gemma-4-31b', + metadata: { + display_name: 'Gemma 4 31B', + pricing: { + input_per_million: 0.1, + output_per_million: 0.2, + pricing_tbd: false, + }, + capabilities: { tools: true, vision: true }, + limits: { + max_context_length: 256_000, + max_output_tokens: 16_384, + max_images: 8, + }, + }, + }); + expect(mapped).toMatchObject({ + id: 'neuralwatt:gemma-4-31b', + modalities: { input: ['text', 'image'], output: ['text'] }, + max_images: 8, + }); + }); +}); + +// ── Construction ──────────────────────────────────────────────────── + +describe('NeuralwattProvider construction', () => { + it('points the OpenAI SDK at the Neuralwatt base URL with the configured key', () => { + makeProvider(); + expect(openAICtor).toHaveBeenCalledTimes(1); + expect(openAICtor).toHaveBeenCalledWith({ + apiKey: 'test-key', + baseURL: 'https://api.neuralwatt.com/v1', + }); + }); + + it('honours an apiBaseUrl override', () => { + makeProvider({ apiBaseUrl: 'https://custom.neuralwatt.example/v1' }); + expect(openAICtor).toHaveBeenLastCalledWith({ + apiKey: 'test-key', + baseURL: 'https://custom.neuralwatt.example/v1', + }); + }); +}); + +// ── Model catalog + quota ─────────────────────────────────────────── + +describe('NeuralwattProvider model catalog', () => { + it('returns the neuralwatt-prefixed default model id', () => { + const { provider } = makeProvider(); + expect(provider.getDefaultModel()).toBe(NEURALWATT_DEFAULT_MODEL); + }); + + it('sends the API key as a bearer token on the catalog fetch', async () => { + const { provider } = makeProvider(); + await provider.models(); + const modelsCall = axiosRequestMock.mock.calls.find( + ([args]) => + typeof args?.url === 'string' && args.url.endsWith('/models'), + ); + expect(modelsCall?.[0]).toMatchObject({ + url: 'https://api.neuralwatt.com/v1/models', + headers: { Authorization: 'Bearer test-key' }, + }); + }); + + it('list() prefixes ids and skips deprecated / pricing_tbd entries', async () => { + const { provider } = makeProvider(); + const ids = await provider.list(); + expect(ids).toContain('neuralwatt:deepseek-v4-flash'); + expect(ids).toContain('neuralwatt:zai-org/GLM-5.1-FP8'); + expect(ids).toContain('GLM-5.1-FP8'); + expect(ids).not.toContain('neuralwatt:coming-soon-model'); + expect(ids).not.toContain('neuralwatt:deprecated-model'); + }); + + it('caches the model list in kv after the first axios round-trip', async () => { + const { provider } = makeProvider(); + await provider.models(); + await provider.models(); + const modelsCalls = axiosRequestMock.mock.calls.filter( + ([args]) => + typeof args?.url === 'string' && args.url.endsWith('/models'), + ); + expect(modelsCalls).toHaveLength(1); + }); + + it('caches accounting_method from /quota', async () => { + const { provider } = makeProvider(); + await expect(provider.getAccountingMethod()).resolves.toBe('energy'); + await expect(provider.getAccountingMethod()).resolves.toBe('energy'); + const quotaCalls = axiosRequestMock.mock.calls.filter( + ([args]) => + typeof args?.url === 'string' && args.url.endsWith('/quota'), + ); + expect(quotaCalls).toHaveLength(1); + }); +}); + +// ── Request shape ─────────────────────────────────────────────────── + +describe('NeuralwattProvider.complete request shape', () => { + const baseCompletion = { + choices: [ + { + message: { content: 'hi', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + cost: { request_cost_usd: 0 }, + energy: { + energy_kwh: 0.000001, + energy_joules: 3.6, + measurement_available: true, + }, + }; + + it('strips the neuralwatt: prefix from the wire model id', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'neuralwatt:deepseek-v4-flash', + messages: [{ role: 'user', content: 'hello' }], + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect(args.model).toBe('deepseek-v4-flash'); + }); + + it('only sets stream_options.include_usage when streaming', async () => { + const { provider } = makeProvider(); + + createMock.mockResolvedValueOnce(baseCompletion); + await withTestActor(() => + provider.complete({ + model: 'neuralwatt:deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + stream: false, + }), + ); + expect(createMock.mock.calls[0]![0].stream).toBe(false); + expect('stream_options' in createMock.mock.calls[0]![0]).toBe(false); + + createMock.mockReturnValueOnce(asAsyncIterable([])); + await withTestActor(() => + provider.complete({ + model: 'neuralwatt:deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + stream: true, + }), + ); + expect(createMock.mock.calls[1]![0].stream_options).toEqual({ + include_usage: true, + }); + }); + + it('rejects image content on a text-only catalog model', async () => { + const { provider } = makeProvider(); + await expect( + withTestActor(() => + provider.complete({ + model: 'neuralwatt:deepseek-v4-flash', + messages: [ + { + role: 'user', + content: [ + { + type: 'image_url', + image_url: { + url: 'https://example.com/cat.png', + }, + }, + ], + }, + ], + }), + ), + ).rejects.toMatchObject({ + statusCode: 400, + message: expect.stringContaining('does not support image input'), + }); + expect(createMock).not.toHaveBeenCalled(); + }); + + it('prefers a vision catalog model when the prompt has images and no model was named', async () => { + // Seed a vision model into the catalog payload. + axiosRequestMock.mockImplementation(async (opts: { url?: string }) => { + if (opts.url?.endsWith('/quota')) { + return { + data: { balance: { accounting_method: 'energy' } }, + }; + } + return { + data: { + data: [ + ...SAMPLE_API_MODELS, + { + id: 'gemma-4-31b', + metadata: { + display_name: 'Gemma 4 31B', + pricing: { + input_per_million: 0.1, + output_per_million: 0.2, + pricing_tbd: false, + }, + capabilities: { + tools: true, + vision: true, + }, + limits: { + max_context_length: 256_000, + max_output_tokens: 16_384, + }, + }, + }, + ], + }, + }; + }); + + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'a cat', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + cost: { request_cost_usd: 0 }, + }); + + await withTestActor(() => + provider.complete({ + model: '', + messages: [ + { + role: 'user', + content: [ + { + type: 'image_url', + image_url: { + url: 'data:image/png;base64,abc', + }, + }, + ], + }, + ], + }), + ); + + expect(createMock.mock.calls[0]![0].model).toBe('gemma-4-31b'); + }); +}); + +// ── Non-stream metering ───────────────────────────────────────────── + +describe('NeuralwattProvider.complete non-stream output', () => { + it('bills from cost.request_cost_usd and records energy units at zero cost', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'hi there', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 10 }, + }, + cost: { request_cost_usd: 0.0001 }, + energy: { + energy_kwh: 0.00000145, + energy_joules: 5.23, + measurement_available: true, + }, + }); + + const result = (await withTestActor(() => + provider.complete({ + model: 'neuralwatt:deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + }), + )) as { usage: Record }; + + expect(recordSpy).toHaveBeenCalledTimes(1); + const [usage, , prefix, overrides] = recordSpy.mock.calls[0]!; + expect(prefix).toBe('neuralwatt:deepseek-v4-flash'); + expect(usage).toMatchObject({ + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 10, + energy_kwh: 0.00000145, + energy_joules: 5.23, + billedUsage: 1, + }); + expect(overrides).toMatchObject({ + prompt_tokens: 0, + completion_tokens: 0, + cached_tokens: 0, + energy_kwh: 0, + energy_joules: 0, + billedUsage: 0.0001 * 100_000_000, + }); + expect(result.usage.usd_cents).toBe(0.0001 * 100); + expect(result.usage.accounting_method).toBe('energy'); + }); + + it('falls back to catalog token pricing when request_cost_usd is absent', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce({ + choices: [ + { + message: { content: 'ok', role: 'assistant' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 100, + completion_tokens: 50, + prompt_tokens_details: { cached_tokens: 10 }, + }, + }); + + await withTestActor(() => + provider.complete({ + model: 'neuralwatt:deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + // usdPerMToken: $0.14/M → 14 cents per MTok unit in costs map + const promptRate = 0.14 * 100; + const completionRate = 0.28 * 100; + const cachedRate = 0.014 * 100; + const [usage, , , overrides] = recordSpy.mock.calls[0]!; + expect(usage).toMatchObject({ + prompt_tokens: 100, + completion_tokens: 50, + cached_tokens: 10, + }); + expect(overrides).toMatchObject({ + prompt_tokens: 100 * promptRate, + completion_tokens: 50 * completionRate, + cached_tokens: 10 * cachedRate, + }); + }); +}); + +// ── Streaming ─────────────────────────────────────────────────────── + +describe('NeuralwattProvider.complete streaming', () => { + it('streams text deltas and meters final-chunk cost + energy', async () => { + const { provider } = makeProvider(); + createMock.mockReturnValueOnce( + asAsyncIterable([ + { + choices: [ + { + delta: { content: 'Hello' }, + finish_reason: null, + }, + ], + }, + { + choices: [ + { + delta: { content: '!' }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 10, + completion_tokens: 2, + }, + cost: { request_cost_usd: 0.00005 }, + energy: { + energy_kwh: 0.000002, + energy_joules: 7.2, + measurement_available: true, + }, + }, + ]), + ); + + const result = (await withTestActor(() => + provider.complete({ + model: 'neuralwatt:deepseek-v4-flash', + messages: [{ role: 'user', content: 'hi' }], + stream: true, + }), + )) as { + stream: true; + init_chat_stream: (p: { + chatStream: AIChatStream; + }) => Promise; + }; + + const { chatStream, events } = makeCapturingChatStream(); + await result.init_chat_stream({ chatStream }); + + const textEvents = events().filter((e) => e.type === 'text'); + expect(textEvents.map((e) => e.text).join('')).toBe('Hello!'); + + expect(recordSpy).toHaveBeenCalledTimes(1); + const [usage, , , overrides] = recordSpy.mock.calls[0]!; + expect(usage).toMatchObject({ + prompt_tokens: 10, + completion_tokens: 2, + energy_kwh: 0.000002, + energy_joules: 7.2, + billedUsage: 1, + }); + expect(overrides).toMatchObject({ + billedUsage: 0.00005 * 100_000_000, + energy_kwh: 0, + }); + }); +}); diff --git a/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts new file mode 100644 index 000000000..a1a55e044 --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts @@ -0,0 +1,384 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import axios from 'axios'; +import { OpenAI } from 'openai'; +import { ChatCompletionCreateParams } from 'openai/resources'; +import { Context } from '../../../../core/context.js'; +import { HttpError } from '../../../../core/http/HttpError.js'; +import type { MeteringService } from '../../../../services/metering/MeteringService.js'; +import { kv } from '../../../../util/kvSingleton.js'; +import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; +import type { + IChatModel, + IChatProvider, + IChatCompleteResult, + ICompleteArguments, +} from '../../types.js'; +import { inlineHttpImageUrls } from '../moonshot/imageHandling.js'; +import { + mapNeuralwattApiModel, + messagesHaveImageContent, + modelSupportsVision, + NEURALWATT_DEFAULT_MODEL, + NEURALWATT_ID_PREFIX, + stripNeuralwattPrefix, + type NeuralwattAccountingMethod, + type NeuralwattApiModel, + type NeuralwattCost, + type NeuralwattEnergy, +} from './models.js'; + +const DEFAULT_API_BASE_URL = 'https://api.neuralwatt.com/v1'; +const KV_MODELS_KEY = 'neuralwattChat:models'; +const KV_QUOTA_KEY = 'neuralwattChat:quota'; +const CACHE_TTL_SEC = 15 * 60; + +type NeuralwattUsage = OpenAI.Completions.CompletionUsage & { + request_cost_usd?: number; + energy_kwh?: number; + energy_joules?: number; + measurement_available?: boolean; +}; + +export class NeuralwattProvider implements IChatProvider { + #meteringService: MeteringService; + + #openai: OpenAI; + + #apiKey: string; + + #apiBaseUrl: string = DEFAULT_API_BASE_URL; + + constructor( + config: { apiBaseUrl?: string; apiKey: string }, + meteringService: MeteringService, + ) { + this.#apiBaseUrl = config.apiBaseUrl || DEFAULT_API_BASE_URL; + this.#apiKey = config.apiKey; + this.#openai = new OpenAI({ + apiKey: config.apiKey, + baseURL: this.#apiBaseUrl, + }); + this.#meteringService = meteringService; + } + + getDefaultModel() { + return NEURALWATT_DEFAULT_MODEL; + } + + async list() { + const models = await this.models(); + const modelNames: string[] = []; + for (const model of models) { + modelNames.push(model.id); + if (model.aliases) modelNames.push(...model.aliases); + } + return modelNames; + } + + async models(): Promise { + let apiModels = kv.get(KV_MODELS_KEY) as + | NeuralwattApiModel[] + | undefined; + if (!apiModels) { + try { + const resp = await axios.request({ + method: 'GET', + url: `${this.#apiBaseUrl}/models`, + headers: { + Authorization: `Bearer ${this.#apiKey}`, + }, + }); + apiModels = resp.data.data ?? []; + kv.set(KV_MODELS_KEY, apiModels, { EX: CACHE_TTL_SEC }); + } catch (e) { + console.error( + 'Failed to fetch Neuralwatt models:', + (e as Error).message, + ); + } + } + if (!apiModels) return []; + + const coerced: IChatModel[] = []; + for (const model of apiModels) { + if (model.metadata?.deprecated) continue; + const mapped = mapNeuralwattApiModel(model); + if (mapped) coerced.push(mapped); + } + return coerced; + } + + /** + * Cached account accounting method (`energy` | `token`) from + * `GET /v1/quota`. Used only to annotate returned usage — billing + * always prefers `cost.request_cost_usd` on the completion. + */ + async getAccountingMethod(): Promise< + NeuralwattAccountingMethod | undefined + > { + let method = kv.get(KV_QUOTA_KEY) as + | NeuralwattAccountingMethod + | undefined; + if (method === 'energy' || method === 'token') return method; + + try { + const resp = await axios.request({ + method: 'GET', + url: `${this.#apiBaseUrl}/quota`, + headers: { + Authorization: `Bearer ${this.#apiKey}`, + }, + }); + const raw = resp.data?.balance?.accounting_method; + if (raw === 'energy' || raw === 'token') { + method = raw; + kv.set(KV_QUOTA_KEY, method, { EX: CACHE_TTL_SEC }); + return method; + } + } catch (e) { + console.error( + 'Failed to fetch Neuralwatt quota:', + (e as Error).message, + ); + } + return undefined; + } + + async complete({ + messages, + stream, + model, + tools, + max_tokens, + temperature, + reasoning_effort, + reasoning, + }: ICompleteArguments): Promise { + // Catalog carries per-model vision / reasoning_effort flags from + // Neuralwatt `GET /models` — resolve against it before shaping the + // upstream request so image-bearing prompts land on a vision model. + const availableModels = await this.models(); + const hasImages = messagesHaveImageContent(messages ?? []); + const modelLower = (model ?? '').toLowerCase(); + let modelUsed = + availableModels.find((m) => + [m.id, ...(m.aliases || [])].some( + (id) => id.toLowerCase() === modelLower, + ), + ) ?? undefined; + + if (!modelUsed) { + if (hasImages) { + modelUsed = availableModels.find((m) => + modelSupportsVision(m), + ); + } + modelUsed = + modelUsed || + availableModels.find((m) => m.id === this.getDefaultModel()) || + availableModels[0]; + } + + if (!modelUsed) { + throw new Error('No Neuralwatt models available'); + } + + if (hasImages && !modelSupportsVision(modelUsed)) { + throw new HttpError( + 400, + `Model ${modelUsed.id} does not support image input`, + { legacyCode: 'bad_request' }, + ); + } + + const modelIdForParams = stripNeuralwattPrefix(modelUsed.id); + const actor = Context.get('actor'); + const accountingMethod = await this.getAccountingMethod(); + + // Vision models: Neuralwatt (like Moonshot) expects inline data URLs + // rather than remote http(s) fetches for image_url parts. + if (modelSupportsVision(modelUsed)) { + await inlineHttpImageUrls(messages); + } + + messages = await OpenAIUtil.process_input_messages(messages); + + const requestedReasoningEffort = + reasoning_effort ?? reasoning?.effort; + const supportsReasoningEffort = modelUsed.reasoning_effort === true; + + const completionParams = { + messages, + model: modelIdForParams, + ...(tools ? { tools } : {}), + ...(max_tokens !== undefined ? { max_tokens } : {}), + ...(temperature !== undefined ? { temperature } : {}), + ...(supportsReasoningEffort && requestedReasoningEffort + ? { reasoning_effort: requestedReasoningEffort } + : {}), + stream, + ...(stream + ? { + stream_options: { include_usage: true }, + } + : {}), + } as ChatCompletionCreateParams; + + const completion = + await this.#openai.chat.completions.create(completionParams); + + const usage_calculator = ({ + usage, + cost, + energy, + }: { + usage: NeuralwattUsage; + cost?: NeuralwattCost; + energy?: NeuralwattEnergy; + }) => { + // Non-streaming spreads the full completion into this call; + // streaming merges top-level cost/energy onto `usage` via the + // index_usage_from_stream_chunk deviation below. + const requestCostUsd = + typeof cost?.request_cost_usd === 'number' + ? cost.request_cost_usd + : typeof usage.request_cost_usd === 'number' + ? usage.request_cost_usd + : undefined; + + const energyBlock = energy ?? { + energy_kwh: usage.energy_kwh, + energy_joules: usage.energy_joules, + measurement_available: usage.measurement_available, + }; + + const trackedTokens = OpenAIUtil.extractMeteredUsage(usage); + const energyUnits: Record = {}; + if ( + energyBlock.measurement_available !== false && + typeof energyBlock.energy_kwh === 'number' && + Number.isFinite(energyBlock.energy_kwh) && + energyBlock.energy_kwh > 0 + ) { + energyUnits.energy_kwh = energyBlock.energy_kwh; + } + if ( + energyBlock.measurement_available !== false && + typeof energyBlock.energy_joules === 'number' && + Number.isFinite(energyBlock.energy_joules) && + energyBlock.energy_joules > 0 + ) { + energyUnits.energy_joules = energyBlock.energy_joules; + } + + const annotate = (tracked: Record) => { + const out: Record = { ...tracked }; + if (accountingMethod) { + out.accounting_method = accountingMethod; + } + return out; + }; + + if ( + typeof requestCostUsd === 'number' && + Number.isFinite(requestCostUsd) + ) { + const billedTrackedUsage = { + ...trackedTokens, + ...energyUnits, + billedUsage: 1, + }; + const costOverwrites = Object.fromEntries( + Object.keys(billedTrackedUsage).map((k) => [k, 0]), + ); + costOverwrites.billedUsage = requestCostUsd * 100_000_000; + this.#meteringService.utilRecordUsageObject( + billedTrackedUsage, + actor!, + modelUsed.id, + costOverwrites, + ); + const result = annotate(billedTrackedUsage); + result.usd_cents = requestCostUsd * 100; + return result; + } + + // Fallback: catalog token rates (preflight / when Neuralwatt + // omits request_cost_usd). + const trackedUsage = { ...trackedTokens, ...energyUnits }; + const costOverwrites = Object.fromEntries( + Object.entries(trackedUsage).map(([k, v]) => { + if (k === 'energy_kwh' || k === 'energy_joules') { + return [k, 0]; + } + return [k, (modelUsed.costs[k] ?? 0) * v]; + }), + ); + this.#meteringService.utilRecordUsageObject( + trackedUsage, + actor!, + modelUsed.id, + costOverwrites, + ); + return annotate(trackedUsage); + }; + + return OpenAIUtil.handle_completion_output({ + deviations: { + index_usage_from_stream_chunk: (chunk: { + usage?: NeuralwattUsage; + cost?: NeuralwattCost; + energy?: NeuralwattEnergy; + }) => { + if (!chunk.usage) return chunk.usage; + return { + ...chunk.usage, + ...(typeof chunk.cost?.request_cost_usd === 'number' + ? { + request_cost_usd: + chunk.cost.request_cost_usd, + } + : {}), + ...(chunk.energy + ? { + energy_kwh: chunk.energy.energy_kwh, + energy_joules: chunk.energy.energy_joules, + measurement_available: + chunk.energy.measurement_available, + } + : {}), + }; + }, + }, + usage_calculator, + stream, + completion, + }); + } + + checkModeration( + _text: string, + ): ReturnType { + throw new Error('Method not implemented.'); + } +} + +export { NEURALWATT_ID_PREFIX, NEURALWATT_DEFAULT_MODEL }; diff --git a/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts b/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts new file mode 100644 index 000000000..6e35ddc1d --- /dev/null +++ b/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts @@ -0,0 +1,186 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { IChatModel } from '../../types.js'; +import { usdPerMToken } from '../../utils/pricing.js'; + +export const NEURALWATT_ID_PREFIX = 'neuralwatt:'; +export const NEURALWATT_DEFAULT_MODEL = 'neuralwatt:deepseek-v4-flash'; + +/** + * Shape of one entry in Neuralwatt's `GET /v1/models` catalog. + * Pricing is USD per million tokens under `metadata.pricing` and is used + * only for Puter preflight estimates / token-cost fallback. Live billing + * uses each completion's `cost.request_cost_usd`. + */ + +export type NeuralwattApiModel = { + id: string; + object?: string; + created?: number; + owned_by?: string; + max_model_len?: number; + metadata?: { + display_name?: string; + description?: string | null; + provider?: string; + huggingface_id?: string | null; + pricing?: { + input_per_million?: number; + output_per_million?: number; + cached_input_per_million?: number | null; + cached_output_per_million?: number | null; + currency?: string; + pricing_tbd?: boolean; + }; + capabilities?: { + tools?: boolean; + json_mode?: boolean; + vision?: boolean; + reasoning?: boolean; + reasoning_effort?: boolean; + streaming?: boolean; + system_role?: boolean; + developer_role?: boolean; + }; + limits?: { + max_context_length?: number | null; + max_output_tokens?: number | null; + max_images?: number | null; + }; + deprecated?: boolean; + deprecated_message?: string | null; + }; +}; + +export type NeuralwattAccountingMethod = 'energy' | 'token'; + +export type NeuralwattEnergy = { + energy_joules?: number; + energy_kwh?: number; + measurement_available?: boolean; + avg_power_watts?: number; + duration_seconds?: number; + attribution_method?: string; + attribution_ratio?: number; +}; + +export type NeuralwattCost = { + request_cost_usd?: number; + cache_savings_usd?: number; + allowance_remaining_usd?: number; +}; + +/** Strip the Puter-facing `neuralwatt:` prefix for upstream API calls. */ +export const stripNeuralwattPrefix = (modelId: string): string => + modelId.startsWith(NEURALWATT_ID_PREFIX) + ? modelId.slice(NEURALWATT_ID_PREFIX.length) + : modelId; + +/** + * Map a Neuralwatt catalog entry to Puter's `IChatModel`. Returns `null` + * when pricing is TBD (placeholders) so the model is not offered for + * preflight credit checks until Neuralwatt publishes real rates. + */ +export const mapNeuralwattApiModel = ( + model: NeuralwattApiModel, +): IChatModel | null => { + const pricing = model.metadata?.pricing; + if (pricing?.pricing_tbd) return null; + + const inputUsd = Number(pricing?.input_per_million ?? 0); + const outputUsd = Number(pricing?.output_per_million ?? 0); + const cachedUsd = + pricing?.cached_input_per_million == null + ? 0 + : Number(pricing.cached_input_per_million); + + const caps = model.metadata?.capabilities; + const limits = model.metadata?.limits; + const context = + limits?.max_context_length ?? model.max_model_len ?? undefined; + const maxTokens = limits?.max_output_tokens ?? context ?? 0; + + const inputModalities = ['text']; + if (caps?.vision) inputModalities.push('image'); + + const displayName = model.metadata?.display_name || model.id; + const pathTail = model.id.includes('/') + ? model.id.split('/').slice(1).join('/') + : undefined; + + return { + id: `${NEURALWATT_ID_PREFIX}${model.id}`, + name: `${displayName} (Neuralwatt)`, + aliases: [ + model.id, + `neuralwatt/${model.id}`, + ...(pathTail && pathTail !== model.id ? [pathTail] : []), + ], + context, + max_tokens: maxTokens, + costs_currency: 'usd-cents', + input_cost_key: 'prompt_tokens', + output_cost_key: 'completion_tokens', + costs: usdPerMToken(inputUsd, outputUsd, cachedUsd), + modalities: { input: inputModalities, output: ['text'] }, + tool_call: caps?.tools === true, + // Surfaced from Neuralwatt `/models` so `complete()` can gate + // reasoning_effort without re-fetching the catalog entry. + reasoning_effort: caps?.reasoning_effort === true, + ...(typeof limits?.max_images === 'number' + ? { max_images: limits.max_images } + : {}), + ...(model.metadata?.description + ? { description: model.metadata.description } + : {}), + ...(model.created + ? { + release_date: new Date(model.created * 1000) + .toISOString() + .slice(0, 10), + } + : {}), + }; +}; + +/** True when the Puter model catalog entry advertises image input. */ +export const modelSupportsVision = (model: IChatModel): boolean => + Array.isArray(model.modalities?.input) && + model.modalities.input.includes('image'); + +/** + * Detect image / puter_path parts so we can prefer a vision-capable model + * from the Neuralwatt catalog (or reject a text-only pick). + */ +export const messagesHaveImageContent = ( + messages: Array<{ content?: unknown }>, +): boolean => { + for (const message of messages) { + if (!Array.isArray(message.content)) continue; + for (const part of message.content as Array>) { + if (!part || typeof part !== 'object') continue; + if (part.type === 'image_url' || part.image_url) return true; + if (typeof part.puter_path === 'string' && part.puter_path) { + return true; + } + } + } + return false; +};