diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts index fd88feddb..2f97c291c 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -95,11 +95,9 @@ import { isToolChoice, toolChoiceFromWire } from './utils/openaiParams.js'; import { costKeys, isFreeModel, - isOutputCostKey, - isPerCallCostKey, longContextMultipliers, - trackedInputTokens, trackedOutputTokens, + usageCost, usageDetailsFromUsage, } from './utils/pricing.js'; import { @@ -1218,126 +1216,6 @@ export class ChatCompletionDriver extends PuterDriver { }; } - // Compute per-token cost in microcents (1 cent = 1_000_000 microCents). - // Shape-agnostic: multiplies every usage key by its matching rate in - // `model.costs`. Returns `null` when cost data is unavailable. - // - // `usageCosts`, when given, carries the provider's own per-key µ¢ cost - // (set from `chatStream.usageCosts` / `res.usageCosts`) — used verbatim - // for that key so the ledger and the reported cost match exactly instead - // of being recomputed off this model's cost table. - #computeCost( - usage: Record, - model: IChatModel, - usageCosts?: Record, - ): { - inputKey: string; - outputKey: string; - inputTokens: number; - outputTokens: number; - inputMicroCents: number; - outputMicroCents: number; - totalMicroCents: number; - } | null { - const { inputKey, outputKey } = costKeys(model); - - const costs = model.costs; - if (!costs) return null; - - const outputRateRaw = costs[outputKey]; - const outputRate = - typeof outputRateRaw === 'number' && Number.isFinite(outputRateRaw) - ? outputRateRaw - : undefined; - - const isOutputKey = (key: string) => isOutputCostKey(key, outputKey); - const multipliers = longContextMultipliers( - model, - trackedInputTokens(usage, model), - ); - - let inputMicroCents = 0; - let outputMicroCents = 0; - let sawAnyRate = false; - - for (const [key, rawAmount] of Object.entries(usage)) { - if (typeof rawAmount !== 'number' || !Number.isFinite(rawAmount)) { - continue; - } - - if (key === 'usd_cents') continue; - if (key === 'tokens') continue; - - const override = usageCosts?.[key]; - if (typeof override === 'number' && Number.isFinite(override)) { - sawAnyRate = true; - if (isOutputKey(key)) outputMicroCents += override; - else inputMicroCents += override; - continue; - } - // Advisor usage is priced at the advisor model's own rates - // (metered separately, under `claude:`), never off this - // (executor) model's cost table — without an override there is - // simply nothing to price it at here. - if (key.startsWith('advisor_')) continue; - - // thinking_tokens → output rate fallback - let rate = costs[key]; - if (typeof rate !== 'number' || !Number.isFinite(rate)) { - if (isOutputKey(key) && outputRate !== undefined) { - rate = outputRate; - } else if (!isOutputKey(key)) { - const inputRateRaw = costs[inputKey]; - if ( - typeof inputRateRaw === 'number' && - Number.isFinite(inputRateRaw) - ) { - rate = inputRateRaw; - } else { - continue; - } - } else { - continue; - } - } - - sawAnyRate = true; - if (isOutputKey(key)) { - outputMicroCents += rawAmount * rate * multipliers.output; - } else { - inputMicroCents += - rawAmount * - rate * - (isPerCallCostKey(key) ? 1 : multipliers.input); - } - } - - if (!sawAnyRate) return null; - - inputMicroCents = Math.max(0, Math.round(inputMicroCents)); - outputMicroCents = Math.max(0, Math.round(outputMicroCents)); - - const inputTokens = Number( - usage[inputKey] ?? usage.prompt_tokens ?? usage.input_tokens ?? 0, - ); - const outputTokens = Number( - usage[outputKey] ?? - usage.completion_tokens ?? - usage.output_tokens ?? - 0, - ); - - return { - inputKey, - outputKey, - inputTokens, - outputTokens, - inputMicroCents, - outputMicroCents, - totalMicroCents: inputMicroCents + outputMicroCents, - }; - } - /** * The credit and subscription gate for one upstream attempt. * @@ -1550,7 +1428,7 @@ export class ChatCompletionDriver extends PuterDriver { [outputKey]: outputTokens, }; - const cost = this.#computeCost(usage, model); + const cost = usageCost(usage, model); this.#aiMetering.utilRecordUsageObject( { [`estimated_${inputKey}`]: inputTokens, @@ -1593,12 +1471,13 @@ export class ChatCompletionDriver extends PuterDriver { ) { return; } - const cost = this.#computeCost(usage, model, usageCosts); + const cost = usageCost(usage, model, usageCosts); if (!cost) { (usage as Record).usd_cents = null; return; } - usage.usd_cents = cost.totalMicroCents / 1_000_000; + usage.usd_cents = + (cost.inputMicroCents + cost.outputMicroCents) / 1_000_000; } // Compute per-token cost in microcents using the model's cost map, @@ -1621,10 +1500,24 @@ export class ChatCompletionDriver extends PuterDriver { usageCosts, } = params; - const cost = this.#computeCost(usage, model, usageCosts); + const cost = usageCost(usage, model, usageCosts); const { inputKey, outputKey } = costKeys(model); - const inputTokens = cost?.inputTokens ?? 0; - const outputTokens = cost?.outputTokens ?? 0; + const inputTokens = cost + ? Number( + usage[inputKey] ?? + usage.prompt_tokens ?? + usage.input_tokens ?? + 0, + ) + : 0; + const outputTokens = cost + ? Number( + usage[outputKey] ?? + usage.completion_tokens ?? + usage.output_tokens ?? + 0, + ) + : 0; const inputMicroCents = cost?.inputMicroCents ?? 0; const outputMicroCents = cost?.outputMicroCents ?? 0; diff --git a/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts b/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts index aa9204eeb..9b9e157ca 100644 --- a/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts @@ -187,15 +187,16 @@ describe('AlibabaProvider model catalog', () => { ['qwen3.8-flash', 15, 47, 1.6], ['qwen3.8-omni-flash', 15, 47, 1.6], ['qwen3.8-max-0902', 200, 600, 25], - ['qwen3.7-max-2026-05-17', 250, 750, 0], - ['deepseek-v4.1-flash', 30, 120, 0], + // No cache discount: cached reads bill at the input rate. + ['qwen3.7-max-2026-05-17', 250, 750, undefined], + ['deepseek-v4.1-flash', 30, 120, undefined], ])('exposes %s at its list price', (id, input, output, cached) => { const model = ALIBABA_MODELS.find((m) => m.id === id); expect(model?.costs).toMatchObject({ prompt_tokens: input, completion_tokens: output, - cached_tokens: cached, }); + expect(model?.costs.cached_tokens).toBe(cached); }); it.each([ @@ -593,14 +594,15 @@ describe('AlibabaProvider.complete non-stream output', () => { (m) => m.id === 'qwen3.8-max', )!; const [usage, , prefix, overrides] = recordSpy.mock.calls[0]!; + // Cached reads come out of `prompt_tokens`; they're priced once. expect(usage).toEqual({ - prompt_tokens: 50, + prompt_tokens: 35, completion_tokens: 20, cached_tokens: 15, }); expect(prefix).toBe('alibaba:qwen3.8-max'); expect(overrides).toEqual({ - prompt_tokens: 50 * Number(model.costs.prompt_tokens), + prompt_tokens: 35 * Number(model.costs.prompt_tokens), completion_tokens: 20 * Number(model.costs.completion_tokens), cached_tokens: 15 * Number(model.costs.cached_tokens ?? 0), }); diff --git a/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.ts b/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.ts index 261e75181..fbd3d8588 100644 --- a/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.ts +++ b/src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.ts @@ -26,6 +26,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; import { ALIBABA_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; type AlibabaConfig = { apiKey: string; @@ -100,20 +101,16 @@ export class AlibabaProvider implements IChatProvider { ); return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverride = Object.fromEntries( - Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * modelUsed.costs[k]]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor!, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, + actor, this.meteringModelKey(modelUsed.id), - costsOverride, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/alibaba/models.ts b/src/backend/drivers/ai-chat/providers/alibaba/models.ts index 5f68e69bb..4ed928a6a 100644 --- a/src/backend/drivers/ai-chat/providers/alibaba/models.ts +++ b/src/backend/drivers/ai-chat/providers/alibaba/models.ts @@ -41,7 +41,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 160, completion_tokens: 640, - cached_tokens: 0, }, }, @@ -65,7 +64,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 120, - cached_tokens: 0, }, }, { @@ -87,7 +85,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 240, - cached_tokens: 0, }, }, { @@ -241,7 +238,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 5, completion_tokens: 40, - cached_tokens: 0, }, }, @@ -265,7 +261,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 30, completion_tokens: 150, - cached_tokens: 0, }, }, @@ -291,7 +286,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 60, completion_tokens: 360, - cached_tokens: 0, }, }, { @@ -315,7 +309,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 320, - cached_tokens: 0, }, }, { @@ -339,7 +332,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 25, completion_tokens: 200, - cached_tokens: 0, }, }, { @@ -363,7 +355,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 30, completion_tokens: 240, - cached_tokens: 0, }, }, @@ -389,7 +380,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 24.8, completion_tokens: 148.5, - cached_tokens: 0, }, }, { @@ -413,7 +403,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 60, completion_tokens: 360, - cached_tokens: 0, }, }, @@ -437,7 +426,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 20, completion_tokens: 160, - cached_tokens: 0, }, }, // -- Omni models (text interface, audio costs excluded) --------- @@ -463,7 +451,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 43, completion_tokens: 166, - cached_tokens: 0, }, }, // -- Translation models ----------------------------------------- @@ -486,7 +473,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 246, completion_tokens: 737, - cached_tokens: 0, }, }, @@ -509,7 +495,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 120, - cached_tokens: 0, }, }, { @@ -530,7 +515,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 120, - cached_tokens: 0, }, }, { @@ -551,7 +535,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 120, - cached_tokens: 0, }, }, // 'qwen-plus-latest' floats to the newest snapshot; kept on the newest @@ -574,7 +557,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 120, - cached_tokens: 0, }, }, // -- Qwen3.7 Max snapshots and preview ------------------------ @@ -596,7 +578,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 250, completion_tokens: 750, - cached_tokens: 0, }, }, { @@ -617,7 +598,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 250, completion_tokens: 750, - cached_tokens: 0, }, }, { @@ -638,7 +618,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 250, completion_tokens: 750, - cached_tokens: 0, }, }, // No vendor-documented release date for the preview; omitted rather @@ -660,7 +639,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 250, completion_tokens: 750, - cached_tokens: 0, }, }, // -- Qwen3.5 Plus snapshots ----------------------------------- @@ -683,7 +661,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 240, - cached_tokens: 0, }, }, { @@ -705,7 +682,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 240, - cached_tokens: 0, }, }, // -- Qwen3.5 / Qwen3.7 / Qwen3.8 Flash ----------------------- @@ -793,7 +769,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 50, completion_tokens: 300, - cached_tokens: 0, }, }, { @@ -814,7 +789,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 200, completion_tokens: 600, - cached_tokens: 0, }, }, // -- Omni additions (text interface, audio costs excluded) ---- @@ -839,7 +813,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 40, completion_tokens: 220, - cached_tokens: 0, }, }, { @@ -863,7 +836,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 140, completion_tokens: 830, - cached_tokens: 0, }, }, { @@ -909,7 +881,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 16, completion_tokens: 49, - cached_tokens: 0, }, }, // No vendor-documented release date; omitted rather than guessed. @@ -930,7 +901,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 12, completion_tokens: 36, - cached_tokens: 0, }, }, // -- Third-party models hosted on Model Studio ------------------ @@ -953,7 +923,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 140, completion_tokens: 440, - cached_tokens: 0, }, }, { @@ -973,7 +942,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 140, completion_tokens: 440, - cached_tokens: 0, }, }, // No vendor-documented release date for the preview; omitted rather @@ -994,7 +962,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 280, completion_tokens: 880, - cached_tokens: 0, }, }, // Time-of-day pricing upstream (busy $0.44/$1.32, idle $0.22/$0.66 @@ -1018,7 +985,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 44, completion_tokens: 132, - cached_tokens: 0, }, }, // Time-of-day pricing upstream (busy $0.30/$1.20, idle $0.15/$0.60 @@ -1041,7 +1007,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 30, completion_tokens: 120, - cached_tokens: 0, }, }, { @@ -1062,7 +1027,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 240, completion_tokens: 480, - cached_tokens: 0, }, }, // Time-of-day pricing upstream (busy $1.32/$3.96, idle $0.66/$1.98 @@ -1085,7 +1049,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 132, completion_tokens: 396, - cached_tokens: 0, }, }, { @@ -1106,7 +1069,6 @@ export const ALIBABA_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 95, completion_tokens: 400, - cached_tokens: 0, }, }, ]; diff --git a/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts index 2051e66f2..19bcd7ce8 100644 --- a/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts @@ -524,7 +524,7 @@ describe('BytePlusProvider.complete non-stream output', () => { finish_reason: 'stop', }); expect((result as { usage: unknown }).usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); @@ -538,14 +538,14 @@ describe('BytePlusProvider.complete non-stream output', () => { expect(recordSpy).toHaveBeenCalledTimes(1); const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!; expect(usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); expect(actor).toBe(SYSTEM_ACTOR); expect(prefix).toBe('byteplus:seed-2-0-mini-260428'); expect(overrides.prompt_tokens).toBeCloseTo( - 100 * Number(seed20Mini.costs.prompt_tokens), + 90 * Number(seed20Mini.costs.prompt_tokens), 5, ); expect(overrides.completion_tokens).toBeCloseTo( @@ -707,7 +707,7 @@ describe('BytePlusProvider.complete streaming', () => { const usageEvent = events.find((e) => e.type === 'usage'); expect(usageEvent?.usage).toEqual({ - prompt_tokens: 4, + prompt_tokens: 3, completion_tokens: 2, cached_tokens: 1, }); @@ -719,7 +719,7 @@ describe('BytePlusProvider.complete streaming', () => { const [, , prefix, overrides] = recordSpy.mock.calls[0]!; expect(prefix).toBe('byteplus:seed-2-0-mini-260428'); expect(overrides.prompt_tokens).toBeCloseTo( - 4 * Number(seed20Mini.costs.prompt_tokens), + 3 * Number(seed20Mini.costs.prompt_tokens), 5, ); expect(overrides.completion_tokens).toBeCloseTo( diff --git a/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts index 07b53a1e3..4e2661cf2 100644 --- a/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts +++ b/src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts @@ -28,6 +28,7 @@ import { openAICompatParams } from '../../utils/openaiParams.js'; import { BYTEPLUS_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; type BytePlusConfig = { apiKey: string; @@ -139,26 +140,16 @@ export class BytePlusProvider implements IChatProvider { ); const result = await OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = usage - ? OpenAIUtil.extractMeteredUsage(usage) - : { - prompt_tokens: 0, - completion_tokens: 0, - cached_tokens: 0, - }; - const costsOverride = Object.fromEntries( - Object.entries(trackedUsage).map(([key, value]) => { - return [key, value * Number(modelUsed.costs[key] ?? 0)]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor!, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, + actor, this.meteringModelKey(modelUsed.id), - costsOverride, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.test.ts b/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.test.ts index 436a4f50f..f5c427307 100644 --- a/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.test.ts @@ -529,8 +529,9 @@ describe('DeepSeekProvider.complete non-stream output', () => { message: { content: 'hi there', role: 'assistant' }, finish_reason: 'stop', }); + // Cached reads come out of `prompt_tokens`; they're priced once. expect((result as { usage: unknown }).usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); @@ -539,14 +540,14 @@ describe('DeepSeekProvider.complete non-stream output', () => { expect(recordSpy).toHaveBeenCalledTimes(1); const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!; expect(usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); expect(actor).toBe(SYSTEM_ACTOR); expect(prefix).toBe('deepseek:deepseek-v4-pro'); expect(overrides).toEqual({ - prompt_tokens: 100 * Number(chat.costs.prompt_tokens), + prompt_tokens: 90 * Number(chat.costs.prompt_tokens), completion_tokens: 50 * Number(chat.costs.completion_tokens), cached_tokens: 10 * Number(chat.costs.cached_tokens ?? 0), }); @@ -667,7 +668,7 @@ describe('DeepSeekProvider.complete streaming', () => { const usageEvent = events.find((e) => e.type === 'usage'); expect(usageEvent?.usage).toEqual({ - prompt_tokens: 4, + prompt_tokens: 3, completion_tokens: 2, cached_tokens: 1, }); @@ -677,7 +678,7 @@ describe('DeepSeekProvider.complete streaming', () => { const [, , prefix, overrides] = recordSpy.mock.calls[0]!; expect(prefix).toBe('deepseek:deepseek-v4-pro'); expect(overrides).toEqual({ - prompt_tokens: 4 * Number(chat.costs.prompt_tokens), + prompt_tokens: 3 * Number(chat.costs.prompt_tokens), completion_tokens: 2 * Number(chat.costs.completion_tokens), cached_tokens: 1 * Number(chat.costs.cached_tokens ?? 0), }); diff --git a/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts b/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts index 2e4e9750c..d1e9a4338 100644 --- a/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts +++ b/src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts @@ -27,6 +27,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; import { DEEPSEEK_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; export class DeepSeekProvider implements IChatProvider { #openai: OpenAI; @@ -131,20 +132,16 @@ export class DeepSeekProvider implements IChatProvider { ); return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverrideFromModel = Object.fromEntries( - Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * modelUsed.costs[k]]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor!, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, + actor, this.meteringModelKey(modelUsed.id), - costsOverrideFromModel, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream: stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts index cb8c793c1..afa7b5b3d 100644 --- a/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts +++ b/src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts @@ -29,10 +29,10 @@ import { process_input_messages, } from '../../utils/OpenAIUtil.js'; import { inlineHttpImageUrls } from '../../utils/inlineImages.js'; -import { buildCostsOverride } from '../../utils/pricing.js'; import { GEMINI_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; export class GeminiChatProvider implements IChatProvider { meteringService: MeteringService; @@ -126,15 +126,19 @@ export class GeminiChatProvider implements IChatProvider { // Cast to access Gemini-specific extras passed alongside usage: // - choices: non-stream grounding metadata lives in choices[0].message.extra_content // - extra_content: streaming grounding metadata accumulated by the stream handler - const { usage, choices, extra_content } = args as { - usage: typeof args.usage; - choices?: Array<{ - message?: { - extra_content?: { grounding_metadata?: unknown }; - }; - }>; - extra_content?: { grounding_metadata?: unknown }; - }; + const { usage, choices, extra_content, setUsageCosts } = + args as { + usage: typeof args.usage; + setUsageCosts: typeof args.setUsageCosts; + choices?: Array<{ + message?: { + extra_content?: { + grounding_metadata?: unknown; + }; + }; + }>; + extra_content?: { grounding_metadata?: unknown }; + }; const cached_tokens = usage?.prompt_tokens_details?.cached_tokens ?? 0; @@ -160,18 +164,15 @@ export class GeminiChatProvider implements IChatProvider { : 0, }; - const costsOverrideFromModel = buildCostsOverride( - trackedUsage, + const metered = meterChatUsage( + this.meteringService, + actor, + this.meteringModelKey(modelUsed.id), modelUsed, - ); - this.meteringService.utilRecordUsageObject( trackedUsage, - actor!, - this.meteringModelKey(modelUsed?.id), - costsOverrideFromModel, ); - - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts b/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts index 98992d0ec..bbc9f29a1 100644 --- a/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts +++ b/src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts @@ -27,6 +27,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; import { GROQ_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; export class GroqAIProvider implements IChatProvider { #client: Groq; @@ -100,20 +101,16 @@ export class GroqAIProvider implements IChatProvider { (chunk as { x_groq?: { usage?: CompletionUsage } }).x_groq ?.usage, }, - usage_calculator: ({ usage }) => { - const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverride = Object.fromEntries( - Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * modelUsed.costs[k]]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, actor, this.meteringModelKey(modelUsed.id), - costsOverride, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/groq/models.ts b/src/backend/drivers/ai-chat/providers/groq/models.ts index d30ba660d..5f5f9d798 100644 --- a/src/backend/drivers/ai-chat/providers/groq/models.ts +++ b/src/backend/drivers/ai-chat/providers/groq/models.ts @@ -77,7 +77,6 @@ export const GROQ_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 7.5, completion_tokens: 30, - cached_tokens: 0, }, max_tokens: 65536, }, @@ -97,7 +96,6 @@ export const GROQ_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 80, completion_tokens: 400, - cached_tokens: 0, }, max_tokens: 16384, }, @@ -117,7 +115,6 @@ export const GROQ_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 3, completion_tokens: 3, - cached_tokens: 0, }, max_tokens: 512, }, @@ -137,7 +134,6 @@ export const GROQ_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 4, completion_tokens: 4, - cached_tokens: 0, }, max_tokens: 512, }, @@ -157,7 +153,6 @@ export const GROQ_MODELS: IChatModel[] = [ tokens: 1_000_000, prompt_tokens: 0, completion_tokens: 0, - cached_tokens: 0, }, max_tokens: 4096, }, diff --git a/src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.test.ts b/src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.test.ts index 4dee535c7..7e4776a45 100644 --- a/src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.test.ts @@ -552,7 +552,7 @@ describe('HoonifyProvider.complete non-stream output', () => { finish_reason: 'stop', }); expect((result as { usage: unknown }).usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); @@ -566,14 +566,14 @@ describe('HoonifyProvider.complete non-stream output', () => { expect(recordSpy).toHaveBeenCalledTimes(1); const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!; expect(usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); expect(actor).toBe(SYSTEM_ACTOR); expect(prefix).toBe('hoonify:zai-org/glm-5.2'); expect(overrides.prompt_tokens).toBeCloseTo( - 100 * Number(glm.costs.prompt_tokens), + 90 * Number(glm.costs.prompt_tokens), 5, ); expect(overrides.completion_tokens).toBeCloseTo( @@ -704,7 +704,7 @@ describe('HoonifyProvider.complete streaming', () => { const usageEvent = events.find((e) => e.type === 'usage'); expect(usageEvent?.usage).toEqual({ - prompt_tokens: 4, + prompt_tokens: 3, completion_tokens: 2, cached_tokens: 1, }); @@ -716,7 +716,7 @@ describe('HoonifyProvider.complete streaming', () => { const [, , prefix, overrides] = recordSpy.mock.calls[0]!; expect(prefix).toBe('hoonify:zai-org/glm-5.2'); expect(overrides.prompt_tokens).toBeCloseTo( - 4 * Number(glm.costs.prompt_tokens), + 3 * Number(glm.costs.prompt_tokens), 5, ); expect(overrides.completion_tokens).toBeCloseTo( diff --git a/src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.ts b/src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.ts index 81ec24937..339a2ee13 100644 --- a/src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.ts +++ b/src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.ts @@ -27,6 +27,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; import { openAICompatParams } from '../../utils/openaiParams.js'; import { HOONIFY_MODELS } from './models.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; type HoonifyConfig = { apiBaseUrl?: string; @@ -139,27 +140,16 @@ export class HoonifyProvider implements IChatProvider { ); return await OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = usage - ? OpenAIUtil.extractMeteredUsage(usage) - : { - prompt_tokens: 0, - completion_tokens: 0, - cached_tokens: 0, - }; - const costsOverrideFromModel = Object.fromEntries( - Object.entries(trackedUsage).map(([key, value]) => { - return [key, value * Number(modelUsed.costs[key] ?? 0)]; - }), - ); - // `modelUsed.id` already carries the `hoonify:` namespace. - this.#meteringService.utilRecordUsageObject( - trackedUsage, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, actor, this.meteringModelKey(modelUsed.id), - costsOverrideFromModel, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts b/src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts index 54e94108c..d53ec8cc1 100644 --- a/src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts +++ b/src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts @@ -31,6 +31,7 @@ import type { ICompleteArguments, } from '../../types.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; /** * One upstream offering of a model in Infron's catalog. The same model is often @@ -287,9 +288,11 @@ export class InfronProvider implements IChatProvider { const usage_calculator = ({ usage, cost, + setUsageCosts, }: { usage: InfronUsage; cost?: number; + setUsageCosts: (costs: Record) => void; }) => { // Infron reports `cost` at the top level of the response, not // inside `usage`. Non-streaming calls get it via the spread @@ -306,42 +309,20 @@ export class InfronProvider implements IChatProvider { usage.prompt_tokens_details?.cached_tokens ?? 0, request: 1, }; - if (typeof authoritativeCost === 'number') { - // Bill the gateway-reported cost as a single line item and - // zero the per-token costs so nothing double-bills. - const billedTrackedUsage = { ...trackedUsage, billedUsage: 1 }; - const costOverwrites = Object.fromEntries( - Object.keys(billedTrackedUsage).map((k) => [k, 0]), - ); - costOverwrites.billedUsage = - authoritativeCost * 100_000_000 || 1; - this.#meteringService.utilRecordUsageObject( - billedTrackedUsage, - actor, - this.meteringModelKey(modelUsed.id), - costOverwrites, - ); - (billedTrackedUsage as Record).usd_cents = - authoritativeCost * 100; - return billedTrackedUsage; - } - // Fallback: per-token pricing from the model catalog. - const costOverwrites = Object.fromEntries( - Object.keys(trackedUsage).map((k) => { - return [ - k, - (modelUsed.costs[k] ?? 0) * - trackedUsage[k as keyof typeof trackedUsage], - ]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, + // The gateway-reported cost when there is one; the catalog's + // per-token prices otherwise. + const metered = meterChatUsage( + this.#meteringService, actor, this.meteringModelKey(modelUsed.id), - costOverwrites, + modelUsed, + trackedUsage, + typeof authoritativeCost === 'number' + ? { authoritativeUsd: authoritativeCost } + : {}, ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }; return OpenAIUtil.handle_completion_output({ diff --git a/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts b/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts index d4ef75f05..d9b49dd3b 100644 --- a/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts +++ b/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts @@ -29,12 +29,12 @@ import type { IChatProvider, ICompleteArguments } from '../../types.js'; import { make_openai_tools } from '../../utils/FunctionCalling.js'; import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; import { openAICompatParams } from '../../utils/openaiParams.js'; -import { buildCostsOverride } from '../../utils/pricing.js'; import { processPuterPathUploads } from '../openai/fileUpload.js'; import { META_MODELS, MUSE_SPARK_DEFAULT_MODEL } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; const DEFAULT_API_BASE_URL = 'https://api.meta.ai/v1'; @@ -221,7 +221,7 @@ export class MetaProvider implements IChatProvider { ); return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { + usage_calculator: ({ usage, setUsageCosts }) => { const cachedTokens = usage?.prompt_tokens_details?.cached_tokens ?? 0; // Meta reports cache reads as a subset of `prompt_tokens`, so @@ -234,17 +234,15 @@ export class MetaProvider implements IChatProvider { completion_tokens: usage?.completion_tokens ?? 0, cached_tokens: cachedTokens, }; - const costsOverride = buildCostsOverride( - trackedUsage, - modelUsed, - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor!, + const metered = meterChatUsage( + this.#meteringService, + actor, this.meteringModelKey(modelUsed.id), - costsOverride, + modelUsed, + trackedUsage, ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.test.ts b/src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.test.ts index 79b3dcfea..e41f1301e 100644 --- a/src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.test.ts @@ -349,7 +349,7 @@ describe('MiniMaxProvider.complete output and metering', () => { finish_reason: 'stop', }); expect((result as { usage: unknown }).usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); @@ -357,14 +357,14 @@ describe('MiniMaxProvider.complete output and metering', () => { expect(recordSpy).toHaveBeenCalledTimes(1); const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!; expect(usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); expect(actor).toBe(SYSTEM_ACTOR); expect(prefix).toBe('minimax:minimax-m2.7'); expect(overrides).toEqual({ - prompt_tokens: 100 * 30, + prompt_tokens: 90 * 30, completion_tokens: 50 * 120, cached_tokens: 10 * 6, }); diff --git a/src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.ts b/src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.ts index 740b33dbf..5713bbe14 100644 --- a/src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.ts +++ b/src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.ts @@ -28,6 +28,7 @@ import { openAICompatParams } from '../../utils/openaiParams.js'; import { MINIMAX_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; type MiniMaxConfig = { apiKey: string; @@ -108,26 +109,16 @@ export class MiniMaxProvider implements IChatProvider { ); return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = usage - ? OpenAIUtil.extractMeteredUsage(usage) - : { - prompt_tokens: 0, - completion_tokens: 0, - cached_tokens: 0, - }; - const costsOverride = Object.fromEntries( - Object.entries(trackedUsage).map(([key, value]) => { - return [key, value * Number(modelUsed.costs[key] ?? 0)]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor!, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, + actor, this.meteringModelKey(modelUsed.id), - costsOverride, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts b/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts index c0f2a4807..0e639f911 100644 --- a/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts +++ b/src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts @@ -39,6 +39,7 @@ import { MISTRAL_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { shouldPresentAsOpenAI } from '../../utils/normalizeToOpenAI.js'; import { withSdkTimeout } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; /** * Mistral's reasoning-capable models return `content` as a chunk array rather @@ -402,20 +403,18 @@ export class MistralAIProvider implements IChatProvider { }, completion: completion as ChatCompletionResponse, stream, - usage_calculator: ({ usage }) => { - const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverrideFromModel = Object.fromEntries( - Object.entries(trackedUsage).map(([k, v]) => { - return [k, v * selectedModel.costs[k]]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, + usage_calculator: ({ usage, setUsageCosts }) => { + // `coerceMistralUsage` has already taken cached reads out of + // `prompt_tokens`. + const metered = meterChatUsage( + this.#meteringService, actor, this.meteringModelKey(selectedModel.id), - costsOverrideFromModel, + selectedModel, + OpenAIUtil.extractMeteredUsage(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, }); } diff --git a/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.test.ts b/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.test.ts index 21602c204..c61b7aeee 100644 --- a/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.test.ts @@ -526,7 +526,7 @@ describe('MoonshotProvider.complete non-stream output', () => { finish_reason: 'stop', }); expect((result as { usage: unknown }).usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); @@ -536,14 +536,14 @@ describe('MoonshotProvider.complete non-stream output', () => { expect(recordSpy).toHaveBeenCalledTimes(1); const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!; expect(usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); expect(actor).toBe(SYSTEM_ACTOR); expect(prefix).toBe('moonshotai:kimi-k2.6'); expect(overrides).toEqual({ - prompt_tokens: 100 * Number(kimi.costs.prompt_tokens), + prompt_tokens: 90 * Number(kimi.costs.prompt_tokens), completion_tokens: 50 * Number(kimi.costs.completion_tokens), cached_tokens: 10 * Number(kimi.costs.cached_tokens ?? 0), }); @@ -667,7 +667,7 @@ describe('MoonshotProvider.complete streaming', () => { const usageEvent = events.find((e) => e.type === 'usage'); expect(usageEvent?.usage).toEqual({ - prompt_tokens: 4, + prompt_tokens: 3, completion_tokens: 2, cached_tokens: 1, }); @@ -677,7 +677,7 @@ describe('MoonshotProvider.complete streaming', () => { const [, , prefix, overrides] = recordSpy.mock.calls[0]!; expect(prefix).toBe('moonshotai:kimi-k2.6'); expect(overrides).toEqual({ - prompt_tokens: 4 * Number(kimi.costs.prompt_tokens), + prompt_tokens: 3 * Number(kimi.costs.prompt_tokens), completion_tokens: 2 * Number(kimi.costs.completion_tokens), cached_tokens: 1 * Number(kimi.costs.cached_tokens ?? 0), }); diff --git a/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.ts b/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.ts index fbf5cca2f..ed9308dfc 100644 --- a/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.ts +++ b/src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.ts @@ -31,6 +31,7 @@ import { inlineHttpImageUrls } from '../../utils/inlineImages.js'; import { MOONSHOT_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; export class MoonshotProvider implements IChatProvider { #openai: OpenAI; @@ -109,20 +110,16 @@ export class MoonshotProvider implements IChatProvider { } return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverride = Object.fromEntries( - Object.entries(trackedUsage).map(([key, value]) => { - return [key, value * modelUsed.costs[key]]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, actor, this.meteringModelKey(modelUsed.id), - costsOverride, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts index c75e1b573..bc8f54f50 100644 --- a/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts @@ -563,7 +563,7 @@ describe('NeuralwattProvider.complete non-stream output', () => { const [usage, , prefix, overrides] = recordSpy.mock.calls[0]!; expect(prefix).toBe('neuralwatt:deepseek-v4-flash'); expect(usage).toMatchObject({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, energy_kwh: 0.00000145, @@ -610,13 +610,14 @@ describe('NeuralwattProvider.complete non-stream output', () => { const completionRate = 0.28 * 100; const cachedRate = 0.014 * 100; const [usage, , , overrides] = recordSpy.mock.calls[0]!; + // Cached reads come out of `prompt_tokens`; they're priced once. expect(usage).toMatchObject({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); expect(overrides).toMatchObject({ - prompt_tokens: 100 * promptRate, + prompt_tokens: 90 * promptRate, completion_tokens: 50 * completionRate, cached_tokens: 10 * cachedRate, }); diff --git a/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts index bf474ea3f..3b52dc403 100644 --- a/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts +++ b/src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts @@ -48,6 +48,7 @@ import { type NeuralwattEnergy, } from './models.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; const DEFAULT_API_BASE_URL = 'https://api.neuralwatt.com/v1'; @@ -229,10 +230,12 @@ export class NeuralwattProvider implements IChatProvider { usage, cost, energy, + setUsageCosts, }: { usage: NeuralwattUsage; cost?: NeuralwattCost; energy?: NeuralwattEnergy; + setUsageCosts: (costs: Record) => void; }) => { // Non-streaming spreads the full completion into this call; // streaming merges top-level cost/energy onto `usage` via the @@ -250,7 +253,7 @@ export class NeuralwattProvider implements IChatProvider { measurement_available: usage.measurement_available, }; - const trackedTokens = OpenAIUtil.extractMeteredUsage(usage); + const trackedTokens = OpenAIUtil.splitCachedPrompt(usage); const energyUnits: Record = {}; if ( energyBlock.measurement_available !== false && @@ -277,48 +280,22 @@ export class NeuralwattProvider implements IChatProvider { return out; }; - if ( - typeof requestCostUsd === 'number' && - Number.isFinite(requestCostUsd) - ) { - const billedTrackedUsage = { - ...trackedTokens, - ...energyUnits, - billedUsage: 1, - }; - const costOverwrites = Object.fromEntries( - Object.keys(billedTrackedUsage).map((k) => [k, 0]), - ); - costOverwrites.billedUsage = requestCostUsd * 100_000_000; - this.#meteringService.utilRecordUsageObject( - billedTrackedUsage, - actor!, - this.meteringModelKey(modelUsed.id), - costOverwrites, - ); - const result = annotate(billedTrackedUsage); - result.usd_cents = requestCostUsd * 100; - return result; - } - - // Fallback: catalog token rates (preflight / when Neuralwatt - // omits request_cost_usd). - const trackedUsage = { ...trackedTokens, ...energyUnits }; - const costOverwrites = Object.fromEntries( - Object.entries(trackedUsage).map(([k, v]) => { - if (k === 'energy_kwh' || k === 'energy_joules') { - return [k, 0]; - } - return [k, (modelUsed.costs[k] ?? 0) * v]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor!, + // Billed at `request_cost_usd` when Neuralwatt reports it, at + // the catalog's token rates otherwise. Energy is recorded, never + // priced. + const metered = meterChatUsage( + this.#meteringService, + actor, this.meteringModelKey(modelUsed.id), - costOverwrites, + modelUsed, + { ...trackedTokens, ...energyUnits }, + typeof requestCostUsd === 'number' && + Number.isFinite(requestCostUsd) + ? { authoritativeUsd: requestCostUsd } + : { costOverrides: { energy_kwh: 0, energy_joules: 0 } }, ); - return annotate(trackedUsage); + setUsageCosts(metered.costs); + return annotate(metered.usage); }; return OpenAIUtil.handle_completion_output({ diff --git a/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts b/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts index 909c4cbd8..0a0578868 100644 --- a/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts +++ b/src/backend/drivers/ai-chat/providers/neuralwatt/models.ts @@ -106,10 +106,13 @@ export const mapNeuralwattApiModel = ( const inputUsd = Number(pricing?.input_per_million ?? 0); const outputUsd = Number(pricing?.output_per_million ?? 0); - const cachedUsd = - pricing?.cached_input_per_million == null - ? 0 - : Number(pricing.cached_input_per_million); + const costs = usdPerMToken( + inputUsd, + outputUsd, + Number(pricing?.cached_input_per_million ?? 0), + ); + // No published cache price: cached reads bill at the input rate. + if (pricing?.cached_input_per_million == null) delete costs.cached_tokens; const caps = model.metadata?.capabilities; const limits = model.metadata?.limits; @@ -138,7 +141,7 @@ export const mapNeuralwattApiModel = ( costs_currency: 'usd-cents', input_cost_key: 'prompt_tokens', output_cost_key: 'completion_tokens', - costs: usdPerMToken(inputUsd, outputUsd, cachedUsd), + costs, modalities: { input: inputModalities, output: ['text'] }, tool_call: caps?.tools === true, // Surfaced from Neuralwatt `/models` so `complete()` can gate diff --git a/src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts b/src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts index ae2ea9550..42c9f4ac1 100644 --- a/src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts +++ b/src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts @@ -26,6 +26,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js'; import { IChatModel, IChatProvider, ICompleteArguments } from '../../types.js'; import { ChatCompletionCreateParams } from 'openai/resources/index.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; /** * OllamaService class - Provides integration with Ollama's API for chat * completions Extends BaseService to implement the puter-chat-completion @@ -151,7 +152,7 @@ export class OllamaChatProvider implements IChatProvider { : `ollama:ollama/${model}` : undefined); return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { + usage_calculator: ({ usage, setUsageCosts }) => { const trackedUsage = { prompt: (usage.prompt_tokens ?? 1) - @@ -160,20 +161,24 @@ export class OllamaChatProvider implements IChatProvider { input_cache_read: usage.prompt_tokens_details?.cached_tokens ?? 0, }; - const costOverwrites = Object.fromEntries( - Object.keys(trackedUsage).map((k) => { - return [k, 0]; // override to 0 since local is free - }), + if (!modelIdForMetering) return trackedUsage; + // Local inference is free. + const metered = meterChatUsage( + this.#meteringService, + actor, + modelIdForMetering, + modelDetails ?? ({ costs: {} } as unknown as IChatModel), + trackedUsage, + { + costOverrides: { + prompt: 0, + completion: 0, + input_cache_read: 0, + }, + }, ); - if (modelIdForMetering) { - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor, - modelIdForMetering, - costOverwrites, - ); - } - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts index 08ad6ad1a..606782467 100644 --- a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts @@ -40,13 +40,13 @@ import { clampReasoningEffort, openAICompatParams, } from '../../utils/openaiParams.js'; -import { buildCostsOverride } from '../../utils/pricing.js'; import { processPuterPathUploads } from './fileUpload.js'; import { OPEN_AI_MODELS } from './models.js'; import type { OpenAiResponsesChatProvider } from './OpenAiChatResponsesProvider.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; const isWebSearchTool = (tool: Record): boolean => tool.type === 'web_search' || @@ -284,7 +284,7 @@ export class OpenAiChatProvider implements IChatProvider { ); return OpenAiUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { + usage_calculator: ({ usage, setUsageCosts }) => { const cachedTokens = usage.prompt_tokens_details?.cached_tokens ?? 0; // GPT-5.6 and later bill cache writes at 1.25x input. They're @@ -307,18 +307,15 @@ export class OpenAiChatProvider implements IChatProvider { : {}), }; - const costsOverrideFromModel = buildCostsOverride( - trackedUsage, - modelUsed, - ); - - this.#meteringService.utilRecordUsageObject( - trackedUsage, + const metered = meterChatUsage( + this.#meteringService, actor, - this.meteringModelKey(modelUsed?.id), - costsOverrideFromModel, + this.meteringModelKey(modelUsed.id), + modelUsed, + trackedUsage, ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts index e5cd4941d..c9421d033 100644 --- a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts @@ -37,7 +37,6 @@ import { openAICompatParams, rejectStatefulResponsesFields, } from '../../utils/openaiParams.js'; -import { buildCostsOverride } from '../../utils/pricing.js'; import { responseSamplingParams } from '../../utils/responseSampling.js'; import { processPuterPathUploads } from './fileUpload.js'; import { OPEN_AI_MODELS } from './models.js'; @@ -45,6 +44,7 @@ import { HttpError } from '@heyputer/backend/src/core/http/HttpError.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; const ANTHROPIC_WEB_SEARCH_TYPE = (type: unknown): boolean => type === 'web_search_20250305' || @@ -362,32 +362,30 @@ export class OpenAiResponsesChatProvider implements IChatProvider { : {}), }; - const costsOverrideFromModel = buildCostsOverride( - trackedUsage, + const metered = meterChatUsage( + this.#meteringService, + actor, + this.meteringModelKey(modelUsed.id), modelUsed, - ); - // Priced dynamically: the rate depends on which - // web-search tool variant was requested and whether the model - // is a reasoning model, not a static per-model cost-table - // entry — `buildCostsOverride`'s fallback would otherwise - // price it (wrongly) at the input/output token rate. - if (webSearchCalls) { - const rate = webSearchCallRate(tools, modelUsed.id) ?? 0; - costsOverrideFromModel.web_search_calls = - webSearchCalls * rate; - setUsageCosts?.({ - web_search_calls: - costsOverrideFromModel.web_search_calls, - }); - } - - this.#meteringService.utilRecordUsageObject( trackedUsage, - actor!, - this.meteringModelKey(modelUsed?.id), - costsOverrideFromModel, + // Priced dynamically: the rate depends on which + // web-search tool variant was requested and whether the + // model is a reasoning model, not a static per-model + // cost-table entry — the model's rates would otherwise + // price it (wrongly) at the input/output token rate. + webSearchCalls + ? { + costOverrides: { + web_search_calls: + webSearchCalls * + (webSearchCallRate(tools, modelUsed.id) ?? + 0), + }, + } + : {}, ); - return trackedUsage; + setUsageCosts?.(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts index 0e277a02a..c6d9bd171 100644 --- a/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts +++ b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts @@ -39,6 +39,7 @@ import type { } from '../../types.js'; import { OPEN_ROUTER_MODEL_OVERRIDES } from './modelOverrides.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; type OpenrouterUsage = OpenAI.Completions.CompletionUsage & { cost?: number; @@ -200,62 +201,38 @@ export class OpenRouterProvider implements IChatProvider { } return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }: { usage: OpenrouterUsage }) => { - if (typeof usage.cost === 'number') { - // custom open router logic because they're pricing are weird - const trackedUsage = { - prompt: - (usage.prompt_tokens ?? 0) - - (usage.prompt_tokens_details?.cached_tokens ?? 0), - completion: usage.completion_tokens ?? 0, - input_cache_read: - usage.prompt_tokens_details?.cached_tokens ?? 0, - request: - (usage as unknown as Record) - .request || 1, - billedUsage: 1, - }; - const costOverwrites = Object.fromEntries( - Object.keys(trackedUsage).map((k) => { - return [k, 0]; // make everything else 0 if they don't respect their own pricing - }), - ); - costOverwrites.billedUsage = usage.cost * 100_000_000 || 1; - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor, - this.meteringModelKey(modelUsed.id), - costOverwrites, - ); - (trackedUsage as Record).usd_cents = - usage.cost * 100; - return trackedUsage; - } else { - // custom open router logic because they're pricing are weird - const trackedUsage: Record = { - prompt: - (usage.prompt_tokens ?? 0) - - (usage.prompt_tokens_details?.cached_tokens ?? 0), - completion: usage.completion_tokens ?? 0, - input_cache_read: - usage.prompt_tokens_details?.cached_tokens ?? 0, - request: - (usage as unknown as Record) - .request || 1, - }; - const costOverwrites = Object.fromEntries( - Object.keys(trackedUsage).map((k) => { - return [k, modelUsed.costs[k] * trackedUsage[k]]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor, - this.meteringModelKey(modelUsed.id), - costOverwrites, - ); - return trackedUsage; - } + usage_calculator: ({ + usage, + setUsageCosts, + }: { + usage: OpenrouterUsage; + setUsageCosts: (costs: Record) => void; + }) => { + const trackedUsage = { + prompt: + (usage.prompt_tokens ?? 0) - + (usage.prompt_tokens_details?.cached_tokens ?? 0), + completion: usage.completion_tokens ?? 0, + input_cache_read: + usage.prompt_tokens_details?.cached_tokens ?? 0, + request: + (usage as unknown as Record).request || + 1, + }; + // OpenRouter reports what it billed (`usage.cost`, USD); + // the catalog's prices are the fallback. + const metered = meterChatUsage( + this.#meteringService, + actor, + this.meteringModelKey(modelUsed.id), + modelUsed, + trackedUsage, + typeof usage.cost === 'number' + ? { authoritativeUsd: usage.cost } + : {}, + ); + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts index e2d30a29a..0c09a1a80 100644 --- a/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts +++ b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts @@ -29,11 +29,7 @@ import { isContextLengthError, } from '../../utils/contextLimit.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; - -const TOGETHER_AI_CHAT_COST_MAP: Record = { - prompt_tokens: 'input', - completion_tokens: 'output', -}; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; export class TogetherAIProvider implements IChatProvider { #together: Together; @@ -178,22 +174,16 @@ export class TogetherAIProvider implements IChatProvider { } return OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = OpenAIUtil.extractMeteredUsage(usage); - const costsOverride = Object.fromEntries( - Object.entries(trackedUsage).map(([k, v]) => { - const mappedKey = TOGETHER_AI_CHAT_COST_MAP[k] || k; - return [k, v * modelUsed.costs[mappedKey]]; - }), - ); - - this.#meteringService.utilRecordUsageObject( - trackedUsage, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, actor, this.meteringModelKey(modelUsed.id), - costsOverride, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.test.ts b/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.test.ts index cff7ed4ec..cd298e6db 100644 --- a/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.test.ts @@ -591,7 +591,7 @@ describe('ZAIProvider.complete non-stream output', () => { finish_reason: 'stop', }); expect((result as { usage: unknown }).usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); @@ -603,14 +603,14 @@ describe('ZAIProvider.complete non-stream output', () => { expect(recordSpy).toHaveBeenCalledTimes(1); const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!; expect(usage).toEqual({ - prompt_tokens: 100, + prompt_tokens: 90, completion_tokens: 50, cached_tokens: 10, }); expect(actor).toBe(SYSTEM_ACTOR); expect(prefix).toBe('zai:glm-4.6'); expect(overrides.prompt_tokens).toBeCloseTo( - 100 * Number(glm46.costs.prompt_tokens), + 90 * Number(glm46.costs.prompt_tokens), 5, ); expect(overrides.completion_tokens).toBeCloseTo( @@ -799,7 +799,7 @@ describe('ZAIProvider.complete streaming', () => { const usageEvent = events.find((e) => e.type === 'usage'); expect(usageEvent?.usage).toEqual({ - prompt_tokens: 4, + prompt_tokens: 3, completion_tokens: 2, cached_tokens: 1, }); @@ -809,7 +809,7 @@ describe('ZAIProvider.complete streaming', () => { const [, , prefix, overrides] = recordSpy.mock.calls[0]!; expect(prefix).toBe('zai:glm-4.6'); expect(overrides.prompt_tokens).toBeCloseTo( - 4 * Number(glm46.costs.prompt_tokens), + 3 * Number(glm46.costs.prompt_tokens), 5, ); expect(overrides.completion_tokens).toBeCloseTo( diff --git a/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts b/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts index 8a606ebce..c85019301 100644 --- a/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts +++ b/src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts @@ -29,6 +29,7 @@ import { ZAI_MODELS } from './models.js'; import { modelLookupNames } from '../../utils/modelRouting.js'; import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; +import { meterChatUsage } from '../../utils/meterChatUsage.js'; // Z.AI documents `user_id` as 6-128 characters. const USER_ID_MAX_LENGTH = 128; @@ -153,26 +154,16 @@ export class ZAIProvider implements IChatProvider { ); const result = await OpenAIUtil.handle_completion_output({ - usage_calculator: ({ usage }) => { - const trackedUsage = usage - ? OpenAIUtil.extractMeteredUsage(usage) - : { - prompt_tokens: 0, - completion_tokens: 0, - cached_tokens: 0, - }; - const costsOverrideFromModel = Object.fromEntries( - Object.entries(trackedUsage).map(([key, value]) => { - return [key, value * Number(modelUsed.costs[key] ?? 0)]; - }), - ); - this.#meteringService.utilRecordUsageObject( - trackedUsage, - actor!, + usage_calculator: ({ usage, setUsageCosts }) => { + const metered = meterChatUsage( + this.#meteringService, + actor, this.meteringModelKey(modelUsed.id), - costsOverrideFromModel, + modelUsed, + OpenAIUtil.splitCachedPrompt(usage), ); - return trackedUsage; + setUsageCosts(metered.costs); + return metered.usage; }, stream, completion, diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js index 438cb700d..fbdc34276 100644 --- a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js @@ -487,6 +487,23 @@ export const extractMeteredUsage = (usage) => { }; }; +/** + * Chat Completions usage as it's metered: `prompt_tokens` reports cached reads + * inside its count, so they come out of it and are priced only once, under + * `cached_tokens`. + * + * @param {import('openai/resources/completions.mjs').CompletionUsage + * | undefined} usage + */ +export const splitCachedPrompt = (usage) => { + const cachedTokens = usage?.prompt_tokens_details?.cached_tokens ?? 0; + return { + prompt_tokens: (usage?.prompt_tokens ?? 0) - cachedTokens, + completion_tokens: usage?.completion_tokens ?? 0, + cached_tokens: cachedTokens, + }; +}; + /** * The `{prompt_tokens, completion_tokens, cached_tokens, cache_write_tokens}` * shape every OpenAI-family `usage_calculator` already returns for billing, @@ -674,6 +691,7 @@ export const create_chat_stream_handler = ? usage_calculator({ usage: last_usage, extra_content: last_extra_content, + setUsageCosts: (c) => chatStream.setUsageCosts(c), }) : undefined; // The calculator just metered. Reported here, not only via `end`: @@ -875,6 +893,7 @@ export const handle_completion_output = async ( * @type {Record & { * usage_calculator: (args: { * usage: import('openai/resources/completions.mjs').CompletionUsage; + * setUsageCosts: (costs: Record) => void; * }) => unknown; * }} */ @@ -927,6 +946,9 @@ export const handle_completion_output = async ( ? usage_calculator({ ...completion, usage: completion_usage, + setUsageCosts: (c) => { + ret.usageCosts = c; + }, }) : { input_tokens: completion_usage.prompt_tokens, diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts b/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts index 6917e1537..3a810529d 100644 --- a/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts @@ -31,6 +31,7 @@ import { handle_completion_output_responses_api, process_input_messages, process_input_messages_responses_api, + splitCachedPrompt, toOpenAIChatMessages, usageDetailsFromTrackedUsage, } from './OpenAIUtil.js'; @@ -833,6 +834,73 @@ describe('extractMeteredUsage', () => { }); }); +// ── splitCachedPrompt ─────────────────────────────────────────────── + +describe('splitCachedPrompt', () => { + it('takes cached reads out of prompt_tokens', () => { + expect( + splitCachedPrompt({ + prompt_tokens: 10, + completion_tokens: 5, + prompt_tokens_details: { cached_tokens: 3 }, + }), + ).toEqual({ + prompt_tokens: 7, + completion_tokens: 5, + cached_tokens: 3, + }); + }); + + it('reads a missing usage as nothing used', () => { + expect(splitCachedPrompt(undefined)).toEqual({ + prompt_tokens: 0, + completion_tokens: 0, + cached_tokens: 0, + }); + }); +}); + +// ── recorded usage costs ──────────────────────────────────────────── + +describe('chat completion handlers carry the recorded usage costs', () => { + const usage_calculator = ({ + usage, + setUsageCosts, + }: { + usage: Record; + setUsageCosts: (costs: Record) => void; + }) => { + setUsageCosts({ prompt_tokens: 42 }); + return usage; + }; + + it('onto a non-stream result', async () => { + const result = await handle_completion_output({ + stream: false, + completion: { + choices: [{ message: { content: 'hi' } }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }, + usage_calculator, + }); + expect(result.usageCosts).toEqual({ prompt_tokens: 42 }); + }); + + it('onto the chat stream', async () => { + const { chatStream } = makeCapturingChatStream(); + await create_chat_stream_handler({ + completion: asAsyncIterable([ + { + choices: [{ delta: { content: 'hi' }, finish_reason: 'stop' }], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }, + ]), + usage_calculator, + })({ chatStream }); + expect(chatStream.usageCosts).toEqual({ prompt_tokens: 42 }); + }); +}); + // ── create_chat_stream_handler ────────────────────────────────────── describe('create_chat_stream_handler', () => { diff --git a/src/backend/drivers/ai-chat/utils/meterChatUsage.test.ts b/src/backend/drivers/ai-chat/utils/meterChatUsage.test.ts new file mode 100644 index 000000000..b19e3f065 --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/meterChatUsage.test.ts @@ -0,0 +1,167 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { describe, expect, it } from 'vitest'; +import type { Actor } from '../../../core/actor.js'; +import { ALIBABA_MODELS } from '../providers/alibaba/models.js'; +import { BYTEPLUS_MODELS } from '../providers/byteplus/models.js'; +import { DEEPSEEK_MODELS } from '../providers/deepseek/models.js'; +import { GROQ_MODELS } from '../providers/groq/models.js'; +import { HOONIFY_MODELS } from '../providers/hoonify/models.js'; +import { MINIMAX_MODELS } from '../providers/minimax/models.js'; +import { MISTRAL_MODELS } from '../providers/mistral/models.js'; +import { MOONSHOT_MODELS } from '../providers/moonshot/models.js'; +import { ZAI_MODELS } from '../providers/zai/models.js'; +import type { IChatModel } from '../types.js'; +import { meterChatUsage } from './meterChatUsage.js'; +import { splitCachedPrompt } from './OpenAIUtil.js'; + +type Recorded = [Record, string, Record]; + +const recordingMetering = () => { + const calls: Recorded[] = []; + return { + calls, + metering: { + utilRecordUsageObject: ( + usage: Record, + _actor: Actor, + key: string, + costs?: Record, + ) => { + calls.push([usage, key, costs ?? {}]); + return Promise.resolve({ total: 0 } as never); + }, + }, + }; +}; + +const actor = {} as Actor; +const model = { + id: 'm', + costs_currency: 'usd-cents', + input_cost_key: 'prompt_tokens', + output_cost_key: 'completion_tokens', + costs: { tokens: 1_000_000, prompt_tokens: 3, completion_tokens: 15 }, + max_tokens: 1024, +} as IChatModel; + +describe('meterChatUsage', () => { + it('records the usage at the model rates and returns what it recorded', () => { + const { calls, metering } = recordingMetering(); + const usage = { prompt_tokens: 10, completion_tokens: 2 }; + + const result = meterChatUsage(metering, actor, 'p:m', model, usage); + + expect(calls).toEqual([ + [usage, 'p:m', { prompt_tokens: 30, completion_tokens: 30 }], + ]); + expect(result).toEqual({ + usage, + costs: { prompt_tokens: 30, completion_tokens: 30 }, + }); + }); + + it('lets a cost override win over the model rates', () => { + const { calls, metering } = recordingMetering(); + meterChatUsage( + metering, + actor, + 'p:m', + model, + { prompt_tokens: 10, web_search_calls: 2 }, + { costOverrides: { web_search_calls: 1000 } }, + ); + expect(calls[0]![2]).toEqual({ + prompt_tokens: 30, + web_search_calls: 1000, + }); + }); + + it('bills an authoritative USD cost as one line and zeroes the tokens', () => { + const { calls, metering } = recordingMetering(); + const result = meterChatUsage( + metering, + actor, + 'gw:m', + model, + { prompt: 10, completion: 2 }, + { authoritativeUsd: 0.25 }, + ); + + expect(calls).toEqual([ + [ + { prompt: 10, completion: 2, billedUsage: 1 }, + 'gw:m', + { prompt: 0, completion: 0, billedUsage: 25_000_000 }, + ], + ]); + expect(result.usage).toEqual({ + prompt: 10, + completion: 2, + billedUsage: 1, + usd_cents: 25, + }); + expect(result.costs.billedUsage).toBe(25_000_000); + }); +}); + +// Every catalog the hand-rolled `amount * costs[key]` pricing used to bill. +// A cached read is priced once, at the model's cache rate, or at its input +// rate when it publishes none; it is never free on a paid model. +describe('cached reads on OpenAI-compatible catalogs', () => { + const catalogs: [string, readonly IChatModel[]][] = [ + ['alibaba', ALIBABA_MODELS], + ['byteplus', BYTEPLUS_MODELS], + ['deepseek', DEEPSEEK_MODELS], + ['groq', GROQ_MODELS], + ['hoonify', HOONIFY_MODELS], + ['minimax', MINIMAX_MODELS], + ['mistral', MISTRAL_MODELS], + ['moonshotai', MOONSHOT_MODELS], + ['zai', ZAI_MODELS], + ]; + const cases = catalogs.flatMap(([provider, models]) => + models.map((m) => [`${provider}:${m.id}`, m] as const), + ); + + it.each(cases)('%s', (_name, m) => { + const { calls, metering } = recordingMetering(); + meterChatUsage( + metering, + actor, + 'k', + m, + splitCachedPrompt({ + prompt_tokens: 1_000_000, + completion_tokens: 100_000, + prompt_tokens_details: { cached_tokens: 200_000 }, + total_tokens: 1_100_000, + }), + ); + const costs = m.costs as Record; + const cachedRate = costs.cached_tokens ?? costs.prompt_tokens; + expect(calls[0]![2]).toEqual({ + prompt_tokens: 800_000 * costs.prompt_tokens, + completion_tokens: 100_000 * costs.completion_tokens, + cached_tokens: 200_000 * cachedRate, + }); + if (costs.prompt_tokens > 0) expect(cachedRate).toBeGreaterThan(0); + }); +}); diff --git a/src/backend/drivers/ai-chat/utils/meterChatUsage.ts b/src/backend/drivers/ai-chat/utils/meterChatUsage.ts new file mode 100644 index 000000000..ae1ec2069 --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/meterChatUsage.ts @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import type { Actor } from '../../../core/actor.js'; +import type { MeteringService } from '../../../services/metering/MeteringService.js'; +import type { IChatModel } from '../types.js'; +import { buildCostsOverride } from './pricing.js'; + +/** Microcents per US dollar. */ +const MICROCENTS_PER_USD = 100_000_000; + +export interface MeterChatUsageOptions { + /** + * What the upstream says it billed, in USD. Recorded as one `billedUsage` + * line instead of pricing the tokens, which are still recorded at zero. + */ + authoritativeUsd?: number; + /** Per-key µ¢ costs that win over the model's rates. */ + costOverrides?: Record; +} + +/** + * Records one completion's usage under `meteringKey` and returns the usage to + * report along with the µ¢ each key was recorded at, so the driver can quote + * the same cost the ledger holds. + */ +export const meterChatUsage = ( + metering: Pick, + actor: Actor | undefined, + meteringKey: string, + model: IChatModel, + usage: Record, + { authoritativeUsd, costOverrides }: MeterChatUsageOptions = {}, +): { usage: Record; costs: Record } => { + if (authoritativeUsd !== undefined) { + const billed = { ...usage, billedUsage: 1 }; + const costs: Record = Object.fromEntries( + Object.keys(billed).map((key) => [key, 0]), + ); + costs.billedUsage = authoritativeUsd * MICROCENTS_PER_USD; + metering.utilRecordUsageObject(billed, actor!, meteringKey, costs); + return { + usage: { ...billed, usd_cents: authoritativeUsd * 100 }, + costs, + }; + } + const costs = { ...buildCostsOverride(usage, model), ...costOverrides }; + metering.utilRecordUsageObject(usage, actor!, meteringKey, costs); + return { usage, costs }; +}; diff --git a/src/backend/drivers/ai-chat/utils/pricing.test.ts b/src/backend/drivers/ai-chat/utils/pricing.test.ts index 20048545c..4e0e42112 100644 --- a/src/backend/drivers/ai-chat/utils/pricing.test.ts +++ b/src/backend/drivers/ai-chat/utils/pricing.test.ts @@ -26,6 +26,7 @@ import { longContextMultipliers, trackedInputTokens, trackedOutputTokens, + usageCost, usageDetailsFromUsage, usdPerMToken, } from './pricing.js'; @@ -350,3 +351,82 @@ describe('long-context pricing', () => { }); }); }); + +describe('usageCost', () => { + const rates = model({ + prompt_tokens: 3, + completion_tokens: 15, + cached_tokens: 0.3, + }); + + it('splits the priced usage into input and output sides', () => { + expect( + usageCost( + { prompt_tokens: 100, completion_tokens: 10, cached_tokens: 50 }, + rates, + ), + ).toEqual({ inputMicroCents: 315, outputMicroCents: 150 }); + }); + + it('agrees with what buildCostsOverride records', () => { + const usage = { + prompt_tokens: 1234, + completion_tokens: 567, + cached_tokens: 89, + grounding_requests: 1, + }; + const recorded = Object.values(buildCostsOverride(usage, rates)).reduce( + (a, b) => a + b, + 0, + ); + const cost = usageCost(usage, rates)!; + expect(cost.inputMicroCents + cost.outputMicroCents).toBe( + Math.round(recorded), + ); + }); + + it('takes the costs a provider recorded over the model rates', () => { + expect( + usageCost( + { prompt_tokens: 100, billedUsage: 1, usd_cents: 0.5 }, + rates, + { prompt_tokens: 0, billedUsage: 500_000 }, + ), + ).toEqual({ inputMicroCents: 500_000, outputMicroCents: 0 }); + }); + + it('prices advisor usage only through a recorded cost', () => { + expect( + usageCost( + { prompt_tokens: 10, advisor_output_tokens: 1000 }, + rates, + ), + ).toEqual({ inputMicroCents: 30, outputMicroCents: 0 }); + expect( + usageCost( + { prompt_tokens: 10, advisor_output_tokens: 1000 }, + rates, + { advisor_output_tokens: 7 }, + ), + ).toEqual({ inputMicroCents: 30, outputMicroCents: 7 }); + }); + + it('is null when nothing in the usage can be priced', () => { + expect( + usageCost({ prompt_tokens: 10, completion_tokens: 5 }, model({})), + ).toBeNull(); + }); + + it('skips usd_cents and non-numeric annotations', () => { + expect( + usageCost( + { + prompt_tokens: 10, + usd_cents: 99, + accounting_method: 'energy', + }, + rates, + ), + ).toEqual({ inputMicroCents: 30, outputMicroCents: 0 }); + }); +}); diff --git a/src/backend/drivers/ai-chat/utils/pricing.ts b/src/backend/drivers/ai-chat/utils/pricing.ts index be8edb1c3..87d6f24b8 100644 --- a/src/backend/drivers/ai-chat/utils/pricing.ts +++ b/src/backend/drivers/ai-chat/utils/pricing.ts @@ -161,8 +161,8 @@ export const isFreeModel = (model: IChatModel): boolean => { * A fallback [[UsageDetails]] for providers that haven't been taught to report * their own (every non-Claude provider, for now). `inputTokens` and * `outputTokens` reuse the same tracked-token accounting the credit gate and - * `#computeCost` already agree on, so a provider with no detail of its own - * still gets a usable breakdown. + * `usageCost` already agree on, so a provider with no detail of its own still + * gets a usable breakdown. */ export const usageDetailsFromUsage = ( usage: Record, @@ -172,6 +172,52 @@ export const usageDetailsFromUsage = ( outputTokens: trackedOutputTokens(usage, model), }); +/** A usage key/amount pair that carries a price, rather than a scale or a label. */ +const isBillable = (key: string, amount: unknown): amount is number => + key !== 'tokens' && key !== 'usd_cents' && isRate(amount); + +/** + * The µ¢ each usage key costs at the model's rates. A key the model doesn't + * price falls back to the output rate when it is output-denominated and the + * input rate otherwise; a key neither covers is left out. + */ +const priceUsage = ( + trackedUsage: Record, + model: IChatModel, +): Record => { + const { inputKey, outputKey } = costKeys(model); + + const costs = model.costs ?? {}; + const inputRate = isRate(costs[inputKey]) ? costs[inputKey] : undefined; + const outputRate = isRate(costs[outputKey]) ? costs[outputKey] : undefined; + + const multipliers = longContextMultipliers( + model, + trackedInputTokens(trackedUsage, model), + ); + + const priced: Record = {}; + for (const [key, amount] of Object.entries(trackedUsage)) { + if (!isBillable(key, amount)) continue; + + const isOutput = isOutputCostKey(key, outputKey); + const rate = isRate(costs[key]) + ? costs[key] + : isOutput + ? outputRate + : inputRate; + if (rate === undefined) continue; + + const multiplier = isPerCallCostKey(key) + ? 1 + : isOutput + ? multipliers.output + : multipliers.input; + priced[key] = amount * rate * multiplier; + } + return priced; +}; + /** * Prices a tracked-usage object against a model's cost table. * @@ -187,35 +233,51 @@ export const buildCostsOverride = ( trackedUsage: Record, model: IChatModel, ): Record => { - const { inputKey, outputKey } = costKeys(model); - - const costs = model.costs ?? {}; - const inputRate = isRate(costs[inputKey]) ? costs[inputKey] : undefined; - const outputRate = isRate(costs[outputKey]) ? costs[outputKey] : undefined; - - const multipliers = longContextMultipliers( - model, - trackedInputTokens(trackedUsage, model), - ); - + const priced = priceUsage(trackedUsage, model); const overrides: Record = {}; - for (const [key, amount] of Object.entries(trackedUsage)) { + for (const key of Object.keys(trackedUsage)) { // `tokens` is a scale descriptor ("costs expressed per N tokens"), // not a per-unit rate. if (key === 'tokens') continue; - - const isOutput = isOutputCostKey(key, outputKey); - const rate = isRate(costs[key]) - ? costs[key] - : ((isOutput ? outputRate : inputRate) ?? 0); - - const multiplier = isPerCallCostKey(key) - ? 1 - : isOutput - ? multipliers.output - : multipliers.input; - overrides[key] = amount * rate * multiplier; + overrides[key] = priced[key] ?? 0; } - return overrides; }; + +/** + * What a usage object costs in µ¢, split into its input and output sides. + * `recordedCosts` are the per-key costs a provider already metered with and win + * over the model's rates, so the figure matches the ledger. Null when nothing + * in the usage could be priced. + */ +export const usageCost = ( + usage: Record, + model: IChatModel, + recordedCosts?: Record, +): { inputMicroCents: number; outputMicroCents: number } | null => { + const { outputKey } = costKeys(model); + const priced = priceUsage(usage, model); + let input = 0; + let output = 0; + let sawAnyCost = false; + for (const [key, amount] of Object.entries(usage)) { + if (!isBillable(key, amount)) continue; + const recorded = recordedCosts?.[key]; + // Advisor usage is priced at the advisor model's own rates, so only a + // recorded cost can price it here. + const cost = isRate(recorded) + ? recorded + : key.startsWith('advisor_') + ? undefined + : priced[key]; + if (cost === undefined) continue; + sawAnyCost = true; + if (isOutputCostKey(key, outputKey)) output += cost; + else input += cost; + } + if (!sawAnyCost) return null; + return { + inputMicroCents: Math.max(0, Math.round(input)), + outputMicroCents: Math.max(0, Math.round(output)), + }; +};