lee/puter
1
0 Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2026-10-13 23:29:50 +00:00
Code Issues Actions 8 Packages Projects Releases Wiki Activity

refactor(ai-chat): meter every OpenAI-family completion through one pricing path

Browse Source 
meterChatUsage records a completion's usage at buildCostsOverride's rates, or
as one billedUsage line when the gateway reports what it billed, and returns
the per-key costs it recorded. Providers hand those to the driver through the
usageCosts channel, now carried by the Chat Completions handlers too, so the
usd_cents and cost-calculated figures equal the ledger. The driver's own
#computeCost is replaced by pricing.ts usageCost, which shares the rate
resolution with buildCostsOverride.

Billing changes, all toward what the upstream charges:
- DeepSeek, Moonshot, Alibaba, Groq, MiniMax, Z.AI, Hoonify, BytePlus and the
  Neuralwatt catalog fallback took cached reads out of prompt_tokens nowhere,
  so a cached token was billed at input + cache rate. splitCachedPrompt bills
  it once, at the cache rate; usage.prompt_tokens on those routes now excludes
  cached reads, as it already did for OpenAI, Gemini and Meta.
- Catalog entries that listed cached_tokens: 0 (38 Alibaba, 5 Groq) relied on
  that double count to bill cached reads at the input rate. The zero is
  dropped, so they keep exactly that price through the input-rate fallback.
  Neuralwatt models with no published cache price likewise.
- An unpriced key falls back to the input/output rate instead of NaN (0), e.g.
  OpenRouter cached reads on its no-cost fallback path.
- A gateway-reported cost of 0 records 0 µ¢ rather than 1 µ¢.

Claude, Azure and xAI keep their metering; xAI's cached-token handling waits
on the open Grok decision.
This commit is contained in:
Daniel Salazar committed 2026-10-10 19:05:48 -04:00
1 parent c6b2d5dd42
commit fc25ad21fc
36 files changed
+793 -597
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/backend/drivers/ai-chat/ChatCompletionDriver.ts
+22 -129
View File
@@ -95,11 +95,9 @@ import { isToolChoice, toolChoiceFromWire } from './utils/openaiParams.js';
import {
costKeys,
isFreeModel,
isOutputCostKey,
isPerCallCostKey,
longContextMultipliers,
trackedInputTokens,
trackedOutputTokens,
usageCost,
usageDetailsFromUsage,
} from './utils/pricing.js';
import {
@@ -1218,126 +1216,6 @@ export class ChatCompletionDriver extends PuterDriver {
};
}
// Compute per-token cost in microcents (1 cent = 1_000_000 microCents).
// Shape-agnostic: multiplies every usage key by its matching rate in
// `model.costs`. Returns `null` when cost data is unavailable.
//
// `usageCosts`, when given, carries the provider's own per-key µ¢ cost
// (set from `chatStream.usageCosts` / `res.usageCosts`) — used verbatim
// for that key so the ledger and the reported cost match exactly instead
// of being recomputed off this model's cost table.
#computeCost(
usage: Record<string, number>,
model: IChatModel,
usageCosts?: Record<string, number>,
): {
inputKey: string;
outputKey: string;
inputTokens: number;
outputTokens: number;
inputMicroCents: number;
outputMicroCents: number;
totalMicroCents: number;
} | null {
const { inputKey, outputKey } = costKeys(model);
const costs = model.costs;
if (!costs) return null;
const outputRateRaw = costs[outputKey];
const outputRate =
typeof outputRateRaw === 'number' && Number.isFinite(outputRateRaw)
? outputRateRaw
: undefined;
const isOutputKey = (key: string) => isOutputCostKey(key, outputKey);
const multipliers = longContextMultipliers(
model,
trackedInputTokens(usage, model),
);
let inputMicroCents = 0;
let outputMicroCents = 0;
let sawAnyRate = false;
for (const [key, rawAmount] of Object.entries(usage)) {
if (typeof rawAmount !== 'number' || !Number.isFinite(rawAmount)) {
continue;
}
if (key === 'usd_cents') continue;
if (key === 'tokens') continue;
const override = usageCosts?.[key];
if (typeof override === 'number' && Number.isFinite(override)) {
sawAnyRate = true;
if (isOutputKey(key)) outputMicroCents += override;
else inputMicroCents += override;
continue;
}
// Advisor usage is priced at the advisor model's own rates
// (metered separately, under `claude:<advisor>`), never off this
// (executor) model's cost table — without an override there is
// simply nothing to price it at here.
if (key.startsWith('advisor_')) continue;
// thinking_tokens → output rate fallback
let rate = costs[key];
if (typeof rate !== 'number' || !Number.isFinite(rate)) {
if (isOutputKey(key) && outputRate !== undefined) {
rate = outputRate;
} else if (!isOutputKey(key)) {
const inputRateRaw = costs[inputKey];
if (
typeof inputRateRaw === 'number' &&
Number.isFinite(inputRateRaw)
) {
rate = inputRateRaw;
} else {
continue;
}
} else {
continue;
}
}
sawAnyRate = true;
if (isOutputKey(key)) {
outputMicroCents += rawAmount * rate * multipliers.output;
} else {
inputMicroCents +=
rawAmount *
rate *
(isPerCallCostKey(key) ? 1 : multipliers.input);
}
}
if (!sawAnyRate) return null;
inputMicroCents = Math.max(0, Math.round(inputMicroCents));
outputMicroCents = Math.max(0, Math.round(outputMicroCents));
const inputTokens = Number(
usage[inputKey] ?? usage.prompt_tokens ?? usage.input_tokens ?? 0,
);
const outputTokens = Number(
usage[outputKey] ??
usage.completion_tokens ??
usage.output_tokens ??
0,
);
return {
inputKey,
outputKey,
inputTokens,
outputTokens,
inputMicroCents,
outputMicroCents,
totalMicroCents: inputMicroCents + outputMicroCents,
};
}
/**
* The credit and subscription gate for one upstream attempt.
*
@@ -1550,7 +1428,7 @@ export class ChatCompletionDriver extends PuterDriver {
[outputKey]: outputTokens,
};
const cost = this.#computeCost(usage, model);
const cost = usageCost(usage, model);
this.#aiMetering.utilRecordUsageObject(
{
[`estimated_${inputKey}`]: inputTokens,
@@ -1593,12 +1471,13 @@ export class ChatCompletionDriver extends PuterDriver {
) {
return;
}
const cost = this.#computeCost(usage, model, usageCosts);
const cost = usageCost(usage, model, usageCosts);
if (!cost) {
(usage as Record<string, number | null>).usd_cents = null;
return;
}
usage.usd_cents = cost.totalMicroCents / 1_000_000;
usage.usd_cents =
(cost.inputMicroCents + cost.outputMicroCents) / 1_000_000;
}
// Compute per-token cost in microcents using the model's cost map,
@@ -1621,10 +1500,24 @@ export class ChatCompletionDriver extends PuterDriver {
usageCosts,
} = params;
const cost = this.#computeCost(usage, model, usageCosts);
const cost = usageCost(usage, model, usageCosts);
const { inputKey, outputKey } = costKeys(model);
const inputTokens = cost?.inputTokens ?? 0;
const outputTokens = cost?.outputTokens ?? 0;
const inputTokens = cost
? Number(
usage[inputKey] ??
usage.prompt_tokens ??
usage.input_tokens ??
0,
)
: 0;
const outputTokens = cost
? Number(
usage[outputKey] ??
usage.completion_tokens ??
usage.output_tokens ??
0,
)
: 0;
const inputMicroCents = cost?.inputMicroCents ?? 0;
const outputMicroCents = cost?.outputMicroCents ?? 0;
src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.test.ts
+7 -5
View File
@@ -187,15 +187,16 @@ describe('AlibabaProvider model catalog', () => {
['qwen3.8-flash', 15, 47, 1.6],
['qwen3.8-omni-flash', 15, 47, 1.6],
['qwen3.8-max-0902', 200, 600, 25],
['qwen3.7-max-2026-05-17', 250, 750, 0],
['deepseek-v4.1-flash', 30, 120, 0],
// No cache discount: cached reads bill at the input rate.
['qwen3.7-max-2026-05-17', 250, 750, undefined],
['deepseek-v4.1-flash', 30, 120, undefined],
])('exposes %s at its list price', (id, input, output, cached) => {
const model = ALIBABA_MODELS.find((m) => m.id === id);
expect(model?.costs).toMatchObject({
prompt_tokens: input,
completion_tokens: output,
cached_tokens: cached,
});
expect(model?.costs.cached_tokens).toBe(cached);
});
it.each([
@@ -593,14 +594,15 @@ describe('AlibabaProvider.complete non-stream output', () => {
(m) => m.id === 'qwen3.8-max',
)!;
const [usage, , prefix, overrides] = recordSpy.mock.calls[0]!;
// Cached reads come out of `prompt_tokens`; they're priced once.
expect(usage).toEqual({
prompt_tokens: 50,
prompt_tokens: 35,
completion_tokens: 20,
cached_tokens: 15,
});
expect(prefix).toBe('alibaba:qwen3.8-max');
expect(overrides).toEqual({
prompt_tokens: 50 * Number(model.costs.prompt_tokens),
prompt_tokens: 35 * Number(model.costs.prompt_tokens),
completion_tokens: 20 * Number(model.costs.completion_tokens),
cached_tokens: 15 * Number(model.costs.cached_tokens ?? 0),
});
src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.ts
+9 -12
View File
@@ -26,6 +26,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js';
import { ALIBABA_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
type AlibabaConfig = {
apiKey: string;
@@ -100,20 +101,16 @@ export class AlibabaProvider implements IChatProvider {
);
return OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
const trackedUsage = OpenAIUtil.extractMeteredUsage(usage);
const costsOverride = Object.fromEntries(
Object.entries(trackedUsage).map(([k, v]) => {
return [k, v * modelUsed.costs[k]];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverride,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/alibaba/models.ts
-38
View File
@@ -41,7 +41,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 160,
completion_tokens: 640,
cached_tokens: 0,
},
},
@@ -65,7 +64,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 120,
cached_tokens: 0,
},
},
{
@@ -87,7 +85,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 240,
cached_tokens: 0,
},
},
{
@@ -241,7 +238,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 5,
completion_tokens: 40,
cached_tokens: 0,
},
},
@@ -265,7 +261,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 30,
completion_tokens: 150,
cached_tokens: 0,
},
},
@@ -291,7 +286,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 60,
completion_tokens: 360,
cached_tokens: 0,
},
},
{
@@ -315,7 +309,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 320,
cached_tokens: 0,
},
},
{
@@ -339,7 +332,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 25,
completion_tokens: 200,
cached_tokens: 0,
},
},
{
@@ -363,7 +355,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 30,
completion_tokens: 240,
cached_tokens: 0,
},
},
@@ -389,7 +380,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 24.8,
completion_tokens: 148.5,
cached_tokens: 0,
},
},
{
@@ -413,7 +403,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 60,
completion_tokens: 360,
cached_tokens: 0,
},
},
@@ -437,7 +426,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 20,
completion_tokens: 160,
cached_tokens: 0,
},
},
// -- Omni models (text interface, audio costs excluded) ---------
@@ -463,7 +451,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 43,
completion_tokens: 166,
cached_tokens: 0,
},
},
// -- Translation models -----------------------------------------
@@ -486,7 +473,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 246,
completion_tokens: 737,
cached_tokens: 0,
},
},
@@ -509,7 +495,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 120,
cached_tokens: 0,
},
},
{
@@ -530,7 +515,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 120,
cached_tokens: 0,
},
},
{
@@ -551,7 +535,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 120,
cached_tokens: 0,
},
},
// 'qwen-plus-latest' floats to the newest snapshot; kept on the newest
@@ -574,7 +557,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 120,
cached_tokens: 0,
},
},
// -- Qwen3.7 Max snapshots and preview ------------------------
@@ -596,7 +578,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 250,
completion_tokens: 750,
cached_tokens: 0,
},
},
{
@@ -617,7 +598,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 250,
completion_tokens: 750,
cached_tokens: 0,
},
},
{
@@ -638,7 +618,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 250,
completion_tokens: 750,
cached_tokens: 0,
},
},
// No vendor-documented release date for the preview; omitted rather
@@ -660,7 +639,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 250,
completion_tokens: 750,
cached_tokens: 0,
},
},
// -- Qwen3.5 Plus snapshots -----------------------------------
@@ -683,7 +661,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 240,
cached_tokens: 0,
},
},
{
@@ -705,7 +682,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 240,
cached_tokens: 0,
},
},
// -- Qwen3.5 / Qwen3.7 / Qwen3.8 Flash -----------------------
@@ -793,7 +769,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 50,
completion_tokens: 300,
cached_tokens: 0,
},
},
{
@@ -814,7 +789,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 200,
completion_tokens: 600,
cached_tokens: 0,
},
},
// -- Omni additions (text interface, audio costs excluded) ----
@@ -839,7 +813,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 40,
completion_tokens: 220,
cached_tokens: 0,
},
},
{
@@ -863,7 +836,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 140,
completion_tokens: 830,
cached_tokens: 0,
},
},
{
@@ -909,7 +881,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 16,
completion_tokens: 49,
cached_tokens: 0,
},
},
// No vendor-documented release date; omitted rather than guessed.
@@ -930,7 +901,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 12,
completion_tokens: 36,
cached_tokens: 0,
},
},
// -- Third-party models hosted on Model Studio ------------------
@@ -953,7 +923,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 140,
completion_tokens: 440,
cached_tokens: 0,
},
},
{
@@ -973,7 +942,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 140,
completion_tokens: 440,
cached_tokens: 0,
},
},
// No vendor-documented release date for the preview; omitted rather
@@ -994,7 +962,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 280,
completion_tokens: 880,
cached_tokens: 0,
},
},
// Time-of-day pricing upstream (busy $0.44/$1.32, idle $0.22/$0.66
@@ -1018,7 +985,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 44,
completion_tokens: 132,
cached_tokens: 0,
},
},
// Time-of-day pricing upstream (busy $0.30/$1.20, idle $0.15/$0.60
@@ -1041,7 +1007,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 30,
completion_tokens: 120,
cached_tokens: 0,
},
},
{
@@ -1062,7 +1027,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 240,
completion_tokens: 480,
cached_tokens: 0,
},
},
// Time-of-day pricing upstream (busy $1.32/$3.96, idle $0.66/$1.98
@@ -1085,7 +1049,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 132,
completion_tokens: 396,
cached_tokens: 0,
},
},
{
@@ -1106,7 +1069,6 @@ export const ALIBABA_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 95,
completion_tokens: 400,
cached_tokens: 0,
},
},
];
src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.test.ts
+5 -5
View File
@@ -524,7 +524,7 @@ describe('BytePlusProvider.complete non-stream output', () => {
finish_reason: 'stop',
});
expect((result as { usage: unknown }).usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
@@ -538,14 +538,14 @@ describe('BytePlusProvider.complete non-stream output', () => {
expect(recordSpy).toHaveBeenCalledTimes(1);
const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!;
expect(usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
expect(actor).toBe(SYSTEM_ACTOR);
expect(prefix).toBe('byteplus:seed-2-0-mini-260428');
expect(overrides.prompt_tokens).toBeCloseTo(
100 * Number(seed20Mini.costs.prompt_tokens),
90 * Number(seed20Mini.costs.prompt_tokens),
5,
);
expect(overrides.completion_tokens).toBeCloseTo(
@@ -707,7 +707,7 @@ describe('BytePlusProvider.complete streaming', () => {
const usageEvent = events.find((e) => e.type === 'usage');
expect(usageEvent?.usage).toEqual({
prompt_tokens: 4,
prompt_tokens: 3,
completion_tokens: 2,
cached_tokens: 1,
});
@@ -719,7 +719,7 @@ describe('BytePlusProvider.complete streaming', () => {
const [, , prefix, overrides] = recordSpy.mock.calls[0]!;
expect(prefix).toBe('byteplus:seed-2-0-mini-260428');
expect(overrides.prompt_tokens).toBeCloseTo(
4 * Number(seed20Mini.costs.prompt_tokens),
3 * Number(seed20Mini.costs.prompt_tokens),
5,
);
expect(overrides.completion_tokens).toBeCloseTo(
src/backend/drivers/ai-chat/providers/byteplus/BytePlusProvider.ts
+9 -18
View File
@@ -28,6 +28,7 @@ import { openAICompatParams } from '../../utils/openaiParams.js';
import { BYTEPLUS_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
type BytePlusConfig = {
apiKey: string;
@@ -139,26 +140,16 @@ export class BytePlusProvider implements IChatProvider {
);
const result = await OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
const trackedUsage = usage
? OpenAIUtil.extractMeteredUsage(usage)
: {
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
};
const costsOverride = Object.fromEntries(
Object.entries(trackedUsage).map(([key, value]) => {
return [key, value * Number(modelUsed.costs[key] ?? 0)];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverride,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.test.ts
+6 -5
View File
@@ -529,8 +529,9 @@ describe('DeepSeekProvider.complete non-stream output', () => {
message: { content: 'hi there', role: 'assistant' },
finish_reason: 'stop',
});
// Cached reads come out of `prompt_tokens`; they're priced once.
expect((result as { usage: unknown }).usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
@@ -539,14 +540,14 @@ describe('DeepSeekProvider.complete non-stream output', () => {
expect(recordSpy).toHaveBeenCalledTimes(1);
const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!;
expect(usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
expect(actor).toBe(SYSTEM_ACTOR);
expect(prefix).toBe('deepseek:deepseek-v4-pro');
expect(overrides).toEqual({
prompt_tokens: 100 * Number(chat.costs.prompt_tokens),
prompt_tokens: 90 * Number(chat.costs.prompt_tokens),
completion_tokens: 50 * Number(chat.costs.completion_tokens),
cached_tokens: 10 * Number(chat.costs.cached_tokens ?? 0),
});
@@ -667,7 +668,7 @@ describe('DeepSeekProvider.complete streaming', () => {
const usageEvent = events.find((e) => e.type === 'usage');
expect(usageEvent?.usage).toEqual({
prompt_tokens: 4,
prompt_tokens: 3,
completion_tokens: 2,
cached_tokens: 1,
});
@@ -677,7 +678,7 @@ describe('DeepSeekProvider.complete streaming', () => {
const [, , prefix, overrides] = recordSpy.mock.calls[0]!;
expect(prefix).toBe('deepseek:deepseek-v4-pro');
expect(overrides).toEqual({
prompt_tokens: 4 * Number(chat.costs.prompt_tokens),
prompt_tokens: 3 * Number(chat.costs.prompt_tokens),
completion_tokens: 2 * Number(chat.costs.completion_tokens),
cached_tokens: 1 * Number(chat.costs.cached_tokens ?? 0),
});
src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts
+9 -12
View File
@@ -27,6 +27,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js';
import { DEEPSEEK_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
export class DeepSeekProvider implements IChatProvider {
#openai: OpenAI;
@@ -131,20 +132,16 @@ export class DeepSeekProvider implements IChatProvider {
);
return OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
const trackedUsage = OpenAIUtil.extractMeteredUsage(usage);
const costsOverrideFromModel = Object.fromEntries(
Object.entries(trackedUsage).map(([k, v]) => {
return [k, v * modelUsed.costs[k]];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverrideFromModel,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream: stream,
completion,
src/backend/drivers/ai-chat/providers/gemini/GeminiChatProvider.ts
+20 -19
View File
@@ -29,10 +29,10 @@ import {
process_input_messages,
} from '../../utils/OpenAIUtil.js';
import { inlineHttpImageUrls } from '../../utils/inlineImages.js';
import { buildCostsOverride } from '../../utils/pricing.js';
import { GEMINI_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
export class GeminiChatProvider implements IChatProvider {
meteringService: MeteringService;
@@ -126,15 +126,19 @@ export class GeminiChatProvider implements IChatProvider {
// Cast to access Gemini-specific extras passed alongside usage:
// - choices: non-stream grounding metadata lives in choices[0].message.extra_content
// - extra_content: streaming grounding metadata accumulated by the stream handler
const { usage, choices, extra_content } = args as {
usage: typeof args.usage;
choices?: Array<{
message?: {
extra_content?: { grounding_metadata?: unknown };
};
}>;
extra_content?: { grounding_metadata?: unknown };
};
const { usage, choices, extra_content, setUsageCosts } =
args as {
usage: typeof args.usage;
setUsageCosts: typeof args.setUsageCosts;
choices?: Array<{
message?: {
extra_content?: {
grounding_metadata?: unknown;
};
};
}>;
extra_content?: { grounding_metadata?: unknown };
};
const cached_tokens =
usage?.prompt_tokens_details?.cached_tokens ?? 0;
@@ -160,18 +164,15 @@ export class GeminiChatProvider implements IChatProvider {
: 0,
};
const costsOverrideFromModel = buildCostsOverride(
trackedUsage,
const metered = meterChatUsage(
this.meteringService,
actor,
this.meteringModelKey(modelUsed.id),
modelUsed,
);
this.meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
this.meteringModelKey(modelUsed?.id),
costsOverrideFromModel,
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts
+8 -11
View File
@@ -27,6 +27,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js';
import { GROQ_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
export class GroqAIProvider implements IChatProvider {
#client: Groq;
@@ -100,20 +101,16 @@ export class GroqAIProvider implements IChatProvider {
(chunk as { x_groq?: { usage?: CompletionUsage } }).x_groq
?.usage,
},
usage_calculator: ({ usage }) => {
const trackedUsage = OpenAIUtil.extractMeteredUsage(usage);
const costsOverride = Object.fromEntries(
Object.entries(trackedUsage).map(([k, v]) => {
return [k, v * modelUsed.costs[k]];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverride,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/groq/models.ts
-5
View File
@@ -77,7 +77,6 @@ export const GROQ_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 7.5,
completion_tokens: 30,
cached_tokens: 0,
},
max_tokens: 65536,
},
@@ -97,7 +96,6 @@ export const GROQ_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 80,
completion_tokens: 400,
cached_tokens: 0,
},
max_tokens: 16384,
},
@@ -117,7 +115,6 @@ export const GROQ_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 3,
completion_tokens: 3,
cached_tokens: 0,
},
max_tokens: 512,
},
@@ -137,7 +134,6 @@ export const GROQ_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 4,
completion_tokens: 4,
cached_tokens: 0,
},
max_tokens: 512,
},
@@ -157,7 +153,6 @@ export const GROQ_MODELS: IChatModel[] = [
tokens: 1_000_000,
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
},
max_tokens: 4096,
},
src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.test.ts
+5 -5
View File
@@ -552,7 +552,7 @@ describe('HoonifyProvider.complete non-stream output', () => {
finish_reason: 'stop',
});
expect((result as { usage: unknown }).usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
@@ -566,14 +566,14 @@ describe('HoonifyProvider.complete non-stream output', () => {
expect(recordSpy).toHaveBeenCalledTimes(1);
const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!;
expect(usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
expect(actor).toBe(SYSTEM_ACTOR);
expect(prefix).toBe('hoonify:zai-org/glm-5.2');
expect(overrides.prompt_tokens).toBeCloseTo(
100 * Number(glm.costs.prompt_tokens),
90 * Number(glm.costs.prompt_tokens),
5,
);
expect(overrides.completion_tokens).toBeCloseTo(
@@ -704,7 +704,7 @@ describe('HoonifyProvider.complete streaming', () => {
const usageEvent = events.find((e) => e.type === 'usage');
expect(usageEvent?.usage).toEqual({
prompt_tokens: 4,
prompt_tokens: 3,
completion_tokens: 2,
cached_tokens: 1,
});
@@ -716,7 +716,7 @@ describe('HoonifyProvider.complete streaming', () => {
const [, , prefix, overrides] = recordSpy.mock.calls[0]!;
expect(prefix).toBe('hoonify:zai-org/glm-5.2');
expect(overrides.prompt_tokens).toBeCloseTo(
4 * Number(glm.costs.prompt_tokens),
3 * Number(glm.costs.prompt_tokens),
5,
);
expect(overrides.completion_tokens).toBeCloseTo(
src/backend/drivers/ai-chat/providers/hoonify/HoonifyProvider.ts
+8 -18
View File
@@ -27,6 +27,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js';
import { openAICompatParams } from '../../utils/openaiParams.js';
import { HOONIFY_MODELS } from './models.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
type HoonifyConfig = {
apiBaseUrl?: string;
@@ -139,27 +140,16 @@ export class HoonifyProvider implements IChatProvider {
);
return await OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
const trackedUsage = usage
? OpenAIUtil.extractMeteredUsage(usage)
: {
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
};
const costsOverrideFromModel = Object.fromEntries(
Object.entries(trackedUsage).map(([key, value]) => {
return [key, value * Number(modelUsed.costs[key] ?? 0)];
}),
);
// `modelUsed.id` already carries the `hoonify:` namespace.
this.#meteringService.utilRecordUsageObject(
trackedUsage,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverrideFromModel,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/infron/InfronProvider.ts
+14 -33
View File
@@ -31,6 +31,7 @@ import type {
ICompleteArguments,
} from '../../types.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
/**
* One upstream offering of a model in Infron's catalog. The same model is often
@@ -287,9 +288,11 @@ export class InfronProvider implements IChatProvider {
const usage_calculator = ({
usage,
cost,
setUsageCosts,
}: {
usage: InfronUsage;
cost?: number;
setUsageCosts: (costs: Record<string, number>) => void;
}) => {
// Infron reports `cost` at the top level of the response, not
// inside `usage`. Non-streaming calls get it via the spread
@@ -306,42 +309,20 @@ export class InfronProvider implements IChatProvider {
usage.prompt_tokens_details?.cached_tokens ?? 0,
request: 1,
};
if (typeof authoritativeCost === 'number') {
// Bill the gateway-reported cost as a single line item and
// zero the per-token costs so nothing double-bills.
const billedTrackedUsage = { ...trackedUsage, billedUsage: 1 };
const costOverwrites = Object.fromEntries(
Object.keys(billedTrackedUsage).map((k) => [k, 0]),
);
costOverwrites.billedUsage =
authoritativeCost * 100_000_000 || 1;
this.#meteringService.utilRecordUsageObject(
billedTrackedUsage,
actor,
this.meteringModelKey(modelUsed.id),
costOverwrites,
);
(billedTrackedUsage as Record<string, number>).usd_cents =
authoritativeCost * 100;
return billedTrackedUsage;
}
// Fallback: per-token pricing from the model catalog.
const costOverwrites = Object.fromEntries(
Object.keys(trackedUsage).map((k) => {
return [
k,
(modelUsed.costs[k] ?? 0) *
trackedUsage[k as keyof typeof trackedUsage],
];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
// The gateway-reported cost when there is one; the catalog's
// per-token prices otherwise.
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costOverwrites,
modelUsed,
trackedUsage,
typeof authoritativeCost === 'number'
? { authoritativeUsd: authoritativeCost }
: {},
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
};
return OpenAIUtil.handle_completion_output({
src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts
+9 -11
View File
@@ -29,12 +29,12 @@ import type { IChatProvider, ICompleteArguments } from '../../types.js';
import { make_openai_tools } from '../../utils/FunctionCalling.js';
import * as OpenAIUtil from '../../utils/OpenAIUtil.js';
import { openAICompatParams } from '../../utils/openaiParams.js';
import { buildCostsOverride } from '../../utils/pricing.js';
import { processPuterPathUploads } from '../openai/fileUpload.js';
import { META_MODELS, MUSE_SPARK_DEFAULT_MODEL } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
const DEFAULT_API_BASE_URL = 'https://api.meta.ai/v1';
@@ -221,7 +221,7 @@ export class MetaProvider implements IChatProvider {
);
return OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
usage_calculator: ({ usage, setUsageCosts }) => {
const cachedTokens =
usage?.prompt_tokens_details?.cached_tokens ?? 0;
// Meta reports cache reads as a subset of `prompt_tokens`, so
@@ -234,17 +234,15 @@ export class MetaProvider implements IChatProvider {
completion_tokens: usage?.completion_tokens ?? 0,
cached_tokens: cachedTokens,
};
const costsOverride = buildCostsOverride(
trackedUsage,
modelUsed,
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverride,
modelUsed,
trackedUsage,
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.test.ts
+3 -3
View File
@@ -349,7 +349,7 @@ describe('MiniMaxProvider.complete output and metering', () => {
finish_reason: 'stop',
});
expect((result as { usage: unknown }).usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
@@ -357,14 +357,14 @@ describe('MiniMaxProvider.complete output and metering', () => {
expect(recordSpy).toHaveBeenCalledTimes(1);
const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!;
expect(usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
expect(actor).toBe(SYSTEM_ACTOR);
expect(prefix).toBe('minimax:minimax-m2.7');
expect(overrides).toEqual({
prompt_tokens: 100 * 30,
prompt_tokens: 90 * 30,
completion_tokens: 50 * 120,
cached_tokens: 10 * 6,
});
src/backend/drivers/ai-chat/providers/minimax/MiniMaxProvider.ts
+9 -18
View File
@@ -28,6 +28,7 @@ import { openAICompatParams } from '../../utils/openaiParams.js';
import { MINIMAX_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
type MiniMaxConfig = {
apiKey: string;
@@ -108,26 +109,16 @@ export class MiniMaxProvider implements IChatProvider {
);
return OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
const trackedUsage = usage
? OpenAIUtil.extractMeteredUsage(usage)
: {
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
};
const costsOverride = Object.fromEntries(
Object.entries(trackedUsage).map(([key, value]) => {
return [key, value * Number(modelUsed.costs[key] ?? 0)];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverride,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts
+10 -11
View File
@@ -39,6 +39,7 @@ import { MISTRAL_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { shouldPresentAsOpenAI } from '../../utils/normalizeToOpenAI.js';
import { withSdkTimeout } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
/**
* Mistral's reasoning-capable models return `content` as a chunk array rather
@@ -402,20 +403,18 @@ export class MistralAIProvider implements IChatProvider {
},
completion: completion as ChatCompletionResponse,
stream,
usage_calculator: ({ usage }) => {
const trackedUsage = OpenAIUtil.extractMeteredUsage(usage);
const costsOverrideFromModel = Object.fromEntries(
Object.entries(trackedUsage).map(([k, v]) => {
return [k, v * selectedModel.costs[k]];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
usage_calculator: ({ usage, setUsageCosts }) => {
// `coerceMistralUsage` has already taken cached reads out of
// `prompt_tokens`.
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(selectedModel.id),
costsOverrideFromModel,
selectedModel,
OpenAIUtil.extractMeteredUsage(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
});
}
src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.test.ts
+5 -5
View File
@@ -526,7 +526,7 @@ describe('MoonshotProvider.complete non-stream output', () => {
finish_reason: 'stop',
});
expect((result as { usage: unknown }).usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
@@ -536,14 +536,14 @@ describe('MoonshotProvider.complete non-stream output', () => {
expect(recordSpy).toHaveBeenCalledTimes(1);
const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!;
expect(usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
expect(actor).toBe(SYSTEM_ACTOR);
expect(prefix).toBe('moonshotai:kimi-k2.6');
expect(overrides).toEqual({
prompt_tokens: 100 * Number(kimi.costs.prompt_tokens),
prompt_tokens: 90 * Number(kimi.costs.prompt_tokens),
completion_tokens: 50 * Number(kimi.costs.completion_tokens),
cached_tokens: 10 * Number(kimi.costs.cached_tokens ?? 0),
});
@@ -667,7 +667,7 @@ describe('MoonshotProvider.complete streaming', () => {
const usageEvent = events.find((e) => e.type === 'usage');
expect(usageEvent?.usage).toEqual({
prompt_tokens: 4,
prompt_tokens: 3,
completion_tokens: 2,
cached_tokens: 1,
});
@@ -677,7 +677,7 @@ describe('MoonshotProvider.complete streaming', () => {
const [, , prefix, overrides] = recordSpy.mock.calls[0]!;
expect(prefix).toBe('moonshotai:kimi-k2.6');
expect(overrides).toEqual({
prompt_tokens: 4 * Number(kimi.costs.prompt_tokens),
prompt_tokens: 3 * Number(kimi.costs.prompt_tokens),
completion_tokens: 2 * Number(kimi.costs.completion_tokens),
cached_tokens: 1 * Number(kimi.costs.cached_tokens ?? 0),
});
src/backend/drivers/ai-chat/providers/moonshot/MoonshotProvider.ts
+8 -11
View File
@@ -31,6 +31,7 @@ import { inlineHttpImageUrls } from '../../utils/inlineImages.js';
import { MOONSHOT_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
export class MoonshotProvider implements IChatProvider {
#openai: OpenAI;
@@ -109,20 +110,16 @@ export class MoonshotProvider implements IChatProvider {
}
return OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
const trackedUsage = OpenAIUtil.extractMeteredUsage(usage);
const costsOverride = Object.fromEntries(
Object.entries(trackedUsage).map(([key, value]) => {
return [key, value * modelUsed.costs[key]];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverride,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.test.ts
+4 -3
View File
@@ -563,7 +563,7 @@ describe('NeuralwattProvider.complete non-stream output', () => {
const [usage, , prefix, overrides] = recordSpy.mock.calls[0]!;
expect(prefix).toBe('neuralwatt:deepseek-v4-flash');
expect(usage).toMatchObject({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
energy_kwh: 0.00000145,
@@ -610,13 +610,14 @@ describe('NeuralwattProvider.complete non-stream output', () => {
const completionRate = 0.28 * 100;
const cachedRate = 0.014 * 100;
const [usage, , , overrides] = recordSpy.mock.calls[0]!;
// Cached reads come out of `prompt_tokens`; they're priced once.
expect(usage).toMatchObject({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
expect(overrides).toMatchObject({
prompt_tokens: 100 * promptRate,
prompt_tokens: 90 * promptRate,
completion_tokens: 50 * completionRate,
cached_tokens: 10 * cachedRate,
});
src/backend/drivers/ai-chat/providers/neuralwatt/NeuralwattProvider.ts
+18 -41
View File
@@ -48,6 +48,7 @@ import {
type NeuralwattEnergy,
} from './models.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
const DEFAULT_API_BASE_URL = 'https://api.neuralwatt.com/v1';
@@ -229,10 +230,12 @@ export class NeuralwattProvider implements IChatProvider {
usage,
cost,
energy,
setUsageCosts,
}: {
usage: NeuralwattUsage;
cost?: NeuralwattCost;
energy?: NeuralwattEnergy;
setUsageCosts: (costs: Record<string, number>) => void;
}) => {
// Non-streaming spreads the full completion into this call;
// streaming merges top-level cost/energy onto `usage` via the
@@ -250,7 +253,7 @@ export class NeuralwattProvider implements IChatProvider {
measurement_available: usage.measurement_available,
};
const trackedTokens = OpenAIUtil.extractMeteredUsage(usage);
const trackedTokens = OpenAIUtil.splitCachedPrompt(usage);
const energyUnits: Record<string, number> = {};
if (
energyBlock.measurement_available !== false &&
@@ -277,48 +280,22 @@ export class NeuralwattProvider implements IChatProvider {
return out;
};
if (
typeof requestCostUsd === 'number' &&
Number.isFinite(requestCostUsd)
) {
const billedTrackedUsage = {
...trackedTokens,
...energyUnits,
billedUsage: 1,
};
const costOverwrites = Object.fromEntries(
Object.keys(billedTrackedUsage).map((k) => [k, 0]),
);
costOverwrites.billedUsage = requestCostUsd * 100_000_000;
this.#meteringService.utilRecordUsageObject(
billedTrackedUsage,
actor!,
this.meteringModelKey(modelUsed.id),
costOverwrites,
);
const result = annotate(billedTrackedUsage);
result.usd_cents = requestCostUsd * 100;
return result;
}
// Fallback: catalog token rates (preflight / when Neuralwatt
// omits request_cost_usd).
const trackedUsage = { ...trackedTokens, ...energyUnits };
const costOverwrites = Object.fromEntries(
Object.entries(trackedUsage).map(([k, v]) => {
if (k === 'energy_kwh' || k === 'energy_joules') {
return [k, 0];
}
return [k, (modelUsed.costs[k] ?? 0) * v];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
// Billed at `request_cost_usd` when Neuralwatt reports it, at
// the catalog's token rates otherwise. Energy is recorded, never
// priced.
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costOverwrites,
modelUsed,
{ ...trackedTokens, ...energyUnits },
typeof requestCostUsd === 'number' &&
Number.isFinite(requestCostUsd)
? { authoritativeUsd: requestCostUsd }
: { costOverrides: { energy_kwh: 0, energy_joules: 0 } },
);
return annotate(trackedUsage);
setUsageCosts(metered.costs);
return annotate(metered.usage);
};
return OpenAIUtil.handle_completion_output({
src/backend/drivers/ai-chat/providers/neuralwatt/models.ts
+8 -5
View File
@@ -106,10 +106,13 @@ export const mapNeuralwattApiModel = (
const inputUsd = Number(pricing?.input_per_million ?? 0);
const outputUsd = Number(pricing?.output_per_million ?? 0);
const cachedUsd =
pricing?.cached_input_per_million == null
? 0
: Number(pricing.cached_input_per_million);
const costs = usdPerMToken(
inputUsd,
outputUsd,
Number(pricing?.cached_input_per_million ?? 0),
);
// No published cache price: cached reads bill at the input rate.
if (pricing?.cached_input_per_million == null) delete costs.cached_tokens;
const caps = model.metadata?.capabilities;
const limits = model.metadata?.limits;
@@ -138,7 +141,7 @@ export const mapNeuralwattApiModel = (
costs_currency: 'usd-cents',
input_cost_key: 'prompt_tokens',
output_cost_key: 'completion_tokens',
costs: usdPerMToken(inputUsd, outputUsd, cachedUsd),
costs,
modalities: { input: inputModalities, output: ['text'] },
tool_call: caps?.tools === true,
// Surfaced from Neuralwatt `/models` so `complete()` can gate
src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts
+19 -14
View File
@@ -26,6 +26,7 @@ import * as OpenAIUtil from '../../utils/OpenAIUtil.js';
import { IChatModel, IChatProvider, ICompleteArguments } from '../../types.js';
import { ChatCompletionCreateParams } from 'openai/resources/index.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
/**
* OllamaService class - Provides integration with Ollama's API for chat
* completions Extends BaseService to implement the puter-chat-completion
@@ -151,7 +152,7 @@ export class OllamaChatProvider implements IChatProvider {
: `ollama:ollama/${model}`
: undefined);
return OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
usage_calculator: ({ usage, setUsageCosts }) => {
const trackedUsage = {
prompt:
(usage.prompt_tokens ?? 1) -
@@ -160,20 +161,24 @@ export class OllamaChatProvider implements IChatProvider {
input_cache_read:
usage.prompt_tokens_details?.cached_tokens ?? 0,
};
const costOverwrites = Object.fromEntries(
Object.keys(trackedUsage).map((k) => {
return [k, 0]; // override to 0 since local is free
}),
if (!modelIdForMetering) return trackedUsage;
// Local inference is free.
const metered = meterChatUsage(
this.#meteringService,
actor,
modelIdForMetering,
modelDetails ?? ({ costs: {} } as unknown as IChatModel),
trackedUsage,
{
costOverrides: {
prompt: 0,
completion: 0,
input_cache_read: 0,
},
},
);
if (modelIdForMetering) {
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor,
modelIdForMetering,
costOverwrites,
);
}
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts
+9 -12
View File
@@ -40,13 +40,13 @@ import {
clampReasoningEffort,
openAICompatParams,
} from '../../utils/openaiParams.js';
import { buildCostsOverride } from '../../utils/pricing.js';
import { processPuterPathUploads } from './fileUpload.js';
import { OPEN_AI_MODELS } from './models.js';
import type { OpenAiResponsesChatProvider } from './OpenAiChatResponsesProvider.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
const isWebSearchTool = (tool: Record<string, unknown>): boolean =>
tool.type === 'web_search' ||
@@ -284,7 +284,7 @@ export class OpenAiChatProvider implements IChatProvider {
);
return OpenAiUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
usage_calculator: ({ usage, setUsageCosts }) => {
const cachedTokens =
usage.prompt_tokens_details?.cached_tokens ?? 0;
// GPT-5.6 and later bill cache writes at 1.25x input. They're
@@ -307,18 +307,15 @@ export class OpenAiChatProvider implements IChatProvider {
: {}),
};
const costsOverrideFromModel = buildCostsOverride(
trackedUsage,
modelUsed,
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed?.id),
costsOverrideFromModel,
this.meteringModelKey(modelUsed.id),
modelUsed,
trackedUsage,
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts
+22 -24
View File
@@ -37,7 +37,6 @@ import {
openAICompatParams,
rejectStatefulResponsesFields,
} from '../../utils/openaiParams.js';
import { buildCostsOverride } from '../../utils/pricing.js';
import { responseSamplingParams } from '../../utils/responseSampling.js';
import { processPuterPathUploads } from './fileUpload.js';
import { OPEN_AI_MODELS } from './models.js';
@@ -45,6 +44,7 @@ import { HttpError } from '@heyputer/backend/src/core/http/HttpError.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
const ANTHROPIC_WEB_SEARCH_TYPE = (type: unknown): boolean =>
type === 'web_search_20250305' ||
@@ -362,32 +362,30 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
: {}),
};
const costsOverrideFromModel = buildCostsOverride(
trackedUsage,
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
modelUsed,
);
// Priced dynamically: the rate depends on which
// web-search tool variant was requested and whether the model
// is a reasoning model, not a static per-model cost-table
// entry — `buildCostsOverride`'s fallback would otherwise
// price it (wrongly) at the input/output token rate.
if (webSearchCalls) {
const rate = webSearchCallRate(tools, modelUsed.id) ?? 0;
costsOverrideFromModel.web_search_calls =
webSearchCalls * rate;
setUsageCosts?.({
web_search_calls:
costsOverrideFromModel.web_search_calls,
});
}
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
this.meteringModelKey(modelUsed?.id),
costsOverrideFromModel,
// Priced dynamically: the rate depends on which
// web-search tool variant was requested and whether the
// model is a reasoning model, not a static per-model
// cost-table entry — the model's rates would otherwise
// price it (wrongly) at the input/output token rate.
webSearchCalls
? {
costOverrides: {
web_search_calls:
webSearchCalls *
(webSearchCallRate(tools, modelUsed.id) ??
0),
},
}
: {},
);
return trackedUsage;
setUsageCosts?.(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts
+33 -56
View File
@@ -39,6 +39,7 @@ import type {
} from '../../types.js';
import { OPEN_ROUTER_MODEL_OVERRIDES } from './modelOverrides.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
type OpenrouterUsage = OpenAI.Completions.CompletionUsage & {
cost?: number;
@@ -200,62 +201,38 @@ export class OpenRouterProvider implements IChatProvider {
}
return OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }: { usage: OpenrouterUsage }) => {
if (typeof usage.cost === 'number') {
// custom open router logic because they're pricing are weird
const trackedUsage = {
prompt:
(usage.prompt_tokens ?? 0) -
(usage.prompt_tokens_details?.cached_tokens ?? 0),
completion: usage.completion_tokens ?? 0,
input_cache_read:
usage.prompt_tokens_details?.cached_tokens ?? 0,
request:
(usage as unknown as Record<string, number>)
.request || 1,
billedUsage: 1,
};
const costOverwrites = Object.fromEntries(
Object.keys(trackedUsage).map((k) => {
return [k, 0]; // make everything else 0 if they don't respect their own pricing
}),
);
costOverwrites.billedUsage = usage.cost * 100_000_000 || 1;
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor,
this.meteringModelKey(modelUsed.id),
costOverwrites,
);
(trackedUsage as Record<string, number>).usd_cents =
usage.cost * 100;
return trackedUsage;
} else {
// custom open router logic because they're pricing are weird
const trackedUsage: Record<string, number> = {
prompt:
(usage.prompt_tokens ?? 0) -
(usage.prompt_tokens_details?.cached_tokens ?? 0),
completion: usage.completion_tokens ?? 0,
input_cache_read:
usage.prompt_tokens_details?.cached_tokens ?? 0,
request:
(usage as unknown as Record<string, number>)
.request || 1,
};
const costOverwrites = Object.fromEntries(
Object.keys(trackedUsage).map((k) => {
return [k, modelUsed.costs[k] * trackedUsage[k]];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor,
this.meteringModelKey(modelUsed.id),
costOverwrites,
);
return trackedUsage;
}
usage_calculator: ({
usage,
setUsageCosts,
}: {
usage: OpenrouterUsage;
setUsageCosts: (costs: Record<string, number>) => void;
}) => {
const trackedUsage = {
prompt:
(usage.prompt_tokens ?? 0) -
(usage.prompt_tokens_details?.cached_tokens ?? 0),
completion: usage.completion_tokens ?? 0,
input_cache_read:
usage.prompt_tokens_details?.cached_tokens ?? 0,
request:
(usage as unknown as Record<string, number>).request ||
1,
};
// OpenRouter reports what it billed (`usage.cost`, USD);
// the catalog's prices are the fallback.
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
modelUsed,
trackedUsage,
typeof usage.cost === 'number'
? { authoritativeUsd: usage.cost }
: {},
);
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts
+8 -18
View File
@@ -29,11 +29,7 @@ import {
isContextLengthError,
} from '../../utils/contextLimit.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
const TOGETHER_AI_CHAT_COST_MAP: Record<string, string> = {
prompt_tokens: 'input',
completion_tokens: 'output',
};
import { meterChatUsage } from '../../utils/meterChatUsage.js';
export class TogetherAIProvider implements IChatProvider {
#together: Together;
@@ -178,22 +174,16 @@ export class TogetherAIProvider implements IChatProvider {
}
return OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
const trackedUsage = OpenAIUtil.extractMeteredUsage(usage);
const costsOverride = Object.fromEntries(
Object.entries(trackedUsage).map(([k, v]) => {
const mappedKey = TOGETHER_AI_CHAT_COST_MAP[k] || k;
return [k, v * modelUsed.costs[mappedKey]];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverride,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/providers/zai/ZAIProvider.test.ts
+5 -5
View File
@@ -591,7 +591,7 @@ describe('ZAIProvider.complete non-stream output', () => {
finish_reason: 'stop',
});
expect((result as { usage: unknown }).usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
@@ -603,14 +603,14 @@ describe('ZAIProvider.complete non-stream output', () => {
expect(recordSpy).toHaveBeenCalledTimes(1);
const [usage, actor, prefix, overrides] = recordSpy.mock.calls[0]!;
expect(usage).toEqual({
prompt_tokens: 100,
prompt_tokens: 90,
completion_tokens: 50,
cached_tokens: 10,
});
expect(actor).toBe(SYSTEM_ACTOR);
expect(prefix).toBe('zai:glm-4.6');
expect(overrides.prompt_tokens).toBeCloseTo(
100 * Number(glm46.costs.prompt_tokens),
90 * Number(glm46.costs.prompt_tokens),
5,
);
expect(overrides.completion_tokens).toBeCloseTo(
@@ -799,7 +799,7 @@ describe('ZAIProvider.complete streaming', () => {
const usageEvent = events.find((e) => e.type === 'usage');
expect(usageEvent?.usage).toEqual({
prompt_tokens: 4,
prompt_tokens: 3,
completion_tokens: 2,
cached_tokens: 1,
});
@@ -809,7 +809,7 @@ describe('ZAIProvider.complete streaming', () => {
const [, , prefix, overrides] = recordSpy.mock.calls[0]!;
expect(prefix).toBe('zai:glm-4.6');
expect(overrides.prompt_tokens).toBeCloseTo(
4 * Number(glm46.costs.prompt_tokens),
3 * Number(glm46.costs.prompt_tokens),
5,
);
expect(overrides.completion_tokens).toBeCloseTo(
src/backend/drivers/ai-chat/providers/zai/ZAIProvider.ts
+9 -18
View File
@@ -29,6 +29,7 @@ import { ZAI_MODELS } from './models.js';
import { modelLookupNames } from '../../utils/modelRouting.js';
import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import { meterChatUsage } from '../../utils/meterChatUsage.js';
// Z.AI documents `user_id` as 6-128 characters.
const USER_ID_MAX_LENGTH = 128;
@@ -153,26 +154,16 @@ export class ZAIProvider implements IChatProvider {
);
const result = await OpenAIUtil.handle_completion_output({
usage_calculator: ({ usage }) => {
const trackedUsage = usage
? OpenAIUtil.extractMeteredUsage(usage)
: {
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
};
const costsOverrideFromModel = Object.fromEntries(
Object.entries(trackedUsage).map(([key, value]) => {
return [key, value * Number(modelUsed.costs[key] ?? 0)];
}),
);
this.#meteringService.utilRecordUsageObject(
trackedUsage,
actor!,
usage_calculator: ({ usage, setUsageCosts }) => {
const metered = meterChatUsage(
this.#meteringService,
actor,
this.meteringModelKey(modelUsed.id),
costsOverrideFromModel,
modelUsed,
OpenAIUtil.splitCachedPrompt(usage),
);
return trackedUsage;
setUsageCosts(metered.costs);
return metered.usage;
},
stream,
completion,
src/backend/drivers/ai-chat/utils/OpenAIUtil.js
+22
View File
@@ -487,6 +487,23 @@ export const extractMeteredUsage = (usage) => {
};
};
/**
* Chat Completions usage as it's metered: `prompt_tokens` reports cached reads
* inside its count, so they come out of it and are priced only once, under
* `cached_tokens`.
*
* @param {import('openai/resources/completions.mjs').CompletionUsage
* | undefined} usage
*/
export const splitCachedPrompt = (usage) => {
const cachedTokens = usage?.prompt_tokens_details?.cached_tokens ?? 0;
return {
prompt_tokens: (usage?.prompt_tokens ?? 0) - cachedTokens,
completion_tokens: usage?.completion_tokens ?? 0,
cached_tokens: cachedTokens,
};
};
/**
* The `{prompt_tokens, completion_tokens, cached_tokens, cache_write_tokens}`
* shape every OpenAI-family `usage_calculator` already returns for billing,
@@ -674,6 +691,7 @@ export const create_chat_stream_handler =
? usage_calculator({
usage: last_usage,
extra_content: last_extra_content,
setUsageCosts: (c) => chatStream.setUsageCosts(c),
})
: undefined;
// The calculator just metered. Reported here, not only via `end`:
@@ -875,6 +893,7 @@ export const handle_completion_output = async (
* @type {Record<string, unknown> & {
* usage_calculator: (args: {
* usage: import('openai/resources/completions.mjs').CompletionUsage;
* setUsageCosts: (costs: Record<string, number>) => void;
* }) => unknown;
* }}
*/
@@ -927,6 +946,9 @@ export const handle_completion_output = async (
? usage_calculator({
...completion,
usage: completion_usage,
setUsageCosts: (c) => {
ret.usageCosts = c;
},
})
: {
input_tokens: completion_usage.prompt_tokens,
src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts
+68
View File
@@ -31,6 +31,7 @@ import {
handle_completion_output_responses_api,
process_input_messages,
process_input_messages_responses_api,
splitCachedPrompt,
toOpenAIChatMessages,
usageDetailsFromTrackedUsage,
} from './OpenAIUtil.js';
@@ -833,6 +834,73 @@ describe('extractMeteredUsage', () => {
});
});
// ── splitCachedPrompt ───────────────────────────────────────────────
describe('splitCachedPrompt', () => {
it('takes cached reads out of prompt_tokens', () => {
expect(
splitCachedPrompt({
prompt_tokens: 10,
completion_tokens: 5,
prompt_tokens_details: { cached_tokens: 3 },
}),
).toEqual({
prompt_tokens: 7,
completion_tokens: 5,
cached_tokens: 3,
});
});
it('reads a missing usage as nothing used', () => {
expect(splitCachedPrompt(undefined)).toEqual({
prompt_tokens: 0,
completion_tokens: 0,
cached_tokens: 0,
});
});
});
// ── recorded usage costs ────────────────────────────────────────────
describe('chat completion handlers carry the recorded usage costs', () => {
const usage_calculator = ({
usage,
setUsageCosts,
}: {
usage: Record<string, number>;
setUsageCosts: (costs: Record<string, number>) => void;
}) => {
setUsageCosts({ prompt_tokens: 42 });
return usage;
};
it('onto a non-stream result', async () => {
const result = await handle_completion_output({
stream: false,
completion: {
choices: [{ message: { content: 'hi' } }],
usage: { prompt_tokens: 1, completion_tokens: 1 },
},
usage_calculator,
});
expect(result.usageCosts).toEqual({ prompt_tokens: 42 });
});
it('onto the chat stream', async () => {
const { chatStream } = makeCapturingChatStream();
await create_chat_stream_handler({
completion: asAsyncIterable([
{
choices: [{ delta: { content: 'hi' }, finish_reason: 'stop' }],
usage: { prompt_tokens: 1, completion_tokens: 1 },
},
]),
usage_calculator,
})({ chatStream });
expect(chatStream.usageCosts).toEqual({ prompt_tokens: 42 });
});
});
// ── create_chat_stream_handler ──────────────────────────────────────
describe('create_chat_stream_handler', () => {
src/backend/drivers/ai-chat/utils/meterChatUsage.test.ts
+167
View File
@@ -0,0 +1,167 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { describe, expect, it } from 'vitest';
import type { Actor } from '../../../core/actor.js';
import { ALIBABA_MODELS } from '../providers/alibaba/models.js';
import { BYTEPLUS_MODELS } from '../providers/byteplus/models.js';
import { DEEPSEEK_MODELS } from '../providers/deepseek/models.js';
import { GROQ_MODELS } from '../providers/groq/models.js';
import { HOONIFY_MODELS } from '../providers/hoonify/models.js';
import { MINIMAX_MODELS } from '../providers/minimax/models.js';
import { MISTRAL_MODELS } from '../providers/mistral/models.js';
import { MOONSHOT_MODELS } from '../providers/moonshot/models.js';
import { ZAI_MODELS } from '../providers/zai/models.js';
import type { IChatModel } from '../types.js';
import { meterChatUsage } from './meterChatUsage.js';
import { splitCachedPrompt } from './OpenAIUtil.js';
type Recorded = [Record<string, number>, string, Record<string, number>];
const recordingMetering = () => {
const calls: Recorded[] = [];
return {
calls,
metering: {
utilRecordUsageObject: (
usage: Record<string, number>,
_actor: Actor,
key: string,
costs?: Record<string, number>,
) => {
calls.push([usage, key, costs ?? {}]);
return Promise.resolve({ total: 0 } as never);
},
},
};
};
const actor = {} as Actor;
const model = {
id: 'm',
costs_currency: 'usd-cents',
input_cost_key: 'prompt_tokens',
output_cost_key: 'completion_tokens',
costs: { tokens: 1_000_000, prompt_tokens: 3, completion_tokens: 15 },
max_tokens: 1024,
} as IChatModel;
describe('meterChatUsage', () => {
it('records the usage at the model rates and returns what it recorded', () => {
const { calls, metering } = recordingMetering();
const usage = { prompt_tokens: 10, completion_tokens: 2 };
const result = meterChatUsage(metering, actor, 'p:m', model, usage);
expect(calls).toEqual([
[usage, 'p:m', { prompt_tokens: 30, completion_tokens: 30 }],
]);
expect(result).toEqual({
usage,
costs: { prompt_tokens: 30, completion_tokens: 30 },
});
});
it('lets a cost override win over the model rates', () => {
const { calls, metering } = recordingMetering();
meterChatUsage(
metering,
actor,
'p:m',
model,
{ prompt_tokens: 10, web_search_calls: 2 },
{ costOverrides: { web_search_calls: 1000 } },
);
expect(calls[0]![2]).toEqual({
prompt_tokens: 30,
web_search_calls: 1000,
});
});
it('bills an authoritative USD cost as one line and zeroes the tokens', () => {
const { calls, metering } = recordingMetering();
const result = meterChatUsage(
metering,
actor,
'gw:m',
model,
{ prompt: 10, completion: 2 },
{ authoritativeUsd: 0.25 },
);
expect(calls).toEqual([
[
{ prompt: 10, completion: 2, billedUsage: 1 },
'gw:m',
{ prompt: 0, completion: 0, billedUsage: 25_000_000 },
],
]);
expect(result.usage).toEqual({
prompt: 10,
completion: 2,
billedUsage: 1,
usd_cents: 25,
});
expect(result.costs.billedUsage).toBe(25_000_000);
});
});
// Every catalog the hand-rolled `amount * costs[key]` pricing used to bill.
// A cached read is priced once, at the model's cache rate, or at its input
// rate when it publishes none; it is never free on a paid model.
describe('cached reads on OpenAI-compatible catalogs', () => {
const catalogs: [string, readonly IChatModel[]][] = [
['alibaba', ALIBABA_MODELS],
['byteplus', BYTEPLUS_MODELS],
['deepseek', DEEPSEEK_MODELS],
['groq', GROQ_MODELS],
['hoonify', HOONIFY_MODELS],
['minimax', MINIMAX_MODELS],
['mistral', MISTRAL_MODELS],
['moonshotai', MOONSHOT_MODELS],
['zai', ZAI_MODELS],
];
const cases = catalogs.flatMap(([provider, models]) =>
models.map((m) => [`${provider}:${m.id}`, m] as const),
);
it.each(cases)('%s', (_name, m) => {
const { calls, metering } = recordingMetering();
meterChatUsage(
metering,
actor,
'k',
m,
splitCachedPrompt({
prompt_tokens: 1_000_000,
completion_tokens: 100_000,
prompt_tokens_details: { cached_tokens: 200_000 },
total_tokens: 1_100_000,
}),
);
const costs = m.costs as Record<string, number>;
const cachedRate = costs.cached_tokens ?? costs.prompt_tokens;
expect(calls[0]![2]).toEqual({
prompt_tokens: 800_000 * costs.prompt_tokens,
completion_tokens: 100_000 * costs.completion_tokens,
cached_tokens: 200_000 * cachedRate,
});
if (costs.prompt_tokens > 0) expect(cachedRate).toBeGreaterThan(0);
});
});
src/backend/drivers/ai-chat/utils/meterChatUsage.ts
+66
View File
@@ -0,0 +1,66 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import type { Actor } from '../../../core/actor.js';
import type { MeteringService } from '../../../services/metering/MeteringService.js';
import type { IChatModel } from '../types.js';
import { buildCostsOverride } from './pricing.js';
/** Microcents per US dollar. */
const MICROCENTS_PER_USD = 100_000_000;
export interface MeterChatUsageOptions {
/**
* What the upstream says it billed, in USD. Recorded as one `billedUsage`
* line instead of pricing the tokens, which are still recorded at zero.
*/
authoritativeUsd?: number;
/** Per-key µ¢ costs that win over the model's rates. */
costOverrides?: Record<string, number>;
}
/**
* Records one completion's usage under `meteringKey` and returns the usage to
* report along with the µ¢ each key was recorded at, so the driver can quote
* the same cost the ledger holds.
*/
export const meterChatUsage = (
metering: Pick<MeteringService, 'utilRecordUsageObject'>,
actor: Actor | undefined,
meteringKey: string,
model: IChatModel,
usage: Record<string, number>,
{ authoritativeUsd, costOverrides }: MeterChatUsageOptions = {},
): { usage: Record<string, number>; costs: Record<string, number> } => {
if (authoritativeUsd !== undefined) {
const billed = { ...usage, billedUsage: 1 };
const costs: Record<string, number> = Object.fromEntries(
Object.keys(billed).map((key) => [key, 0]),
);
costs.billedUsage = authoritativeUsd * MICROCENTS_PER_USD;
metering.utilRecordUsageObject(billed, actor!, meteringKey, costs);
return {
usage: { ...billed, usd_cents: authoritativeUsd * 100 },
costs,
};
}
const costs = { ...buildCostsOverride(usage, model), ...costOverrides };
metering.utilRecordUsageObject(usage, actor!, meteringKey, costs);
return { usage, costs };
};
src/backend/drivers/ai-chat/utils/pricing.test.ts
+80
View File
@@ -26,6 +26,7 @@ import {
longContextMultipliers,
trackedInputTokens,
trackedOutputTokens,
usageCost,
usageDetailsFromUsage,
usdPerMToken,
} from './pricing.js';
@@ -350,3 +351,82 @@ describe('long-context pricing', () => {
});
});
});
describe('usageCost', () => {
const rates = model({
prompt_tokens: 3,
completion_tokens: 15,
cached_tokens: 0.3,
});
it('splits the priced usage into input and output sides', () => {
expect(
usageCost(
{ prompt_tokens: 100, completion_tokens: 10, cached_tokens: 50 },
rates,
),
).toEqual({ inputMicroCents: 315, outputMicroCents: 150 });
});
it('agrees with what buildCostsOverride records', () => {
const usage = {
prompt_tokens: 1234,
completion_tokens: 567,
cached_tokens: 89,
grounding_requests: 1,
};
const recorded = Object.values(buildCostsOverride(usage, rates)).reduce(
(a, b) => a + b,
0,
);
const cost = usageCost(usage, rates)!;
expect(cost.inputMicroCents + cost.outputMicroCents).toBe(
Math.round(recorded),
);
});
it('takes the costs a provider recorded over the model rates', () => {
expect(
usageCost(
{ prompt_tokens: 100, billedUsage: 1, usd_cents: 0.5 },
rates,
{ prompt_tokens: 0, billedUsage: 500_000 },
),
).toEqual({ inputMicroCents: 500_000, outputMicroCents: 0 });
});
it('prices advisor usage only through a recorded cost', () => {
expect(
usageCost(
{ prompt_tokens: 10, advisor_output_tokens: 1000 },
rates,
),
).toEqual({ inputMicroCents: 30, outputMicroCents: 0 });
expect(
usageCost(
{ prompt_tokens: 10, advisor_output_tokens: 1000 },
rates,
{ advisor_output_tokens: 7 },
),
).toEqual({ inputMicroCents: 30, outputMicroCents: 7 });
});
it('is null when nothing in the usage can be priced', () => {
expect(
usageCost({ prompt_tokens: 10, completion_tokens: 5 }, model({})),
).toBeNull();
});
it('skips usd_cents and non-numeric annotations', () => {
expect(
usageCost(
{
prompt_tokens: 10,
usd_cents: 99,
accounting_method: 'energy',
},
rates,
),
).toEqual({ inputMicroCents: 30, outputMicroCents: 0 });
});
});
src/backend/drivers/ai-chat/utils/pricing.ts
+89 -27
View File
@@ -161,8 +161,8 @@ export const isFreeModel = (model: IChatModel): boolean => {
* A fallback [[UsageDetails]] for providers that haven't been taught to report
* their own (every non-Claude provider, for now). `inputTokens` and
* `outputTokens` reuse the same tracked-token accounting the credit gate and
* `#computeCost` already agree on, so a provider with no detail of its own
* still gets a usable breakdown.
* `usageCost` already agree on, so a provider with no detail of its own still
* gets a usable breakdown.
*/
export const usageDetailsFromUsage = (
usage: Record<string, unknown>,
@@ -172,6 +172,52 @@ export const usageDetailsFromUsage = (
outputTokens: trackedOutputTokens(usage, model),
});
/** A usage key/amount pair that carries a price, rather than a scale or a label. */
const isBillable = (key: string, amount: unknown): amount is number =>
key !== 'tokens' && key !== 'usd_cents' && isRate(amount);
/**
* The µ¢ each usage key costs at the model's rates. A key the model doesn't
* price falls back to the output rate when it is output-denominated and the
* input rate otherwise; a key neither covers is left out.
*/
const priceUsage = (
trackedUsage: Record<string, unknown>,
model: IChatModel,
): Record<string, number> => {
const { inputKey, outputKey } = costKeys(model);
const costs = model.costs ?? {};
const inputRate = isRate(costs[inputKey]) ? costs[inputKey] : undefined;
const outputRate = isRate(costs[outputKey]) ? costs[outputKey] : undefined;
const multipliers = longContextMultipliers(
model,
trackedInputTokens(trackedUsage, model),
);
const priced: Record<string, number> = {};
for (const [key, amount] of Object.entries(trackedUsage)) {
if (!isBillable(key, amount)) continue;
const isOutput = isOutputCostKey(key, outputKey);
const rate = isRate(costs[key])
? costs[key]
: isOutput
? outputRate
: inputRate;
if (rate === undefined) continue;
const multiplier = isPerCallCostKey(key)
? 1
: isOutput
? multipliers.output
: multipliers.input;
priced[key] = amount * rate * multiplier;
}
return priced;
};
/**
* Prices a tracked-usage object against a model's cost table.
*
@@ -187,35 +233,51 @@ export const buildCostsOverride = (
trackedUsage: Record<string, number>,
model: IChatModel,
): Record<string, number> => {
const { inputKey, outputKey } = costKeys(model);
const costs = model.costs ?? {};
const inputRate = isRate(costs[inputKey]) ? costs[inputKey] : undefined;
const outputRate = isRate(costs[outputKey]) ? costs[outputKey] : undefined;
const multipliers = longContextMultipliers(
model,
trackedInputTokens(trackedUsage, model),
);
const priced = priceUsage(trackedUsage, model);
const overrides: Record<string, number> = {};
for (const [key, amount] of Object.entries(trackedUsage)) {
for (const key of Object.keys(trackedUsage)) {
// `tokens` is a scale descriptor ("costs expressed per N tokens"),
// not a per-unit rate.
if (key === 'tokens') continue;
const isOutput = isOutputCostKey(key, outputKey);
const rate = isRate(costs[key])
? costs[key]
: ((isOutput ? outputRate : inputRate) ?? 0);
const multiplier = isPerCallCostKey(key)
? 1
: isOutput
? multipliers.output
: multipliers.input;
overrides[key] = amount * rate * multiplier;
overrides[key] = priced[key] ?? 0;
}
return overrides;
};
/**
* What a usage object costs in µ¢, split into its input and output sides.
* `recordedCosts` are the per-key costs a provider already metered with and win
* over the model's rates, so the figure matches the ledger. Null when nothing
* in the usage could be priced.
*/
export const usageCost = (
usage: Record<string, unknown>,
model: IChatModel,
recordedCosts?: Record<string, number>,
): { inputMicroCents: number; outputMicroCents: number } | null => {
const { outputKey } = costKeys(model);
const priced = priceUsage(usage, model);
let input = 0;
let output = 0;
let sawAnyCost = false;
for (const [key, amount] of Object.entries(usage)) {
if (!isBillable(key, amount)) continue;
const recorded = recordedCosts?.[key];
// Advisor usage is priced at the advisor model's own rates, so only a
// recorded cost can price it here.
const cost = isRate(recorded)
? recorded
: key.startsWith('advisor_')
? undefined
: priced[key];
if (cost === undefined) continue;
sawAnyCost = true;
if (isOutputCostKey(key, outputKey)) output += cost;
else input += cost;
}
if (!sawAnyCost) return null;
return {
inputMicroCents: Math.max(0, Math.round(input)),
outputMicroCents: Math.max(0, Math.round(output)),
};
};