lee/puter
1
0 Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2026-11-07 06:23:05 +00:00
Code Issues Actions 12 Packages Projects Releases Wiki Activity

fix(ai): meter OpenAI cache writes by lifetime

Browse Source 
GPT-5.6 and later write prompt-cache entries with a 30m or 12h lifetime
and bill them at 1.25x and 2.5x input; every write was metered at the
30m rate. The OpenAI calculators now split `cache_write_tokens_details`
into `cache_write_tokens` (30m) and `cache_write_12h_tokens` (12h), the
GPT-5.6+ catalog carries the 12h rate, and `usageDetails` reports
`cacheWrite30mTokens` / `cacheWrite12hTokens` instead of labelling the
writes as 5m entries.

Callers pick the lifetime with `prompt_cache_options` ({ mode, ttl }) on
`puter.ai.chat`, `/openai/v1/responses` and `/openai/v1/chat/completions`.
The Chat Completions provider hands a 12h request to the Responses
provider, the only API that writes 12h entries. `prompt_cache_retention`
is forwarded on Chat Completions too, and the hyphenated `in-memory`
spelling maps to the `in_memory` value the upstreams accept.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
404oopsandClaude Fable 5.1 committed 2026-11-03 00:52:33 +01:00
1 parent 58456ae87c
commit 89f070f7d0
21 files changed
+749 -87
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/backend/controllers/puterai/PuterAIController.shapes.test.ts
+31
View File
@@ -201,6 +201,7 @@ describe('PuterAIController.openaiResponses parameter forwarding', () => {
context_management: [{ type: 'compaction' }],
prompt_cache_key: 'ck',
prompt_cache_retention: '24h',
prompt_cache_options: { ttl: '12h', prewarm: true },
store: true,
truncation: 'auto',
service_tier: 'default',
@@ -226,6 +227,9 @@ describe('PuterAIController.openaiResponses parameter forwarding', () => {
context_management: [{ type: 'compaction' }],
prompt_cache_key: 'ck',
prompt_cache_retention: '24h',
// Forwarded as sent; the provider keeps only the fields it
// accepts.
prompt_cache_options: { ttl: '12h', prewarm: true },
// forced false regardless of the client's request.
store: false,
truncation: 'auto',
@@ -639,6 +643,33 @@ describe('PuterAIController.openaiResponses output items', () => {
// -- /openai/v1/chat/completions edges -------------------------------
describe('PuterAIController.openaiChatCompletions translation edges', () => {
it('forwards the prompt-cache fields to the driver', async () => {
const completeSpy = stubChatComplete({
message: { role: 'assistant', content: 'ok' },
});
const { res } = makeRes();
await controller.openaiChatCompletions(
makeReq({
body: {
model: 'gpt-test',
messages: [{ role: 'user', content: 'hi' }],
prompt_cache_key: 'ck',
prompt_cache_retention: 'in_memory',
prompt_cache_options: { ttl: '30m' },
},
}),
res,
);
const args = completeSpy.mock.calls[0]![0] as Record<string, unknown>;
expect(args).toMatchObject({
prompt_cache_key: 'ck',
prompt_cache_retention: 'in_memory',
prompt_cache_options: { ttl: '30m' },
});
});
it('derives tool_calls from tool_use content blocks when the message has none', async () => {
stubChatComplete({
message: {
src/backend/controllers/puterai/PuterAIController.ts
+29 -9
View File
@@ -447,6 +447,7 @@ export class PuterAIController extends PuterController {
body.reasoning_effort as ICompleteArguments['reasoning_effort'],
}
: {}),
...promptCacheArgsFromBody(body),
...openaiCompatProvider(body),
...(body.fallback === false ? { fallback: false } : {}),
};
@@ -842,15 +843,7 @@ export class PuterAIController extends PuterController {
body.compaction as ICompleteArguments['compaction'],
}
: {}),
...(body.prompt_cache_key
? { prompt_cache_key: String(body.prompt_cache_key) }
: {}),
...(body.prompt_cache_retention
? {
prompt_cache_retention:
body.prompt_cache_retention as ICompleteArguments['prompt_cache_retention'],
}
: {}),
...promptCacheArgsFromBody(body),
// forced off regardless of what the caller sent — OpenAI
// defaults `store` to true, which would persist state server-side
// on OpenAI's end that this route gives no way to retrieve.
@@ -1343,6 +1336,33 @@ const openaiCompatProvider = (
return { provider: DEFAULTS.openaiChat };
};
/**
* The prompt-cache fields both OpenAI-shaped routes forward as sent; the
* provider checks the values.
*/
const promptCacheArgsFromBody = (
body: Record<string, unknown>,
): Pick<
ICompleteArguments,
'prompt_cache_key' | 'prompt_cache_retention' | 'prompt_cache_options'
> => ({
...(body.prompt_cache_key
? { prompt_cache_key: String(body.prompt_cache_key) }
: {}),
...(body.prompt_cache_retention
? {
prompt_cache_retention:
body.prompt_cache_retention as ICompleteArguments['prompt_cache_retention'],
}
: {}),
...(body.prompt_cache_options
? {
prompt_cache_options:
body.prompt_cache_options as ICompleteArguments['prompt_cache_options'],
}
: {}),
});
const randomId = (): string => crypto.randomUUID().replace(/-/g, '');
const generateId = (prefix: string): string => `${prefix}_${randomId()}`;
src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.test.ts
+16
View File
@@ -449,6 +449,22 @@ describe('AzureResponsesProvider.complete request shape', () => {
expect(args.service_tier).toBe('default');
});
it('sends the wire spelling of the in-memory cache retention', async () => {
const provider = makeProvider();
responsesCreateMock.mockResolvedValueOnce(okResponse);
await withTestActor(() =>
provider.complete({
model: 'gpt-5.3-codex',
messages: [{ role: 'user', content: 'hi' }],
prompt_cache_retention: 'in-memory',
} as never),
);
const [args] = responsesCreateMock.mock.calls[0]!;
expect(args.prompt_cache_retention).toBe('in_memory');
});
it.each([
['previous_response_id', 'resp_1'],
['conversation', 'conv_1'],
src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts
+6 -2
View File
@@ -35,6 +35,7 @@ import * as OpenAiUtil from '../../utils/OpenAIUtil.js';
import {
clampReasoningEffort,
openAICompatParams,
promptCacheRetentionToWire,
rejectStatefulResponsesFields,
} from '../../utils/openaiParams.js';
import { buildCostsOverride } from '../../utils/pricing.js';
@@ -209,6 +210,9 @@ export class AzureResponsesProvider implements IChatProvider {
const userIdentifier = upstreamUserIdentifier(actor);
// Cache key defaults to the actor identifier; see upstreamUserIdentifier.
const cacheKey = prompt_cache_key ?? userIdentifier;
const cacheRetention = promptCacheRetentionToWire(
prompt_cache_retention,
);
// Resolve any `puter_path` content parts into inline base64 data URLs
// before the Responses API sees them.
@@ -307,8 +311,8 @@ export class AzureResponsesProvider implements IChatProvider {
...(instructions !== undefined ? { instructions } : {}),
...(metadata !== undefined ? { metadata } : {}),
...(cacheKey !== undefined ? { prompt_cache_key: cacheKey } : {}),
...(prompt_cache_retention !== undefined
? { prompt_cache_retention }
...(cacheRetention !== undefined
? { prompt_cache_retention: cacheRetention }
: {}),
...(store !== undefined ? { store } : {}),
...(max_tokens !== undefined
src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts
+4 -6
View File
@@ -23,6 +23,7 @@ import { HttpError } from '../../../../core/http/HttpError.js';
import type { MeteringService } from '../../../../services/metering/MeteringService.js';
import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js';
import type { IChatModel, ICompleteArguments } from '../../types.js';
import { promptCacheRetentionToWire } from '../../utils/openaiParams.js';
import { sdkClientOptions } from '../../utils/sdkClient.js';
import {
type ChatProviderConfig,
@@ -107,12 +108,9 @@ export class MetaProvider extends OpenAICompatProvider {
? requestedEffort
: undefined;
// Puter spells the in-memory retention with a hyphen; Meta's enum
// uses an underscore.
const cacheRetention =
args.prompt_cache_retention === 'in-memory'
? 'in_memory'
: args.prompt_cache_retention;
const cacheRetention = promptCacheRetentionToWire(
args.prompt_cache_retention,
);
// The identifier is Puter's abuse attribution, so `custom` can't
// override it. Cache key defaults to it; see upstreamUserIdentifier.
src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.test.ts
+71 -1
View File
@@ -249,6 +249,44 @@ describe('OpenAiChatProvider.complete argument validation', () => {
expect(result).toEqual({ delegated: true });
});
it('delegates a 12h cache lifetime to the Responses provider, which is the only API that writes it', async () => {
const { provider } = makeProvider();
const sibling = {
complete: vi.fn().mockResolvedValue({ delegated: true }),
};
provider.setResponsesProvider(
sibling as unknown as Parameters<
typeof provider.setResponsesProvider
>[0],
);
const params = {
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'hi' }],
prompt_cache_options: { ttl: '12h' as const },
};
const result = await withTestActor(() => provider.complete(params));
expect(createMock).not.toHaveBeenCalled();
expect(sibling.complete).toHaveBeenCalledWith(params);
expect(result).toEqual({ delegated: true });
});
it('throws 400 for a 12h cache lifetime without a Responses sibling provider', async () => {
const { provider } = makeProvider();
await expect(
withTestActor(() =>
provider.complete({
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'hi' }],
prompt_cache_options: { ttl: '12h' },
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
it('requestPricing delegates to the Responses sibling (web_search\'s own hold), and no-ops without one', async () => {
const { provider } = makeProvider();
const args = {
@@ -349,6 +387,31 @@ describe('OpenAiChatProvider.complete request shape', () => {
expect(args.safety_identifier).toBe('puter-u42');
});
it('forwards 30m prompt_cache_options and the wire spelling of prompt_cache_retention', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'hello' }],
prompt_cache_retention: 'in-memory',
prompt_cache_options: {
mode: 'explicit',
ttl: '30m',
prewarm: true,
} as never,
}),
);
const [args] = createMock.mock.calls[0]!;
expect(args.prompt_cache_retention).toBe('in_memory');
expect(args.prompt_cache_options).toEqual({
mode: 'explicit',
ttl: '30m',
});
});
it.each([
'gpt-6-astra',
'gpt-6.1-sol',
@@ -665,7 +728,7 @@ describe('OpenAiChatProvider.complete non-stream output', () => {
},
});
await withTestActor(() =>
const result = await withTestActor(() =>
provider.complete({
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'hi' }],
@@ -685,6 +748,13 @@ describe('OpenAiChatProvider.complete non-stream output', () => {
cached_tokens: 1000 * Number(luna.costs.cached_tokens),
cache_write_tokens: 3000 * Number(luna.costs.cache_write_tokens),
});
// Chat Completions only writes OpenAI's default 30m entries.
expect((result as { usageDetails: unknown }).usageDetails).toEqual({
inputTokens: 1000,
outputTokens: 12,
cacheReadTokens: 1000,
cacheWrite30mTokens: 3000,
});
});
it('zeroes cached_tokens when prompt_tokens_details is missing', async () => {
src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts
+36 -22
View File
@@ -36,6 +36,8 @@ import * as OpenAiUtil from '../../utils/OpenAIUtil.js';
import {
clampReasoningEffort,
openAICompatParams,
promptCacheOptionsToWire,
promptCacheRetentionToWire,
} from '../../utils/openaiParams.js';
import { OPEN_AI_MODELS } from './models.js';
import type { OpenAiResponsesChatProvider } from './OpenAiChatResponsesProvider.js';
@@ -126,8 +128,11 @@ export class OpenAiChatProvider implements IChatProvider {
temperature,
text,
prompt_cache_key,
prompt_cache_retention,
prompt_cache_options,
} = params;
let { messages, model } = params;
const cacheOptions = promptCacheOptionsToWire(prompt_cache_options);
if (
tools?.filter((e) => isWebSearchTool(e as Record<string, unknown>))
.length
@@ -157,6 +162,18 @@ export class OpenAiChatProvider implements IChatProvider {
}
return await this.#responsesProvider.complete(params);
}
// Only the Responses API writes 12h cache entries; Chat Completions
// rejects the lifetime.
if (cacheOptions?.ttl === '12h') {
if (!this.#responsesProvider) {
throw new HttpError(
400,
'a 12h prompt cache lifetime requires the OpenAI Responses provider, which is not configured',
{ legacyCode: 'bad_request' },
);
}
return await this.#responsesProvider.complete(params);
}
// Validate messages
if (!Array.isArray(messages)) {
throw new HttpError(400, '`messages` must be an array', {
@@ -180,6 +197,9 @@ export class OpenAiChatProvider implements IChatProvider {
const userIdentifier = upstreamUserIdentifier(actor);
// Cache key defaults to the actor identifier; see upstreamUserIdentifier.
const cacheKey = prompt_cache_key ?? userIdentifier;
const cacheRetention = promptCacheRetentionToWire(
prompt_cache_retention,
);
// Strip Anthropic-only shape a fallback-replayed message can carry
// (thinking/server-tool blocks, cache_control, citations) before the
@@ -216,6 +236,12 @@ export class OpenAiChatProvider implements IChatProvider {
user: userIdentifier,
safety_identifier: userIdentifier,
...(cacheKey !== undefined ? { prompt_cache_key: cacheKey } : {}),
...(cacheRetention !== undefined
? { prompt_cache_retention: cacheRetention }
: {}),
...(cacheOptions !== undefined
? { prompt_cache_options: cacheOptions }
: {}),
messages: messages,
model: modelUsed.id,
...(mappedTools?.length ? { tools: mappedTools } : {}),
@@ -254,28 +280,11 @@ export class OpenAiChatProvider implements IChatProvider {
);
return OpenAiUtil.handle_completion_output({
usage_calculator: ({ usage, setUsageCosts }) => {
const cachedTokens =
usage.prompt_tokens_details?.cached_tokens ?? 0;
// GPT-5.6 and later bill cache writes at 1.25x input. They're
// reported inside `prompt_tokens`, like cached reads.
// The SDK doesn't type `cache_write_tokens` yet.
const cacheWriteTokens =
(
usage.prompt_tokens_details as
{ cache_write_tokens?: number } | undefined
)?.cache_write_tokens ?? 0;
const trackedUsage = {
prompt_tokens:
(usage.prompt_tokens ?? 0) -
cachedTokens -
cacheWriteTokens,
completion_tokens: usage.completion_tokens ?? 0,
cached_tokens: cachedTokens,
...(cacheWriteTokens
? { cache_write_tokens: cacheWriteTokens }
: {}),
};
usage_calculator: ({ usage, setUsageCosts, setUsageDetails }) => {
// GPT-5.6 and later report cache writes inside
// `prompt_tokens`, like cached reads; Chat Completions only
// writes the default 30m entries, billed at 1.25x input.
const trackedUsage = OpenAiUtil.splitCachedPrompt(usage);
const metered = meterChatUsage(
this.#meteringService,
@@ -285,6 +294,11 @@ export class OpenAiChatProvider implements IChatProvider {
trackedUsage,
);
setUsageCosts(metered.costs);
setUsageDetails(
OpenAiUtil.usageDetailsFromTrackedUsage(metered.usage, {
writeTtl: '30m',
})!,
);
return metered.usage;
},
stream,
src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.test.ts
+174 -1
View File
@@ -246,6 +246,7 @@ describe('OpenAiResponsesChatProvider model catalog', () => {
prompt_tokens: input,
cached_tokens: cached,
cache_write_tokens: input * 1.25,
cache_write_12h_tokens: input * 2.5,
completion_tokens: output,
},
context: 1_050_000,
@@ -844,7 +845,7 @@ describe('OpenAiResponsesChatProvider.complete non-stream output', () => {
},
});
await withTestActor(() =>
const result = await withTestActor(() =>
provider.complete({
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'hi' }],
@@ -867,6 +868,115 @@ describe('OpenAiResponsesChatProvider.complete non-stream output', () => {
cached_tokens: 1000 * Number(luna.costs.cached_tokens),
cache_write_tokens: 3000 * Number(luna.costs.cache_write_tokens),
});
// Writes without a per-lifetime breakdown are OpenAI's default 30m
// entries, and the neutral breakdown says so.
expect((result as { usageDetails: unknown }).usageDetails).toEqual({
inputTokens: 1000,
outputTokens: 20,
cacheReadTokens: 1000,
cacheWrite30mTokens: 3000,
});
});
it('bills 12h cache writes at 2.5x input and the other writes at 1.25x', async () => {
const luna = OPEN_AI_MODELS.find((m) => m.id === 'gpt-6-luna')!;
const { provider } = makeProvider();
// One request can write both lifetimes: an explicit 12h breakpoint
// plus the implicit 30m one OpenAI adds at the end of the prompt.
responsesCreateMock.mockResolvedValueOnce({
output: [{ role: 'assistant' }],
output_text: 'ok',
usage: {
input_tokens: 6026,
output_tokens: 5,
input_tokens_details: {
cached_tokens: 0,
cache_write_tokens: 6023,
cache_write_tokens_details: {
ttl_12h_tokens: 6012,
ttl_30m_tokens: 11,
},
},
},
});
const result = await withTestActor(() =>
provider.complete({
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'hi' }],
prompt_cache_options: { ttl: '12h' },
}),
);
const [usage, , , overrides] = recordSpy.mock.calls[0]!;
expect(usage).toEqual({
prompt_tokens: 3,
completion_tokens: 5,
cached_tokens: 0,
cache_write_tokens: 11,
cache_write_12h_tokens: 6012,
});
expect(luna.costs.cache_write_12h_tokens).toBe(
Number(luna.costs.prompt_tokens) * 2.5,
);
expect(overrides).toEqual({
prompt_tokens: 3 * Number(luna.costs.prompt_tokens),
completion_tokens: 5 * Number(luna.costs.completion_tokens),
cached_tokens: 0,
cache_write_tokens: 11 * Number(luna.costs.cache_write_tokens),
cache_write_12h_tokens:
6012 * Number(luna.costs.cache_write_12h_tokens),
});
expect((result as { usageDetails: unknown }).usageDetails).toEqual({
inputTokens: 3,
outputTokens: 5,
cacheWrite30mTokens: 11,
cacheWrite12hTokens: 6012,
});
});
it('forwards prompt_cache_options and the wire spelling of prompt_cache_retention', async () => {
const { provider } = makeProvider();
responsesCreateMock.mockResolvedValueOnce({
output: [{ role: 'assistant' }],
output_text: 'ok',
usage: { input_tokens: 1, output_tokens: 1 },
});
await withTestActor(() =>
provider.complete({
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'hi' }],
prompt_cache_retention: 'in-memory',
prompt_cache_options: {
mode: 'explicit',
ttl: '12h',
comparison_response_id: 'resp_1',
} as never,
}),
);
const [args] = responsesCreateMock.mock.calls[0]!;
expect(args.prompt_cache_retention).toBe('in_memory');
expect(args.prompt_cache_options).toEqual({
mode: 'explicit',
ttl: '12h',
});
});
it('rejects a cache lifetime OpenAI does not offer before calling upstream', async () => {
const { provider } = makeProvider();
await expect(
withTestActor(() =>
provider.complete({
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'hi' }],
prompt_cache_options: { ttl: '2h' } as never,
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(responsesCreateMock).not.toHaveBeenCalled();
});
it.each([
@@ -1068,6 +1178,69 @@ describe('OpenAiResponsesChatProvider.complete streaming', () => {
});
});
it('labels streamed cache writes by lifetime on the final usage chunk', async () => {
const luna = OPEN_AI_MODELS.find((m) => m.id === 'gpt-6-luna')!;
const { provider } = makeProvider();
responsesCreateMock.mockReturnValueOnce(
asAsyncIterable([
{ type: 'response.output_text.delta', delta: 'ok' },
{
type: 'response.completed',
response: {
usage: {
input_tokens: 6026,
output_tokens: 5,
input_tokens_details: {
cached_tokens: 0,
cache_write_tokens: 6023,
cache_write_tokens_details: {
ttl_12h_tokens: 6012,
ttl_30m_tokens: 11,
},
},
},
},
},
]),
);
const result = await withTestActor(() =>
provider.complete({
model: 'gpt-6-luna',
messages: [{ role: 'user', content: 'say ok' }],
stream: true,
prompt_cache_options: { ttl: '12h' },
}),
);
const harness = makeCapturingChatStream();
await (
result as {
init_chat_stream: (p: { chatStream: unknown }) => Promise<void>;
}
).init_chat_stream({ chatStream: harness.chatStream });
const usageEvent = harness.events().find((e) => e.type === 'usage');
expect(usageEvent?.usage).toEqual({
prompt_tokens: 3,
completion_tokens: 5,
cached_tokens: 0,
cache_write_tokens: 11,
cache_write_12h_tokens: 6012,
});
expect(usageEvent?.usageDetails).toEqual({
inputTokens: 3,
outputTokens: 5,
cacheWrite30mTokens: 11,
cacheWrite12hTokens: 6012,
});
const [, , , overrides] = recordSpy.mock.calls[0]!;
expect(overrides).toMatchObject({
cache_write_tokens: 11 * Number(luna.costs.cache_write_tokens),
cache_write_12h_tokens:
6012 * Number(luna.costs.cache_write_12h_tokens),
});
});
it('emits a tool_use block when response.output_item.done arrives with a function_call', async () => {
const { provider } = makeProvider();
responsesCreateMock.mockReturnValueOnce(
src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts
+27 -19
View File
@@ -32,6 +32,8 @@ import * as OpenAiUtil from '../../utils/OpenAIUtil.js';
import {
clampReasoningEffort,
openAICompatParams,
promptCacheOptionsToWire,
promptCacheRetentionToWire,
rejectStatefulResponsesFields,
} from '../../utils/openaiParams.js';
import { responseSamplingParams } from '../../utils/responseSampling.js';
@@ -150,6 +152,7 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
prompt,
prompt_cache_key,
prompt_cache_retention,
prompt_cache_options,
store,
top_p,
truncation,
@@ -195,6 +198,10 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
const userIdentifier = upstreamUserIdentifier(actor);
// Cache key defaults to the actor identifier; see upstreamUserIdentifier.
const cacheKey = prompt_cache_key ?? userIdentifier;
const cacheRetention = promptCacheRetentionToWire(
prompt_cache_retention,
);
const cacheOptions = promptCacheOptionsToWire(prompt_cache_options);
if (tools) {
// Unravel tools to OpenAI Responses API format
@@ -288,8 +295,11 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
...(instructions !== undefined ? { instructions } : {}),
...(metadata !== undefined ? { metadata } : {}),
...(cacheKey !== undefined ? { prompt_cache_key: cacheKey } : {}),
...(prompt_cache_retention !== undefined
? { prompt_cache_retention }
...(cacheRetention !== undefined
? { prompt_cache_retention: cacheRetention }
: {}),
...(cacheOptions !== undefined
? { prompt_cache_options: cacheOptions }
: {}),
...(store !== undefined ? { store } : {}),
...(max_tokens !== undefined
@@ -310,24 +320,17 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
{ signal: Context.get('upstreamSignal') },
);
return OpenAiUtil.handle_completion_output_responses_api({
usage_calculator: ({ usage, webSearchCalls, setUsageCosts }) => {
const cachedTokens =
(usage as any).input_tokens_details?.cached_tokens ?? 0;
// GPT-5.6 and later bill cache writes at 1.25x input. They're
// reported inside `input_tokens`, like cached reads.
const cacheWriteTokens =
(usage as any).input_tokens_details?.cache_write_tokens ??
0;
usage_calculator: ({
usage,
webSearchCalls,
setUsageCosts,
setUsageDetails,
}) => {
// GPT-5.6 and later report cache writes inside `input_tokens`,
// like cached reads, split by lifetime (30m and 12h entries
// cost 1.25x and 2.5x input to write).
const trackedUsage: Record<string, number> = {
prompt_tokens:
((usage as any).input_tokens ?? 0) -
cachedTokens -
cacheWriteTokens,
completion_tokens: (usage as any).output_tokens ?? 0,
cached_tokens: cachedTokens,
...(cacheWriteTokens
? { cache_write_tokens: cacheWriteTokens }
: {}),
...OpenAiUtil.splitCachedPrompt(usage),
...(webSearchCalls
? { web_search_calls: webSearchCalls }
: {}),
@@ -356,6 +359,11 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
: {},
);
setUsageCosts?.(metered.costs);
setUsageDetails?.(
OpenAiUtil.usageDetailsFromTrackedUsage(metered.usage, {
writeTtl: '30m',
})!,
);
return metered.usage;
},
stream,
src/backend/drivers/ai-chat/providers/openai/models.ts
+10
View File
@@ -23,6 +23,8 @@ import type { IChatModel } from '../../types.js';
// Prompts over 272K input tokens are billed at 2x input (cached reads and
// cache writes included) and 1.5x output for the full request.
// GPT-5.6 and later bill cache writes by the lifetime requested: 1.25x input
// for the default 30m entries, 2.5x for 12h entries.
const GPT_LONG_CONTEXT_PRICING = {
threshold: 272_000,
input_multiplier: 2,
@@ -49,6 +51,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [
prompt_tokens: 100,
cached_tokens: 5,
cache_write_tokens: 100 * 1.25,
cache_write_12h_tokens: 100 * 2.5,
completion_tokens: 500,
},
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
@@ -74,6 +77,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [
prompt_tokens: 200,
cached_tokens: 10,
cache_write_tokens: 200 * 1.25,
cache_write_12h_tokens: 200 * 2.5,
completion_tokens: 1000,
},
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
@@ -99,6 +103,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [
prompt_tokens: 200,
cached_tokens: 20,
cache_write_tokens: 200 * 1.25,
cache_write_12h_tokens: 200 * 2.5,
completion_tokens: 1000,
},
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
@@ -124,6 +129,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [
prompt_tokens: 10,
cached_tokens: 1,
cache_write_tokens: 10 * 1.25,
cache_write_12h_tokens: 10 * 2.5,
completion_tokens: 50,
},
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
@@ -149,6 +155,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [
prompt_tokens: 1000,
cached_tokens: 100,
cache_write_tokens: 1000 * 1.25,
cache_write_12h_tokens: 1000 * 2.5,
completion_tokens: 5000,
},
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
@@ -174,6 +181,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [
prompt_tokens: 400,
cached_tokens: 40,
cache_write_tokens: 400 * 1.25,
cache_write_12h_tokens: 400 * 2.5,
completion_tokens: 2000,
},
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
@@ -198,6 +206,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [
prompt_tokens: 200,
cached_tokens: 20,
cache_write_tokens: 200 * 1.25,
cache_write_12h_tokens: 200 * 2.5,
completion_tokens: 1200,
},
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
@@ -222,6 +231,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [
prompt_tokens: 20,
cached_tokens: 2,
cache_write_tokens: 20 * 1.25,
cache_write_12h_tokens: 20 * 2.5,
completion_tokens: 120,
},
long_context_pricing: GPT_LONG_CONTEXT_PRICING,
src/backend/drivers/ai-chat/types.ts
+22 -2
View File
@@ -145,13 +145,19 @@ export type StopReason =
| 'model_context_window_exceeded'
| (string & {});
/** Provider-neutral usage breakdown. `inputTokens` excludes cache reads/writes. */
/**
* Provider-neutral usage breakdown. `inputTokens` excludes cache reads/writes.
* Cache writes are keyed by the lifetime the entry was written with: Claude and
* Moonshot write 5m and 1h entries, OpenAI writes 30m and 12h entries.
*/
export interface UsageDetails {
inputTokens: number;
outputTokens: number; // includes reasoning
cacheReadTokens?: number;
cacheWrite5mTokens?: number;
cacheWrite30mTokens?: number;
cacheWrite1hTokens?: number;
cacheWrite12hTokens?: number;
reasoningTokens?: number;
webSearchRequests?: number;
webFetchRequests?: number;
@@ -168,10 +174,22 @@ export interface UsageDetails {
outputTokens: number;
cacheReadTokens?: number;
cacheWrite5mTokens?: number;
cacheWrite30mTokens?: number;
cacheWrite1hTokens?: number;
cacheWrite12hTokens?: number;
}>;
}
/**
* OpenAI `prompt_cache_options`, GPT-5.6 and later. `ttl` is the minimum
* lifetime of every cache entry the request writes; `12h` entries cost twice
* what `30m` entries do to write and only the Responses API writes them.
*/
export interface PromptCacheOptions {
mode?: 'implicit' | 'explicit';
ttl?: '30m' | '12h';
}
export interface ICompleteArguments {
messages: PuterMessage[];
provider?: string;
@@ -234,7 +252,9 @@ export interface ICompleteArguments {
metadata?: Record<string, string>;
prompt?: unknown;
prompt_cache_key?: string;
prompt_cache_retention?: 'in-memory' | '24h' | undefined;
/** `in-memory` is accepted as a spelling of OpenAI's `in_memory`. */
prompt_cache_retention?: 'in-memory' | 'in_memory' | '24h' | undefined;
prompt_cache_options?: PromptCacheOptions;
store?: boolean;
top_p?: number;
truncation?: 'auto' | 'disabled' | undefined;
src/backend/drivers/ai-chat/utils/OpenAIUtil.js
+78 -20
View File
@@ -481,43 +481,86 @@ export const extractMeteredUsage = (usage) => {
};
/**
* Chat Completions usage as it's metered: `prompt_tokens` reports cached reads
* inside its count, so they come out of it and are priced only once, under
* `cached_tokens`.
* @typedef {{
* cached_tokens?: number | null;
* cache_write_tokens?: number | null;
* cache_write_tokens_details?: {
* ttl_30m_tokens?: number | null;
* ttl_12h_tokens?: number | null;
* } | null;
* }} OpenAiCacheDetails
*/
/**
* OpenAI-shaped usage as it's metered. The prompt count reports cached reads
* and cache writes inside it, so they come out of it and are priced once each
* under their own keys: `cached_tokens` for reads, `cache_write_tokens` for
* writes at the model's default lifetime and `cache_write_12h_tokens` for
* writes at OpenAI's 12h lifetime. Without the per-lifetime breakdown every
* write is a default-lifetime write: Chat Completions writes no 12h entries,
* and Responses reports the breakdown whenever a request asked for them. Reads
* the Chat Completions (`prompt_tokens`) and Responses (`input_tokens`)
* spellings alike.
*
* @param {import('openai/resources/completions.mjs').CompletionUsage | undefined} usage
* @param {{
* prompt_tokens?: number;
* completion_tokens?: number;
* prompt_tokens_details?: OpenAiCacheDetails | null;
* input_tokens?: number;
* output_tokens?: number;
* input_tokens_details?: OpenAiCacheDetails | null;
* }
* | undefined} usage
*/
export const splitCachedPrompt = (usage) => {
const details = usage?.prompt_tokens_details;
const details = usage?.prompt_tokens_details ?? usage?.input_tokens_details;
const cachedTokens = details?.cached_tokens ?? 0;
// Moonshot counts the prompt tokens it wrote to its prefix cache here;
// they're billed at the model's write rate, not the plain prompt rate.
// Moonshot and GPT-5.6+ count the prompt tokens they wrote to the prefix
// cache here; they're billed at the model's write rate, not the prompt rate.
const writeTokens =
typeof details?.cache_write_tokens === 'number'
? details.cache_write_tokens
: undefined;
const reported12h = details?.cache_write_tokens_details?.ttl_12h_tokens;
const write12hTokens =
typeof reported12h === 'number'
? Math.min(reported12h, writeTokens ?? 0)
: undefined;
return {
prompt_tokens:
(usage?.prompt_tokens ?? 0) - cachedTokens - (writeTokens ?? 0),
completion_tokens: usage?.completion_tokens ?? 0,
(usage?.prompt_tokens ?? usage?.input_tokens ?? 0) -
cachedTokens -
(writeTokens ?? 0),
completion_tokens:
usage?.completion_tokens ?? usage?.output_tokens ?? 0,
cached_tokens: cachedTokens,
...(writeTokens !== undefined
? { cache_write_tokens: writeTokens }
? { cache_write_tokens: writeTokens - (write12hTokens ?? 0) }
: {}),
...(write12hTokens !== undefined
? { cache_write_12h_tokens: write12hTokens }
: {}),
};
};
/**
* The `{prompt_tokens, completion_tokens, cached_tokens, cache_write_tokens}`
* shape every OpenAI-family `usage_calculator` already returns for billing,
* mapped to the provider-neutral `UsageDetails` the end-of-stream/non-stream
* result carries. `inputTokens`/`outputTokens` are the same numbers already
* billed — this never introduces a new figure, only exposes the breakdown.
* The `{prompt_tokens, completion_tokens, cached_tokens, cache_write_tokens,
* cache_write_12h_tokens}` shape every OpenAI-family `usage_calculator` already
* returns for billing, mapped to the provider-neutral `UsageDetails` the
* end-of-stream/non-stream result carries. `inputTokens`/`outputTokens` are the
* same numbers already billed — this never introduces a new figure, only
* exposes the breakdown.
*
* @param {Record<string, number> | undefined | null} trackedUsage
* @param {{ writeTtl?: '5m' | '30m' }} [opts] The lifetime `cache_write_tokens`
* were written with: 5m unless the provider says otherwise (OpenAI's default
* entries live 30m).
* @returns {import('../types.js').UsageDetails | undefined}
*/
export const usageDetailsFromTrackedUsage = (trackedUsage) => {
export const usageDetailsFromTrackedUsage = (
trackedUsage,
{ writeTtl = '5m' } = {},
) => {
if (!trackedUsage || typeof trackedUsage !== 'object') return undefined;
// OpenRouter, Infron and Ollama track `{prompt, completion,
// input_cache_read}` instead; any other shape is left to the driver.
@@ -526,12 +569,19 @@ export const usageDetailsFromTrackedUsage = (trackedUsage) => {
if (input === undefined && output === undefined) return undefined;
const cacheRead =
trackedUsage.cached_tokens ?? trackedUsage.input_cache_read;
const cacheWrite = trackedUsage.cache_write_tokens;
return {
inputTokens: input ?? 0,
outputTokens: output ?? 0,
...(cacheRead ? { cacheReadTokens: cacheRead } : {}),
...(trackedUsage.cache_write_tokens
? { cacheWrite5mTokens: trackedUsage.cache_write_tokens }
...(cacheWrite && writeTtl === '30m'
? { cacheWrite30mTokens: cacheWrite }
: {}),
...(cacheWrite && writeTtl !== '30m'
? { cacheWrite5mTokens: cacheWrite }
: {}),
...(trackedUsage.cache_write_12h_tokens
? { cacheWrite12hTokens: trackedUsage.cache_write_12h_tokens }
: {}),
};
};
@@ -864,12 +914,16 @@ export const create_chat_stream_handler_responses_api =
// Missing usage is left undefined rather than fed to the calculator —
// see the sibling handler above, including why usage is reported
// before the block flushes.
let usageDetails;
const usage = last_usage
? usage_calculator({
usage: last_usage,
webSearchCalls,
tool_usage: toolUsage,
setUsageCosts: (c) => chatStream.setUsageCosts(c),
setUsageDetails: (d) => {
usageDetails = d;
},
})
: undefined;
chatStream.reportUsage(usage);
@@ -877,7 +931,7 @@ export const create_chat_stream_handler_responses_api =
reason:
incompleteReason ?? (sawFunctionCall ? 'tool_use' : 'end_turn'),
});
const usageDetails = usageDetailsFromTrackedUsage(usage);
usageDetails ??= usageDetailsFromTrackedUsage(usage);
if (usageDetails) chatStream.setUsageDetails(usageDetails);
textblock.end();
@@ -1015,6 +1069,7 @@ export const handle_completion_output = async (
* webSearchCalls?: number;
* tool_usage?: Record<string, unknown>;
* setUsageCosts?: (costs: Record<string, number>) => void;
* setUsageDetails?: (details: import('../types').UsageDetails) => void;
* }) => unknown} params.usage_calculator
* @param {() => Promise<void>} [params.finally_fn]
* @returns {ReturnType<import('../types').IChatProvider['complete']>}
@@ -1156,12 +1211,15 @@ export const handle_completion_output_responses_api = async ({
setUsageCosts: (c) => {
ret.usageCosts = c;
},
setUsageDetails: (d) => {
ret.usageDetails = d;
},
})
: {
input_tokens: completion.usage.input_tokens,
output_tokens: completion.usage.output_tokens,
};
if (usage_calculator) {
if (usage_calculator && ret.usageDetails === undefined) {
const usageDetails = usageDetailsFromTrackedUsage(ret.usage);
if (usageDetails) ret.usageDetails = usageDetails;
}
src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts
+94
View File
@@ -247,6 +247,40 @@ describe('usageDetailsFromTrackedUsage', () => {
});
});
it('labels cache writes by the lifetime the provider says it wrote them with', () => {
expect(
usageDetailsFromTrackedUsage(
{
prompt_tokens: 10,
completion_tokens: 5,
cached_tokens: 3,
cache_write_tokens: 11,
cache_write_12h_tokens: 6012,
},
{ writeTtl: '30m' },
),
).toEqual({
inputTokens: 10,
outputTokens: 5,
cacheReadTokens: 3,
cacheWrite30mTokens: 11,
cacheWrite12hTokens: 6012,
});
// Zero counts, as OpenAI reports on a cache hit, add no field.
expect(
usageDetailsFromTrackedUsage(
{
prompt_tokens: 10,
completion_tokens: 5,
cached_tokens: 3,
cache_write_tokens: 0,
cache_write_12h_tokens: 0,
},
{ writeTtl: '30m' },
),
).toEqual({ inputTokens: 10, outputTokens: 5, cacheReadTokens: 3 });
});
it('returns undefined for a missing/non-object usage', () => {
expect(usageDetailsFromTrackedUsage(undefined)).toBeUndefined();
expect(usageDetailsFromTrackedUsage(null)).toBeUndefined();
@@ -962,6 +996,66 @@ describe('splitCachedPrompt', () => {
cached_tokens: 3,
});
});
it('reads the Responses usage spelling', () => {
expect(
splitCachedPrompt({
input_tokens: 6026,
output_tokens: 5,
input_tokens_details: { cached_tokens: 0, cache_write_tokens: 6023 },
}),
).toEqual({
prompt_tokens: 3,
completion_tokens: 5,
cached_tokens: 0,
cache_write_tokens: 6023,
});
});
it('splits 12h writes out of the write count when the breakdown is reported', () => {
// One request can write both lifetimes: an explicit 12h breakpoint
// plus the implicit 30m one OpenAI adds at the end of the prompt.
expect(
splitCachedPrompt({
input_tokens: 6026,
output_tokens: 5,
input_tokens_details: {
cached_tokens: 0,
cache_write_tokens: 6023,
cache_write_tokens_details: {
ttl_12h_tokens: 6012,
ttl_30m_tokens: 11,
},
},
}),
).toEqual({
prompt_tokens: 3,
completion_tokens: 5,
cached_tokens: 0,
cache_write_tokens: 11,
cache_write_12h_tokens: 6012,
});
});
it('never reports more 12h writes than writes in total', () => {
expect(
splitCachedPrompt({
input_tokens: 100,
output_tokens: 1,
input_tokens_details: {
cached_tokens: 0,
cache_write_tokens: 40,
cache_write_tokens_details: { ttl_12h_tokens: 50 },
},
}),
).toEqual({
prompt_tokens: 60,
completion_tokens: 1,
cached_tokens: 0,
cache_write_tokens: 0,
cache_write_12h_tokens: 40,
});
});
});
// ── recorded usage costs ────────────────────────────────────────────
src/backend/drivers/ai-chat/utils/openaiParams.test.ts
+49
View File
@@ -27,6 +27,8 @@ import {
outputFormatFromResponsesText,
outputFormatToResponseFormat,
outputFormatToResponsesText,
promptCacheOptionsToWire,
promptCacheRetentionToWire,
rejectStatefulResponsesFields,
toolChoiceFromWire,
toolChoiceToWire,
@@ -334,3 +336,50 @@ describe('rejectStatefulResponsesFields', () => {
).toThrow(/previous_response_id/);
});
});
describe('promptCacheRetentionToWire', () => {
it('spells the in-memory retention the way the upstreams expect', () => {
expect(promptCacheRetentionToWire('in-memory')).toBe('in_memory');
expect(promptCacheRetentionToWire('in_memory')).toBe('in_memory');
expect(promptCacheRetentionToWire('24h')).toBe('24h');
expect(promptCacheRetentionToWire(undefined)).toBeUndefined();
});
});
describe('promptCacheOptionsToWire', () => {
it('forwards mode and ttl and nothing else', () => {
expect(
promptCacheOptionsToWire({
mode: 'explicit',
ttl: '12h',
prewarm: true,
comparison_response_id: 'resp_1',
}),
).toEqual({ mode: 'explicit', ttl: '12h' });
expect(promptCacheOptionsToWire({ ttl: '30m' })).toEqual({
ttl: '30m',
});
});
it('sends nothing for an absent or empty object', () => {
expect(promptCacheOptionsToWire(undefined)).toBeUndefined();
expect(promptCacheOptionsToWire(null)).toBeUndefined();
expect(promptCacheOptionsToWire({})).toBeUndefined();
expect(promptCacheOptionsToWire({ prewarm: true })).toBeUndefined();
});
it('rejects a lifetime or mode OpenAI does not offer', () => {
expect(() => promptCacheOptionsToWire({ ttl: '2h' })).toThrow(
expect.objectContaining({ statusCode: 400 }),
);
expect(() => promptCacheOptionsToWire({ ttl: '24h' })).toThrow(
/30m.*12h/,
);
expect(() => promptCacheOptionsToWire({ mode: 'auto' })).toThrow(
/implicit.*explicit/,
);
expect(() => promptCacheOptionsToWire('12h')).toThrow(
/must be an object/,
);
});
});
src/backend/drivers/ai-chat/utils/openaiParams.ts
+55
View File
@@ -29,6 +29,7 @@ import { HttpError } from '../../../core/http/HttpError.js';
import type {
ICompleteArguments,
OutputFormat,
PromptCacheOptions,
ReasoningEffort,
ToolChoice,
} from '../types.js';
@@ -305,6 +306,60 @@ export const openAICompatParams = (
return out;
};
// -- prompt caching ----------------------------------------------------------
/**
* The wire spelling of `prompt_cache_retention`. Puter also accepts the
* hyphenated `in-memory`, which every OpenAI-shaped upstream rejects.
*/
export const promptCacheRetentionToWire = (
retention: ICompleteArguments['prompt_cache_retention'],
): 'in_memory' | '24h' | undefined =>
retention === 'in-memory' ? 'in_memory' : retention;
const PROMPT_CACHE_MODES = new Set(['implicit', 'explicit']);
const PROMPT_CACHE_TTLS = new Set(['30m', '12h']);
/**
* `prompt_cache_options` as forwarded upstream: `mode` and `ttl` only, each
* checked against the values OpenAI accepts, so a bad value fails here with a
* clear message instead of as an upstream 400 the fallback loop may hide.
* `comparison_response_id` names a stored response, which Puter never keeps,
* and `prewarm` is dropped with it.
*/
export const promptCacheOptionsToWire = (
options: unknown,
): PromptCacheOptions | undefined => {
if (options === undefined || options === null) return undefined;
if (typeof options !== 'object' || Array.isArray(options)) {
throw new HttpError(400, '`prompt_cache_options` must be an object', {
legacyCode: 'bad_request',
});
}
const { mode, ttl } = options as Record<string, unknown>;
if (mode !== undefined && !PROMPT_CACHE_MODES.has(mode as string)) {
throw new HttpError(
400,
'`prompt_cache_options.mode` must be `implicit` or `explicit`',
{ legacyCode: 'bad_request' },
);
}
if (ttl !== undefined && !PROMPT_CACHE_TTLS.has(ttl as string)) {
throw new HttpError(
400,
'`prompt_cache_options.ttl` must be `30m` or `12h`',
{ legacyCode: 'bad_request' },
);
}
const wire: PromptCacheOptions = {
...(mode !== undefined
? { mode: mode as PromptCacheOptions['mode'] }
: {}),
...(ttl !== undefined ? { ttl: ttl as PromptCacheOptions['ttl'] } : {}),
};
return Object.keys(wire).length ? wire : undefined;
};
// -- Responses server-side state ---------------------------------------------
const STATEFUL_RESPONSES_FIELDS = [
src/docs/src/AI/chat.md
+15
View File
@@ -314,6 +314,21 @@ You can find the implementation in our [prompt caching example](/playground/ai-c
Moonshot's Kimi K3 and Kimi K3.1 Pro models cache prompt prefixes on their own and take no `cache_control`. Their `usage` reports the tokens read from the cache as `cached_tokens` and the tokens written to it as `cache_write_tokens`. Each is billed at the model's own rate from the [AI models list](https://developer.puter.com/ai/models/).
OpenAI models cache prompt prefixes on their own too. Requests from the same app and user share one cache by default, and you can pick the cache with `prompt_cache_key`. Their `usage` reports the tokens read from the cache as `cached_tokens`.
GPT-5.6 and later models charge for cache writes and let you choose how long an entry lives. Pass `prompt_cache_options` with a `ttl` of `"30m"` (the default) or `"12h"`. The `usage` reports the tokens written with the 30 minute lifetime as `cache_write_tokens` and the tokens written with the 12 hour lifetime as `cache_write_12h_tokens`. Writing a 30 minute entry costs 1.25 times the model's input rate, and writing a 12 hour entry costs 2.5 times the input rate. Reads cost the same for both lifetimes. The 12 hour lifetime is only available through the OpenAI Responses API, so a request with `ttl: "12h"` is sent there.
```js
puter.ai.chat('<your message>', {
model: 'gpt-6-luna',
prompt_cache_options: { ttl: '12h' }
});
```
Older OpenAI models charge nothing for cache writes. Pass `prompt_cache_retention` with `"in_memory"` for an entry that lives a few minutes or `"24h"` for one that can live up to a day. The 24 hour option is the default, and GPT-5.5 and later models only support it.
The `usageDetails` object on every result carries the same counts under provider-neutral names, with one field per lifetime, such as `cacheWrite30mTokens` and `cacheWrite12hTokens`. See the [ChatResponse](/Objects/chatresponse/) object for the full list.
## Compaction
For long, multi-turn conversations that you keep on the client, enable
src/docs/src/Objects/chatresponse.md
+1 -1
View File
@@ -64,7 +64,7 @@ Extra detail on why generation stopped (e.g. a refusal classification). Provider
#### `usageDetails` (Object)
A provider-neutral usage breakdown, present alongside `usage` on every result. `inputTokens` excludes cache reads/writes; `outputTokens` includes reasoning tokens. Fields: `inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWrite5mTokens`, `cacheWrite1hTokens`, `reasoningTokens`, `webSearchRequests`, `webFetchRequests`, `speed` (`"fast"` or `"standard"`), `serviceTier`, and `iterations` (a per-pass breakdown when the upstream ran several passes, e.g. compaction or an advisor sub-call).
A provider-neutral usage breakdown, present alongside `usage` on every result. `inputTokens` excludes cache reads/writes; `outputTokens` includes reasoning tokens. Fields: `inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWrite5mTokens`, `cacheWrite30mTokens`, `cacheWrite1hTokens`, `cacheWrite12hTokens`, `reasoningTokens`, `webSearchRequests`, `webFetchRequests`, `speed` (`"fast"` or `"standard"`), `serviceTier`, and `iterations` (a per-pass breakdown when the upstream ran several passes, e.g. compaction or an advisor sub-call). Each `cacheWrite*Tokens` field counts the tokens written to the cache with that lifetime: Claude and Moonshot write 5 minute and 1 hour entries, OpenAI writes 30 minute and 12 hour entries.
Prefer `usageDetails` over `usage` for anything beyond the raw billing totals. Claude's `usage.thinking_tokens` is informational: it's included in `output_tokens` and not billed separately — `usageDetails.reasoningTokens` carries the same count.
src/docs/src/Objects/chatresponsechunk.md
+1 -1
View File
@@ -102,7 +102,7 @@ Extra detail on why generation stopped (e.g. a refusal classification). Provider
#### `usageDetails` (Object)
A provider-neutral usage breakdown, present on the final `usage` chunk. `inputTokens` excludes cache reads/writes; `outputTokens` includes reasoning tokens. Fields: `inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWrite5mTokens`, `cacheWrite1hTokens`, `reasoningTokens`, `webSearchRequests`, `webFetchRequests`, `speed` (`"fast"` or `"standard"`), `serviceTier`, and `iterations` (a per-pass breakdown when the upstream ran several passes, e.g. compaction or an advisor sub-call).
A provider-neutral usage breakdown, present on the final `usage` chunk. `inputTokens` excludes cache reads/writes; `outputTokens` includes reasoning tokens. Fields: `inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWrite5mTokens`, `cacheWrite30mTokens`, `cacheWrite1hTokens`, `cacheWrite12hTokens`, `reasoningTokens`, `webSearchRequests`, `webFetchRequests`, `speed` (`"fast"` or `"standard"`), `serviceTier`, and `iterations` (a per-pass breakdown when the upstream ran several passes, e.g. compaction or an advisor sub-call). Each `cacheWrite*Tokens` field counts the tokens written to the cache with that lifetime: Claude and Moonshot write 5 minute and 1 hour entries, OpenAI writes 30 minute and 12 hour entries.
#### `message` (String)
src/puter-js/src/modules/ai/ai.test.js
+14
View File
@@ -171,6 +171,20 @@ describe('ai.chat driver payloads', () => {
});
});
it('chat(prompt, options) forwards the OpenAI prompt-cache options', async () => {
await ai.chat('hello', {
prompt_cache_key: 'ck',
prompt_cache_retention: '24h',
prompt_cache_options: { mode: 'explicit', ttl: '12h' },
});
expect(lastBody().args).toEqual({
messages: [{ content: 'hello' }],
prompt_cache_key: 'ck',
prompt_cache_retention: '24h',
prompt_cache_options: { mode: 'explicit', ttl: '12h' },
});
});
it('chat(prompt, options) forwards temperature: 0 and max_tokens: 0', async () => {
await ai.chat('hello', {
temperature: 0,
src/puter-js/src/modules/ai/chat.js
+3
View File
@@ -27,6 +27,9 @@ const PARAMS_TO_PASS = [
'outputFormat',
'thinking',
'streamToolInput',
'prompt_cache_key',
'prompt_cache_retention',
'prompt_cache_options',
];
// Parameters renamed between the SDK's camelCase option and the driver's
src/puter-js/src/modules/ai/types.js
+13 -3
View File
@@ -115,6 +115,14 @@
* @property {boolean} [streamToolInput] Streaming only: emit incremental `tool_use_start` /
* `tool_input_delta` chunks as a tool call's arguments are generated, instead of a single `tool_use`
* chunk once the call completes.
* @property {string} [prompt_cache_key] OpenAI models: which prompt cache the request reads from and
* writes to. Defaults to one cache per app and user.
* @property {'in_memory' | '24h'} [prompt_cache_retention] OpenAI models before GPT-5.6: how long a
* cache entry may live. `'24h'` is the default and the only value GPT-5.5 and later accept.
* @property {{ mode?: 'implicit' | 'explicit', ttl?: '30m' | '12h' }} [prompt_cache_options] OpenAI
* GPT-5.6 and later: `ttl` is the minimum lifetime of every cache entry the request writes. A `'12h'`
* entry costs twice as much to write as the default `'30m'` one and is served through the Responses
* API. `mode: 'explicit'` only writes at the `prompt_cache_breakpoint` markers in your messages.
*/
/**
@@ -132,14 +140,16 @@
* @property {number} inputTokens
* @property {number} outputTokens
* @property {number} [cacheReadTokens]
* @property {number} [cacheWrite5mTokens]
* @property {number} [cacheWrite1hTokens]
* @property {number} [cacheWrite5mTokens] Tokens written to the cache with a 5 minute lifetime (Claude, Moonshot).
* @property {number} [cacheWrite30mTokens] Tokens written with a 30 minute lifetime (OpenAI's default).
* @property {number} [cacheWrite1hTokens] Tokens written with a 1 hour lifetime (Claude, Moonshot).
* @property {number} [cacheWrite12hTokens] Tokens written with a 12 hour lifetime (OpenAI, `prompt_cache_options.ttl: '12h'`).
* @property {number} [reasoningTokens]
* @property {number} [webSearchRequests]
* @property {number} [webFetchRequests]
* @property {'fast' | 'standard'} [speed]
* @property {string} [serviceTier]
* @property {Array<{ type: string, model?: string, inputTokens: number, outputTokens: number, cacheReadTokens?: number, cacheWrite5mTokens?: number, cacheWrite1hTokens?: number }>} [iterations]
* @property {Array<{ type: string, model?: string, inputTokens: number, outputTokens: number, cacheReadTokens?: number, cacheWrite5mTokens?: number, cacheWrite30mTokens?: number, cacheWrite1hTokens?: number, cacheWrite12hTokens?: number }>} [iterations]
* Per-pass breakdown when the upstream ran several passes (compaction, an advisor sub-call).
*/