diff --git a/src/backend/controllers/puterai/PuterAIController.shapes.test.ts b/src/backend/controllers/puterai/PuterAIController.shapes.test.ts index 77631e7f9..42c7b1849 100644 --- a/src/backend/controllers/puterai/PuterAIController.shapes.test.ts +++ b/src/backend/controllers/puterai/PuterAIController.shapes.test.ts @@ -201,6 +201,7 @@ describe('PuterAIController.openaiResponses parameter forwarding', () => { context_management: [{ type: 'compaction' }], prompt_cache_key: 'ck', prompt_cache_retention: '24h', + prompt_cache_options: { ttl: '12h', prewarm: true }, store: true, truncation: 'auto', service_tier: 'default', @@ -226,6 +227,9 @@ describe('PuterAIController.openaiResponses parameter forwarding', () => { context_management: [{ type: 'compaction' }], prompt_cache_key: 'ck', prompt_cache_retention: '24h', + // Forwarded as sent; the provider keeps only the fields it + // accepts. + prompt_cache_options: { ttl: '12h', prewarm: true }, // forced false regardless of the client's request. store: false, truncation: 'auto', @@ -639,6 +643,33 @@ describe('PuterAIController.openaiResponses output items', () => { // -- /openai/v1/chat/completions edges ------------------------------- describe('PuterAIController.openaiChatCompletions translation edges', () => { + it('forwards the prompt-cache fields to the driver', async () => { + const completeSpy = stubChatComplete({ + message: { role: 'assistant', content: 'ok' }, + }); + + const { res } = makeRes(); + await controller.openaiChatCompletions( + makeReq({ + body: { + model: 'gpt-test', + messages: [{ role: 'user', content: 'hi' }], + prompt_cache_key: 'ck', + prompt_cache_retention: 'in_memory', + prompt_cache_options: { ttl: '30m' }, + }, + }), + res, + ); + + const args = completeSpy.mock.calls[0]![0] as Record; + expect(args).toMatchObject({ + prompt_cache_key: 'ck', + prompt_cache_retention: 'in_memory', + prompt_cache_options: { ttl: '30m' }, + }); + }); + it('derives tool_calls from tool_use content blocks when the message has none', async () => { stubChatComplete({ message: { diff --git a/src/backend/controllers/puterai/PuterAIController.ts b/src/backend/controllers/puterai/PuterAIController.ts index 02c27bb31..3ea11b39a 100644 --- a/src/backend/controllers/puterai/PuterAIController.ts +++ b/src/backend/controllers/puterai/PuterAIController.ts @@ -447,6 +447,7 @@ export class PuterAIController extends PuterController { body.reasoning_effort as ICompleteArguments['reasoning_effort'], } : {}), + ...promptCacheArgsFromBody(body), ...openaiCompatProvider(body), ...(body.fallback === false ? { fallback: false } : {}), }; @@ -842,15 +843,7 @@ export class PuterAIController extends PuterController { body.compaction as ICompleteArguments['compaction'], } : {}), - ...(body.prompt_cache_key - ? { prompt_cache_key: String(body.prompt_cache_key) } - : {}), - ...(body.prompt_cache_retention - ? { - prompt_cache_retention: - body.prompt_cache_retention as ICompleteArguments['prompt_cache_retention'], - } - : {}), + ...promptCacheArgsFromBody(body), // forced off regardless of what the caller sent — OpenAI // defaults `store` to true, which would persist state server-side // on OpenAI's end that this route gives no way to retrieve. @@ -1343,6 +1336,33 @@ const openaiCompatProvider = ( return { provider: DEFAULTS.openaiChat }; }; +/** + * The prompt-cache fields both OpenAI-shaped routes forward as sent; the + * provider checks the values. + */ +const promptCacheArgsFromBody = ( + body: Record, +): Pick< + ICompleteArguments, + 'prompt_cache_key' | 'prompt_cache_retention' | 'prompt_cache_options' +> => ({ + ...(body.prompt_cache_key + ? { prompt_cache_key: String(body.prompt_cache_key) } + : {}), + ...(body.prompt_cache_retention + ? { + prompt_cache_retention: + body.prompt_cache_retention as ICompleteArguments['prompt_cache_retention'], + } + : {}), + ...(body.prompt_cache_options + ? { + prompt_cache_options: + body.prompt_cache_options as ICompleteArguments['prompt_cache_options'], + } + : {}), +}); + const randomId = (): string => crypto.randomUUID().replace(/-/g, ''); const generateId = (prefix: string): string => `${prefix}_${randomId()}`; diff --git a/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.test.ts b/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.test.ts index 2d1d249ab..2b5164743 100644 --- a/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.test.ts @@ -449,6 +449,22 @@ describe('AzureResponsesProvider.complete request shape', () => { expect(args.service_tier).toBe('default'); }); + it('sends the wire spelling of the in-memory cache retention', async () => { + const provider = makeProvider(); + responsesCreateMock.mockResolvedValueOnce(okResponse); + + await withTestActor(() => + provider.complete({ + model: 'gpt-5.3-codex', + messages: [{ role: 'user', content: 'hi' }], + prompt_cache_retention: 'in-memory', + } as never), + ); + + const [args] = responsesCreateMock.mock.calls[0]!; + expect(args.prompt_cache_retention).toBe('in_memory'); + }); + it.each([ ['previous_response_id', 'resp_1'], ['conversation', 'conv_1'], diff --git a/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts b/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts index 9152b691e..e98720737 100644 --- a/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts +++ b/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts @@ -35,6 +35,7 @@ import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; import { clampReasoningEffort, openAICompatParams, + promptCacheRetentionToWire, rejectStatefulResponsesFields, } from '../../utils/openaiParams.js'; import { buildCostsOverride } from '../../utils/pricing.js'; @@ -209,6 +210,9 @@ export class AzureResponsesProvider implements IChatProvider { const userIdentifier = upstreamUserIdentifier(actor); // Cache key defaults to the actor identifier; see upstreamUserIdentifier. const cacheKey = prompt_cache_key ?? userIdentifier; + const cacheRetention = promptCacheRetentionToWire( + prompt_cache_retention, + ); // Resolve any `puter_path` content parts into inline base64 data URLs // before the Responses API sees them. @@ -307,8 +311,8 @@ export class AzureResponsesProvider implements IChatProvider { ...(instructions !== undefined ? { instructions } : {}), ...(metadata !== undefined ? { metadata } : {}), ...(cacheKey !== undefined ? { prompt_cache_key: cacheKey } : {}), - ...(prompt_cache_retention !== undefined - ? { prompt_cache_retention } + ...(cacheRetention !== undefined + ? { prompt_cache_retention: cacheRetention } : {}), ...(store !== undefined ? { store } : {}), ...(max_tokens !== undefined diff --git a/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts b/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts index c0ca50e02..f3e94615f 100644 --- a/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts +++ b/src/backend/drivers/ai-chat/providers/meta/MetaProvider.ts @@ -23,6 +23,7 @@ import { HttpError } from '../../../../core/http/HttpError.js'; import type { MeteringService } from '../../../../services/metering/MeteringService.js'; import { upstreamUserIdentifier } from '../../../util/upstreamIdentifier.js'; import type { IChatModel, ICompleteArguments } from '../../types.js'; +import { promptCacheRetentionToWire } from '../../utils/openaiParams.js'; import { sdkClientOptions } from '../../utils/sdkClient.js'; import { type ChatProviderConfig, @@ -107,12 +108,9 @@ export class MetaProvider extends OpenAICompatProvider { ? requestedEffort : undefined; - // Puter spells the in-memory retention with a hyphen; Meta's enum - // uses an underscore. - const cacheRetention = - args.prompt_cache_retention === 'in-memory' - ? 'in_memory' - : args.prompt_cache_retention; + const cacheRetention = promptCacheRetentionToWire( + args.prompt_cache_retention, + ); // The identifier is Puter's abuse attribution, so `custom` can't // override it. Cache key defaults to it; see upstreamUserIdentifier. diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.test.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.test.ts index 56a7c02bc..4aa943a4b 100644 --- a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.test.ts @@ -249,6 +249,44 @@ describe('OpenAiChatProvider.complete argument validation', () => { expect(result).toEqual({ delegated: true }); }); + it('delegates a 12h cache lifetime to the Responses provider, which is the only API that writes it', async () => { + const { provider } = makeProvider(); + const sibling = { + complete: vi.fn().mockResolvedValue({ delegated: true }), + }; + provider.setResponsesProvider( + sibling as unknown as Parameters< + typeof provider.setResponsesProvider + >[0], + ); + + const params = { + model: 'gpt-6-luna', + messages: [{ role: 'user', content: 'hi' }], + prompt_cache_options: { ttl: '12h' as const }, + }; + const result = await withTestActor(() => provider.complete(params)); + + expect(createMock).not.toHaveBeenCalled(); + expect(sibling.complete).toHaveBeenCalledWith(params); + expect(result).toEqual({ delegated: true }); + }); + + it('throws 400 for a 12h cache lifetime without a Responses sibling provider', async () => { + const { provider } = makeProvider(); + + await expect( + withTestActor(() => + provider.complete({ + model: 'gpt-6-luna', + messages: [{ role: 'user', content: 'hi' }], + prompt_cache_options: { ttl: '12h' }, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(createMock).not.toHaveBeenCalled(); + }); + it('requestPricing delegates to the Responses sibling (web_search\'s own hold), and no-ops without one', async () => { const { provider } = makeProvider(); const args = { @@ -349,6 +387,31 @@ describe('OpenAiChatProvider.complete request shape', () => { expect(args.safety_identifier).toBe('puter-u42'); }); + it('forwards 30m prompt_cache_options and the wire spelling of prompt_cache_retention', async () => { + const { provider } = makeProvider(); + createMock.mockResolvedValueOnce(baseCompletion); + + await withTestActor(() => + provider.complete({ + model: 'gpt-6-luna', + messages: [{ role: 'user', content: 'hello' }], + prompt_cache_retention: 'in-memory', + prompt_cache_options: { + mode: 'explicit', + ttl: '30m', + prewarm: true, + } as never, + }), + ); + + const [args] = createMock.mock.calls[0]!; + expect(args.prompt_cache_retention).toBe('in_memory'); + expect(args.prompt_cache_options).toEqual({ + mode: 'explicit', + ttl: '30m', + }); + }); + it.each([ 'gpt-6-astra', 'gpt-6.1-sol', @@ -665,7 +728,7 @@ describe('OpenAiChatProvider.complete non-stream output', () => { }, }); - await withTestActor(() => + const result = await withTestActor(() => provider.complete({ model: 'gpt-6-luna', messages: [{ role: 'user', content: 'hi' }], @@ -685,6 +748,13 @@ describe('OpenAiChatProvider.complete non-stream output', () => { cached_tokens: 1000 * Number(luna.costs.cached_tokens), cache_write_tokens: 3000 * Number(luna.costs.cache_write_tokens), }); + // Chat Completions only writes OpenAI's default 30m entries. + expect((result as { usageDetails: unknown }).usageDetails).toEqual({ + inputTokens: 1000, + outputTokens: 12, + cacheReadTokens: 1000, + cacheWrite30mTokens: 3000, + }); }); it('zeroes cached_tokens when prompt_tokens_details is missing', async () => { diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts index 92e05e4e5..553712fcd 100644 --- a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts @@ -36,6 +36,8 @@ import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; import { clampReasoningEffort, openAICompatParams, + promptCacheOptionsToWire, + promptCacheRetentionToWire, } from '../../utils/openaiParams.js'; import { OPEN_AI_MODELS } from './models.js'; import type { OpenAiResponsesChatProvider } from './OpenAiChatResponsesProvider.js'; @@ -126,8 +128,11 @@ export class OpenAiChatProvider implements IChatProvider { temperature, text, prompt_cache_key, + prompt_cache_retention, + prompt_cache_options, } = params; let { messages, model } = params; + const cacheOptions = promptCacheOptionsToWire(prompt_cache_options); if ( tools?.filter((e) => isWebSearchTool(e as Record)) .length @@ -157,6 +162,18 @@ export class OpenAiChatProvider implements IChatProvider { } return await this.#responsesProvider.complete(params); } + // Only the Responses API writes 12h cache entries; Chat Completions + // rejects the lifetime. + if (cacheOptions?.ttl === '12h') { + if (!this.#responsesProvider) { + throw new HttpError( + 400, + 'a 12h prompt cache lifetime requires the OpenAI Responses provider, which is not configured', + { legacyCode: 'bad_request' }, + ); + } + return await this.#responsesProvider.complete(params); + } // Validate messages if (!Array.isArray(messages)) { throw new HttpError(400, '`messages` must be an array', { @@ -180,6 +197,9 @@ export class OpenAiChatProvider implements IChatProvider { const userIdentifier = upstreamUserIdentifier(actor); // Cache key defaults to the actor identifier; see upstreamUserIdentifier. const cacheKey = prompt_cache_key ?? userIdentifier; + const cacheRetention = promptCacheRetentionToWire( + prompt_cache_retention, + ); // Strip Anthropic-only shape a fallback-replayed message can carry // (thinking/server-tool blocks, cache_control, citations) before the @@ -216,6 +236,12 @@ export class OpenAiChatProvider implements IChatProvider { user: userIdentifier, safety_identifier: userIdentifier, ...(cacheKey !== undefined ? { prompt_cache_key: cacheKey } : {}), + ...(cacheRetention !== undefined + ? { prompt_cache_retention: cacheRetention } + : {}), + ...(cacheOptions !== undefined + ? { prompt_cache_options: cacheOptions } + : {}), messages: messages, model: modelUsed.id, ...(mappedTools?.length ? { tools: mappedTools } : {}), @@ -254,28 +280,11 @@ export class OpenAiChatProvider implements IChatProvider { ); return OpenAiUtil.handle_completion_output({ - usage_calculator: ({ usage, setUsageCosts }) => { - const cachedTokens = - usage.prompt_tokens_details?.cached_tokens ?? 0; - // GPT-5.6 and later bill cache writes at 1.25x input. They're - // reported inside `prompt_tokens`, like cached reads. - // The SDK doesn't type `cache_write_tokens` yet. - const cacheWriteTokens = - ( - usage.prompt_tokens_details as - { cache_write_tokens?: number } | undefined - )?.cache_write_tokens ?? 0; - const trackedUsage = { - prompt_tokens: - (usage.prompt_tokens ?? 0) - - cachedTokens - - cacheWriteTokens, - completion_tokens: usage.completion_tokens ?? 0, - cached_tokens: cachedTokens, - ...(cacheWriteTokens - ? { cache_write_tokens: cacheWriteTokens } - : {}), - }; + usage_calculator: ({ usage, setUsageCosts, setUsageDetails }) => { + // GPT-5.6 and later report cache writes inside + // `prompt_tokens`, like cached reads; Chat Completions only + // writes the default 30m entries, billed at 1.25x input. + const trackedUsage = OpenAiUtil.splitCachedPrompt(usage); const metered = meterChatUsage( this.#meteringService, @@ -285,6 +294,11 @@ export class OpenAiChatProvider implements IChatProvider { trackedUsage, ); setUsageCosts(metered.costs); + setUsageDetails( + OpenAiUtil.usageDetailsFromTrackedUsage(metered.usage, { + writeTtl: '30m', + })!, + ); return metered.usage; }, stream, diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.test.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.test.ts index 509e6ec4d..7ce8889fa 100644 --- a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.test.ts @@ -246,6 +246,7 @@ describe('OpenAiResponsesChatProvider model catalog', () => { prompt_tokens: input, cached_tokens: cached, cache_write_tokens: input * 1.25, + cache_write_12h_tokens: input * 2.5, completion_tokens: output, }, context: 1_050_000, @@ -844,7 +845,7 @@ describe('OpenAiResponsesChatProvider.complete non-stream output', () => { }, }); - await withTestActor(() => + const result = await withTestActor(() => provider.complete({ model: 'gpt-6-luna', messages: [{ role: 'user', content: 'hi' }], @@ -867,6 +868,115 @@ describe('OpenAiResponsesChatProvider.complete non-stream output', () => { cached_tokens: 1000 * Number(luna.costs.cached_tokens), cache_write_tokens: 3000 * Number(luna.costs.cache_write_tokens), }); + // Writes without a per-lifetime breakdown are OpenAI's default 30m + // entries, and the neutral breakdown says so. + expect((result as { usageDetails: unknown }).usageDetails).toEqual({ + inputTokens: 1000, + outputTokens: 20, + cacheReadTokens: 1000, + cacheWrite30mTokens: 3000, + }); + }); + + it('bills 12h cache writes at 2.5x input and the other writes at 1.25x', async () => { + const luna = OPEN_AI_MODELS.find((m) => m.id === 'gpt-6-luna')!; + const { provider } = makeProvider(); + // One request can write both lifetimes: an explicit 12h breakpoint + // plus the implicit 30m one OpenAI adds at the end of the prompt. + responsesCreateMock.mockResolvedValueOnce({ + output: [{ role: 'assistant' }], + output_text: 'ok', + usage: { + input_tokens: 6026, + output_tokens: 5, + input_tokens_details: { + cached_tokens: 0, + cache_write_tokens: 6023, + cache_write_tokens_details: { + ttl_12h_tokens: 6012, + ttl_30m_tokens: 11, + }, + }, + }, + }); + + const result = await withTestActor(() => + provider.complete({ + model: 'gpt-6-luna', + messages: [{ role: 'user', content: 'hi' }], + prompt_cache_options: { ttl: '12h' }, + }), + ); + + const [usage, , , overrides] = recordSpy.mock.calls[0]!; + expect(usage).toEqual({ + prompt_tokens: 3, + completion_tokens: 5, + cached_tokens: 0, + cache_write_tokens: 11, + cache_write_12h_tokens: 6012, + }); + expect(luna.costs.cache_write_12h_tokens).toBe( + Number(luna.costs.prompt_tokens) * 2.5, + ); + expect(overrides).toEqual({ + prompt_tokens: 3 * Number(luna.costs.prompt_tokens), + completion_tokens: 5 * Number(luna.costs.completion_tokens), + cached_tokens: 0, + cache_write_tokens: 11 * Number(luna.costs.cache_write_tokens), + cache_write_12h_tokens: + 6012 * Number(luna.costs.cache_write_12h_tokens), + }); + expect((result as { usageDetails: unknown }).usageDetails).toEqual({ + inputTokens: 3, + outputTokens: 5, + cacheWrite30mTokens: 11, + cacheWrite12hTokens: 6012, + }); + }); + + it('forwards prompt_cache_options and the wire spelling of prompt_cache_retention', async () => { + const { provider } = makeProvider(); + responsesCreateMock.mockResolvedValueOnce({ + output: [{ role: 'assistant' }], + output_text: 'ok', + usage: { input_tokens: 1, output_tokens: 1 }, + }); + + await withTestActor(() => + provider.complete({ + model: 'gpt-6-luna', + messages: [{ role: 'user', content: 'hi' }], + prompt_cache_retention: 'in-memory', + prompt_cache_options: { + mode: 'explicit', + ttl: '12h', + comparison_response_id: 'resp_1', + } as never, + }), + ); + + const [args] = responsesCreateMock.mock.calls[0]!; + expect(args.prompt_cache_retention).toBe('in_memory'); + expect(args.prompt_cache_options).toEqual({ + mode: 'explicit', + ttl: '12h', + }); + }); + + it('rejects a cache lifetime OpenAI does not offer before calling upstream', async () => { + const { provider } = makeProvider(); + + await expect( + withTestActor(() => + provider.complete({ + model: 'gpt-6-luna', + messages: [{ role: 'user', content: 'hi' }], + prompt_cache_options: { ttl: '2h' } as never, + }), + ), + ).rejects.toMatchObject({ statusCode: 400 }); + expect(responsesCreateMock).not.toHaveBeenCalled(); }); it.each([ @@ -1068,6 +1178,69 @@ describe('OpenAiResponsesChatProvider.complete streaming', () => { }); }); + it('labels streamed cache writes by lifetime on the final usage chunk', async () => { + const luna = OPEN_AI_MODELS.find((m) => m.id === 'gpt-6-luna')!; + const { provider } = makeProvider(); + responsesCreateMock.mockReturnValueOnce( + asAsyncIterable([ + { type: 'response.output_text.delta', delta: 'ok' }, + { + type: 'response.completed', + response: { + usage: { + input_tokens: 6026, + output_tokens: 5, + input_tokens_details: { + cached_tokens: 0, + cache_write_tokens: 6023, + cache_write_tokens_details: { + ttl_12h_tokens: 6012, + ttl_30m_tokens: 11, + }, + }, + }, + }, + }, + ]), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'gpt-6-luna', + messages: [{ role: 'user', content: 'say ok' }], + stream: true, + prompt_cache_options: { ttl: '12h' }, + }), + ); + const harness = makeCapturingChatStream(); + await ( + result as { + init_chat_stream: (p: { chatStream: unknown }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }); + + const usageEvent = harness.events().find((e) => e.type === 'usage'); + expect(usageEvent?.usage).toEqual({ + prompt_tokens: 3, + completion_tokens: 5, + cached_tokens: 0, + cache_write_tokens: 11, + cache_write_12h_tokens: 6012, + }); + expect(usageEvent?.usageDetails).toEqual({ + inputTokens: 3, + outputTokens: 5, + cacheWrite30mTokens: 11, + cacheWrite12hTokens: 6012, + }); + const [, , , overrides] = recordSpy.mock.calls[0]!; + expect(overrides).toMatchObject({ + cache_write_tokens: 11 * Number(luna.costs.cache_write_tokens), + cache_write_12h_tokens: + 6012 * Number(luna.costs.cache_write_12h_tokens), + }); + }); + it('emits a tool_use block when response.output_item.done arrives with a function_call', async () => { const { provider } = makeProvider(); responsesCreateMock.mockReturnValueOnce( diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts index 0f1deee10..79e5352dd 100644 --- a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts @@ -32,6 +32,8 @@ import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; import { clampReasoningEffort, openAICompatParams, + promptCacheOptionsToWire, + promptCacheRetentionToWire, rejectStatefulResponsesFields, } from '../../utils/openaiParams.js'; import { responseSamplingParams } from '../../utils/responseSampling.js'; @@ -150,6 +152,7 @@ export class OpenAiResponsesChatProvider implements IChatProvider { prompt, prompt_cache_key, prompt_cache_retention, + prompt_cache_options, store, top_p, truncation, @@ -195,6 +198,10 @@ export class OpenAiResponsesChatProvider implements IChatProvider { const userIdentifier = upstreamUserIdentifier(actor); // Cache key defaults to the actor identifier; see upstreamUserIdentifier. const cacheKey = prompt_cache_key ?? userIdentifier; + const cacheRetention = promptCacheRetentionToWire( + prompt_cache_retention, + ); + const cacheOptions = promptCacheOptionsToWire(prompt_cache_options); if (tools) { // Unravel tools to OpenAI Responses API format @@ -288,8 +295,11 @@ export class OpenAiResponsesChatProvider implements IChatProvider { ...(instructions !== undefined ? { instructions } : {}), ...(metadata !== undefined ? { metadata } : {}), ...(cacheKey !== undefined ? { prompt_cache_key: cacheKey } : {}), - ...(prompt_cache_retention !== undefined - ? { prompt_cache_retention } + ...(cacheRetention !== undefined + ? { prompt_cache_retention: cacheRetention } + : {}), + ...(cacheOptions !== undefined + ? { prompt_cache_options: cacheOptions } : {}), ...(store !== undefined ? { store } : {}), ...(max_tokens !== undefined @@ -310,24 +320,17 @@ export class OpenAiResponsesChatProvider implements IChatProvider { { signal: Context.get('upstreamSignal') }, ); return OpenAiUtil.handle_completion_output_responses_api({ - usage_calculator: ({ usage, webSearchCalls, setUsageCosts }) => { - const cachedTokens = - (usage as any).input_tokens_details?.cached_tokens ?? 0; - // GPT-5.6 and later bill cache writes at 1.25x input. They're - // reported inside `input_tokens`, like cached reads. - const cacheWriteTokens = - (usage as any).input_tokens_details?.cache_write_tokens ?? - 0; + usage_calculator: ({ + usage, + webSearchCalls, + setUsageCosts, + setUsageDetails, + }) => { + // GPT-5.6 and later report cache writes inside `input_tokens`, + // like cached reads, split by lifetime (30m and 12h entries + // cost 1.25x and 2.5x input to write). const trackedUsage: Record = { - prompt_tokens: - ((usage as any).input_tokens ?? 0) - - cachedTokens - - cacheWriteTokens, - completion_tokens: (usage as any).output_tokens ?? 0, - cached_tokens: cachedTokens, - ...(cacheWriteTokens - ? { cache_write_tokens: cacheWriteTokens } - : {}), + ...OpenAiUtil.splitCachedPrompt(usage), ...(webSearchCalls ? { web_search_calls: webSearchCalls } : {}), @@ -356,6 +359,11 @@ export class OpenAiResponsesChatProvider implements IChatProvider { : {}, ); setUsageCosts?.(metered.costs); + setUsageDetails?.( + OpenAiUtil.usageDetailsFromTrackedUsage(metered.usage, { + writeTtl: '30m', + })!, + ); return metered.usage; }, stream, diff --git a/src/backend/drivers/ai-chat/providers/openai/models.ts b/src/backend/drivers/ai-chat/providers/openai/models.ts index 241e26418..378480b9e 100644 --- a/src/backend/drivers/ai-chat/providers/openai/models.ts +++ b/src/backend/drivers/ai-chat/providers/openai/models.ts @@ -23,6 +23,8 @@ import type { IChatModel } from '../../types.js'; // Prompts over 272K input tokens are billed at 2x input (cached reads and // cache writes included) and 1.5x output for the full request. +// GPT-5.6 and later bill cache writes by the lifetime requested: 1.25x input +// for the default 30m entries, 2.5x for 12h entries. const GPT_LONG_CONTEXT_PRICING = { threshold: 272_000, input_multiplier: 2, @@ -49,6 +51,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ prompt_tokens: 100, cached_tokens: 5, cache_write_tokens: 100 * 1.25, + cache_write_12h_tokens: 100 * 2.5, completion_tokens: 500, }, long_context_pricing: GPT_LONG_CONTEXT_PRICING, @@ -74,6 +77,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ prompt_tokens: 200, cached_tokens: 10, cache_write_tokens: 200 * 1.25, + cache_write_12h_tokens: 200 * 2.5, completion_tokens: 1000, }, long_context_pricing: GPT_LONG_CONTEXT_PRICING, @@ -99,6 +103,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ prompt_tokens: 200, cached_tokens: 20, cache_write_tokens: 200 * 1.25, + cache_write_12h_tokens: 200 * 2.5, completion_tokens: 1000, }, long_context_pricing: GPT_LONG_CONTEXT_PRICING, @@ -124,6 +129,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ prompt_tokens: 10, cached_tokens: 1, cache_write_tokens: 10 * 1.25, + cache_write_12h_tokens: 10 * 2.5, completion_tokens: 50, }, long_context_pricing: GPT_LONG_CONTEXT_PRICING, @@ -149,6 +155,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ prompt_tokens: 1000, cached_tokens: 100, cache_write_tokens: 1000 * 1.25, + cache_write_12h_tokens: 1000 * 2.5, completion_tokens: 5000, }, long_context_pricing: GPT_LONG_CONTEXT_PRICING, @@ -174,6 +181,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ prompt_tokens: 400, cached_tokens: 40, cache_write_tokens: 400 * 1.25, + cache_write_12h_tokens: 400 * 2.5, completion_tokens: 2000, }, long_context_pricing: GPT_LONG_CONTEXT_PRICING, @@ -198,6 +206,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ prompt_tokens: 200, cached_tokens: 20, cache_write_tokens: 200 * 1.25, + cache_write_12h_tokens: 200 * 2.5, completion_tokens: 1200, }, long_context_pricing: GPT_LONG_CONTEXT_PRICING, @@ -222,6 +231,7 @@ export const OPEN_AI_MODELS: IChatModel[] = [ prompt_tokens: 20, cached_tokens: 2, cache_write_tokens: 20 * 1.25, + cache_write_12h_tokens: 20 * 2.5, completion_tokens: 120, }, long_context_pricing: GPT_LONG_CONTEXT_PRICING, diff --git a/src/backend/drivers/ai-chat/types.ts b/src/backend/drivers/ai-chat/types.ts index 3208e47de..bd59eee28 100644 --- a/src/backend/drivers/ai-chat/types.ts +++ b/src/backend/drivers/ai-chat/types.ts @@ -145,13 +145,19 @@ export type StopReason = | 'model_context_window_exceeded' | (string & {}); -/** Provider-neutral usage breakdown. `inputTokens` excludes cache reads/writes. */ +/** + * Provider-neutral usage breakdown. `inputTokens` excludes cache reads/writes. + * Cache writes are keyed by the lifetime the entry was written with: Claude and + * Moonshot write 5m and 1h entries, OpenAI writes 30m and 12h entries. + */ export interface UsageDetails { inputTokens: number; outputTokens: number; // includes reasoning cacheReadTokens?: number; cacheWrite5mTokens?: number; + cacheWrite30mTokens?: number; cacheWrite1hTokens?: number; + cacheWrite12hTokens?: number; reasoningTokens?: number; webSearchRequests?: number; webFetchRequests?: number; @@ -168,10 +174,22 @@ export interface UsageDetails { outputTokens: number; cacheReadTokens?: number; cacheWrite5mTokens?: number; + cacheWrite30mTokens?: number; cacheWrite1hTokens?: number; + cacheWrite12hTokens?: number; }>; } +/** + * OpenAI `prompt_cache_options`, GPT-5.6 and later. `ttl` is the minimum + * lifetime of every cache entry the request writes; `12h` entries cost twice + * what `30m` entries do to write and only the Responses API writes them. + */ +export interface PromptCacheOptions { + mode?: 'implicit' | 'explicit'; + ttl?: '30m' | '12h'; +} + export interface ICompleteArguments { messages: PuterMessage[]; provider?: string; @@ -234,7 +252,9 @@ export interface ICompleteArguments { metadata?: Record; prompt?: unknown; prompt_cache_key?: string; - prompt_cache_retention?: 'in-memory' | '24h' | undefined; + /** `in-memory` is accepted as a spelling of OpenAI's `in_memory`. */ + prompt_cache_retention?: 'in-memory' | 'in_memory' | '24h' | undefined; + prompt_cache_options?: PromptCacheOptions; store?: boolean; top_p?: number; truncation?: 'auto' | 'disabled' | undefined; diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js index fd42cb5d8..6c8ef5d1a 100644 --- a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js @@ -481,43 +481,86 @@ export const extractMeteredUsage = (usage) => { }; /** - * Chat Completions usage as it's metered: `prompt_tokens` reports cached reads - * inside its count, so they come out of it and are priced only once, under - * `cached_tokens`. + * @typedef {{ + * cached_tokens?: number | null; + * cache_write_tokens?: number | null; + * cache_write_tokens_details?: { + * ttl_30m_tokens?: number | null; + * ttl_12h_tokens?: number | null; + * } | null; + * }} OpenAiCacheDetails + */ + +/** + * OpenAI-shaped usage as it's metered. The prompt count reports cached reads + * and cache writes inside it, so they come out of it and are priced once each + * under their own keys: `cached_tokens` for reads, `cache_write_tokens` for + * writes at the model's default lifetime and `cache_write_12h_tokens` for + * writes at OpenAI's 12h lifetime. Without the per-lifetime breakdown every + * write is a default-lifetime write: Chat Completions writes no 12h entries, + * and Responses reports the breakdown whenever a request asked for them. Reads + * the Chat Completions (`prompt_tokens`) and Responses (`input_tokens`) + * spellings alike. * - * @param {import('openai/resources/completions.mjs').CompletionUsage | undefined} usage + * @param {{ + * prompt_tokens?: number; + * completion_tokens?: number; + * prompt_tokens_details?: OpenAiCacheDetails | null; + * input_tokens?: number; + * output_tokens?: number; + * input_tokens_details?: OpenAiCacheDetails | null; + * } + * | undefined} usage */ export const splitCachedPrompt = (usage) => { - const details = usage?.prompt_tokens_details; + const details = usage?.prompt_tokens_details ?? usage?.input_tokens_details; const cachedTokens = details?.cached_tokens ?? 0; - // Moonshot counts the prompt tokens it wrote to its prefix cache here; - // they're billed at the model's write rate, not the plain prompt rate. + // Moonshot and GPT-5.6+ count the prompt tokens they wrote to the prefix + // cache here; they're billed at the model's write rate, not the prompt rate. const writeTokens = typeof details?.cache_write_tokens === 'number' ? details.cache_write_tokens : undefined; + const reported12h = details?.cache_write_tokens_details?.ttl_12h_tokens; + const write12hTokens = + typeof reported12h === 'number' + ? Math.min(reported12h, writeTokens ?? 0) + : undefined; return { prompt_tokens: - (usage?.prompt_tokens ?? 0) - cachedTokens - (writeTokens ?? 0), - completion_tokens: usage?.completion_tokens ?? 0, + (usage?.prompt_tokens ?? usage?.input_tokens ?? 0) - + cachedTokens - + (writeTokens ?? 0), + completion_tokens: + usage?.completion_tokens ?? usage?.output_tokens ?? 0, cached_tokens: cachedTokens, ...(writeTokens !== undefined - ? { cache_write_tokens: writeTokens } + ? { cache_write_tokens: writeTokens - (write12hTokens ?? 0) } + : {}), + ...(write12hTokens !== undefined + ? { cache_write_12h_tokens: write12hTokens } : {}), }; }; /** - * The `{prompt_tokens, completion_tokens, cached_tokens, cache_write_tokens}` - * shape every OpenAI-family `usage_calculator` already returns for billing, - * mapped to the provider-neutral `UsageDetails` the end-of-stream/non-stream - * result carries. `inputTokens`/`outputTokens` are the same numbers already - * billed — this never introduces a new figure, only exposes the breakdown. + * The `{prompt_tokens, completion_tokens, cached_tokens, cache_write_tokens, + * cache_write_12h_tokens}` shape every OpenAI-family `usage_calculator` already + * returns for billing, mapped to the provider-neutral `UsageDetails` the + * end-of-stream/non-stream result carries. `inputTokens`/`outputTokens` are the + * same numbers already billed — this never introduces a new figure, only + * exposes the breakdown. * * @param {Record | undefined | null} trackedUsage + * @param {{ writeTtl?: '5m' | '30m' }} [opts] The lifetime `cache_write_tokens` + * were written with: 5m unless the provider says otherwise (OpenAI's default + * entries live 30m). * @returns {import('../types.js').UsageDetails | undefined} */ -export const usageDetailsFromTrackedUsage = (trackedUsage) => { +export const usageDetailsFromTrackedUsage = ( + trackedUsage, + { writeTtl = '5m' } = {}, +) => { if (!trackedUsage || typeof trackedUsage !== 'object') return undefined; // OpenRouter, Infron and Ollama track `{prompt, completion, // input_cache_read}` instead; any other shape is left to the driver. @@ -526,12 +569,19 @@ export const usageDetailsFromTrackedUsage = (trackedUsage) => { if (input === undefined && output === undefined) return undefined; const cacheRead = trackedUsage.cached_tokens ?? trackedUsage.input_cache_read; + const cacheWrite = trackedUsage.cache_write_tokens; return { inputTokens: input ?? 0, outputTokens: output ?? 0, ...(cacheRead ? { cacheReadTokens: cacheRead } : {}), - ...(trackedUsage.cache_write_tokens - ? { cacheWrite5mTokens: trackedUsage.cache_write_tokens } + ...(cacheWrite && writeTtl === '30m' + ? { cacheWrite30mTokens: cacheWrite } + : {}), + ...(cacheWrite && writeTtl !== '30m' + ? { cacheWrite5mTokens: cacheWrite } + : {}), + ...(trackedUsage.cache_write_12h_tokens + ? { cacheWrite12hTokens: trackedUsage.cache_write_12h_tokens } : {}), }; }; @@ -864,12 +914,16 @@ export const create_chat_stream_handler_responses_api = // Missing usage is left undefined rather than fed to the calculator — // see the sibling handler above, including why usage is reported // before the block flushes. + let usageDetails; const usage = last_usage ? usage_calculator({ usage: last_usage, webSearchCalls, tool_usage: toolUsage, setUsageCosts: (c) => chatStream.setUsageCosts(c), + setUsageDetails: (d) => { + usageDetails = d; + }, }) : undefined; chatStream.reportUsage(usage); @@ -877,7 +931,7 @@ export const create_chat_stream_handler_responses_api = reason: incompleteReason ?? (sawFunctionCall ? 'tool_use' : 'end_turn'), }); - const usageDetails = usageDetailsFromTrackedUsage(usage); + usageDetails ??= usageDetailsFromTrackedUsage(usage); if (usageDetails) chatStream.setUsageDetails(usageDetails); textblock.end(); @@ -1015,6 +1069,7 @@ export const handle_completion_output = async ( * webSearchCalls?: number; * tool_usage?: Record; * setUsageCosts?: (costs: Record) => void; + * setUsageDetails?: (details: import('../types').UsageDetails) => void; * }) => unknown} params.usage_calculator * @param {() => Promise} [params.finally_fn] * @returns {ReturnType} @@ -1156,12 +1211,15 @@ export const handle_completion_output_responses_api = async ({ setUsageCosts: (c) => { ret.usageCosts = c; }, + setUsageDetails: (d) => { + ret.usageDetails = d; + }, }) : { input_tokens: completion.usage.input_tokens, output_tokens: completion.usage.output_tokens, }; - if (usage_calculator) { + if (usage_calculator && ret.usageDetails === undefined) { const usageDetails = usageDetailsFromTrackedUsage(ret.usage); if (usageDetails) ret.usageDetails = usageDetails; } diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts b/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts index 4aa408000..f283f65a9 100644 --- a/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts @@ -247,6 +247,40 @@ describe('usageDetailsFromTrackedUsage', () => { }); }); + it('labels cache writes by the lifetime the provider says it wrote them with', () => { + expect( + usageDetailsFromTrackedUsage( + { + prompt_tokens: 10, + completion_tokens: 5, + cached_tokens: 3, + cache_write_tokens: 11, + cache_write_12h_tokens: 6012, + }, + { writeTtl: '30m' }, + ), + ).toEqual({ + inputTokens: 10, + outputTokens: 5, + cacheReadTokens: 3, + cacheWrite30mTokens: 11, + cacheWrite12hTokens: 6012, + }); + // Zero counts, as OpenAI reports on a cache hit, add no field. + expect( + usageDetailsFromTrackedUsage( + { + prompt_tokens: 10, + completion_tokens: 5, + cached_tokens: 3, + cache_write_tokens: 0, + cache_write_12h_tokens: 0, + }, + { writeTtl: '30m' }, + ), + ).toEqual({ inputTokens: 10, outputTokens: 5, cacheReadTokens: 3 }); + }); + it('returns undefined for a missing/non-object usage', () => { expect(usageDetailsFromTrackedUsage(undefined)).toBeUndefined(); expect(usageDetailsFromTrackedUsage(null)).toBeUndefined(); @@ -962,6 +996,66 @@ describe('splitCachedPrompt', () => { cached_tokens: 3, }); }); + + it('reads the Responses usage spelling', () => { + expect( + splitCachedPrompt({ + input_tokens: 6026, + output_tokens: 5, + input_tokens_details: { cached_tokens: 0, cache_write_tokens: 6023 }, + }), + ).toEqual({ + prompt_tokens: 3, + completion_tokens: 5, + cached_tokens: 0, + cache_write_tokens: 6023, + }); + }); + + it('splits 12h writes out of the write count when the breakdown is reported', () => { + // One request can write both lifetimes: an explicit 12h breakpoint + // plus the implicit 30m one OpenAI adds at the end of the prompt. + expect( + splitCachedPrompt({ + input_tokens: 6026, + output_tokens: 5, + input_tokens_details: { + cached_tokens: 0, + cache_write_tokens: 6023, + cache_write_tokens_details: { + ttl_12h_tokens: 6012, + ttl_30m_tokens: 11, + }, + }, + }), + ).toEqual({ + prompt_tokens: 3, + completion_tokens: 5, + cached_tokens: 0, + cache_write_tokens: 11, + cache_write_12h_tokens: 6012, + }); + }); + + it('never reports more 12h writes than writes in total', () => { + expect( + splitCachedPrompt({ + input_tokens: 100, + output_tokens: 1, + input_tokens_details: { + cached_tokens: 0, + cache_write_tokens: 40, + cache_write_tokens_details: { ttl_12h_tokens: 50 }, + }, + }), + ).toEqual({ + prompt_tokens: 60, + completion_tokens: 1, + cached_tokens: 0, + cache_write_tokens: 0, + cache_write_12h_tokens: 40, + }); + }); }); // ── recorded usage costs ──────────────────────────────────────────── diff --git a/src/backend/drivers/ai-chat/utils/openaiParams.test.ts b/src/backend/drivers/ai-chat/utils/openaiParams.test.ts index bd2801ca8..cc7ed5ff5 100644 --- a/src/backend/drivers/ai-chat/utils/openaiParams.test.ts +++ b/src/backend/drivers/ai-chat/utils/openaiParams.test.ts @@ -27,6 +27,8 @@ import { outputFormatFromResponsesText, outputFormatToResponseFormat, outputFormatToResponsesText, + promptCacheOptionsToWire, + promptCacheRetentionToWire, rejectStatefulResponsesFields, toolChoiceFromWire, toolChoiceToWire, @@ -334,3 +336,50 @@ describe('rejectStatefulResponsesFields', () => { ).toThrow(/previous_response_id/); }); }); + +describe('promptCacheRetentionToWire', () => { + it('spells the in-memory retention the way the upstreams expect', () => { + expect(promptCacheRetentionToWire('in-memory')).toBe('in_memory'); + expect(promptCacheRetentionToWire('in_memory')).toBe('in_memory'); + expect(promptCacheRetentionToWire('24h')).toBe('24h'); + expect(promptCacheRetentionToWire(undefined)).toBeUndefined(); + }); +}); + +describe('promptCacheOptionsToWire', () => { + it('forwards mode and ttl and nothing else', () => { + expect( + promptCacheOptionsToWire({ + mode: 'explicit', + ttl: '12h', + prewarm: true, + comparison_response_id: 'resp_1', + }), + ).toEqual({ mode: 'explicit', ttl: '12h' }); + expect(promptCacheOptionsToWire({ ttl: '30m' })).toEqual({ + ttl: '30m', + }); + }); + + it('sends nothing for an absent or empty object', () => { + expect(promptCacheOptionsToWire(undefined)).toBeUndefined(); + expect(promptCacheOptionsToWire(null)).toBeUndefined(); + expect(promptCacheOptionsToWire({})).toBeUndefined(); + expect(promptCacheOptionsToWire({ prewarm: true })).toBeUndefined(); + }); + + it('rejects a lifetime or mode OpenAI does not offer', () => { + expect(() => promptCacheOptionsToWire({ ttl: '2h' })).toThrow( + expect.objectContaining({ statusCode: 400 }), + ); + expect(() => promptCacheOptionsToWire({ ttl: '24h' })).toThrow( + /30m.*12h/, + ); + expect(() => promptCacheOptionsToWire({ mode: 'auto' })).toThrow( + /implicit.*explicit/, + ); + expect(() => promptCacheOptionsToWire('12h')).toThrow( + /must be an object/, + ); + }); +}); diff --git a/src/backend/drivers/ai-chat/utils/openaiParams.ts b/src/backend/drivers/ai-chat/utils/openaiParams.ts index eca62e486..8bb87d1d0 100644 --- a/src/backend/drivers/ai-chat/utils/openaiParams.ts +++ b/src/backend/drivers/ai-chat/utils/openaiParams.ts @@ -29,6 +29,7 @@ import { HttpError } from '../../../core/http/HttpError.js'; import type { ICompleteArguments, OutputFormat, + PromptCacheOptions, ReasoningEffort, ToolChoice, } from '../types.js'; @@ -305,6 +306,60 @@ export const openAICompatParams = ( return out; }; +// -- prompt caching ---------------------------------------------------------- + +/** + * The wire spelling of `prompt_cache_retention`. Puter also accepts the + * hyphenated `in-memory`, which every OpenAI-shaped upstream rejects. + */ +export const promptCacheRetentionToWire = ( + retention: ICompleteArguments['prompt_cache_retention'], +): 'in_memory' | '24h' | undefined => + retention === 'in-memory' ? 'in_memory' : retention; + +const PROMPT_CACHE_MODES = new Set(['implicit', 'explicit']); +const PROMPT_CACHE_TTLS = new Set(['30m', '12h']); + +/** + * `prompt_cache_options` as forwarded upstream: `mode` and `ttl` only, each + * checked against the values OpenAI accepts, so a bad value fails here with a + * clear message instead of as an upstream 400 the fallback loop may hide. + * `comparison_response_id` names a stored response, which Puter never keeps, + * and `prewarm` is dropped with it. + */ +export const promptCacheOptionsToWire = ( + options: unknown, +): PromptCacheOptions | undefined => { + if (options === undefined || options === null) return undefined; + if (typeof options !== 'object' || Array.isArray(options)) { + throw new HttpError(400, '`prompt_cache_options` must be an object', { + legacyCode: 'bad_request', + }); + } + const { mode, ttl } = options as Record; + if (mode !== undefined && !PROMPT_CACHE_MODES.has(mode as string)) { + throw new HttpError( + 400, + '`prompt_cache_options.mode` must be `implicit` or `explicit`', + { legacyCode: 'bad_request' }, + ); + } + if (ttl !== undefined && !PROMPT_CACHE_TTLS.has(ttl as string)) { + throw new HttpError( + 400, + '`prompt_cache_options.ttl` must be `30m` or `12h`', + { legacyCode: 'bad_request' }, + ); + } + const wire: PromptCacheOptions = { + ...(mode !== undefined + ? { mode: mode as PromptCacheOptions['mode'] } + : {}), + ...(ttl !== undefined ? { ttl: ttl as PromptCacheOptions['ttl'] } : {}), + }; + return Object.keys(wire).length ? wire : undefined; +}; + // -- Responses server-side state --------------------------------------------- const STATEFUL_RESPONSES_FIELDS = [ diff --git a/src/docs/src/AI/chat.md b/src/docs/src/AI/chat.md index 48ebf291f..1192baaf6 100755 --- a/src/docs/src/AI/chat.md +++ b/src/docs/src/AI/chat.md @@ -314,6 +314,21 @@ You can find the implementation in our [prompt caching example](/playground/ai-c Moonshot's Kimi K3 and Kimi K3.1 Pro models cache prompt prefixes on their own and take no `cache_control`. Their `usage` reports the tokens read from the cache as `cached_tokens` and the tokens written to it as `cache_write_tokens`. Each is billed at the model's own rate from the [AI models list](https://developer.puter.com/ai/models/). +OpenAI models cache prompt prefixes on their own too. Requests from the same app and user share one cache by default, and you can pick the cache with `prompt_cache_key`. Their `usage` reports the tokens read from the cache as `cached_tokens`. + +GPT-5.6 and later models charge for cache writes and let you choose how long an entry lives. Pass `prompt_cache_options` with a `ttl` of `"30m"` (the default) or `"12h"`. The `usage` reports the tokens written with the 30 minute lifetime as `cache_write_tokens` and the tokens written with the 12 hour lifetime as `cache_write_12h_tokens`. Writing a 30 minute entry costs 1.25 times the model's input rate, and writing a 12 hour entry costs 2.5 times the input rate. Reads cost the same for both lifetimes. The 12 hour lifetime is only available through the OpenAI Responses API, so a request with `ttl: "12h"` is sent there. + +```js +puter.ai.chat('', { + model: 'gpt-6-luna', + prompt_cache_options: { ttl: '12h' } +}); +``` + +Older OpenAI models charge nothing for cache writes. Pass `prompt_cache_retention` with `"in_memory"` for an entry that lives a few minutes or `"24h"` for one that can live up to a day. The 24 hour option is the default, and GPT-5.5 and later models only support it. + +The `usageDetails` object on every result carries the same counts under provider-neutral names, with one field per lifetime, such as `cacheWrite30mTokens` and `cacheWrite12hTokens`. See the [ChatResponse](/Objects/chatresponse/) object for the full list. + ## Compaction For long, multi-turn conversations that you keep on the client, enable diff --git a/src/docs/src/Objects/chatresponse.md b/src/docs/src/Objects/chatresponse.md index 628f6a678..2fc6a81ca 100644 --- a/src/docs/src/Objects/chatresponse.md +++ b/src/docs/src/Objects/chatresponse.md @@ -64,7 +64,7 @@ Extra detail on why generation stopped (e.g. a refusal classification). Provider #### `usageDetails` (Object) -A provider-neutral usage breakdown, present alongside `usage` on every result. `inputTokens` excludes cache reads/writes; `outputTokens` includes reasoning tokens. Fields: `inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWrite5mTokens`, `cacheWrite1hTokens`, `reasoningTokens`, `webSearchRequests`, `webFetchRequests`, `speed` (`"fast"` or `"standard"`), `serviceTier`, and `iterations` (a per-pass breakdown when the upstream ran several passes, e.g. compaction or an advisor sub-call). +A provider-neutral usage breakdown, present alongside `usage` on every result. `inputTokens` excludes cache reads/writes; `outputTokens` includes reasoning tokens. Fields: `inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWrite5mTokens`, `cacheWrite30mTokens`, `cacheWrite1hTokens`, `cacheWrite12hTokens`, `reasoningTokens`, `webSearchRequests`, `webFetchRequests`, `speed` (`"fast"` or `"standard"`), `serviceTier`, and `iterations` (a per-pass breakdown when the upstream ran several passes, e.g. compaction or an advisor sub-call). Each `cacheWrite*Tokens` field counts the tokens written to the cache with that lifetime: Claude and Moonshot write 5 minute and 1 hour entries, OpenAI writes 30 minute and 12 hour entries. Prefer `usageDetails` over `usage` for anything beyond the raw billing totals. Claude's `usage.thinking_tokens` is informational: it's included in `output_tokens` and not billed separately — `usageDetails.reasoningTokens` carries the same count. diff --git a/src/docs/src/Objects/chatresponsechunk.md b/src/docs/src/Objects/chatresponsechunk.md index b5a147f78..28104d9ed 100644 --- a/src/docs/src/Objects/chatresponsechunk.md +++ b/src/docs/src/Objects/chatresponsechunk.md @@ -102,7 +102,7 @@ Extra detail on why generation stopped (e.g. a refusal classification). Provider #### `usageDetails` (Object) -A provider-neutral usage breakdown, present on the final `usage` chunk. `inputTokens` excludes cache reads/writes; `outputTokens` includes reasoning tokens. Fields: `inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWrite5mTokens`, `cacheWrite1hTokens`, `reasoningTokens`, `webSearchRequests`, `webFetchRequests`, `speed` (`"fast"` or `"standard"`), `serviceTier`, and `iterations` (a per-pass breakdown when the upstream ran several passes, e.g. compaction or an advisor sub-call). +A provider-neutral usage breakdown, present on the final `usage` chunk. `inputTokens` excludes cache reads/writes; `outputTokens` includes reasoning tokens. Fields: `inputTokens`, `outputTokens`, `cacheReadTokens`, `cacheWrite5mTokens`, `cacheWrite30mTokens`, `cacheWrite1hTokens`, `cacheWrite12hTokens`, `reasoningTokens`, `webSearchRequests`, `webFetchRequests`, `speed` (`"fast"` or `"standard"`), `serviceTier`, and `iterations` (a per-pass breakdown when the upstream ran several passes, e.g. compaction or an advisor sub-call). Each `cacheWrite*Tokens` field counts the tokens written to the cache with that lifetime: Claude and Moonshot write 5 minute and 1 hour entries, OpenAI writes 30 minute and 12 hour entries. #### `message` (String) diff --git a/src/puter-js/src/modules/ai/ai.test.js b/src/puter-js/src/modules/ai/ai.test.js index ff31c1938..f5f14dc9e 100644 --- a/src/puter-js/src/modules/ai/ai.test.js +++ b/src/puter-js/src/modules/ai/ai.test.js @@ -171,6 +171,20 @@ describe('ai.chat driver payloads', () => { }); }); + it('chat(prompt, options) forwards the OpenAI prompt-cache options', async () => { + await ai.chat('hello', { + prompt_cache_key: 'ck', + prompt_cache_retention: '24h', + prompt_cache_options: { mode: 'explicit', ttl: '12h' }, + }); + expect(lastBody().args).toEqual({ + messages: [{ content: 'hello' }], + prompt_cache_key: 'ck', + prompt_cache_retention: '24h', + prompt_cache_options: { mode: 'explicit', ttl: '12h' }, + }); + }); + it('chat(prompt, options) forwards temperature: 0 and max_tokens: 0', async () => { await ai.chat('hello', { temperature: 0, diff --git a/src/puter-js/src/modules/ai/chat.js b/src/puter-js/src/modules/ai/chat.js index 0f4384a48..9d1093fa1 100644 --- a/src/puter-js/src/modules/ai/chat.js +++ b/src/puter-js/src/modules/ai/chat.js @@ -27,6 +27,9 @@ const PARAMS_TO_PASS = [ 'outputFormat', 'thinking', 'streamToolInput', + 'prompt_cache_key', + 'prompt_cache_retention', + 'prompt_cache_options', ]; // Parameters renamed between the SDK's camelCase option and the driver's diff --git a/src/puter-js/src/modules/ai/types.js b/src/puter-js/src/modules/ai/types.js index 0e6aa7218..2b9552cf4 100644 --- a/src/puter-js/src/modules/ai/types.js +++ b/src/puter-js/src/modules/ai/types.js @@ -115,6 +115,14 @@ * @property {boolean} [streamToolInput] Streaming only: emit incremental `tool_use_start` / * `tool_input_delta` chunks as a tool call's arguments are generated, instead of a single `tool_use` * chunk once the call completes. + * @property {string} [prompt_cache_key] OpenAI models: which prompt cache the request reads from and + * writes to. Defaults to one cache per app and user. + * @property {'in_memory' | '24h'} [prompt_cache_retention] OpenAI models before GPT-5.6: how long a + * cache entry may live. `'24h'` is the default and the only value GPT-5.5 and later accept. + * @property {{ mode?: 'implicit' | 'explicit', ttl?: '30m' | '12h' }} [prompt_cache_options] OpenAI + * GPT-5.6 and later: `ttl` is the minimum lifetime of every cache entry the request writes. A `'12h'` + * entry costs twice as much to write as the default `'30m'` one and is served through the Responses + * API. `mode: 'explicit'` only writes at the `prompt_cache_breakpoint` markers in your messages. */ /** @@ -132,14 +140,16 @@ * @property {number} inputTokens * @property {number} outputTokens * @property {number} [cacheReadTokens] - * @property {number} [cacheWrite5mTokens] - * @property {number} [cacheWrite1hTokens] + * @property {number} [cacheWrite5mTokens] Tokens written to the cache with a 5 minute lifetime (Claude, Moonshot). + * @property {number} [cacheWrite30mTokens] Tokens written with a 30 minute lifetime (OpenAI's default). + * @property {number} [cacheWrite1hTokens] Tokens written with a 1 hour lifetime (Claude, Moonshot). + * @property {number} [cacheWrite12hTokens] Tokens written with a 12 hour lifetime (OpenAI, `prompt_cache_options.ttl: '12h'`). * @property {number} [reasoningTokens] * @property {number} [webSearchRequests] * @property {number} [webFetchRequests] * @property {'fast' | 'standard'} [speed] * @property {string} [serviceTier] - * @property {Array<{ type: string, model?: string, inputTokens: number, outputTokens: number, cacheReadTokens?: number, cacheWrite5mTokens?: number, cacheWrite1hTokens?: number }>} [iterations] + * @property {Array<{ type: string, model?: string, inputTokens: number, outputTokens: number, cacheReadTokens?: number, cacheWrite5mTokens?: number, cacheWrite30mTokens?: number, cacheWrite1hTokens?: number, cacheWrite12hTokens?: number }>} [iterations] * Per-pass breakdown when the upstream ran several passes (compaction, an advisor sub-call). */