diff --git a/extensions/metering.ts b/extensions/metering.ts index d74fc2e9b..16295860a 100644 --- a/extensions/metering.ts +++ b/extensions/metering.ts @@ -1,4 +1,3 @@ -import type { Request, Response } from 'express'; import { Context } from '@heyputer/backend/src/core'; import { HttpError } from '@heyputer/backend/src/core/http'; import { @@ -6,6 +5,7 @@ import { driversContainers, } from '@heyputer/backend/src/exports'; import { extension } from '@heyputer/backend/src/extensions'; +import type { Request, Response } from 'express'; const services = extension.import('service'); const clients = extension.import('client'); diff --git a/src/backend/controllers/puterai/PuterAIController.test.ts b/src/backend/controllers/puterai/PuterAIController.test.ts index fbabdd028..e8ed260fd 100644 --- a/src/backend/controllers/puterai/PuterAIController.test.ts +++ b/src/backend/controllers/puterai/PuterAIController.test.ts @@ -1506,6 +1506,212 @@ describe('PuterAIController.anthropicMessages streaming + helpers', () => { }); }); +// ── Inline compaction ─────────────────────────────────────────────── + +describe('PuterAIController inline compaction', () => { + // Extract the single `event: compaction\ndata: {...}` SSE frame. + const compactionFrame = (out: string): string | null => { + const m = out.match(/event: compaction\ndata: [^\n]*\n\n/); + return m ? m[0] : null; + }; + + it('forwards the compaction opt-in to the driver (/responses)', async () => { + const spy = stubChatComplete({ + message: { role: 'assistant', content: 'ok' }, + finish_reason: 'stop', + }); + await controller.openaiResponses( + makeReq({ + body: { model: 'gpt-test', input: 'hi', compaction: true }, + actor: makeUserActor(), + }), + makeRes().res, + ); + expect(spy.mock.calls[0]![0].compaction).toBe(true); + }); + + it('round-trips a compaction `input` item into a messages compaction item', async () => { + const spy = stubChatComplete({ + message: { role: 'assistant', content: 'ack' }, + finish_reason: 'stop', + }); + await controller.openaiResponses( + makeReq({ + body: { + model: 'gpt-test', + input: [ + { + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }, + ], + }, + actor: makeUserActor(), + }), + makeRes().res, + ); + expect(spy.mock.calls[0]![0].messages).toContainEqual({ + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }); + }); + + it('emits a canonical compaction SSE event and output item (/responses stream)', async () => { + stubChatComplete({ + dataType: 'stream', + content_type: 'application/x-ndjson', + stream: ndjsonStreamFrom([ + { type: 'text', text: 'hi' }, + { type: 'compaction', id: 'cmpct_9', encrypted_content: 'ENC9' }, + { type: 'usage', usage: { prompt_tokens: 1, completion_tokens: 1 } }, + ]), + }); + const { res, captured } = makeRes(); + await controller.openaiResponses( + makeReq({ + body: { + model: 'gpt-test', + input: 'hi', + stream: true, + compaction: true, + }, + actor: makeUserActor(), + }), + res, + ); + await new Promise((resolve) => setImmediate(resolve)); + + const out = captured.written.join(''); + expect(compactionFrame(out)).toBe( + 'event: compaction\ndata: {"type":"compaction","id":"cmpct_9","encrypted_content":"ENC9"}\n\n', + ); + // Native shape also lands in the final response.completed output[]. + const completed = out + .split('event: response.completed\n')[1] + ?.split('\n\n')[0]; + expect(completed).toContain('"type":"compaction"'); + expect(completed).toContain('"encrypted_content":"ENC9"'); + }); + + it('emits the compaction item in non-streaming /responses output', async () => { + stubChatComplete({ + message: { role: 'assistant', content: 'done' }, + finish_reason: 'stop', + compaction: { id: 'cmpct_n', encrypted_content: 'ENCN' }, + }); + const { res, captured } = makeRes(); + await controller.openaiResponses( + makeReq({ + body: { model: 'gpt-test', input: 'hi', compaction: true }, + actor: makeUserActor(), + }), + res, + ); + const body = captured.body as { output: Array> }; + expect(body.output).toContainEqual( + expect.objectContaining({ + type: 'compaction', + encrypted_content: 'ENCN', + }), + ); + }); + + it('emits an identical canonical compaction SSE event on the Anthropic surface', async () => { + // /responses frame + stubChatComplete({ + dataType: 'stream', + content_type: 'application/x-ndjson', + stream: ndjsonStreamFrom([ + { type: 'compaction', id: 'cmpct_x', encrypted_content: 'ENCX' }, + ]), + }); + const r1 = makeRes(); + await controller.openaiResponses( + makeReq({ + body: { model: 'gpt-test', input: 'hi', stream: true }, + actor: makeUserActor(), + }), + r1.res, + ); + await new Promise((resolve) => setImmediate(resolve)); + + // /anthropic/v1/messages frame + stubChatComplete({ + dataType: 'stream', + content_type: 'application/x-ndjson', + stream: ndjsonStreamFrom([ + { type: 'compaction', id: 'cmpct_x', encrypted_content: 'ENCX' }, + ]), + }); + const r2 = makeRes(); + await controller.anthropicMessages( + makeReq({ + body: { + model: 'claude-test', + messages: [{ role: 'user', content: 'hi' }], + stream: true, + }, + actor: makeUserActor(), + }), + r2.res, + ); + await new Promise((resolve) => setImmediate(resolve)); + + const a = compactionFrame(r1.captured.written.join('')); + const b = compactionFrame(r2.captured.written.join('')); + expect(a).not.toBeNull(); + expect(a).toBe(b); // byte-identical streaming shape across providers + }); + + it('renders a native compaction content block in non-streaming /messages', async () => { + stubChatComplete({ + message: { role: 'assistant', content: 'done' }, + finish_reason: 'stop', + compaction: { id: 'cmpct_m', encrypted_content: 'ENCM' }, + }); + const { res, captured } = makeRes(); + await controller.anthropicMessages( + makeReq({ + body: { + model: 'claude-test', + messages: [{ role: 'user', content: 'hi' }], + }, + actor: makeUserActor(), + }), + res, + ); + const body = captured.body as { content: Array> }; + expect(body.content).toContainEqual({ + type: 'compaction', + id: 'cmpct_m', + encrypted_content: 'ENCM', + }); + }); + + it('does not emit compaction frames for a normal stream (regression)', async () => { + stubChatComplete({ + dataType: 'stream', + content_type: 'application/x-ndjson', + stream: ndjsonStreamFrom([ + { type: 'text', text: 'hello' }, + { type: 'usage', usage: { prompt_tokens: 1, completion_tokens: 1 } }, + ]), + }); + const { res, captured } = makeRes(); + await controller.openaiResponses( + makeReq({ + body: { model: 'gpt-test', input: 'hi', stream: true }, + actor: makeUserActor(), + }), + res, + ); + await new Promise((resolve) => setImmediate(resolve)); + expect(captured.written.join('')).not.toContain('event: compaction'); + }); +}); + // ── Model details listing ─────────────────────────────────────────── describe('PuterAIController model details', () => { diff --git a/src/backend/controllers/puterai/PuterAIController.ts b/src/backend/controllers/puterai/PuterAIController.ts index b33adf4b5..0ac3a7ed2 100644 --- a/src/backend/controllers/puterai/PuterAIController.ts +++ b/src/backend/controllers/puterai/PuterAIController.ts @@ -587,6 +587,15 @@ export class PuterAIController extends PuterController { ? { metadata: body.metadata as Record } : {}), ...(body.conversation ? { conversation: body.conversation } : {}), + ...(body.context_management !== undefined + ? { context_management: body.context_management } + : {}), + ...(body.compaction !== undefined + ? { + compaction: + body.compaction as ICompleteArguments['compaction'], + } + : {}), ...(body.previous_response_id ? { previous_response_id: String(body.previous_response_id) } : {}), @@ -750,6 +759,30 @@ export class PuterAIController extends PuterController { output_index: outputIndex, item, }); + } else if (ev.type === 'compaction') { + // Native shape in the final `output[]`, plus the + // canonical SSE event shared with the Anthropic surface. + const item = { + type: 'compaction', + ...(ev.id !== undefined ? { id: ev.id } : {}), + encrypted_content: ev.encrypted_content, + }; + const outputIndex = output.length; + output.push(item); + sendEvent({ + type: 'response.output_item.added', + output_index: outputIndex, + item, + }); + sendEvent({ + type: 'response.output_item.done', + output_index: outputIndex, + item, + }); + writeCompactionEvent(res, { + id: ev.id, + encrypted_content: ev.encrypted_content, + }); } else if (ev.type === 'usage') { usage = buildResponsesUsage( ev.usage as Record, @@ -869,6 +902,15 @@ export class PuterAIController extends PuterController { ...(body.max_tokens !== undefined ? { max_tokens: Number(body.max_tokens) } : {}), + ...(body.context_management !== undefined + ? { context_management: body.context_management } + : {}), + ...(body.compaction !== undefined + ? { + compaction: + body.compaction as ICompleteArguments['compaction'], + } + : {}), ...(body.provider ? { provider: toStringOrEmpty(body.provider) } : { provider: DEFAULTS.anthropic }), @@ -968,6 +1010,14 @@ export class PuterAIController extends PuterController { }, }); closeBlock(); + } else if (ev.type === 'compaction') { + // Close any open content block, then emit the canonical + // compaction SSE event — byte-identical to /responses. + closeBlock(); + writeCompactionEvent(res, { + id: ev.id, + encrypted_content: ev.encrypted_content, + }); } else if (ev.type === 'usage') { usage = ev.usage as Record; } @@ -1026,6 +1076,18 @@ export class PuterAIController extends PuterController { if (textContent) contentBlocks.push({ type: 'text', text: textContent }); contentBlocks.push(...toolUseBlocks); + // Native Anthropic-shaped compaction block (non-streaming bodies stay + // provider-native, unlike the unified streaming event). + const compaction = ( + messageResult as { compaction?: Record } + ).compaction; + if (compaction) { + contentBlocks.push({ + type: 'compaction', + ...(compaction.id !== undefined ? { id: compaction.id } : {}), + encrypted_content: compaction.encrypted_content, + }); + } if (contentBlocks.length === 0) contentBlocks.push({ type: 'text', text: '' }); @@ -1082,6 +1144,25 @@ const setSseHeaders = (res: Response): void => { res.setHeader('Connection', 'keep-alive'); }; +/** + * Inline-compaction is emitted in a single canonical SSE shape that is + * byte-identical across the `/responses` and `/anthropic/v1/messages` streaming + * surfaces, so a streaming client parses compaction the same way regardless of + * which upstream served the request. (Non-streaming bodies stay provider-native.) + */ +const writeCompactionEvent = ( + res: Response, + compaction: { id?: unknown; encrypted_content?: unknown }, +): void => { + const payload = { + type: 'compaction', + ...(compaction.id !== undefined ? { id: compaction.id } : {}), + encrypted_content: compaction.encrypted_content, + }; + res.write(`event: compaction\n`); + res.write(`data: ${JSON.stringify(payload)}\n\n`); +}; + /** * The chat driver returns either a stream-result envelope or a plain * message result. Proxy routes invoked with `stream: true` expect the @@ -1325,6 +1406,17 @@ const responseInputToMessages = (input: unknown): unknown[] => { if (!item || typeof item !== 'object') continue; const it = item as Record; + if (it.type === 'compaction') { + // Round-tripped compaction artifact: preserve it as a bare item so + // `normalize_single_message` wraps it into an internal compaction + // content block (providers map it back to their native input shape). + messages.push({ + type: 'compaction', + ...(it.id !== undefined ? { id: it.id } : {}), + encrypted_content: it.encrypted_content, + }); + continue; + } if (it.type === 'function_call_output') { messages.push({ role: 'tool', @@ -1416,6 +1508,16 @@ const responseOutputFromResult = ( }); } + const compaction = (result as { compaction?: Record }) + .compaction; + if (compaction) { + output.push({ + id: (compaction.id as string | undefined) || generateId('cmpct'), + type: 'compaction', + encrypted_content: compaction.encrypted_content, + }); + } + return output; }; diff --git a/src/backend/drivers/ai-chat/providers/azure/AzureChatProvider.ts b/src/backend/drivers/ai-chat/providers/azure/AzureChatProvider.ts index c05d0b5f9..1820100c0 100644 --- a/src/backend/drivers/ai-chat/providers/azure/AzureChatProvider.ts +++ b/src/backend/drivers/ai-chat/providers/azure/AzureChatProvider.ts @@ -26,6 +26,10 @@ import type { MeteringService } from '../../../../services/metering/MeteringServ import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import { + messagesHaveCompaction, + wantsCompaction, +} from '../../utils/compaction.js'; import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; import { processPuterPathUploads } from '../openai/fileUpload.js'; import { AZURE_MODELS } from './models.js'; @@ -141,6 +145,20 @@ export class AzureChatProvider implements IChatProvider { } return await this.#responsesProvider.complete(params); } + // Inline compaction is a Responses-API feature; chat.completions can't + // express `context_management` or a `compaction` content block. + // Delegate to the Responses provider when the caller opted in OR when + // the messages carry a round-tripped compaction artifact. + if (wantsCompaction(params) || messagesHaveCompaction(messages)) { + if (!this.#responsesProvider) { + throw new HttpError( + 400, + 'compaction requires the OpenAI Responses provider, which is not configured', + { legacyCode: 'bad_request' }, + ); + } + return await this.#responsesProvider.complete(params); + } // Validate messages if (!Array.isArray(messages)) { throw new HttpError(400, '`messages` must be an array', { diff --git a/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts b/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts index 1b7a8db62..130a4b40e 100644 --- a/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts +++ b/src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts @@ -25,6 +25,7 @@ import type { MeteringService } from '../../../../services/metering/MeteringServ import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import { toOpenAiContextManagement } from '../../utils/compaction.js'; import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; import { processPuterPathUploads } from '../openai/fileUpload.js'; import { AZURE_MODELS } from './models.js'; @@ -109,6 +110,8 @@ export class AzureResponsesProvider implements IChatProvider { parallel_tool_calls, include, conversation, + compaction, + context_management, previous_response_id, instructions, metadata, @@ -182,6 +185,13 @@ export class AzureResponsesProvider implements IChatProvider { const supportsReasoningControls = typeof model === 'string' && model.startsWith('gpt-5'); + // Translate the neutral compaction opt-in (or pass a raw + // `context_management` payload through) to OpenAI's Responses shape. + const contextManagement = toOpenAiContextManagement({ + compaction, + context_management, + }); + const completionParams: ResponseCreateParams = { user: userIdentifier, safety_identifier: userIdentifier, @@ -193,6 +203,9 @@ export class AzureResponsesProvider implements IChatProvider { ? { parallel_tool_calls } : {}), ...(include !== undefined ? { include } : {}), + ...(contextManagement !== undefined + ? { context_management: contextManagement } + : {}), ...(conversation !== undefined ? { conversation } : {}), ...(previous_response_id !== undefined ? { previous_response_id } diff --git a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts index 0a0e1538d..32f7b3dcc 100644 --- a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts @@ -618,6 +618,185 @@ describe('ClaudeProvider.complete streaming', () => { }); }); +// ── Inline compaction ─────────────────────────────────────────────── + +describe('ClaudeProvider.complete compaction', () => { + it('translates the neutral opt-in to context_management + the compaction beta', async () => { + const { provider } = makeProvider(); + messagesCreateMock.mockResolvedValueOnce({ + content: [{ type: 'text', text: 'hi' }], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + + await withTestActor(() => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [{ role: 'user', content: 'hi' }], + compaction: { trigger_tokens: 50000 }, + }), + ); + + const [args] = messagesCreateMock.mock.calls[0]!; + expect(args.context_management).toEqual({ + edits: [ + { + type: 'compact_20260112', + trigger: { type: 'input_tokens', value: 50000 }, + }, + ], + }); + expect(args.betas).toContain('compact-2026-01-12'); + }); + + it('emits a canonical compaction event from a streamed compaction block', async () => { + const { provider } = makeProvider(); + messagesStreamMock.mockReturnValueOnce( + makeStreamLike( + [ + { type: 'message_start' }, + { + type: 'content_block_start', + content_block: { + type: 'compaction', + id: 'cmpct_1', + content: 'ENC', // Anthropic carries the summary here + }, + }, + { type: 'content_block_stop' }, + { + type: 'message_delta', + usage: { input_tokens: 1, output_tokens: 1 }, + }, + { type: 'message_stop' }, + ], + { input_tokens: 1, output_tokens: 1 }, + ), + ); + + const result = await withTestActor(() => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [{ role: 'user', content: 'hi' }], + stream: true, + compaction: true, + }), + ); + const harness = makeCapturingChatStream(); + await ( + result as { + init_chat_stream: (p: { chatStream: unknown }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }); + + const compaction = harness + .events() + .find((e) => e.type === 'compaction'); + expect(compaction).toEqual({ + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }); + }); + + it('enables the compaction beta when a round-tripped compaction block is resent (no opt-in)', async () => { + const { provider } = makeProvider(); + messagesCreateMock.mockResolvedValueOnce({ + content: [{ type: 'text', text: 'ok' }], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + + await withTestActor(() => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [ + { role: 'user', content: 'continue' }, + { + role: 'assistant', + content: [ + { + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }, + ], + }, + ], + // note: no `compaction`/`context_management` opt-in + }), + ); + + const [args] = messagesCreateMock.mock.calls[0]!; + expect(args.betas).toContain('compact-2026-01-12'); + // No new compaction was requested, so no context_management is sent. + expect(args.context_management).toBeUndefined(); + }); + + it('surfaces a compaction block from a non-streaming response as result.compaction', async () => { + const { provider } = makeProvider(); + messagesCreateMock.mockResolvedValueOnce({ + content: [ + { type: 'text', text: 'done' }, + { + type: 'compaction', + id: 'cmpct_2', + content: 'ENC2', // Anthropic carries the summary here + }, + ], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + + const result = (await withTestActor(() => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [{ role: 'user', content: 'hi' }], + compaction: true, + }), + )) as { compaction?: { id?: string; encrypted_content: string } }; + + // Anthropic's `content` is surfaced under the unified `encrypted_content`. + expect(result.compaction).toEqual({ + type: 'compaction', + id: 'cmpct_2', + encrypted_content: 'ENC2', + }); + }); + + it('maps a round-tripped compaction block back to Anthropic `content` on input', async () => { + const { provider } = makeProvider(); + messagesCreateMock.mockResolvedValueOnce({ + content: [{ type: 'text', text: 'ok' }], + usage: { input_tokens: 1, output_tokens: 1 }, + }); + + await withTestActor(() => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [ + { role: 'user', content: 'continue' }, + { + role: 'assistant', + content: [ + { type: 'compaction', encrypted_content: 'SUMMARY' }, + ], + }, + ], + }), + ); + + const [args] = messagesCreateMock.mock.calls[0]!; + const compactionBlock = args.messages + .flatMap((m: { content?: unknown[] }) => + Array.isArray(m.content) ? m.content : [], + ) + .find((c: { type?: string }) => c?.type === 'compaction'); + // Internal `encrypted_content` carrier → Anthropic native `content`. + expect(compactionBlock).toEqual({ + type: 'compaction', + content: 'SUMMARY', + }); + }); +}); + // ── Moderation ────────────────────────────────────────────────────── describe('ClaudeProvider.checkModeration', () => { diff --git a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts index f90f7289c..2fca2e566 100644 --- a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts +++ b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts @@ -34,6 +34,10 @@ import type { ICompleteArguments, IChatCompleteResult, } from '../../types.js'; +import { + messagesHaveCompaction, + toAnthropicContextManagement, +} from '../../utils/compaction.js'; import { make_claude_tools } from '../../utils/FunctionCalling.js'; import { extract_and_remove_system_messages } from '../../utils/Messages.js'; import type { @@ -44,6 +48,11 @@ import type { import { FILES_API_BETA, processPuterPathUploads } from './fileUpload.js'; import { CLAUDE_MODELS } from './models.js'; +// Anthropic inline-compaction beta. The vendored SDK (0.68.0) doesn't type the +// `compact_20260112` edit or the `compaction` content block, so the request +// params and streamed/returned blocks are handled with `as any` casts. +const COMPACTION_BETA = 'compact-2026-01-12'; + export class ClaudeProvider implements IChatProvider { anthropic: Anthropic; @@ -97,9 +106,18 @@ export class ClaudeProvider implements IChatProvider { temperature, reasoning, reasoning_effort, + compaction, + context_management, }: ICompleteArguments): Promise { tools = make_claude_tools(tools); + // Translate the neutral compaction opt-in (or pass a raw + // `context_management` payload through) to Anthropic's beta shape. + const contextManagement = toAnthropicContextManagement({ + compaction, + context_management, + }); + // eslint-disable-next-line @typescript-eslint/no-explicit-any let system_prompts: string | any[]; [system_prompts, messages] = @@ -218,6 +236,23 @@ export class ClaudeProvider implements IChatProvider { return message; }); + // Map round-tripped compaction blocks back to Anthropic's native shape. + // The internal/unified carrier field is `encrypted_content`; Anthropic's + // compaction block uses `content` (a plaintext summary). + // eslint-disable-next-line @typescript-eslint/no-explicit-any + messages = messages.map((message: any) => { + if (!Array.isArray(message.content)) return message; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + message.content = message.content.map((part: any) => { + if (part?.type !== 'compaction') return part; + return { + type: 'compaction', + content: part.content ?? part.encrypted_content ?? '', + }; + }); + return message; + }); + const modelUsed = this.models().find((m) => [m.id, ...(m.aliases || [])].includes(model), @@ -262,6 +297,18 @@ export class ClaudeProvider implements IChatProvider { actor, ); const usesBetaFiles = uploadedFileIds.length > 0; + // The compaction beta is needed both to *request* compaction + // (contextManagement) and to *accept a round-tripped* compaction block + // back as input (messagesHaveCompaction). + const usesCompaction = + !!contextManagement || messagesHaveCompaction(messages); + // Compaction and Files API both require the beta endpoint; combine their + // beta headers and route through `beta.messages.*` if either is active. + const betas = [ + ...(usesBetaFiles ? [FILES_API_BETA] : []), + ...(usesCompaction ? [COMPACTION_BETA] : []), + ]; + const usesBeta = betas.length > 0; const sdkParams: MessageCreateParams & { betas?: string[]; @@ -291,7 +338,11 @@ export class ClaudeProvider implements IChatProvider { ...(supportsEffort && requestedReasoningEffort ? { output_config: { effort: requestedReasoningEffort } } : {}), - ...(usesBetaFiles ? { betas: [FILES_API_BETA] } : {}), + // Cast: `context_management` compaction edits aren't typed in SDK 0.68.0. + ...(contextManagement + ? { context_management: contextManagement as any } + : {}), + ...(usesBeta ? { betas } : {}), } as MessageCreateParams & { betas?: string[] }; const cleanupUploads = async () => { @@ -315,13 +366,23 @@ export class ClaudeProvider implements IChatProvider { }: { chatStream: AIChatStream; }) => { - const completion = usesBetaFiles + const completion = usesBeta ? this.anthropic.beta.messages.stream(sdkParams) : this.anthropic.messages.stream(sdkParams); const usageSum: Record = {}; let message, contentBlock; let currentContentBlockType: string | null = null; + // Inline-compaction block is an untyped beta block; capture its + // artifact across start/delta and emit on stop. Anthropic carries + // the summary in `content` (plaintext) — unlike OpenAI's + // `encrypted_content` — so read `content` first. + let compactionData: { + id?: string; + payload: string; + buffer: string; + } | null = null; + let emittedCompaction = false; for await (const event of completion) { if (event.type === 'message_delta') { const meteredData = this.#usageFormatterUtil( @@ -345,6 +406,24 @@ export class ClaudeProvider implements IChatProvider { } if (event.type === 'content_block_start') { currentContentBlockType = event.content_block.type; + if ( + (event.content_block.type as string) === + 'compaction' + ) { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const block = event.content_block as any; + compactionData = { + id: block.id, + // Anthropic uses `content`; fall back to + // `encrypted_content` in case the field varies. + payload: + block.content ?? + block.encrypted_content ?? + '', + buffer: '', + }; + continue; + } if (event.content_block.type === 'tool_use') { contentBlock = message!.contentBlock({ type: event.content_block.type, @@ -363,12 +442,42 @@ export class ClaudeProvider implements IChatProvider { continue; } if (event.type === 'content_block_stop') { + if (currentContentBlockType === 'compaction') { + const encrypted_content = + compactionData?.payload || + compactionData?.buffer || + ''; + // Only emit (and mark done) if we actually captured + // the summary; otherwise let the finalMessage + // fallback recover it from the complete block. + if (encrypted_content) { + chatStream.compaction({ + id: compactionData?.id, + encrypted_content, + }); + emittedCompaction = true; + } + compactionData = null; + currentContentBlockType = null; + continue; + } contentBlock!.end(); contentBlock = null; currentContentBlockType = null; continue; } if (event.type === 'content_block_delta') { + if (currentContentBlockType === 'compaction') { + // Capture any streamed payload for the artifact. + // eslint-disable-next-line @typescript-eslint/no-explicit-any + const d = event.delta as any; + const chunk = + d.partial_json ?? d.text ?? d.data ?? ''; + if (typeof chunk === 'string' && compactionData) { + compactionData.buffer += chunk; + } + continue; + } if (event.delta.type === 'input_json_delta') { (contentBlock as AIChatToolUseStream)!.addPartialJSON( event.delta.partial_json, @@ -391,18 +500,37 @@ export class ClaudeProvider implements IChatProvider { // signature_delta — ignored } } - const finalUsage = await completion + const finalMessage = await completion .finalMessage() - .then((msg) => - this.#usageFormatterUtil( - msg.usage as Usage | BetaUsage, - ), - ) .catch(() => null); - if (finalUsage) { + if (finalMessage) { + const finalUsage = this.#usageFormatterUtil( + finalMessage.usage as Usage | BetaUsage, + ); for (const [key, value] of Object.entries(finalUsage)) { usageSum[key] = value; } + // Fallback: some SDK versions surface the compaction block + // only in the final message, not as a streamed block. + if (!emittedCompaction) { + const block = ( + (finalMessage.content as unknown[]) ?? [] + ).find( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (c: any) => c?.type === 'compaction', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ) as any; + if (block) { + chatStream.compaction({ + id: block.id, + encrypted_content: + block.content ?? + block.encrypted_content ?? + '', + }); + emittedCompaction = true; + } + } } chatStream.end(usageSum); const costsOverrideFromModel = @@ -423,7 +551,7 @@ export class ClaudeProvider implements IChatProvider { } try { - const msg = await (usesBetaFiles + const msg = await (usesBeta ? this.anthropic.beta.messages.create(sdkParams) : this.anthropic.messages.create(sdkParams)); const usage = this.#usageFormatterUtil( @@ -440,7 +568,37 @@ export class ClaudeProvider implements IChatProvider { costsOverrideFromModel, ); - return { message: msg, usage, finish_reason: 'stop' }; + // Surface any inline-compaction artifact for stateless round-trip. + const compactionBlock = ( + ((msg as Message).content as unknown[]) ?? [] + ).find( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + (c: any) => c?.type === 'compaction', + // eslint-disable-next-line @typescript-eslint/no-explicit-any + ) as any; + + return { + message: msg, + usage, + finish_reason: 'stop', + ...(compactionBlock + ? { + compaction: { + // `type` makes the artifact a drop-in `messages` + // item for the round-trip (symmetric with the + // streaming compaction chunk). + type: 'compaction' as const, + ...(compactionBlock.id !== undefined + ? { id: compactionBlock.id } + : {}), + encrypted_content: + compactionBlock.content ?? + compactionBlock.encrypted_content ?? + '', + }, + } + : {}), + }; } finally { await cleanupUploads(); } diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts index a0abeb64b..166a46019 100644 --- a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts @@ -26,6 +26,10 @@ import type { MeteringService } from '../../../../services/metering/MeteringServ import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import { + messagesHaveCompaction, + wantsCompaction, +} from '../../utils/compaction.js'; import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; import { processPuterPathUploads } from './fileUpload.js'; import { OPEN_AI_MODELS } from './models.js'; @@ -125,6 +129,20 @@ export class OpenAiChatProvider implements IChatProvider { } return await this.#responsesProvider.complete(params); } + // Inline compaction is a Responses-API feature; chat.completions can't + // express `context_management` or a `compaction` content block. + // Delegate to the sibling Responses provider when the caller opted in + // OR when the messages carry a round-tripped compaction artifact. + if (wantsCompaction(params) || messagesHaveCompaction(messages)) { + if (!this.#responsesProvider) { + throw new HttpError( + 400, + 'compaction requires the OpenAI Responses provider, which is not configured', + { legacyCode: 'bad_request' }, + ); + } + return await this.#responsesProvider.complete(params); + } // Validate messages if (!Array.isArray(messages)) { throw new HttpError(400, '`messages` must be an array', { diff --git a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts index 708c4b139..94605b385 100644 --- a/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts +++ b/src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts @@ -25,6 +25,7 @@ import type { MeteringService } from '../../../../services/metering/MeteringServ import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js'; import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js'; import type { IChatProvider, ICompleteArguments } from '../../types.js'; +import { toOpenAiContextManagement } from '../../utils/compaction.js'; import * as OpenAiUtil from '../../utils/OpenAIUtil.js'; import { processPuterPathUploads } from './fileUpload.js'; import { OPEN_AI_MODELS } from './models.js'; @@ -100,6 +101,8 @@ export class OpenAiResponsesChatProvider implements IChatProvider { parallel_tool_calls, include, conversation, + compaction, + context_management, previous_response_id, instructions, metadata, @@ -177,6 +180,13 @@ export class OpenAiResponsesChatProvider implements IChatProvider { const supportsReasoningControls = typeof model === 'string' && model.startsWith('gpt-5'); + // Translate the neutral compaction opt-in (or pass a raw + // `context_management` payload through) to OpenAI's Responses shape. + const contextManagement = toOpenAiContextManagement({ + compaction, + context_management, + }); + const completionParams: ResponseCreateParams = { user: userIdentifier, safety_identifier: userIdentifier, @@ -188,6 +198,9 @@ export class OpenAiResponsesChatProvider implements IChatProvider { ? { parallel_tool_calls } : {}), ...(include !== undefined ? { include } : {}), + ...(contextManagement !== undefined + ? { context_management: contextManagement } + : {}), ...(conversation !== undefined ? { conversation } : {}), ...(previous_response_id !== undefined ? { previous_response_id } diff --git a/src/backend/drivers/ai-chat/types.ts b/src/backend/drivers/ai-chat/types.ts index 1af4c8972..60d8d1e38 100644 --- a/src/backend/drivers/ai-chat/types.ts +++ b/src/backend/drivers/ai-chat/types.ts @@ -67,6 +67,19 @@ export interface ICompleteArguments { parallel_tool_calls?: boolean; include?: unknown[]; conversation?: unknown; + /** + * Provider-neutral inline-compaction opt-in. `true` enables compaction with + * provider defaults; `{ trigger_tokens }` sets the token threshold at which + * the upstream summarizes earlier context. Each provider translates this to + * its own SDK shape (OpenAI `context_management:[{type:'compaction',...}]`, + * Anthropic `context_management:{edits:[{type:'compact_20260112'}]}`). + */ + compaction?: boolean | { trigger_tokens?: number }; + /** + * Escape hatch: provider-native `context_management` payload, passed through + * untouched (used by `/responses` callers sending the OpenAI-native array). + */ + context_management?: unknown; previous_response_id?: string; instructions?: string | PuterMessage[]; metadata?: Record; @@ -116,6 +129,13 @@ export interface IChatMessageResult { finally_fn?: never; normalized?: boolean; via_ai_chat_service?: boolean; + /** + * Inline-compaction artifact, present when the upstream compacted earlier + * context during this (non-streaming) response. Carries `type:'compaction'` + * so it's a drop-in `messages` item — the caller resends it on the next turn + * in place of the summarized history. See [[ICompleteArguments]]. + */ + compaction?: { type: 'compaction'; id?: string; encrypted_content: string }; } export type IChatCompleteResult = IChatStreamResult | IChatMessageResult; diff --git a/src/backend/drivers/ai-chat/utils/Messages.js b/src/backend/drivers/ai-chat/utils/Messages.js index a7a032e97..63b29b87d 100644 --- a/src/backend/drivers/ai-chat/utils/Messages.js +++ b/src/backend/drivers/ai-chat/utils/Messages.js @@ -49,6 +49,23 @@ export const normalize_single_message = (message, params = {}) => { legacyCode: 'bad_request', }); } + // A round-tripped inline-compaction artifact may be supplied as a bare + // top-level item (the shape the client received from the stream). Wrap it + // into an internal compaction content block so it survives normalization; + // each provider maps it back to its native input shape (OpenAI top-level + // input item, Anthropic content block). + if (!message.role && !message.content && message.type === 'compaction') { + return { + role: 'assistant', + content: [ + { + type: 'compaction', + ...(message.id !== undefined ? { id: message.id } : {}), + encrypted_content: message.encrypted_content, + }, + ], + }; + } if (!message.role) { message.role = params.role; } diff --git a/src/backend/drivers/ai-chat/utils/Messages.test.ts b/src/backend/drivers/ai-chat/utils/Messages.test.ts index ac2f8adf9..f692ec502 100644 --- a/src/backend/drivers/ai-chat/utils/Messages.test.ts +++ b/src/backend/drivers/ai-chat/utils/Messages.test.ts @@ -46,6 +46,22 @@ describe('normalize_single_message', () => { expect(result.role).toBe('system'); }); + it('wraps a bare round-tripped compaction item into a compaction content block', () => { + const result = normalize_single_message({ + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }); + expect(result.role).toBe('assistant'); + expect(result.content).toEqual([ + { + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }, + ]); + }); + it('throws 400 when message is null/undefined/array', () => { expect(() => normalize_single_message(null)).toThrow( expect.objectContaining({ statusCode: 400 }), diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js index 71ddde6cd..0e28ebd4a 100644 --- a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js @@ -119,6 +119,24 @@ export const process_input_messages_responses_api = async (messages) => { continue; } + // Round-tripped inline-compaction artifact → Responses compaction input + // item (`{ type:'compaction', encrypted_content, id }` at top level). + if (Array.isArray(msg.content)) { + const compactionBlock = msg.content.find( + (c) => c && c.type === 'compaction', + ); + if (compactionBlock) { + msg.type = 'compaction'; + msg.encrypted_content = compactionBlock.encrypted_content; + if (compactionBlock.id !== undefined) { + msg.id = compactionBlock.id; + } + delete msg.role; + delete msg.content; + continue; + } + } + if (!msg.content) continue; if (typeof msg.content !== 'object') continue; @@ -355,6 +373,19 @@ export const create_chat_stream_handler_responses_api = last_usage = chunk.response.usage; } + if ( + chunk.type === 'response.output_item.done' && + chunk.item?.type === 'compaction' + ) { + // Inline compaction fired mid-response — normalize the artifact + // into the canonical internal compaction event. + chatStream.compaction({ + id: chunk.item.id, + encrypted_content: chunk.item.encrypted_content, + }); + continue; + } + if ( chunk.type === 'response.output_item.done' && chunk.item?.type === 'function_call' @@ -488,10 +519,14 @@ export const handle_completion_output_responses_api = async ({ ...(item.id ? { canonical_id: item.id } : {}), })); + // Inline-compaction artifact, if the upstream compacted this turn. + const compactionItem = output.find((item) => item?.type === 'compaction'); + const is_empty = completion.output_text.trim() === ''; - if (is_empty && responseToolCalls.length < 1) { + if (is_empty && responseToolCalls.length < 1 && !compactionItem) { // GPT refuses to generate an empty response if you ask it to, // so this will probably only happen on an error condition. + // A compaction-only output is legitimate, so don't reject it. throw new HttpError(400, 'an empty response was generated', { legacyCode: 'bad_response', }); @@ -523,6 +558,18 @@ export const handle_completion_output_responses_api = async ({ }; ret.role = output.find((item) => item?.role)?.role ?? 'assistant'; + if (compactionItem) { + // Include `type` so the artifact is a drop-in `messages` item for the + // stateless round-trip — symmetric with the streaming compaction chunk. + ret.compaction = { + type: 'compaction', + ...(compactionItem.id !== undefined + ? { id: compactionItem.id } + : {}), + encrypted_content: compactionItem.encrypted_content, + }; + } + delete ret.type; ret.usage = usage_calculator diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts b/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts index fbce447eb..7fc2488c5 100644 --- a/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts @@ -232,6 +232,30 @@ describe('process_input_messages_responses_api', () => { expect(out!.output).toBe('part-apart-bpart-c'); }); + it('rewrites a round-tripped compaction content block into a top-level compaction item', async () => { + const messages: Array> = [ + { + role: 'assistant', + content: [ + { + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }, + ], + }, + ]; + + const [out] = (await process_input_messages_responses_api( + messages, + )) as Array>; + expect(out!.type).toBe('compaction'); + expect(out!.encrypted_content).toBe('ENC'); + expect(out!.id).toBe('cmpct_1'); + expect(out!.role).toBeUndefined(); + expect(out!.content).toBeUndefined(); + }); + it('upgrades user/system text blocks to input_text', async () => { const messages: Array> = [ { @@ -539,6 +563,39 @@ describe('create_chat_stream_handler_responses_api', () => { }); }); + it('emits a compaction event when a compaction output_item completes', async () => { + const completion = asAsyncIterable([ + { + type: 'response.output_item.done', + item: { + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }, + }, + { + type: 'response.completed', + response: { usage: { input_tokens: 1, output_tokens: 2 } }, + }, + ]); + const init = create_chat_stream_handler_responses_api({ + deviations: undefined, + completion, + usage_calculator: () => ({}), + }); + const harness = makeCapturingChatStream(); + await init({ chatStream: harness.chatStream }); + + const compaction = harness + .events() + .find((e: { type: string }) => e.type === 'compaction'); + expect(compaction).toEqual({ + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }); + }); + it('emits a tool_use block when a function_call output_item completes', async () => { const completion = asAsyncIterable([ { @@ -765,6 +822,31 @@ describe('handle_completion_output_responses_api non-stream', () => { ).rejects.toMatchObject({ statusCode: 400 }); }); + it('attaches a compaction artifact and allows a compaction-only output', async () => { + const completion = { + output: [ + { + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }, + ], + output_text: ' ', + usage: { input_tokens: 1, output_tokens: 0 }, + }; + const result = await handle_completion_output_responses_api({ + deviations: undefined, + stream: false, + completion, + }); + // Compaction-only output is not rejected as "empty". + expect(result.compaction).toEqual({ + type: 'compaction', + id: 'cmpct_1', + encrypted_content: 'ENC', + }); + }); + it('runs moderation against output_text when a moderate fn is supplied', async () => { const completion = { output: [{ role: 'assistant' }], diff --git a/src/backend/drivers/ai-chat/utils/Streaming.js b/src/backend/drivers/ai-chat/utils/Streaming.js index 95a11dc87..fa4d12f0b 100644 --- a/src/backend/drivers/ai-chat/utils/Streaming.js +++ b/src/backend/drivers/ai-chat/utils/Streaming.js @@ -103,6 +103,25 @@ export class AIChatStream { this.stream.end(); } + /** + * Emit a canonical compaction event into the NDJSON stream. Both the OpenAI + * and Anthropic providers normalize their native inline-compaction artifact + * to this single shape, so downstream consumers (controllers, puter.js) see + * an identical `{ type: 'compaction', id, encrypted_content }` chunk + * regardless of which upstream served the request. + * + * @param {{ id?: string, encrypted_content: string }} compaction + */ + compaction({ id, encrypted_content }) { + this.stream.write( + `${JSON.stringify({ + type: 'compaction', + ...(id !== undefined ? { id } : {}), + encrypted_content, + })}\n`, + ); + } + message() { return new AIChatMessageStream(this); } diff --git a/src/backend/drivers/ai-chat/utils/compaction.js b/src/backend/drivers/ai-chat/utils/compaction.js new file mode 100644 index 000000000..9ab764bb0 --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/compaction.js @@ -0,0 +1,124 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Inline-compaction translation helpers. + * + * The driver-facing surface exposes a single provider-neutral opt-in + * (`compaction: boolean | { trigger_tokens }` on `ICompleteArguments`), plus a + * raw `context_management` escape hatch for callers hitting `/responses` with + * the OpenAI-native shape. These helpers map that neutral opt-in to each + * provider's SDK shape so the providers stay free of opt-in-parsing logic. + */ + +/** + * @param {boolean | { trigger_tokens?: number } | undefined} compaction + * @returns {{ enabled: boolean, trigger_tokens?: number }} + */ +const readCompaction = (compaction) => { + if (compaction === true) return { enabled: true }; + if (compaction && typeof compaction === 'object') { + return { + enabled: true, + ...(typeof compaction.trigger_tokens === 'number' + ? { trigger_tokens: compaction.trigger_tokens } + : {}), + }; + } + return { enabled: false }; +}; + +/** + * Build OpenAI Responses `context_management` from the neutral opt-in. A raw + * `context_management` passthrough (already in OpenAI shape) wins. + * + * @param {{ compaction?: boolean | { trigger_tokens?: number }, context_management?: unknown }} args + * @returns {Array<{ type: 'compaction', compact_threshold?: number }> | undefined} + */ +export const toOpenAiContextManagement = (args) => { + if (args.context_management !== undefined) { + return /** @type {any} */ (args.context_management); + } + const { enabled, trigger_tokens } = readCompaction(args.compaction); + if (!enabled) return undefined; + return [ + { + type: 'compaction', + ...(trigger_tokens !== undefined + ? { compact_threshold: trigger_tokens } + : {}), + }, + ]; +}; + +/** + * Build Anthropic `context_management` (beta `compact-2026-01-12`) from the + * neutral opt-in. A raw `context_management` passthrough wins. + * + * @param {{ compaction?: boolean | { trigger_tokens?: number }, context_management?: unknown }} args + * @returns {{ edits: Array> } | undefined} + */ +export const toAnthropicContextManagement = (args) => { + if (args.context_management !== undefined) { + return /** @type {any} */ (args.context_management); + } + const { enabled, trigger_tokens } = readCompaction(args.compaction); + if (!enabled) return undefined; + return { + edits: [ + { + type: 'compact_20260112', + ...(trigger_tokens !== undefined + ? { + trigger: { + type: 'input_tokens', + value: trigger_tokens, + }, + } + : {}), + }, + ], + }; +}; + +/** + * Whether the request opted into inline compaction by any route. + * + * @param {{ compaction?: boolean | { trigger_tokens?: number }, context_management?: unknown }} args + */ +export const wantsCompaction = (args) => + args.context_management !== undefined || + readCompaction(args.compaction).enabled; + +/** + * Whether the (normalized) message list carries a round-tripped compaction + * artifact. Such a request must route through a compaction-capable surface even + * if it didn't request *new* compaction — chat.completions can't represent a + * compaction content block, and Anthropic needs its compaction beta to accept + * one as input. + * + * @param {unknown} messages + */ +export const messagesHaveCompaction = (messages) => + Array.isArray(messages) && + messages.some( + (m) => + Array.isArray(m?.content) && + m.content.some((c) => c && c.type === 'compaction'), + ); diff --git a/src/backend/drivers/ai-chat/utils/compaction.test.ts b/src/backend/drivers/ai-chat/utils/compaction.test.ts new file mode 100644 index 000000000..18e1db534 --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/compaction.test.ts @@ -0,0 +1,132 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { describe, expect, it } from 'vitest'; +import { + messagesHaveCompaction, + toAnthropicContextManagement, + toOpenAiContextManagement, + wantsCompaction, +} from './compaction.js'; + +describe('toOpenAiContextManagement', () => { + it('returns undefined when compaction is off', () => { + expect(toOpenAiContextManagement({})).toBeUndefined(); + expect( + toOpenAiContextManagement({ compaction: false }), + ).toBeUndefined(); + }); + + it('builds a compaction entry from `true`', () => { + expect(toOpenAiContextManagement({ compaction: true })).toEqual([ + { type: 'compaction' }, + ]); + }); + + it('maps trigger_tokens to compact_threshold', () => { + expect( + toOpenAiContextManagement({ compaction: { trigger_tokens: 1000 } }), + ).toEqual([{ type: 'compaction', compact_threshold: 1000 }]); + }); + + it('passes a raw context_management payload through verbatim', () => { + const raw = [{ type: 'compaction', compact_threshold: 5 }]; + expect( + toOpenAiContextManagement({ + compaction: true, + context_management: raw, + }), + ).toBe(raw); + }); +}); + +describe('toAnthropicContextManagement', () => { + it('returns undefined when compaction is off', () => { + expect(toAnthropicContextManagement({})).toBeUndefined(); + }); + + it('builds a compact_20260112 edit from `true`', () => { + expect(toAnthropicContextManagement({ compaction: true })).toEqual({ + edits: [{ type: 'compact_20260112' }], + }); + }); + + it('maps trigger_tokens to an input_tokens trigger', () => { + expect( + toAnthropicContextManagement({ + compaction: { trigger_tokens: 2000 }, + }), + ).toEqual({ + edits: [ + { + type: 'compact_20260112', + trigger: { type: 'input_tokens', value: 2000 }, + }, + ], + }); + }); + + it('passes a raw context_management payload through verbatim', () => { + const raw = { edits: [{ type: 'compact_20260112' }] }; + expect( + toAnthropicContextManagement({ context_management: raw }), + ).toBe(raw); + }); +}); + +describe('wantsCompaction', () => { + it('is false without opt-in', () => { + expect(wantsCompaction({})).toBe(false); + expect(wantsCompaction({ compaction: false })).toBe(false); + }); + + it('is true for the neutral opt-in or a raw passthrough', () => { + expect(wantsCompaction({ compaction: true })).toBe(true); + expect(wantsCompaction({ compaction: { trigger_tokens: 1 } })).toBe( + true, + ); + expect(wantsCompaction({ context_management: [] })).toBe(true); + }); +}); + +describe('messagesHaveCompaction', () => { + it('detects a round-tripped compaction content block', () => { + expect( + messagesHaveCompaction([ + { role: 'user', content: [{ type: 'text', text: 'hi' }] }, + { + role: 'assistant', + content: [ + { type: 'compaction', encrypted_content: 'ENC' }, + ], + }, + ]), + ).toBe(true); + }); + + it('is false for ordinary messages or non-arrays', () => { + expect( + messagesHaveCompaction([ + { role: 'user', content: [{ type: 'text', text: 'hi' }] }, + ]), + ).toBe(false); + expect(messagesHaveCompaction(undefined)).toBe(false); + expect(messagesHaveCompaction('nope')).toBe(false); + }); +}); diff --git a/src/backend/package.json b/src/backend/package.json index 80350b347..a10e23c0c 100644 --- a/src/backend/package.json +++ b/src/backend/package.json @@ -9,7 +9,7 @@ "bench": "vitest bench --config=vitest.bench.config.ts --run" }, "dependencies": { - "@anthropic-ai/sdk": "^0.68.0", + "@anthropic-ai/sdk": "^0.105.0", "@aws-sdk/client-dynamodb": "^3.490.0", "@aws-sdk/client-polly": "^3.1028.0", "@aws-sdk/client-s3": "^3.1028.0", diff --git a/src/backend/services/selfhosted/DefaultUserService.ts b/src/backend/services/selfhosted/DefaultUserService.ts index edf568a8c..f91559452 100644 --- a/src/backend/services/selfhosted/DefaultUserService.ts +++ b/src/backend/services/selfhosted/DefaultUserService.ts @@ -24,6 +24,8 @@ import { PuterService } from '../types.js'; import type { UserRow } from '../../stores/user/UserStore.js'; import { generateDefaultFsentries } from '../../util/userProvisioning.js'; import type { AppIconService } from '../appIcon/AppIconService.js'; +import { LOCAL_UNLIMITED_USER } from '../../data/subPolicies/localUnlimitedUserPolicy.js'; +import { UNLIMITED_SUBSCRIPTION } from '../metering/consts.js'; const USERNAME = 'admin'; const ADMIN_GROUP_UID = 'ca342a5e-b13d-4dee-9048-58b11a57cc55'; @@ -43,6 +45,20 @@ const ADMIN_STORAGE_BYTES = 10 * 1024 * 1024 * 1024; */ export class DefaultUserService extends PuterService { override async onServerStart(): Promise { + // Dev convenience: grant the bootstrap `admin` user unlimited metering. + // Gated to env === 'dev' so prod deployments never get a free-usage + // actor. Registering the policy makes its `'unlimited'` id resolvable + // (extraPolicies is always in the available set); the resolver returns + // null for everyone else, so all other users keep their normal tier. + if (this.config.env === 'dev') { + this.services.metering.registerPolicy(LOCAL_UNLIMITED_USER); + this.services.metering.registerSubscriptionResolver((actor) => + actor.user?.username === USERNAME + ? UNLIMITED_SUBSCRIPTION + : null, + ); + } + if (this.config.no_default_user) return; let user = await this.stores.user.getByUsername(USERNAME); let tmpPassword: string; diff --git a/src/docs/src/AI/chat.md b/src/docs/src/AI/chat.md index 794a8405b..a5e0bc251 100755 --- a/src/docs/src/AI/chat.md +++ b/src/docs/src/AI/chat.md @@ -34,6 +34,7 @@ An object containing the following properties: - `tools` (Array) (Optional) - Function definitions the AI can call. See [Function Calling](#function-calling) for details. - `reasoning_effort` / `reasoning.effort` (String) (Optional) - Controls how much effort reasoning models spend thinking. Supported values: `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`. Lower values give faster responses with less reasoning. OpenAI models only. - `verbosity` / `text.verbosity` (String) (Optional) - Controls how long or short responses are. Supported values: `low`, `medium`, and `high`. Lower values give shorter responses. OpenAI models only. +- `compaction` (Boolean | Object) (Optional) - Opt into inline context compaction for long conversations. Pass `true` to enable it with provider defaults, or `{ trigger_tokens: number }` to set the token threshold at which earlier context is summarized. When the model compacts, you receive a `compaction` chunk while streaming (or a `compaction` field on the result when not streaming) containing an opaque `encrypted_content` summary. Resend that item in `messages` on the next turn in place of the summarized history. The compaction chunk shape is identical across providers, so the same code works whether `model` is an OpenAI or Anthropic model. See [Compaction](#compaction). #### `testMode` (Boolean) (Optional) @@ -183,6 +184,105 @@ Pass in the `cache_control` parameter inside the object in the `messages` array. You can find the implementation in our [prompt caching example](/playground/ai-claude-cache-control/). Find more details about cache control in [Anthropic documentation](https://platform.claude.com/docs/en/build-with-claude/prompt-caching). +## Compaction + +For long, multi-turn conversations that you keep on the client, enable +`compaction` so the model can summarize earlier context before it overflows the +context window. When it fires, the model summarizes the older turns into a single +**compaction artifact** and answers from that summary instead of the full +history — so the request stays small. + +You get the artifact back as a `compaction` item (a stream chunk when streaming, +or `result.compaction` when not). On the next turn you resend that one item **in +place of the turns it replaced**, instead of the raw history. The item shape is +the same across providers, so the same code works for OpenAI and Anthropic models. + +#### Enabling it + +Pass `compaction` in the options object: + +- `compaction: true` — enable with provider defaults. +- `compaction: { trigger_tokens: 60000 }` — set the token threshold at which the + model compacts. + +The artifact is `{ type: 'compaction', id, encrypted_content }` — a drop-in +`messages` item. `encrypted_content` is an opaque payload; treat it as a black +box and just carry it forward. + +#### Streaming + +```js +const resp = await puter.ai.chat(messages, { + model: 'gpt-5.4', // or 'claude-opus-4-8' — same code + stream: true, + compaction: { trigger_tokens: 60000 }, +}); + +let text = ''; +let compaction = null; +for await ( const part of resp ) { + if ( part.type === 'text' ) text += part.text; + else if ( part.type === 'compaction' ) compaction = part; // { type, id, encrypted_content } + else if ( part.type === 'error' ) console.error('stream error:', part.message); +} + +// Next turn: the artifact stands in for the compacted history. Place it where +// that history was (before the new user turn) and keep compaction enabled so it +// can compact again later. +if ( compaction ) { + const next = await puter.ai.chat( + [ + { role: 'system', content: 'You are a helpful assistant.' }, + compaction, + { role: 'user', content: 'now compare the two approaches' }, + ], + { model: 'gpt-5.4', stream: true, compaction: true } + ); + for await ( const part of next ) { + if ( part.type === 'text' ) document.write(part.text); + } +} +``` + +#### Non-streaming + +```js +const result = await puter.ai.chat(messages, { + model: 'gpt-5.4', + compaction: { trigger_tokens: 60000 }, +}); +console.log(result.message.content); + +if ( result.compaction ) { + // result.compaction is { type: 'compaction', id, encrypted_content } — the + // same drop-in item you get from the stream. Resend it next turn: + const next = await puter.ai.chat( + [ + { role: 'system', content: 'You are a helpful assistant.' }, + result.compaction, + { role: 'user', content: 'now compare the two approaches' }, + ], + { model: 'gpt-5.4', compaction: true } + ); + console.log(next.message.content); +} +``` + +#### Notes + +- **Keep `compaction` enabled on every turn** of the conversation so it can + compact again as the conversation keeps growing. +- **Place the artifact where the compacted history was** — after your system + prompt, before the new user message. +- **It only fires once the context is large enough.** Anthropic models require a + minimum threshold of **50,000 tokens**, and the conversation must actually + exceed your `trigger_tokens`. OpenAI models don't enforce that floor, so they + can compact smaller conversations. If nothing compacts, your input was below + the threshold. +- **Handle the `error` chunk** when streaming — provider errors (e.g. a + `trigger_tokens` below a provider's minimum) arrive as an `error` chunk, not a + thrown exception. + ## Image Generation (Gemini Image Models) Certain Gemini models can generate and edit images as part of a chat conversation. These models accept text and image inputs, and return text and images in the response. diff --git a/src/docs/src/Objects/chatresponse.md b/src/docs/src/Objects/chatresponse.md index 8d4305e5b..dc4762594 100644 --- a/src/docs/src/Objects/chatresponse.md +++ b/src/docs/src/Objects/chatresponse.md @@ -22,3 +22,7 @@ An object containing the chat message data. - `cache_control` (Object) - An optional object controlling prompt caching for this message. Contains a `type` (String) property. - `images` (Array) - An array of image content objects associated with the message. Each object contains a `type` (String) and an `image_url` object with a `url` (String) property. + +#### `compaction` (Object) + +Present only on non-streaming responses where the model compacted earlier context (see [Compaction](/AI/chat#compaction)). A drop-in `messages` item of the form `{ type: 'compaction', id, encrypted_content }` — resend it on the next turn in place of the summarized history. Absent when no compaction occurred. diff --git a/src/docs/src/Objects/chatresponsechunk.md b/src/docs/src/Objects/chatresponsechunk.md index 562248177..5f56c6132 100644 --- a/src/docs/src/Objects/chatresponsechunk.md +++ b/src/docs/src/Objects/chatresponsechunk.md @@ -16,6 +16,7 @@ The kind of chunk. One of: - `"text"` - A portion of the response text. - `"reasoning"` - A portion of the model's reasoning/thinking output. - `"tool_use"` - A tool/function the model wants to call. +- `"compaction"` - An inline-compaction summary of earlier context (when `compaction` is enabled). - `"extra_content"` - Provider-specific metadata. - `"usage"` - Token usage totals, emitted as the final chunk. @@ -29,7 +30,11 @@ A portion of the model's reasoning output. Present on `reasoning` chunks. #### `id` (String) -The unique identifier for the tool call. Present on `tool_use` chunks. +The unique identifier for the tool call (`tool_use` chunks) or the compaction item (`compaction` chunks). + +#### `encrypted_content` (String) + +The opaque/encrypted compaction summary. Present on `compaction` chunks. The shape is identical across providers — resend this item in `messages` on the next turn in place of the summarized history. #### `name` (String) diff --git a/src/puter-js/src/modules/AI.js b/src/puter-js/src/modules/AI.js index c1205ea2c..4dc2bf096 100644 --- a/src/puter-js/src/modules/AI.js +++ b/src/puter-js/src/modules/AI.js @@ -813,8 +813,10 @@ class AI { requestParams.provider = requestParams.provider || userParams.driver; } - // Additional parameters to pass from userParams to requestParams - const PARAMS_TO_PASS = ['tools', 'response', 'reasoning', 'reasoning_effort', 'text', 'verbosity', 'provider', 'image_config']; + // Additional parameters to pass from userParams to requestParams. + // `compaction` (provider-neutral inline-compaction opt-in) and the raw + // `context_management` escape hatch flow straight through to the driver. + const PARAMS_TO_PASS = ['tools', 'response', 'reasoning', 'reasoning_effort', 'text', 'verbosity', 'provider', 'image_config', 'compaction', 'context_management']; for ( const name of PARAMS_TO_PASS ) { if ( userParams[name] ) { requestParams[name] = userParams[name]; diff --git a/src/puter-js/test/ai.test.js b/src/puter-js/test/ai.test.js index d491b76e7..70c7041a5 100644 --- a/src/puter-js/test/ai.test.js +++ b/src/puter-js/test/ai.test.js @@ -139,6 +139,43 @@ const testChatStreamingCore = async function(model) { } }; +const testChatCompactionCore = async function(model) { + // Opting into inline compaction must not break a normal streamed response. + // We can't reliably force the model to compact (needs a huge context), so + // this asserts the opt-in streams cleanly and that any compaction chunk + // that does appear carries a provider-independent encrypted_content. + const result = await puter.ai.chat("Count from 1 to 5", { + model: model, + stream: true, + max_tokens: 100, + compaction: true, + }); + + assert(typeof result === 'object' && result !== null, "compaction chat should return an object"); + assert(typeof result[Symbol.asyncIterator] === 'function', "compaction chat should be streamable"); + + let chunkCount = 0; + let compaction = null; + for await (const chunk of result) { + assert(typeof chunk === 'object', "each streaming chunk should be an object"); + if (chunk.type === 'compaction') compaction = chunk; + if (++chunkCount >= 10) break; + } + assert(chunkCount > 0, "compaction streaming should produce at least one chunk"); + + if (compaction) { + assert(typeof compaction.encrypted_content === 'string', + "a compaction chunk should carry an encrypted_content string"); + + // Round-trip: the artifact can be resent as a message item. + const next = await puter.ai.chat([ + { role: 'user', content: 'continue' }, + compaction, + ], { model: model }); + assert(typeof next === 'object' && next !== null, "resending a compaction item should succeed"); + } +}; + // Function to generate test functions for a specific model const generateTestsForModel = function(model) { const modelName = model.replace(/[^a-zA-Z0-9]/g, '_'); // Sanitize model name for function names @@ -195,6 +232,19 @@ const generateTestsForModel = function(model) { } } }, + + [`testChatCompaction_${modelName}`]: { + name: `testChatCompaction_${modelName}`, + description: `Test AI chat inline compaction opt-in and round-trip using ${model} model`, + test: async function() { + try { + await testChatCompactionCore(model); + pass(`testChatCompaction_${modelName} passed`); + } catch (error) { + fail(`testChatCompaction_${modelName} failed:`, error); + } + } + }, }; }; diff --git a/src/puter-js/types/modules/ai.d.ts b/src/puter-js/types/modules/ai.d.ts index ccacdc5e1..5196244e6 100644 --- a/src/puter-js/types/modules/ai.d.ts +++ b/src/puter-js/types/modules/ai.d.ts @@ -68,6 +68,20 @@ export interface ChatOptions { * - `image_size`: output quality/resolution; must be one of the model's supported quality levels. */ image_config?: { aspect_ratio: string, image_size: string }; + /** + * Provider-neutral inline-compaction opt-in for long stateless + * conversations. `true` enables it with provider defaults; an object sets + * the token threshold at which earlier context is summarized. When the + * upstream compacts, you receive a `"compaction"` chunk (streaming) or a + * `compaction` field on the result (non-streaming) — resend it in `messages` + * on the next turn in place of the summarized history. + */ + compaction?: boolean | { trigger_tokens?: number }; + /** + * Escape hatch: a provider-native `context_management` payload, passed + * through untouched. Prefer `compaction` for provider portability. + */ + context_management?: unknown; } export interface StreamingChatOptions extends ChatOptions { @@ -77,6 +91,13 @@ export interface StreamingChatOptions extends ChatOptions { export interface ChatResponse { message?: ChatMessage; choices?: unknown; + /** + * Inline-compaction artifact, present when the upstream compacted earlier + * context during this (non-streaming) response. Carries `type:'compaction'` + * so you can push it straight into `messages` on the next turn in place of + * the summarized history (same shape as the streaming `compaction` chunk). + */ + compaction?: { type: 'compaction'; id?: string; encrypted_content: string }; } /** @@ -84,18 +105,24 @@ export interface ChatResponse { * discriminator; which other fields are present depends on that `type`. */ export interface ChatResponseChunk { - /** The kind of chunk: `"text"`, `"reasoning"`, `"tool_use"`, `"extra_content"`, or `"usage"`. */ + /** The kind of chunk: `"text"`, `"reasoning"`, `"tool_use"`, `"compaction"`, `"extra_content"`, or `"usage"`. */ type: string; /** Text delta. Present on `"text"` chunks. */ text?: string; /** Reasoning/thinking delta. Present on `"reasoning"` chunks. */ reasoning?: string; - /** Tool call id. Present on `"tool_use"` chunks. */ + /** Tool call id (`"tool_use"`) or compaction item id (`"compaction"`). */ id?: string; /** Tool/function name. Present on `"tool_use"` chunks. */ name?: string; /** Parsed tool call arguments. Present on `"tool_use"` chunks. */ input?: unknown; + /** + * Opaque/encrypted compaction summary. Present on `"compaction"` chunks — + * the same shape regardless of which provider served the request. Resend it + * in `messages` on the next turn in place of the summarized history. + */ + encrypted_content?: string; /** Provider-specific extra metadata. */ extra_content?: unknown; /** Token usage totals. Present on the final `"usage"` chunk. */