lee/puter
1
0
Fork 0
mirror of https://github.com/HeyPuter/puter.git synced 2026-08-28 08:56:58 +00:00
Code Issues Actions 3 Packages Projects Releases Wiki Activity

Alpha: compaction support for OpenAI and Anthropic

Browse Source
This commit is contained in:
Neal Shah
2026-06-21 18:41:04 -04:00
parent b576f01d12
commit 3d45e76801
25 changed files with 1387 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
extensions/metering.ts
+1 -1
View File
@@ -1,4 +1,3 @@
import type { Request, Response } from 'express';
import { Context } from '@heyputer/backend/src/core';
import { HttpError } from '@heyputer/backend/src/core/http';
import {
@@ -6,6 +5,7 @@ import {
driversContainers,
} from '@heyputer/backend/src/exports';
import { extension } from '@heyputer/backend/src/extensions';
import type { Request, Response } from 'express';
const services = extension.import('service');
const clients = extension.import('client');
src/backend/controllers/puterai/PuterAIController.test.ts
+206
View File
@@ -1506,6 +1506,212 @@ describe('PuterAIController.anthropicMessages streaming + helpers', () => {
});
});
// ── Inline compaction ───────────────────────────────────────────────
describe('PuterAIController inline compaction', () => {
// Extract the single `event: compaction\ndata: {...}` SSE frame.
const compactionFrame = (out: string): string | null => {
const m = out.match(/event: compaction\ndata: [^\n]*\n\n/);
return m ? m[0] : null;
};
it('forwards the compaction opt-in to the driver (/responses)', async () => {
const spy = stubChatComplete({
message: { role: 'assistant', content: 'ok' },
finish_reason: 'stop',
});
await controller.openaiResponses(
makeReq({
body: { model: 'gpt-test', input: 'hi', compaction: true },
actor: makeUserActor(),
}),
makeRes().res,
);
expect(spy.mock.calls[0]![0].compaction).toBe(true);
});
it('round-trips a compaction `input` item into a messages compaction item', async () => {
const spy = stubChatComplete({
message: { role: 'assistant', content: 'ack' },
finish_reason: 'stop',
});
await controller.openaiResponses(
makeReq({
body: {
model: 'gpt-test',
input: [
{
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
},
],
},
actor: makeUserActor(),
}),
makeRes().res,
);
expect(spy.mock.calls[0]![0].messages).toContainEqual({
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
});
});
it('emits a canonical compaction SSE event and output item (/responses stream)', async () => {
stubChatComplete({
dataType: 'stream',
content_type: 'application/x-ndjson',
stream: ndjsonStreamFrom([
{ type: 'text', text: 'hi' },
{ type: 'compaction', id: 'cmpct_9', encrypted_content: 'ENC9' },
{ type: 'usage', usage: { prompt_tokens: 1, completion_tokens: 1 } },
]),
});
const { res, captured } = makeRes();
await controller.openaiResponses(
makeReq({
body: {
model: 'gpt-test',
input: 'hi',
stream: true,
compaction: true,
},
actor: makeUserActor(),
}),
res,
);
await new Promise<void>((resolve) => setImmediate(resolve));
const out = captured.written.join('');
expect(compactionFrame(out)).toBe(
'event: compaction\ndata: {"type":"compaction","id":"cmpct_9","encrypted_content":"ENC9"}\n\n',
);
// Native shape also lands in the final response.completed output[].
const completed = out
.split('event: response.completed\n')[1]
?.split('\n\n')[0];
expect(completed).toContain('"type":"compaction"');
expect(completed).toContain('"encrypted_content":"ENC9"');
});
it('emits the compaction item in non-streaming /responses output', async () => {
stubChatComplete({
message: { role: 'assistant', content: 'done' },
finish_reason: 'stop',
compaction: { id: 'cmpct_n', encrypted_content: 'ENCN' },
});
const { res, captured } = makeRes();
await controller.openaiResponses(
makeReq({
body: { model: 'gpt-test', input: 'hi', compaction: true },
actor: makeUserActor(),
}),
res,
);
const body = captured.body as { output: Array<Record<string, unknown>> };
expect(body.output).toContainEqual(
expect.objectContaining({
type: 'compaction',
encrypted_content: 'ENCN',
}),
);
});
it('emits an identical canonical compaction SSE event on the Anthropic surface', async () => {
// /responses frame
stubChatComplete({
dataType: 'stream',
content_type: 'application/x-ndjson',
stream: ndjsonStreamFrom([
{ type: 'compaction', id: 'cmpct_x', encrypted_content: 'ENCX' },
]),
});
const r1 = makeRes();
await controller.openaiResponses(
makeReq({
body: { model: 'gpt-test', input: 'hi', stream: true },
actor: makeUserActor(),
}),
r1.res,
);
await new Promise<void>((resolve) => setImmediate(resolve));
// /anthropic/v1/messages frame
stubChatComplete({
dataType: 'stream',
content_type: 'application/x-ndjson',
stream: ndjsonStreamFrom([
{ type: 'compaction', id: 'cmpct_x', encrypted_content: 'ENCX' },
]),
});
const r2 = makeRes();
await controller.anthropicMessages(
makeReq({
body: {
model: 'claude-test',
messages: [{ role: 'user', content: 'hi' }],
stream: true,
},
actor: makeUserActor(),
}),
r2.res,
);
await new Promise<void>((resolve) => setImmediate(resolve));
const a = compactionFrame(r1.captured.written.join(''));
const b = compactionFrame(r2.captured.written.join(''));
expect(a).not.toBeNull();
expect(a).toBe(b); // byte-identical streaming shape across providers
});
it('renders a native compaction content block in non-streaming /messages', async () => {
stubChatComplete({
message: { role: 'assistant', content: 'done' },
finish_reason: 'stop',
compaction: { id: 'cmpct_m', encrypted_content: 'ENCM' },
});
const { res, captured } = makeRes();
await controller.anthropicMessages(
makeReq({
body: {
model: 'claude-test',
messages: [{ role: 'user', content: 'hi' }],
},
actor: makeUserActor(),
}),
res,
);
const body = captured.body as { content: Array<Record<string, unknown>> };
expect(body.content).toContainEqual({
type: 'compaction',
id: 'cmpct_m',
encrypted_content: 'ENCM',
});
});
it('does not emit compaction frames for a normal stream (regression)', async () => {
stubChatComplete({
dataType: 'stream',
content_type: 'application/x-ndjson',
stream: ndjsonStreamFrom([
{ type: 'text', text: 'hello' },
{ type: 'usage', usage: { prompt_tokens: 1, completion_tokens: 1 } },
]),
});
const { res, captured } = makeRes();
await controller.openaiResponses(
makeReq({
body: { model: 'gpt-test', input: 'hi', stream: true },
actor: makeUserActor(),
}),
res,
);
await new Promise<void>((resolve) => setImmediate(resolve));
expect(captured.written.join('')).not.toContain('event: compaction');
});
});
// ── Model details listing ───────────────────────────────────────────
describe('PuterAIController model details', () => {
src/backend/controllers/puterai/PuterAIController.ts
+102
View File
@@ -587,6 +587,15 @@ export class PuterAIController extends PuterController {
? { metadata: body.metadata as Record<string, string> }
: {}),
...(body.conversation ? { conversation: body.conversation } : {}),
...(body.context_management !== undefined
? { context_management: body.context_management }
: {}),
...(body.compaction !== undefined
? {
compaction:
body.compaction as ICompleteArguments['compaction'],
}
: {}),
...(body.previous_response_id
? { previous_response_id: String(body.previous_response_id) }
: {}),
@@ -750,6 +759,30 @@ export class PuterAIController extends PuterController {
output_index: outputIndex,
item,
});
} else if (ev.type === 'compaction') {
// Native shape in the final `output[]`, plus the
// canonical SSE event shared with the Anthropic surface.
const item = {
type: 'compaction',
...(ev.id !== undefined ? { id: ev.id } : {}),
encrypted_content: ev.encrypted_content,
};
const outputIndex = output.length;
output.push(item);
sendEvent({
type: 'response.output_item.added',
output_index: outputIndex,
item,
});
sendEvent({
type: 'response.output_item.done',
output_index: outputIndex,
item,
});
writeCompactionEvent(res, {
id: ev.id,
encrypted_content: ev.encrypted_content,
});
} else if (ev.type === 'usage') {
usage = buildResponsesUsage(
ev.usage as Record<string, unknown>,
@@ -869,6 +902,15 @@ export class PuterAIController extends PuterController {
...(body.max_tokens !== undefined
? { max_tokens: Number(body.max_tokens) }
: {}),
...(body.context_management !== undefined
? { context_management: body.context_management }
: {}),
...(body.compaction !== undefined
? {
compaction:
body.compaction as ICompleteArguments['compaction'],
}
: {}),
...(body.provider
? { provider: toStringOrEmpty(body.provider) }
: { provider: DEFAULTS.anthropic }),
@@ -968,6 +1010,14 @@ export class PuterAIController extends PuterController {
},
});
closeBlock();
} else if (ev.type === 'compaction') {
// Close any open content block, then emit the canonical
// compaction SSE event — byte-identical to /responses.
closeBlock();
writeCompactionEvent(res, {
id: ev.id,
encrypted_content: ev.encrypted_content,
});
} else if (ev.type === 'usage') {
usage = ev.usage as Record<string, unknown>;
}
@@ -1026,6 +1076,18 @@ export class PuterAIController extends PuterController {
if (textContent)
contentBlocks.push({ type: 'text', text: textContent });
contentBlocks.push(...toolUseBlocks);
// Native Anthropic-shaped compaction block (non-streaming bodies stay
// provider-native, unlike the unified streaming event).
const compaction = (
messageResult as { compaction?: Record<string, unknown> }
).compaction;
if (compaction) {
contentBlocks.push({
type: 'compaction',
...(compaction.id !== undefined ? { id: compaction.id } : {}),
encrypted_content: compaction.encrypted_content,
});
}
if (contentBlocks.length === 0)
contentBlocks.push({ type: 'text', text: '' });
@@ -1082,6 +1144,25 @@ const setSseHeaders = (res: Response): void => {
res.setHeader('Connection', 'keep-alive');
};
/**
* Inline-compaction is emitted in a single canonical SSE shape that is
* byte-identical across the `/responses` and `/anthropic/v1/messages` streaming
* surfaces, so a streaming client parses compaction the same way regardless of
* which upstream served the request. (Non-streaming bodies stay provider-native.)
*/
const writeCompactionEvent = (
res: Response,
compaction: { id?: unknown; encrypted_content?: unknown },
): void => {
const payload = {
type: 'compaction',
...(compaction.id !== undefined ? { id: compaction.id } : {}),
encrypted_content: compaction.encrypted_content,
};
res.write(`event: compaction\n`);
res.write(`data: ${JSON.stringify(payload)}\n\n`);
};
/**
* The chat driver returns either a stream-result envelope or a plain
* message result. Proxy routes invoked with `stream: true` expect the
@@ -1325,6 +1406,17 @@ const responseInputToMessages = (input: unknown): unknown[] => {
if (!item || typeof item !== 'object') continue;
const it = item as Record<string, unknown>;
if (it.type === 'compaction') {
// Round-tripped compaction artifact: preserve it as a bare item so
// `normalize_single_message` wraps it into an internal compaction
// content block (providers map it back to their native input shape).
messages.push({
type: 'compaction',
...(it.id !== undefined ? { id: it.id } : {}),
encrypted_content: it.encrypted_content,
});
continue;
}
if (it.type === 'function_call_output') {
messages.push({
role: 'tool',
@@ -1416,6 +1508,16 @@ const responseOutputFromResult = (
});
}
const compaction = (result as { compaction?: Record<string, unknown> })
.compaction;
if (compaction) {
output.push({
id: (compaction.id as string | undefined) || generateId('cmpct'),
type: 'compaction',
encrypted_content: compaction.encrypted_content,
});
}
return output;
};
src/backend/drivers/ai-chat/providers/azure/AzureChatProvider.ts
+18
View File
@@ -26,6 +26,10 @@ import type { MeteringService } from '../../../../services/metering/MeteringServ
import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js';
import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js';
import type { IChatProvider, ICompleteArguments } from '../../types.js';
import {
messagesHaveCompaction,
wantsCompaction,
} from '../../utils/compaction.js';
import * as OpenAiUtil from '../../utils/OpenAIUtil.js';
import { processPuterPathUploads } from '../openai/fileUpload.js';
import { AZURE_MODELS } from './models.js';
@@ -141,6 +145,20 @@ export class AzureChatProvider implements IChatProvider {
}
return await this.#responsesProvider.complete(params);
}
// Inline compaction is a Responses-API feature; chat.completions can't
// express `context_management` or a `compaction` content block.
// Delegate to the Responses provider when the caller opted in OR when
// the messages carry a round-tripped compaction artifact.
if (wantsCompaction(params) || messagesHaveCompaction(messages)) {
if (!this.#responsesProvider) {
throw new HttpError(
400,
'compaction requires the OpenAI Responses provider, which is not configured',
{ legacyCode: 'bad_request' },
);
}
return await this.#responsesProvider.complete(params);
}
// Validate messages
if (!Array.isArray(messages)) {
throw new HttpError(400, '`messages` must be an array', {
src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts
+13
View File
@@ -25,6 +25,7 @@ import type { MeteringService } from '../../../../services/metering/MeteringServ
import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js';
import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js';
import type { IChatProvider, ICompleteArguments } from '../../types.js';
import { toOpenAiContextManagement } from '../../utils/compaction.js';
import * as OpenAiUtil from '../../utils/OpenAIUtil.js';
import { processPuterPathUploads } from '../openai/fileUpload.js';
import { AZURE_MODELS } from './models.js';
@@ -109,6 +110,8 @@ export class AzureResponsesProvider implements IChatProvider {
parallel_tool_calls,
include,
conversation,
compaction,
context_management,
previous_response_id,
instructions,
metadata,
@@ -182,6 +185,13 @@ export class AzureResponsesProvider implements IChatProvider {
const supportsReasoningControls =
typeof model === 'string' && model.startsWith('gpt-5');
// Translate the neutral compaction opt-in (or pass a raw
// `context_management` payload through) to OpenAI's Responses shape.
const contextManagement = toOpenAiContextManagement({
compaction,
context_management,
});
const completionParams: ResponseCreateParams = {
user: userIdentifier,
safety_identifier: userIdentifier,
@@ -193,6 +203,9 @@ export class AzureResponsesProvider implements IChatProvider {
? { parallel_tool_calls }
: {}),
...(include !== undefined ? { include } : {}),
...(contextManagement !== undefined
? { context_management: contextManagement }
: {}),
...(conversation !== undefined ? { conversation } : {}),
...(previous_response_id !== undefined
? { previous_response_id }
src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts
+179
View File
@@ -618,6 +618,185 @@ describe('ClaudeProvider.complete streaming', () => {
});
});
// ── Inline compaction ───────────────────────────────────────────────
describe('ClaudeProvider.complete compaction', () => {
it('translates the neutral opt-in to context_management + the compaction beta', async () => {
const { provider } = makeProvider();
messagesCreateMock.mockResolvedValueOnce({
content: [{ type: 'text', text: 'hi' }],
usage: { input_tokens: 1, output_tokens: 1 },
});
await withTestActor(() =>
provider.complete({
model: 'claude-haiku-4-5-20251001',
messages: [{ role: 'user', content: 'hi' }],
compaction: { trigger_tokens: 50000 },
}),
);
const [args] = messagesCreateMock.mock.calls[0]!;
expect(args.context_management).toEqual({
edits: [
{
type: 'compact_20260112',
trigger: { type: 'input_tokens', value: 50000 },
},
],
});
expect(args.betas).toContain('compact-2026-01-12');
});
it('emits a canonical compaction event from a streamed compaction block', async () => {
const { provider } = makeProvider();
messagesStreamMock.mockReturnValueOnce(
makeStreamLike(
[
{ type: 'message_start' },
{
type: 'content_block_start',
content_block: {
type: 'compaction',
id: 'cmpct_1',
content: 'ENC', // Anthropic carries the summary here
},
},
{ type: 'content_block_stop' },
{
type: 'message_delta',
usage: { input_tokens: 1, output_tokens: 1 },
},
{ type: 'message_stop' },
],
{ input_tokens: 1, output_tokens: 1 },
),
);
const result = await withTestActor(() =>
provider.complete({
model: 'claude-haiku-4-5-20251001',
messages: [{ role: 'user', content: 'hi' }],
stream: true,
compaction: true,
}),
);
const harness = makeCapturingChatStream();
await (
result as {
init_chat_stream: (p: { chatStream: unknown }) => Promise<void>;
}
).init_chat_stream({ chatStream: harness.chatStream });
const compaction = harness
.events()
.find((e) => e.type === 'compaction');
expect(compaction).toEqual({
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
});
});
it('enables the compaction beta when a round-tripped compaction block is resent (no opt-in)', async () => {
const { provider } = makeProvider();
messagesCreateMock.mockResolvedValueOnce({
content: [{ type: 'text', text: 'ok' }],
usage: { input_tokens: 1, output_tokens: 1 },
});
await withTestActor(() =>
provider.complete({
model: 'claude-haiku-4-5-20251001',
messages: [
{ role: 'user', content: 'continue' },
{
role: 'assistant',
content: [
{
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
},
],
},
],
// note: no `compaction`/`context_management` opt-in
}),
);
const [args] = messagesCreateMock.mock.calls[0]!;
expect(args.betas).toContain('compact-2026-01-12');
// No new compaction was requested, so no context_management is sent.
expect(args.context_management).toBeUndefined();
});
it('surfaces a compaction block from a non-streaming response as result.compaction', async () => {
const { provider } = makeProvider();
messagesCreateMock.mockResolvedValueOnce({
content: [
{ type: 'text', text: 'done' },
{
type: 'compaction',
id: 'cmpct_2',
content: 'ENC2', // Anthropic carries the summary here
},
],
usage: { input_tokens: 1, output_tokens: 1 },
});
const result = (await withTestActor(() =>
provider.complete({
model: 'claude-haiku-4-5-20251001',
messages: [{ role: 'user', content: 'hi' }],
compaction: true,
}),
)) as { compaction?: { id?: string; encrypted_content: string } };
// Anthropic's `content` is surfaced under the unified `encrypted_content`.
expect(result.compaction).toEqual({
type: 'compaction',
id: 'cmpct_2',
encrypted_content: 'ENC2',
});
});
it('maps a round-tripped compaction block back to Anthropic `content` on input', async () => {
const { provider } = makeProvider();
messagesCreateMock.mockResolvedValueOnce({
content: [{ type: 'text', text: 'ok' }],
usage: { input_tokens: 1, output_tokens: 1 },
});
await withTestActor(() =>
provider.complete({
model: 'claude-haiku-4-5-20251001',
messages: [
{ role: 'user', content: 'continue' },
{
role: 'assistant',
content: [
{ type: 'compaction', encrypted_content: 'SUMMARY' },
],
},
],
}),
);
const [args] = messagesCreateMock.mock.calls[0]!;
const compactionBlock = args.messages
.flatMap((m: { content?: unknown[] }) =>
Array.isArray(m.content) ? m.content : [],
)
.find((c: { type?: string }) => c?.type === 'compaction');
// Internal `encrypted_content` carrier → Anthropic native `content`.
expect(compactionBlock).toEqual({
type: 'compaction',
content: 'SUMMARY',
});
});
});
// ── Moderation ──────────────────────────────────────────────────────
describe('ClaudeProvider.checkModeration', () => {
src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts
+169 -11
View File
@@ -34,6 +34,10 @@ import type {
ICompleteArguments,
IChatCompleteResult,
} from '../../types.js';
import {
messagesHaveCompaction,
toAnthropicContextManagement,
} from '../../utils/compaction.js';
import { make_claude_tools } from '../../utils/FunctionCalling.js';
import { extract_and_remove_system_messages } from '../../utils/Messages.js';
import type {
@@ -44,6 +48,11 @@ import type {
import { FILES_API_BETA, processPuterPathUploads } from './fileUpload.js';
import { CLAUDE_MODELS } from './models.js';
// Anthropic inline-compaction beta. The vendored SDK (0.68.0) doesn't type the
// `compact_20260112` edit or the `compaction` content block, so the request
// params and streamed/returned blocks are handled with `as any` casts.
const COMPACTION_BETA = 'compact-2026-01-12';
export class ClaudeProvider implements IChatProvider {
anthropic: Anthropic;
@@ -97,9 +106,18 @@ export class ClaudeProvider implements IChatProvider {
temperature,
reasoning,
reasoning_effort,
compaction,
context_management,
}: ICompleteArguments): Promise<IChatCompleteResult> {
tools = make_claude_tools(tools);
// Translate the neutral compaction opt-in (or pass a raw
// `context_management` payload through) to Anthropic's beta shape.
const contextManagement = toAnthropicContextManagement({
compaction,
context_management,
});
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let system_prompts: string | any[];
[system_prompts, messages] =
@@ -218,6 +236,23 @@ export class ClaudeProvider implements IChatProvider {
return message;
});
// Map round-tripped compaction blocks back to Anthropic's native shape.
// The internal/unified carrier field is `encrypted_content`; Anthropic's
// compaction block uses `content` (a plaintext summary).
// eslint-disable-next-line @typescript-eslint/no-explicit-any
messages = messages.map((message: any) => {
if (!Array.isArray(message.content)) return message;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
message.content = message.content.map((part: any) => {
if (part?.type !== 'compaction') return part;
return {
type: 'compaction',
content: part.content ?? part.encrypted_content ?? '',
};
});
return message;
});
const modelUsed =
this.models().find((m) =>
[m.id, ...(m.aliases || [])].includes(model),
@@ -262,6 +297,18 @@ export class ClaudeProvider implements IChatProvider {
actor,
);
const usesBetaFiles = uploadedFileIds.length > 0;
// The compaction beta is needed both to *request* compaction
// (contextManagement) and to *accept a round-tripped* compaction block
// back as input (messagesHaveCompaction).
const usesCompaction =
!!contextManagement || messagesHaveCompaction(messages);
// Compaction and Files API both require the beta endpoint; combine their
// beta headers and route through `beta.messages.*` if either is active.
const betas = [
...(usesBetaFiles ? [FILES_API_BETA] : []),
...(usesCompaction ? [COMPACTION_BETA] : []),
];
const usesBeta = betas.length > 0;
const sdkParams: MessageCreateParams & {
betas?: string[];
@@ -291,7 +338,11 @@ export class ClaudeProvider implements IChatProvider {
...(supportsEffort && requestedReasoningEffort
? { output_config: { effort: requestedReasoningEffort } }
: {}),
...(usesBetaFiles ? { betas: [FILES_API_BETA] } : {}),
// Cast: `context_management` compaction edits aren't typed in SDK 0.68.0.
...(contextManagement
? { context_management: contextManagement as any }
: {}),
...(usesBeta ? { betas } : {}),
} as MessageCreateParams & { betas?: string[] };
const cleanupUploads = async () => {
@@ -315,13 +366,23 @@ export class ClaudeProvider implements IChatProvider {
}: {
chatStream: AIChatStream;
}) => {
const completion = usesBetaFiles
const completion = usesBeta
? this.anthropic.beta.messages.stream(sdkParams)
: this.anthropic.messages.stream(sdkParams);
const usageSum: Record<string, number> = {};
let message, contentBlock;
let currentContentBlockType: string | null = null;
// Inline-compaction block is an untyped beta block; capture its
// artifact across start/delta and emit on stop. Anthropic carries
// the summary in `content` (plaintext) — unlike OpenAI's
// `encrypted_content` — so read `content` first.
let compactionData: {
id?: string;
payload: string;
buffer: string;
} | null = null;
let emittedCompaction = false;
for await (const event of completion) {
if (event.type === 'message_delta') {
const meteredData = this.#usageFormatterUtil(
@@ -345,6 +406,24 @@ export class ClaudeProvider implements IChatProvider {
}
if (event.type === 'content_block_start') {
currentContentBlockType = event.content_block.type;
if (
(event.content_block.type as string) ===
'compaction'
) {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const block = event.content_block as any;
compactionData = {
id: block.id,
// Anthropic uses `content`; fall back to
// `encrypted_content` in case the field varies.
payload:
block.content ??
block.encrypted_content ??
'',
buffer: '',
};
continue;
}
if (event.content_block.type === 'tool_use') {
contentBlock = message!.contentBlock({
type: event.content_block.type,
@@ -363,12 +442,42 @@ export class ClaudeProvider implements IChatProvider {
continue;
}
if (event.type === 'content_block_stop') {
if (currentContentBlockType === 'compaction') {
const encrypted_content =
compactionData?.payload ||
compactionData?.buffer ||
'';
// Only emit (and mark done) if we actually captured
// the summary; otherwise let the finalMessage
// fallback recover it from the complete block.
if (encrypted_content) {
chatStream.compaction({
id: compactionData?.id,
encrypted_content,
});
emittedCompaction = true;
}
compactionData = null;
currentContentBlockType = null;
continue;
}
contentBlock!.end();
contentBlock = null;
currentContentBlockType = null;
continue;
}
if (event.type === 'content_block_delta') {
if (currentContentBlockType === 'compaction') {
// Capture any streamed payload for the artifact.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
const d = event.delta as any;
const chunk =
d.partial_json ?? d.text ?? d.data ?? '';
if (typeof chunk === 'string' && compactionData) {
compactionData.buffer += chunk;
}
continue;
}
if (event.delta.type === 'input_json_delta') {
(contentBlock as AIChatToolUseStream)!.addPartialJSON(
event.delta.partial_json,
@@ -391,18 +500,37 @@ export class ClaudeProvider implements IChatProvider {
// signature_delta — ignored
}
}
const finalUsage = await completion
const finalMessage = await completion
.finalMessage()
.then((msg) =>
this.#usageFormatterUtil(
msg.usage as Usage | BetaUsage,
),
)
.catch(() => null);
if (finalUsage) {
if (finalMessage) {
const finalUsage = this.#usageFormatterUtil(
finalMessage.usage as Usage | BetaUsage,
);
for (const [key, value] of Object.entries(finalUsage)) {
usageSum[key] = value;
}
// Fallback: some SDK versions surface the compaction block
// only in the final message, not as a streamed block.
if (!emittedCompaction) {
const block = (
(finalMessage.content as unknown[]) ?? []
).find(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(c: any) => c?.type === 'compaction',
// eslint-disable-next-line @typescript-eslint/no-explicit-any
) as any;
if (block) {
chatStream.compaction({
id: block.id,
encrypted_content:
block.content ??
block.encrypted_content ??
'',
});
emittedCompaction = true;
}
}
}
chatStream.end(usageSum);
const costsOverrideFromModel =
@@ -423,7 +551,7 @@ export class ClaudeProvider implements IChatProvider {
}
try {
const msg = await (usesBetaFiles
const msg = await (usesBeta
? this.anthropic.beta.messages.create(sdkParams)
: this.anthropic.messages.create(sdkParams));
const usage = this.#usageFormatterUtil(
@@ -440,7 +568,37 @@ export class ClaudeProvider implements IChatProvider {
costsOverrideFromModel,
);
return { message: msg, usage, finish_reason: 'stop' };
// Surface any inline-compaction artifact for stateless round-trip.
const compactionBlock = (
((msg as Message).content as unknown[]) ?? []
).find(
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(c: any) => c?.type === 'compaction',
// eslint-disable-next-line @typescript-eslint/no-explicit-any
) as any;
return {
message: msg,
usage,
finish_reason: 'stop',
...(compactionBlock
? {
compaction: {
// `type` makes the artifact a drop-in `messages`
// item for the round-trip (symmetric with the
// streaming compaction chunk).
type: 'compaction' as const,
...(compactionBlock.id !== undefined
? { id: compactionBlock.id }
: {}),
encrypted_content:
compactionBlock.content ??
compactionBlock.encrypted_content ??
'',
},
}
: {}),
};
} finally {
await cleanupUploads();
}
src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts
+18
View File
@@ -26,6 +26,10 @@ import type { MeteringService } from '../../../../services/metering/MeteringServ
import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js';
import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js';
import type { IChatProvider, ICompleteArguments } from '../../types.js';
import {
messagesHaveCompaction,
wantsCompaction,
} from '../../utils/compaction.js';
import * as OpenAiUtil from '../../utils/OpenAIUtil.js';
import { processPuterPathUploads } from './fileUpload.js';
import { OPEN_AI_MODELS } from './models.js';
@@ -125,6 +129,20 @@ export class OpenAiChatProvider implements IChatProvider {
}
return await this.#responsesProvider.complete(params);
}
// Inline compaction is a Responses-API feature; chat.completions can't
// express `context_management` or a `compaction` content block.
// Delegate to the sibling Responses provider when the caller opted in
// OR when the messages carry a round-tripped compaction artifact.
if (wantsCompaction(params) || messagesHaveCompaction(messages)) {
if (!this.#responsesProvider) {
throw new HttpError(
400,
'compaction requires the OpenAI Responses provider, which is not configured',
{ legacyCode: 'bad_request' },
);
}
return await this.#responsesProvider.complete(params);
}
// Validate messages
if (!Array.isArray(messages)) {
throw new HttpError(400, '`messages` must be an array', {
src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts
+13
View File
@@ -25,6 +25,7 @@ import type { MeteringService } from '../../../../services/metering/MeteringServ
import type { FSEntryStore } from '../../../../stores/fs/FSEntryStore.js';
import type { S3ObjectStore } from '../../../../stores/fs/S3ObjectStore.js';
import type { IChatProvider, ICompleteArguments } from '../../types.js';
import { toOpenAiContextManagement } from '../../utils/compaction.js';
import * as OpenAiUtil from '../../utils/OpenAIUtil.js';
import { processPuterPathUploads } from './fileUpload.js';
import { OPEN_AI_MODELS } from './models.js';
@@ -100,6 +101,8 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
parallel_tool_calls,
include,
conversation,
compaction,
context_management,
previous_response_id,
instructions,
metadata,
@@ -177,6 +180,13 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
const supportsReasoningControls =
typeof model === 'string' && model.startsWith('gpt-5');
// Translate the neutral compaction opt-in (or pass a raw
// `context_management` payload through) to OpenAI's Responses shape.
const contextManagement = toOpenAiContextManagement({
compaction,
context_management,
});
const completionParams: ResponseCreateParams = {
user: userIdentifier,
safety_identifier: userIdentifier,
@@ -188,6 +198,9 @@ export class OpenAiResponsesChatProvider implements IChatProvider {
? { parallel_tool_calls }
: {}),
...(include !== undefined ? { include } : {}),
...(contextManagement !== undefined
? { context_management: contextManagement }
: {}),
...(conversation !== undefined ? { conversation } : {}),
...(previous_response_id !== undefined
? { previous_response_id }
src/backend/drivers/ai-chat/types.ts
+20
View File
@@ -67,6 +67,19 @@ export interface ICompleteArguments {
parallel_tool_calls?: boolean;
include?: unknown[];
conversation?: unknown;
/**
* Provider-neutral inline-compaction opt-in. `true` enables compaction with
* provider defaults; `{ trigger_tokens }` sets the token threshold at which
* the upstream summarizes earlier context. Each provider translates this to
* its own SDK shape (OpenAI `context_management:[{type:'compaction',...}]`,
* Anthropic `context_management:{edits:[{type:'compact_20260112'}]}`).
*/
compaction?: boolean | { trigger_tokens?: number };
/**
* Escape hatch: provider-native `context_management` payload, passed through
* untouched (used by `/responses` callers sending the OpenAI-native array).
*/
context_management?: unknown;
previous_response_id?: string;
instructions?: string | PuterMessage[];
metadata?: Record<string, string>;
@@ -116,6 +129,13 @@ export interface IChatMessageResult {
finally_fn?: never;
normalized?: boolean;
via_ai_chat_service?: boolean;
/**
* Inline-compaction artifact, present when the upstream compacted earlier
* context during this (non-streaming) response. Carries `type:'compaction'`
* so it's a drop-in `messages` item — the caller resends it on the next turn
* in place of the summarized history. See [[ICompleteArguments]].
*/
compaction?: { type: 'compaction'; id?: string; encrypted_content: string };
}
export type IChatCompleteResult = IChatStreamResult | IChatMessageResult;
src/backend/drivers/ai-chat/utils/Messages.js
+17
View File
@@ -49,6 +49,23 @@ export const normalize_single_message = (message, params = {}) => {
legacyCode: 'bad_request',
});
}
// A round-tripped inline-compaction artifact may be supplied as a bare
// top-level item (the shape the client received from the stream). Wrap it
// into an internal compaction content block so it survives normalization;
// each provider maps it back to its native input shape (OpenAI top-level
// input item, Anthropic content block).
if (!message.role && !message.content && message.type === 'compaction') {
return {
role: 'assistant',
content: [
{
type: 'compaction',
...(message.id !== undefined ? { id: message.id } : {}),
encrypted_content: message.encrypted_content,
},
],
};
}
if (!message.role) {
message.role = params.role;
}
src/backend/drivers/ai-chat/utils/Messages.test.ts
+16
View File
@@ -46,6 +46,22 @@ describe('normalize_single_message', () => {
expect(result.role).toBe('system');
});
it('wraps a bare round-tripped compaction item into a compaction content block', () => {
const result = normalize_single_message({
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
});
expect(result.role).toBe('assistant');
expect(result.content).toEqual([
{
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
},
]);
});
it('throws 400 when message is null/undefined/array', () => {
expect(() => normalize_single_message(null)).toThrow(
expect.objectContaining({ statusCode: 400 }),
src/backend/drivers/ai-chat/utils/OpenAIUtil.js
+48 -1
View File
@@ -119,6 +119,24 @@ export const process_input_messages_responses_api = async (messages) => {
continue;
}
// Round-tripped inline-compaction artifact → Responses compaction input
// item (`{ type:'compaction', encrypted_content, id }` at top level).
if (Array.isArray(msg.content)) {
const compactionBlock = msg.content.find(
(c) => c && c.type === 'compaction',
);
if (compactionBlock) {
msg.type = 'compaction';
msg.encrypted_content = compactionBlock.encrypted_content;
if (compactionBlock.id !== undefined) {
msg.id = compactionBlock.id;
}
delete msg.role;
delete msg.content;
continue;
}
}
if (!msg.content) continue;
if (typeof msg.content !== 'object') continue;
@@ -355,6 +373,19 @@ export const create_chat_stream_handler_responses_api =
last_usage = chunk.response.usage;
}
if (
chunk.type === 'response.output_item.done' &&
chunk.item?.type === 'compaction'
) {
// Inline compaction fired mid-response — normalize the artifact
// into the canonical internal compaction event.
chatStream.compaction({
id: chunk.item.id,
encrypted_content: chunk.item.encrypted_content,
});
continue;
}
if (
chunk.type === 'response.output_item.done' &&
chunk.item?.type === 'function_call'
@@ -488,10 +519,14 @@ export const handle_completion_output_responses_api = async ({
...(item.id ? { canonical_id: item.id } : {}),
}));
// Inline-compaction artifact, if the upstream compacted this turn.
const compactionItem = output.find((item) => item?.type === 'compaction');
const is_empty = completion.output_text.trim() === '';
if (is_empty && responseToolCalls.length < 1) {
if (is_empty && responseToolCalls.length < 1 && !compactionItem) {
// GPT refuses to generate an empty response if you ask it to,
// so this will probably only happen on an error condition.
// A compaction-only output is legitimate, so don't reject it.
throw new HttpError(400, 'an empty response was generated', {
legacyCode: 'bad_response',
});
@@ -523,6 +558,18 @@ export const handle_completion_output_responses_api = async ({
};
ret.role = output.find((item) => item?.role)?.role ?? 'assistant';
if (compactionItem) {
// Include `type` so the artifact is a drop-in `messages` item for the
// stateless round-trip — symmetric with the streaming compaction chunk.
ret.compaction = {
type: 'compaction',
...(compactionItem.id !== undefined
? { id: compactionItem.id }
: {}),
encrypted_content: compactionItem.encrypted_content,
};
}
delete ret.type;
ret.usage = usage_calculator
src/backend/drivers/ai-chat/utils/OpenAIUtil.test.ts
+82
View File
@@ -232,6 +232,30 @@ describe('process_input_messages_responses_api', () => {
expect(out!.output).toBe('part-apart-bpart-c');
});
it('rewrites a round-tripped compaction content block into a top-level compaction item', async () => {
const messages: Array<Record<string, unknown>> = [
{
role: 'assistant',
content: [
{
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
},
],
},
];
const [out] = (await process_input_messages_responses_api(
messages,
)) as Array<Record<string, unknown>>;
expect(out!.type).toBe('compaction');
expect(out!.encrypted_content).toBe('ENC');
expect(out!.id).toBe('cmpct_1');
expect(out!.role).toBeUndefined();
expect(out!.content).toBeUndefined();
});
it('upgrades user/system text blocks to input_text', async () => {
const messages: Array<Record<string, unknown>> = [
{
@@ -539,6 +563,39 @@ describe('create_chat_stream_handler_responses_api', () => {
});
});
it('emits a compaction event when a compaction output_item completes', async () => {
const completion = asAsyncIterable([
{
type: 'response.output_item.done',
item: {
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
},
},
{
type: 'response.completed',
response: { usage: { input_tokens: 1, output_tokens: 2 } },
},
]);
const init = create_chat_stream_handler_responses_api({
deviations: undefined,
completion,
usage_calculator: () => ({}),
});
const harness = makeCapturingChatStream();
await init({ chatStream: harness.chatStream });
const compaction = harness
.events()
.find((e: { type: string }) => e.type === 'compaction');
expect(compaction).toEqual({
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
});
});
it('emits a tool_use block when a function_call output_item completes', async () => {
const completion = asAsyncIterable([
{
@@ -765,6 +822,31 @@ describe('handle_completion_output_responses_api non-stream', () => {
).rejects.toMatchObject({ statusCode: 400 });
});
it('attaches a compaction artifact and allows a compaction-only output', async () => {
const completion = {
output: [
{
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
},
],
output_text: ' ',
usage: { input_tokens: 1, output_tokens: 0 },
};
const result = await handle_completion_output_responses_api({
deviations: undefined,
stream: false,
completion,
});
// Compaction-only output is not rejected as "empty".
expect(result.compaction).toEqual({
type: 'compaction',
id: 'cmpct_1',
encrypted_content: 'ENC',
});
});
it('runs moderation against output_text when a moderate fn is supplied', async () => {
const completion = {
output: [{ role: 'assistant' }],
src/backend/drivers/ai-chat/utils/Streaming.js
+19
View File
@@ -103,6 +103,25 @@ export class AIChatStream {
this.stream.end();
}
/**
* Emit a canonical compaction event into the NDJSON stream. Both the OpenAI
* and Anthropic providers normalize their native inline-compaction artifact
* to this single shape, so downstream consumers (controllers, puter.js) see
* an identical `{ type: 'compaction', id, encrypted_content }` chunk
* regardless of which upstream served the request.
*
* @param {{ id?: string, encrypted_content: string }} compaction
*/
compaction({ id, encrypted_content }) {
this.stream.write(
`${JSON.stringify({
type: 'compaction',
...(id !== undefined ? { id } : {}),
encrypted_content,
})}\n`,
);
}
message() {
return new AIChatMessageStream(this);
}
src/backend/drivers/ai-chat/utils/compaction.js
+124
View File
@@ -0,0 +1,124 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
/**
* Inline-compaction translation helpers.
*
* The driver-facing surface exposes a single provider-neutral opt-in
* (`compaction: boolean | { trigger_tokens }` on `ICompleteArguments`), plus a
* raw `context_management` escape hatch for callers hitting `/responses` with
* the OpenAI-native shape. These helpers map that neutral opt-in to each
* provider's SDK shape so the providers stay free of opt-in-parsing logic.
*/
/**
* @param {boolean | { trigger_tokens?: number } | undefined} compaction
* @returns {{ enabled: boolean, trigger_tokens?: number }}
*/
const readCompaction = (compaction) => {
if (compaction === true) return { enabled: true };
if (compaction && typeof compaction === 'object') {
return {
enabled: true,
...(typeof compaction.trigger_tokens === 'number'
? { trigger_tokens: compaction.trigger_tokens }
: {}),
};
}
return { enabled: false };
};
/**
* Build OpenAI Responses `context_management` from the neutral opt-in. A raw
* `context_management` passthrough (already in OpenAI shape) wins.
*
* @param {{ compaction?: boolean | { trigger_tokens?: number }, context_management?: unknown }} args
* @returns {Array<{ type: 'compaction', compact_threshold?: number }> | undefined}
*/
export const toOpenAiContextManagement = (args) => {
if (args.context_management !== undefined) {
return /** @type {any} */ (args.context_management);
}
const { enabled, trigger_tokens } = readCompaction(args.compaction);
if (!enabled) return undefined;
return [
{
type: 'compaction',
...(trigger_tokens !== undefined
? { compact_threshold: trigger_tokens }
: {}),
},
];
};
/**
* Build Anthropic `context_management` (beta `compact-2026-01-12`) from the
* neutral opt-in. A raw `context_management` passthrough wins.
*
* @param {{ compaction?: boolean | { trigger_tokens?: number }, context_management?: unknown }} args
* @returns {{ edits: Array<Record<string, unknown>> } | undefined}
*/
export const toAnthropicContextManagement = (args) => {
if (args.context_management !== undefined) {
return /** @type {any} */ (args.context_management);
}
const { enabled, trigger_tokens } = readCompaction(args.compaction);
if (!enabled) return undefined;
return {
edits: [
{
type: 'compact_20260112',
...(trigger_tokens !== undefined
? {
trigger: {
type: 'input_tokens',
value: trigger_tokens,
},
}
: {}),
},
],
};
};
/**
* Whether the request opted into inline compaction by any route.
*
* @param {{ compaction?: boolean | { trigger_tokens?: number }, context_management?: unknown }} args
*/
export const wantsCompaction = (args) =>
args.context_management !== undefined ||
readCompaction(args.compaction).enabled;
/**
* Whether the (normalized) message list carries a round-tripped compaction
* artifact. Such a request must route through a compaction-capable surface even
* if it didn't request *new* compaction — chat.completions can't represent a
* compaction content block, and Anthropic needs its compaction beta to accept
* one as input.
*
* @param {unknown} messages
*/
export const messagesHaveCompaction = (messages) =>
Array.isArray(messages) &&
messages.some(
(m) =>
Array.isArray(m?.content) &&
m.content.some((c) => c && c.type === 'compaction'),
);
src/backend/drivers/ai-chat/utils/compaction.test.ts
+132
View File
@@ -0,0 +1,132 @@
/*
* Copyright (C) 2024-present Puter Technologies Inc.
*
* This file is part of Puter.
*
* Puter is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published
* by the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import { describe, expect, it } from 'vitest';
import {
messagesHaveCompaction,
toAnthropicContextManagement,
toOpenAiContextManagement,
wantsCompaction,
} from './compaction.js';
describe('toOpenAiContextManagement', () => {
it('returns undefined when compaction is off', () => {
expect(toOpenAiContextManagement({})).toBeUndefined();
expect(
toOpenAiContextManagement({ compaction: false }),
).toBeUndefined();
});
it('builds a compaction entry from `true`', () => {
expect(toOpenAiContextManagement({ compaction: true })).toEqual([
{ type: 'compaction' },
]);
});
it('maps trigger_tokens to compact_threshold', () => {
expect(
toOpenAiContextManagement({ compaction: { trigger_tokens: 1000 } }),
).toEqual([{ type: 'compaction', compact_threshold: 1000 }]);
});
it('passes a raw context_management payload through verbatim', () => {
const raw = [{ type: 'compaction', compact_threshold: 5 }];
expect(
toOpenAiContextManagement({
compaction: true,
context_management: raw,
}),
).toBe(raw);
});
});
describe('toAnthropicContextManagement', () => {
it('returns undefined when compaction is off', () => {
expect(toAnthropicContextManagement({})).toBeUndefined();
});
it('builds a compact_20260112 edit from `true`', () => {
expect(toAnthropicContextManagement({ compaction: true })).toEqual({
edits: [{ type: 'compact_20260112' }],
});
});
it('maps trigger_tokens to an input_tokens trigger', () => {
expect(
toAnthropicContextManagement({
compaction: { trigger_tokens: 2000 },
}),
).toEqual({
edits: [
{
type: 'compact_20260112',
trigger: { type: 'input_tokens', value: 2000 },
},
],
});
});
it('passes a raw context_management payload through verbatim', () => {
const raw = { edits: [{ type: 'compact_20260112' }] };
expect(
toAnthropicContextManagement({ context_management: raw }),
).toBe(raw);
});
});
describe('wantsCompaction', () => {
it('is false without opt-in', () => {
expect(wantsCompaction({})).toBe(false);
expect(wantsCompaction({ compaction: false })).toBe(false);
});
it('is true for the neutral opt-in or a raw passthrough', () => {
expect(wantsCompaction({ compaction: true })).toBe(true);
expect(wantsCompaction({ compaction: { trigger_tokens: 1 } })).toBe(
true,
);
expect(wantsCompaction({ context_management: [] })).toBe(true);
});
});
describe('messagesHaveCompaction', () => {
it('detects a round-tripped compaction content block', () => {
expect(
messagesHaveCompaction([
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
{
role: 'assistant',
content: [
{ type: 'compaction', encrypted_content: 'ENC' },
],
},
]),
).toBe(true);
});
it('is false for ordinary messages or non-arrays', () => {
expect(
messagesHaveCompaction([
{ role: 'user', content: [{ type: 'text', text: 'hi' }] },
]),
).toBe(false);
expect(messagesHaveCompaction(undefined)).toBe(false);
expect(messagesHaveCompaction('nope')).toBe(false);
});
});
src/backend/package.json
+1 -1
View File
@@ -9,7 +9,7 @@
"bench": "vitest bench --config=vitest.bench.config.ts --run"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.68.0",
"@anthropic-ai/sdk": "^0.105.0",
"@aws-sdk/client-dynamodb": "^3.490.0",
"@aws-sdk/client-polly": "^3.1028.0",
"@aws-sdk/client-s3": "^3.1028.0",
src/backend/services/selfhosted/DefaultUserService.ts
+16
View File
@@ -24,6 +24,8 @@ import { PuterService } from '../types.js';
import type { UserRow } from '../../stores/user/UserStore.js';
import { generateDefaultFsentries } from '../../util/userProvisioning.js';
import type { AppIconService } from '../appIcon/AppIconService.js';
import { LOCAL_UNLIMITED_USER } from '../../data/subPolicies/localUnlimitedUserPolicy.js';
import { UNLIMITED_SUBSCRIPTION } from '../metering/consts.js';
const USERNAME = 'admin';
const ADMIN_GROUP_UID = 'ca342a5e-b13d-4dee-9048-58b11a57cc55';
@@ -43,6 +45,20 @@ const ADMIN_STORAGE_BYTES = 10 * 1024 * 1024 * 1024;
*/
export class DefaultUserService extends PuterService {
override async onServerStart(): Promise<void> {
// Dev convenience: grant the bootstrap `admin` user unlimited metering.
// Gated to env === 'dev' so prod deployments never get a free-usage
// actor. Registering the policy makes its `'unlimited'` id resolvable
// (extraPolicies is always in the available set); the resolver returns
// null for everyone else, so all other users keep their normal tier.
if (this.config.env === 'dev') {
this.services.metering.registerPolicy(LOCAL_UNLIMITED_USER);
this.services.metering.registerSubscriptionResolver((actor) =>
actor.user?.username === USERNAME
? UNLIMITED_SUBSCRIPTION
: null,
);
}
if (this.config.no_default_user) return;
let user = await this.stores.user.getByUsername(USERNAME);
let tmpPassword: string;
src/docs/src/AI/chat.md
+100
View File
@@ -34,6 +34,7 @@ An object containing the following properties:
- `tools` (Array) (Optional) - Function definitions the AI can call. See [Function Calling](#function-calling) for details.
- `reasoning_effort` / `reasoning.effort` (String) (Optional) - Controls how much effort reasoning models spend thinking. Supported values: `none`, `minimal`, `low`, `medium`, `high`, and `xhigh`. Lower values give faster responses with less reasoning. OpenAI models only.
- `verbosity` / `text.verbosity` (String) (Optional) - Controls how long or short responses are. Supported values: `low`, `medium`, and `high`. Lower values give shorter responses. OpenAI models only.
- `compaction` (Boolean | Object) (Optional) - Opt into inline context compaction for long conversations. Pass `true` to enable it with provider defaults, or `{ trigger_tokens: number }` to set the token threshold at which earlier context is summarized. When the model compacts, you receive a `compaction` chunk while streaming (or a `compaction` field on the result when not streaming) containing an opaque `encrypted_content` summary. Resend that item in `messages` on the next turn in place of the summarized history. The compaction chunk shape is identical across providers, so the same code works whether `model` is an OpenAI or Anthropic model. See [Compaction](#compaction).
#### `testMode` (Boolean) (Optional)
@@ -183,6 +184,105 @@ Pass in the `cache_control` parameter inside the object in the `messages` array.
You can find the implementation in our [prompt caching example](/playground/ai-claude-cache-control/). Find more details about cache control in [Anthropic documentation](https://platform.claude.com/docs/en/build-with-claude/prompt-caching).
## Compaction
For long, multi-turn conversations that you keep on the client, enable
`compaction` so the model can summarize earlier context before it overflows the
context window. When it fires, the model summarizes the older turns into a single
**compaction artifact** and answers from that summary instead of the full
history — so the request stays small.
You get the artifact back as a `compaction` item (a stream chunk when streaming,
or `result.compaction` when not). On the next turn you resend that one item **in
place of the turns it replaced**, instead of the raw history. The item shape is
the same across providers, so the same code works for OpenAI and Anthropic models.
#### Enabling it
Pass `compaction` in the options object:
- `compaction: true` — enable with provider defaults.
- `compaction: { trigger_tokens: 60000 }` — set the token threshold at which the
model compacts.
The artifact is `{ type: 'compaction', id, encrypted_content }` — a drop-in
`messages` item. `encrypted_content` is an opaque payload; treat it as a black
box and just carry it forward.
#### Streaming
```js
const resp = await puter.ai.chat(messages, {
model: 'gpt-5.4', // or 'claude-opus-4-8' — same code
stream: true,
compaction: { trigger_tokens: 60000 },
});
let text = '';
let compaction = null;
for await ( const part of resp ) {
if ( part.type === 'text' ) text += part.text;
else if ( part.type === 'compaction' ) compaction = part; // { type, id, encrypted_content }
else if ( part.type === 'error' ) console.error('stream error:', part.message);
}
// Next turn: the artifact stands in for the compacted history. Place it where
// that history was (before the new user turn) and keep compaction enabled so it
// can compact again later.
if ( compaction ) {
const next = await puter.ai.chat(
[
{ role: 'system', content: 'You are a helpful assistant.' },
compaction,
{ role: 'user', content: 'now compare the two approaches' },
],
{ model: 'gpt-5.4', stream: true, compaction: true }
);
for await ( const part of next ) {
if ( part.type === 'text' ) document.write(part.text);
}
}
```
#### Non-streaming
```js
const result = await puter.ai.chat(messages, {
model: 'gpt-5.4',
compaction: { trigger_tokens: 60000 },
});
console.log(result.message.content);
if ( result.compaction ) {
// result.compaction is { type: 'compaction', id, encrypted_content } — the
// same drop-in item you get from the stream. Resend it next turn:
const next = await puter.ai.chat(
[
{ role: 'system', content: 'You are a helpful assistant.' },
result.compaction,
{ role: 'user', content: 'now compare the two approaches' },
],
{ model: 'gpt-5.4', compaction: true }
);
console.log(next.message.content);
}
```
#### Notes
- **Keep `compaction` enabled on every turn** of the conversation so it can
compact again as the conversation keeps growing.
- **Place the artifact where the compacted history was** — after your system
prompt, before the new user message.
- **It only fires once the context is large enough.** Anthropic models require a
minimum threshold of **50,000 tokens**, and the conversation must actually
exceed your `trigger_tokens`. OpenAI models don't enforce that floor, so they
can compact smaller conversations. If nothing compacts, your input was below
the threshold.
- **Handle the `error` chunk** when streaming — provider errors (e.g. a
`trigger_tokens` below a provider's minimum) arrive as an `error` chunk, not a
thrown exception.
## Image Generation (Gemini Image Models)
Certain Gemini models can generate and edit images as part of a chat conversation. These models accept text and image inputs, and return text and images in the response.
src/docs/src/Objects/chatresponse.md
+4
View File
@@ -22,3 +22,7 @@ An object containing the chat message data.
- `cache_control` (Object) - An optional object controlling prompt caching for this message. Contains a `type` (String) property.
- `images` (Array) - An array of image content objects associated with the message. Each object contains a `type` (String) and an `image_url` object with a `url` (String) property.
#### `compaction` (Object)
Present only on non-streaming responses where the model compacted earlier context (see [Compaction](/AI/chat#compaction)). A drop-in `messages` item of the form `{ type: 'compaction', id, encrypted_content }` — resend it on the next turn in place of the summarized history. Absent when no compaction occurred.
src/docs/src/Objects/chatresponsechunk.md
+6 -1
View File
@@ -16,6 +16,7 @@ The kind of chunk. One of:
- `"text"` - A portion of the response text.
- `"reasoning"` - A portion of the model's reasoning/thinking output.
- `"tool_use"` - A tool/function the model wants to call.
- `"compaction"` - An inline-compaction summary of earlier context (when `compaction` is enabled).
- `"extra_content"` - Provider-specific metadata.
- `"usage"` - Token usage totals, emitted as the final chunk.
@@ -29,7 +30,11 @@ A portion of the model's reasoning output. Present on `reasoning` chunks.
#### `id` (String)
The unique identifier for the tool call. Present on `tool_use` chunks.
The unique identifier for the tool call (`tool_use` chunks) or the compaction item (`compaction` chunks).
#### `encrypted_content` (String)
The opaque/encrypted compaction summary. Present on `compaction` chunks. The shape is identical across providers — resend this item in `messages` on the next turn in place of the summarized history.
#### `name` (String)
src/puter-js/src/modules/AI.js
+4 -2
View File
@@ -813,8 +813,10 @@ class AI {
requestParams.provider = requestParams.provider || userParams.driver;
}
// Additional parameters to pass from userParams to requestParams
const PARAMS_TO_PASS = ['tools', 'response', 'reasoning', 'reasoning_effort', 'text', 'verbosity', 'provider', 'image_config'];
// Additional parameters to pass from userParams to requestParams.
// `compaction` (provider-neutral inline-compaction opt-in) and the raw
// `context_management` escape hatch flow straight through to the driver.
const PARAMS_TO_PASS = ['tools', 'response', 'reasoning', 'reasoning_effort', 'text', 'verbosity', 'provider', 'image_config', 'compaction', 'context_management'];
for ( const name of PARAMS_TO_PASS ) {
if ( userParams[name] ) {
requestParams[name] = userParams[name];
src/puter-js/test/ai.test.js
+50
View File
@@ -139,6 +139,43 @@ const testChatStreamingCore = async function(model) {
}
};
const testChatCompactionCore = async function(model) {
// Opting into inline compaction must not break a normal streamed response.
// We can't reliably force the model to compact (needs a huge context), so
// this asserts the opt-in streams cleanly and that any compaction chunk
// that does appear carries a provider-independent encrypted_content.
const result = await puter.ai.chat("Count from 1 to 5", {
model: model,
stream: true,
max_tokens: 100,
compaction: true,
});
assert(typeof result === 'object' && result !== null, "compaction chat should return an object");
assert(typeof result[Symbol.asyncIterator] === 'function', "compaction chat should be streamable");
let chunkCount = 0;
let compaction = null;
for await (const chunk of result) {
assert(typeof chunk === 'object', "each streaming chunk should be an object");
if (chunk.type === 'compaction') compaction = chunk;
if (++chunkCount >= 10) break;
}
assert(chunkCount > 0, "compaction streaming should produce at least one chunk");
if (compaction) {
assert(typeof compaction.encrypted_content === 'string',
"a compaction chunk should carry an encrypted_content string");
// Round-trip: the artifact can be resent as a message item.
const next = await puter.ai.chat([
{ role: 'user', content: 'continue' },
compaction,
], { model: model });
assert(typeof next === 'object' && next !== null, "resending a compaction item should succeed");
}
};
// Function to generate test functions for a specific model
const generateTestsForModel = function(model) {
const modelName = model.replace(/[^a-zA-Z0-9]/g, '_'); // Sanitize model name for function names
@@ -195,6 +232,19 @@ const generateTestsForModel = function(model) {
}
}
},
[`testChatCompaction_${modelName}`]: {
name: `testChatCompaction_${modelName}`,
description: `Test AI chat inline compaction opt-in and round-trip using ${model} model`,
test: async function() {
try {
await testChatCompactionCore(model);
pass(`testChatCompaction_${modelName} passed`);
} catch (error) {
fail(`testChatCompaction_${modelName} failed:`, error);
}
}
},
};
};
src/puter-js/types/modules/ai.d.ts
Vendored
+29 -2
View File
@@ -68,6 +68,20 @@ export interface ChatOptions {
* - `image_size`: output quality/resolution; must be one of the model's supported quality levels.
*/
image_config?: { aspect_ratio: string, image_size: string };
/**
* Provider-neutral inline-compaction opt-in for long stateless
* conversations. `true` enables it with provider defaults; an object sets
* the token threshold at which earlier context is summarized. When the
* upstream compacts, you receive a `"compaction"` chunk (streaming) or a
* `compaction` field on the result (non-streaming) — resend it in `messages`
* on the next turn in place of the summarized history.
*/
compaction?: boolean | { trigger_tokens?: number };
/**
* Escape hatch: a provider-native `context_management` payload, passed
* through untouched. Prefer `compaction` for provider portability.
*/
context_management?: unknown;
}
export interface StreamingChatOptions extends ChatOptions {
@@ -77,6 +91,13 @@ export interface StreamingChatOptions extends ChatOptions {
export interface ChatResponse {
message?: ChatMessage;
choices?: unknown;
/**
* Inline-compaction artifact, present when the upstream compacted earlier
* context during this (non-streaming) response. Carries `type:'compaction'`
* so you can push it straight into `messages` on the next turn in place of
* the summarized history (same shape as the streaming `compaction` chunk).
*/
compaction?: { type: 'compaction'; id?: string; encrypted_content: string };
}
/**
@@ -84,18 +105,24 @@ export interface ChatResponse {
* discriminator; which other fields are present depends on that `type`.
*/
export interface ChatResponseChunk {
/** The kind of chunk: `"text"`, `"reasoning"`, `"tool_use"`, `"extra_content"`, or `"usage"`. */
/** The kind of chunk: `"text"`, `"reasoning"`, `"tool_use"`, `"compaction"`, `"extra_content"`, or `"usage"`. */
type: string;
/** Text delta. Present on `"text"` chunks. */
text?: string;
/** Reasoning/thinking delta. Present on `"reasoning"` chunks. */
reasoning?: string;
/** Tool call id. Present on `"tool_use"` chunks. */
/** Tool call id (`"tool_use"`) or compaction item id (`"compaction"`). */
id?: string;
/** Tool/function name. Present on `"tool_use"` chunks. */
name?: string;
/** Parsed tool call arguments. Present on `"tool_use"` chunks. */
input?: unknown;
/**
* Opaque/encrypted compaction summary. Present on `"compaction"` chunks —
* the same shape regardless of which provider served the request. Resend it
* in `messages` on the next turn in place of the summarized history.
*/
encrypted_content?: string;
/** Provider-specific extra metadata. */
extra_content?: unknown;
/** Token usage totals. Present on the final `"usage"` chunk. */