diff --git a/src/backend/controllers/puterai/PuterAIController.shapes.test.ts b/src/backend/controllers/puterai/PuterAIController.shapes.test.ts index 8050e2c7f..54ecb5580 100644 --- a/src/backend/controllers/puterai/PuterAIController.shapes.test.ts +++ b/src/backend/controllers/puterai/PuterAIController.shapes.test.ts @@ -1478,7 +1478,7 @@ describe('PuterAIController model listing edges', () => { vi.spyOn(server.drivers.aiVideo, 'models').mockResolvedValueOnce([ { id: 'veo-test' }, { id: 'fake' }, - { id: 'model-fallback-test-1' }, + { id: 'costly' }, ] as never); const { res, captured } = makeRes(); diff --git a/src/backend/controllers/puterai/PuterAIController.test.ts b/src/backend/controllers/puterai/PuterAIController.test.ts index d87445416..a674b4c85 100644 --- a/src/backend/controllers/puterai/PuterAIController.test.ts +++ b/src/backend/controllers/puterai/PuterAIController.test.ts @@ -360,7 +360,8 @@ describe('PuterAIController.openaiChatCompletions', () => { res, ); - // Driver was given the user's messages and the default chat provider. + // Driver was given the user's messages and no provider pin, so it is + // free to pick the preferred healthy route for the model. expect(completeSpy).toHaveBeenCalledTimes(1); const completeArgs = completeSpy.mock.calls[0]![0]; expect(completeArgs.model).toBe('gpt-test'); @@ -368,7 +369,7 @@ describe('PuterAIController.openaiChatCompletions', () => { { role: 'user', content: 'hi' }, ]); expect(completeArgs.stream).toBe(false); - expect(completeArgs.provider).toBe('openai-completion'); + expect(completeArgs.provider).toBeUndefined(); // Wire routes translate shapes themselves; the driver is pinned // provider-native so the release-date cutoff can't change them. expect(completeArgs.normalize).toBe(false); @@ -395,6 +396,53 @@ describe('PuterAIController.openaiChatCompletions', () => { expect((body.id as string).startsWith('chatcmpl-')).toBe(true); }); + it('forwards an explicit provider as a pin instead of choosing one itself', async () => { + const completeSpy = stubChatComplete({ + message: { role: 'assistant', content: 'hi there' }, + finish_reason: 'stop', + usage: { prompt_tokens: 4, completion_tokens: 2 }, + }); + + const { res } = makeRes(); + await controller.openaiChatCompletions( + makeReq({ + body: { + model: 'gpt-test', + provider: 'openai-completion', + messages: [{ role: 'user', content: 'hi' }], + }, + actor: makeUserActor(), + }), + res, + ); + + const completeArgs = completeSpy.mock.calls[0]![0]; + expect(completeArgs.provider).toBe('openai-completion'); + }); + + it('still pins OpenAI when the caller names no model, so the default model does not move', async () => { + const completeSpy = stubChatComplete({ + message: { role: 'assistant', content: 'hi there' }, + finish_reason: 'stop', + usage: { prompt_tokens: 4, completion_tokens: 2 }, + }); + + const { res } = makeRes(); + await controller.openaiChatCompletions( + makeReq({ + body: { messages: [{ role: 'user', content: 'hi' }] }, + actor: makeUserActor(), + }), + res, + ); + + // With no model there is no route to prefer — unpinned, this would + // take its default model from whichever provider the driver picks. + expect(completeSpy.mock.calls[0]![0].provider).toBe( + 'openai-completion', + ); + }); + it('streams chat completion deltas as SSE chunks ending with [DONE]', async () => { stubChatComplete({ // The controller's expectStream() checks the DriverStreamResult @@ -548,6 +596,59 @@ describe('PuterAIController.openaiCompletions', () => { }); expect((body.id as string).startsWith('cmpl-')).toBe(true); }); + + it('leaves provider unset for a named model so the driver can route it', async () => { + const completeSpy = stubChatComplete({ + message: { role: 'assistant', content: 'response' }, + finish_reason: 'stop', + }); + const { res } = makeRes(); + await controller.openaiCompletions( + makeReq({ + body: { model: 'gpt-test', prompt: 'hello' }, + actor: makeUserActor(), + }), + res, + ); + expect(completeSpy.mock.calls[0]![0].provider).toBeUndefined(); + }); + + it('forwards an explicit provider as a pin', async () => { + const completeSpy = stubChatComplete({ + message: { role: 'assistant', content: 'response' }, + finish_reason: 'stop', + }); + const { res } = makeRes(); + await controller.openaiCompletions( + makeReq({ + body: { + model: 'gpt-test', + provider: 'openai-completion', + prompt: 'hello', + }, + actor: makeUserActor(), + }), + res, + ); + expect(completeSpy.mock.calls[0]![0].provider).toBe( + 'openai-completion', + ); + }); + + it('still pins OpenAI when the caller names no model', async () => { + const completeSpy = stubChatComplete({ + message: { role: 'assistant', content: 'response' }, + finish_reason: 'stop', + }); + const { res } = makeRes(); + await controller.openaiCompletions( + makeReq({ body: { prompt: 'hello' }, actor: makeUserActor() }), + res, + ); + expect(completeSpy.mock.calls[0]![0].provider).toBe( + 'openai-completion', + ); + }); }); // ── /openai/v1/responses ──────────────────────────────────────────── diff --git a/src/backend/controllers/puterai/PuterAIController.ts b/src/backend/controllers/puterai/PuterAIController.ts index da0e4b346..5ae8e0c41 100644 --- a/src/backend/controllers/puterai/PuterAIController.ts +++ b/src/backend/controllers/puterai/PuterAIController.ts @@ -286,9 +286,8 @@ export class PuterAIController extends PuterController { legacyCode: 'internal_error', }); const models = await driver.list(); - const HIDDEN = ['costly', 'fake', 'abuse', 'model-fallback-test-1']; res.json({ - models: models?.filter((m) => !HIDDEN.includes(m)), + models: models?.filter((m) => !HIDDEN_MODELS.includes(m)), }); }; } @@ -301,9 +300,8 @@ export class PuterAIController extends PuterController { legacyCode: 'internal_error', }); const models = await driver.models(); - const HIDDEN = ['costly', 'fake', 'abuse', 'model-fallback-test-1']; res.json({ - models: models?.filter((m) => !HIDDEN.includes(m.id)), + models: models?.filter((m) => !HIDDEN_MODELS.includes(m.id)), }); }; } @@ -341,9 +339,7 @@ export class PuterAIController extends PuterController { ? { temperature: Number(body.temperature) } : {}), ...(finiteMaxTokens(body.max_tokens) ?? {}), - ...(body.provider - ? { provider: toStringOrEmpty(body.provider) } - : { provider: DEFAULTS.openaiChat }), + ...openaiCompatProvider(body), }; const result = await this.#driver().complete(completeArgs); @@ -499,9 +495,7 @@ export class PuterAIController extends PuterController { ? { temperature: Number(body.temperature) } : {}), ...(finiteMaxTokens(body.max_tokens) ?? {}), - ...(body.provider - ? { provider: toStringOrEmpty(body.provider) } - : { provider: DEFAULTS.openaiCompletion }), + ...openaiCompatProvider(body), }; const completionId = `cmpl-${randomId()}`; @@ -589,8 +583,7 @@ export class PuterAIController extends PuterController { text: extractTextContent( ( messageResult.message as - | Record - | undefined + Record | undefined )?.content, ), index: 0, @@ -612,6 +605,9 @@ export class PuterAIController extends PuterController { const body = asRecord(req.body); const stream = !!body.stream; + // Pinned, unlike the chat/completions routes: the translators below + // read one provider's native Responses shape, so the preferred-route + // and unhealthy-route logic cannot be allowed to swap it out. const providerName = toStringOrEmpty(body.provider) || DEFAULTS.openaiResponses; if (providerName !== DEFAULTS.openaiResponses) { @@ -991,6 +987,9 @@ export class PuterAIController extends PuterController { body.compaction as ICompleteArguments['compaction'], } : {}), + // Pinned for the same reason as /openai/v1/responses: this route + // translates Anthropic's native shape and cannot take whatever + // the preferred healthy route happens to return. ...(body.provider ? { provider: toStringOrEmpty(body.provider) } : { provider: DEFAULTS.anthropic }), @@ -1201,11 +1200,28 @@ export class PuterAIController extends PuterController { const DEFAULTS = { openaiChat: 'openai-completion', - openaiCompletion: 'openai-completion', openaiResponses: 'openai-responses', anthropic: 'claude', } as const; +/** Test-only chat models, kept out of the public listings. */ +const HIDDEN_MODELS = ['costly', 'fake', 'abuse']; + +/** + * How the OpenAI-compat routes pin a provider. An explicit one is the caller's + * choice; a named model is left unpinned so the driver picks the preferred + * healthy route, as it does for puter.js callers. A request with no model has + * no route to prefer, so it stays pinned and keeps taking its default model + * from OpenAI rather than silently switching to another vendor's. + */ +const openaiCompatProvider = ( + body: Record, +): { provider?: string } => { + if (body.provider) return { provider: toStringOrEmpty(body.provider) }; + if (toStringOrEmpty(body.model)) return {}; + return { provider: DEFAULTS.openaiChat }; +}; + const randomId = (): string => crypto.randomUUID().replace(/-/g, ''); const generateId = (prefix: string): string => `${prefix}_${randomId()}`; diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts index 4d23b785f..473ae64b5 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts @@ -47,6 +47,7 @@ import { withTestActor } from '../integrationTestUtil.js'; import { ChatCompletionDriver } from './ChatCompletionDriver.js'; import { clearUnhealthyRoutes, + isRouteUnhealthy, markRouteUnhealthy, } from './utils/providerHealth.js'; @@ -418,6 +419,31 @@ describe('ChatCompletionDriver unhealthy-route skipping', () => { expect(attempts[0]).toMatchObject({ provider: 'infron' }); }); + + it('marks a route only for failures that indict the route, not the request', async () => { + const complete = () => + withTestActor(() => + driver.complete({ + model: 'deepseek-v4-pro', + messages: [{ role: 'user', content: 'hi' }], + }), + ).catch(() => undefined); + + // A 400 is the upstream judging this prompt; the next caller's may + // be fine, so the route stays in rotation. + createMock.mockRejectedValue( + Object.assign(new Error('invalid request'), { status: 400 }), + ); + await complete(); + expect(isRouteUnhealthy('deepseek', 'deepseek-v4-pro')).toBe(false); + + // A 503 says the route itself is down and is remembered. + createMock.mockRejectedValue( + Object.assign(new Error('service unavailable'), { status: 503 }), + ); + await complete(); + expect(isRouteUnhealthy('deepseek', 'deepseek-v4-pro')).toBe(true); + }); }); // A transport timeout carries no status, so the classifier used to lump a diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.test.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.test.ts index da58e631e..a966c6b8c 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.test.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.test.ts @@ -202,9 +202,9 @@ describe('ChatCompletionDriver.complete auth and model resolution', () => { ).rejects.toMatchObject({ statusCode: 400 }); }); - it('falls back to the provider default model when neither model nor provider is given (claude is the hard-coded default provider)', async () => { - // Without `claude` in providers config, the driver tries - // `claude` as the default provider but it isn't registered, so + it('falls back to the provider default model when neither model nor provider is given (azure-openai is the hard-coded default provider)', async () => { + // Without `azure-openai` in providers config, the driver tries + // `azure-openai` as the default provider but it isn't registered, so // `args.model` stays undefined and `#resolveModel` returns null // — surfaces as 400. await expect( diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts index af001868f..fdc7a2793 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -1114,10 +1114,9 @@ export class ChatCompletionDriver extends PuterDriver { ); } - // Azure AI Foundry (OpenAI + xAI Grok). Registered before the regular - // OpenAI/xAI providers so that since its costs mirror theirs but - // Azure is preferred for us, it takes precedence in the per-model - // bucket + // Azure AI Foundry (OpenAI + xAI Grok). Its costs mirror the vendors' + // and it is preferred for us; `PREFERRED_PROVIDERS` in modelRouting + // puts it ahead of them in the per-model bucket. const azureOpenai = providers['azure-openai']; const azureOpenaiKey = readKey(azureOpenai); const azureOpenaiURL = azureOpenai?.apiURL as string | undefined; diff --git a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts index 73c6f233a..3fe829ad6 100644 --- a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.test.ts @@ -44,24 +44,35 @@ import { type MockInstance, } from 'vitest'; +import { v4 as uuidv4 } from 'uuid'; + +import type { Actor } from '../../../../core/actor.js'; import { SYSTEM_ACTOR } from '../../../../core/actor.js'; import type { MeteringService } from '../../../../services/metering/MeteringService.js'; import { PuterServer } from '../../../../server.js'; import { setupTestServer } from '../../../../testUtil.js'; +import { generateDefaultFsentries } from '../../../../util/userProvisioning.js'; import { withTestActor } from '../../../integrationTestUtil.js'; import { AIChatStream } from '../../utils/Streaming.js'; +import { FILES_API_BETA } from './fileUpload.js'; import { CLAUDE_MODELS } from './models.js'; import { ClaudeProvider } from './ClaudeProvider.js'; // ── Anthropic SDK mock ────────────────────────────────────────────── -const { messagesCreateMock, messagesStreamMock, anthropicCtor } = vi.hoisted( - () => ({ - messagesCreateMock: vi.fn(), - messagesStreamMock: vi.fn(), - anthropicCtor: vi.fn(), - }), -); +const { + messagesCreateMock, + messagesStreamMock, + anthropicCtor, + filesUploadMock, + filesDeleteMock, +} = vi.hoisted(() => ({ + messagesCreateMock: vi.fn(), + messagesStreamMock: vi.fn(), + anthropicCtor: vi.fn(), + filesUploadMock: vi.fn(), + filesDeleteMock: vi.fn(), +})); vi.mock('@anthropic-ai/sdk', () => { const Anthropic = vi.fn().mockImplementation(function ( @@ -76,14 +87,20 @@ vi.mock('@anthropic-ai/sdk', () => { // Beta files surface — only consulted when puter_path uploads run, so // tests that exercise text-only paths never hit these stubs. this.beta = { - files: { delete: vi.fn() }, + files: { upload: filesUploadMock, delete: filesDeleteMock }, messages: { create: messagesCreateMock, stream: messagesStreamMock, }, }; }); - return { default: Anthropic }; + return { + default: Anthropic, + toFile: async (data: unknown, filename: string) => ({ + data, + filename, + }), + }; }); // ── Test harness ──────────────────────────────────────────────────── @@ -122,11 +139,13 @@ const asAsyncIterable = (items: T[]): AsyncIterable => ({ const makeStreamLike = (events: unknown[], finalUsage?: unknown) => { // Anthropic's `messages.stream(...)` returns an object that is itself - // both an async iterable (the events) AND has a `.finalMessage()` - // promise. The provider awaits both. + // an async iterable (the events), has a `.withResponse()` promise that + // settles once the upstream accepts the request, AND has a + // `.finalMessage()` promise. The provider awaits all three. const iter = asAsyncIterable(events); return { [Symbol.asyncIterator]: iter[Symbol.asyncIterator].bind(iter), + withResponse: () => Promise.resolve({}), finalMessage: () => Promise.resolve({ usage: finalUsage ?? { input_tokens: 0, output_tokens: 0 }, @@ -134,6 +153,47 @@ const makeStreamLike = (events: unknown[], finalUsage?: unknown) => { }; }; +/** + * A user with one real FS entry, for the `puter_path` upload branch. Only the + * Files API calls are stubbed; the read goes through the wired FSService. + */ +const makeUserWithFile = async () => { + const username = `clsp-${Math.random().toString(36).slice(2, 10)}`; + const created = await server.stores.user.create({ + username, + uuid: uuidv4(), + password: null, + email: `${username}@test.local`, + free_storage: 100 * 1024 * 1024, + requires_email_confirmation: false, + }); + await generateDefaultFsentries( + server.clients.db, + server.stores.user, + created, + ); + const user = (await server.stores.user.getById(created.id))!; + const actor = { + user: { + id: user.id, + uuid: user.uuid, + username: user.username, + email: user.email ?? null, + email_confirmed: true, + } as Actor['user'], + }; + const path = `/${username}/Documents/pic.png`; + await withTestActor( + () => + server.services.fs.write(user.id, { + fileMetadata: { path, size: 4, contentType: 'image/png' }, + fileContent: Buffer.from([0x89, 0x50, 0x4e, 0x47]), + }), + actor, + ); + return { actor, path }; +}; + const makeCapturingChatStream = () => { const chunks: string[] = []; const sink = new Writable({ @@ -158,6 +218,8 @@ beforeEach(() => { messagesCreateMock.mockReset(); messagesStreamMock.mockReset(); anthropicCtor.mockReset(); + filesUploadMock.mockReset(); + filesDeleteMock.mockReset(); recordSpy = vi.spyOn(server.services.metering, 'utilRecordUsageObject'); }); @@ -881,6 +943,111 @@ describe('ClaudeProvider.complete non-stream output', () => { // ── Streaming deltas ──────────────────────────────────────────────── describe('ClaudeProvider.complete streaming', () => { + it('rejects from complete() when the upstream refuses the stream, so the driver can fall back', async () => { + const { provider } = makeProvider(); + const refused = Object.assign(new Error('Overloaded'), { + status: 529, + }); + messagesStreamMock.mockReturnValueOnce({ + ...makeStreamLike([]), + withResponse: () => Promise.reject(refused), + }); + + // Thrown here, before a populator exists — a populator that failed + // later would already have a 200 on the wire. + await expect( + withTestActor(() => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [{ role: 'user', content: 'say hi' }], + stream: true, + }), + ), + ).rejects.toBe(refused); + }); + + it('deletes the uploaded files and hands back the puter_path when the stream is refused', async () => { + const { provider } = makeProvider(); + const { actor, path } = await makeUserWithFile(); + filesUploadMock.mockResolvedValue({ id: 'file_stream_1' }); + const refused = Object.assign(new Error('Overloaded'), { + status: 529, + }); + messagesStreamMock.mockReturnValueOnce({ + ...makeStreamLike([]), + withResponse: () => Promise.reject(refused), + }); + + const part: Record = { puter_path: path }; + await expect( + withTestActor( + () => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [{ role: 'user', content: [part] }], + stream: true, + }), + actor, + ), + ).rejects.toBe(refused); + + expect(filesUploadMock).toHaveBeenCalledTimes(1); + expect(filesDeleteMock).toHaveBeenCalledWith('file_stream_1', { + betas: [FILES_API_BETA], + }); + // The driver reuses this object on the fallback route, which has no + // way to resolve a file we just deleted from our own account. + expect(part).toEqual({ puter_path: path }); + }); + + it('surfaces a failure the event iterator swallowed instead of ending the stream clean', async () => { + const { provider } = makeProvider(); + const dropped = Object.assign(new Error('Overloaded'), { + status: 529, + }); + // The SDK hands an error only to a reader already waiting on it; one + // that lands earlier leaves the iterator reporting a plain end of + // stream, so `errored` is the only thing left to go on. + messagesStreamMock.mockReturnValueOnce({ + ...makeStreamLike([ + { type: 'message_start' }, + { + type: 'content_block_start', + content_block: { type: 'text' }, + }, + { + type: 'content_block_delta', + delta: { type: 'text_delta', text: 'half an ans' }, + }, + ]), + errored: true, + finalMessage: () => Promise.reject(dropped), + }); + + const result = await withTestActor(() => + provider.complete({ + model: 'claude-haiku-4-5-20251001', + messages: [{ role: 'user', content: 'say hi' }], + stream: true, + }), + ); + + const harness = makeCapturingChatStream(); + await expect( + ( + result as { + init_chat_stream: (p: { + chatStream: unknown; + }) => Promise; + } + ).init_chat_stream({ chatStream: harness.chatStream }), + ).rejects.toBe(dropped); + + // A truncated response reported as a success would also have been + // billed for the tokens it did produce. + expect(recordSpy).not.toHaveBeenCalled(); + }); + it('streams text_delta events as text and meters usage from message_delta + finalMessage', async () => { const { provider } = makeProvider(); messagesStreamMock.mockReturnValueOnce( diff --git a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts index 819281e93..5e3555ae4 100644 --- a/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts +++ b/src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts @@ -329,13 +329,14 @@ export class ClaudeProvider implements IChatProvider { // Upload any `puter_path` parts to Anthropic's Files API and rewrite // them in-place to reference the returned `file_id`. Must happen // before sdkParams snapshots `messages`. - const { fileIds: uploadedFileIds } = await processPuterPathUploads( - this.anthropic, - messages, - this.#stores, - this.#fsService, - actor, - ); + const { fileIds: uploadedFileIds, restore: restoreUploads } = + await processPuterPathUploads( + this.anthropic, + messages, + this.#stores, + this.#fsService, + actor, + ); const usesBetaFiles = uploadedFileIds.length > 0; // The compaction beta is needed both to *request* compaction // (contextManagement) and to *accept a round-tripped* compaction block @@ -403,14 +404,28 @@ export class ClaudeProvider implements IChatProvider { }; if (stream) { + const completion = usesBeta + ? this.anthropic.beta.messages.stream(sdkParams) + : this.anthropic.messages.stream(sdkParams); + // Subscribed before the request is awaited: the SDK only queues + // events for iterators that already exist. + const events = completion[Symbol.asyncIterator](); + + // The driver's fallback loop only sees what this method throws, so + // the upstream has to accept the request before a populator exists. + try { + await completion.withResponse(); + } catch (e) { + await cleanupUploads(); + restoreUploads(); + throw e; + } + const init_chat_stream = async ({ chatStream, }: { chatStream: AIChatStream; }) => { - const completion = usesBeta - ? this.anthropic.beta.messages.stream(sdkParams) - : this.anthropic.messages.stream(sdkParams); const usageSum: Record = {}; let message, contentBlock; @@ -425,7 +440,9 @@ export class ClaudeProvider implements IChatProvider { buffer: string; } | null = null; let emittedCompaction = false; - for await (const event of completion) { + for await (const event of { + [Symbol.asyncIterator]: () => events, + }) { if (event.type === 'message_delta') { const meteredData = this.#usageFormatterUtil( (event?.usage ?? {}) as Usage | BetaUsage, @@ -542,6 +559,11 @@ export class ClaudeProvider implements IChatProvider { // signature_delta — ignored } } + // The SDK only rejects event readers that were already + // waiting, so a failure that landed before this loop started + // pulling ends it silently rather than throwing. + if (completion.errored) await completion.finalMessage(); + const finalMessage = await completion .finalMessage() .catch((): null => null); @@ -645,6 +667,9 @@ export class ClaudeProvider implements IChatProvider { } : {}), }; + } catch (e) { + restoreUploads(); + throw e; } finally { await cleanupUploads(); } diff --git a/src/backend/drivers/ai-chat/providers/claude/fileUpload.test.ts b/src/backend/drivers/ai-chat/providers/claude/fileUpload.test.ts index c8a7908ba..d09cbe357 100644 --- a/src/backend/drivers/ai-chat/providers/claude/fileUpload.test.ts +++ b/src/backend/drivers/ai-chat/providers/claude/fileUpload.test.ts @@ -130,7 +130,7 @@ describe('claude processPuterPathUploads uploading', () => { ); expect(upload).not.toHaveBeenCalled(); - expect(result).toEqual({ fileIds: [] }); + expect(result.fileIds).toEqual([]); }); it('uploads an image and rewrites the part to an image block referencing the file_id', async () => { @@ -232,6 +232,56 @@ describe('claude processPuterPathUploads uploading', () => { // -- Rejection paths ------------------------------------------------- +describe('claude processPuterPathUploads restoring', () => { + it('puts a rewritten part back to the puter_path it arrived as', async () => { + const { actor, userId } = await makeUser(); + const { client } = makeAnthropicStub(); + const username = actor.user!.username!; + const path = `/${username}/Documents/pic.png`; + await writeFile( + actor, + userId, + path, + Buffer.from([0x89, 0x50, 0x4e, 0x47]), + 'image/png', + ); + + const part: Record = { puter_path: path }; + const result = await processPuterPathUploads( + client, + [{ content: [part] }], + server.stores, + server.services.fs, + actor, + ); + expect(part.source).toEqual({ type: 'file', file_id: 'file_1' }); + + // A failed request deletes the uploads, and the driver hands these + // same objects to the next route — which can only use the path. + result.restore(); + expect(part).toEqual({ puter_path: path }); + }); + + it('undoes the inline text error swapped in for a part that could not load', async () => { + const { actor } = await makeUser(); + const { client } = makeAnthropicStub(); + const path = `/${actor.user!.username!}/Documents/missing.png`; + + const part: Record = { puter_path: path }; + const result = await processPuterPathUploads( + client, + [{ content: [part] }], + server.stores, + server.services.fs, + actor, + ); + expect(part.type).toBe('text'); + + result.restore(); + expect(part).toEqual({ puter_path: path }); + }); +}); + describe('claude processPuterPathUploads rejection paths', () => { it('replaces the part with an inline error when the caller is unauthenticated', async () => { const { client, upload } = makeAnthropicStub(); diff --git a/src/backend/drivers/ai-chat/providers/claude/fileUpload.ts b/src/backend/drivers/ai-chat/providers/claude/fileUpload.ts index 3ac16799e..bdaac2fc5 100644 --- a/src/backend/drivers/ai-chat/providers/claude/fileUpload.ts +++ b/src/backend/drivers/ai-chat/providers/claude/fileUpload.ts @@ -39,6 +39,12 @@ interface ContentPart { export interface ClaudeUploadResult { /** File IDs uploaded this request; caller deletes them after completion. */ fileIds: string[]; + /** + * Puts every rewritten part back to the `puter_path` it arrived as. Call it + * when the request fails: the driver hands the same message objects to the + * next fallback route, which cannot resolve our `file_id`s. + */ + restore: () => void; } /** @@ -66,15 +72,29 @@ export async function processPuterPathUploads( if (part?.puter_path) parts.push(part); } } - if (parts.length === 0) return { fileIds: [] }; + if (parts.length === 0) return { fileIds: [], restore: () => {} }; const fileIds: string[] = []; + const restores: Array<() => void> = []; await Promise.all( parts.map((part) => - processPart(part, anthropic, stores, fsService, actor, fileIds), + processPart( + part, + anthropic, + stores, + fsService, + actor, + fileIds, + restores, + ), ), ); - return { fileIds }; + return { + fileIds, + restore: () => { + for (const undo of restores) undo(); + }, + }; } async function processPart( @@ -84,8 +104,19 @@ async function processPart( fsService: FSService, actor: Actor | undefined, fileIds: string[], + restores: Array<() => void>, ): Promise { const path = part.puter_path!; + // Only the path needs to come back: `type`/`source` on success and + // `type`/`text` on failure are the sole fields written below, and the + // next provider's uploader keys off `puter_path` alone. Undoing those by + // name rather than snapshotting the part keeps nothing else alive. + restores.push(() => { + delete part.type; + delete part.text; + delete part.source; + part.puter_path = path; + }); delete part.puter_path; if (!actor?.user?.id) { diff --git a/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.test.ts b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.test.ts index ac62960cc..5c3aa10ac 100644 --- a/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.test.ts @@ -82,8 +82,7 @@ let recordSpy: MockInstance; const KV_KEY = 'togetherai:models'; // Together's `models.list()` returns API-shaped rows; the provider -// coerces them to IChatModel and prepends a synthetic -// `model-fallback-test-1` row at the end. Costs (per million): +// coerces them to IChatModel. Costs (per million): // Llama-3.1-8B: input=18, output=18; Qwen-7B: input=20, output=20. const SAMPLE_API_MODELS = [ { @@ -208,8 +207,6 @@ describe('TogetherAIProvider model catalog', () => { 'togetherai/meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo', ); expect(ids).toContain('Meta-Llama-3.1-8B-Instruct-Turbo'); - // The synthetic fallback-test model is appended. - expect(ids).toContain('model-fallback-test-1'); }); it('reserves headroom under the context length for the output cap', async () => { @@ -357,22 +354,6 @@ describe('TogetherAIProvider.complete request shape', () => { }); }); - it('throws synthetic model-fallback-test-1 BEFORE hitting the SDK', async () => { - const { provider } = makeProvider(); - - await expect( - withTestActor(() => - provider.complete({ - model: 'model-fallback-test-1', - messages: [{ role: 'user', content: 'hi' }], - }), - ), - ).rejects.toThrow(/Model Fallback Test 1/); - - expect(createMock).not.toHaveBeenCalled(); - expect(recordSpy).not.toHaveBeenCalled(); - }); - // Together's APIError carries the whole response body on `.error`, so the // provider message sits at `.error.error.message` — one level deeper than // the OpenAI SDK puts it. diff --git a/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts index 968c13eff..19b06ab93 100644 --- a/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts +++ b/src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts @@ -102,20 +102,6 @@ export class TogetherAIProvider implements IChatProvider { } } - models.push({ - id: 'model-fallback-test-1', - name: 'Model Fallback Test 1', - context: 1000, - costs_currency: 'usd-cents', - input_cost_key: 'input', - output_cost_key: 'output', - costs: { - tokens: 1_000_000, - prompt_tokens: 10, - completion_tokens: 10, - }, - max_tokens: 1000, - }); kv.set(this.#kvKey, models, { EX: 15 * 60 }); return models; } @@ -132,10 +118,6 @@ export class TogetherAIProvider implements IChatProvider { max_tokens, temperature, }: ICompleteArguments): ReturnType { - if (model === 'model-fallback-test-1') { - throw new Error('Model Fallback Test 1'); - } - const actor = Context.get('actor'); const models = await this.models(); const modelLower = model.toLowerCase(); diff --git a/src/backend/drivers/ai-chat/utils/modelRouting.test.ts b/src/backend/drivers/ai-chat/utils/modelRouting.test.ts index abc36f431..a725b60dc 100644 --- a/src/backend/drivers/ai-chat/utils/modelRouting.test.ts +++ b/src/backend/drivers/ai-chat/utils/modelRouting.test.ts @@ -133,12 +133,44 @@ describe('compareModelPreference', () => { const long = { ...geminiModel('gemini-2.5-flash'), id: 'some-vendor/gemini-x-2025-preview', - provider: 'azure-openai', + provider: 'xai', }; expect(winner(long, short).id).toBe('gemini-x'); }); + it('serves Azure ahead of the vendor it fronts, whichever registered first', () => { + // Azure's catalog copies the vendor's prices, so cost cannot decide + // and the winner used to be whichever provider registered first. + const azure = { + ...geminiModel('gemini-2.5-flash'), + id: 'gpt-x', + provider: 'azure-openai', + }; + const vendor = { ...azure, provider: 'openai-completion' }; + + expect(winner(vendor, azure).provider).toBe('azure-openai'); + expect(winner(azure, vendor).provider).toBe('azure-openai'); + }); + + it('keeps Azure first even when its copied cost table drifts higher', () => { + const azure = { + ...geminiModel('gemini-2.5-pro'), + id: 'gpt-x', + provider: 'azure-openai', + }; + const vendor = { + ...geminiModel('gemini-2.5-flash-lite'), + id: 'gpt-x', + provider: 'openai-completion', + }; + expect(azure.costs.prompt_tokens).toBeGreaterThan( + vendor.costs.prompt_tokens as number, + ); + + expect(winner(vendor, azure).provider).toBe('azure-openai'); + }); + it('leaves a reseller serving models no vendor provider carries', () => { // The image-preview models are absent from GEMINI_MODELS, so the // gateway is the only route and must stay the winner. diff --git a/src/backend/drivers/ai-chat/utils/modelRouting.ts b/src/backend/drivers/ai-chat/utils/modelRouting.ts index 956f5de5e..339216525 100644 --- a/src/backend/drivers/ai-chat/utils/modelRouting.ts +++ b/src/backend/drivers/ai-chat/utils/modelRouting.ts @@ -33,13 +33,25 @@ export const AGGREGATOR_PROVIDERS = new Set([ 'hoonify', ]); -// Lower rank is served first. `openrouter` and `together-ai` sit at the very -// bottom, in that order, behind the other resellers. +/** + * Direct providers served ahead of every other direct provider, at any price. + * Azure AI Foundry fronts OpenAI and xAI models at mirrored prices, so naming + * it here keeps it first even if the two cost tables drift. The rank is + * unconditional, not scoped to the vendors it fronts: any other provider that + * ever publishes one of these model ids loses the bucket however cheaply it + * quotes. + */ +const PREFERRED_PROVIDERS = new Set(['azure-openai', 'azure-openai-responses']); + +// Lower rank is served first: preferred direct providers, then the other +// direct vendors, then the resellers with `openrouter` and `together-ai` at +// the very bottom, in that order. const providerRank = (provider?: string): number => { - if (provider === 'together-ai') return 3; - if (provider === 'openrouter') return 2; - if (provider && AGGREGATOR_PROVIDERS.has(provider)) return 1; - return 0; + if (provider === 'together-ai') return 4; + if (provider === 'openrouter') return 3; + if (provider && AGGREGATOR_PROVIDERS.has(provider)) return 2; + if (provider && PREFERRED_PROVIDERS.has(provider)) return 0; + return 1; }; /** @@ -85,11 +97,12 @@ export const isIdentityKey = (key: string): boolean => /** * Orders the candidates that share a model bucket; the first one gets served. * - * Direct vendors outrank resellers regardless of quoted price. Resellers - * advertise a floor price across their upstream routes, so on price alone they - * undercut the vendor's list price and capture traffic for models we hold a - * direct integration for. Within a rank, cheapest input cost wins and ties - * break by shorter id — usually the official name over a qualified one. + * Preferred direct providers outrank the other direct vendors, and direct + * vendors outrank resellers, regardless of quoted price. Resellers advertise a + * floor price across their upstream routes, so on price alone they undercut the + * vendor's list price and capture traffic for models we hold a direct + * integration for. Within a rank, cheapest input cost wins and ties break by + * shorter id — usually the official name over a qualified one. */ export const compareModelPreference = ( a: IChatModel,