From f2445fdd2d32734ded5d1e0a2707f2cde35fbae3 Mon Sep 17 00:00:00 2001 From: 404oops Date: Wed, 14 Oct 2026 14:47:38 +0200 Subject: [PATCH] Handle model access denials and filter batch chat variants --- ...CompletionDriver.availability.http.test.ts | 199 ++++++++++ .../ChatCompletionDriver.edges.test.ts | 74 ++++ .../ChatCompletionDriver.routing.test.ts | 367 ++++++++++++++++++ .../drivers/ai-chat/ChatCompletionDriver.ts | 34 +- .../openrouter/OpenRouterProvider.test.ts | 149 +++++++ .../openrouter/OpenRouterProvider.ts | 41 +- .../drivers/ai-chat/utils/OpenAIUtil.js | 2 +- 7 files changed, 851 insertions(+), 15 deletions(-) create mode 100644 src/backend/drivers/ai-chat/ChatCompletionDriver.availability.http.test.ts diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.availability.http.test.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.availability.http.test.ts new file mode 100644 index 000000000..08ae37540 --- /dev/null +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.availability.http.test.ts @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { createServer, type Server } from 'node:http'; +import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest'; +import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js'; +import { clearUnhealthyRoutes } from './utils/providerHealth.js'; +import { makeActor } from '../../core/actor.js'; + +const policyMessage = + 'Policy Violation: this user has been blocked for a previous policy violation. Learn more: https://vendor.test/private'; +const routes = [ + '/drivers/call', + '/puterai/openai/v1/chat/completions', + '/puterai/anthropic/v1/messages', +]; +const cases = [ + { + name: 'policy block', + status: 403, + body: { error: { code: 403, message: policyMessage } }, + expected: [400, 404, 404], + }, + { + name: 'in-band policy block', + status: 200, + body: { error: { code: 403, message: policyMessage } }, + expected: [400, 404, 404], + }, + { + name: 'bad credentials', + status: 401, + body: { error: { code: 401, message: 'Invalid API key' } }, + expected: [500, 500, 500], + }, + { + name: 'generic forbidden', + status: 403, + body: { error: { code: 403, message: 'Forbidden' } }, + expected: [500, 500, 500], + }, + { + name: 'credit exhaustion', + status: 403, + body: { + error: { + code: 'insufficient_user_quota', + message: 'user quota is not enough', + }, + }, + expected: [503, 503, 503], + }, + { + name: 'outage', + status: 503, + body: { error: { code: 503, message: 'service unavailable' } }, + expected: [400, 400, 502], + }, + { + name: 'empty choices', + status: 200, + body: { choices: [] }, + expected: [400, 400, 400], + }, +]; + +describe('model availability through real HTTP', () => { + let upstream: Server; + let env: PuterTestEnv; + let response = cases[0]; + beforeAll(async () => { + upstream = createServer((req, res) => { + res.setHeader('content-type', 'application/json'); + if (req.url === '/v1/models') { + const model = { + id: 'vendor/availability-test', + name: 'Availability test', + context_length: 4096, + top_provider: { max_completion_tokens: 256 }, + pricing: { prompt: 0.000001, completion: 0.000001 }, + }; + res.end( + JSON.stringify({ + data: [ + model, + { + ...model, + id: `${model.id}:batch`, + name: 'Batch only', + }, + ], + }), + ); + return; + } + req.resume(); + req.on('end', () => { + res.statusCode = response.status; + res.end(JSON.stringify(response.body)); + }); + }); + await new Promise((resolve) => + upstream.listen(0, '127.0.0.1', resolve), + ); + const port = (upstream.address() as { port: number }).port; + env = await setupPuterTestEnv({ + providers: { + openrouter: { + apiKey: 'synthetic-key', + apiBaseUrl: `http://127.0.0.1:${port}/v1`, + }, + }, + meteringEnforcement: { subscriptions: false }, + } as never); + }, 120000); + afterAll(async () => { + await env?.shutdown(); + upstream?.closeAllConnections(); + if (upstream) + await new Promise((resolve) => + upstream.close(() => resolve()), + ); + }); + beforeEach(() => clearUnhealthyRoutes()); + + it('excludes batch IDs and aliases', async () => { + const models = await env.server.drivers.aiChat.models(); + expect( + models.some((m) => m.id === 'openrouter:vendor/availability-test'), + ).toBe(true); + expect(JSON.stringify(models)).not.toContain(':batch'); + }); + for (const [index, route] of routes.entries()) { + it.each(cases)( + `$name on ${route}`, + async (current) => { + response = current; + const testUser = index === 2 ? env.users.other : env.users.user; + const args = { + model: 'openrouter:vendor/availability-test', + messages: [{ role: 'user', content: 'hi' }], + max_tokens: 16, + stream: false, + }; + const res = await fetch(new URL(route, env.apiOrigin), { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${testUser.apiToken}`, + Origin: env.apiOrigin, + }, + body: JSON.stringify( + route === '/drivers/call' + ? { + interface: 'puter-chat-completion', + service: 'ai-chat', + method: 'complete', + args, + } + : args, + ), + }); + const body = await res.text(); + expect(res.status, body).toBe(current.expected[index]); + if (current.name.includes('policy block')) { + expect(body).toMatch(/model.*(not available|unavailable)/i); + expect(body).not.toMatch( + /vendor\.test|policy violation|private/i, + ); + } + const user = await env.server.stores.user.getByUsername( + testUser.username, + ); + expect( + await env.server.services.metering.getOutstandingHolds( + makeActor({ user: user! }), + ), + ).toBe(0); + }, + 30000, + ); + } +}); diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts index c485a6949..1892148c2 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.edges.test.ts @@ -539,6 +539,80 @@ describe('ChatCompletionDriver exhausted-chain classification', () => { }); }); + it('maps an upstream 403 model_not_found to the model-unavailable 400, not an auth failure, and redacts its URL', async () => { + const err = await errorFor({ + status: 403, + message: + 'The model `fake` is not available to your account. See https://vendor.test/models (request id: req-secret)', + error: { + code: 'model_not_found', + message: 'The model `fake` is not available to your account.', + }, + }); + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'upstream_bad_request', + fields: expect.objectContaining({ upstreamStatus: 404 }), + }); + expect(err.noAlarm).toBeFalsy(); + expect(err.message).toBe('Model not available: fake'); + expect(JSON.stringify(attemptsFrom(err))).not.toMatch( + /http|request id/i, + ); + }); + + it.each([ + ['no code', {}], + ['another code', { error: { code: 'access_denied' } }], + ])( + 'keeps a 403 with %s as 500 upstream_auth_failed', + async (_label, extra) => { + const err = await errorFor({ + status: 403, + message: 'Forbidden', + ...extra, + }); + expect(err).toMatchObject({ + statusCode: 500, + legacyCode: 'upstream_auth_failed', + }); + }, + ); + + it('keeps a 404 whose message reads like a credential failure an auth failure', async () => { + const err = await errorFor( + Object.assign(new Error('Unauthorized'), { status: 404 }), + ); + expect(err).toMatchObject({ + statusCode: 500, + legacyCode: 'upstream_auth_failed', + }); + }); + + it('keeps a 401 model_not_found an auth failure', async () => { + const err = await errorFor({ + status: 401, + message: 'invalid api key', + error: { code: 'model_not_found' }, + }); + expect(err).toMatchObject({ + statusCode: 500, + legacyCode: 'upstream_auth_failed', + }); + }); + + it('classifies a 403 insufficient_user_quota as credit exhaustion', async () => { + const err = await errorFor({ + status: 403, + message: 'user quota is not enough', + error: { code: 'insufficient_user_quota' }, + }); + expect(err).toMatchObject({ + statusCode: 503, + legacyCode: 'upstream_credits_exhausted', + }); + }); + it('sniffs a status out of the message when the provider throws a bare Error', async () => { const err = await errorFor(new Error('provider blew up with 503')); expect(err).toMatchObject({ diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts index 03061c3ad..95e4d43b1 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts @@ -40,6 +40,12 @@ import { type MockInstance, } from 'vitest'; +import type { Request } from 'express'; +import { + renderAnthropicError, + renderOpenAIError, +} from '../../controllers/puterai/wireErrors.js'; +import { Context } from '../../core/context.js'; import { HttpError } from '../../core/http/HttpError.js'; import { PuterServer } from '../../server.js'; import { setupTestServer } from '../../testUtil.js'; @@ -143,6 +149,7 @@ const OPENROUTER_CATALOG = [ // also the alias the Meta provider publishes — the two have to land in // one bucket rather than becoming separate models. 'meta/muse-spark-1.2', + 'openai/o1-pro', ].map((id) => ({ id, name: `${id} (via OpenRouter)`, @@ -475,6 +482,366 @@ describe('ChatCompletionDriver muse spark routing', () => { }); }); +describe('ChatCompletionDriver model-unavailable routes', () => { + // Shaped like the OpenAI SDK's APIError for an OpenAI-convention body. + const modelNotFound = () => + Object.assign( + new Error('403 The model is not available to your account.'), + { + status: 403, + error: { + code: 'model_not_found', + message: 'The model is not available to your account.', + }, + }, + ); + const statusError = (status: number, message: string) => + Object.assign(new Error(message), { status }); + const reply = { + choices: [ + { + message: { role: 'assistant', content: 'hi' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }; + + const complete = (model: string, provider?: string) => + withTestActor(() => + driver.complete({ + model, + ...(provider ? { provider } : {}), + messages: [{ role: 'user', content: 'hi' }], + }), + ).catch((e: unknown) => e as HttpError); + + const statusesOf = (err: HttpError) => + ( + err as unknown as { fields: { attempts: { status?: number }[] } } + ).fields.attempts.map((a) => a.status); + + it('falls back past a gated route and marks only that route', async () => { + createMock + .mockRejectedValueOnce(modelNotFound()) + .mockResolvedValueOnce(reply); + + await expect(complete('deepseek-v4-pro')).resolves.toMatchObject({ + message: { content: 'hi' }, + }); + expect(isRouteUnhealthy('deepseek', 'deepseek-v4-pro')).toBe(true); + expect( + isRouteUnhealthy('infron', 'infron:deepseek/deepseek-v4-pro'), + ).toBe(false); + }); + + it('keeps serving a model from its vendor after the gateway route is gated', async () => { + createMock + .mockRejectedValueOnce(modelNotFound()) + .mockResolvedValueOnce(reply) + .mockResolvedValueOnce(reply); + + await expect( + complete('muse-spark-1.2', 'openrouter'), + ).resolves.toMatchObject({ message: { content: 'hi' } }); + expect(createMock.mock.calls.at(-1)![0].model).toBe('muse-spark-1.2'); + expect( + isRouteUnhealthy('openrouter', 'openrouter:meta/muse-spark-1.2'), + ).toBe(true); + expect(isRouteUnhealthy('meta', 'muse-spark-1.2')).toBe(false); + + await expect(complete('meta/muse-spark-1.2')).resolves.toMatchObject({ + message: { content: 'hi' }, + }); + expect(createMock.mock.calls.at(-1)![0].model).toBe('muse-spark-1.2'); + }); + + it('keeps a gated route marked for as long as bad credentials', async () => { + vi.useFakeTimers({ toFake: ['Date'] }); + const start = Date.now(); + try { + createMock + .mockRejectedValueOnce(modelNotFound()) + .mockResolvedValueOnce(reply); + await complete('deepseek-v4-pro'); + + vi.setSystemTime(start + 61 * 1000); + expect(isRouteUnhealthy('deepseek', 'deepseek-v4-pro')).toBe(true); + vi.setSystemTime(start + (10 * 60 + 1) * 1000); + expect(isRouteUnhealthy('deepseek', 'deepseek-v4-pro')).toBe(false); + } finally { + vi.useRealTimers(); + } + }); + + it('tells the caller the model is unavailable when every route is gated or missing', async () => { + createMock + .mockRejectedValueOnce(modelNotFound()) + .mockRejectedValue(statusError(404, 'model not found')); + + const err = await complete('deepseek-v4-pro'); + + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'upstream_bad_request', + fields: expect.objectContaining({ upstreamStatus: 404 }), + }); + expect(statusesOf(err)).toEqual([403, 404, 404]); + }); + + it('still pages when a gated route sits beside a bad credential', async () => { + createMock + .mockRejectedValueOnce(modelNotFound()) + .mockRejectedValue(statusError(401, 'invalid api key')); + + const err = await complete('deepseek-v4-pro'); + + expect(err).toMatchObject({ + statusCode: 500, + legacyCode: 'upstream_auth_failed', + }); + expect(statusesOf(err)).toEqual([403, 401, 401]); + }); + + it('classifies a gated route beside an outage as a mixed upstream failure', async () => { + createMock + .mockRejectedValueOnce(modelNotFound()) + .mockRejectedValue(statusError(503, 'service unavailable')); + + const err = await complete('deepseek-v4-pro'); + + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'upstream_failed', + }); + expect(statusesOf(err)).toEqual([403, 503, 503]); + }); +}); + +describe('ChatCompletionDriver OpenRouter account policy block', () => { + const POLICY_MESSAGE = + 'Policy Violation: this user has been blocked for a previous policy violation. Learn more: https://openrouter.ai/docs/policy (request id: req-secret)'; + // Shaped like the OpenAI SDK's APIError for OpenRouter's error body. + const policyBlock = () => + Object.assign(new Error(`403 ${POLICY_MESSAGE}`), { + status: 403, + code: 403, + error: { code: 403, message: POLICY_MESSAGE }, + }); + const reply = { + choices: [ + { + message: { role: 'assistant', content: 'hi' }, + finish_reason: 'stop', + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1 }, + }; + + const complete = (model: string, provider?: string) => + withTestActor(() => + driver.complete({ + model, + ...(provider ? { provider } : {}), + messages: [{ role: 'user', content: 'hi' }], + }), + ).catch((e: unknown) => e as HttpError); + + const attemptsOf = (err: HttpError) => + ( + err as unknown as { + fields: { + attempts: { + provider: string; + status?: number; + code?: unknown; + }[]; + }; + } + ).fields.attempts; + + it.each(['openai/o1-pro', 'openrouter:openai/o1-pro', 'o1-pro'])( + 'tells a caller asking for %s that the model is unavailable', + async (model) => { + createMock.mockRejectedValueOnce(policyBlock()); + + const err = await complete(model); + + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'upstream_bad_request', + message: 'Model not available: openrouter:openai/o1-pro', + fields: expect.objectContaining({ upstreamStatus: 404 }), + }); + expect(JSON.stringify(err.fields)).not.toMatch( + /policy|http|request id/i, + ); + }, + ); + + it('treats the in-band policy block of a 200 with no choices the same way', async () => { + createMock.mockResolvedValueOnce({ + error: { code: 403, message: POLICY_MESSAGE }, + }); + + const err = await complete('openai/o1-pro'); + + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'upstream_bad_request', + fields: expect.objectContaining({ upstreamStatus: 404 }), + }); + expect(attemptsOf(err)).toEqual([ + expect.objectContaining({ + provider: 'openrouter', + status: 403, + code: 'model_not_found', + }), + ]); + }); + + it('keeps a no-choice completion without the policy block a request-level 400', async () => { + createMock.mockResolvedValueOnce({}); + + const err = await complete('openai/o1-pro'); + + expect(err).toMatchObject({ + statusCode: 400, + legacyCode: 'upstream_bad_request', + message: 'no completion was returned', + }); + expect(err.fields?.upstreamStatus).toBe(400); + expect(isRouteUnhealthy('openrouter', 'openrouter:openai/o1-pro')).toBe( + false, + ); + }); + + it('falls back from a policy-blocked gateway route to the vendor, under the Anthropic route too', async () => { + for (const strict of [false, true]) { + clearUnhealthyRoutes(); + createMock + .mockRejectedValueOnce(policyBlock()) + .mockResolvedValueOnce(reply); + + const res = await withTestActor(() => { + if (strict) Context.set('strictUpstreamErrors', true); + return driver.complete({ + model: 'muse-spark-1.2', + provider: 'openrouter', + messages: [{ role: 'user', content: 'hi' }], + }); + }); + + expect(res).toMatchObject({ message: { content: 'hi' } }); + expect(createMock.mock.calls.at(-1)![0].model).toBe( + 'muse-spark-1.2', + ); + expect( + isRouteUnhealthy( + 'openrouter', + 'openrouter:meta/muse-spark-1.2', + ), + ).toBe(true); + expect(isRouteUnhealthy('meta', 'muse-spark-1.2')).toBe(false); + } + }); + + it('renders a gated route beside a missing one as not found on both wire formats', async () => { + createMock + .mockRejectedValueOnce(policyBlock()) + .mockRejectedValueOnce( + Object.assign(new Error('model not found'), { status: 404 }), + ); + + const err = await complete('muse-spark-1.2', 'openrouter'); + + expect(attemptsOf(err).map((a) => a.status)).toEqual([403, 404]); + const anthropic = renderAnthropicError(err, {} as Request); + const openai = renderOpenAIError(err, {} as Request); + expect(anthropic.status).toBe(404); + expect(anthropic.body).toMatchObject({ + error: { + type: 'not_found_error', + message: 'Model not available: openrouter:meta/muse-spark-1.2', + }, + }); + expect(openai.status).toBe(404); + expect(openai.body).toMatchObject({ + error: { + type: 'not_found_error', + message: 'Model not available: openrouter:meta/muse-spark-1.2', + }, + }); + expect(JSON.stringify([anthropic.body, openai.body])).not.toMatch( + /policy|http|request id/i, + ); + }); + + it('leaves the same body from another provider an auth failure', async () => { + createMock.mockRejectedValue(policyBlock()); + + const err = await complete('muse-spark-1.2'); + + expect(err).toMatchObject({ + statusCode: 500, + legacyCode: 'upstream_auth_failed', + }); + expect(attemptsOf(err)).toEqual([ + expect.objectContaining({ + provider: 'meta', + status: 403, + code: 403, + }), + expect.objectContaining({ + provider: 'openrouter', + status: 403, + code: 'model_not_found', + }), + ]); + }); + + it.each([ + [ + 'a generic 403', + { status: 403, error: { code: 403, message: 'Forbidden' } }, + 500, + 'upstream_auth_failed', + ], + [ + 'a 401', + { status: 401, error: { code: 401, message: 'User not found.' } }, + 500, + 'upstream_auth_failed', + ], + [ + 'a 402', + { + status: 402, + error: { code: 402, message: 'Insufficient credits' }, + }, + 503, + 'upstream_credits_exhausted', + ], + [ + 'a 503', + { status: 503, error: { code: 503, message: POLICY_MESSAGE } }, + 400, + 'upstream_provider_unavailable', + ], + ])( + 'keeps %s from OpenRouter classified as before', + async (_label, shape, statusCode, legacyCode) => { + createMock.mockRejectedValueOnce( + Object.assign(new Error(shape.error.message), shape), + ); + + const err = await complete('openai/o1-pro'); + + expect(err).toMatchObject({ statusCode, legacyCode }); + }, + ); +}); + describe('ChatCompletionDriver duplicate-model fallback', () => { // deepseek-v4-pro is served directly by DeepSeek, by Infron, and twice by // OpenRouter (two upstream orgs) — four routes in one bucket. diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts index eef6cd91c..4fbcde51f 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -309,10 +309,17 @@ const isRateLimit = (a: ProviderAttempt) => a.status === 429 || /rate[\s_-]?limit|too many requests|quota/i.test(a.error); +const isModelAccessDenied = (a: ProviderAttempt) => + a.status === 403 && a.code === 'model_not_found'; + +const isModelUnavailable = (a: ProviderAttempt) => + a.status === 404 || isModelAccessDenied(a); + const isAuthFailure = (a: ProviderAttempt) => - a.status === 401 || - a.status === 403 || - /unauthorized|forbidden|invalid api key/i.test(a.error); + !isModelAccessDenied(a) && + (a.status === 401 || + a.status === 403 || + /unauthorized|forbidden|invalid api key/i.test(a.error)); const isUpstream5xx = (a: ProviderAttempt) => (a.status !== undefined && a.status >= 500) || @@ -333,7 +340,7 @@ const isUpstream5xx = (a: ProviderAttempt) => */ const isRouteLevelFailure = (a: ProviderAttempt) => a.status === undefined || - a.status === 404 || + isModelUnavailable(a) || a.status === 408 || isCreditExhaustion(a) || isRateLimit(a) || @@ -373,8 +380,6 @@ const requestLevelError = (a: ProviderAttempt): HttpError => const isOverloaded = (a: ProviderAttempt) => a.status === 529 || a.upstreamType === 'overloaded_error'; -const isModelUnavailable = (a: ProviderAttempt) => a.status === 404; - /** * The HTTP status an in-band mid-stream error chunk reports — a stream * populator failure never goes through `classifyAttempts` (there is no fallback @@ -404,11 +409,12 @@ const streamErrorStatus = (a: ProviderAttempt): number => { * - All credit-exhausted → 503 `upstream_credits_exhausted` (alerted) * - All rate-limited → 429 `upstream_rate_limited` (alerted, unless every attempt * was on a free model — see `allModelsFree`) - * - All auth failures → 500 `upstream_auth_failed` (paged: our config) + * - All auth failures, or any alongside 403 `model_not_found` → 500 + * `upstream_auth_failed` (paged: our config) * - All overloaded (529 / `overloaded_error`) → 400 * `upstream_provider_unavailable`, `upstreamStatus: 529` (no page) - * - All model-not-found (404) → 400 `upstream_bad_request`, `upstreamStatus: 404` - * (no page) + * - All model-not-found (404, or 403 `model_not_found`) → 400 + * `upstream_bad_request`, `upstreamStatus: 404` (no page) * - All upstream 5xx or dropped connections → 400 `upstream_provider_unavailable` * (no page) * - All upstream 4xx (other) → 400 `upstream_bad_request` (no page) @@ -449,7 +455,10 @@ const classifyAttempts = ( noAlarm: allModelsFree, }); } - if (attempts.every(isAuthFailure)) { + if ( + attempts.some(isAuthFailure) && + attempts.every((a) => isAuthFailure(a) || isModelAccessDenied(a)) + ) { return new HttpError(500, 'AI provider authentication failed', { legacyCode: 'upstream_auth_failed', fields, @@ -462,7 +471,10 @@ const classifyAttempts = ( }); } if (attempts.every(isModelUnavailable)) { - return new HttpError(400, attempts[0].error, { + const message = attempts.some(isModelAccessDenied) + ? `Model not available: ${attempts[0].model}` + : attempts[0].error; + return new HttpError(400, message, { legacyCode: 'upstream_bad_request', fields: { ...fields, upstreamStatus: 404 }, }); diff --git a/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.test.ts b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.test.ts index fef4926bf..1f2a55962 100644 --- a/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.test.ts +++ b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.test.ts @@ -143,6 +143,23 @@ const SAMPLE_API_MODELS = [ top_provider: { max_completion_tokens: 32768 }, expiration_date: '2026-10-09', }, + { + // Batch variants aren't served on chat/completions — filtered out. + id: 'openai/gpt-6-luna:batch', + name: 'GPT-6 Luna (batch)', + created: 1714564800, + context_length: 128000, + pricing: { prompt: 0.000005, completion: 0.000015 }, + top_provider: { max_completion_tokens: 16000 }, + }, + { + id: 'openai/gpt-6-luna:free', + name: 'GPT-6 Luna (free)', + created: 1714564800, + context_length: 128000, + pricing: { prompt: 0, completion: 0 }, + top_provider: { max_completion_tokens: 16000 }, + }, { // 'openrouter/auto' is filtered out — disallowed. id: 'openrouter/auto', @@ -268,6 +285,18 @@ describe('OpenRouterProvider model catalog', () => { expect(ids).not.toContain('qwen/qwen3-max'); }); + it('filters out batch variants and keeps their base model', async () => { + const { provider } = makeProvider(); + const ids = modelLookupNames(await provider.models()); + expect(ids).not.toContain('openrouter:openai/gpt-6-luna:batch'); + expect(ids).not.toContain('openai/gpt-6-luna:batch'); + expect(ids).not.toContain('gpt-6-luna:batch'); + expect(ids).toContain('openrouter:openai/gpt-6-luna'); + expect(ids).toContain('gpt-6-luna'); + expect(ids).toContain('openrouter:openai/gpt-6-luna:free'); + expect(ids).toContain('gpt-6-luna:free'); + }); + it('coerces string prices and normalizes conditional tiers', async () => { const { provider } = makeProvider(); const models = await provider.models(); @@ -610,6 +639,126 @@ describe('OpenRouterProvider.complete request shape', () => { }); }); +describe('OpenRouterProvider account policy block', () => { + const POLICY_MESSAGE = + 'Policy Violation: this user has been blocked for a previous policy violation. Learn more: https://openrouter.ai/docs/policy (request id: req-secret)'; + + // Shaped like the OpenAI SDK's APIError for OpenRouter's error body. + const apiError = (status: number, body: Record) => + Object.assign(new Error(`${status} ${String(body.message)}`), { + status, + error: body, + code: body.code, + }); + + const complete = () => + withTestActor(() => + makeProvider().provider.complete({ + model: 'openrouter:openai/gpt-6-luna', + messages: [{ role: 'user', content: 'hi' }], + }), + ); + + const expectModelUnavailable = async () => { + const err = await complete().catch((e: unknown) => e); + expect(err).toMatchObject({ + name: 'UpstreamResponseError', + status: 403, + code: 'model_not_found', + provider: 'openrouter', + message: 'Model not available: openrouter:openai/gpt-6-luna', + }); + expect(JSON.stringify(err)).not.toMatch(/policy|http|request id/i); + expect((err as Error).message).not.toMatch(/policy|http/i); + }; + + it('reports a numeric 403 policy block as the model being unavailable', async () => { + createMock.mockRejectedValueOnce( + apiError(403, { code: 403, message: POLICY_MESSAGE }), + ); + await expectModelUnavailable(); + expect(createMock).toHaveBeenCalledTimes(1); + }); + + it('reports the in-band 403 policy block of a 200 with no choices the same way', async () => { + createMock.mockResolvedValueOnce({ + error: { code: 403, message: POLICY_MESSAGE }, + }); + await expectModelUnavailable(); + expect(recordSpy).not.toHaveBeenCalled(); + }); + + it.each([ + ['a generic 403', apiError(403, { code: 403, message: 'Forbidden' })], + [ + 'a key limit 403', + apiError(403, { + code: 403, + message: 'Key limit exceeded (daily limit).', + }), + ], + [ + 'a moderation 403', + apiError(403, { + code: 403, + message: + 'Your chosen model requires moderation and your input was flagged', + metadata: { reasons: ['violence'] }, + }), + ], + [ + 'a similar 403 wording', + apiError(403, { + code: 403, + message: 'Policy Violation: this key has been blocked.', + }), + ], + [ + 'the policy wording without its numeric code', + apiError(403, { message: POLICY_MESSAGE }), + ], + [ + 'the policy wording on a 401', + apiError(401, { code: 401, message: POLICY_MESSAGE }), + ], + [ + 'the policy wording on a 503', + apiError(503, { code: 403, message: POLICY_MESSAGE }), + ], + [ + 'a 402 credit exhaustion', + apiError(402, { code: 402, message: 'Insufficient credits' }), + ], + ])('rethrows %s unchanged', async (_label, thrown) => { + createMock.mockRejectedValueOnce(thrown); + await expect(complete()).rejects.toBe(thrown); + }); + + it.each([ + ['no error body', {}, 'no completion was returned'], + [ + 'an in-band error with another code', + { error: { code: 401, message: POLICY_MESSAGE } }, + POLICY_MESSAGE, + ], + [ + 'an in-band error with other wording', + { error: { code: 403, message: 'Provider returned error' } }, + 'Provider returned error', + ], + ])( + 'keeps a no-choice completion with %s as bad_response', + async (_label, completion, message) => { + createMock.mockResolvedValueOnce(completion); + await expect(complete()).rejects.toMatchObject({ + statusCode: 400, + legacyCode: 'bad_response', + message, + }); + }, + ); +}); + // ── Non-stream completion: cost calculator branches ───────────────── describe('OpenRouterProvider.complete non-stream output', () => { diff --git a/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts index 452d673e3..efe694414 100644 --- a/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts +++ b/src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts @@ -19,7 +19,8 @@ import axios from 'axios'; import { OpenAI } from 'openai'; -import { HttpError } from '../../../../core/http/HttpError.js'; +import { HttpError, isHttpError } from '../../../../core/http/HttpError.js'; +import { UpstreamResponseError } from '../../../util/upstreamErrors.js'; import type { MeteringService } from '../../../../services/metering/MeteringService.js'; import type { IChatModel, ICompleteArguments } from '../../types.js'; import { cachedRemoteCatalog } from '../../utils/cachedRemoteCatalog.js'; @@ -37,6 +38,29 @@ type OpenrouterUsage = OpenAI.Completions.CompletionUsage & { cost?: number; }; +const POLICY_BLOCK_MESSAGE = + 'Policy Violation: this user has been blocked for a previous policy violation.'; + +const upstreamErrorBody = (err: unknown): unknown => { + if (isHttpError(err)) { + return err.legacyCode === 'bad_response' ? err.cause : undefined; + } + const e = err as { status?: unknown; error?: unknown } | undefined; + return e?.status === 403 ? e.error : undefined; +}; + +const isPolicyBlock = (err: unknown): boolean => { + const { code, message } = (upstreamErrorBody(err) ?? {}) as { + code?: unknown; + message?: unknown; + }; + return ( + code === 403 && + typeof message === 'string' && + message.startsWith(POLICY_BLOCK_MESSAGE) + ); +}; + export class OpenRouterProvider extends OpenAICompatProvider { #apiBaseUrl: string; @@ -78,7 +102,17 @@ export class OpenRouterProvider extends OpenAICompatProvider { }, ); } - return super.complete(args, resolved); + try { + return await super.complete(args, resolved); + } catch (e) { + if (!isPolicyBlock(e)) throw e; + throw new UpstreamResponseError( + `Model not available: ${args.model}`, + 403, + 'openrouter', + 'model_not_found', + ); + } } override async models(): Promise { @@ -105,7 +139,8 @@ export class OpenRouterProvider extends OpenAICompatProvider { for (const model of models) { if ( typeof model.id !== 'string' || - model.id.includes('openrouter/auto') + model.id.includes('openrouter/auto') || + model.id.endsWith(':batch') ) { continue; } diff --git a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js index 74b3faec9..3a7d5974e 100644 --- a/src/backend/drivers/ai-chat/utils/OpenAIUtil.js +++ b/src/backend/drivers/ai-chat/utils/OpenAIUtil.js @@ -907,7 +907,7 @@ export const handle_completion_output = async ( typeof upstreamMessage === 'string' && upstreamMessage ? upstreamMessage : 'no completion was returned', - { legacyCode: 'bad_response' }, + { legacyCode: 'bad_response', cause: completion?.error }, ); }