Handle unavailable chat model routes and exclude batch variants (#4230)

* Handle model access denials and filter batch chat variants * Raise a low alarm when a provider denies model access A 403 model_not_found still returns 400 upstream_bad_request, but now alarms at info so a lost provider entitlement gets noticed. HttpError gains an alarmSeverity option the terminal alarm gate honors. --------- Co-authored-by: 404oops <oops404@util.404oops.com> Co-authored-by: Daniel Salazar <daniel.salazar@puter.com>
2026-10-21 19:11:10 +00:00 · 2026-10-16 23:21:46 -07:00
1 parent 471b900462
commit 145d54cf69
@@ -81,6 +81,8 @@ export type LegacyErrorCodes =
 * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/).
 */

+import type { PagerSeverity } from '../../types';
+
 /** Options accepted by `HttpError`. All optional. */
 export interface HttpErrorOptions {
    /** Underlying error. Set as the standard `Error.cause`. */
@@ -111,6 +113,11 @@ export interface HttpErrorOptions {
     * serialized to the client.
     */
    noAlarm?: boolean;
+    /**
+     * Alarm at this severity even where the 4xx / `upstream_` rules would skip
+     * it. `noAlarm` still wins. Never serialized to the client.
+     */
+    alarmSeverity?: PagerSeverity;
 }

 /**
@@ -140,6 +147,7 @@ export class HttpError extends Error {
    readonly code?: string;
    readonly fields?: Record<string, unknown>;
    readonly noAlarm?: boolean;
+    readonly alarmSeverity?: PagerSeverity;

    constructor(
        statusCode: number,
@@ -156,6 +164,7 @@ export class HttpError extends Error {
        this.code = options.code;
        this.fields = options.fields;
        this.noAlarm = options.noAlarm;
+        this.alarmSeverity = options.alarmSeverity;
    }
 }

@@ -0,0 +1,199 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { createServer, type Server } from 'node:http';
+import { afterAll, beforeAll, beforeEach, describe, expect, it } from 'vitest';
+import { setupPuterTestEnv, type PuterTestEnv } from '../../testUtil.js';
+import { clearUnhealthyRoutes } from './utils/providerHealth.js';
+import { makeActor } from '../../core/actor.js';
+
+const policyMessage =
+    'Policy Violation: this user has been blocked for a previous policy violation. Learn more: https://vendor.test/private';
+const routes = [
+    '/drivers/call',
+    '/puterai/openai/v1/chat/completions',
+    '/puterai/anthropic/v1/messages',
+];
+const cases = [
+    {
+        name: 'policy block',
+        status: 403,
+        body: { error: { code: 403, message: policyMessage } },
+        expected: [400, 404, 404],
+    },
+    {
+        name: 'in-band policy block',
+        status: 200,
+        body: { error: { code: 403, message: policyMessage } },
+        expected: [400, 404, 404],
+    },
+    {
+        name: 'bad credentials',
+        status: 401,
+        body: { error: { code: 401, message: 'Invalid API key' } },
+        expected: [500, 500, 500],
+    },
+    {
+        name: 'generic forbidden',
+        status: 403,
+        body: { error: { code: 403, message: 'Forbidden' } },
+        expected: [500, 500, 500],
+    },
+    {
+        name: 'credit exhaustion',
+        status: 403,
+        body: {
+            error: {
+                code: 'insufficient_user_quota',
+                message: 'user quota is not enough',
+            },
+        },
+        expected: [503, 503, 503],
+    },
+    {
+        name: 'outage',
+        status: 503,
+        body: { error: { code: 503, message: 'service unavailable' } },
+        expected: [400, 400, 502],
+    },
+    {
+        name: 'empty choices',
+        status: 200,
+        body: { choices: [] },
+        expected: [400, 400, 400],
+    },
+];
+
+describe('model availability through real HTTP', () => {
+    let upstream: Server;
+    let env: PuterTestEnv;
+    let response = cases[0];
+    beforeAll(async () => {
+        upstream = createServer((req, res) => {
+            res.setHeader('content-type', 'application/json');
+            if (req.url === '/v1/models') {
+                const model = {
+                    id: 'vendor/availability-test',
+                    name: 'Availability test',
+                    context_length: 4096,
+                    top_provider: { max_completion_tokens: 256 },
+                    pricing: { prompt: 0.000001, completion: 0.000001 },
+                };
+                res.end(
+                    JSON.stringify({
+                        data: [
+                            model,
+                            {
+                                ...model,
+                                id: `${model.id}:batch`,
+                                name: 'Batch only',
+                            },
+                        ],
+                    }),
+                );
+                return;
+            }
+            req.resume();
+            req.on('end', () => {
+                res.statusCode = response.status;
+                res.end(JSON.stringify(response.body));
+            });
+        });
+        await new Promise<void>((resolve) =>
+            upstream.listen(0, '127.0.0.1', resolve),
+        );
+        const port = (upstream.address() as { port: number }).port;
+        env = await setupPuterTestEnv({
+            providers: {
+                openrouter: {
+                    apiKey: 'synthetic-key',
+                    apiBaseUrl: `http://127.0.0.1:${port}/v1`,
+                },
+            },
+            meteringEnforcement: { subscriptions: false },
+        } as never);
+    }, 120000);
+    afterAll(async () => {
+        await env?.shutdown();
+        upstream?.closeAllConnections();
+        if (upstream)
+            await new Promise<void>((resolve) =>
+                upstream.close(() => resolve()),
+            );
+    });
+    beforeEach(() => clearUnhealthyRoutes());
+
+    it('excludes batch IDs and aliases', async () => {
+        const models = await env.server.drivers.aiChat.models();
+        expect(
+            models.some((m) => m.id === 'openrouter:vendor/availability-test'),
+        ).toBe(true);
+        expect(JSON.stringify(models)).not.toContain(':batch');
+    });
+    for (const [index, route] of routes.entries()) {
+        it.each(cases)(
+            `$name on ${route}`,
+            async (current) => {
+                response = current;
+                const testUser = index === 2 ? env.users.other : env.users.user;
+                const args = {
+                    model: 'openrouter:vendor/availability-test',
+                    messages: [{ role: 'user', content: 'hi' }],
+                    max_tokens: 16,
+                    stream: false,
+                };
+                const res = await fetch(new URL(route, env.apiOrigin), {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                        Authorization: `Bearer ${testUser.apiToken}`,
+                        Origin: env.apiOrigin,
+                    },
+                    body: JSON.stringify(
+                        route === '/drivers/call'
+                            ? {
+                                  interface: 'puter-chat-completion',
+                                  service: 'ai-chat',
+                                  method: 'complete',
+                                  args,
+                              }
+                            : args,
+                    ),
+                });
+                const body = await res.text();
+                expect(res.status, body).toBe(current.expected[index]);
+                if (current.name.includes('policy block')) {
+                    expect(body).toMatch(/model.*(not available|unavailable)/i);
+                    expect(body).not.toMatch(
+                        /vendor\.test|policy violation|private/i,
+                    );
+                }
+                const user = await env.server.stores.user.getByUsername(
+                    testUser.username,
+                );
+                expect(
+                    await env.server.services.metering.getOutstandingHolds(
+                        makeActor({ user: user! }),
+                    ),
+                ).toBe(0);
+            },
+            30000,
+        );
+    }
+});
@@ -537,6 +537,82 @@ describe('ChatCompletionDriver exhausted-chain classification', () => {
            legacyCode: 'upstream_bad_request',
            fields: expect.objectContaining({ upstreamStatus: 404 }),
        });
+        expect(err.alarmSeverity).toBeUndefined();
+    });
+
+    it('maps an upstream 403 model_not_found to the model-unavailable 400, not an auth failure, and redacts its URL', async () => {
+        const err = await errorFor({
+            status: 403,
+            message:
+                'The model `fake` is not available to your account. See https://vendor.test/models (request id: req-secret)',
+            error: {
+                code: 'model_not_found',
+                message: 'The model `fake` is not available to your account.',
+            },
+        });
+        expect(err).toMatchObject({
+            statusCode: 400,
+            legacyCode: 'upstream_bad_request',
+            fields: expect.objectContaining({ upstreamStatus: 404 }),
+        });
+        expect(err.noAlarm).toBeFalsy();
+        expect(err.alarmSeverity).toBe('info');
+        expect(err.message).toBe('Model not available: fake');
+        expect(JSON.stringify(attemptsFrom(err))).not.toMatch(
+            /http|request id/i,
+        );
+    });
+
+    it.each([
+        ['no code', {}],
+        ['another code', { error: { code: 'access_denied' } }],
+    ])(
+        'keeps a 403 with %s as 500 upstream_auth_failed',
+        async (_label, extra) => {
+            const err = await errorFor({
+                status: 403,
+                message: 'Forbidden',
+                ...extra,
+            });
+            expect(err).toMatchObject({
+                statusCode: 500,
+                legacyCode: 'upstream_auth_failed',
+            });
+        },
+    );
+
+    it('keeps a 404 whose message reads like a credential failure an auth failure', async () => {
+        const err = await errorFor(
+            Object.assign(new Error('Unauthorized'), { status: 404 }),
+        );
+        expect(err).toMatchObject({
+            statusCode: 500,
+            legacyCode: 'upstream_auth_failed',
+        });
+    });
+
+    it('keeps a 401 model_not_found an auth failure', async () => {
+        const err = await errorFor({
+            status: 401,
+            message: 'invalid api key',
+            error: { code: 'model_not_found' },
+        });
+        expect(err).toMatchObject({
+            statusCode: 500,
+            legacyCode: 'upstream_auth_failed',
+        });
+    });
+
+    it('classifies a 403 insufficient_user_quota as credit exhaustion', async () => {
+        const err = await errorFor({
+            status: 403,
+            message: 'user quota is not enough',
+            error: { code: 'insufficient_user_quota' },
+        });
+        expect(err).toMatchObject({
+            statusCode: 503,
+            legacyCode: 'upstream_credits_exhausted',
+        });
    });

    it('sniffs a status out of the message when the provider throws a bare Error', async () => {
@@ -40,6 +40,12 @@ import {
    type MockInstance,
 } from 'vitest';

+import type { Request } from 'express';
+import {
+    renderAnthropicError,
+    renderOpenAIError,
+} from '../../controllers/puterai/wireErrors.js';
+import { Context } from '../../core/context.js';
 import { HttpError } from '../../core/http/HttpError.js';
 import { PuterServer } from '../../server.js';
 import { setupTestServer } from '../../testUtil.js';
@@ -143,6 +149,7 @@ const OPENROUTER_CATALOG = [
    // also the alias the Meta provider publishes — the two have to land in
    // one bucket rather than becoming separate models.
    'meta/muse-spark-1.2',
+    'openai/o1-pro',
 ].map((id) => ({
    id,
    name: `${id} (via OpenRouter)`,
@@ -475,6 +482,366 @@ describe('ChatCompletionDriver muse spark routing', () => {
    });
 });

+describe('ChatCompletionDriver model-unavailable routes', () => {
+    // Shaped like the OpenAI SDK's APIError for an OpenAI-convention body.
+    const modelNotFound = () =>
+        Object.assign(
+            new Error('403 The model is not available to your account.'),
+            {
+                status: 403,
+                error: {
+                    code: 'model_not_found',
+                    message: 'The model is not available to your account.',
+                },
+            },
+        );
+    const statusError = (status: number, message: string) =>
+        Object.assign(new Error(message), { status });
+    const reply = {
+        choices: [
+            {
+                message: { role: 'assistant', content: 'hi' },
+                finish_reason: 'stop',
+            },
+        ],
+        usage: { prompt_tokens: 1, completion_tokens: 1 },
+    };
+
+    const complete = (model: string, provider?: string) =>
+        withTestActor(() =>
+            driver.complete({
+                model,
+                ...(provider ? { provider } : {}),
+                messages: [{ role: 'user', content: 'hi' }],
+            }),
+        ).catch((e: unknown) => e as HttpError);
+
+    const statusesOf = (err: HttpError) =>
+        (
+            err as unknown as { fields: { attempts: { status?: number }[] } }
+        ).fields.attempts.map((a) => a.status);
+
+    it('falls back past a gated route and marks only that route', async () => {
+        createMock
+            .mockRejectedValueOnce(modelNotFound())
+            .mockResolvedValueOnce(reply);
+
+        await expect(complete('deepseek-v4-pro')).resolves.toMatchObject({
+            message: { content: 'hi' },
+        });
+        expect(isRouteUnhealthy('deepseek', 'deepseek-v4-pro')).toBe(true);
+        expect(
+            isRouteUnhealthy('infron', 'infron:deepseek/deepseek-v4-pro'),
+        ).toBe(false);
+    });
+
+    it('keeps serving a model from its vendor after the gateway route is gated', async () => {
+        createMock
+            .mockRejectedValueOnce(modelNotFound())
+            .mockResolvedValueOnce(reply)
+            .mockResolvedValueOnce(reply);
+
+        await expect(
+            complete('muse-spark-1.2', 'openrouter'),
+        ).resolves.toMatchObject({ message: { content: 'hi' } });
+        expect(createMock.mock.calls.at(-1)![0].model).toBe('muse-spark-1.2');
+        expect(
+            isRouteUnhealthy('openrouter', 'openrouter:meta/muse-spark-1.2'),
+        ).toBe(true);
+        expect(isRouteUnhealthy('meta', 'muse-spark-1.2')).toBe(false);
+
+        await expect(complete('meta/muse-spark-1.2')).resolves.toMatchObject({
+            message: { content: 'hi' },
+        });
+        expect(createMock.mock.calls.at(-1)![0].model).toBe('muse-spark-1.2');
+    });
+
+    it('keeps a gated route marked for as long as bad credentials', async () => {
+        vi.useFakeTimers({ toFake: ['Date'] });
+        const start = Date.now();
+        try {
+            createMock
+                .mockRejectedValueOnce(modelNotFound())
+                .mockResolvedValueOnce(reply);
+            await complete('deepseek-v4-pro');
+
+            vi.setSystemTime(start + 61 * 1000);
+            expect(isRouteUnhealthy('deepseek', 'deepseek-v4-pro')).toBe(true);
+            vi.setSystemTime(start + (10 * 60 + 1) * 1000);
+            expect(isRouteUnhealthy('deepseek', 'deepseek-v4-pro')).toBe(false);
+        } finally {
+            vi.useRealTimers();
+        }
+    });
+
+    it('tells the caller the model is unavailable when every route is gated or missing', async () => {
+        createMock
+            .mockRejectedValueOnce(modelNotFound())
+            .mockRejectedValue(statusError(404, 'model not found'));
+
+        const err = await complete('deepseek-v4-pro');
+
+        expect(err).toMatchObject({
+            statusCode: 400,
+            legacyCode: 'upstream_bad_request',
+            fields: expect.objectContaining({ upstreamStatus: 404 }),
+        });
+        expect(statusesOf(err)).toEqual([403, 404, 404]);
+    });
+
+    it('still pages when a gated route sits beside a bad credential', async () => {
+        createMock
+            .mockRejectedValueOnce(modelNotFound())
+            .mockRejectedValue(statusError(401, 'invalid api key'));
+
+        const err = await complete('deepseek-v4-pro');
+
+        expect(err).toMatchObject({
+            statusCode: 500,
+            legacyCode: 'upstream_auth_failed',
+        });
+        expect(statusesOf(err)).toEqual([403, 401, 401]);
+    });
+
+    it('classifies a gated route beside an outage as a mixed upstream failure', async () => {
+        createMock
+            .mockRejectedValueOnce(modelNotFound())
+            .mockRejectedValue(statusError(503, 'service unavailable'));
+
+        const err = await complete('deepseek-v4-pro');
+
+        expect(err).toMatchObject({
+            statusCode: 400,
+            legacyCode: 'upstream_failed',
+        });
+        expect(statusesOf(err)).toEqual([403, 503, 503]);
+    });
+});
+
+describe('ChatCompletionDriver OpenRouter account policy block', () => {
+    const POLICY_MESSAGE =
+        'Policy Violation: this user has been blocked for a previous policy violation. Learn more: https://openrouter.ai/docs/policy (request id: req-secret)';
+    // Shaped like the OpenAI SDK's APIError for OpenRouter's error body.
+    const policyBlock = () =>
+        Object.assign(new Error(`403 ${POLICY_MESSAGE}`), {
+            status: 403,
+            code: 403,
+            error: { code: 403, message: POLICY_MESSAGE },
+        });
+    const reply = {
+        choices: [
+            {
+                message: { role: 'assistant', content: 'hi' },
+                finish_reason: 'stop',
+            },
+        ],
+        usage: { prompt_tokens: 1, completion_tokens: 1 },
+    };
+
+    const complete = (model: string, provider?: string) =>
+        withTestActor(() =>
+            driver.complete({
+                model,
+                ...(provider ? { provider } : {}),
+                messages: [{ role: 'user', content: 'hi' }],
+            }),
+        ).catch((e: unknown) => e as HttpError);
+
+    const attemptsOf = (err: HttpError) =>
+        (
+            err as unknown as {
+                fields: {
+                    attempts: {
+                        provider: string;
+                        status?: number;
+                        code?: unknown;
+                    }[];
+                };
+            }
+        ).fields.attempts;
+
+    it.each(['openai/o1-pro', 'openrouter:openai/o1-pro', 'o1-pro'])(
+        'tells a caller asking for %s that the model is unavailable',
+        async (model) => {
+            createMock.mockRejectedValueOnce(policyBlock());
+
+            const err = await complete(model);
+
+            expect(err).toMatchObject({
+                statusCode: 400,
+                legacyCode: 'upstream_bad_request',
+                message: 'Model not available: openrouter:openai/o1-pro',
+                fields: expect.objectContaining({ upstreamStatus: 404 }),
+            });
+            expect(JSON.stringify(err.fields)).not.toMatch(
+                /policy|http|request id/i,
+            );
+        },
+    );
+
+    it('treats the in-band policy block of a 200 with no choices the same way', async () => {
+        createMock.mockResolvedValueOnce({
+            error: { code: 403, message: POLICY_MESSAGE },
+        });
+
+        const err = await complete('openai/o1-pro');
+
+        expect(err).toMatchObject({
+            statusCode: 400,
+            legacyCode: 'upstream_bad_request',
+            fields: expect.objectContaining({ upstreamStatus: 404 }),
+        });
+        expect(attemptsOf(err)).toEqual([
+            expect.objectContaining({
+                provider: 'openrouter',
+                status: 403,
+                code: 'model_not_found',
+            }),
+        ]);
+    });
+
+    it('keeps a no-choice completion without the policy block a request-level 400', async () => {
+        createMock.mockResolvedValueOnce({});
+
+        const err = await complete('openai/o1-pro');
+
+        expect(err).toMatchObject({
+            statusCode: 400,
+            legacyCode: 'upstream_bad_request',
+            message: 'no completion was returned',
+        });
+        expect(err.fields?.upstreamStatus).toBe(400);
+        expect(isRouteUnhealthy('openrouter', 'openrouter:openai/o1-pro')).toBe(
+            false,
+        );
+    });
+
+    it('falls back from a policy-blocked gateway route to the vendor, under the Anthropic route too', async () => {
+        for (const strict of [false, true]) {
+            clearUnhealthyRoutes();
+            createMock
+                .mockRejectedValueOnce(policyBlock())
+                .mockResolvedValueOnce(reply);
+
+            const res = await withTestActor(() => {
+                if (strict) Context.set('strictUpstreamErrors', true);
+                return driver.complete({
+                    model: 'muse-spark-1.2',
+                    provider: 'openrouter',
+                    messages: [{ role: 'user', content: 'hi' }],
+                });
+            });
+
+            expect(res).toMatchObject({ message: { content: 'hi' } });
+            expect(createMock.mock.calls.at(-1)![0].model).toBe(
+                'muse-spark-1.2',
+            );
+            expect(
+                isRouteUnhealthy(
+                    'openrouter',
+                    'openrouter:meta/muse-spark-1.2',
+                ),
+            ).toBe(true);
+            expect(isRouteUnhealthy('meta', 'muse-spark-1.2')).toBe(false);
+        }
+    });
+
+    it('renders a gated route beside a missing one as not found on both wire formats', async () => {
+        createMock
+            .mockRejectedValueOnce(policyBlock())
+            .mockRejectedValueOnce(
+                Object.assign(new Error('model not found'), { status: 404 }),
+            );
+
+        const err = await complete('muse-spark-1.2', 'openrouter');
+
+        expect(attemptsOf(err).map((a) => a.status)).toEqual([403, 404]);
+        const anthropic = renderAnthropicError(err, {} as Request);
+        const openai = renderOpenAIError(err, {} as Request);
+        expect(anthropic.status).toBe(404);
+        expect(anthropic.body).toMatchObject({
+            error: {
+                type: 'not_found_error',
+                message: 'Model not available: openrouter:meta/muse-spark-1.2',
+            },
+        });
+        expect(openai.status).toBe(404);
+        expect(openai.body).toMatchObject({
+            error: {
+                type: 'not_found_error',
+                message: 'Model not available: openrouter:meta/muse-spark-1.2',
+            },
+        });
+        expect(JSON.stringify([anthropic.body, openai.body])).not.toMatch(
+            /policy|http|request id/i,
+        );
+    });
+
+    it('leaves the same body from another provider an auth failure', async () => {
+        createMock.mockRejectedValue(policyBlock());
+
+        const err = await complete('muse-spark-1.2');
+
+        expect(err).toMatchObject({
+            statusCode: 500,
+            legacyCode: 'upstream_auth_failed',
+        });
+        expect(attemptsOf(err)).toEqual([
+            expect.objectContaining({
+                provider: 'meta',
+                status: 403,
+                code: 403,
+            }),
+            expect.objectContaining({
+                provider: 'openrouter',
+                status: 403,
+                code: 'model_not_found',
+            }),
+        ]);
+    });
+
+    it.each([
+        [
+            'a generic 403',
+            { status: 403, error: { code: 403, message: 'Forbidden' } },
+            500,
+            'upstream_auth_failed',
+        ],
+        [
+            'a 401',
+            { status: 401, error: { code: 401, message: 'User not found.' } },
+            500,
+            'upstream_auth_failed',
+        ],
+        [
+            'a 402',
+            {
+                status: 402,
+                error: { code: 402, message: 'Insufficient credits' },
+            },
+            503,
+            'upstream_credits_exhausted',
+        ],
+        [
+            'a 503',
+            { status: 503, error: { code: 503, message: POLICY_MESSAGE } },
+            400,
+            'upstream_provider_unavailable',
+        ],
+    ])(
+        'keeps %s from OpenRouter classified as before',
+        async (_label, shape, statusCode, legacyCode) => {
+            createMock.mockRejectedValueOnce(
+                Object.assign(new Error(shape.error.message), shape),
+            );
+
+            const err = await complete('openai/o1-pro');
+
+            expect(err).toMatchObject({ statusCode, legacyCode });
+        },
+    );
+});
+
 describe('ChatCompletionDriver duplicate-model fallback', () => {
    // deepseek-v4-pro is served directly by DeepSeek, by Infron, and twice by
    // OpenRouter (two upstream orgs) — four routes in one bucket.
@@ -309,10 +309,17 @@ const isRateLimit = (a: ProviderAttempt) =>
    a.status === 429 ||
    /rate[\s_-]?limit|too many requests|quota/i.test(a.error);

+const isModelAccessDenied = (a: ProviderAttempt) =>
+    a.status === 403 && a.code === 'model_not_found';
+
+const isModelUnavailable = (a: ProviderAttempt) =>
+    a.status === 404 || isModelAccessDenied(a);
+
 const isAuthFailure = (a: ProviderAttempt) =>
-    a.status === 401 ||
-    a.status === 403 ||
-    /unauthorized|forbidden|invalid api key/i.test(a.error);
+    !isModelAccessDenied(a) &&
+    (a.status === 401 ||
+        a.status === 403 ||
+        /unauthorized|forbidden|invalid api key/i.test(a.error));

 const isUpstream5xx = (a: ProviderAttempt) =>
    (a.status !== undefined && a.status >= 500) ||
@@ -333,7 +340,7 @@ const isUpstream5xx = (a: ProviderAttempt) =>
 */
 const isRouteLevelFailure = (a: ProviderAttempt) =>
    a.status === undefined ||
-    a.status === 404 ||
+    isModelUnavailable(a) ||
    a.status === 408 ||
    isCreditExhaustion(a) ||
    isRateLimit(a) ||
@@ -373,8 +380,6 @@ const requestLevelError = (a: ProviderAttempt): HttpError =>
 const isOverloaded = (a: ProviderAttempt) =>
    a.status === 529 || a.upstreamType === 'overloaded_error';

-const isModelUnavailable = (a: ProviderAttempt) => a.status === 404;
-
 /**
 * The HTTP status an in-band mid-stream error chunk reports — a stream
 * populator failure never goes through `classifyAttempts` (there is no fallback
@@ -404,11 +409,12 @@ const streamErrorStatus = (a: ProviderAttempt): number => {
 * - All credit-exhausted → 503 `upstream_credits_exhausted` (alerted)
 * - All rate-limited → 429 `upstream_rate_limited` (alerted, unless every attempt
 *   was on a free model — see `allModelsFree`)
- * - All auth failures → 500 `upstream_auth_failed` (paged: our config)
+ * - All auth failures, or any alongside 403 `model_not_found` → 500
+ *   `upstream_auth_failed` (paged: our config)
 * - All overloaded (529 / `overloaded_error`) → 400
 *   `upstream_provider_unavailable`, `upstreamStatus: 529` (no page)
- * - All model-not-found (404) → 400 `upstream_bad_request`, `upstreamStatus: 404`
- *   (no page)
+ * - All model-not-found (404, or 403 `model_not_found`) → 400
+ *   `upstream_bad_request`, `upstreamStatus: 404` (no page)
 * - All upstream 5xx or dropped connections → 400 `upstream_provider_unavailable`
 *   (no page)
 * - All upstream 4xx (other) → 400 `upstream_bad_request` (no page)
@@ -449,7 +455,10 @@ const classifyAttempts = (
            noAlarm: allModelsFree,
        });
    }
-    if (attempts.every(isAuthFailure)) {
+    if (
+        attempts.some(isAuthFailure) &&
+        attempts.every((a) => isAuthFailure(a) || isModelAccessDenied(a))
+    ) {
        return new HttpError(500, 'AI provider authentication failed', {
            legacyCode: 'upstream_auth_failed',
            fields,
@@ -462,9 +471,16 @@ const classifyAttempts = (
        });
    }
    if (attempts.every(isModelUnavailable)) {
-        return new HttpError(400, attempts[0].error, {
+        const accessDenied = attempts.some(isModelAccessDenied);
+        const message = accessDenied
+            ? `Model not available: ${attempts[0].model}`
+            : attempts[0].error;
+        return new HttpError(400, message, {
            legacyCode: 'upstream_bad_request',
            fields: { ...fields, upstreamStatus: 404 },
+            // Our provider account lost access to the model; low alarm so
+            // someone notices.
+            ...(accessDenied ? { alarmSeverity: 'info' as const } : {}),
        });
    }
    if (
@@ -143,6 +143,23 @@ const SAMPLE_API_MODELS = [
        top_provider: { max_completion_tokens: 32768 },
        expiration_date: '2026-10-09',
    },
+    {
+        // Batch variants aren't served on chat/completions — filtered out.
+        id: 'openai/gpt-6-luna:batch',
+        name: 'GPT-6 Luna (batch)',
+        created: 1714564800,
+        context_length: 128000,
+        pricing: { prompt: 0.000005, completion: 0.000015 },
+        top_provider: { max_completion_tokens: 16000 },
+    },
+    {
+        id: 'openai/gpt-6-luna:free',
+        name: 'GPT-6 Luna (free)',
+        created: 1714564800,
+        context_length: 128000,
+        pricing: { prompt: 0, completion: 0 },
+        top_provider: { max_completion_tokens: 16000 },
+    },
    {
        // 'openrouter/auto' is filtered out — disallowed.
        id: 'openrouter/auto',
@@ -268,6 +285,18 @@ describe('OpenRouterProvider model catalog', () => {
        expect(ids).not.toContain('qwen/qwen3-max');
    });

+    it('filters out batch variants and keeps their base model', async () => {
+        const { provider } = makeProvider();
+        const ids = modelLookupNames(await provider.models());
+        expect(ids).not.toContain('openrouter:openai/gpt-6-luna:batch');
+        expect(ids).not.toContain('openai/gpt-6-luna:batch');
+        expect(ids).not.toContain('gpt-6-luna:batch');
+        expect(ids).toContain('openrouter:openai/gpt-6-luna');
+        expect(ids).toContain('gpt-6-luna');
+        expect(ids).toContain('openrouter:openai/gpt-6-luna:free');
+        expect(ids).toContain('gpt-6-luna:free');
+    });
+
    it('coerces string prices and normalizes conditional tiers', async () => {
        const { provider } = makeProvider();
        const models = await provider.models();
@@ -610,6 +639,126 @@ describe('OpenRouterProvider.complete request shape', () => {
    });
 });

+describe('OpenRouterProvider account policy block', () => {
+    const POLICY_MESSAGE =
+        'Policy Violation: this user has been blocked for a previous policy violation. Learn more: https://openrouter.ai/docs/policy (request id: req-secret)';
+
+    // Shaped like the OpenAI SDK's APIError for OpenRouter's error body.
+    const apiError = (status: number, body: Record<string, unknown>) =>
+        Object.assign(new Error(`${status} ${String(body.message)}`), {
+            status,
+            error: body,
+            code: body.code,
+        });
+
+    const complete = () =>
+        withTestActor(() =>
+            makeProvider().provider.complete({
+                model: 'openrouter:openai/gpt-6-luna',
+                messages: [{ role: 'user', content: 'hi' }],
+            }),
+        );
+
+    const expectModelUnavailable = async () => {
+        const err = await complete().catch((e: unknown) => e);
+        expect(err).toMatchObject({
+            name: 'UpstreamResponseError',
+            status: 403,
+            code: 'model_not_found',
+            provider: 'openrouter',
+            message: 'Model not available: openrouter:openai/gpt-6-luna',
+        });
+        expect(JSON.stringify(err)).not.toMatch(/policy|http|request id/i);
+        expect((err as Error).message).not.toMatch(/policy|http/i);
+    };
+
+    it('reports a numeric 403 policy block as the model being unavailable', async () => {
+        createMock.mockRejectedValueOnce(
+            apiError(403, { code: 403, message: POLICY_MESSAGE }),
+        );
+        await expectModelUnavailable();
+        expect(createMock).toHaveBeenCalledTimes(1);
+    });
+
+    it('reports the in-band 403 policy block of a 200 with no choices the same way', async () => {
+        createMock.mockResolvedValueOnce({
+            error: { code: 403, message: POLICY_MESSAGE },
+        });
+        await expectModelUnavailable();
+        expect(recordSpy).not.toHaveBeenCalled();
+    });
+
+    it.each([
+        ['a generic 403', apiError(403, { code: 403, message: 'Forbidden' })],
+        [
+            'a key limit 403',
+            apiError(403, {
+                code: 403,
+                message: 'Key limit exceeded (daily limit).',
+            }),
+        ],
+        [
+            'a moderation 403',
+            apiError(403, {
+                code: 403,
+                message:
+                    'Your chosen model requires moderation and your input was flagged',
+                metadata: { reasons: ['violence'] },
+            }),
+        ],
+        [
+            'a similar 403 wording',
+            apiError(403, {
+                code: 403,
+                message: 'Policy Violation: this key has been blocked.',
+            }),
+        ],
+        [
+            'the policy wording without its numeric code',
+            apiError(403, { message: POLICY_MESSAGE }),
+        ],
+        [
+            'the policy wording on a 401',
+            apiError(401, { code: 401, message: POLICY_MESSAGE }),
+        ],
+        [
+            'the policy wording on a 503',
+            apiError(503, { code: 403, message: POLICY_MESSAGE }),
+        ],
+        [
+            'a 402 credit exhaustion',
+            apiError(402, { code: 402, message: 'Insufficient credits' }),
+        ],
+    ])('rethrows %s unchanged', async (_label, thrown) => {
+        createMock.mockRejectedValueOnce(thrown);
+        await expect(complete()).rejects.toBe(thrown);
+    });
+
+    it.each([
+        ['no error body', {}, 'no completion was returned'],
+        [
+            'an in-band error with another code',
+            { error: { code: 401, message: POLICY_MESSAGE } },
+            POLICY_MESSAGE,
+        ],
+        [
+            'an in-band error with other wording',
+            { error: { code: 403, message: 'Provider returned error' } },
+            'Provider returned error',
+        ],
+    ])(
+        'keeps a no-choice completion with %s as bad_response',
+        async (_label, completion, message) => {
+            createMock.mockResolvedValueOnce(completion);
+            await expect(complete()).rejects.toMatchObject({
+                statusCode: 400,
+                legacyCode: 'bad_response',
+                message,
+            });
+        },
+    );
+});
+
 // ── Non-stream completion: cost calculator branches ─────────────────

 describe('OpenRouterProvider.complete non-stream output', () => {
@@ -19,7 +19,8 @@

 import axios from 'axios';
 import { OpenAI } from 'openai';
-import { HttpError } from '../../../../core/http/HttpError.js';
+import { HttpError, isHttpError } from '../../../../core/http/HttpError.js';
+import { UpstreamResponseError } from '../../../util/upstreamErrors.js';
 import type { MeteringService } from '../../../../services/metering/MeteringService.js';
 import type { IChatModel, ICompleteArguments } from '../../types.js';
 import { cachedRemoteCatalog } from '../../utils/cachedRemoteCatalog.js';
@@ -37,6 +38,29 @@ type OpenrouterUsage = OpenAI.Completions.CompletionUsage & {
    cost?: number;
 };

+const POLICY_BLOCK_MESSAGE =
+    'Policy Violation: this user has been blocked for a previous policy violation.';
+
+const upstreamErrorBody = (err: unknown): unknown => {
+    if (isHttpError(err)) {
+        return err.legacyCode === 'bad_response' ? err.cause : undefined;
+    }
+    const e = err as { status?: unknown; error?: unknown } | undefined;
+    return e?.status === 403 ? e.error : undefined;
+};
+
+const isPolicyBlock = (err: unknown): boolean => {
+    const { code, message } = (upstreamErrorBody(err) ?? {}) as {
+        code?: unknown;
+        message?: unknown;
+    };
+    return (
+        code === 403 &&
+        typeof message === 'string' &&
+        message.startsWith(POLICY_BLOCK_MESSAGE)
+    );
+};
+
 export class OpenRouterProvider extends OpenAICompatProvider {
    #apiBaseUrl: string;

@@ -78,7 +102,17 @@ export class OpenRouterProvider extends OpenAICompatProvider {
                },
            );
        }
-        return super.complete(args, resolved);
+        try {
+            return await super.complete(args, resolved);
+        } catch (e) {
+            if (!isPolicyBlock(e)) throw e;
+            throw new UpstreamResponseError(
+                `Model not available: ${args.model}`,
+                403,
+                'openrouter',
+                'model_not_found',
+            );
+        }
    }

    override async models(): Promise<IChatModel[]> {
@@ -105,7 +139,8 @@ export class OpenRouterProvider extends OpenAICompatProvider {
        for (const model of models) {
            if (
                typeof model.id !== 'string' ||
-                model.id.includes('openrouter/auto')
+                model.id.includes('openrouter/auto') ||
+                model.id.endsWith(':batch')
            ) {
                continue;
            }
@@ -907,7 +907,7 @@ export const handle_completion_output = async (
            typeof upstreamMessage === 'string' && upstreamMessage
                ? upstreamMessage
                : 'no completion was returned',
-            { legacyCode: 'bad_response' },
+            { legacyCode: 'bad_response', cause: completion?.error },
        );
    }

@@ -924,6 +924,27 @@ describe('PuterServer HTTP alarm gate', () => {
                    });
                }) as unknown as RequestHandler,
            },
+            {
+                method: 'get',
+                path: '/model-unavailable',
+                options: {},
+                handler: (() => {
+                    throw new HttpError(400, 'Model not available: m', {
+                        legacyCode: 'upstream_bad_request',
+                        alarmSeverity: 'info',
+                    });
+                }) as unknown as RequestHandler,
+            },
+            {
+                method: 'get',
+                path: '/upstream-bad-request',
+                options: {},
+                handler: (() => {
+                    throw new HttpError(400, 'bad tool schema', {
+                        legacyCode: 'upstream_bad_request',
+                    });
+                }) as unknown as RequestHandler,
+            },
        );
        port = await allocateEphemeralPort();
        server = await setupTestServer(
@@ -1035,6 +1056,25 @@ describe('PuterServer HTTP alarm gate', () => {
            ],
        });
    });
+
+    it("alarms a 4xx at the thrower's alarmSeverity", async () => {
+        const { id, severity } = await raisedFor('/model-unavailable', 400);
+        expect(id).toBe(
+            'http_400:GET:/model-unavailable:upstream_bad_request:Model not available: m',
+        );
+        expect(severity).toBe('info');
+    });
+
+    it('does not alarm an upstream 4xx without one', async () => {
+        const alarm = vi
+            .spyOn(server.clients.alarm, 'create')
+            .mockImplementation(() => undefined);
+        const res = await rawRequest(port, '/upstream-bad-request', {
+            host: 'puter.localhost',
+        });
+        expect(res.status).toBe(400);
+        expect(alarm).not.toHaveBeenCalled();
+    });
 });

 /**
@@ -793,7 +793,9 @@ export class PuterServer {
                    if (isHttp && err.noAlarm) return;
                    const status = isHttp ? err.statusCode : 500;
                    const legacyCode = isHttp ? (err.legacyCode ?? '') : '';
-                    const forcedSeverity = FORCED_ALERT_CODES.get(legacyCode);
+                    const forcedSeverity =
+                        (isHttp ? err.alarmSeverity : undefined) ??
+                        FORCED_ALERT_CODES.get(legacyCode);
                    if (!forcedSeverity) {
                        if (status < 500) return;
                        if (SKIP_ALERT_PREFIXES.test(legacyCode)) return;
@@ -802,7 +804,7 @@ export class PuterServer {
                        ? err instanceof Error
                            ? err.message
                            : String(err)
-                        : status >= 500
+                        : status >= 500 || err.alarmSeverity
                          ? `${err.legacyCode || err.code || 'http'}:${err.message}`
                          : err.legacyCode || err.code || err.message;
                    const routePath =