fix(ai): retry a fast-mode 429 at standard speed on Claude

Fast mode has its own quota. Its 429 used to count as a route failure,
so the request was replayed on a reseller and the Claude route for that
model was marked unhealthy. The provider now retries once without speed.
This commit is contained in:
Daniel Salazar committed 2026-10-05 20:54:01 -07:00
1 parent c2fc45a8d1
commit f21cd51edf
2 files changed
+72

No files matched your search

@@ -2019,6 +2019,51 @@ describe('ClaudeProvider metering', () => {
expect('fast_input_tokens' in result.usage).toBe(false);
});
it('retries at standard speed on Claude when fast mode is rate limited', async () => {
const { provider } = makeProvider();
const fastLimited = Object.assign(
new Error(
"This request would exceed your organization's rate limit of 0 fast mode input tokens per minute",
),
{ status: 429 },
);
messagesCreateMock.mockRejectedValueOnce(fastLimited).mockResolvedValueOnce({
content: [{ type: 'text', text: 'hi' }],
usage: { input_tokens: 10, output_tokens: 5, speed: 'standard' },
});
const result = (await withTestActor(() =>
provider.complete({
model: 'claude-opus-5-5',
messages: [{ role: 'user', content: 'hi' }],
speed: 'fast',
}),
)) as { usage: Record<string, number> };
expect(messagesCreateMock).toHaveBeenCalledTimes(2);
expect(messagesCreateMock.mock.calls[0]![0].speed).toBe('fast');
expect('speed' in messagesCreateMock.mock.calls[1]![0]).toBe(false);
expect(result.usage.input_tokens).toBe(10);
expect('fast_input_tokens' in result.usage).toBe(false);
});
it('does not retry an ordinary 429 on a fast-mode request', async () => {
const { provider } = makeProvider();
const limited = Object.assign(new Error('rate limited'), { status: 429 });
messagesCreateMock.mockRejectedValueOnce(limited);
await expect(
withTestActor(() =>
provider.complete({
model: 'claude-opus-5-5',
messages: [{ role: 'user', content: 'hi' }],
speed: 'fast',
}),
),
).rejects.toBe(limited);
expect(messagesCreateMock).toHaveBeenCalledTimes(1);
});
it('meters web_search_requests at 1,000,000 µ¢ per request', async () => {
const { provider } = makeProvider();
messagesCreateMock.mockResolvedValueOnce({
@@ -108,6 +108,16 @@ const toAnthropicMediaPart = (part: any): any => {
return part;
};
/** A 429 from fast mode's separate quota, not the model's normal limits. */
const isFastModeRateLimit = (e: unknown): boolean => {
const err = e as { status?: unknown; message?: unknown };
return (
err?.status === 429 &&
typeof err.message === 'string' &&
/fast mode/i.test(err.message)
);
};
// Models whose current turn rejects a non-default temperature/top_p/top_k
// outright. Fable 5/5.1, Sonnet 5/5.5, and Opus 4.7+.
const OMITS_SAMPLING_PARAMS = new Set([
@@ -354,6 +364,23 @@ export class ClaudeProvider implements IChatProvider {
}
async complete(args: ICompleteArguments): Promise<IChatCompleteResult> {
try {
return await this.#completeOnce(args);
} catch (e) {
// Fast mode has its own quota. Anthropic's advice on that 429 is to
// drop `speed` and run at standard speed; doing it here keeps the
// request on Claude instead of marking the route unhealthy and
// replaying it on a reseller.
if (args.speed === 'fast' && isFastModeRateLimit(e)) {
return await this.#completeOnce({ ...args, speed: undefined });
}
throw e;
}
}
async #completeOnce(
args: ICompleteArguments,
): Promise<IChatCompleteResult> {
const {
sdkParams,
usesBeta,