mirror of
https://github.com/HeyPuter/puter.git
synced 2026-10-14 07:39:40 +00:00
fix(ai): retry a fast-mode 429 at standard speed on Claude
Fast mode has its own quota. Its 429 used to count as a route failure, so the request was replayed on a reseller and the Claude route for that model was marked unhealthy. The provider now retries once without speed.
This commit is contained in:
1 parent
c2fc45a8d1
commit
f21cd51edf
2 files changed
+72
No files matched your search
@@ -2019,6 +2019,51 @@ describe('ClaudeProvider metering', () => {
|
||||
expect('fast_input_tokens' in result.usage).toBe(false);
|
||||
});
|
||||
|
||||
it('retries at standard speed on Claude when fast mode is rate limited', async () => {
|
||||
const { provider } = makeProvider();
|
||||
const fastLimited = Object.assign(
|
||||
new Error(
|
||||
"This request would exceed your organization's rate limit of 0 fast mode input tokens per minute",
|
||||
),
|
||||
{ status: 429 },
|
||||
);
|
||||
messagesCreateMock.mockRejectedValueOnce(fastLimited).mockResolvedValueOnce({
|
||||
content: [{ type: 'text', text: 'hi' }],
|
||||
usage: { input_tokens: 10, output_tokens: 5, speed: 'standard' },
|
||||
});
|
||||
|
||||
const result = (await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'claude-opus-5-5',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
speed: 'fast',
|
||||
}),
|
||||
)) as { usage: Record<string, number> };
|
||||
|
||||
expect(messagesCreateMock).toHaveBeenCalledTimes(2);
|
||||
expect(messagesCreateMock.mock.calls[0]![0].speed).toBe('fast');
|
||||
expect('speed' in messagesCreateMock.mock.calls[1]![0]).toBe(false);
|
||||
expect(result.usage.input_tokens).toBe(10);
|
||||
expect('fast_input_tokens' in result.usage).toBe(false);
|
||||
});
|
||||
|
||||
it('does not retry an ordinary 429 on a fast-mode request', async () => {
|
||||
const { provider } = makeProvider();
|
||||
const limited = Object.assign(new Error('rate limited'), { status: 429 });
|
||||
messagesCreateMock.mockRejectedValueOnce(limited);
|
||||
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'claude-opus-5-5',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
speed: 'fast',
|
||||
}),
|
||||
),
|
||||
).rejects.toBe(limited);
|
||||
expect(messagesCreateMock).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it('meters web_search_requests at 1,000,000 µ¢ per request', async () => {
|
||||
const { provider } = makeProvider();
|
||||
messagesCreateMock.mockResolvedValueOnce({
|
||||
|
||||
@@ -108,6 +108,16 @@ const toAnthropicMediaPart = (part: any): any => {
|
||||
return part;
|
||||
};
|
||||
|
||||
/** A 429 from fast mode's separate quota, not the model's normal limits. */
|
||||
const isFastModeRateLimit = (e: unknown): boolean => {
|
||||
const err = e as { status?: unknown; message?: unknown };
|
||||
return (
|
||||
err?.status === 429 &&
|
||||
typeof err.message === 'string' &&
|
||||
/fast mode/i.test(err.message)
|
||||
);
|
||||
};
|
||||
|
||||
// Models whose current turn rejects a non-default temperature/top_p/top_k
|
||||
// outright. Fable 5/5.1, Sonnet 5/5.5, and Opus 4.7+.
|
||||
const OMITS_SAMPLING_PARAMS = new Set([
|
||||
@@ -354,6 +364,23 @@ export class ClaudeProvider implements IChatProvider {
|
||||
}
|
||||
|
||||
async complete(args: ICompleteArguments): Promise<IChatCompleteResult> {
|
||||
try {
|
||||
return await this.#completeOnce(args);
|
||||
} catch (e) {
|
||||
// Fast mode has its own quota. Anthropic's advice on that 429 is to
|
||||
// drop `speed` and run at standard speed; doing it here keeps the
|
||||
// request on Claude instead of marking the route unhealthy and
|
||||
// replaying it on a reseller.
|
||||
if (args.speed === 'fast' && isFastModeRateLimit(e)) {
|
||||
return await this.#completeOnce({ ...args, speed: undefined });
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async #completeOnce(
|
||||
args: ICompleteArguments,
|
||||
): Promise<IChatCompleteResult> {
|
||||
const {
|
||||
sdkParams,
|
||||
usesBeta,
|
||||
|
||||
Reference in new issue
Block a user