mirror of
https://github.com/HeyPuter/puter.git
synced 2026-10-14 07:39:40 +00:00
perf(ai-chat): resolve the model once in the driver and keep its map fresh
The driver hands each provider its own catalog entry for the route it chose, so providers stop re-resolving the id per request; the OpenAI-compatible providers only look an id up (indexed, case-insensitively) for a direct caller, and an unknown id is a 400 instead of silently becoming the default model. Gateways with mixed-case ids (Infron, Neuralwatt) used to miss the driver's lowercased id and fall through to that default. The model map is now built from every provider's catalog in parallel, with a failing catalog left out rather than aborting the build, swapped in whole, awaited in onServerStart, and rebuilt every 15 minutes so gateway catalog changes reach routing without a restart.
This commit is contained in:
1 parent
2bcb169a2b
commit
ddff917630
21 files changed
+299
-209
No files matched your search
@@ -93,14 +93,8 @@ const makeDriver = async (config: Record<string, unknown>) => {
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
d.onServerStart();
|
||||
// `onServerStart` kicks off the model map without awaiting it.
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const models = await d.models();
|
||||
if (models.length > 1) return d;
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
}
|
||||
throw new Error('model map never populated');
|
||||
await d.onServerStart();
|
||||
return d;
|
||||
};
|
||||
|
||||
let fullDriver: ChatCompletionDriver;
|
||||
|
||||
@@ -80,13 +80,8 @@ const makeDriver = async () => {
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
d.onServerStart();
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const m = await d.models();
|
||||
if (m.length > 0) return d;
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
}
|
||||
throw new Error('ChatCompletionDriver model map never populated in test');
|
||||
await d.onServerStart();
|
||||
return d;
|
||||
};
|
||||
|
||||
const fakeCatalog = (modalities?: IChatModel['modalities']): IChatModel[] => [
|
||||
|
||||
@@ -66,13 +66,8 @@ const makeDriver = async () => {
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
d.onServerStart();
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const m = await d.models();
|
||||
if (m.length > 0) return d;
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
}
|
||||
throw new Error('ChatCompletionDriver model map never populated in test');
|
||||
await d.onServerStart();
|
||||
return d;
|
||||
};
|
||||
|
||||
const drain = async (stream: Readable): Promise<void> => {
|
||||
|
||||
@@ -49,6 +49,7 @@ import {
|
||||
isRouteUnhealthy,
|
||||
markRouteUnhealthy,
|
||||
} from './utils/providerHealth.js';
|
||||
import { InfronProvider } from './providers/infron/InfronProvider.js';
|
||||
|
||||
// -- OpenAI SDK mock ------------------------------------------------
|
||||
// Gemini reaches Google through `new openai.OpenAI()` (default export) and
|
||||
@@ -104,6 +105,17 @@ const GATEWAY_CATALOG = [
|
||||
min_prompt_price: 0.3,
|
||||
min_completion_price: 2.5,
|
||||
},
|
||||
{
|
||||
// A gateway id with capitals, which the driver's map lowercases.
|
||||
id: 'Qwen/QwQ-Max',
|
||||
display_name: 'Qwen: QwQ Max',
|
||||
category_type: 'LLM',
|
||||
supported_endpoint_types: ['openai'],
|
||||
context_length: 131_072,
|
||||
max_output_tokens: 8_192,
|
||||
min_prompt_price: 1.6,
|
||||
min_completion_price: 6.4,
|
||||
},
|
||||
{
|
||||
// A vendor we integrate with directly that isn't Google — the case
|
||||
// resold duplicates used to be dropped for.
|
||||
@@ -167,19 +179,7 @@ beforeAll(async () => {
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
driver.onServerStart();
|
||||
// `onServerStart` doesn't await `#buildModelMap`, and the gateway
|
||||
// catalogs resolve on a microtask — poll until both gateways land.
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const ids = await driver.list();
|
||||
if (
|
||||
ids.some((id) => id.startsWith('infron:')) &&
|
||||
ids.some((id) => id.startsWith('openrouter:'))
|
||||
) {
|
||||
break;
|
||||
}
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
}
|
||||
await driver.onServerStart();
|
||||
});
|
||||
|
||||
// Every failure in this file marks the route it hit. Without this the first
|
||||
@@ -253,6 +253,58 @@ describe('ChatCompletionDriver gemini routing', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('ChatCompletionDriver hands providers their own catalog entry', () => {
|
||||
it('sends a gateway model under its own casing, not the lowercased map id', async () => {
|
||||
createMock.mockResolvedValueOnce({
|
||||
choices: [
|
||||
{
|
||||
message: { role: 'assistant', content: 'hi' },
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1 },
|
||||
});
|
||||
const record = vi.spyOn(
|
||||
server.services.metering,
|
||||
'utilRecordUsageObject',
|
||||
);
|
||||
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model: 'qwen/qwq-max',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls.at(-1)![0].model).toBe('Qwen/QwQ-Max');
|
||||
expect(record.mock.calls.at(-1)![2]).toBe('infron:Qwen/QwQ-Max');
|
||||
record.mockRestore();
|
||||
});
|
||||
|
||||
it('leaves the provider nothing to look up per request', async () => {
|
||||
createMock.mockResolvedValueOnce({
|
||||
choices: [
|
||||
{
|
||||
message: { role: 'assistant', content: 'hi' },
|
||||
finish_reason: 'stop',
|
||||
},
|
||||
],
|
||||
usage: { prompt_tokens: 1, completion_tokens: 1 },
|
||||
});
|
||||
const catalog = vi.spyOn(InfronProvider.prototype, 'models');
|
||||
|
||||
await withTestActor(() =>
|
||||
driver.complete({
|
||||
model: 'qwen/qwq-max',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(catalog).not.toHaveBeenCalled();
|
||||
catalog.mockRestore();
|
||||
});
|
||||
});
|
||||
|
||||
describe('ChatCompletionDriver muse spark routing', () => {
|
||||
it('serves Muse Spark from Meta, with OpenRouter only as fallback', async () => {
|
||||
const attempts = await attemptsFor('muse-spark-1.2');
|
||||
|
||||
@@ -54,6 +54,7 @@ import { withTestActor } from '../integrationTestUtil.js';
|
||||
import { COUNT_TOKENS, ChatCompletionDriver } from './ChatCompletionDriver.js';
|
||||
import { ClaudeProvider } from './providers/claude/ClaudeProvider.js';
|
||||
import { FakeChatProvider } from './providers/FakeChatProvider.js';
|
||||
import { OllamaChatProvider } from './providers/ollama/OllamaProvider.js';
|
||||
import type {
|
||||
IChatCompleteResult,
|
||||
IChatProvider,
|
||||
@@ -78,17 +79,8 @@ const makeDriver = async () => {
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
d.onServerStart();
|
||||
// `onServerStart` kicks off `#buildModelMap` without awaiting it.
|
||||
// Poll `models()` until the map is populated — production never
|
||||
// serves a request before the kernel has finished booting, but in
|
||||
// tests we hit the driver before microtasks have drained.
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const m = await d.models();
|
||||
if (m.length > 0) return d;
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
}
|
||||
throw new Error('ChatCompletionDriver model map never populated in test');
|
||||
await d.onServerStart();
|
||||
return d;
|
||||
};
|
||||
|
||||
beforeAll(async () => {
|
||||
@@ -184,6 +176,58 @@ describe('ChatCompletionDriver model catalog', () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe('ChatCompletionDriver model map', () => {
|
||||
const newDriver = (providers: Record<string, unknown>) =>
|
||||
new ChatCompletionDriver(
|
||||
{ providers } as never,
|
||||
server.clients,
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
|
||||
it('is built by the time onServerStart resolves', async () => {
|
||||
const d = newDriver({ ollama: { enabled: false } });
|
||||
await d.onServerStart();
|
||||
expect((await d.models()).length).toBeGreaterThan(0);
|
||||
d.onServerShutdown();
|
||||
});
|
||||
|
||||
it('leaves out a provider whose catalog fails instead of failing the build', async () => {
|
||||
vi.spyOn(console, 'warn').mockImplementation(() => {});
|
||||
vi.spyOn(OllamaChatProvider.prototype, 'models').mockRejectedValue(
|
||||
new Error('catalog down'),
|
||||
);
|
||||
const d = newDriver({ ollama: {} });
|
||||
await d.onServerStart();
|
||||
expect((await d.models()).map((m) => m.provider)).toContain(
|
||||
'fake-chat',
|
||||
);
|
||||
d.onServerShutdown();
|
||||
});
|
||||
|
||||
it('rebuilds on a timer, picking up catalog changes', async () => {
|
||||
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
|
||||
try {
|
||||
const real = FakeChatProvider.prototype.models;
|
||||
const models = vi.spyOn(FakeChatProvider.prototype, 'models');
|
||||
const d = newDriver({ ollama: { enabled: false } });
|
||||
await d.onServerStart();
|
||||
expect(await d.list()).not.toContain('added-later');
|
||||
|
||||
models.mockImplementation(async function (this: FakeChatProvider) {
|
||||
const catalog = await real.call(this);
|
||||
return [{ ...catalog[0]!, id: 'added-later' }, ...catalog];
|
||||
});
|
||||
await vi.advanceTimersByTimeAsync(15 * 60 * 1000);
|
||||
|
||||
expect(await d.list()).toContain('added-later');
|
||||
d.onServerShutdown();
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── Auth + model resolution ─────────────────────────────────────────
|
||||
|
||||
describe('ChatCompletionDriver.complete auth and model resolution', () => {
|
||||
@@ -1133,13 +1177,8 @@ describe('ChatCompletionDriver Claude usage.thinking_tokens', () => {
|
||||
server.stores,
|
||||
server.services,
|
||||
);
|
||||
d.onServerStart();
|
||||
for (let i = 0; i < 200; i++) {
|
||||
const m = await d.models();
|
||||
if (m.some((model) => model.id === THINKING_MODEL)) return d;
|
||||
await new Promise((r) => setTimeout(r, 5));
|
||||
}
|
||||
throw new Error('claude driver model map never populated in test');
|
||||
await d.onServerStart();
|
||||
return d;
|
||||
};
|
||||
|
||||
it('adds thinking_tokens to the non-stream wire usage, priced only off the real output', async () => {
|
||||
|
||||
@@ -104,6 +104,7 @@ import {
|
||||
isRouteUnhealthy,
|
||||
markRouteUnhealthy,
|
||||
} from './utils/providerHealth.js';
|
||||
import { CATALOG_TTL_MS } from './utils/cachedRemoteCatalog.js';
|
||||
import { fromFinishReason } from './utils/stopReason.js';
|
||||
import { AIChatStream } from './utils/Streaming.js';
|
||||
import {
|
||||
@@ -491,6 +492,9 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
|
||||
#providers: Record<string, IChatProvider> = Object.create(null);
|
||||
#modelIdMap: Record<string, IChatModel[]> = Object.create(null);
|
||||
/** Each bucket entry's own catalog entry, which keeps the vendor's casing. */
|
||||
#catalogEntries = new WeakMap<IChatModel, IChatModel>();
|
||||
#modelMapRefresh: ReturnType<typeof setInterval> | undefined;
|
||||
|
||||
/** Metering scoped to this driver. Lazy: services wire up after drivers. */
|
||||
get #aiMetering(): AiMeteringService {
|
||||
@@ -501,9 +505,18 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
);
|
||||
}
|
||||
|
||||
override onServerStart() {
|
||||
override async onServerStart() {
|
||||
this.#registerProviders();
|
||||
this.#buildModelMap();
|
||||
await this.#buildModelMap();
|
||||
// Gateway catalogs change under us; rebuild once they've expired.
|
||||
this.#modelMapRefresh = setInterval(() => {
|
||||
void this.#buildModelMap();
|
||||
}, CATALOG_TTL_MS);
|
||||
this.#modelMapRefresh.unref?.();
|
||||
}
|
||||
|
||||
override onServerShutdown() {
|
||||
clearInterval(this.#modelMapRefresh);
|
||||
}
|
||||
|
||||
// -- Interface methods -------------------------------------------
|
||||
@@ -745,11 +758,14 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
if (!useFakeProvider) {
|
||||
await this.#resolvePuterPaths(attemptProvider, args, actor);
|
||||
}
|
||||
return attemptProvider.complete({
|
||||
...args,
|
||||
model: attemptModel.id,
|
||||
provider: attemptModel.provider,
|
||||
});
|
||||
return attemptProvider.complete(
|
||||
{
|
||||
...args,
|
||||
model: attemptModel.id,
|
||||
provider: attemptModel.provider,
|
||||
},
|
||||
this.#catalogEntries.get(attemptModel),
|
||||
);
|
||||
};
|
||||
|
||||
try {
|
||||
@@ -1831,12 +1847,28 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
*
|
||||
* Entries join a bucket by identity key (see `isIdentityKey`); display
|
||||
* names remain addressable but never merge two providers' entries.
|
||||
*
|
||||
* Built into a fresh map that replaces the old one whole, so a refresh
|
||||
* never serves a half-built map. A provider whose catalog fails is left out
|
||||
* of this build rather than failing it.
|
||||
*/
|
||||
async #buildModelMap() {
|
||||
for (const providerName in this.#providers) {
|
||||
const provider = this.#providers[providerName];
|
||||
const names = Object.keys(this.#providers);
|
||||
const catalogs = await Promise.allSettled(
|
||||
names.map(async (name) => this.#providers[name].models()),
|
||||
);
|
||||
const modelIdMap: Record<string, IChatModel[]> = Object.create(null);
|
||||
|
||||
for (const entry of await provider.models()) {
|
||||
names.forEach((providerName, i) => {
|
||||
const catalog = catalogs[i]!;
|
||||
if (catalog.status === 'rejected') {
|
||||
console.warn(
|
||||
`[ai-chat] ${providerName} catalog unavailable: ${(catalog.reason as Error)?.message ?? catalog.reason}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const entry of catalog.value) {
|
||||
// Catalogs are module-level constants shared by every driver
|
||||
// instance, so they are read and never written: normalizing
|
||||
// the id or appending puterId in place would accumulate across
|
||||
@@ -1874,19 +1906,23 @@ export class ChatCompletionDriver extends PuterDriver {
|
||||
const bucket =
|
||||
keys
|
||||
.filter(isIdentityKey)
|
||||
.map((key) => this.#modelIdMap[key])
|
||||
.map((key) => modelIdMap[key])
|
||||
.find(Boolean) ?? [];
|
||||
bucket.push({ ...model, provider: providerName });
|
||||
const routed = { ...model, provider: providerName };
|
||||
this.#catalogEntries.set(routed, entry);
|
||||
bucket.push(routed);
|
||||
|
||||
// First registration owns a key: a name already claimed by
|
||||
// another model keeps pointing where it did.
|
||||
for (const key of keys) {
|
||||
this.#modelIdMap[key] ??= bucket;
|
||||
modelIdMap[key] ??= bucket;
|
||||
}
|
||||
|
||||
bucket.sort(compareModelPreference);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
this.#modelIdMap = modelIdMap;
|
||||
}
|
||||
|
||||
#resolveModel(modelId: string, provider?: string): IChatModel | null {
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
import type { CompletionUsage } from 'openai/resources/completions.mjs';
|
||||
import type { Actor } from '../../../core/actor.js';
|
||||
import { Context } from '../../../core/context.js';
|
||||
import { HttpError } from '../../../core/http/HttpError.js';
|
||||
import type { MeteringService } from '../../../services/metering/MeteringService.js';
|
||||
import type {
|
||||
IChatCompleteResult,
|
||||
@@ -162,9 +163,12 @@ export class OpenAICompatProvider implements IChatProvider {
|
||||
throw new Error('Method not implemented.');
|
||||
}
|
||||
|
||||
async complete(args: ICompleteArguments): Promise<IChatCompleteResult> {
|
||||
async complete(
|
||||
args: ICompleteArguments,
|
||||
resolved?: IChatModel,
|
||||
): Promise<IChatCompleteResult> {
|
||||
const options = this.#options;
|
||||
const model = await this.resolveModel(args.model);
|
||||
const model = resolved ?? (await this.resolveModel(args.model));
|
||||
const actor = Context.get('actor');
|
||||
const signal = Context.get('abortSignal');
|
||||
|
||||
@@ -271,7 +275,10 @@ export class OpenAICompatProvider implements IChatProvider {
|
||||
});
|
||||
}
|
||||
|
||||
/** The catalog entry for a requested id or alias, case-insensitively. */
|
||||
/**
|
||||
* The catalog entry for a requested id or alias, case-insensitively, for a
|
||||
* caller that didn't pass the one the driver resolved.
|
||||
*/
|
||||
protected async resolveModel(requested: string): Promise<IChatModel> {
|
||||
const models = await this.models();
|
||||
let index = this.#index.get(models);
|
||||
@@ -285,11 +292,11 @@ export class OpenAICompatProvider implements IChatProvider {
|
||||
}
|
||||
this.#index.set(models, index);
|
||||
}
|
||||
const model =
|
||||
index.get(normalizeModelKey(requested ?? '')) ??
|
||||
index.get(normalizeModelKey(this.getDefaultModel()));
|
||||
const model = index.get(normalizeModelKey(requested ?? ''));
|
||||
if (!model) {
|
||||
throw new Error(`Model not found: ${requested}`);
|
||||
throw new HttpError(400, `Model not found: ${requested}`, {
|
||||
legacyCode: 'bad_request',
|
||||
});
|
||||
}
|
||||
return model;
|
||||
}
|
||||
|
||||
@@ -476,20 +476,18 @@ describe('BytePlusProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe(
|
||||
'seed-2-0-lite-260428',
|
||||
);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -465,24 +465,18 @@ describe('DeepSeekProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe('deepseek-flash');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
'deepseek:deepseek-flash',
|
||||
expect.any(Object),
|
||||
);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
// DeepSeek discontinued the chat/reasoner names on 2026-07-24.
|
||||
|
||||
@@ -393,18 +393,18 @@ describe('GeminiChatProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe('gemini-2.5-flash');
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each([
|
||||
|
||||
@@ -367,24 +367,18 @@ describe('GroqAIProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe('openai/gpt-oss-20b');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
'groq:openai/gpt-oss-20b',
|
||||
expect.any(Object),
|
||||
);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -498,26 +498,18 @@ describe('HoonifyProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe(
|
||||
'google/gemma-4-31B-it',
|
||||
);
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
'hoonify:google/gemma-4-31b-it',
|
||||
expect.any(Object),
|
||||
);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -498,17 +498,18 @@ describe('MetaProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
createMock.mockResolvedValueOnce(OK_COMPLETION);
|
||||
await complete(makeProvider(), { model: 'totally-not-a-real-model' });
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const provider = makeProvider();
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe('muse-spark-1.2');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
'meta:muse-spark-1.2',
|
||||
expect.any(Object),
|
||||
);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ export class MetaProvider extends OpenAICompatProvider {
|
||||
this.#fsService = fsService;
|
||||
}
|
||||
|
||||
override async complete(args: ICompleteArguments) {
|
||||
override async complete(args: ICompleteArguments, resolved?: IChatModel) {
|
||||
if (!Array.isArray(args.messages)) {
|
||||
throw new HttpError(400, '`messages` must be an array', {
|
||||
legacyCode: 'bad_request',
|
||||
@@ -107,7 +107,7 @@ export class MetaProvider extends OpenAICompatProvider {
|
||||
this.#fsService,
|
||||
Context.get('actor'),
|
||||
);
|
||||
return super.complete(args);
|
||||
return super.complete(args, resolved);
|
||||
}
|
||||
|
||||
protected override vendorParams(
|
||||
|
||||
@@ -474,24 +474,18 @@ describe('MoonshotProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe('kimi-k2.6');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
'moonshotai:kimi-k2.6',
|
||||
expect.any(Object),
|
||||
);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -154,8 +154,8 @@ export class NeuralwattProvider extends OpenAICompatProvider {
|
||||
},
|
||||
});
|
||||
|
||||
override async complete(args: ICompleteArguments) {
|
||||
const model = await this.resolveModel(args.model);
|
||||
override async complete(args: ICompleteArguments, resolved?: IChatModel) {
|
||||
const model = resolved ?? (await this.resolveModel(args.model));
|
||||
if (
|
||||
messagesHaveImageContent(args.messages ?? []) &&
|
||||
!modelSupportsVision(model)
|
||||
@@ -167,7 +167,7 @@ export class NeuralwattProvider extends OpenAICompatProvider {
|
||||
);
|
||||
}
|
||||
this.#accountingMethod = await this.getAccountingMethod();
|
||||
return super.complete(args);
|
||||
return super.complete(args, model);
|
||||
}
|
||||
|
||||
protected override vendorParams(
|
||||
|
||||
@@ -438,7 +438,7 @@ describe('OpenRouterProvider.complete request shape', () => {
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'openrouter:anthropic/claude-sonnet',
|
||||
model: 'openrouter:anthropic/claude-haiku-4.5',
|
||||
messages: [
|
||||
{
|
||||
role: 'user',
|
||||
|
||||
@@ -72,7 +72,7 @@ export class OpenRouterProvider extends OpenAICompatProvider {
|
||||
this.#apiBaseUrl = apiBaseUrl;
|
||||
}
|
||||
|
||||
override async complete(args: ICompleteArguments) {
|
||||
override async complete(args: ICompleteArguments, resolved?: IChatModel) {
|
||||
if (args.model === 'openrouter/auto') {
|
||||
throw new HttpError(
|
||||
400,
|
||||
@@ -87,7 +87,7 @@ export class OpenRouterProvider extends OpenAICompatProvider {
|
||||
},
|
||||
);
|
||||
}
|
||||
return super.complete(args);
|
||||
return super.complete(args, resolved);
|
||||
}
|
||||
|
||||
override async models(): Promise<IChatModel[]> {
|
||||
|
||||
@@ -613,20 +613,18 @@ describe('TogetherAIProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe(
|
||||
'meta-llama/Llama-3.3-70B-Instruct-Turbo',
|
||||
);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -539,24 +539,18 @@ describe('ZAIProvider model resolution', () => {
|
||||
);
|
||||
});
|
||||
|
||||
it('falls back to the default model when given an unknown id', async () => {
|
||||
it('rejects an unknown id instead of substituting the default', async () => {
|
||||
const { provider } = makeProvider();
|
||||
createMock.mockResolvedValueOnce(baseCompletion);
|
||||
|
||||
await withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
);
|
||||
|
||||
expect(createMock.mock.calls[0]![0].model).toBe('glm-5.1');
|
||||
expect(recordSpy).toHaveBeenCalledWith(
|
||||
expect.any(Object),
|
||||
expect.anything(),
|
||||
'zai:glm-5.1',
|
||||
expect.any(Object),
|
||||
);
|
||||
await expect(
|
||||
withTestActor(() =>
|
||||
provider.complete({
|
||||
model: 'totally-not-a-real-model',
|
||||
messages: [{ role: 'user', content: 'hi' }],
|
||||
}),
|
||||
),
|
||||
).rejects.toMatchObject({ statusCode: 400 });
|
||||
expect(createMock).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -293,7 +293,14 @@ export interface IChatProvider {
|
||||
models(extra_params?: unknown): IChatModel[] | Promise<IChatModel[]>;
|
||||
list(): string[] | Promise<string[]>;
|
||||
getDefaultModel(): string;
|
||||
complete(arg: ICompleteArguments): Promise<IChatCompleteResult>;
|
||||
/**
|
||||
* `model` is this provider's own catalog entry for `arg.model`, as the
|
||||
* driver resolved it; a provider resolves `arg.model` itself without it.
|
||||
*/
|
||||
complete(
|
||||
arg: ICompleteArguments,
|
||||
model?: IChatModel,
|
||||
): Promise<IChatCompleteResult>;
|
||||
checkModeration(
|
||||
text: string,
|
||||
): Promise<{ flagged: boolean; categories?: string[] }> | void;
|
||||
|
||||
Reference in new issue
Block a user