perf(ai-chat): resolve the model once in the driver and keep its map fresh

The driver hands each provider its own catalog entry for the route it chose,
so providers stop re-resolving the id per request; the OpenAI-compatible
providers only look an id up (indexed, case-insensitively) for a direct
caller, and an unknown id is a 400 instead of silently becoming the default
model. Gateways with mixed-case ids (Infron, Neuralwatt) used to miss the
driver's lowercased id and fall through to that default.

The model map is now built from every provider's catalog in parallel, with a
failing catalog left out rather than aborting the build, swapped in whole,
awaited in onServerStart, and rebuilt every 15 minutes so gateway catalog
changes reach routing without a restart.
This commit is contained in:
Daniel Salazar committed 2026-10-10 20:01:50 -04:00
1 parent 2bcb169a2b
commit ddff917630
21 files changed
+299 -209

No files matched your search

@@ -93,14 +93,8 @@ const makeDriver = async (config: Record<string, unknown>) => {
server.stores,
server.services,
);
d.onServerStart();
// `onServerStart` kicks off the model map without awaiting it.
for (let i = 0; i < 200; i++) {
const models = await d.models();
if (models.length > 1) return d;
await new Promise((r) => setTimeout(r, 5));
}
throw new Error('model map never populated');
await d.onServerStart();
return d;
};
let fullDriver: ChatCompletionDriver;
@@ -80,13 +80,8 @@ const makeDriver = async () => {
server.stores,
server.services,
);
d.onServerStart();
for (let i = 0; i < 200; i++) {
const m = await d.models();
if (m.length > 0) return d;
await new Promise((r) => setTimeout(r, 5));
}
throw new Error('ChatCompletionDriver model map never populated in test');
await d.onServerStart();
return d;
};
const fakeCatalog = (modalities?: IChatModel['modalities']): IChatModel[] => [
@@ -66,13 +66,8 @@ const makeDriver = async () => {
server.stores,
server.services,
);
d.onServerStart();
for (let i = 0; i < 200; i++) {
const m = await d.models();
if (m.length > 0) return d;
await new Promise((r) => setTimeout(r, 5));
}
throw new Error('ChatCompletionDriver model map never populated in test');
await d.onServerStart();
return d;
};
const drain = async (stream: Readable): Promise<void> => {
@@ -49,6 +49,7 @@ import {
isRouteUnhealthy,
markRouteUnhealthy,
} from './utils/providerHealth.js';
import { InfronProvider } from './providers/infron/InfronProvider.js';
// -- OpenAI SDK mock ------------------------------------------------
// Gemini reaches Google through `new openai.OpenAI()` (default export) and
@@ -104,6 +105,17 @@ const GATEWAY_CATALOG = [
min_prompt_price: 0.3,
min_completion_price: 2.5,
},
{
// A gateway id with capitals, which the driver's map lowercases.
id: 'Qwen/QwQ-Max',
display_name: 'Qwen: QwQ Max',
category_type: 'LLM',
supported_endpoint_types: ['openai'],
context_length: 131_072,
max_output_tokens: 8_192,
min_prompt_price: 1.6,
min_completion_price: 6.4,
},
{
// A vendor we integrate with directly that isn't Google — the case
// resold duplicates used to be dropped for.
@@ -167,19 +179,7 @@ beforeAll(async () => {
server.stores,
server.services,
);
driver.onServerStart();
// `onServerStart` doesn't await `#buildModelMap`, and the gateway
// catalogs resolve on a microtask — poll until both gateways land.
for (let i = 0; i < 200; i++) {
const ids = await driver.list();
if (
ids.some((id) => id.startsWith('infron:')) &&
ids.some((id) => id.startsWith('openrouter:'))
) {
break;
}
await new Promise((r) => setTimeout(r, 5));
}
await driver.onServerStart();
});
// Every failure in this file marks the route it hit. Without this the first
@@ -253,6 +253,58 @@ describe('ChatCompletionDriver gemini routing', () => {
});
});
describe('ChatCompletionDriver hands providers their own catalog entry', () => {
it('sends a gateway model under its own casing, not the lowercased map id', async () => {
createMock.mockResolvedValueOnce({
choices: [
{
message: { role: 'assistant', content: 'hi' },
finish_reason: 'stop',
},
],
usage: { prompt_tokens: 1, completion_tokens: 1 },
});
const record = vi.spyOn(
server.services.metering,
'utilRecordUsageObject',
);
await withTestActor(() =>
driver.complete({
model: 'qwen/qwq-max',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls.at(-1)![0].model).toBe('Qwen/QwQ-Max');
expect(record.mock.calls.at(-1)![2]).toBe('infron:Qwen/QwQ-Max');
record.mockRestore();
});
it('leaves the provider nothing to look up per request', async () => {
createMock.mockResolvedValueOnce({
choices: [
{
message: { role: 'assistant', content: 'hi' },
finish_reason: 'stop',
},
],
usage: { prompt_tokens: 1, completion_tokens: 1 },
});
const catalog = vi.spyOn(InfronProvider.prototype, 'models');
await withTestActor(() =>
driver.complete({
model: 'qwen/qwq-max',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(catalog).not.toHaveBeenCalled();
catalog.mockRestore();
});
});
describe('ChatCompletionDriver muse spark routing', () => {
it('serves Muse Spark from Meta, with OpenRouter only as fallback', async () => {
const attempts = await attemptsFor('muse-spark-1.2');
@@ -54,6 +54,7 @@ import { withTestActor } from '../integrationTestUtil.js';
import { COUNT_TOKENS, ChatCompletionDriver } from './ChatCompletionDriver.js';
import { ClaudeProvider } from './providers/claude/ClaudeProvider.js';
import { FakeChatProvider } from './providers/FakeChatProvider.js';
import { OllamaChatProvider } from './providers/ollama/OllamaProvider.js';
import type {
IChatCompleteResult,
IChatProvider,
@@ -78,17 +79,8 @@ const makeDriver = async () => {
server.stores,
server.services,
);
d.onServerStart();
// `onServerStart` kicks off `#buildModelMap` without awaiting it.
// Poll `models()` until the map is populated — production never
// serves a request before the kernel has finished booting, but in
// tests we hit the driver before microtasks have drained.
for (let i = 0; i < 200; i++) {
const m = await d.models();
if (m.length > 0) return d;
await new Promise((r) => setTimeout(r, 5));
}
throw new Error('ChatCompletionDriver model map never populated in test');
await d.onServerStart();
return d;
};
beforeAll(async () => {
@@ -184,6 +176,58 @@ describe('ChatCompletionDriver model catalog', () => {
});
});
describe('ChatCompletionDriver model map', () => {
const newDriver = (providers: Record<string, unknown>) =>
new ChatCompletionDriver(
{ providers } as never,
server.clients,
server.stores,
server.services,
);
it('is built by the time onServerStart resolves', async () => {
const d = newDriver({ ollama: { enabled: false } });
await d.onServerStart();
expect((await d.models()).length).toBeGreaterThan(0);
d.onServerShutdown();
});
it('leaves out a provider whose catalog fails instead of failing the build', async () => {
vi.spyOn(console, 'warn').mockImplementation(() => {});
vi.spyOn(OllamaChatProvider.prototype, 'models').mockRejectedValue(
new Error('catalog down'),
);
const d = newDriver({ ollama: {} });
await d.onServerStart();
expect((await d.models()).map((m) => m.provider)).toContain(
'fake-chat',
);
d.onServerShutdown();
});
it('rebuilds on a timer, picking up catalog changes', async () => {
vi.useFakeTimers({ toFake: ['setInterval', 'clearInterval'] });
try {
const real = FakeChatProvider.prototype.models;
const models = vi.spyOn(FakeChatProvider.prototype, 'models');
const d = newDriver({ ollama: { enabled: false } });
await d.onServerStart();
expect(await d.list()).not.toContain('added-later');
models.mockImplementation(async function (this: FakeChatProvider) {
const catalog = await real.call(this);
return [{ ...catalog[0]!, id: 'added-later' }, ...catalog];
});
await vi.advanceTimersByTimeAsync(15 * 60 * 1000);
expect(await d.list()).toContain('added-later');
d.onServerShutdown();
} finally {
vi.useRealTimers();
}
});
});
// ── Auth + model resolution ─────────────────────────────────────────
describe('ChatCompletionDriver.complete auth and model resolution', () => {
@@ -1133,13 +1177,8 @@ describe('ChatCompletionDriver Claude usage.thinking_tokens', () => {
server.stores,
server.services,
);
d.onServerStart();
for (let i = 0; i < 200; i++) {
const m = await d.models();
if (m.some((model) => model.id === THINKING_MODEL)) return d;
await new Promise((r) => setTimeout(r, 5));
}
throw new Error('claude driver model map never populated in test');
await d.onServerStart();
return d;
};
it('adds thinking_tokens to the non-stream wire usage, priced only off the real output', async () => {
@@ -104,6 +104,7 @@ import {
isRouteUnhealthy,
markRouteUnhealthy,
} from './utils/providerHealth.js';
import { CATALOG_TTL_MS } from './utils/cachedRemoteCatalog.js';
import { fromFinishReason } from './utils/stopReason.js';
import { AIChatStream } from './utils/Streaming.js';
import {
@@ -491,6 +492,9 @@ export class ChatCompletionDriver extends PuterDriver {
#providers: Record<string, IChatProvider> = Object.create(null);
#modelIdMap: Record<string, IChatModel[]> = Object.create(null);
/** Each bucket entry's own catalog entry, which keeps the vendor's casing. */
#catalogEntries = new WeakMap<IChatModel, IChatModel>();
#modelMapRefresh: ReturnType<typeof setInterval> | undefined;
/** Metering scoped to this driver. Lazy: services wire up after drivers. */
get #aiMetering(): AiMeteringService {
@@ -501,9 +505,18 @@ export class ChatCompletionDriver extends PuterDriver {
);
}
override onServerStart() {
override async onServerStart() {
this.#registerProviders();
this.#buildModelMap();
await this.#buildModelMap();
// Gateway catalogs change under us; rebuild once they've expired.
this.#modelMapRefresh = setInterval(() => {
void this.#buildModelMap();
}, CATALOG_TTL_MS);
this.#modelMapRefresh.unref?.();
}
override onServerShutdown() {
clearInterval(this.#modelMapRefresh);
}
// -- Interface methods -------------------------------------------
@@ -745,11 +758,14 @@ export class ChatCompletionDriver extends PuterDriver {
if (!useFakeProvider) {
await this.#resolvePuterPaths(attemptProvider, args, actor);
}
return attemptProvider.complete({
...args,
model: attemptModel.id,
provider: attemptModel.provider,
});
return attemptProvider.complete(
{
...args,
model: attemptModel.id,
provider: attemptModel.provider,
},
this.#catalogEntries.get(attemptModel),
);
};
try {
@@ -1831,12 +1847,28 @@ export class ChatCompletionDriver extends PuterDriver {
*
* Entries join a bucket by identity key (see `isIdentityKey`); display
* names remain addressable but never merge two providers' entries.
*
* Built into a fresh map that replaces the old one whole, so a refresh
* never serves a half-built map. A provider whose catalog fails is left out
* of this build rather than failing it.
*/
async #buildModelMap() {
for (const providerName in this.#providers) {
const provider = this.#providers[providerName];
const names = Object.keys(this.#providers);
const catalogs = await Promise.allSettled(
names.map(async (name) => this.#providers[name].models()),
);
const modelIdMap: Record<string, IChatModel[]> = Object.create(null);
for (const entry of await provider.models()) {
names.forEach((providerName, i) => {
const catalog = catalogs[i]!;
if (catalog.status === 'rejected') {
console.warn(
`[ai-chat] ${providerName} catalog unavailable: ${(catalog.reason as Error)?.message ?? catalog.reason}`,
);
return;
}
for (const entry of catalog.value) {
// Catalogs are module-level constants shared by every driver
// instance, so they are read and never written: normalizing
// the id or appending puterId in place would accumulate across
@@ -1874,19 +1906,23 @@ export class ChatCompletionDriver extends PuterDriver {
const bucket =
keys
.filter(isIdentityKey)
.map((key) => this.#modelIdMap[key])
.map((key) => modelIdMap[key])
.find(Boolean) ?? [];
bucket.push({ ...model, provider: providerName });
const routed = { ...model, provider: providerName };
this.#catalogEntries.set(routed, entry);
bucket.push(routed);
// First registration owns a key: a name already claimed by
// another model keeps pointing where it did.
for (const key of keys) {
this.#modelIdMap[key] ??= bucket;
modelIdMap[key] ??= bucket;
}
bucket.sort(compareModelPreference);
}
}
});
this.#modelIdMap = modelIdMap;
}
#resolveModel(modelId: string, provider?: string): IChatModel | null {
@@ -20,6 +20,7 @@
import type { CompletionUsage } from 'openai/resources/completions.mjs';
import type { Actor } from '../../../core/actor.js';
import { Context } from '../../../core/context.js';
import { HttpError } from '../../../core/http/HttpError.js';
import type { MeteringService } from '../../../services/metering/MeteringService.js';
import type {
IChatCompleteResult,
@@ -162,9 +163,12 @@ export class OpenAICompatProvider implements IChatProvider {
throw new Error('Method not implemented.');
}
async complete(args: ICompleteArguments): Promise<IChatCompleteResult> {
async complete(
args: ICompleteArguments,
resolved?: IChatModel,
): Promise<IChatCompleteResult> {
const options = this.#options;
const model = await this.resolveModel(args.model);
const model = resolved ?? (await this.resolveModel(args.model));
const actor = Context.get('actor');
const signal = Context.get('abortSignal');
@@ -271,7 +275,10 @@ export class OpenAICompatProvider implements IChatProvider {
});
}
/** The catalog entry for a requested id or alias, case-insensitively. */
/**
* The catalog entry for a requested id or alias, case-insensitively, for a
* caller that didn't pass the one the driver resolved.
*/
protected async resolveModel(requested: string): Promise<IChatModel> {
const models = await this.models();
let index = this.#index.get(models);
@@ -285,11 +292,11 @@ export class OpenAICompatProvider implements IChatProvider {
}
this.#index.set(models, index);
}
const model =
index.get(normalizeModelKey(requested ?? '')) ??
index.get(normalizeModelKey(this.getDefaultModel()));
const model = index.get(normalizeModelKey(requested ?? ''));
if (!model) {
throw new Error(`Model not found: ${requested}`);
throw new HttpError(400, `Model not found: ${requested}`, {
legacyCode: 'bad_request',
});
}
return model;
}
@@ -476,20 +476,18 @@ describe('BytePlusProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
it('rejects an unknown id instead of substituting the default', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls[0]![0].model).toBe(
'seed-2-0-lite-260428',
);
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
});
@@ -465,24 +465,18 @@ describe('DeepSeekProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
it('rejects an unknown id instead of substituting the default', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls[0]![0].model).toBe('deepseek-flash');
expect(recordSpy).toHaveBeenCalledWith(
expect.any(Object),
expect.anything(),
'deepseek:deepseek-flash',
expect.any(Object),
);
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
// DeepSeek discontinued the chat/reasoner names on 2026-07-24.
@@ -393,18 +393,18 @@ describe('GeminiChatProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
it('rejects an unknown id instead of substituting the default', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls[0]![0].model).toBe('gemini-2.5-flash');
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
it.each([
@@ -367,24 +367,18 @@ describe('GroqAIProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
it('rejects an unknown id instead of substituting the default', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls[0]![0].model).toBe('openai/gpt-oss-20b');
expect(recordSpy).toHaveBeenCalledWith(
expect.any(Object),
expect.anything(),
'groq:openai/gpt-oss-20b',
expect.any(Object),
);
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
});
@@ -498,26 +498,18 @@ describe('HoonifyProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
it('rejects an unknown id instead of substituting the default', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls[0]![0].model).toBe(
'google/gemma-4-31B-it',
);
expect(recordSpy).toHaveBeenCalledWith(
expect.any(Object),
expect.anything(),
'hoonify:google/gemma-4-31b-it',
expect.any(Object),
);
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
});
@@ -498,17 +498,18 @@ describe('MetaProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
createMock.mockResolvedValueOnce(OK_COMPLETION);
await complete(makeProvider(), { model: 'totally-not-a-real-model' });
it('rejects an unknown id instead of substituting the default', async () => {
const provider = makeProvider();
expect(createMock.mock.calls[0]![0].model).toBe('muse-spark-1.2');
expect(recordSpy).toHaveBeenCalledWith(
expect.any(Object),
expect.anything(),
'meta:muse-spark-1.2',
expect.any(Object),
);
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
});
@@ -93,7 +93,7 @@ export class MetaProvider extends OpenAICompatProvider {
this.#fsService = fsService;
}
override async complete(args: ICompleteArguments) {
override async complete(args: ICompleteArguments, resolved?: IChatModel) {
if (!Array.isArray(args.messages)) {
throw new HttpError(400, '`messages` must be an array', {
legacyCode: 'bad_request',
@@ -107,7 +107,7 @@ export class MetaProvider extends OpenAICompatProvider {
this.#fsService,
Context.get('actor'),
);
return super.complete(args);
return super.complete(args, resolved);
}
protected override vendorParams(
@@ -474,24 +474,18 @@ describe('MoonshotProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
it('rejects an unknown id instead of substituting the default', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls[0]![0].model).toBe('kimi-k2.6');
expect(recordSpy).toHaveBeenCalledWith(
expect.any(Object),
expect.anything(),
'moonshotai:kimi-k2.6',
expect.any(Object),
);
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
});
@@ -154,8 +154,8 @@ export class NeuralwattProvider extends OpenAICompatProvider {
},
});
override async complete(args: ICompleteArguments) {
const model = await this.resolveModel(args.model);
override async complete(args: ICompleteArguments, resolved?: IChatModel) {
const model = resolved ?? (await this.resolveModel(args.model));
if (
messagesHaveImageContent(args.messages ?? []) &&
!modelSupportsVision(model)
@@ -167,7 +167,7 @@ export class NeuralwattProvider extends OpenAICompatProvider {
);
}
this.#accountingMethod = await this.getAccountingMethod();
return super.complete(args);
return super.complete(args, model);
}
protected override vendorParams(
@@ -438,7 +438,7 @@ describe('OpenRouterProvider.complete request shape', () => {
await withTestActor(() =>
provider.complete({
model: 'openrouter:anthropic/claude-sonnet',
model: 'openrouter:anthropic/claude-haiku-4.5',
messages: [
{
role: 'user',
@@ -72,7 +72,7 @@ export class OpenRouterProvider extends OpenAICompatProvider {
this.#apiBaseUrl = apiBaseUrl;
}
override async complete(args: ICompleteArguments) {
override async complete(args: ICompleteArguments, resolved?: IChatModel) {
if (args.model === 'openrouter/auto') {
throw new HttpError(
400,
@@ -87,7 +87,7 @@ export class OpenRouterProvider extends OpenAICompatProvider {
},
);
}
return super.complete(args);
return super.complete(args, resolved);
}
override async models(): Promise<IChatModel[]> {
@@ -613,20 +613,18 @@ describe('TogetherAIProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
it('rejects an unknown id instead of substituting the default', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls[0]![0].model).toBe(
'meta-llama/Llama-3.3-70B-Instruct-Turbo',
);
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
});
@@ -539,24 +539,18 @@ describe('ZAIProvider model resolution', () => {
);
});
it('falls back to the default model when given an unknown id', async () => {
it('rejects an unknown id instead of substituting the default', async () => {
const { provider } = makeProvider();
createMock.mockResolvedValueOnce(baseCompletion);
await withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
);
expect(createMock.mock.calls[0]![0].model).toBe('glm-5.1');
expect(recordSpy).toHaveBeenCalledWith(
expect.any(Object),
expect.anything(),
'zai:glm-5.1',
expect.any(Object),
);
await expect(
withTestActor(() =>
provider.complete({
model: 'totally-not-a-real-model',
messages: [{ role: 'user', content: 'hi' }],
}),
),
).rejects.toMatchObject({ statusCode: 400 });
expect(createMock).not.toHaveBeenCalled();
});
});
+8 -1
View File
@@ -293,7 +293,14 @@ export interface IChatProvider {
models(extra_params?: unknown): IChatModel[] | Promise<IChatModel[]>;
list(): string[] | Promise<string[]>;
getDefaultModel(): string;
complete(arg: ICompleteArguments): Promise<IChatCompleteResult>;
/**
* `model` is this provider's own catalog entry for `arg.model`, as the
* driver resolved it; a provider resolves `arg.model` itself without it.
*/
complete(
arg: ICompleteArguments,
model?: IChatModel,
): Promise<IChatCompleteResult>;
checkModeration(
text: string,
): Promise<{ flagged: boolean; categories?: string[] }> | void;