mirror of
https://github.com/HeyPuter/puter.git
synced 2026-09-22 05:06:10 +00:00
dev: de-couple OpenAI SDK utilities from service
Docker Image CI / build-and-push-image (push) Waiting to run
Maintain Release Merge PR / update-release-pr (push) Waiting to run
release-please / release-please (push) Waiting to run
test / test (18.x) (push) Waiting to run
test / test (20.x) (push) Waiting to run
test / test (22.x) (push) Waiting to run
Docker Image CI / build-and-push-image (push) Waiting to run
Maintain Release Merge PR / update-release-pr (push) Waiting to run
release-please / release-please (push) Waiting to run
test / test (18.x) (push) Waiting to run
test / test (20.x) (push) Waiting to run
test / test (22.x) (push) Waiting to run
This change will be helpful for making behavior across OpenAI-compatible APIs consistent.
This commit is contained in:
KernelDeimos
@@ -24,6 +24,7 @@ const { TypedValue } = require('../../services/drivers/meta/Runtime');
|
||||
const { Context } = require('../../util/context');
|
||||
const smol = require('@heyputer/putility').libs.smol;
|
||||
const { nou } = require('../../util/langutil');
|
||||
const OpenAIUtil = require('./lib/OpenAIUtil');
|
||||
const { TeePromise } = require('@heyputer/putility').libs.promise;
|
||||
|
||||
|
||||
@@ -157,51 +158,7 @@ class OpenAICompletionService extends BaseService {
|
||||
return model_names;
|
||||
},
|
||||
|
||||
/**
|
||||
* AI Chat completion method.
|
||||
* See AIChatService for more details.
|
||||
*/
|
||||
async complete ({ messages, test_mode, stream, model, tools }) {
|
||||
|
||||
// for now this code (also in AIChatService.js) needs to be
|
||||
// duplicated because this hasn't been moved to be under
|
||||
// the centralised controller yet
|
||||
const svc_event = this.services.get('event');
|
||||
const event = {
|
||||
allow: true,
|
||||
intended_service: 'openai',
|
||||
parameters: { messages }
|
||||
};
|
||||
await svc_event.emit('ai.prompt.validate', event);
|
||||
if ( ! event.allow ) {
|
||||
test_mode = true;
|
||||
}
|
||||
|
||||
if ( test_mode ) {
|
||||
const { LoremIpsum } = require('lorem-ipsum');
|
||||
const li = new LoremIpsum({
|
||||
sentencesPerParagraph: {
|
||||
max: 8,
|
||||
min: 4
|
||||
},
|
||||
wordsPerSentence: {
|
||||
max: 20,
|
||||
min: 12
|
||||
},
|
||||
});
|
||||
return {
|
||||
"index": 0,
|
||||
"message": {
|
||||
"role": "assistant",
|
||||
"content": li.generateParagraphs(
|
||||
Math.floor(Math.random() * 3) + 1
|
||||
),
|
||||
},
|
||||
"logprobs": null,
|
||||
"finish_reason": "stop"
|
||||
}
|
||||
}
|
||||
|
||||
return await this.complete(messages, {
|
||||
model: model,
|
||||
tools,
|
||||
@@ -283,50 +240,8 @@ class OpenAICompletionService extends BaseService {
|
||||
// Here's something fun; the documentation shows `type: 'image_url'` in
|
||||
// objects that contain an image url, but everything still works if
|
||||
// that's missing. We normalise it here so the token count code works.
|
||||
for ( const msg of messages ) {
|
||||
if ( ! msg.content ) continue;
|
||||
if ( typeof msg.content !== 'object' ) continue;
|
||||
messages = await OpenAIUtil.process_input_messages(messages);
|
||||
|
||||
const content = msg.content;
|
||||
|
||||
for ( const o of content ) {
|
||||
if ( ! o.hasOwnProperty('image_url') ) continue;
|
||||
if ( o.type ) continue;
|
||||
o.type = 'image_url';
|
||||
}
|
||||
|
||||
// coerce tool calls
|
||||
for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
|
||||
const content_block = content[i];
|
||||
|
||||
if ( content_block.type === 'tool_use' ) {
|
||||
if ( ! msg.hasOwnProperty('tool_calls') ) {
|
||||
msg.tool_calls = [];
|
||||
}
|
||||
msg.tool_calls.push({
|
||||
id: content_block.id,
|
||||
type: 'function',
|
||||
function: {
|
||||
name: content_block.name,
|
||||
arguments: JSON.stringify(content_block.input),
|
||||
}
|
||||
});
|
||||
content.splice(i, 1);
|
||||
}
|
||||
}
|
||||
|
||||
// coerce tool results
|
||||
// (we assume multiple tool results were already split into separate messages)
|
||||
for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
|
||||
const content_block = content[i];
|
||||
if ( content_block.type !== 'tool_result' ) continue;
|
||||
msg.role = 'tool';
|
||||
msg.tool_call_id = content_block.tool_use_id;
|
||||
msg.content = content_block.content;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('MODEL IN USE ------- ', model);
|
||||
const completion = await this.openai.chat.completions.create({
|
||||
user: user_private_uid,
|
||||
messages: messages,
|
||||
@@ -339,99 +254,10 @@ class OpenAICompletionService extends BaseService {
|
||||
} : {}),
|
||||
});
|
||||
|
||||
if ( stream ) {
|
||||
let usage_promise = new TeePromise();
|
||||
|
||||
const init_chat_stream = async ({ chatStream }) => {
|
||||
const message = chatStream.message();
|
||||
let textblock = message.contentBlock({ type: 'text' });
|
||||
let toolblock = null;
|
||||
let mode = 'text';
|
||||
const tool_call_blocks = [];
|
||||
|
||||
for await ( const chunk of completion ) {
|
||||
console.log('CHUNK', chunk, JSON.stringify(chunk?.choices?.[0]?.delta ?? null));
|
||||
if ( chunk.usage ) {
|
||||
usage_promise.resolve({
|
||||
input_tokens: chunk.usage.prompt_tokens,
|
||||
output_tokens: chunk.usage.completion_tokens,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if ( chunk.choices.length < 1 ) continue;
|
||||
|
||||
const choice = chunk.choices[0];
|
||||
|
||||
if ( ! nou(choice.delta.content) ) {
|
||||
if ( mode === 'tool' ) {
|
||||
toolblock.end();
|
||||
mode = 'text';
|
||||
textblock = message.contentBlock({ type: 'text' });
|
||||
}
|
||||
textblock.addText(choice.delta.content);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ( ! nou(choice.delta.tool_calls) ) {
|
||||
if ( mode === 'text' ) {
|
||||
mode = 'tool';
|
||||
textblock.end();
|
||||
}
|
||||
for ( const tool_call of choice.delta.tool_calls ) {
|
||||
if ( ! tool_call_blocks[tool_call.index] ) {
|
||||
toolblock = message.contentBlock({
|
||||
type: 'tool_use',
|
||||
id: tool_call.id,
|
||||
name: tool_call.function.name,
|
||||
});
|
||||
tool_call_blocks[tool_call.index] = toolblock;
|
||||
} else {
|
||||
toolblock = tool_call_blocks[tool_call.index];
|
||||
}
|
||||
toolblock.addPartialJSON(tool_call.function.arguments);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ( mode === 'text' ) textblock.end();
|
||||
if ( mode === 'tool' ) toolblock.end();
|
||||
message.end();
|
||||
chatStream.end();
|
||||
};
|
||||
|
||||
return new TypedValue({ $: 'ai-chat-intermediate' }, {
|
||||
stream: true,
|
||||
init_chat_stream,
|
||||
usage_promise: usage_promise,
|
||||
});
|
||||
return retval;
|
||||
}
|
||||
|
||||
|
||||
this.log.info('how many choices?: ' + completion.choices.length);
|
||||
|
||||
const is_empty = completion.choices?.[0]?.message?.content?.trim() === '';
|
||||
if ( is_empty ) {
|
||||
// GPT refuses to generate an empty response if you ask it to,
|
||||
// so this will probably only happen on an error condition.
|
||||
throw new Error('an empty response was generated');
|
||||
}
|
||||
|
||||
// We need to moderate the completion too
|
||||
const mod_text = completion.choices[0].message.content;
|
||||
if ( moderation && mod_text !== null ) {
|
||||
const moderation_result = await this.check_moderation(mod_text);
|
||||
if ( moderation_result.flagged ) {
|
||||
throw new Error('message is not allowed');
|
||||
}
|
||||
}
|
||||
|
||||
const ret = completion.choices[0];
|
||||
ret.usage = {
|
||||
input_tokens: completion.usage.prompt_tokens,
|
||||
output_tokens: completion.usage.completion_tokens,
|
||||
};
|
||||
return ret;
|
||||
return OpenAIUtil.handle_completion_output({
|
||||
stream, completion,
|
||||
moderate: moderation && this.check_moderation.bind(this),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,157 @@
|
||||
const putility = require("@heyputer/putility");
|
||||
const { TypedValue } = require("../../../services/drivers/meta/Runtime");
|
||||
const { nou } = require("../../../util/langutil");
|
||||
|
||||
module.exports = class OpenAIUtil {
|
||||
/**
|
||||
* Process input messages from Puter's normalized format to OpenAI's format
|
||||
* May make changes in-place.
|
||||
*
|
||||
* @param {Array<Message>} messages - array of normalized messages
|
||||
* @returns {Array<Message>} - array of messages in OpenAI format
|
||||
*/
|
||||
static process_input_messages = async (messages) => {
|
||||
for ( const msg of messages ) {
|
||||
if ( ! msg.content ) continue;
|
||||
if ( typeof msg.content !== 'object' ) continue;
|
||||
|
||||
const content = msg.content;
|
||||
|
||||
for ( const o of content ) {
|
||||
if ( ! o.hasOwnProperty('image_url') ) continue;
|
||||
if ( o.type ) continue;
|
||||
o.type = 'image_url';
|
||||
}
|
||||
|
||||
// coerce tool calls
|
||||
for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
|
||||
const content_block = content[i];
|
||||
|
||||
if ( content_block.type === 'tool_use' ) {
|
||||
if ( ! msg.hasOwnProperty('tool_calls') ) {
|
||||
msg.tool_calls = [];
|
||||
}
|
||||
msg.tool_calls.push({
|
||||
id: content_block.id,
|
||||
type: 'function',
|
||||
function: {
|
||||
name: content_block.name,
|
||||
arguments: JSON.stringify(content_block.input),
|
||||
}
|
||||
});
|
||||
content.splice(i, 1);
|
||||
}
|
||||
}
|
||||
|
||||
// coerce tool results
|
||||
// (we assume multiple tool results were already split into separate messages)
|
||||
for ( let i = content.length - 1 ; i >= 0 ; i-- ) {
|
||||
const content_block = content[i];
|
||||
if ( content_block.type !== 'tool_result' ) continue;
|
||||
msg.role = 'tool';
|
||||
msg.tool_call_id = content_block.tool_use_id;
|
||||
msg.content = content_block.content;
|
||||
}
|
||||
}
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
static create_chat_stream_handler = ({
|
||||
completion,
|
||||
}) => async ({ chatStream }) => {
|
||||
const message = chatStream.message();
|
||||
let textblock = message.contentBlock({ type: 'text' });
|
||||
let toolblock = null;
|
||||
let mode = 'text';
|
||||
const tool_call_blocks = [];
|
||||
|
||||
for await ( const chunk of completion ) {
|
||||
if ( chunk.usage ) {
|
||||
usage_promise.resolve({
|
||||
input_tokens: chunk.usage.prompt_tokens,
|
||||
output_tokens: chunk.usage.completion_tokens,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if ( chunk.choices.length < 1 ) continue;
|
||||
|
||||
const choice = chunk.choices[0];
|
||||
|
||||
if ( ! nou(choice.delta.content) ) {
|
||||
if ( mode === 'tool' ) {
|
||||
toolblock.end();
|
||||
mode = 'text';
|
||||
textblock = message.contentBlock({ type: 'text' });
|
||||
}
|
||||
textblock.addText(choice.delta.content);
|
||||
continue;
|
||||
}
|
||||
|
||||
if ( ! nou(choice.delta.tool_calls) ) {
|
||||
if ( mode === 'text' ) {
|
||||
mode = 'tool';
|
||||
textblock.end();
|
||||
}
|
||||
for ( const tool_call of choice.delta.tool_calls ) {
|
||||
if ( ! tool_call_blocks[tool_call.index] ) {
|
||||
toolblock = message.contentBlock({
|
||||
type: 'tool_use',
|
||||
id: tool_call.id,
|
||||
name: tool_call.function.name,
|
||||
});
|
||||
tool_call_blocks[tool_call.index] = toolblock;
|
||||
} else {
|
||||
toolblock = tool_call_blocks[tool_call.index];
|
||||
}
|
||||
toolblock.addPartialJSON(tool_call.function.arguments);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ( mode === 'text' ) textblock.end();
|
||||
if ( mode === 'tool' ) toolblock.end();
|
||||
message.end();
|
||||
chatStream.end();
|
||||
};
|
||||
|
||||
static async handle_completion_output ({
|
||||
stream, completion, moderate
|
||||
}) {
|
||||
if ( stream ) {
|
||||
let usage_promise = new putility.libs.promise.TeePromise();
|
||||
|
||||
const init_chat_stream =
|
||||
OpenAIUtil.create_chat_stream_handler({ completion });
|
||||
|
||||
return new TypedValue({ $: 'ai-chat-intermediate' }, {
|
||||
stream: true,
|
||||
init_chat_stream,
|
||||
usage_promise: usage_promise,
|
||||
});
|
||||
}
|
||||
|
||||
const is_empty = completion.choices?.[0]?.message?.content?.trim() === '';
|
||||
if ( is_empty ) {
|
||||
// GPT refuses to generate an empty response if you ask it to,
|
||||
// so this will probably only happen on an error condition.
|
||||
throw new Error('an empty response was generated');
|
||||
}
|
||||
|
||||
// We need to moderate the completion too
|
||||
const mod_text = completion.choices[0].message.content;
|
||||
if ( moderate && mod_text !== null ) {
|
||||
const moderation_result = await moderate(mod_text);
|
||||
if ( moderation_result.flagged ) {
|
||||
throw new Error('message is not allowed');
|
||||
}
|
||||
}
|
||||
|
||||
const ret = completion.choices[0];
|
||||
ret.usage = {
|
||||
input_tokens: completion.usage.prompt_tokens,
|
||||
output_tokens: completion.usage.completion_tokens,
|
||||
};
|
||||
return ret;
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user