diff --git a/src/backend/src/modules/puterai/ClaudeService.js b/src/backend/src/modules/puterai/ClaudeService.js index 7ca6e82bf..0ee08d750 100644 --- a/src/backend/src/modules/puterai/ClaudeService.js +++ b/src/backend/src/modules/puterai/ClaudeService.js @@ -213,6 +213,7 @@ class ClaudeService extends BaseService { state = STATES.ready; for await ( const event of completion ) { + // console.log('EVENT', event); const input_tokens = (event?.usage ?? event?.message?.usage)?.input_tokens; const output_tokens = diff --git a/src/backend/src/modules/puterai/OpenAICompletionService.js b/src/backend/src/modules/puterai/OpenAICompletionService.js index d22ceee36..aebbd6f64 100644 --- a/src/backend/src/modules/puterai/OpenAICompletionService.js +++ b/src/backend/src/modules/puterai/OpenAICompletionService.js @@ -321,6 +321,7 @@ class OpenAICompletionService extends BaseService { }, stream); (async () => { for await ( const chunk of completion ) { + // console.log('CHUNK', chunk, JSON.stringify(chunk?.choices?.[0]?.delta ?? null)); entire.push(chunk); if ( chunk.usage ) { usage_promise.resolve({ diff --git a/src/backend/src/modules/puterai/experiment/stream_claude.js b/src/backend/src/modules/puterai/experiment/stream_claude.js new file mode 100644 index 000000000..7475207ff --- /dev/null +++ b/src/backend/src/modules/puterai/experiment/stream_claude.js @@ -0,0 +1,58 @@ +const { nou } = require('../../../util/langutil'); +const Streaming = require('../lib/Streaming'); +// const claude_sample = require('../samples/claude-1'); +const claude_sample = require('../samples/claude-tools-1'); + +const echo_stream = { + write: data => { + console.log(data); + } +}; + +const chatStream = new Streaming.AIChatStream({ stream: echo_stream }); + +let message; +let contentBlock; +for (const event of claude_sample) { + if ( event.type === 'message_start' ) { + message = chatStream.message(); + continue; + } + if ( event.type === 'message_stop' ) { + message.end(); + message = null; + continue; + } + + if ( event.type === 'content_block_start' ) { + if ( event.content_block.type === 'tool_use' ) { + contentBlock = message.contentBlock({ + type: event.content_block.type, + id: event.content_block.id, + name: event.content_block.name, + }); + continue; + } + contentBlock = message.contentBlock({ + type: event.content_block.type, + }); + continue; + } + + if ( event.type === 'content_block_stop' ) { + contentBlock.end(); + contentBlock = null; + continue; + } + + if ( event.type === 'content_block_delta' ) { + if ( event.delta.type === 'input_json_delta' ) { + contentBlock.addPartialJSON(event.delta.partial_json); + continue; + } + if ( event.delta.type === 'text_delta' ) { + contentBlock.addText(event.delta.text); + continue; + } + } +} diff --git a/src/backend/src/modules/puterai/experiment/stream_openai.js b/src/backend/src/modules/puterai/experiment/stream_openai.js new file mode 100644 index 000000000..a863d8e69 --- /dev/null +++ b/src/backend/src/modules/puterai/experiment/stream_openai.js @@ -0,0 +1,62 @@ +const { nou } = require('../../../util/langutil'); +const FunctionCalling = require('../lib/FunctionCalling'); +const Streaming = require('../lib/Streaming'); +const openai_fish = require('../samples/openai-tools-1'); + +const echo_stream = { + write: data => { + console.log(data); + } +}; + + +const chatStream = new Streaming.AIChatStream({ + stream: echo_stream, +}); + +const message = chatStream.message(); +let textblock = message.contentBlock({ type: 'text' }); +let toolblock = null; +let mode = 'text'; + +const tool_call_blocks = []; + +for ( const chunk of openai_fish ) { + if ( chunk.usage ) continue; + if ( chunk.choices.length < 1 ) continue; + + const choice = chunk.choices[0]; + + if ( ! nou(choice.delta.content) ) { + if ( mode === 'tool' ) { + toolblock.end(); + mode = 'text'; + textblock = message.contentBlock({ type: 'text' }); + } + textblock.addText(choice.delta.content); + continue; + } + + if ( ! nou(choice.delta.tool_calls) ) { + if ( mode === 'text' ) { + mode = 'tool'; + textblock.end(); + } + for ( const tool_call of choice.delta.tool_calls ) { + if ( ! tool_call_blocks[tool_call.index] ) { + toolblock = message.contentBlock({ + type: 'tool_use', + id: tool_call.function.name, + }); + tool_call_blocks[tool_call.index] = toolblock; + } else { + toolblock = tool_call_blocks[tool_call.index]; + } + toolblock.addPartialJSON(tool_call.function.arguments); + } + } +} + +if ( mode === 'text' ) textblock.end(); +if ( mode === 'tool' ) toolblock.end(); +message.end(); diff --git a/src/backend/src/modules/puterai/lib/Streaming.js b/src/backend/src/modules/puterai/lib/Streaming.js new file mode 100644 index 000000000..cb594ed16 --- /dev/null +++ b/src/backend/src/modules/puterai/lib/Streaming.js @@ -0,0 +1,84 @@ +/** + * Assign the properties of the override object to the original object, + * like Object.assign, except properties are ordered so override properties + * are enumerated first. + * + * @param {*} original + * @param {*} override + */ +const objectAssignTop = (original, override) => { + let o = { + ...original, + ...override, + }; + o = { + ...override, + ...original, + }; + return o; +} + +class AIChatConstructStream { + constructor (chatStream, params) { + this.chatStream = chatStream; + if ( this._start ) this._start(params); + } + end () { + if ( this._end ) this._end(); + } +} + +class AIChatTextStream extends AIChatConstructStream { + addText (text) { + const json = JSON.stringify({ + text, + }); + this.chatStream.stream.write(json + '\n'); + } +} + +class AIChatToolUseStream extends AIChatConstructStream { + _start (params) { + this.contentBlock = params; + this.buffer = ''; + } + addPartialJSON (partial_json) { + this.buffer += partial_json; + } + _end () { + const str = JSON.stringify(objectAssignTop({ + ...this.contentBlock, + input: JSON.parse(this.buffer), + ...( ! this.contentBlock.text ? { text: "" } : {}), + }, { + type: 'tool_use', + })); + this.chatStream.stream.write(str + '\n'); + } +} + +class AIChatMessageStream extends AIChatConstructStream { + contentBlock ({ type, ...params }) { + if ( type === 'tool_use' ) { + return new AIChatToolUseStream(this.chatStream, params); + } + if ( type === 'text' ) { + return new AIChatTextStream(this.chatStream, params); + } + throw new Error(`Unknown content block type: ${type}`); + } +} + +class AIChatStream { + constructor ({ stream }) { + this.stream = stream; + } + + message () { + return new AIChatMessageStream(this); + } +} + +module.exports = class Streaming { + static AIChatStream = AIChatStream; +} diff --git a/src/backend/src/modules/puterai/samples/claude-1.js b/src/backend/src/modules/puterai/samples/claude-1.js new file mode 100644 index 000000000..705892b7b --- /dev/null +++ b/src/backend/src/modules/puterai/samples/claude-1.js @@ -0,0 +1,65 @@ +module.exports = [ + { + type: 'message_start', + message: { + id: 'msg_01KKQeaUDpMzNovH9utP5qJc', + type: 'message', + role: 'assistant', + model: 'claude-3-5-sonnet-20241022', + content: [], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 82, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + output_tokens: 1 + } + } + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' } + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'Some' } + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: ' species of fish, like the electric' } + }, + { + type: 'content_block_delta', + index: 0, + delta: { + type: 'text_delta', + text: ' eel, can generate powerful electrical' + } + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: ' charges of up to 860 ' } + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'volts to stun prey an' } + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'd defend themselves.' } + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'message_delta', + delta: { stop_reason: 'end_turn', stop_sequence: null }, + usage: { output_tokens: 35 } + }, + { type: 'message_stop' }, +] \ No newline at end of file diff --git a/src/backend/src/modules/puterai/samples/claude-tools-1.js b/src/backend/src/modules/puterai/samples/claude-tools-1.js new file mode 100644 index 000000000..41a5495da --- /dev/null +++ b/src/backend/src/modules/puterai/samples/claude-tools-1.js @@ -0,0 +1,76 @@ +module.exports = [ + { + type: 'message_start', + message: { + id: 'msg_01GAy4THpFyFJcpxqWXBMrvx', + type: 'message', + role: 'assistant', + model: 'claude-3-5-sonnet-20241022', + content: [], + stop_reason: null, + stop_sequence: null, + usage: { + input_tokens: 458, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + output_tokens: 1 + } + } + }, + { + type: 'content_block_start', + index: 0, + content_block: { type: 'text', text: '' } + }, + { + type: 'content_block_delta', + index: 0, + delta: { type: 'text_delta', text: 'I' } + }, + { + type: 'content_block_delta', + index: 0, + delta: { + type: 'text_delta', + text: "'ll check the weather in Vancouver for you." + } + }, + { type: 'content_block_stop', index: 0 }, + { + type: 'content_block_start', + index: 1, + content_block: { + type: 'tool_use', + id: 'toolu_01E12jeyCenTtntPBk1j7rgc', + name: 'get_weather', + input: {} + } + }, + { + type: 'content_block_delta', + index: 1, + delta: { type: 'input_json_delta', partial_json: '' } + }, + { + type: 'content_block_delta', + index: 1, + delta: { type: 'input_json_delta', partial_json: '{"location"' } + }, + { + type: 'content_block_delta', + index: 1, + delta: { type: 'input_json_delta', partial_json: ': "Van' } + }, + { + type: 'content_block_delta', + index: 1, + delta: { type: 'input_json_delta', partial_json: 'couver"}' } + }, + { type: 'content_block_stop', index: 1 }, + { + type: 'message_delta', + delta: { stop_reason: 'tool_use', stop_sequence: null }, + usage: { output_tokens: 64 } + }, + { type: 'message_stop' }, +] diff --git a/src/backend/src/modules/puterai/samples/openai-1.js b/src/backend/src/modules/puterai/samples/openai-1.js new file mode 100644 index 000000000..af606ed72 --- /dev/null +++ b/src/backend/src/modules/puterai/samples/openai-1.js @@ -0,0 +1,46 @@ +module.exports = [ + { + id: 'chatcmpl-AvspmQTvFBBjKsFhHYhyiphFmKMY8', + object: 'chat.completion.chunk', + created: 1738358842, + model: 'gpt-4o-mini-2024-07-18', + service_tier: 'default', + system_fingerprint: 'fp_bd83329f63', + choices: [ + { + index: 0, + delta: { + role: "assistant", + content: "", + refusal: null + }, + logprobs: null, + finish_reason: null + } + ], + usage: null + }, + ...[ + `Fish`, ` are`, ` diverse`, ` aquatic`, ` creatures`, ` that`, ` play`, + ` a`, ` crucial`, ` role`, ` in`, ` marine`, ` ecosystems`, ` and`, + ` human`, ` diets`, `.` + ].map(str => ({ + id: 'chatcmpl-AvspmQTvFBBjKsFhHYhyiphFmKMY8', + object: 'chat.completion.chunk', + created: 1738358842, + model: 'gpt-4o-mini-2024-07-18', + service_tier: 'default', + system_fingerprint: 'fp_bd83329f63', + choices: [ + { + index: 0, + delta: { + content: str + }, + logprobs: null, + finish_reason: null + } + ], + usage: null + })), +]; diff --git a/src/backend/src/modules/puterai/samples/openai-tools-1.js b/src/backend/src/modules/puterai/samples/openai-tools-1.js new file mode 100644 index 000000000..3c75cc5dc --- /dev/null +++ b/src/backend/src/modules/puterai/samples/openai-tools-1.js @@ -0,0 +1,102 @@ +module.exports = [ + { + id: 'chatcmpl-Avqr6AwmQoEFLXuwf1llkKknIR4Ry', + object: 'chat.completion.chunk', + created: 1738351236, + model: 'gpt-4o-mini-2024-07-18', + service_tier: 'default', + system_fingerprint: 'fp_72ed7ab54c', + choices: [ + { + index: 0, + delta: { + role: "assistant", + content: null, + tool_calls: [ + { + index: 0, + id: "call_ULl8cRKFQbYeJSIZ3giLAg6r", + type: "function", + function: { + name: "get_weather", + arguments: "" + } + } + ], + refusal: null + }, + logprobs: null, + finish_reason: null + } + ], + usage: null + }, + ...[ + `{"`, `location`, `":"`, + `V`, `ancouver`, + `"}` + ].map(str => ({ + id: 'chatcmpl-Avqr6AwmQoEFLXuwf1llkKknIR4Ry', + object: 'chat.completion.chunk', + created: 1738351236, + model: 'gpt-4o-mini-2024-07-18', + service_tier: 'default', + system_fingerprint: 'fp_72ed7ab54c', + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + function: { + arguments: str, + } + } + ] + }, + logprobs: null, + finish_reason: null + } + ], + usage: null + })), + { + id: 'chatcmpl-Avqr6AwmQoEFLXuwf1llkKknIR4Ry', + object: 'chat.completion.chunk', + created: 1738351236, + model: 'gpt-4o-mini-2024-07-18', + service_tier: 'default', + system_fingerprint: 'fp_72ed7ab54c', + choices: [ + { + index: 0, + delta: {}, + logprobs: null, + finish_reason: 'tool_calls' + } + ], + usage: null + }, + { + id: 'chatcmpl-Avqr6AwmQoEFLXuwf1llkKknIR4Ry', + object: 'chat.completion.chunk', + created: 1738351236, + model: 'gpt-4o-mini-2024-07-18', + service_tier: 'default', + system_fingerprint: 'fp_72ed7ab54c', + choices: [], + usage: { + prompt_tokens: 62, + completion_tokens: 16, + total_tokens: 78, + prompt_tokens_details: { cached_tokens: 0, audio_tokens: 0 }, + completion_tokens_details: { + reasoning_tokens: 0, + audio_tokens: 0, + accepted_prediction_tokens: 0, + rejected_prediction_tokens: 0 + } + } + } +];