dev: PoC for stream normalization

src/backend/src/modules/puterai/ClaudeService.js

@@ -213,6 +213,7 @@ class ClaudeService extends BaseService {
                         state = STATES.ready;
 
                         for await ( const event of completion ) {
+                            // console.log('EVENT', event);
                             const input_tokens =
                                 (event?.usage ?? event?.message?.usage)?.input_tokens;
                             const output_tokens =

src/backend/src/modules/puterai/OpenAICompletionService.js

@@ -321,6 +321,7 @@ class OpenAICompletionService extends BaseService {
             }, stream);
             (async () => {
                 for await ( const chunk of completion ) {
+                    // console.log('CHUNK', chunk, JSON.stringify(chunk?.choices?.[0]?.delta ?? null));
                     entire.push(chunk);
                     if ( chunk.usage ) {
                         usage_promise.resolve({

src/backend/src/modules/puterai/experiment/stream_claude.js

@@ -0,0 +1,58 @@
+const { nou } = require('../../../util/langutil');
+const Streaming = require('../lib/Streaming');
+// const claude_sample = require('../samples/claude-1');
+const claude_sample = require('../samples/claude-tools-1');
+
+const echo_stream = {
+    write: data => {
+        console.log(data);
+    }
+};
+
+const chatStream = new Streaming.AIChatStream({ stream: echo_stream });
+
+let message;
+let contentBlock;
+for (const event of claude_sample) {
+    if ( event.type === 'message_start' ) {
+        message = chatStream.message();
+        continue;
+    }
+    if ( event.type === 'message_stop' ) {
+        message.end();
+        message = null;
+        continue;
+    }
+
+    if ( event.type === 'content_block_start' ) {
+        if ( event.content_block.type === 'tool_use' ) {
+            contentBlock = message.contentBlock({
+                type: event.content_block.type,
+                id: event.content_block.id,
+                name: event.content_block.name,
+            });
+            continue;
+        }
+        contentBlock = message.contentBlock({
+            type: event.content_block.type,
+        });
+        continue;
+    }
+
+    if ( event.type === 'content_block_stop' ) {
+        contentBlock.end();
+        contentBlock = null;
+        continue;
+    }
+
+    if ( event.type === 'content_block_delta' ) {
+        if ( event.delta.type === 'input_json_delta' ) {
+            contentBlock.addPartialJSON(event.delta.partial_json);
+            continue;
+        }
+        if ( event.delta.type === 'text_delta' ) {
+            contentBlock.addText(event.delta.text);
+            continue;
+        }
+    }
+}

src/backend/src/modules/puterai/experiment/stream_openai.js

@@ -0,0 +1,62 @@
+const { nou } = require('../../../util/langutil');
+const FunctionCalling = require('../lib/FunctionCalling');
+const Streaming = require('../lib/Streaming');
+const openai_fish = require('../samples/openai-tools-1');
+
+const echo_stream = {
+    write: data => {
+        console.log(data);
+    }
+};
+
+
+const chatStream = new Streaming.AIChatStream({
+    stream: echo_stream,
+});
+
+const message = chatStream.message();
+let textblock = message.contentBlock({ type: 'text' });
+let toolblock = null;
+let mode = 'text';
+
+const tool_call_blocks = [];
+
+for ( const chunk of openai_fish ) {
+    if ( chunk.usage ) continue;
+    if ( chunk.choices.length < 1 ) continue;
+
+    const choice = chunk.choices[0];
+
+    if ( ! nou(choice.delta.content) ) {
+        if ( mode === 'tool' ) {
+            toolblock.end();
+            mode = 'text';
+            textblock = message.contentBlock({ type: 'text' });
+        }
+        textblock.addText(choice.delta.content);
+        continue;
+    }
+
+    if (  ! nou(choice.delta.tool_calls) ) {
+        if ( mode === 'text' ) {
+            mode = 'tool';
+            textblock.end();
+        }
+        for ( const tool_call of choice.delta.tool_calls ) {
+            if ( ! tool_call_blocks[tool_call.index] ) {
+                toolblock = message.contentBlock({
+                    type: 'tool_use',
+                    id: tool_call.function.name,
+                });
+                tool_call_blocks[tool_call.index] = toolblock;
+            } else {
+                toolblock = tool_call_blocks[tool_call.index];
+            }
+            toolblock.addPartialJSON(tool_call.function.arguments);
+        }
+    }
+}
+
+if ( mode === 'text' ) textblock.end();
+if ( mode === 'tool' ) toolblock.end();
+message.end();

src/backend/src/modules/puterai/lib/Streaming.js

@@ -0,0 +1,84 @@
+/**
+ * Assign the properties of the override object to the original object,
+ * like Object.assign, except properties are ordered so override properties
+ * are enumerated first.
+ * 
+ * @param {*} original 
+ * @param {*} override 
+ */
+const objectAssignTop = (original, override) => {
+    let o = {
+        ...original,
+        ...override,
+    };
+    o = {
+        ...override,
+        ...original,
+    };
+    return o;
+}
+
+class AIChatConstructStream {
+    constructor (chatStream, params) {
+        this.chatStream = chatStream;
+        if ( this._start ) this._start(params);
+    }
+    end () {
+        if ( this._end ) this._end();
+    }
+}
+
+class AIChatTextStream extends AIChatConstructStream {
+    addText (text) {
+        const json = JSON.stringify({
+            text,
+        });
+        this.chatStream.stream.write(json + '\n');
+    }
+}
+
+class AIChatToolUseStream extends AIChatConstructStream {
+    _start (params) {
+        this.contentBlock = params;
+        this.buffer = '';
+    }
+    addPartialJSON (partial_json) {
+        this.buffer += partial_json;
+    }
+    _end () {
+        const str = JSON.stringify(objectAssignTop({
+            ...this.contentBlock,
+            input: JSON.parse(this.buffer),
+            ...( ! this.contentBlock.text ? { text: "" } : {}),
+        }, {
+            type: 'tool_use',
+        }));
+        this.chatStream.stream.write(str + '\n');
+    }
+}
+
+class AIChatMessageStream extends AIChatConstructStream {
+    contentBlock ({ type, ...params }) {
+        if ( type === 'tool_use' ) {
+            return new AIChatToolUseStream(this.chatStream, params);
+        }
+        if ( type === 'text' ) {
+            return new AIChatTextStream(this.chatStream, params);
+        }
+        throw new Error(`Unknown content block type: ${type}`);
+    }
+}
+
+class AIChatStream {
+    constructor ({ stream }) {
+        this.stream = stream;
+    }
+
+    message () {
+        return new AIChatMessageStream(this);
+    }
+}
+
+module.exports = class Streaming {
+    static AIChatStream  = AIChatStream;
+}

src/backend/src/modules/puterai/samples/claude-1.js

@@ -0,0 +1,65 @@
+module.exports = [
+    {
+        type: 'message_start',
+        message: {
+            id: 'msg_01KKQeaUDpMzNovH9utP5qJc',
+            type: 'message',
+            role: 'assistant',
+            model: 'claude-3-5-sonnet-20241022',
+            content: [],
+            stop_reason: null,
+            stop_sequence: null,
+            usage: {
+                input_tokens: 82,
+                cache_creation_input_tokens: 0,
+                cache_read_input_tokens: 0,
+                output_tokens: 1
+            }
+        }
+    },
+    {
+        type: 'content_block_start',
+        index: 0,
+        content_block: { type: 'text', text: '' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 0,
+        delta: { type: 'text_delta', text: 'Some' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 0,
+        delta: { type: 'text_delta', text: ' species of fish, like the electric' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 0,
+        delta: {
+            type: 'text_delta',
+            text: ' eel, can generate powerful electrical'
+        }
+    },
+    {
+        type: 'content_block_delta',
+        index: 0,
+        delta: { type: 'text_delta', text: ' charges of up to 860 ' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 0,
+        delta: { type: 'text_delta', text: 'volts to stun prey an' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 0,
+        delta: { type: 'text_delta', text: 'd defend themselves.' }
+    },
+    { type: 'content_block_stop', index: 0 },
+    {
+        type: 'message_delta',
+        delta: { stop_reason: 'end_turn', stop_sequence: null },
+        usage: { output_tokens: 35 }
+    },
+    { type: 'message_stop' },
+]

src/backend/src/modules/puterai/samples/claude-tools-1.js

@@ -0,0 +1,76 @@
+module.exports = [
+    {
+        type: 'message_start',
+        message: {
+            id: 'msg_01GAy4THpFyFJcpxqWXBMrvx',
+            type: 'message',
+            role: 'assistant',
+            model: 'claude-3-5-sonnet-20241022',
+            content: [],
+            stop_reason: null,
+            stop_sequence: null,
+            usage: {
+                input_tokens: 458,
+                cache_creation_input_tokens: 0,
+                cache_read_input_tokens: 0,
+                output_tokens: 1
+            }
+        }
+    },
+    {
+        type: 'content_block_start',
+        index: 0,
+        content_block: { type: 'text', text: '' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 0,
+        delta: { type: 'text_delta', text: 'I' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 0,
+        delta: {
+            type: 'text_delta',
+            text: "'ll check the weather in Vancouver for you."
+        }
+    },
+    { type: 'content_block_stop', index: 0 },
+    {
+        type: 'content_block_start',
+        index: 1,
+        content_block: {
+            type: 'tool_use',
+            id: 'toolu_01E12jeyCenTtntPBk1j7rgc',
+            name: 'get_weather',
+            input: {}
+        }
+    },
+    {
+        type: 'content_block_delta',
+        index: 1,
+        delta: { type: 'input_json_delta', partial_json: '' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 1,
+        delta: { type: 'input_json_delta', partial_json: '{"location"' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 1,
+        delta: { type: 'input_json_delta', partial_json: ': "Van' }
+    },
+    {
+        type: 'content_block_delta',
+        index: 1,
+        delta: { type: 'input_json_delta', partial_json: 'couver"}' }
+    },
+    { type: 'content_block_stop', index: 1 },
+    {
+        type: 'message_delta',
+        delta: { stop_reason: 'tool_use', stop_sequence: null },
+        usage: { output_tokens: 64 }
+    },
+    { type: 'message_stop' },
+]

src/backend/src/modules/puterai/samples/openai-1.js

@@ -0,0 +1,46 @@
+module.exports = [
+    {
+        id: 'chatcmpl-AvspmQTvFBBjKsFhHYhyiphFmKMY8',
+        object: 'chat.completion.chunk',
+        created: 1738358842,
+        model: 'gpt-4o-mini-2024-07-18',
+        service_tier: 'default',
+        system_fingerprint: 'fp_bd83329f63',
+        choices: [
+            {
+                index: 0,
+                delta: {
+                    role: "assistant",
+                    content: "",
+                    refusal: null
+                },
+                logprobs: null,
+                finish_reason: null
+            }
+        ],
+        usage: null
+    },
+    ...[
+        `Fish`, ` are`, ` diverse`, ` aquatic`, ` creatures`, ` that`, ` play`,
+        ` a`, ` crucial`, ` role`, ` in`, ` marine`, ` ecosystems`, ` and`,
+        ` human`, ` diets`, `.`
+    ].map(str => ({
+        id: 'chatcmpl-AvspmQTvFBBjKsFhHYhyiphFmKMY8',
+        object: 'chat.completion.chunk',
+        created: 1738358842,
+        model: 'gpt-4o-mini-2024-07-18',
+        service_tier: 'default',
+        system_fingerprint: 'fp_bd83329f63',
+        choices: [
+          {
+            index: 0,
+            delta: {
+                content: str
+            },
+            logprobs: null,
+            finish_reason: null
+        }
+        ],
+        usage: null
+    })),
+];

src/backend/src/modules/puterai/samples/openai-tools-1.js

@@ -0,0 +1,102 @@
+module.exports = [
+    {
+        id: 'chatcmpl-Avqr6AwmQoEFLXuwf1llkKknIR4Ry',
+        object: 'chat.completion.chunk',
+        created: 1738351236,
+        model: 'gpt-4o-mini-2024-07-18',
+        service_tier: 'default',
+        system_fingerprint: 'fp_72ed7ab54c',
+        choices: [
+            {
+                index: 0,
+                delta: {
+                    role: "assistant",
+                    content: null,
+                    tool_calls: [
+                        {
+                            index: 0,
+                            id: "call_ULl8cRKFQbYeJSIZ3giLAg6r",
+                            type: "function",
+                            function: {
+                                name: "get_weather",
+                                arguments: ""
+                            }
+                        }
+                    ],
+                    refusal: null
+                },
+                logprobs: null,
+                finish_reason: null
+            }
+        ],
+        usage: null
+    },
+    ...[
+        `{"`, `location`, `":"`,
+        `V`, `ancouver`,
+        `"}`
+    ].map(str => ({
+        id: 'chatcmpl-Avqr6AwmQoEFLXuwf1llkKknIR4Ry',
+        object: 'chat.completion.chunk',
+        created: 1738351236,
+        model: 'gpt-4o-mini-2024-07-18',
+        service_tier: 'default',
+        system_fingerprint: 'fp_72ed7ab54c',
+        choices: [
+            {
+                index: 0,
+                delta: {
+                    tool_calls: [
+                        {
+                            index: 0,
+                            function: {
+                                arguments: str,
+                            }
+                        }
+                    ]
+                },
+                logprobs: null,
+                finish_reason: null
+            }
+        ],
+        usage: null
+    })),
+    {
+        id: 'chatcmpl-Avqr6AwmQoEFLXuwf1llkKknIR4Ry',
+        object: 'chat.completion.chunk',
+        created: 1738351236,
+        model: 'gpt-4o-mini-2024-07-18',
+        service_tier: 'default',
+        system_fingerprint: 'fp_72ed7ab54c',
+        choices: [
+            {
+                index: 0,
+                delta: {},
+                logprobs: null,
+                finish_reason: 'tool_calls'
+            }
+        ],
+        usage: null
+    },
+    {
+        id: 'chatcmpl-Avqr6AwmQoEFLXuwf1llkKknIR4Ry',
+        object: 'chat.completion.chunk',
+        created: 1738351236,
+        model: 'gpt-4o-mini-2024-07-18',
+        service_tier: 'default',
+        system_fingerprint: 'fp_72ed7ab54c',
+        choices: [],
+        usage: {
+            prompt_tokens: 62,
+            completion_tokens: 16,
+            total_tokens: 78,
+            prompt_tokens_details: { cached_tokens: 0, audio_tokens: 0 },
+            completion_tokens_details: {
+                reasoning_tokens: 0,
+                audio_tokens: 0,
+                accepted_prediction_tokens: 0,
+                rejected_prediction_tokens: 0
+            }
+        }
+    }
+];