From 7b93d7c15bcc35006f51522f3be3ceae27e9b149 Mon Sep 17 00:00:00 2001 From: Nariman Jelveh Date: Tue, 11 Nov 2025 17:06:44 -0800 Subject: [PATCH] Add Mistral OCR support and imrpove `img2txt` API (#1957) Introduces Mistral OCR integration in backend and updates the `AIInterfaceService` and `MistralAIService` to support new OCR options and response normalization. Updates metering cost map for OCR and annotation usage. Refactors `DriverService` to support interface-specific service aliases. Expands the puter.js `AI.img2txt` API to support flexible options and provider selection, including Mistral OCR. --- package-lock.json | 27 +-- .../src/modules/puterai/AIChatService.js | 3 +- .../src/modules/puterai/AIInterfaceService.js | 29 +++ .../src/modules/puterai/MistralAIService.js | 217 ++++++++++++++++++ .../costMaps/mistralCostMap.ts | 7 +- .../src/services/drivers/DriverService.js | 31 ++- src/puter-js/index.d.ts | 16 ++ src/puter-js/src/modules/AI.js | 110 ++++++--- 8 files changed, 374 insertions(+), 66 deletions(-) diff --git a/package-lock.json b/package-lock.json index f5f933f64..90dd3bb5f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -904,7 +904,6 @@ "integrity": "sha512-2BCOP7TN8M+gVDj7/ht3hsaO/B/n5oDbiAyyvnRlNOs+u1o+JWNYTQrmpuNp1/Wq2gcFrI01JAW+paEKDMx/CA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@babel/code-frame": "^7.27.1", "@babel/generator": "^7.28.3", @@ -3048,7 +3047,6 @@ "resolved": "https://registry.npmjs.org/@jimp/custom/-/custom-0.22.12.tgz", "integrity": "sha512-xcmww1O/JFP2MrlGUMd3Q78S3Qu6W3mYTXYuIqFq33EorgYHV/HqymHfXy9GjiCJ7OI+7lWx6nYFOzU7M4rd1Q==", "license": "MIT", - "peer": true, "dependencies": { "@jimp/core": "^0.22.12" } @@ -3085,7 +3083,6 @@ "resolved": "https://registry.npmjs.org/@jimp/plugin-blit/-/plugin-blit-0.22.12.tgz", "integrity": "sha512-xslz2ZoFZOPLY8EZ4dC29m168BtDx95D6K80TzgUi8gqT7LY6CsajWO0FAxDwHz6h0eomHMfyGX0stspBrTKnQ==", "license": "MIT", - "peer": true, "dependencies": { "@jimp/utils": "^0.22.12" }, @@ -3098,7 +3095,6 @@ "resolved": "https://registry.npmjs.org/@jimp/plugin-blur/-/plugin-blur-0.22.12.tgz", "integrity": "sha512-S0vJADTuh1Q9F+cXAwFPlrKWzDj2F9t/9JAbUvaaDuivpyWuImEKXVz5PUZw2NbpuSHjwssbTpOZ8F13iJX4uw==", "license": "MIT", - "peer": true, "dependencies": { "@jimp/utils": "^0.22.12" }, @@ -3123,7 +3119,6 @@ "resolved": "https://registry.npmjs.org/@jimp/plugin-color/-/plugin-color-0.22.12.tgz", "integrity": "sha512-xImhTE5BpS8xa+mAN6j4sMRWaUgUDLoaGHhJhpC+r7SKKErYDR0WQV4yCE4gP+N0gozD0F3Ka1LUSaMXrn7ZIA==", "license": "MIT", - "peer": true, "dependencies": { "@jimp/utils": "^0.22.12", "tinycolor2": "^1.6.0" @@ -3167,7 +3162,6 @@ "resolved": "https://registry.npmjs.org/@jimp/plugin-crop/-/plugin-crop-0.22.12.tgz", "integrity": "sha512-FNuUN0OVzRCozx8XSgP9MyLGMxNHHJMFt+LJuFjn1mu3k0VQxrzqbN06yIl46TVejhyAhcq5gLzqmSCHvlcBVw==", "license": "MIT", - "peer": true, "dependencies": { "@jimp/utils": "^0.22.12" }, @@ -3291,7 +3285,6 @@ "resolved": "https://registry.npmjs.org/@jimp/plugin-resize/-/plugin-resize-0.22.12.tgz", "integrity": "sha512-3NyTPlPbTnGKDIbaBgQ3HbE6wXbAlFfxHVERmrbqAi8R3r6fQPxpCauA8UVDnieg5eo04D0T8nnnNIX//i/sXg==", "license": "MIT", - "peer": true, "dependencies": { "@jimp/utils": "^0.22.12" }, @@ -3304,7 +3297,6 @@ "resolved": "https://registry.npmjs.org/@jimp/plugin-rotate/-/plugin-rotate-0.22.12.tgz", "integrity": "sha512-9YNEt7BPAFfTls2FGfKBVgwwLUuKqy+E8bDGGEsOqHtbuhbshVGxN2WMZaD4gh5IDWvR+emmmPPWGgaYNYt1gA==", "license": "MIT", - "peer": true, "dependencies": { "@jimp/utils": "^0.22.12" }, @@ -3320,7 +3312,6 @@ "resolved": "https://registry.npmjs.org/@jimp/plugin-scale/-/plugin-scale-0.22.12.tgz", "integrity": "sha512-dghs92qM6MhHj0HrV2qAwKPMklQtjNpoYgAB94ysYpsXslhRTiPisueSIELRwZGEr0J0VUxpUY7HgJwlSIgGZw==", "license": "MIT", - "peer": true, "dependencies": { "@jimp/utils": "^0.22.12" }, @@ -3628,7 +3619,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api/-/api-1.4.1.tgz", "integrity": "sha512-O2yRJce1GOc6PAy3QxFM4NzFiWzvScDC1/5ihYBL6BUEVdq0XMWN01sppE+H6bBXbaFYipjwFLEWLg5PaSOThA==", "license": "Apache-2.0", - "peer": true, "engines": { "node": ">=8.0.0" } @@ -3638,7 +3628,6 @@ "resolved": "https://registry.npmjs.org/@opentelemetry/api-logs/-/api-logs-0.49.1.tgz", "integrity": "sha512-kaNl/T7WzyMUQHQlVq7q0oV4Kev6+0xFwqzofryC66jgGMacd0QH5TwfpbUwSTby+SdAdprAe5UKMvBw4tKS5Q==", "license": "Apache-2.0", - "peer": true, "dependencies": { "@opentelemetry/api": "^1.0.0" }, @@ -7273,7 +7262,6 @@ "integrity": "sha512-6JSSaBZmsKvEkbRUkf7Zj7dru/8ZCrJxAqArcLaVMee5907JdtEbKGsZ7zNiIm/UAkpGUkaSMZEXShnN2D1HZA==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.46.1", "@typescript-eslint/types": "8.46.1", @@ -7816,8 +7804,7 @@ "version": "5.5.0", "resolved": "https://registry.npmjs.org/@xterm/xterm/-/xterm-5.5.0.tgz", "integrity": "sha512-hqJHYaQb5OptNunnyAnkHyM8aCjZ1MEIDTQu1iIbbTD/xops91NB5yq1ZK/dC2JDbVWtF23zUtl9JE2NqwT87A==", - "license": "MIT", - "peer": true + "license": "MIT" }, "node_modules/@xtuc/ieee754": { "version": "1.2.0", @@ -7867,7 +7854,6 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.15.0.tgz", "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "license": "MIT", - "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -8557,7 +8543,6 @@ } ], "license": "MIT", - "peer": true, "dependencies": { "baseline-browser-mapping": "^2.8.9", "caniuse-lite": "^1.0.30001746", @@ -8812,7 +8797,6 @@ "resolved": "https://registry.npmjs.org/chai/-/chai-5.3.3.tgz", "integrity": "sha512-4zNhdJD/iOjSH0A05ea+Ke6MU5mmpQcbQsSOkgdaUMJ9zTlDTD/GYlwohmIE2u0gaxHYiVHEn1Fw9mZ/ktJWgw==", "license": "MIT", - "peer": true, "dependencies": { "assertion-error": "^2.0.1", "check-error": "^2.1.1", @@ -10549,7 +10533,6 @@ "resolved": "https://registry.npmjs.org/eslint/-/eslint-9.37.0.tgz", "integrity": "sha512-XyLmROnACWqSxiGYArdef1fItQd47weqB7iwtfr9JHwRrqIXZdcFMvvEcL9xHCmL0SNsOvF0c42lWyM1U5dgig==", "license": "MIT", - "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -16322,7 +16305,6 @@ "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.17.1.tgz", "integrity": "sha512-B/gBuNg5SiMTrPkC+A2+cW0RszwxYmn6VYxB/inlBStS5nx6xHIt/ehKRhIMhqusl7a8LjQoZnjCs5vhwxOQ1g==", "license": "MIT", - "peer": true, "dependencies": { "fast-deep-equal": "^3.1.3", "fast-uri": "^3.0.1", @@ -18003,7 +17985,6 @@ "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.9.3.tgz", "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "license": "Apache-2.0", - "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -18230,7 +18211,6 @@ "integrity": "sha512-4nVGliEpxmhCL8DslSAUdxlB6+SMrhB0a1v5ijlh1xB1nEPuy1mxaHxysVucLHuWryAxLWg6a5ei+U4TLn/rFg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", @@ -18496,7 +18476,6 @@ "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.102.1.tgz", "integrity": "sha512-7h/weGm9d/ywQ6qzJ+Xy+r9n/3qgp/thalBbpOi5i223dPXKi04IBtqPN9nTd+jBc7QKfvDbaBnFipYp4sJAUQ==", "license": "MIT", - "peer": true, "dependencies": { "@types/eslint-scope": "^3.7.7", "@types/estree": "^1.0.8", @@ -18546,7 +18525,6 @@ "integrity": "sha512-pIDJHIEI9LR0yxHXQ+Qh95k2EvXpWzZ5l+d+jIo+RdSm9MiHfzazIxwwni/p7+x4eJZuvG1AJwgC4TNQ7NRgsg==", "dev": true, "license": "MIT", - "peer": true, "dependencies": { "@discoveryjs/json-ext": "^0.5.0", "@webpack-cli/configtest": "^2.1.1", @@ -18778,7 +18756,6 @@ "resolved": "https://registry.npmjs.org/winston/-/winston-3.18.3.tgz", "integrity": "sha512-NoBZauFNNWENgsnC9YpgyYwOVrl2m58PpQ8lNHjV3kosGs7KJ7Npk9pCUE+WJlawVSe8mykWDKWFSVfs3QO9ww==", "license": "MIT", - "peer": true, "dependencies": { "@colors/colors": "^1.6.0", "@dabh/diagnostics": "^2.0.8", @@ -18983,7 +18960,6 @@ "resolved": "https://registry.npmjs.org/ws/-/ws-8.18.3.tgz", "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==", "license": "MIT", - "peer": true, "engines": { "node": ">=10.0.0" }, @@ -19317,7 +19293,6 @@ "resolved": "https://registry.npmjs.org/zod/-/zod-3.25.76.tgz", "integrity": "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ==", "license": "MIT", - "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/src/backend/src/modules/puterai/AIChatService.js b/src/backend/src/modules/puterai/AIChatService.js index 1196f1d85..e348fa6b4 100644 --- a/src/backend/src/modules/puterai/AIChatService.js +++ b/src/backend/src/modules/puterai/AIChatService.js @@ -122,7 +122,8 @@ class AIChatService extends BaseService { const svc_driver = this.services.get('driver'); for ( const provider of this.providers ) { svc_driver.register_service_alias('ai-chat', - provider.service_name); + provider.service_name, + { iface: 'puter-chat-completion' }); } } diff --git a/src/backend/src/modules/puterai/AIInterfaceService.js b/src/backend/src/modules/puterai/AIInterfaceService.js index 7031d814b..b86c7e30a 100644 --- a/src/backend/src/modules/puterai/AIInterfaceService.js +++ b/src/backend/src/modules/puterai/AIInterfaceService.js @@ -47,6 +47,35 @@ class AIInterfaceService extends BaseService { source: { type: 'file', }, + model: { + type: 'string', + optional: true, + }, + pages: { + type: 'json', + subtype: 'array', + optional: true, + }, + includeImageBase64: { + type: 'flag', + optional: true, + }, + imageLimit: { + type: 'number', + optional: true, + }, + imageMinSize: { + type: 'number', + optional: true, + }, + bboxAnnotationFormat: { + type: 'json', + optional: true, + }, + documentAnnotationFormat: { + type: 'json', + optional: true, + }, }, result: { type: { diff --git a/src/backend/src/modules/puterai/MistralAIService.js b/src/backend/src/modules/puterai/MistralAIService.js index 1a7e81f07..c609aa485 100644 --- a/src/backend/src/modules/puterai/MistralAIService.js +++ b/src/backend/src/modules/puterai/MistralAIService.js @@ -22,6 +22,9 @@ const BaseService = require('../../services/BaseService'); const axios = require('axios'); const OpenAIUtil = require('./lib/OpenAIUtil'); const { Context } = require('../../util/context'); +const APIError = require('../../api/APIError'); +const mime = require('mime-types'); +const path = require('path'); /** * MistralAIService class extends BaseService to provide integration with the Mistral AI API. @@ -310,6 +313,69 @@ class MistralAIService extends BaseService { return 'mistral-large-latest'; } static IMPLEMENTS = { + 'driver-capabilities': { + supports_test_mode(iface, method_name) { + return iface === 'puter-ocr' && method_name === 'recognize'; + }, + }, + 'puter-ocr': { + async recognize({ + source, + model, + pages, + includeImageBase64, + imageLimit, + imageMinSize, + bboxAnnotationFormat, + documentAnnotationFormat, + test_mode, + }) { + if ( test_mode ) { + return this._sampleOcrResponse(); + } + if ( ! source ) { + throw APIError.create('missing_required_argument', { + interface_name: 'puter-ocr', + method_name: 'recognize', + arg_name: 'source', + }); + } + + const document = await this._buildDocumentChunkFromSource(source); + const payload = { + model: model ?? 'mistral-ocr-latest', + document, + }; + if ( Array.isArray(pages) ) { + payload.pages = pages; + } + if ( typeof includeImageBase64 === 'boolean' ) { + payload.includeImageBase64 = includeImageBase64; + } + if ( typeof imageLimit === 'number' ) { + payload.imageLimit = imageLimit; + } + if ( typeof imageMinSize === 'number' ) { + payload.imageMinSize = imageMinSize; + } + if ( bboxAnnotationFormat !== undefined ) { + payload.bboxAnnotationFormat = bboxAnnotationFormat; + } + if ( documentAnnotationFormat !== undefined ) { + payload.documentAnnotationFormat = documentAnnotationFormat; + } + + const response = await this.client.ocr.process(payload); + const annotationsRequested = ( + payload.documentAnnotationFormat !== undefined || + payload.bboxAnnotationFormat !== undefined + ); + this._recordOcrUsage(response, payload.model, { + annotationsRequested, + }); + return this._normalizeOcrResponse(response); + }, + }, 'puter-chat-completion': { /** * Returns a list of available models and their details. @@ -399,6 +465,157 @@ class MistralAIService extends BaseService { }, }, }; + + async _buildDocumentChunkFromSource(fileFacade) { + const dataUrl = await this._safeFileValue(fileFacade, 'data_url'); + const webUrl = await this._safeFileValue(fileFacade, 'web_url'); + const filePath = await this._safeFileValue(fileFacade, 'path'); + const fsNode = await this._safeFileValue(fileFacade, 'fs-node'); + const fileName = filePath ? path.basename(filePath) : fsNode?.name; + const inferredMime = this._inferMimeFromName(fileName); + + if ( webUrl ) { + return this._chunkFromUrl(webUrl, fileName, inferredMime); + } + if ( dataUrl ) { + const mimeFromUrl = this._extractMimeFromDataUrl(dataUrl) ?? inferredMime; + return this._chunkFromUrl(dataUrl, fileName, mimeFromUrl); + } + + const buffer = await this._safeFileValue(fileFacade, 'buffer'); + if ( ! buffer ) { + throw APIError.create('field_invalid', null, { + key: 'source', + expected: 'file, data URL, or web URL', + }); + } + const mimeType = inferredMime ?? 'application/octet-stream'; + const generatedDataUrl = this._createDataUrl(buffer, mimeType); + return this._chunkFromUrl(generatedDataUrl, fileName, mimeType); + } + + async _safeFileValue(fileFacade, key) { + if ( ! fileFacade || typeof fileFacade.get !== 'function' ) return undefined; + const maybeCache = fileFacade.values?.values; + if ( maybeCache && Object.prototype.hasOwnProperty.call(maybeCache, key) ) { + return maybeCache[key]; + } + try { + return await fileFacade.get(key); + } catch (e) { + return undefined; + } + } + + _chunkFromUrl(url, fileName, mimeType) { + const lowerName = fileName?.toLowerCase(); + const urlLooksPdf = /\.pdf($|\?)/i.test(url); + const mimeLooksPdf = mimeType?.includes('pdf'); + const isPdf = mimeLooksPdf || urlLooksPdf || (lowerName ? lowerName.endsWith('.pdf') : false); + + if ( isPdf ) { + const chunk = { + type: 'document_url', + documentUrl: url, + }; + if ( fileName ) { + chunk.documentName = fileName; + } + return chunk; + } + + return { + type: 'image_url', + imageUrl: { + url, + }, + }; + } + + _inferMimeFromName(name) { + if ( ! name ) return undefined; + return mime.lookup(name) || undefined; + } + + _extractMimeFromDataUrl(url) { + if ( typeof url !== 'string' ) return undefined; + const match = url.match(/^data:([^;,]+)[;,]/); + return match ? match[1] : undefined; + } + + _createDataUrl(buffer, mimeType) { + return `data:${mimeType || 'application/octet-stream'};base64,${buffer.toString('base64')}`; + } + + _normalizeOcrResponse(response) { + if ( ! response ) return {}; + const normalized = { + model: response.model, + pages: response.pages ?? [], + usage_info: response.usageInfo, + }; + const blocks = []; + if ( Array.isArray(response.pages) ) { + for ( const page of response.pages ) { + if ( typeof page?.markdown !== 'string' ) continue; + const lines = page.markdown.split('\n').map(line => line.trim()).filter(Boolean); + for ( const line of lines ) { + blocks.push({ + type: 'text/mistral:LINE', + text: line, + page: page.index, + }); + } + } + } + normalized.blocks = blocks; + if ( blocks.length ) { + normalized.text = blocks.map(block => block.text).join('\n'); + } else if ( Array.isArray(response.pages) ) { + normalized.text = response.pages.map(page => page?.markdown || '').join('\n\n').trim(); + } + return normalized; + } + + _recordOcrUsage(response, model, { annotationsRequested } = {}) { + try { + if ( ! this.meteringService ) return; + const actor = Context.get('actor'); + if ( ! actor ) return; + const pagesProcessed = + response?.usageInfo?.pagesProcessed ?? + (Array.isArray(response?.pages) ? response.pages.length : 1); + this.meteringService.incrementUsage(actor, 'mistral-ocr:ocr:page', pagesProcessed); + if ( annotationsRequested ) { + this.meteringService.incrementUsage(actor, 'mistral-ocr:annotations:page', pagesProcessed); + } + } catch (e) { + // ignore metering failures to avoid blocking OCR results + } + } + + _sampleOcrResponse() { + const markdown = 'Sample OCR output (test mode).'; + return { + model: 'mistral-ocr-latest', + pages: [ + { + index: 0, + markdown, + images: [], + dimensions: null, + }, + ], + blocks: [ + { + type: 'text/mistral:LINE', + text: markdown, + page: 0, + }, + ], + text: markdown, + }; + } } module.exports = { MistralAIService }; diff --git a/src/backend/src/services/MeteringService/costMaps/mistralCostMap.ts b/src/backend/src/services/MeteringService/costMaps/mistralCostMap.ts index 70a84b242..9e5976926 100644 --- a/src/backend/src/services/MeteringService/costMaps/mistralCostMap.ts +++ b/src/backend/src/services/MeteringService/costMaps/mistralCostMap.ts @@ -57,4 +57,9 @@ export const MISTRAL_COST_MAP = { 'mistral:open-mistral-nemo:completion_tokens': 10, 'mistral:mistral-ocr-latest:prompt_tokens': 100, 'mistral:mistral-ocr-latest:completion_tokens': 300, -}; \ No newline at end of file + // OCR page-based pricing (values in microcents/page) + // $1 / 1000 pages -> $0.001 per page -> 100000 microcents + 'mistral-ocr:ocr:page': 100000, + // $3 / 1000 pages -> $0.003 per page -> 300000 microcents + 'mistral-ocr:annotations:page': 300000, +}; diff --git a/src/backend/src/services/drivers/DriverService.js b/src/backend/src/services/drivers/DriverService.js index ceb595766..906025f17 100644 --- a/src/backend/src/services/drivers/DriverService.js +++ b/src/backend/src/services/drivers/DriverService.js @@ -86,6 +86,7 @@ class DriverService extends BaseService { this.interface_to_implementation = {}; this.interface_to_test_service = {}; this.service_aliases = {}; + this.interface_service_aliases = {}; } _init () { @@ -121,13 +122,12 @@ class DriverService extends BaseService { }, 'no_implementation_available': { status: 502, - message: ({ - iface, - interface_name, - driver - }) => `No implementation available for ` + - (iface ?? interface_name) ? 'interface' : 'driver' + - ' ' + quot(iface ?? interface_name ?? driver) + '.', + message: ({ iface, interface_name, driver }) => { + const has_interface = (iface ?? interface_name) !== undefined; + const target_type = has_interface ? 'interface' : 'driver'; + const target_name = quot(iface ?? interface_name ?? driver); + return `No implementation available for ${target_type} ${target_name}.`; + }, }, }); } @@ -219,7 +219,15 @@ class DriverService extends BaseService { this.interface_to_test_service[interface_name] = service_name; } - register_service_alias (service_name, alias) { + register_service_alias (service_name, alias, options = {}) { + const iface = options.iface; + if ( iface ) { + if ( ! this.interface_service_aliases[iface] ) { + this.interface_service_aliases[iface] = {}; + } + this.interface_service_aliases[iface][alias] = service_name; + return; + } this.service_aliases[alias] = service_name; } @@ -323,7 +331,12 @@ class DriverService extends BaseService { response_metadata: {}, test_mode, }; - driver = this.service_aliases[driver] ?? driver; + const iface_aliases = this.interface_service_aliases[iface]; + if ( iface_aliases && iface_aliases[driver] ) { + driver = iface_aliases[driver]; + } else { + driver = this.service_aliases[driver] ?? driver; + } const service = this.get_service_or_throw_(driver, iface); diff --git a/src/puter-js/index.d.ts b/src/puter-js/index.d.ts index 212ddd48c..4d9ed1abc 100644 --- a/src/puter-js/index.d.ts +++ b/src/puter-js/index.d.ts @@ -47,6 +47,9 @@ interface AI { chat(messages: ChatMessage[], testMode?: boolean, options?: NonStreamingChatOptions): Promise; img2txt(image: string | File | Blob, testMode?: boolean): Promise; + img2txt(image: string | File | Blob, options?: Img2TxtOptions): Promise; + img2txt(image: string | File | Blob, testMode?: boolean, options?: Img2TxtOptions): Promise; + img2txt(options: Img2TxtOptions): Promise; txt2img(prompt: string, testMode?: boolean): Promise; txt2img(prompt: string, options?: Txt2ImgOptions): Promise; @@ -148,6 +151,19 @@ interface Txt2VidOptions { test_mode?: boolean; } +interface Img2TxtOptions { + source?: string | File | Blob; + provider?: 'aws-textract' | 'mistral'; + model?: string; + pages?: number[]; + includeImageBase64?: boolean; + imageLimit?: number; + imageMinSize?: number; + bboxAnnotationFormat?: Record; + documentAnnotationFormat?: Record; + testMode?: boolean; +} + interface Txt2SpeechOptions { language?: string; voice?: string; diff --git a/src/puter-js/src/modules/AI.js b/src/puter-js/src/modules/AI.js index 2f6e19e80..d8ec4ad84 100644 --- a/src/puter-js/src/modules/AI.js +++ b/src/puter-js/src/modules/AI.js @@ -118,48 +118,100 @@ class AI{ } img2txt = async (...args) => { - let MAX_INPUT_SIZE = 10 * 1024 * 1024; - let options = {}; - let testMode = false; - - // Check that the argument is not undefined or null - if(!args){ - throw({message: 'Arguments are required', code: 'arguments_required'}); + const MAX_INPUT_SIZE = 10 * 1024 * 1024; + if (!args || args.length === 0) { + throw { message: 'Arguments are required', code: 'arguments_required' }; } - // if argument is string transform it to the object that the API expects - if (typeof args[0] === 'string' || args[0] instanceof Blob) { + const isBlobLike = (value) => { + if (typeof Blob === 'undefined') return false; + return value instanceof Blob || (typeof File !== 'undefined' && value instanceof File); + }; + const isPlainObject = (value) => value && typeof value === 'object' && !Array.isArray(value) && !isBlobLike(value); + const normalizeProvider = (value) => { + if (!value) return 'aws-textract'; + const normalized = String(value).toLowerCase(); + if (['aws', 'textract', 'aws-textract'].includes(normalized)) return 'aws-textract'; + if (['mistral', 'mistral-ocr'].includes(normalized)) return 'mistral'; + return 'aws-textract'; + }; + + let options = {}; + if (isPlainObject(args[0])) { + options = { ...args[0] }; + } else { options.source = args[0]; } - // if input is a blob, transform it to a data URI - if (args[0].source instanceof Blob) { - options.source = await utils.blobToDataUri(args[0].source); + let testMode = false; + for (let i = 1; i < args.length; i++) { + const value = args[i]; + if (typeof value === 'boolean') { + testMode = testMode || value; + } else if (isPlainObject(value)) { + options = { ...options, ...value }; + } } - // check input size - if (options.source.length > this.MAX_INPUT_SIZE) { + if (typeof options.testMode === 'boolean') { + testMode = options.testMode; + } + + const provider = normalizeProvider(options.provider); + delete options.provider; + delete options.testMode; + + if (!options.source) { + throw { message: 'Source is required', code: 'source_required' }; + } + + if (isBlobLike(options.source)) { + options.source = await utils.blobToDataUri(options.source); + } else if (options.source?.source && isBlobLike(options.source.source)) { + // Support shape { source: Blob } + options.source = await utils.blobToDataUri(options.source.source); + } + + if (typeof options.source === 'string' && + options.source.startsWith('data:') && + options.source.length > MAX_INPUT_SIZE) { throw { message: 'Input size cannot be larger than ' + MAX_INPUT_SIZE, code: 'input_too_large' }; } - // determine if test mode is enabled - if (typeof args[1] === 'boolean' && args[1] === true || - typeof args[2] === 'boolean' && args[2] === true || - typeof args[3] === 'boolean' && args[3] === true) { - testMode = true; - } - - return await utils.make_driver_method(['source'], 'puter-ocr', 'aws-textract', 'recognize', { - test_mode: testMode ?? false, - transform: async (result) => { + const toText = (result) => { + if (!result) return ''; + if (Array.isArray(result.blocks) && result.blocks.length) { let str = ''; - for (let i = 0; i < result?.blocks?.length; i++) { - if("text/textract:LINE" === result.blocks[i].type) - str += result.blocks[i].text + "\n"; + for (const block of result.blocks) { + if (typeof block?.text !== 'string') continue; + if (!block.type || block.type === 'text/textract:LINE' || block.type.startsWith('text/')) { + str += block.text + '\n'; + } } - return str; + if (str.trim()) return str; } - }).call(this, options); + if (Array.isArray(result.pages) && result.pages.length) { + const markdown = result.pages + .map(page => (page?.markdown || '').trim()) + .filter(Boolean) + .join('\n\n'); + if (markdown.trim()) return markdown; + } + if (typeof result.document_annotation === 'string') { + return result.document_annotation; + } + if (typeof result.text === 'string') { + return result.text; + } + return ''; + }; + + const driverCall = utils.make_driver_method(['source'], 'puter-ocr', provider, 'recognize', { + test_mode: testMode ?? false, + transform: async (result) => toText(result), + }); + + return await driverCall.call(this, options); } txt2speech = async (...args) => {