From 0491d5fea7c35d786ffd6e007e80c1dc1e44db7a Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Sat, 8 Aug 2026 06:09:49 -0700 Subject: [PATCH] fix: mcp better uploads (#3528) --- src/backend/vitest.config.ts | 3 + src/docs/src/mcp.md | 5 +- src/mcp-connector/README.md | 36 +++- src/mcp-connector/mcpb/manifest.json | 5 +- src/mcp-connector/src/tools.js | 204 ++++++++++++++++++++- src/mcp-connector/src/tools.upload.test.js | 129 +++++++++++++ 6 files changed, 377 insertions(+), 5 deletions(-) create mode 100644 src/mcp-connector/src/tools.upload.test.js diff --git a/src/backend/vitest.config.ts b/src/backend/vitest.config.ts index b91ef30b8..236720f87 100644 --- a/src/backend/vitest.config.ts +++ b/src/backend/vitest.config.ts @@ -109,6 +109,9 @@ export default defineConfig(({ mode }) => ({ include: [ 'src/backend/**/*.test.{js,ts}', 'extensions/**/*.test.{js,ts}', + // The MCP connector's signed-upload tools call the `/fs` HTTP API + // directly, so their tests need a booted backend (`setupPuterTestEnv`). + 'src/mcp-connector/**/*.test.{js,ts}', ], // Root is the repo root so that the file transformer (which // applies `lowerDecoratorsPlugin`) sees both src/backend and diff --git a/src/docs/src/mcp.md b/src/docs/src/mcp.md index c99e097ba..bbb7a590c 100644 --- a/src/docs/src/mcp.md +++ b/src/docs/src/mcp.md @@ -97,7 +97,10 @@ The Puter MCP server exposes the following tools, grouped by category. Each one ### Filesystem -- `fs_write_file`: Create or overwrite a file in your Puter filesystem. +- `fs_write_file`: Create or overwrite a file in your Puter filesystem from inline content. +- `fs_start_upload`: Get a presigned URL to upload a local file directly to storage, without sending its bytes through the agent. Preferred for large or binary files. +- `fs_complete_upload`: Finalize an upload started with `fs_start_upload` — this is what creates the file. +- `fs_abort_upload`: Discard an upload without creating a file. - `fs_read_file`: Read a file's contents (UTF-8 text, or base64 for binary), optionally just a byte window of it. - `fs_readdir`: List the files and subdirectories in a directory. - `fs_mkdir`: Create a directory, optionally creating missing parents. diff --git a/src/mcp-connector/README.md b/src/mcp-connector/README.md index 0c1b450b0..25fafb4a4 100644 --- a/src/mcp-connector/README.md +++ b/src/mcp-connector/README.md @@ -21,7 +21,10 @@ OAuth "Sign in with Puter" flow the Worker hosts itself (see | --- | --- | | `fs_read_file` | Read a file (UTF-8 or base64; optional byte offset/length window). | | `fs_stat` | Stat a file or directory (size, type, timestamps, uid). | -| `fs_write_file` | Create/overwrite a file (UTF-8 or base64 content). | +| `fs_write_file` | Create/overwrite a file from inline content (UTF-8 or base64). | +| `fs_start_upload` | Get a presigned URL to upload a local file out of band. | +| `fs_complete_upload` | Finalize an upload started with `fs_start_upload`. | +| `fs_abort_upload` | Discard an upload without creating a file. | | `fs_mkdir` | Create a directory (optionally creating missing parents). | | `fs_delete` | Delete a file or directory (recursive by default). | | `fs_readdir` | List the entries of a directory. | @@ -29,6 +32,37 @@ OAuth "Sign in with Puter" flow the Worker hosts itself (see | `fs_move` | Move a file or directory to another location (also renames). | | `fs_rename` | Rename a file or directory in place. | +#### Uploading a local file + +`fs_write_file` carries its content inline, which means a file on disk has to be +base64-encoded through the agent's context before it reaches the server — slow +for anything large, and impossible past the client's message size limit. + +`fs_start_upload` avoids that entirely by handing back a presigned storage URL. +The bytes go straight from the machine holding the file to storage; neither this +server nor the conversation ever sees them. + +``` +fs_start_upload({ path: "~/uploads/build.zip", size: 48210433, local_path: "./build.zip" }) + -> { upload_id, url, upload_command, expires_at, ... } + +# run the returned upload_command: +curl -sS --fail-with-body -X PUT -H 'Content-Type: application/zip' \ + --upload-file './build.zip' 'https://...' + +fs_complete_upload({ upload_id }) -> the created file entry +``` + +The file does not exist in Puter until `fs_complete_upload` succeeds; use +`fs_abort_upload` to release a session whose upload failed. Two constraints come +from the signing itself: `size` must be the file's exact byte count (`wc -c`), +and the PUT must send the same `Content-Type` that was signed — storage rejects +a mismatch on either. The emitted `upload_command` already gets both right. + +Uploads larger than the server's single-PUT ceiling would need a multipart part +dance that these tools don't implement; `fs_start_upload` detects that case, +releases the session, and says so. + ### Hosting (static websites) Publishing a website in Puter means creating a hosting subdomain served at `https://.puter.site`, backed by a directory in your Puter filesystem. diff --git a/src/mcp-connector/mcpb/manifest.json b/src/mcp-connector/mcpb/manifest.json index 01dce4fa6..9e269f3d9 100644 --- a/src/mcp-connector/mcpb/manifest.json +++ b/src/mcp-connector/mcpb/manifest.json @@ -30,7 +30,10 @@ { "name": "whoami", "description": "Get the authenticated user's info (username, uuid, home directory). Use it to build valid paths." }, { "name": "fs_read_file", "description": "Read a file (UTF-8 or base64; optional byte offset/length window)." }, { "name": "fs_stat", "description": "Stat a file or directory (size, type, timestamps, uid)." }, - { "name": "fs_write_file", "description": "Create or overwrite a file (UTF-8 or base64 content)." }, + { "name": "fs_write_file", "description": "Create or overwrite a file from inline content (UTF-8 or base64)." }, + { "name": "fs_start_upload", "description": "Get a presigned URL to upload a local file out of band." }, + { "name": "fs_complete_upload", "description": "Finalize an upload started with fs_start_upload." }, + { "name": "fs_abort_upload", "description": "Discard an upload without creating a file." }, { "name": "fs_mkdir", "description": "Create a directory (optionally creating missing parents)." }, { "name": "fs_delete", "description": "Delete a file or directory (recursive by default)." }, { "name": "fs_readdir", "description": "List the entries of a directory." }, diff --git a/src/mcp-connector/src/tools.js b/src/mcp-connector/src/tools.js index 08b2ded0d..209fbe722 100644 --- a/src/mcp-connector/src/tools.js +++ b/src/mcp-connector/src/tools.js @@ -116,6 +116,66 @@ async function fetchDocText(url) { return resp.text(); } +// ----- signed (out-of-band) uploads ---------------------------------------- +// fs_write_file carries every byte inline, so a large file has to be base64'd +// through the caller's context before it ever reaches us. The fs_*_upload tools +// hand back a presigned storage URL instead: the caller PUTs the file straight +// to storage and then finalizes, and the bytes never pass through this server +// or the conversation. + +/** POST JSON to a Puter API endpoint as the caller, returning the parsed body. */ +async function postApi(puter, endpoint, payload) { + const resp = await fetch(`${puter.APIOrigin}${endpoint}`, { + method: 'POST', + headers: { + Authorization: `Bearer ${puter.authToken}`, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(payload), + }); + const text = await resp.text(); + let body = null; + if (text) { + try { + body = JSON.parse(text); + } catch { + body = text; + } + } + if (!resp.ok) { + const message = (body && (body.message || body.error?.message || body.error)) + || (typeof body === 'string' && body) + || `Request failed with status ${resp.status}`; + const error = new Error(typeof message === 'string' ? message : JSON.stringify(message)); + error.status = resp.status; + throw error; + } + return body; +} + +/** Single-quote a string for safe interpolation into the emitted shell command. */ +function shellQuote(value) { + return `'${String(value).replace(/'/g, `'\\''`)}'`; +} + +/** Best-effort content type from a file extension, for the signed PUT. */ +const UPLOAD_MIME_BY_EXT = { + txt: 'text/plain', md: 'text/markdown', csv: 'text/csv', html: 'text/html', + css: 'text/css', js: 'application/javascript', json: 'application/json', + xml: 'application/xml', pdf: 'application/pdf', zip: 'application/zip', + gz: 'application/gzip', tar: 'application/x-tar', png: 'image/png', + jpg: 'image/jpeg', jpeg: 'image/jpeg', gif: 'image/gif', webp: 'image/webp', + svg: 'image/svg+xml', ico: 'image/x-icon', mp3: 'audio/mpeg', wav: 'audio/wav', + ogg: 'audio/ogg', mp4: 'video/mp4', webm: 'video/webm', mov: 'video/quicktime', +}; + +function guessContentType(path) { + const name = String(path).split('/').pop() || ''; + const dot = name.lastIndexOf('.'); + if (dot <= 0) return 'application/octet-stream'; + return UPLOAD_MIME_BY_EXT[name.slice(dot + 1).toLowerCase()] || 'application/octet-stream'; +} + /** The directory part of a Puter path ('' when the path has no directory part). */ function parentPath(path) { const trimmed = String(path).replace(/\/+$/, ''); @@ -241,8 +301,11 @@ export const TOOLS = [ { name: 'fs_write_file', description: - 'Write (create or overwrite) a file in Puter. Provide content as UTF-8 text, ' + - 'or set encoding="base64" to write binary data. Equivalent to PuterJS puter.fs.write(path, data).', + 'Write (create or overwrite) a file in Puter from content you supply inline. Provide content ' + + 'as UTF-8 text, or set encoding="base64" to write binary data. Best for text you are ' + + 'generating anyway; for a file that already exists on disk — especially a large or binary ' + + 'one — use fs_start_upload instead, which moves the bytes out of band. Equivalent to ' + + 'PuterJS puter.fs.write(path, data).', inputSchema: { type: 'object', properties: { @@ -274,6 +337,143 @@ export const TOOLS = [ }); }, }, + { + name: 'fs_start_upload', + description: + 'Begin an out-of-band file upload and get back a presigned URL to PUT the bytes to. ' + + 'PREFER THIS OVER fs_write_file for any file you already have on disk, and for anything ' + + 'large or binary: the bytes go straight from your machine to storage instead of being ' + + 'base64-encoded through this conversation. Three steps: (1) call this with the destination ' + + 'path and the file\'s exact byte size, (2) run the returned `upload_command` in a shell, ' + + '(3) call fs_complete_upload with the returned upload_id — the file does not exist in Puter ' + + 'until you do. Get the exact size with `wc -c < file` and pass it verbatim; a wrong size is ' + + 'rejected. The URL expires (see expires_at), so upload promptly. If the upload fails, call ' + + 'fs_abort_upload rather than leaving the session dangling.', + inputSchema: { + type: 'object', + properties: { + path: { type: 'string', description: `Destination file path in Puter. ${HOME_PATH_NOTE}` }, + size: { + type: 'integer', + minimum: 0, + description: 'Exact size of the file in bytes (`wc -c < file`). Must match what you upload.', + }, + local_path: { + type: 'string', + description: 'Path of the local file being uploaded. Only used to write the `upload_command` for you; this server never reads it.', + }, + content_type: { + type: 'string', + description: 'MIME type. Defaults to a guess from the destination extension. The upload MUST send this exact value as its Content-Type header — `upload_command` already does.', + }, + expires_in: { + type: 'integer', + description: 'Seconds the upload URL stays valid. Clamped to 60..3600 by the server. Defaults to 900.', + }, + overwrite: { type: 'boolean', default: true, description: 'Overwrite an existing file.' }, + create_missing_parents: { + type: 'boolean', + default: true, + description: 'Create missing parent directories.', + }, + dedupe_name: { + type: 'boolean', + default: false, + description: 'Auto-rename instead of overwriting if the file exists.', + }, + }, + required: ['path', 'size'], + }, + async handler(puter, { + path, size, local_path, content_type, expires_in, + overwrite = true, create_missing_parents = true, dedupe_name = false, + }) { + const contentType = content_type || guessContentType(path); + const startResponse = await postApi(puter, '/fs/startBatchWrite', [{ + fileMetadata: { + path, + size, + contentType, + overwrite, + dedupeName: dedupe_name, + createMissingParents: create_missing_parents, + }, + uploadMode: 'single', + ...(expires_in ? { expiresInSeconds: expires_in } : {}), + }]); + const started = Array.isArray(startResponse) ? startResponse[0] : null; + + if (!started || !started.sessionId) { + throw new Error( + 'This Puter server did not return a presigned upload URL. Use fs_write_file instead.', + ); + } + + // The server upgrades anything past its single-PUT ceiling to a + // multipart upload, which needs a per-part ETag dance we don't + // support here. Release the session rather than leaving it pending. + if (started.uploadMode !== 'single' || !started.url) { + await postApi(puter, '/fs/abortWrite', { uploadId: started.sessionId }).catch(() => {}); + throw new Error( + `File is too large for a single-shot signed upload (${size} bytes). ` + + 'Split it into smaller files, or use fs_write_file if it is small enough to inline.', + ); + } + + const target = local_path ? shellQuote(local_path) : ''; + return { + upload_id: started.sessionId, + url: started.url, + content_type: started.contentType || contentType, + size, + path, + expires_at: new Date(started.expiresAt).toISOString(), + upload_command: + `curl -sS --fail-with-body -X PUT ` + + `-H ${shellQuote(`Content-Type: ${started.contentType || contentType}`)} ` + + `--upload-file ${target} ${shellQuote(started.url)}`, + next_step: + 'Run upload_command, then call fs_complete_upload with this upload_id. ' + + 'The file is not visible in Puter until that call succeeds.', + }; + }, + }, + { + name: 'fs_complete_upload', + description: + 'Finalize an upload started with fs_start_upload, after the bytes have been PUT to the ' + + 'presigned URL. This is what actually creates the file in Puter — skip it and the upload ' + + 'is discarded. Returns the created file entry.', + inputSchema: { + type: 'object', + properties: { + upload_id: { type: 'string', description: 'The upload_id returned by fs_start_upload.' }, + }, + required: ['upload_id'], + }, + async handler(puter, { upload_id }) { + const response = await postApi(puter, '/fs/completeBatchWrite', [{ uploadId: upload_id }]); + const completed = Array.isArray(response) ? response[0] : response; + return (completed && completed.fsEntry) || completed; + }, + }, + { + name: 'fs_abort_upload', + description: + 'Discard an upload started with fs_start_upload without creating a file. Use this when the ' + + 'PUT failed or you changed your mind, so the pending upload does not linger.', + inputSchema: { + type: 'object', + properties: { + upload_id: { type: 'string', description: 'The upload_id returned by fs_start_upload.' }, + }, + required: ['upload_id'], + }, + async handler(puter, { upload_id }) { + await postApi(puter, '/fs/abortWrite', { uploadId: upload_id }); + return { success: true, aborted: upload_id }; + }, + }, { name: 'fs_mkdir', description: 'Create a directory in Puter (optionally creating missing parents). Equivalent to PuterJS puter.fs.mkdir(path).', diff --git a/src/mcp-connector/src/tools.upload.test.js b/src/mcp-connector/src/tools.upload.test.js new file mode 100644 index 000000000..efabc509b --- /dev/null +++ b/src/mcp-connector/src/tools.upload.test.js @@ -0,0 +1,129 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { setupPuterTestEnv } from '../../backend/testUtil.js'; +import { TOOL_MAP } from './tools.js'; + +/** + * The signed-upload tools talk to `/fs/*` over HTTP rather than through + * puter.js, so the only thing that proves they work is a real round trip: mint + * a URL, PUT bytes to it the way the emitted shell command would, finalize, and + * read the file back. A stubbed API would just re-assert our own assumptions + * about the request shape — which is the part most likely to be wrong. + */ +describe('signed upload tools', () => { + let env; + /** Stands in for the caller's puter.js instance — the tools only read these two. */ + let puter; + + beforeAll(async () => { + env = await setupPuterTestEnv(); + puter = { + APIOrigin: new URL(env.apiOrigin).origin, + authToken: env.users.user.token, + }; + }, 120_000); + + afterAll(async () => { + await env?.shutdown(); + }); + + const call = (name, args) => TOOL_MAP.get(name).handler(puter, args); + + const homePath = (name) => `/${env.users.user.username}/Documents/${name}`; + + it('uploads a file out of band and finalizes it', async () => { + const path = homePath(`signed-upload-${Date.now()}.txt`); + const content = 'streamed straight to storage\n'; + const size = Buffer.byteLength(content); + + const started = await call('fs_start_upload', { + path, + size, + local_path: './local.txt', + }); + + expect(started.upload_id).toEqual(expect.any(String)); + expect(started.url).toMatch(/^https?:\/\//); + expect(started.content_type).toBe('text/plain'); + expect(new Date(started.expires_at).getTime()).toBeGreaterThan(Date.now()); + // The command has to carry the signed content type, or storage 403s. + expect(started.upload_command).toContain("-H 'Content-Type: text/plain'"); + expect(started.upload_command).toContain("--upload-file './local.txt'"); + + // What `upload_command` does, minus the shell. + const put = await fetch(started.url, { + method: 'PUT', + headers: { 'Content-Type': started.content_type }, + body: content, + }); + expect(put.status).toBe(200); + + // Not a file until it is completed. + const completed = await call('fs_complete_upload', { + upload_id: started.upload_id, + }); + expect(completed.path).toBe(path); + expect(completed.size).toBe(size); + + const readBack = await fetch( + new URL( + `/fs/read?path=${encodeURIComponent(path)}&auth_token=${env.users.user.token}`, + env.apiOrigin, + ), + ); + expect(await readBack.text()).toBe(content); + }); + + it('guesses the content type from the destination extension', async () => { + const started = await call('fs_start_upload', { + path: homePath(`signed-upload-${Date.now()}.png`), + size: 3, + }); + expect(started.content_type).toBe('image/png'); + await call('fs_abort_upload', { upload_id: started.upload_id }); + }); + + it('aborts an upload without creating a file', async () => { + const path = homePath(`signed-abort-${Date.now()}.bin`); + const started = await call('fs_start_upload', { path, size: 8 }); + + const aborted = await call('fs_abort_upload', { + upload_id: started.upload_id, + }); + expect(aborted).toEqual({ success: true, aborted: started.upload_id }); + + // Completing an aborted session must not resurrect the file. + await expect( + call('fs_complete_upload', { upload_id: started.upload_id }), + ).rejects.toThrow(); + }); + + it('rejects a file too large for a single-shot upload', async () => { + // Past the server's single-PUT ceiling the backend switches to + // multipart, which these tools deliberately do not implement. + await expect( + call('fs_start_upload', { + path: homePath('signed-too-big.bin'), + size: 1024 * 1024 * 1024, + }), + ).rejects.toThrow(/too large for a single-shot signed upload/); + }); +});