mirror of
https://github.com/HeyPuter/puter.git
synced 2026-09-09 14:55:47 +00:00
Add resilient PDF thumbnails to GUI uploads (#3809)
* Add resilient PDF thumbnails to GUI uploads * Keep GUI image thumbnails working with SDKs lacking the callback context The desktop loads puter.js from js.puter.com by default, and the SDK there predates the thumbnail callback context, so passing the PDF generator made every image upload lose its thumbnail until the SDK deploys. Fall back to the SDK's bundled image generator whenever the running SDK passes no usable context, and cover both paths in the unit and browser tests. * Ignore preparation failures that land after an upload is cancelled Cancelling during preparation already rejects the upload and fires the abort callback. If the step that was in flight then fails, such as a dropped directory that cannot be read, the error callback also fired and the GUI showed an upload error for an upload the user had just cancelled. Skip error reporting once preparation has been aborted. * Give each PDF thumbnail worker four seconds The per-PDF budget covers downloading PDF.js as well as rendering, and the first PDF of a session on a slower connection ran out of time before its assets had even loaded. Four seconds fits that first load on ordinary connections while staying under the five-second batch cap, so one stuck PDF still leaves the rest of the batch a chance.
This commit is contained in:
Nariman Jelveh
Generated
+279
-1
@@ -3438,6 +3438,271 @@
|
||||
"node": ">= 10"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas/-/canvas-1.0.8.tgz",
|
||||
"integrity": "sha512-/SaLcvlqGWdm0HSCWMgHu7cjJiQXfP8/mOY+6dUyV9flQz7sPBBZ+ed2zYtoukojPmxOaL7bm+d/G4GeWWoN7g==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"workspaces": [
|
||||
"e2e/*"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas-android-arm64": "1.0.8",
|
||||
"@napi-rs/canvas-darwin-arm64": "1.0.8",
|
||||
"@napi-rs/canvas-darwin-x64": "1.0.8",
|
||||
"@napi-rs/canvas-linux-arm-gnueabihf": "1.0.8",
|
||||
"@napi-rs/canvas-linux-arm64-gnu": "1.0.8",
|
||||
"@napi-rs/canvas-linux-arm64-musl": "1.0.8",
|
||||
"@napi-rs/canvas-linux-riscv64-gnu": "1.0.8",
|
||||
"@napi-rs/canvas-linux-x64-gnu": "1.0.8",
|
||||
"@napi-rs/canvas-linux-x64-musl": "1.0.8",
|
||||
"@napi-rs/canvas-win32-arm64-msvc": "1.0.8",
|
||||
"@napi-rs/canvas-win32-x64-msvc": "1.0.8"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-android-arm64": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-android-arm64/-/canvas-android-arm64-1.0.8.tgz",
|
||||
"integrity": "sha512-5+nkh8i3gt6lqS/d2jTZ1xAn6tdgtB4Lf1mW6T0Qm5/rXNwBuV1sAEyLEWan5o9gJPU/GuvHR3rvSeZ+FaGrbw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"android"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-arm64": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-arm64/-/canvas-darwin-arm64-1.0.8.tgz",
|
||||
"integrity": "sha512-7jQ47gi+fZ7KJmfc/5rNyy1CYw/cu4kZ0KPIYbo9UUgSdW0bKQJpt+WihEor6s4Lyp7+xc3a+3HeyXmAEbbnPg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-darwin-x64": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-darwin-x64/-/canvas-darwin-x64-1.0.8.tgz",
|
||||
"integrity": "sha512-rRjDMZs9pIRKGxgijwezplKc1RnJsqUokrA9h88bbTkqQ+7ePj0ZN4ZnZDy8Vu0tXs7KRlI2tQLaK4mx9QlxHg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm-gnueabihf": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm-gnueabihf/-/canvas-linux-arm-gnueabihf-1.0.8.tgz",
|
||||
"integrity": "sha512-jGcCd+8ra6Q61xKqZeiItujTpp9a9eRLcQ0jW6qYNku+WpupqOPFPY0SrsuSnXFviJwkpKYT9p7QrB4lsf3LNQ==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-gnu": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-gnu/-/canvas-linux-arm64-gnu-1.0.8.tgz",
|
||||
"integrity": "sha512-od6I2Y7kU7i1SwZYG2EKW8rWz6JiedtPpko4WEe1DDsiikrfaotVBCRaUTM5/yeZKaZ92EatoAS+5xG+6uJlYA==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-arm64-musl": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-arm64-musl/-/canvas-linux-arm64-musl-1.0.8.tgz",
|
||||
"integrity": "sha512-yYkPbJDJiWj6N0gASA3CAvRypZmVpJnxU0DQg3aBhneLDQde9TPLKADsQkobNoJUtTT/lj46aWpzT48PDb3Qcg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-riscv64-gnu": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-riscv64-gnu/-/canvas-linux-riscv64-gnu-1.0.8.tgz",
|
||||
"integrity": "sha512-PB00MSKAp4VwK/xwe6duKxRKmH8UH4GIl1pqHSbxng0jnU9Dr7FwaDypDiqwNFZ774N+8G7mJLGuLtg9NTcQsg==",
|
||||
"cpu": [
|
||||
"riscv64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-gnu": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-gnu/-/canvas-linux-x64-gnu-1.0.8.tgz",
|
||||
"integrity": "sha512-TWM2XWJoitLiIPCvgJh7SriC+L/T9qkYCVzC66AidsZy0QP1hkKzBzVwshCdcA3q6fIn3yE0ISbq4lMJSy8jFw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"glibc"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-linux-x64-musl": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-linux-x64-musl/-/canvas-linux-x64-musl-1.0.8.tgz",
|
||||
"integrity": "sha512-hb20MxKXXb5IB7AAwN8UHz9WRsa2HmdZfjsDCzjElwJoeV1aotVEwFU4FrFQcYQVzsJQLeaCc/2Qdt/0Q72mMg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"libc": [
|
||||
"musl"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-arm64-msvc": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-arm64-msvc/-/canvas-win32-arm64-msvc-1.0.8.tgz",
|
||||
"integrity": "sha512-WwPN08IXE4SkL+FhJyPz/iFnycMAUkbphFIT4cmKLlvbSU0Zfn1R7BGJ3Hqky1S89QUYc0Q4IOScXb/42Re9wQ==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/canvas-win32-x64-msvc": {
|
||||
"version": "1.0.8",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/canvas-win32-x64-msvc/-/canvas-win32-x64-msvc-1.0.8.tgz",
|
||||
"integrity": "sha512-XkrVqKb+pxyba7kjy2LJvABFVBTE0DNpEl7MrG4OYUmaWarrXH+t54z/Czj2YxCKtizYTV4mg6phNm3x24qjhQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
],
|
||||
"engines": {
|
||||
"node": ">= 10"
|
||||
},
|
||||
"funding": {
|
||||
"type": "github",
|
||||
"url": "https://github.com/sponsors/Brooooooklyn"
|
||||
}
|
||||
},
|
||||
"node_modules/@napi-rs/wasm-runtime": {
|
||||
"version": "1.2.3",
|
||||
"resolved": "https://registry.npmjs.org/@napi-rs/wasm-runtime/-/wasm-runtime-1.2.3.tgz",
|
||||
@@ -13652,6 +13917,18 @@
|
||||
"dev": true,
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "6.3.289",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-6.3.289.tgz",
|
||||
"integrity": "sha512-ZHjSVpDa3D6izMq8/04lvkhkATUmL9px6ChPaXc1k6nU2Mrhlg1/7F0bdUqCwUjw3NsPTfPZsMDUU6ZIcRaeQw==",
|
||||
"license": "Apache-2.0",
|
||||
"engines": {
|
||||
"node": ">=22.13.0 || >=24"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@napi-rs/canvas": "^1.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pg": {
|
||||
"version": "8.22.0",
|
||||
"resolved": "https://registry.npmjs.org/pg/-/pg-8.22.0.tgz",
|
||||
@@ -18676,7 +18953,8 @@
|
||||
],
|
||||
"dependencies": {
|
||||
"@prelude.so/js-sdk": "0.12.1",
|
||||
"@thumbmarkjs/thumbmarkjs": "1.9.1"
|
||||
"@thumbmarkjs/thumbmarkjs": "1.9.1",
|
||||
"pdfjs-dist": "6.3.289"
|
||||
},
|
||||
"devDependencies": {
|
||||
"chalk": "^4.1.0",
|
||||
|
||||
@@ -5,6 +5,8 @@ description: Store and manage data in the user's own cloud drive with Puter.js f
|
||||
|
||||
The Cloud Storage API lets you store and manage data in the cloud.
|
||||
|
||||
Local [uploads](/FS/upload/) can optionally generate browser image thumbnails or use a custom thumbnail callback. The callback can delegate to the built-in image generator and respond to upload cancellation. The Puter desktop additionally provides PDF previews without adding a PDF renderer to the SDK.
|
||||
|
||||
It comes with a comprehensive but familiar file system operations including write, read, delete, move, and copy for files, plus powerful directory management features like creating directories, listing contents, and much more.
|
||||
|
||||
With Puter.js, you don't need to worry about setting up storage infrastructure such as configuring buckets, managing CDNs, or ensuring availability, since everything is handled for you. Additionally, with the [User-Pays Model](/user-pays-model/), you don't have to worry about storage or bandwidth costs, as users of your application cover their own usage.
|
||||
|
||||
@@ -31,6 +31,9 @@ A set of key/value pairs that configure the upload process. The following option
|
||||
- `overwrite` (Boolean) - Whether to overwrite the destination file if it already exists. Defaults to `false`.
|
||||
- `dedupeName` (Boolean) - Whether to deduplicate the file name if it already exists. Defaults to `true`. Ignored when `overwrite` is `true`.
|
||||
- `createMissingParents` (Boolean) - Whether to create missing parent directories. Defaults to `false`.
|
||||
- `generateThumbnails` (Boolean) - Generate image thumbnails in the browser before uploading. Defaults to `false`. Unsupported files and generation failures are skipped.
|
||||
- `thumbnailGenerator` (Function) - Optional `(file, context) => string | undefined` callback (which may also return a promise), called once per file instead of the built-in image generator. Return a thumbnail data URL or URL, or `undefined` to skip. Exceptions are ignored. `context.defaultGenerator(file)` delegates to the built-in image generator; `context.signal` is an `AbortSignal` for upload preparation cancellation. Existing one-argument callbacks continue to work. A custom generator enables thumbnail preparation even when `generateThumbnails` is omitted.
|
||||
- `thumbnail` (String) - Optional thumbnail data URL or URL to use when a file has no generated thumbnail. Data URLs exceeding 2 MiB are discarded.
|
||||
|
||||
The following callbacks report on the upload as it runs. `operationId` identifies the upload, so a page running several uploads at once can tell them apart:
|
||||
|
||||
@@ -39,6 +42,8 @@ The following callbacks report on the upload as it runs. `operationId` identifie
|
||||
- `progress` (Function) - Called with `(operationId, progress)` as bytes are sent, where `progress` is a percentage between `0` and `100`.
|
||||
- `abort` (Function) - Called with `(operationId)` if the upload is aborted.
|
||||
|
||||
Cancelling through the `init` request handle during thumbnail preparation rejects with `{ code: 'upload_aborted', message: 'Upload aborted.' }` and prevents the upload from starting. Custom generators should stop their work when `context.signal` aborts and impose their own time and resource budgets; the SDK awaits thumbnail preparation before sending files.
|
||||
|
||||
```js
|
||||
puter.fs.upload(items, './uploads', {
|
||||
progress: (operationId, progress) => {
|
||||
@@ -68,6 +73,22 @@ On `nodejs` and `workers`, where the upload goes through an older batch endpoint
|
||||
|
||||
Directory uploads (dropped directory entries, or `createFileParent`) are supported on `websites` and `apps`. On `nodejs` and `workers` the upload goes through an older batch endpoint that cannot create the directory tree, so a directory upload rejects with `batch_upload_failed`; create the directories with [`puter.fs.mkdir()`](/FS/mkdir/) and upload the files into them instead.
|
||||
|
||||
## Thumbnails
|
||||
|
||||
The built-in generator handles browser-decodable images. PDF rendering is provided separately by the Puter desktop; PDF.js is not included in the SDK. Apps can supply their own renderer through `thumbnailGenerator` and delegate other files to `context.defaultGenerator`:
|
||||
|
||||
```js
|
||||
const file = new File(['Hello!'], 'hello.txt', { type: 'text/plain' });
|
||||
await puter.fs.upload(file, './', {
|
||||
thumbnailGenerator: async (file, { defaultGenerator, signal }) => {
|
||||
if (signal.aborted) return undefined;
|
||||
return defaultGenerator(file);
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
When using signed uploads, a separate thumbnail transfer that fails or exceeds five seconds is skipped and the original file still uploads. Explicit upload cancellation still stops the upload. Errors transferring the original file continue to reject normally.
|
||||
|
||||
## Examples
|
||||
|
||||
<strong class="example-title">Upload a file from a file input</strong>
|
||||
|
||||
@@ -97,6 +97,21 @@ All per minute unless stated:
|
||||
|
||||
Signed-URL routes have no session to key on, so they are bounded per network rather than per account: 3,000 reads/min, 600 writes/min, 60 concurrent.
|
||||
|
||||
The Puter desktop generates PDF upload thumbnails locally with these best-effort budgets. Exceeding them skips the preview and does not reject the original file upload:
|
||||
|
||||
| PDF thumbnail preparation | Limit |
|
||||
| --- | --- |
|
||||
| Input PDF size | 20 MiB |
|
||||
| Active PDF renderers per desktop page | 1 |
|
||||
| Preparation per upload, including queued PDFs | 5 seconds from the first eligible PDF |
|
||||
| Worker lifetime per PDF, including asset loading and cleanup | 4 seconds |
|
||||
| Embedded image or intermediate canvas area | 4,194,304 pixels |
|
||||
| Image resize budget passed to PDF.js | 16 MiB |
|
||||
| Output | First page, at most 128 × 128 pixels, preserving aspect ratio |
|
||||
| Thumbnail payload | 2 MiB |
|
||||
|
||||
The SDK allows five seconds for each separate signed thumbnail transfer. A failed or timed-out thumbnail transfer is skipped; explicit upload cancellation and failures transferring the original file still stop the upload. These are preview budgets, not upload file-size limits. The PDF renderer's memory budgets do not constitute a hard limit on total browser-process memory.
|
||||
|
||||
### WebDAV
|
||||
|
||||
The `dav` host authenticates each request itself, so its limits are bounded per network rather than per account: **600 requests/min** and **10 concurrent**, one ceiling for everyone.
|
||||
|
||||
@@ -42,6 +42,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@prelude.so/js-sdk": "0.12.1",
|
||||
"@thumbmarkjs/thumbmarkjs": "1.9.1"
|
||||
"@thumbmarkjs/thumbmarkjs": "1.9.1",
|
||||
"pdfjs-dist": "6.3.289"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
|
||||
/* eslint-disable no-invalid-this */
|
||||
/* eslint-disable @stylistic/quotes */
|
||||
import { createUploadThumbnailGenerator } from '../../services/pdfThumbnails/index.js';
|
||||
import path from '../../lib/path.js';
|
||||
import open_item from '../../helpers/openItem.js';
|
||||
import UIContextMenu from '../UIContextMenu.js';
|
||||
@@ -1278,6 +1279,7 @@ const TabFiles = {
|
||||
|
||||
puter.fs.upload(files, _this.currentPath, {
|
||||
generateThumbnails: true,
|
||||
thumbnailGenerator: createUploadThumbnailGenerator(),
|
||||
init: async (operation_id, xhr) => {
|
||||
opid = operation_id;
|
||||
// create upload progress window
|
||||
@@ -4716,6 +4718,7 @@ const TabFiles = {
|
||||
|
||||
puter.fs.upload(items, destPath, {
|
||||
generateThumbnails: true,
|
||||
thumbnailGenerator: createUploadThumbnailGenerator(),
|
||||
init: async (operation_id, xhr) => {
|
||||
opid = operation_id;
|
||||
upload_progress_window = await UIWindowProgress({
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { createUploadThumbnailGenerator } from '../services/pdfThumbnails/index.js';
|
||||
import UITaskbarItem from './UITaskbarItem.js';
|
||||
import UIPopover from './UIPopover.js';
|
||||
import launch_app from '../helpers/launchApp.js';
|
||||
@@ -252,6 +253,7 @@ async function UITaskbar (options) {
|
||||
});
|
||||
await puter.fs.upload(new File([content], fileName), window.desktop_path, {
|
||||
generateThumbnails: true,
|
||||
thumbnailGenerator: createUploadThumbnailGenerator(),
|
||||
});
|
||||
} catch ( err ) {
|
||||
console.error('Failed to add shortcut to desktop:', err);
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
import { createUploadThumbnailGenerator } from './services/pdfThumbnails/index.js';
|
||||
import get_html_element_from_options from './helpers/getHtmlElementFromOptions.js';
|
||||
import globToRegExp from './helpers/globToRegExp.js';
|
||||
import item_icon from './helpers/itemIcon.js';
|
||||
@@ -1159,6 +1160,7 @@ window.create_file = async (options) => {
|
||||
try {
|
||||
puter.fs.upload(new File(content, filename), dirname, {
|
||||
generateThumbnails: true,
|
||||
thumbnailGenerator: createUploadThumbnailGenerator(),
|
||||
success: async function (data) {
|
||||
const created_file = $(appendto_element).find(`.item[data-path="${html_encode(dirname)}/${html_encode(data.name)}"]`);
|
||||
if ( created_file.length > 0 ) {
|
||||
@@ -2372,6 +2374,7 @@ window.upload_items = async function (items, dest_path) {
|
||||
// options
|
||||
{
|
||||
generateThumbnails: true,
|
||||
thumbnailGenerator: createUploadThumbnailGenerator(),
|
||||
// init
|
||||
init: async (operation_id, xhr) => {
|
||||
opid = operation_id;
|
||||
@@ -2869,6 +2872,7 @@ window.unzipItem = async function (itemPath) {
|
||||
{
|
||||
createFileParent: true,
|
||||
generateThumbnails: true,
|
||||
thumbnailGenerator: createUploadThumbnailGenerator(),
|
||||
progress: async function (operation_id, op_progress) {
|
||||
progwin.set_progress(op_progress);
|
||||
// update title if window is not visible
|
||||
@@ -3119,6 +3123,7 @@ window.untarItem = async function (itemPath) {
|
||||
{
|
||||
createFileParent: true,
|
||||
generateThumbnails: true,
|
||||
thumbnailGenerator: createUploadThumbnailGenerator(),
|
||||
progress: async function (operation_id, op_progress) {
|
||||
progwin.set_progress(op_progress);
|
||||
if ( document.visibilityState !== 'visible' ) {
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
# PDF upload thumbnails
|
||||
|
||||
Create one `createUploadThumbnailGenerator()` callback per GUI upload. The SDK
|
||||
passes a built-in image generator and preparation cancellation signal as the
|
||||
callback's second argument. Non-PDF files delegate to that image generator; when
|
||||
the running SDK predates the callback context (it then passes only the file),
|
||||
the GUI's bundled copy of the SDK image generator is used instead, so image
|
||||
thumbnails do not depend on deploy order.
|
||||
|
||||
PDF.js and its fonts, CMaps, ICC profiles and WASM decoders are copied into the
|
||||
versioned `/dist/pdf-thumbnails/` directory during the GUI build. Deploy that
|
||||
directory with the GUI. `PDFJS_VERSION` must match the exact GUI dependency;
|
||||
the build rejects mismatches. The SDK and initial GUI bundle contain no PDF.js.
|
||||
Asset requests stay on the GUI origin and begin only for an eligible PDF.
|
||||
|
||||
Each PDF gets a disposable module worker. PDF.js uses its loopback transport
|
||||
inside that worker, so termination stops both parsing and rasterization. Fonts
|
||||
use PDF.js's outline renderer and canvases use OffscreenCanvas. PDFs requiring
|
||||
unavailable DOM rendering features, a password, or resources beyond the budgets
|
||||
retain their normal icon. Empty first pages also retain their normal icon. Browsers lacking the required APIs also skip previews.
|
||||
All budgets are published in the developer docs' rate-limits-and-quotas page.
|
||||
|
||||
Run `npm test -- src/services/pdfThumbnails` from `src/gui` for scheduling tests
|
||||
and Chromium rendering tests (`npx playwright install chromium` if needed).
|
||||
The browser tests use the same asset-copy step as production, render real PDF
|
||||
fixtures, and check first-page pixels, text, rotation, image delegation, failure
|
||||
recovery and termination of CPU-bound work. The SDK upload suites cover callback
|
||||
compatibility and cancellation; its HTTP transport tests cover thumbnail
|
||||
failure, timeout and original-file failure separately.
|
||||
@@ -0,0 +1,179 @@
|
||||
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
|
||||
import { chromium } from '@playwright/test';
|
||||
import express from 'express';
|
||||
import { mkdtemp, readFile, rm } from 'node:fs/promises';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { copyPdfThumbnailAssets } from '../../../tools/copyPdfThumbnailAssets.js';
|
||||
import { createPdf } from '../../../tests/fixtures/pdf.js';
|
||||
|
||||
let browser;
|
||||
let server;
|
||||
let directory;
|
||||
let origin;
|
||||
let workerMode;
|
||||
|
||||
beforeAll(async () => {
|
||||
directory = await mkdtemp(path.join(tmpdir(), 'puter-pdf-thumbnails-'));
|
||||
await copyPdfThumbnailAssets(directory);
|
||||
const app = express();
|
||||
app.get('/dist/pdf-thumbnails/:version/worker.js', (_req, res, next) => {
|
||||
if ( workerMode === 'missing' ) return res.sendStatus(404);
|
||||
if ( workerMode === 'busy' ) return res.type('js').send('self.onmessage = () => { while (true) {} };');
|
||||
next();
|
||||
});
|
||||
app.use('/dist', express.static(directory));
|
||||
// The generator's relative import into the SDK source climbs past /src, which the browser clamps to /puter-js/src.
|
||||
app.use('/puter-js/src', express.static(fileURLToPath(new URL('../../../../puter-js/src/', import.meta.url))));
|
||||
app.use('/src', express.static(fileURLToPath(new URL('../../', import.meta.url))));
|
||||
app.get('/', (_req, res) => res.send(`<!doctype html><title>Thumbnail test</title><script type="module">
|
||||
import { createUploadThumbnailGenerator } from '/src/services/pdfThumbnails/index.js';
|
||||
import { defaultThumbnailGenerator } from '/puter-js/src/modules/FileSystem/operations/upload/thumbnails.js';
|
||||
window.defaultGenerator = defaultThumbnailGenerator;
|
||||
window.createUploadThumbnailGenerator = createUploadThumbnailGenerator;
|
||||
</script>`));
|
||||
server = await new Promise(resolve => {
|
||||
const listener = app.listen(0, '127.0.0.1', () => resolve(listener));
|
||||
});
|
||||
origin = `http://127.0.0.1:${server.address().port}`;
|
||||
browser = await chromium.launch();
|
||||
}, 30000);
|
||||
|
||||
afterAll(async () => {
|
||||
await browser?.close();
|
||||
if ( server ) await new Promise(resolve => server.close(resolve));
|
||||
if ( directory ) await rm(directory, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe('PDF thumbnails in a real browser', () => {
|
||||
it.each([
|
||||
{ rotation: 0, scanned: false },
|
||||
{ rotation: 90, scanned: false },
|
||||
{ rotation: 0, scanned: true },
|
||||
])('renders the first page with rotation and scanned images: %j', async (options) => {
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
await page.goto(origin);
|
||||
await page.waitForFunction(() => window.createUploadThumbnailGenerator);
|
||||
const result = await page.evaluate(async ({ pdf }) => {
|
||||
const thumbnail = await window.createUploadThumbnailGenerator()(new File([pdf], 'test.pdf'));
|
||||
if ( ! thumbnail ) return null;
|
||||
const image = new Image();
|
||||
image.src = thumbnail;
|
||||
await image.decode();
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = image.width;
|
||||
canvas.height = image.height;
|
||||
const ctx = canvas.getContext('2d');
|
||||
ctx.drawImage(image, 0, 0);
|
||||
const pixels = ctx.getImageData(0, 0, image.width, image.height).data;
|
||||
let whitePixels = 0;
|
||||
for ( let i = 0; i < pixels.length; i += 4 ) {
|
||||
if ( pixels[i] > 240 && pixels[i + 1] > 40 && pixels[i + 2] > 40 ) whitePixels++;
|
||||
}
|
||||
return { width: image.width, height: image.height, pixel: [...ctx.getImageData(5, 5, 1, 1).data], hasText: whitePixels > 0 };
|
||||
}, { pdf: createPdf(options) });
|
||||
expect(result).toEqual({
|
||||
width: options.rotation ? 128 : 64,
|
||||
height: options.rotation ? 64 : 128,
|
||||
pixel: [255, 0, 0, 255],
|
||||
hasText: !options.scanned,
|
||||
});
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('retains image thumbnails in mixed uploads and leaves original PDF bytes intact', async () => {
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
await page.goto(origin);
|
||||
await page.waitForFunction(() => window.createUploadThumbnailGenerator);
|
||||
const result = await page.evaluate(async (pdf) => {
|
||||
const canvas = document.createElement('canvas');
|
||||
canvas.width = canvas.height = 10;
|
||||
const image = new File([await new Promise(resolve => canvas.toBlob(resolve))], 'image.png', { type: 'image/png' });
|
||||
const file = new File([pdf], 'document.PDF');
|
||||
const generate = window.createUploadThumbnailGenerator();
|
||||
const thumbnails = await Promise.all([file, image, new File(['notes'], 'notes.txt')]
|
||||
.map(file => generate(file, { defaultGenerator: window.defaultGenerator })));
|
||||
// An SDK without the callback context calls the generator with the file alone.
|
||||
const withoutContext = await generate(image);
|
||||
return { thumbnails, withoutContext, original: await file.text() };
|
||||
}, createPdf());
|
||||
expect(result.thumbnails[0]).toMatch(/^data:image\/png;base64,/);
|
||||
expect(result.thumbnails[1]).toMatch(/^data:image\//);
|
||||
expect(result.thumbnails[2]).toBeUndefined();
|
||||
expect(result.withoutContext).toMatch(/^data:image\//);
|
||||
expect(result.original).toBe(createPdf());
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('skips encrypted, malformed, and oversized-image PDFs without unhandled errors', async () => {
|
||||
const page = await browser.newPage();
|
||||
const errors = [];
|
||||
page.on('pageerror', error => errors.push(error.message));
|
||||
try {
|
||||
await page.goto(origin);
|
||||
await page.waitForFunction(() => window.createUploadThumbnailGenerator);
|
||||
const encrypted = [...await readFile(new URL('../../../tests/fixtures/encrypted.pdf', import.meta.url))];
|
||||
const result = await page.evaluate(async ({ encrypted, oversized }) => {
|
||||
const generate = window.createUploadThumbnailGenerator();
|
||||
return await Promise.all([
|
||||
new File([new Uint8Array(encrypted)], 'encrypted.pdf'),
|
||||
new File(['not a PDF'], 'broken.pdf'),
|
||||
new File([oversized], 'oversized.pdf'),
|
||||
].map(file => generate(file)));
|
||||
}, { encrypted, oversized: createPdf({ scanned: true, imageSize: 4096 }) });
|
||||
expect(result).toEqual([undefined, undefined, undefined]);
|
||||
expect(errors).toEqual([]);
|
||||
} finally {
|
||||
await page.close();
|
||||
}
|
||||
});
|
||||
|
||||
it('recovers after missing renderer assets', async () => {
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
workerMode = 'missing';
|
||||
await page.goto(origin);
|
||||
await page.waitForFunction(() => window.createUploadThumbnailGenerator);
|
||||
const generate = () => page.evaluate(async pdf =>
|
||||
window.createUploadThumbnailGenerator()(new File([pdf], 'document.pdf')), createPdf());
|
||||
expect(await generate()).toBeUndefined();
|
||||
workerMode = undefined;
|
||||
expect(await generate()).toMatch(/^data:image\/png;base64,/);
|
||||
} finally {
|
||||
workerMode = undefined;
|
||||
await page.close();
|
||||
}
|
||||
});
|
||||
|
||||
it.each(['timeout', 'cancel'])('terminates CPU-bound rendering on %s while the page stays responsive', async (mode) => {
|
||||
const page = await browser.newPage();
|
||||
try {
|
||||
workerMode = 'busy';
|
||||
await page.goto(origin);
|
||||
await page.waitForFunction(() => window.createUploadThumbnailGenerator);
|
||||
const result = await page.evaluate(async ({ pdf, mode }) => {
|
||||
const controller = new AbortController();
|
||||
let ticks = 0;
|
||||
const timer = setInterval(() => ticks++, 10);
|
||||
if ( mode === 'cancel' ) setTimeout(() => controller.abort(), 100);
|
||||
const thumbnail = await window.createUploadThumbnailGenerator()(new File([pdf], 'document.pdf'), { signal: controller.signal });
|
||||
clearInterval(timer);
|
||||
return { thumbnail, ticks };
|
||||
}, { pdf: createPdf(), mode });
|
||||
expect(result.thumbnail).toBeUndefined();
|
||||
expect(result.ticks).toBeGreaterThan(2);
|
||||
await expect.poll(() => page.workers().length, { timeout: 5000 }).toBe(0);
|
||||
} finally {
|
||||
workerMode = undefined;
|
||||
await page.close();
|
||||
}
|
||||
}, 10000);
|
||||
|
||||
});
|
||||
@@ -0,0 +1,8 @@
|
||||
export const PDFJS_VERSION = '6.3.289';
|
||||
export const PDF_THUMBNAIL_ASSET_PATH = `/dist/pdf-thumbnails/${PDFJS_VERSION}/`;
|
||||
export const PDF_THUMBNAIL_MAX_FILE_BYTES = 20 * 1024 * 1024;
|
||||
export const PDF_THUMBNAIL_BATCH_TIMEOUT_MS = 5000;
|
||||
export const PDF_THUMBNAIL_JOB_TIMEOUT_MS = 4000;
|
||||
export const PDF_THUMBNAIL_MAX_PIXELS = 4 * 1024 * 1024;
|
||||
export const PDF_THUMBNAIL_DIMENSION = 128;
|
||||
export const PDF_THUMBNAIL_MAX_BYTES = 2 * 1024 * 1024;
|
||||
@@ -0,0 +1,106 @@
|
||||
import { defaultThumbnailGenerator } from '../../../../puter-js/src/modules/FileSystem/operations/upload/thumbnails.js';
|
||||
import {
|
||||
PDF_THUMBNAIL_ASSET_PATH,
|
||||
PDF_THUMBNAIL_MAX_FILE_BYTES,
|
||||
PDF_THUMBNAIL_BATCH_TIMEOUT_MS,
|
||||
PDF_THUMBNAIL_JOB_TIMEOUT_MS,
|
||||
PDF_THUMBNAIL_MAX_BYTES,
|
||||
} from './config.js';
|
||||
|
||||
const pendingJobs = new Set();
|
||||
let activeJob;
|
||||
let startingJobs = false;
|
||||
|
||||
const startNextJob = () => {
|
||||
if ( startingJobs ) return;
|
||||
startingJobs = true;
|
||||
try {
|
||||
while ( !activeJob && pendingJobs.size ) {
|
||||
const job = pendingJobs.values().next().value;
|
||||
pendingJobs.delete(job);
|
||||
activeJob = job;
|
||||
job.start();
|
||||
}
|
||||
} finally {
|
||||
startingJobs = false;
|
||||
}
|
||||
};
|
||||
|
||||
const generatePdfThumbnail = (file, deadline, signal) => new Promise(resolve => {
|
||||
let worker;
|
||||
let settled = false;
|
||||
let jobTimer;
|
||||
let deadlineTimer;
|
||||
const finish = (thumbnail) => {
|
||||
if ( settled ) return;
|
||||
settled = true;
|
||||
clearTimeout(jobTimer);
|
||||
clearTimeout(deadlineTimer);
|
||||
signal?.removeEventListener('abort', onAbort);
|
||||
worker?.terminate();
|
||||
pendingJobs.delete(job);
|
||||
if ( activeJob === job ) activeJob = undefined;
|
||||
resolve(thumbnail);
|
||||
startNextJob();
|
||||
};
|
||||
const onAbort = () => finish();
|
||||
const job = {
|
||||
start: () => {
|
||||
if ( signal?.aborted || Date.now() >= deadline ) {
|
||||
finish();
|
||||
return;
|
||||
}
|
||||
try {
|
||||
// All parsing, file reads and rasterization stay off the desktop thread.
|
||||
worker = new Worker(`${PDF_THUMBNAIL_ASSET_PATH}worker.js`, { type: 'module' });
|
||||
worker.onmessage = ({ data: message }) => {
|
||||
if ( message?.type !== 'thumbnail' ) return;
|
||||
const data = message.thumbnail;
|
||||
const valid = typeof data === 'string' && data.startsWith('data:image/png;base64,') &&
|
||||
data.length <= Math.ceil(PDF_THUMBNAIL_MAX_BYTES / 3) * 4 + 22;
|
||||
finish(valid ? data : undefined);
|
||||
};
|
||||
worker.onerror = (event) => {
|
||||
event.preventDefault();
|
||||
finish();
|
||||
};
|
||||
worker.onmessageerror = () => finish();
|
||||
jobTimer = setTimeout(() => finish(), PDF_THUMBNAIL_JOB_TIMEOUT_MS);
|
||||
worker.postMessage(file);
|
||||
} catch {
|
||||
finish();
|
||||
}
|
||||
},
|
||||
};
|
||||
signal?.addEventListener('abort', onAbort, { once: true });
|
||||
deadlineTimer = setTimeout(() => finish(), Math.max(0, deadline - Date.now()));
|
||||
pendingJobs.add(job);
|
||||
startNextJob();
|
||||
});
|
||||
|
||||
/** Create one callback per upload so queued PDFs share a preparation deadline. */
|
||||
export const createUploadThumbnailGenerator = () => {
|
||||
let deadline;
|
||||
return async (file, context) => {
|
||||
try {
|
||||
if ( context?.signal?.aborted ) return undefined;
|
||||
const isPdf = file.type === 'application/pdf' || /\.pdf$/i.test(file.name || '');
|
||||
if ( ! isPdf ) {
|
||||
// SDKs that predate the callback context pass only the file; images still need thumbnails then.
|
||||
const imageGenerator = typeof context?.defaultGenerator === 'function'
|
||||
? context.defaultGenerator
|
||||
: defaultThumbnailGenerator;
|
||||
return await imageGenerator(file);
|
||||
}
|
||||
if ( ! file.size || file.size > PDF_THUMBNAIL_MAX_FILE_BYTES ||
|
||||
typeof Worker === 'undefined' || typeof OffscreenCanvas === 'undefined' ) {
|
||||
return undefined;
|
||||
}
|
||||
deadline ??= Date.now() + PDF_THUMBNAIL_BATCH_TIMEOUT_MS;
|
||||
if ( Date.now() >= deadline ) return undefined;
|
||||
return await generatePdfThumbnail(file, deadline, context?.signal);
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,172 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import {
|
||||
PDF_THUMBNAIL_BATCH_TIMEOUT_MS,
|
||||
PDF_THUMBNAIL_JOB_TIMEOUT_MS,
|
||||
PDF_THUMBNAIL_MAX_FILE_BYTES,
|
||||
} from './config.js';
|
||||
|
||||
const thumbnail = 'data:image/png;base64,AAAA';
|
||||
const bundledThumbnail = 'data:image/webp;base64,BBBB';
|
||||
const bundledImageGenerator = vi.fn(async () => bundledThumbnail);
|
||||
vi.mock('../../../../puter-js/src/modules/FileSystem/operations/upload/thumbnails.js', () => ({
|
||||
defaultThumbnailGenerator: (...args) => bundledImageGenerator(...args),
|
||||
}));
|
||||
const pdf = () => new File(['%PDF-1.7'], 'document.pdf', { type: 'application/pdf' });
|
||||
let createUploadThumbnailGenerator;
|
||||
let workers;
|
||||
|
||||
beforeEach(async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.resetModules();
|
||||
bundledImageGenerator.mockClear();
|
||||
workers = [];
|
||||
vi.stubGlobal('OffscreenCanvas', class {});
|
||||
vi.stubGlobal('Worker', class {
|
||||
terminate = vi.fn();
|
||||
postMessage = vi.fn();
|
||||
constructor () { workers.push(this); }
|
||||
complete (value = thumbnail) { this.onmessage({ data: { type: 'thumbnail', thumbnail: value } }); }
|
||||
});
|
||||
({ createUploadThumbnailGenerator } = await import('./index.js'));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
await vi.runAllTimersAsync();
|
||||
vi.useRealTimers();
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe('GUI upload thumbnail scheduling', () => {
|
||||
it('delegates images without loading PDF assets', async () => {
|
||||
const file = new File(['image'], 'image.png');
|
||||
const defaultGenerator = vi.fn(async () => thumbnail);
|
||||
expect(await createUploadThumbnailGenerator()(file, { defaultGenerator })).toBe(thumbnail);
|
||||
expect(defaultGenerator).toHaveBeenCalledWith(file);
|
||||
expect(bundledImageGenerator).not.toHaveBeenCalled();
|
||||
expect(workers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each([undefined, {}, { defaultGenerator: 'not a function' }])(
|
||||
'falls back to the bundled image generator when the SDK passes no usable context: %j', async (context) => {
|
||||
const file = new File(['image'], 'image.png');
|
||||
expect(await createUploadThumbnailGenerator()(file, context)).toBe(bundledThumbnail);
|
||||
expect(bundledImageGenerator).toHaveBeenCalledWith(file);
|
||||
expect(workers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('preserves graceful failure of the image generator', async () => {
|
||||
expect(await createUploadThumbnailGenerator()(new File(['x'], 'x.png'), {
|
||||
defaultGenerator: async () => { throw new Error('decode failed'); },
|
||||
})).toBeUndefined();
|
||||
expect(workers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
new File([], 'empty.pdf'),
|
||||
{ name: 'large.pdf', size: PDF_THUMBNAIL_MAX_FILE_BYTES + 1 },
|
||||
])('skips empty and oversized PDFs without reading them', async (file) => {
|
||||
expect(await createUploadThumbnailGenerator()(file)).toBeUndefined();
|
||||
expect(workers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each(['Worker', 'OffscreenCanvas'])('skips PDFs when %s is unavailable', async (feature) => {
|
||||
vi.stubGlobal(feature, undefined);
|
||||
expect(await createUploadThumbnailGenerator()(pdf())).toBeUndefined();
|
||||
expect(workers).toHaveLength(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
new File(['pdf'], 'document.PDF'),
|
||||
new File(['pdf'], 'document', { type: 'application/pdf' }),
|
||||
])('recognizes PDF extensions and MIME types', async (file) => {
|
||||
const result = createUploadThumbnailGenerator()(file);
|
||||
expect(workers[0].postMessage).toHaveBeenCalledWith(file);
|
||||
workers[0].complete();
|
||||
expect(await result).toBe(thumbnail);
|
||||
expect(workers[0].terminate).toHaveBeenCalledOnce();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('ignores PDF.js readiness messages until the thumbnail is ready', async () => {
|
||||
const result = createUploadThumbnailGenerator()(pdf());
|
||||
workers[0].onmessage({ data: { action: 'ready', targetName: 'main' } });
|
||||
expect(workers[0].terminate).not.toHaveBeenCalled();
|
||||
workers[0].complete();
|
||||
expect(await result).toBe(thumbnail);
|
||||
});
|
||||
|
||||
it('serializes PDFs across simultaneous uploads', async () => {
|
||||
const first = createUploadThumbnailGenerator()(pdf());
|
||||
const second = createUploadThumbnailGenerator()(pdf());
|
||||
expect(workers).toHaveLength(1);
|
||||
workers[0].complete();
|
||||
expect(workers).toHaveLength(2);
|
||||
workers[1].complete();
|
||||
expect(await Promise.all([first, second])).toEqual([thumbnail, thumbnail]);
|
||||
});
|
||||
|
||||
it('terminates a stuck worker and advances the queue', async () => {
|
||||
const generate = createUploadThumbnailGenerator();
|
||||
const first = generate(pdf());
|
||||
const second = generate(pdf());
|
||||
await vi.advanceTimersByTimeAsync(PDF_THUMBNAIL_JOB_TIMEOUT_MS);
|
||||
expect(await first).toBeUndefined();
|
||||
expect(workers[0].terminate).toHaveBeenCalledOnce();
|
||||
expect(workers).toHaveLength(2);
|
||||
workers[1].complete();
|
||||
expect(await second).toBe(thumbnail);
|
||||
});
|
||||
|
||||
it('bounds a whole batch including queued files and rejects late results', async () => {
|
||||
const generate = createUploadThumbnailGenerator();
|
||||
const results = Array.from({ length: 20 }, () => generate(pdf()));
|
||||
await vi.advanceTimersByTimeAsync(PDF_THUMBNAIL_BATCH_TIMEOUT_MS);
|
||||
expect(await Promise.all(results)).toEqual(Array(20).fill(undefined));
|
||||
expect(workers).toHaveLength(Math.ceil(PDF_THUMBNAIL_BATCH_TIMEOUT_MS / PDF_THUMBNAIL_JOB_TIMEOUT_MS));
|
||||
expect(workers.every(worker => worker.terminate.mock.calls.length === 1)).toBe(true);
|
||||
workers[0].complete();
|
||||
expect(await generate(pdf())).toBeUndefined();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
|
||||
it('cancels both running and queued jobs and releases abort listeners', async () => {
|
||||
const controller = new AbortController();
|
||||
const remove = vi.spyOn(controller.signal, 'removeEventListener');
|
||||
const generate = createUploadThumbnailGenerator();
|
||||
const first = generate(pdf(), { signal: controller.signal });
|
||||
const second = generate(pdf(), { signal: controller.signal });
|
||||
controller.abort();
|
||||
expect(await Promise.all([first, second])).toEqual([undefined, undefined]);
|
||||
expect(workers).toHaveLength(1);
|
||||
expect(workers[0].terminate).toHaveBeenCalledOnce();
|
||||
expect(remove).toHaveBeenCalledTimes(2);
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
expect(await generate(pdf(), { signal: controller.signal })).toBeUndefined();
|
||||
});
|
||||
|
||||
it.each(['error', 'messageerror', 'invalid', 'empty'])('recovers from a worker %s', async (failure) => {
|
||||
const result = createUploadThumbnailGenerator()(pdf());
|
||||
if ( failure === 'error' ) workers[0].onerror({ preventDefault: vi.fn() });
|
||||
if ( failure === 'messageerror' ) workers[0].onmessageerror();
|
||||
if ( failure === 'invalid' ) workers[0].complete('not an image');
|
||||
if ( failure === 'empty' ) workers[0].onmessage({ data: { type: 'thumbnail' } });
|
||||
expect(await result).toBeUndefined();
|
||||
expect(workers[0].terminate).toHaveBeenCalledOnce();
|
||||
const next = createUploadThumbnailGenerator()(pdf());
|
||||
workers[1].complete();
|
||||
expect(await next).toBe(thumbnail);
|
||||
});
|
||||
|
||||
it('survives worker construction and structured-clone errors', async () => {
|
||||
vi.stubGlobal('Worker', class { constructor () { throw new Error('blocked'); } });
|
||||
expect(await createUploadThumbnailGenerator()(pdf())).toBeUndefined();
|
||||
vi.stubGlobal('Worker', class {
|
||||
terminate = vi.fn();
|
||||
postMessage () { throw new Error('clone failed'); }
|
||||
constructor () { workers.push(this); }
|
||||
});
|
||||
expect(await createUploadThumbnailGenerator()(pdf())).toBeUndefined();
|
||||
expect(workers[0].terminate).toHaveBeenCalledOnce();
|
||||
expect(vi.getTimerCount()).toBe(0);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,88 @@
|
||||
// These imports are served beside this worker by the GUI build.
|
||||
import { getDocument } from './pdf.mjs';
|
||||
import { WorkerMessageHandler } from './pdf.worker.mjs';
|
||||
import {
|
||||
PDF_THUMBNAIL_MAX_PIXELS,
|
||||
PDF_THUMBNAIL_DIMENSION,
|
||||
PDF_THUMBNAIL_MAX_BYTES,
|
||||
} from './config.js';
|
||||
|
||||
// PDF.js's loopback transport keeps its parser in this disposable worker too.
|
||||
globalThis.pdfjsWorker = { WorkerMessageHandler };
|
||||
|
||||
class ThumbnailCanvasFactory {
|
||||
create (width, height) {
|
||||
const canvas = new OffscreenCanvas(1, 1);
|
||||
this.reset({ canvas }, width, height);
|
||||
return { canvas, context: canvas.getContext('2d') };
|
||||
}
|
||||
|
||||
reset ({ canvas }, width, height) {
|
||||
if ( !Number.isFinite(width) || !Number.isFinite(height) || width <= 0 || height <= 0 ||
|
||||
Math.ceil(width) * Math.ceil(height) > PDF_THUMBNAIL_MAX_PIXELS ) {
|
||||
throw new Error('Thumbnail canvas exceeds budget');
|
||||
}
|
||||
canvas.width = Math.ceil(width);
|
||||
canvas.height = Math.ceil(height);
|
||||
}
|
||||
|
||||
destroy (entry) {
|
||||
entry.canvas.width = entry.canvas.height = 0;
|
||||
entry.canvas = entry.context = null;
|
||||
}
|
||||
}
|
||||
|
||||
self.onmessage = async ({ data: file }) => {
|
||||
let loadingTask;
|
||||
let canvas;
|
||||
let thumbnail;
|
||||
try {
|
||||
const assetUrl = directory => new URL(`${directory}/`, import.meta.url).href;
|
||||
loadingTask = getDocument({
|
||||
data: new Uint8Array(await file.arrayBuffer()),
|
||||
cMapUrl: assetUrl('cmaps'),
|
||||
standardFontDataUrl: assetUrl('standard_fonts'),
|
||||
wasmUrl: assetUrl('wasm'),
|
||||
iccUrl: assetUrl('iccs'),
|
||||
useWorkerFetch: true,
|
||||
disableFontFace: true,
|
||||
useSystemFonts: false,
|
||||
enableXfa: false,
|
||||
stopAtErrors: true,
|
||||
verbosity: 0,
|
||||
maxImageSize: PDF_THUMBNAIL_MAX_PIXELS,
|
||||
canvasMaxAreaInBytes: PDF_THUMBNAIL_MAX_PIXELS * 4,
|
||||
CanvasFactory: ThumbnailCanvasFactory,
|
||||
});
|
||||
const pdf = await loadingTask.promise;
|
||||
const page = await pdf.getPage(1);
|
||||
// Resource-limit failures can resolve to an empty operator list.
|
||||
const operators = await page.getOperatorList();
|
||||
if ( ! operators.fnArray.length ) return;
|
||||
const baseViewport = page.getViewport({ scale: 1 });
|
||||
const viewport = page.getViewport({
|
||||
scale: PDF_THUMBNAIL_DIMENSION / Math.max(baseViewport.width, baseViewport.height),
|
||||
});
|
||||
const target = new ThumbnailCanvasFactory().create(viewport.width, viewport.height);
|
||||
canvas = target.canvas;
|
||||
await page.render({
|
||||
canvasContext: target.context,
|
||||
viewport,
|
||||
background: 'rgb(255, 255, 255)',
|
||||
}).promise;
|
||||
const blob = await canvas.convertToBlob({ type: 'image/png' });
|
||||
if ( blob.size <= PDF_THUMBNAIL_MAX_BYTES ) {
|
||||
thumbnail = new FileReaderSync().readAsDataURL(blob);
|
||||
}
|
||||
} catch {
|
||||
// Unsupported, encrypted and malformed PDFs retain their ordinary file icon.
|
||||
} finally {
|
||||
try {
|
||||
await loadingTask?.destroy();
|
||||
} catch {
|
||||
// The parent also terminates this worker, including on cleanup timeout.
|
||||
}
|
||||
if ( canvas ) canvas.width = canvas.height = 0;
|
||||
self.postMessage({ type: 'thumbnail', thumbnail });
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1 @@
|
||||
*.pdf binary
|
||||
Vendored
+18
@@ -0,0 +1,18 @@
|
||||
# PDF thumbnail fixtures
|
||||
|
||||
`pdf.js` builds small test PDFs with correct cross-reference offsets: a red first
|
||||
page with text or an image and a blue second page. It supports page rotation and
|
||||
oversized image declarations without requiring a PDF generation dependency.
|
||||
|
||||
`encrypted.pdf` is a blank 300 × 600 point page generated with pypdf solely for
|
||||
the password-protection failure test:
|
||||
|
||||
```python
|
||||
from pypdf import PdfWriter
|
||||
writer = PdfWriter()
|
||||
writer.add_blank_page(width=300, height=600)
|
||||
writer.encrypt('test-only-password')
|
||||
writer.write('encrypted.pdf')
|
||||
```
|
||||
|
||||
These fixtures contain no user documents or production credentials.
|
||||
Vendored
+59
@@ -0,0 +1,59 @@
|
||||
%PDF-1.3
|
||||
%âãÏÓ
|
||||
1 0 obj
|
||||
<<
|
||||
/Producer <e601e0d3ad>
|
||||
>>
|
||||
endobj
|
||||
2 0 obj
|
||||
<<
|
||||
/Type /Pages
|
||||
/Count 1
|
||||
/Kids [ 4 0 R ]
|
||||
>>
|
||||
endobj
|
||||
3 0 obj
|
||||
<<
|
||||
/Type /Catalog
|
||||
/Pages 2 0 R
|
||||
>>
|
||||
endobj
|
||||
4 0 obj
|
||||
<<
|
||||
/Type /Page
|
||||
/Resources <<
|
||||
>>
|
||||
/MediaBox [ 0.0 0.0 300 600 ]
|
||||
/Parent 2 0 R
|
||||
>>
|
||||
endobj
|
||||
5 0 obj
|
||||
<<
|
||||
/V 2
|
||||
/R 3
|
||||
/Length 128
|
||||
/P 4294967292
|
||||
/Filter /Standard
|
||||
/O <3abc5f38171dfcfb14eed227dc227036ad2a1fd6252e9edd3ab60066f1f3f4ee>
|
||||
/U <0ce16c146f669e4111f4a871423ebba628bf4e5e4e758a4164004e56fffa0108>
|
||||
>>
|
||||
endobj
|
||||
xref
|
||||
0 6
|
||||
0000000000 65535 f
|
||||
0000000015 00000 n
|
||||
0000000059 00000 n
|
||||
0000000118 00000 n
|
||||
0000000167 00000 n
|
||||
0000000261 00000 n
|
||||
trailer
|
||||
<<
|
||||
/Size 6
|
||||
/Root 3 0 R
|
||||
/Info 1 0 R
|
||||
/ID [ <6130323032303032656234643661656263346631626236336234303364663531> <6130323032303032656234643661656263346631626236336234303364663531> ]
|
||||
/Encrypt 5 0 R
|
||||
>>
|
||||
startxref
|
||||
476
|
||||
%%EOF
|
||||
Vendored
+28
@@ -0,0 +1,28 @@
|
||||
// Small, deterministic PDFs with real cross-reference tables, generated without a PDF library.
|
||||
export const createPdf = ({ rotation = 0, scanned = false, imageSize = 1 } = {}) => {
|
||||
const content = scanned
|
||||
? 'q 300 0 0 600 0 0 cm /Im1 Do Q'
|
||||
: '1 0 0 rg 0 0 300 600 re f BT /F1 36 Tf 1 1 1 rg 20 300 Td (Thumbnail) Tj ET';
|
||||
const stream = data => `<< /Length ${data.length} >>\nstream\n${data}\nendstream`;
|
||||
const objects = [
|
||||
'<< /Type /Catalog /Pages 2 0 R >>',
|
||||
'<< /Type /Pages /Kids [3 0 R 7 0 R] /Count 2 >>',
|
||||
`<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 600] /Rotate ${rotation} /Resources << /Font << /F1 4 0 R >> /XObject << /Im1 6 0 R >> >> /Contents 5 0 R >>`,
|
||||
'<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>',
|
||||
stream(content),
|
||||
`<< /Type /XObject /Subtype /Image /Width ${imageSize} /Height ${imageSize} /ColorSpace /DeviceRGB /BitsPerComponent 8 /Filter /ASCIIHexDecode /Length 7 >>\nstream\nff0000>\nendstream`,
|
||||
'<< /Type /Page /Parent 2 0 R /MediaBox [0 0 300 600] /Contents 8 0 R >>',
|
||||
stream('0 0 1 rg 0 0 300 600 re f'),
|
||||
];
|
||||
let pdf = '%PDF-1.7\n';
|
||||
const offsets = [0];
|
||||
for ( const [index, object] of objects.entries() ) {
|
||||
offsets.push(pdf.length);
|
||||
pdf += `${index + 1} 0 obj\n${object}\nendobj\n`;
|
||||
}
|
||||
const xref = pdf.length;
|
||||
pdf += `xref\n0 ${objects.length + 1}\n0000000000 65535 f \n`;
|
||||
for ( const offset of offsets.slice(1) ) pdf += `${String(offset).padStart(10, '0')} 00000 n \n`;
|
||||
pdf += `trailer\n<< /Size ${objects.length + 1} /Root 1 0 R >>\nstartxref\n${xref}\n%%EOF\n`;
|
||||
return pdf;
|
||||
};
|
||||
@@ -0,0 +1,23 @@
|
||||
import { cp, mkdir, readFile } from 'node:fs/promises';
|
||||
import { createRequire } from 'node:module';
|
||||
import path from 'node:path';
|
||||
import { PDFJS_VERSION } from '../src/services/pdfThumbnails/config.js';
|
||||
|
||||
export const copyPdfThumbnailAssets = async (distPath) => {
|
||||
const require = createRequire(import.meta.url);
|
||||
const packagePath = require.resolve('pdfjs-dist/package.json');
|
||||
const metadata = JSON.parse(await readFile(packagePath, 'utf8'));
|
||||
if ( metadata.version !== PDFJS_VERSION ) throw new Error('PDF thumbnail assets must match the pinned PDF.js version');
|
||||
const source = path.dirname(packagePath);
|
||||
const destination = path.join(distPath, 'pdf-thumbnails', PDFJS_VERSION);
|
||||
await mkdir(destination, { recursive: true });
|
||||
for ( const name of ['cmaps', 'standard_fonts', 'wasm', 'iccs', 'LICENSE'] ) {
|
||||
await cp(path.join(source, name), path.join(destination, name), { recursive: true });
|
||||
}
|
||||
for ( const name of ['pdf', 'pdf.worker'] ) {
|
||||
await cp(path.join(source, 'build', `${name}.min.mjs`), path.join(destination, `${name}.mjs`));
|
||||
}
|
||||
for ( const name of ['worker.js', 'config.js'] ) {
|
||||
await cp(new URL(`../src/services/pdfThumbnails/${name}`, import.meta.url), path.join(destination, name));
|
||||
}
|
||||
};
|
||||
@@ -25,6 +25,7 @@ import uglifyjs from 'uglify-js';
|
||||
import { lib_paths, css_paths, js_paths } from './src/static-assets.js';
|
||||
import { fileURLToPath } from 'url';
|
||||
import BaseConfig from './webpack/BaseConfig.cjs';
|
||||
import { copyPdfThumbnailAssets } from './tools/copyPdfThumbnailAssets.js';
|
||||
|
||||
// Polyfill __dirname, which doesn't exist in modules mode
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
@@ -59,6 +60,7 @@ async function build (options) {
|
||||
fs.rmSync(path.join(__dirname, 'dist'), { recursive: true });
|
||||
}
|
||||
fs.mkdirSync(path.join(__dirname, 'dist'));
|
||||
await copyPdfThumbnailAssets(path.join(__dirname, 'dist'));
|
||||
|
||||
// -----------------------------------------------
|
||||
// Concat/merge the JS libraries and save them to ./dist/libs.js
|
||||
|
||||
Vendored
+2
@@ -146,6 +146,8 @@ export type {
|
||||
UploadItems,
|
||||
UploadOperationResult,
|
||||
UploadOptions,
|
||||
ThumbnailGenerator,
|
||||
ThumbnailGeneratorContext,
|
||||
WriteOptions,
|
||||
} from './types/modules/FileSystem/types.js';
|
||||
export type {
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
export const MAX_THUMBNAIL_BYTES = 2 * 1024 * 1024;
|
||||
export const DEFAULT_THUMBNAIL_DIMENSION = 128;
|
||||
export const MIN_THUMBNAIL_DIMENSION = 32;
|
||||
export const THUMBNAIL_UPLOAD_TIMEOUT_MS = 5000;
|
||||
|
||||
// Marks whether the signed batch-write endpoints are known to be available on
|
||||
// the current backend. Cached on the FileSystem module instance so a single
|
||||
|
||||
@@ -51,7 +51,13 @@ const uploadImpl = async function (items, dirPath, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
// Native XHR.abort() does nothing before send(); preparation still needs cancellation.
|
||||
const preparationController = new AbortController();
|
||||
|
||||
const error = (e) => {
|
||||
// Cancelling already settled the upload; a preparation step failing afterwards is not an error.
|
||||
if ( preparationController.signal.aborted ) return;
|
||||
|
||||
// Out of storage: prompt the user to upgrade, then reject as usual.
|
||||
promptIfStorageLimitError(e);
|
||||
|
||||
@@ -84,11 +90,22 @@ const uploadImpl = async function (items, dirPath, options = {}) {
|
||||
// fires at most once even when the signed path falls back to legacy.
|
||||
const flags = { startCallbackFired: false };
|
||||
|
||||
xhr.abort = () => {
|
||||
if ( preparationController.signal.aborted ) return;
|
||||
preparationController.abort();
|
||||
try {
|
||||
options.abort?.(operationId);
|
||||
} finally {
|
||||
reject({ code: 'upload_aborted', message: 'Upload aborted.' });
|
||||
}
|
||||
};
|
||||
|
||||
// Call 'init' callback if provided
|
||||
// init is basically a hook that allows the user to get the operation ID and the XMLHttpRequest object
|
||||
if ( options.init && typeof options.init === 'function' ) {
|
||||
options.init(operationId, xhr);
|
||||
}
|
||||
if ( preparationController.signal.aborted ) return;
|
||||
|
||||
// Normalize the accepted input shapes (DataTransferItemList, FileList,
|
||||
// File, Blob, string, or arrays of these) into a flat list of entries.
|
||||
@@ -98,6 +115,7 @@ const uploadImpl = async function (items, dirPath, options = {}) {
|
||||
} catch (e) {
|
||||
return error(e);
|
||||
}
|
||||
if ( preparationController.signal.aborted ) return;
|
||||
|
||||
// Separate files from directories and tally the upload size.
|
||||
// This executor is async, so anything that throws here would settle
|
||||
@@ -112,10 +130,11 @@ const uploadImpl = async function (items, dirPath, options = {}) {
|
||||
return error({ code: 'EMPTY_UPLOAD', message: 'No files or directories to upload.' });
|
||||
}
|
||||
|
||||
thumbnails = await generateThumbnails(files, options);
|
||||
thumbnails = await generateThumbnails(files, options, preparationController.signal);
|
||||
} catch (e) {
|
||||
return error(e);
|
||||
}
|
||||
if ( preparationController.signal.aborted ) return;
|
||||
|
||||
// Check storage capacity.
|
||||
// We need to check the storage capacity before the upload starts because
|
||||
@@ -141,6 +160,9 @@ const uploadImpl = async function (items, dirPath, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
if ( preparationController.signal.aborted ) return;
|
||||
delete xhr.abort;
|
||||
|
||||
const signedDirectories = dirs.map((dir) => dir.path);
|
||||
|
||||
const signedBatchWriteCapability = this[SIGNED_BATCH_WRITE_CAPABILITY_KEY];
|
||||
|
||||
@@ -50,6 +50,33 @@ afterEach(() => {
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('cancellation during preparation', () => {
|
||||
afterEach(() => {
|
||||
delete globalThis.DataTransferItemList;
|
||||
});
|
||||
|
||||
it('does not report a preparation failure that lands after cancelling', async () => {
|
||||
globalThis.DataTransferItemList = class {};
|
||||
let handle;
|
||||
const error = vi.fn();
|
||||
const abort = vi.fn();
|
||||
globalThis.puter.ui = {
|
||||
getEntriesFromDataTransferItems: async () => {
|
||||
handle.abort();
|
||||
throw new Error('directory read failed');
|
||||
},
|
||||
};
|
||||
await expect(upload.call(fs, new globalThis.DataTransferItemList(), '/user/dir', {
|
||||
init: (_operationId, xhr) => { handle = xhr; },
|
||||
error,
|
||||
abort,
|
||||
})).rejects.toMatchObject({ code: 'upload_aborted' });
|
||||
await new Promise(resolve => setTimeout(resolve, 0));
|
||||
expect(error).not.toHaveBeenCalled();
|
||||
expect(abort).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('storage capacity pre-flight', () => {
|
||||
it('is skipped for an upload below the threshold', async () => {
|
||||
await expect(uploadOf(64)).resolves.toEqual({ uid: 'uploaded' });
|
||||
|
||||
@@ -106,6 +106,7 @@ export const toErrorMessage = (error) => {
|
||||
* url: string,
|
||||
* blob: Blob,
|
||||
* contentType?: string,
|
||||
* timeoutMs?: number,
|
||||
* onProgress?: (deltaBytes: number) => void,
|
||||
* onRequestCreated?: (request: XMLHttpRequest) => void,
|
||||
* onRequestCompleted?: (request: XMLHttpRequest) => void,
|
||||
@@ -116,6 +117,7 @@ export const uploadBlobToSignedUrl = async ({
|
||||
url,
|
||||
blob,
|
||||
contentType,
|
||||
timeoutMs = 0,
|
||||
onProgress,
|
||||
onRequestCreated,
|
||||
onRequestCompleted,
|
||||
@@ -124,6 +126,7 @@ export const uploadBlobToSignedUrl = async ({
|
||||
const request = new XMLHttpRequest();
|
||||
request.open('PUT', url, true);
|
||||
request.withCredentials = false;
|
||||
request.timeout = timeoutMs;
|
||||
|
||||
if ( contentType ) {
|
||||
request.setRequestHeader('Content-Type', contentType);
|
||||
@@ -183,6 +186,11 @@ export const uploadBlobToSignedUrl = async ({
|
||||
reject(error);
|
||||
};
|
||||
|
||||
request.ontimeout = () => {
|
||||
onRequestCompleted?.(request);
|
||||
reject(new Error('Signed upload timed out'));
|
||||
};
|
||||
|
||||
request.send(blob);
|
||||
});
|
||||
};
|
||||
|
||||
@@ -6,6 +6,7 @@
|
||||
import path from 'path-browserify';
|
||||
import {
|
||||
MAX_THUMBNAIL_BYTES,
|
||||
THUMBNAIL_UPLOAD_TIMEOUT_MS,
|
||||
SIGNED_BATCH_WRITE_CAPABILITY_KEY,
|
||||
SIGNED_BATCH_REQUEST_CHUNK_SIZE,
|
||||
SIGNED_BATCH_CHUNK_PIPELINE_CONCURRENCY,
|
||||
@@ -302,21 +303,27 @@ export async function performSignedBatchUpload (ctx) {
|
||||
const thumbnailUploadUrl = startResponse.thumbnailUploadUrl;
|
||||
const thumbnailUrl = startResponse.thumbnailUrl;
|
||||
if ( thumbnailUploadUrl && thumbnailUrl ) {
|
||||
const thumbnailBlob = await dataUrlToBlob(thumbnailData);
|
||||
if ( thumbnailBlob.size <= MAX_THUMBNAIL_BYTES ) {
|
||||
await uploadBlobToSignedUrl({
|
||||
url: thumbnailUploadUrl,
|
||||
blob: thumbnailBlob,
|
||||
contentType: thumbnailBlob.type || parseDataUrlContentType(thumbnailData),
|
||||
onProgress: addSignedProgress,
|
||||
onRequestCreated: (request) => {
|
||||
activeSignedRequests.add(request);
|
||||
},
|
||||
onRequestCompleted: (request) => {
|
||||
activeSignedRequests.delete(request);
|
||||
},
|
||||
});
|
||||
completionThumbnailData = thumbnailUrl;
|
||||
try {
|
||||
const thumbnailBlob = await dataUrlToBlob(thumbnailData);
|
||||
if ( thumbnailBlob.size <= MAX_THUMBNAIL_BYTES ) {
|
||||
await uploadBlobToSignedUrl({
|
||||
url: thumbnailUploadUrl,
|
||||
blob: thumbnailBlob,
|
||||
contentType: thumbnailBlob.type || parseDataUrlContentType(thumbnailData),
|
||||
timeoutMs: THUMBNAIL_UPLOAD_TIMEOUT_MS,
|
||||
onProgress: addSignedProgress,
|
||||
onRequestCreated: (request) => {
|
||||
activeSignedRequests.add(request);
|
||||
},
|
||||
onRequestCompleted: (request) => {
|
||||
activeSignedRequests.delete(request);
|
||||
},
|
||||
});
|
||||
completionThumbnailData = thumbnailUrl;
|
||||
}
|
||||
} catch (error) {
|
||||
if ( signedUploadAborted || error?.aborted ) throw error;
|
||||
// A missing preview must not prevent the original file from uploading.
|
||||
}
|
||||
}
|
||||
} else if ( typeof thumbnailData === 'string' && thumbnailData.length > 0 ) {
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { createServer } from 'node:http';
|
||||
import { createRequire } from 'node:module';
|
||||
|
||||
// Load the dual-export XHR adapter natively; Vite treats its CommonJS branch as a module mutation.
|
||||
const { default: XMLHttpRequestShim } = createRequire(import.meta.url)('../../../../lib/polyfills/xhrshim.js');
|
||||
import upload from './index.js';
|
||||
import { THUMBNAIL_UPLOAD_TIMEOUT_MS } from './constants.js';
|
||||
|
||||
let server;
|
||||
let fs;
|
||||
let thumbnailResponse;
|
||||
let fileResponse;
|
||||
let fileBytes;
|
||||
let completion;
|
||||
let requestHandle;
|
||||
let thumbnailStarted;
|
||||
let onThumbnailStarted;
|
||||
|
||||
beforeEach(async () => {
|
||||
thumbnailResponse = 200;
|
||||
fileResponse = 200;
|
||||
fileBytes = undefined;
|
||||
completion = undefined;
|
||||
thumbnailStarted = new Promise(resolve => { onThumbnailStarted = resolve; });
|
||||
server = createServer(async (req, res) => {
|
||||
const chunks = [];
|
||||
for await ( const chunk of req ) chunks.push(chunk);
|
||||
const body = Buffer.concat(chunks);
|
||||
const origin = fs.APIOrigin;
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
if ( req.url === '/fs/startBatchWrite' ) {
|
||||
res.end(JSON.stringify([{
|
||||
sessionId: 'test-upload', uploadMode: 'single', url: `${origin}/file`,
|
||||
thumbnailUploadUrl: `${origin}/thumbnail`, thumbnailUrl: `${origin}/preview.png`,
|
||||
}]));
|
||||
} else if ( req.url === '/thumbnail' ) {
|
||||
onThumbnailStarted();
|
||||
if ( thumbnailResponse === 'hang' ) return;
|
||||
if ( thumbnailResponse === 'disconnect' ) { req.socket.destroy(); return; }
|
||||
res.statusCode = thumbnailResponse;
|
||||
res.end('{}');
|
||||
} else if ( req.url === '/file' ) {
|
||||
fileBytes = body.toString();
|
||||
res.statusCode = fileResponse;
|
||||
res.end('{}');
|
||||
} else if ( req.url === '/fs/completeBatchWrite' ) {
|
||||
completion = JSON.parse(body)[0];
|
||||
res.end(JSON.stringify([{ uid: 'uploaded-file' }]));
|
||||
} else if ( req.url === '/fs/abortWrite' ) {
|
||||
res.end('{}');
|
||||
} else {
|
||||
res.statusCode = 404;
|
||||
res.end('{}');
|
||||
}
|
||||
});
|
||||
await new Promise(resolve => server.listen(0, '127.0.0.1', resolve));
|
||||
fs = { APIOrigin: `http://127.0.0.1:${server.address().port}`, authToken: 'test-token' };
|
||||
vi.stubGlobal('XMLHttpRequest', XMLHttpRequestShim);
|
||||
vi.stubGlobal('puter', { env: 'gui', authToken: 'test-token', fs });
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
server.closeAllConnections();
|
||||
await new Promise(resolve => server.close(resolve));
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
const uploadFile = () => upload.call(fs, new File(['original PDF bytes'], 'document.pdf'), '/testuser', {
|
||||
thumbnailGenerator: async () => 'data:image/png;base64,AAAA',
|
||||
init: (_id, xhr) => { requestHandle = xhr; },
|
||||
});
|
||||
|
||||
describe('optional thumbnail transfer over HTTP', () => {
|
||||
it('uploads the thumbnail and the original file', async () => {
|
||||
expect(await uploadFile()).toEqual({ uid: 'uploaded-file' });
|
||||
expect(fileBytes).toBe('original PDF bytes');
|
||||
expect(completion.thumbnailData).toBe(`${fs.APIOrigin}/preview.png`);
|
||||
});
|
||||
|
||||
it.each([403, 500, 'disconnect'])('preserves the original when thumbnail transfer fails: %s', async (failure) => {
|
||||
thumbnailResponse = failure;
|
||||
expect(await uploadFile()).toEqual({ uid: 'uploaded-file' });
|
||||
expect(fileBytes).toBe('original PDF bytes');
|
||||
expect(completion).not.toHaveProperty('thumbnailData');
|
||||
});
|
||||
|
||||
it('times out a stalled thumbnail transfer and uploads the file', async () => {
|
||||
thumbnailResponse = 'hang';
|
||||
expect(await uploadFile()).toEqual({ uid: 'uploaded-file' });
|
||||
expect(fileBytes).toBe('original PDF bytes');
|
||||
expect(completion).not.toHaveProperty('thumbnailData');
|
||||
}, THUMBNAIL_UPLOAD_TIMEOUT_MS + 3000);
|
||||
|
||||
it('does not swallow cancellation during thumbnail transfer', async () => {
|
||||
thumbnailResponse = 'hang';
|
||||
const result = uploadFile();
|
||||
const rejection = expect(result).rejects.toMatchObject({ partial: true });
|
||||
await thumbnailStarted;
|
||||
requestHandle.abort();
|
||||
await rejection;
|
||||
expect(fileBytes).toBeUndefined();
|
||||
expect(completion).toBeUndefined();
|
||||
});
|
||||
|
||||
it('still rejects when the original file fails to upload', async () => {
|
||||
fileResponse = 500;
|
||||
await expect(uploadFile()).rejects.toMatchObject({ partial: true });
|
||||
expect(completion).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -118,10 +118,11 @@ export const defaultThumbnailGenerator = async (file) => {
|
||||
* empty array when thumbnail generation was not requested.
|
||||
*
|
||||
* @param {File[]} files
|
||||
* @param {{ generateThumbnails?: boolean, thumbnailGenerator?: (file: File) => Promise<string | undefined> }} options
|
||||
* @param {import('../../types.js').UploadOptions} options
|
||||
* @param {AbortSignal} [signal]
|
||||
* @returns {Promise<Array<string | undefined>>}
|
||||
*/
|
||||
export const generateThumbnails = async (files, options) => {
|
||||
export const generateThumbnails = async (files, options, signal) => {
|
||||
const shouldGenerateThumbnails = options.generateThumbnails || options.thumbnailGenerator;
|
||||
if ( ! files.length || ! shouldGenerateThumbnails ) {
|
||||
return [];
|
||||
@@ -130,7 +131,8 @@ export const generateThumbnails = async (files, options) => {
|
||||
const generator = options.thumbnailGenerator || defaultThumbnailGenerator;
|
||||
return await Promise.all(files.map(async (file) => {
|
||||
try {
|
||||
return await generator(file);
|
||||
if ( signal?.aborted ) return undefined;
|
||||
return await generator(file, { defaultGenerator: defaultThumbnailGenerator, signal });
|
||||
} catch (e) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
@@ -187,8 +187,21 @@
|
||||
* @typedef {FSItemRead & { shares: Share[] }} FSItemWithShares
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} ThumbnailGeneratorContext
|
||||
* @property {(file: File) => Promise<string | undefined>} defaultGenerator Built-in browser image generator.
|
||||
* @property {AbortSignal} [signal] Aborted when upload preparation is cancelled.
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {(file: File, context: ThumbnailGeneratorContext) => string | undefined | Promise<string | undefined>} ThumbnailGenerator
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} UploadOptionsOwn
|
||||
* @property {boolean} [generateThumbnails] Generate browser image thumbnails before uploading. Defaults to `false`.
|
||||
* @property {ThumbnailGenerator} [thumbnailGenerator] Overrides image generation; return `undefined` to skip.
|
||||
* @property {string} [thumbnail] Thumbnail data URL or URL, used when no generated thumbnail is returned.
|
||||
* @property {boolean} [overwrite] Whether to overwrite the destination file if it already exists.
|
||||
* Defaults to `false`.
|
||||
* @property {boolean} [dedupeName] Whether to deduplicate the file name if it already exists. Defaults
|
||||
|
||||
@@ -1185,6 +1185,92 @@ export default suite('fs', {
|
||||
t.assert.equal(await (await t.puter.fs.read(`${dir}/thumb-a.txt`)).text(), 'a');
|
||||
},
|
||||
|
||||
'upload thumbnail callbacks can delegate to the built-in generator': async (t) => {
|
||||
const dir = `${home(t)}/fs-suite-thumb-delegation`;
|
||||
await t.puter.fs.mkdir(dir);
|
||||
let delegated = false;
|
||||
await t.puter.fs.upload(new File(['original bytes'], 'document.txt'), dir, {
|
||||
thumbnailGenerator: async (file, context) => {
|
||||
t.assert.equal(typeof context.defaultGenerator, 'function');
|
||||
t.assert.equal(context.signal?.aborted, false);
|
||||
const result = await context.defaultGenerator(file);
|
||||
t.assert.equal(result, undefined);
|
||||
delegated = true;
|
||||
return result;
|
||||
},
|
||||
});
|
||||
t.assert.equal(delegated, true);
|
||||
t.assert.equal(await (await t.puter.fs.read(`${dir}/document.txt`)).text(), 'original bytes');
|
||||
},
|
||||
|
||||
'upload cancellation during thumbnail generation does not write files': async (t) => {
|
||||
const dir = `${home(t)}/fs-suite-thumb-cancel`;
|
||||
await t.puter.fs.mkdir(dir);
|
||||
let request: XMLHttpRequest;
|
||||
let aborted = 0;
|
||||
let signalAborted = false;
|
||||
let started = false;
|
||||
const error = await t.assert.rejects(() => t.puter.fs.upload(
|
||||
new File(['must not upload'], 'cancelled.pdf'), dir, {
|
||||
init: (_operationId, xhr) => { request = xhr; },
|
||||
start: () => { started = true; },
|
||||
abort: () => { aborted++; },
|
||||
thumbnailGenerator: async (_file, { signal }) => {
|
||||
request.abort();
|
||||
request.abort();
|
||||
signalAborted = signal.aborted;
|
||||
return undefined;
|
||||
},
|
||||
},
|
||||
));
|
||||
t.assert.equal((error as { code: string }).code, 'upload_aborted');
|
||||
t.assert.equal(signalAborted, true);
|
||||
t.assert.equal(started, false);
|
||||
t.assert.equal(aborted, 1);
|
||||
t.assert.equal((await t.puter.fs.readdir(dir)).length, 0);
|
||||
},
|
||||
|
||||
'upload remains cancelled when a pending thumbnail generator finishes late': async (t) => {
|
||||
const dir = `${home(t)}/fs-suite-thumb-cancel-late`;
|
||||
await t.puter.fs.mkdir(dir);
|
||||
let request: XMLHttpRequest;
|
||||
let finish: (thumbnail: string) => void;
|
||||
let entered: () => void;
|
||||
let started = false;
|
||||
const generating = new Promise<void>(resolve => { entered = resolve; });
|
||||
const result = t.puter.fs.upload(new File(['original'], 'cancelled.pdf'), dir, {
|
||||
init: (_operationId, xhr) => { request = xhr; },
|
||||
start: () => { started = true; },
|
||||
thumbnailGenerator: () => new Promise<string>(resolve => {
|
||||
finish = resolve;
|
||||
entered();
|
||||
}),
|
||||
});
|
||||
const rejection = t.assert.rejects(() => result);
|
||||
await generating;
|
||||
request.abort();
|
||||
t.assert.equal(((await rejection) as { code: string }).code, 'upload_aborted');
|
||||
finish(`data:image/png;base64,${TINY_PNG_BASE64}`);
|
||||
await new Promise(resolve => setTimeout(resolve, 0));
|
||||
t.assert.equal(started, false);
|
||||
t.assert.equal((await t.puter.fs.readdir(dir)).length, 0);
|
||||
},
|
||||
|
||||
'upload cancellation in init does not invoke thumbnail generation': async (t) => {
|
||||
const dir = `${home(t)}/fs-suite-thumb-cancel-init`;
|
||||
await t.puter.fs.mkdir(dir);
|
||||
let generated = false;
|
||||
const error = await t.assert.rejects(() => t.puter.fs.upload(
|
||||
new File(['must not upload'], 'cancelled.pdf'), dir, {
|
||||
init: (_operationId, xhr) => xhr.abort(),
|
||||
thumbnailGenerator: async () => { generated = true; return undefined; },
|
||||
},
|
||||
));
|
||||
t.assert.equal((error as { code: string }).code, 'upload_aborted');
|
||||
t.assert.equal(generated, false);
|
||||
t.assert.equal((await t.puter.fs.readdir(dir)).length, 0);
|
||||
},
|
||||
|
||||
'upload survives a thumbnail generator that throws': async (t) => {
|
||||
const dir = `${home(t)}/fs-suite-thumb-throws`;
|
||||
await t.puter.fs.mkdir(dir);
|
||||
|
||||
Reference in New Issue
Block a user