mirror of
https://github.com/HeyPuter/puter.git
synced 2026-09-23 13:46:11 +00:00
cleanup: remove thumbnail service for client side thumbnails (#2468)
* cleanup: remove thumbnail service for client side thumbnails * fix: emit thumbnail create event and not just use raw data * fix: broken test
This commit is contained in:
@@ -64,10 +64,7 @@ something like the following (updated 2025-02-26):
|
||||
"engine": "sqlite",
|
||||
"path": "puter-database.sqlite"
|
||||
},
|
||||
"dynamo" :{"path":"./puter-ddb"},
|
||||
"thumbnails": {
|
||||
"engine": "http"
|
||||
}
|
||||
"dynamo" :{"path":"./puter-ddb"}
|
||||
},
|
||||
"cookie_name": "...",
|
||||
"jwt_secret": "...",
|
||||
|
||||
@@ -119,9 +119,6 @@ const install = async ({ context, services, app, useapi, modapi }) => {
|
||||
// TODO: move these to top level imports or await imports and esm this file
|
||||
|
||||
const { CommandService } = require('./services/CommandService');
|
||||
const { HTTPThumbnailService } = require('./services/thumbnails/HTTPThumbnailService');
|
||||
const { PureJSThumbnailService } = require('./services/thumbnails/PureJSThumbnailService');
|
||||
const { NAPIThumbnailService } = require('./services/thumbnails/NAPIThumbnailService');
|
||||
const { RateLimitService } = require('./services/sla/RateLimitService');
|
||||
const { AuthService } = require('./services/auth/AuthService');
|
||||
const { SLAService } = require('./services/sla/SLAService');
|
||||
@@ -151,7 +148,6 @@ const install = async ({ context, services, app, useapi, modapi }) => {
|
||||
const { MakeProdDebuggingLessAwfulService } = require('./services/MakeProdDebuggingLessAwfulService');
|
||||
const { ConfigurableCountingService } = require('./services/ConfigurableCountingService');
|
||||
const { FSLockService } = require('./services/fs/FSLockService');
|
||||
const { StrategizedService } = require('./services/StrategizedService');
|
||||
const FilesystemAPIService = require('./services/FilesystemAPIService');
|
||||
const ServeGUIService = require('./services/ServeGUIService');
|
||||
const PuterAPIService = require('./services/PuterAPIService');
|
||||
@@ -247,15 +243,6 @@ const install = async ({ context, services, app, useapi, modapi }) => {
|
||||
services.registerService('identification', IdentificationService);
|
||||
services.registerService('auth-audit', AuthAuditService);
|
||||
services.registerService('counting', ConfigurableCountingService);
|
||||
services.registerService('thumbnails', StrategizedService, {
|
||||
strategy_key: 'engine',
|
||||
default_strategy: 'purejs',
|
||||
strategies: {
|
||||
napi: [NAPIThumbnailService],
|
||||
purejs: [PureJSThumbnailService],
|
||||
http: [HTTPThumbnailService],
|
||||
},
|
||||
});
|
||||
services.registerService('__refresh-assocs', RefreshAssociationsService);
|
||||
services.registerService('__prod-debugging', MakeProdDebuggingLessAwfulService);
|
||||
const { EventService } = require('./services/EventService');
|
||||
|
||||
@@ -34,8 +34,5 @@ module.exports = {
|
||||
dynamo: {
|
||||
path: './puter-ddb',
|
||||
},
|
||||
thumbnails: {
|
||||
engine: 'purejs',
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -24,7 +24,7 @@ const UserParam = require('../../api/filesystem/UserParam');
|
||||
const config = require('../../config');
|
||||
const { chkperm, validate_fsentry_name } = require('../../helpers');
|
||||
const { TeePromise } = require('@heyputer/putility').libs.promise;
|
||||
const { pausing_tee, offset_write_stream, stream_to_the_void } = require('../../util/streamutil');
|
||||
const { offset_write_stream } = require('../../util/streamutil');
|
||||
const { TYPE_DIRECTORY } = require('../FSNodeContext');
|
||||
const { LLRead } = require('../ll_operations/ll_read');
|
||||
const { RootNodeSelector, NodePathSelector } = require('../node/selectors');
|
||||
@@ -100,7 +100,7 @@ class HLWrite extends HLFilesystemOperation {
|
||||
- create missing parent directories
|
||||
- overwrite existing files
|
||||
- deduplicate files with the same name
|
||||
// - create thumbnails; this will happen in low-level operation for now
|
||||
- accept client-provided thumbnails
|
||||
- create shortcuts
|
||||
`;
|
||||
|
||||
@@ -136,7 +136,6 @@ class HLWrite extends HLFilesystemOperation {
|
||||
|
||||
static MODULES = {
|
||||
_path: require('path'),
|
||||
mime: require('mime-types'),
|
||||
};
|
||||
|
||||
async _run () {
|
||||
@@ -318,58 +317,13 @@ class HLWrite extends HLFilesystemOperation {
|
||||
this.checkpoint('before thumbnail');
|
||||
|
||||
let thumbnail_promise = new TeePromise();
|
||||
if ( await parent.isAppDataDirectory() || values.no_thumbnail ) {
|
||||
if ( await parent.isAppDataDirectory() || values.no_thumbnail || !values.thumbnail ) {
|
||||
thumbnail_promise.resolve(undefined);
|
||||
} else if ( values.thumbnail ) {
|
||||
// Use the thumbnail provided by the client (base64 string)
|
||||
thumbnail_promise.resolve(values.thumbnail);
|
||||
} else {
|
||||
(async () => {
|
||||
const reason = await (async () => {
|
||||
const { mime } = this.modules;
|
||||
const thumbnails = context.get('services').get('thumbnails');
|
||||
|
||||
const content_type = mime.contentType(target_name);
|
||||
this.log.debug('CONTENT TYPE', content_type);
|
||||
if ( ! content_type ) return 'no content type';
|
||||
if ( ! thumbnails.is_supported_mimetype(content_type) ) return 'unsupported content type';
|
||||
if ( ! thumbnails.is_supported_size(values.file.size) ) return 'too large';
|
||||
|
||||
// Create file object for thumbnail by either using an existing
|
||||
// buffer (ex: /download endpoint) or by forking a stream
|
||||
// (ex: /write and /batch endpoints).
|
||||
const thumb_file = (() => {
|
||||
if ( values.file.buffer ) return values.file;
|
||||
|
||||
const [replace_stream, thumbnail_stream] =
|
||||
pausing_tee(values.file.stream, 2);
|
||||
|
||||
values.file.stream = replace_stream;
|
||||
return { ...values.file, stream: thumbnail_stream };
|
||||
})();
|
||||
|
||||
let thumbnail;
|
||||
try {
|
||||
thumbnail = await thumbnails.thumbify(thumb_file);
|
||||
} catch (e) {
|
||||
stream_to_the_void(thumb_file.stream);
|
||||
return `thumbnail error: ${ e.message}`;
|
||||
}
|
||||
|
||||
const thumbnailData = { url: thumbnail };
|
||||
if ( thumbnailData.url ) {
|
||||
await svc_event.emit('thumbnail.created', thumbnailData); // An extension can modify where this thumbnail is stored
|
||||
}
|
||||
|
||||
thumbnail_promise.resolve(thumbnailData.url);
|
||||
})();
|
||||
if ( reason ) {
|
||||
this.log.debug('REASON', reason);
|
||||
thumbnail_promise.resolve(undefined);
|
||||
|
||||
// values.file.stream = logging_stream(values.file.stream);
|
||||
}
|
||||
})();
|
||||
// Allow extensions to transform client-provided thumbnails before DB write.
|
||||
const thumbnailData = { url: values.thumbnail };
|
||||
await svc_event.emit('thumbnail.created', thumbnailData);
|
||||
thumbnail_promise.resolve(thumbnailData.url);
|
||||
}
|
||||
|
||||
this.checkpoint('before delegate');
|
||||
|
||||
@@ -1,462 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
// TODO: If an RPC protocol is ever used this service can be replaced
|
||||
// with a more general RPCService and a model.
|
||||
|
||||
const axios = require('axios');
|
||||
|
||||
const { TeePromise } = require('@heyputer/putility').libs.promise;
|
||||
const FormData = require('form-data');
|
||||
const { stream_to_the_void, buffer_to_stream } = require('../../util/streamutil');
|
||||
const BaseService = require('../BaseService');
|
||||
|
||||
class ThumbnailOperation extends TeePromise {
|
||||
// static MAX_RECYCLE_COUNT = 5*3;
|
||||
static MAX_RECYCLE_COUNT = 3;
|
||||
constructor (file) {
|
||||
super();
|
||||
this.file = file;
|
||||
this.recycle_count = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Recycles the ThumbnailOperation instance.
|
||||
*
|
||||
* Increments the recycle count and checks if the operation can be recycled again.
|
||||
* If the recycle count exceeds the maximum allowed, the operation is resolved with undefined.
|
||||
*
|
||||
* @returns {boolean} Returns true if the operation can be recycled, false otherwise.
|
||||
*/
|
||||
recycle () {
|
||||
this.recycle_count++;
|
||||
|
||||
if ( this.recycle_count > this.constructor.MAX_RECYCLE_COUNT ) {
|
||||
this.resolve(undefined);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @class HTTPThumbnailService
|
||||
* @extends BaseService
|
||||
* @description
|
||||
* This class implements a service for generating thumbnails from various file types via HTTP requests.
|
||||
* It manages a queue of thumbnail generation operations, handles the execution of these operations,
|
||||
* and provides methods to check file support, manage service status, and interact with an external
|
||||
* thumbnail generation service. The service can be configured to periodically query supported MIME types
|
||||
* and handles file size limitations and recycling of thumbnail generation attempts.
|
||||
*/
|
||||
class HTTPThumbnailService extends BaseService {
|
||||
static STATUS_IDLE = {};
|
||||
static STATUS_RUNNING = {};
|
||||
|
||||
static LIMIT = 400 * 1024 * 1024;
|
||||
|
||||
static MODULES = {
|
||||
setTimeout,
|
||||
axios,
|
||||
};
|
||||
|
||||
static SUPPORTED_MIMETYPES = [
|
||||
'audio/ogg',
|
||||
'audio/wave',
|
||||
'audio/mpeg',
|
||||
'application/ogg',
|
||||
'application/pdf',
|
||||
// 'image/bmp',
|
||||
'image/gif',
|
||||
'image/jpeg',
|
||||
'image/jpg',
|
||||
'image/png',
|
||||
// 'image/tiff',
|
||||
'image/webp',
|
||||
'video/avi',
|
||||
'video/x-msvideo',
|
||||
'video/msvideo',
|
||||
'video/flv',
|
||||
'video/x-flv',
|
||||
'video/mp4',
|
||||
'video/x-matroska',
|
||||
'video/quicktime',
|
||||
'video/webm',
|
||||
];
|
||||
|
||||
constructor (cons) {
|
||||
const { services, my_config } = cons;
|
||||
super(cons);
|
||||
|
||||
this.services = services;
|
||||
this.log = services.get('log-service').create('thumbnail-service');
|
||||
this.errors = services.get('error-service').create(this.log);
|
||||
this.config = my_config;
|
||||
|
||||
this.queue = [];
|
||||
this.status = this.constructor.STATUS_IDLE;
|
||||
|
||||
this.LIMIT = my_config?.limit ?? this.constructor.LIMIT;
|
||||
|
||||
if ( my_config?.query_supported_types !== false ) {
|
||||
/**
|
||||
* Periodically queries the thumbnail service for supported MIME types.
|
||||
*
|
||||
* @memberof HTTPThumbnailService
|
||||
* @private
|
||||
* @method query_supported_mime_types_
|
||||
* @returns {Promise<void>} A promise that resolves when the query is complete.
|
||||
* @notes
|
||||
* - This method is called every minute if `query_supported_types` in the config is not set to false.
|
||||
* - Updates the `SUPPORTED_MIMETYPES` static property of the class with the latest MIME types.
|
||||
*/
|
||||
setInterval(() => {
|
||||
this.query_supported_mime_types_();
|
||||
}, 60 * 1000);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets up the HTTP routes for the thumbnail service.
|
||||
* This method is called during the installation process of the service.
|
||||
*
|
||||
* @param {Object} _ - Unused parameter, typically the context or request object.
|
||||
* @param {Object} options - An object containing the Express application instance.
|
||||
* @param {Object} options.app - The Express application object to mount the routes onto.
|
||||
*/
|
||||
async ['__on_install.routes'] (_, { app }) {
|
||||
/**
|
||||
* Sets up the routes for the thumbnail service.
|
||||
*
|
||||
* This method is called when the service is installed to configure the Express application
|
||||
* with the necessary routes for handling thumbnail-related HTTP requests.
|
||||
*
|
||||
* @param {Object} _ - Unused parameter, part of the installation context.
|
||||
* @param {Object} context - The context object containing the Express application.
|
||||
* @param {Express.Application} context.app - The Express application to configure routes on.
|
||||
*/
|
||||
const r_thumbs = (() => {
|
||||
const require = this.require;
|
||||
const express = require('express');
|
||||
return express.Router();
|
||||
})();
|
||||
|
||||
app.use('/thumbs', r_thumbs);
|
||||
|
||||
r_thumbs.get('/status', (req, res) => {
|
||||
/**
|
||||
* Get the current status of the thumbnail service.
|
||||
* @param {Request} req - Express request object.
|
||||
* @param {Response} res - Express response object.
|
||||
*/
|
||||
const status_as_string = (status) => {
|
||||
switch ( status ) {
|
||||
case this.constructor.STATUS_IDLE:
|
||||
return 'idle';
|
||||
case this.constructor.STATUS_RUNNING:
|
||||
return 'running';
|
||||
default:
|
||||
return 'unknown';
|
||||
}
|
||||
};
|
||||
res.json({
|
||||
status: status_as_string(this.status),
|
||||
queue: this.queue.length,
|
||||
recycle_counts: this.queue.map(job => job.recycle_count),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Initializes the thumbnail service by setting up health checks.
|
||||
* This method is called when the service is installed to ensure
|
||||
* the thumbnail generation service is responsive.
|
||||
*
|
||||
* @async
|
||||
* @returns {Promise<void>} A promise that resolves when initialization is complete.
|
||||
*/
|
||||
async _init () {
|
||||
const services = this.services;
|
||||
const svc_serverHealth = services.get('server-health');
|
||||
|
||||
/**
|
||||
* Initializes the thumbnail service by setting up health checks.
|
||||
* @async
|
||||
* @method
|
||||
* @memberof HTTPThumbnailService
|
||||
* @instance
|
||||
* @description This method adds a health check for the thumbnail service to ensure it's operational.
|
||||
* It uses axios to make a ping request to the thumbnail service.
|
||||
*/
|
||||
svc_serverHealth.add_check('thumbnail-ping', async () => {
|
||||
await axios.request({
|
||||
method: 'get',
|
||||
url: `${this.host_}/ping`,
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
get host_ () {
|
||||
return this.config.host || 'http://127.0.0.1:3101';
|
||||
}
|
||||
|
||||
is_supported_mimetype (mimetype) {
|
||||
return this.constructor.SUPPORTED_MIMETYPES.includes(mimetype);
|
||||
}
|
||||
|
||||
is_supported_size (size) {
|
||||
return size < this.LIMIT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Thumbifies a given file by creating a thumbnail.
|
||||
*
|
||||
* @param {object} file - An object describing the file in the same format
|
||||
* as the file object created by multer. The necessary properties are
|
||||
* `buffer`, `filename`, and `mimetype`.
|
||||
* @returns {Promise<string|undefined>} A Promise that resolves to the base64 encoded thumbnail data URL,
|
||||
* or `undefined` if thumbification fails or is not possible.
|
||||
* @throws Will log errors if thumbification process encounters issues.
|
||||
*/
|
||||
async thumbify (file) {
|
||||
const job = new ThumbnailOperation(file);
|
||||
this.queue.push(job);
|
||||
this.checkShouldExec_();
|
||||
return await job;
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if the thumbnail generation process should start executing.
|
||||
* This method evaluates if the service is in an idle state, has items in the queue,
|
||||
* and is not in test mode before initiating the execution.
|
||||
*
|
||||
* @private
|
||||
*/
|
||||
checkShouldExec_ () {
|
||||
if ( this.test_mode ) {
|
||||
this.test_checked_exec = true;
|
||||
return;
|
||||
}
|
||||
if ( this.status !== this.constructor.STATUS_IDLE ) return;
|
||||
if ( this.queue.length === 0 ) return;
|
||||
this.exec_();
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes thumbnail generation for queued files.
|
||||
*
|
||||
* This method is responsible for processing files in the queue for thumbnail generation.
|
||||
* It handles the transition of service status, manages file size limits, and initiates
|
||||
* the thumbnail generation process for files within the size limit. If errors occur,
|
||||
* it handles the resolution of jobs appropriately and logs errors.
|
||||
*
|
||||
* @private
|
||||
*/
|
||||
async exec_ () {
|
||||
const { setTimeout } = this.modules;
|
||||
|
||||
this.status = this.constructor.STATUS_RUNNING;
|
||||
|
||||
const LIMIT = this.LIMIT;
|
||||
|
||||
// Grab up to 400MB worth of files to send to the thumbnail service.
|
||||
// Resolve any jobs as undefined if they're over the limit.
|
||||
|
||||
let total_size = 0;
|
||||
const queue = [];
|
||||
while ( this.queue.length > 0 ) {
|
||||
const job = this.queue[0];
|
||||
const size = job.file.size;
|
||||
if ( size > LIMIT ) {
|
||||
job.resolve(undefined);
|
||||
if ( job.file.stream ) stream_to_the_void(job.file.stream);
|
||||
this.queue.shift();
|
||||
continue;
|
||||
}
|
||||
if ( total_size + size > LIMIT ) break;
|
||||
total_size += size;
|
||||
queue.push(job);
|
||||
this.queue.shift();
|
||||
}
|
||||
|
||||
if ( queue.length === 0 ) {
|
||||
this.status = this.constructor.STATUS_IDLE;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
return await this.exec_0(queue);
|
||||
} catch ( err ) {
|
||||
await new Promise(resolve => setTimeout(resolve, 200));
|
||||
|
||||
// const new_queue = queue.filter(job => job.recycle());
|
||||
// this.queue = new_queue.concat(this.queue);
|
||||
this.queue = [];
|
||||
for ( const job of queue ) {
|
||||
if ( job.file.stream ) stream_to_the_void(job.file.stream);
|
||||
job.resolve(undefined);
|
||||
}
|
||||
|
||||
this.errors.report('thumbnails-exec', {
|
||||
source: err,
|
||||
trace: true,
|
||||
alarm: true,
|
||||
});
|
||||
} finally {
|
||||
this.status = this.constructor.STATUS_IDLE;
|
||||
this.checkShouldExec_();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes the thumbnail generation process for the given queue of jobs.
|
||||
*
|
||||
* This method attempts to process the provided queue, handling errors gracefully
|
||||
* by recycling jobs or resolving them as undefined if they exceed size limits or
|
||||
* if an error occurs during the request. After execution, it updates the service
|
||||
* status and checks for further executions if needed.
|
||||
*
|
||||
* @param {Array<ThumbnailOperation>} queue - An array of ThumbnailOperation objects
|
||||
* representing the jobs to be processed.
|
||||
* @returns {Promise<any>} - A promise that resolves with the results of the thumbnail
|
||||
* generation or undefined if an error occurred.
|
||||
*/
|
||||
async exec_0 (queue) {
|
||||
this.log.info('starting thumbnail request');
|
||||
const resp = await this.request_({ queue });
|
||||
this.log.info('done thumbnail request');
|
||||
|
||||
if ( resp.status !== 200 ) {
|
||||
this.log.error('Thumbnail service returned non-200 status');
|
||||
throw new Error('Thumbnail service returned non-200 status');
|
||||
}
|
||||
|
||||
const results = resp.data;
|
||||
|
||||
if ( results.length !== queue.length ) {
|
||||
this.log.error('Thumbnail service returned wrong number of results');
|
||||
throw new Error('Thumbnail service returned wrong number of results');
|
||||
}
|
||||
|
||||
for ( let i = 0 ; i < queue.length ; i++ ) {
|
||||
const result = results[i];
|
||||
const job = queue[i];
|
||||
job.resolve(result.encoded
|
||||
&& `data:image/png;base64,${result.encoded}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handles the thumbnail request process by sending queued files to the thumbnail service,
|
||||
* managing the response, and resolving the thumbnail operations accordingly.
|
||||
*
|
||||
* @param {ThumbnailOperation[]} queue - An array of ThumbnailOperation instances representing files to be thumbnailed.
|
||||
* @returns {Promise} A promise that resolves with the thumbnail service response or throws an error if the request fails.
|
||||
*/
|
||||
async request_ ({ queue }) {
|
||||
if ( this.test_mode ) {
|
||||
const results = [];
|
||||
for ( const job of queue ) {
|
||||
if ( job.file?.behavior === 'fail' ) {
|
||||
throw new Error('test fail');
|
||||
}
|
||||
results.push({
|
||||
encoded: 'data:image/png;base64,' +
|
||||
'iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX' +
|
||||
'///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASU' +
|
||||
'VORK5CYII',
|
||||
});
|
||||
}
|
||||
return {
|
||||
status: 200,
|
||||
data: results,
|
||||
};
|
||||
}
|
||||
|
||||
const form = new FormData();
|
||||
for ( const job of queue ) {
|
||||
|
||||
/**
|
||||
* Prepares and sends a request to the thumbnail service for processing multiple files.
|
||||
*
|
||||
* @param {Object} options - Options object containing the queue of files.
|
||||
* @param {Array<ThumbnailOperation>} options.queue - An array of ThumbnailOperation objects to be processed.
|
||||
* @returns {Promise<Object>} A promise that resolves to the response from the thumbnail service.
|
||||
* @throws {Error} If the thumbnail service returns an error or if there's an issue with the request.
|
||||
*/
|
||||
const file_data = job.file.buffer ? (() => {
|
||||
job.file.size = job.file.buffer.length;
|
||||
return buffer_to_stream(job.file.buffer);
|
||||
})() : job.file.stream;
|
||||
|
||||
form.append('file', file_data, {
|
||||
filename: job.file.name ?? job.file.originalname,
|
||||
contentType: job.file.type ?? job.file.mimetype,
|
||||
knownLength: job.file.size,
|
||||
});
|
||||
}
|
||||
|
||||
const resp = await axios.request({
|
||||
method: 'post',
|
||||
url: `${this.host_}/thumbify`,
|
||||
data: form,
|
||||
headers: {
|
||||
'Content-Type': 'multipart/form-data',
|
||||
},
|
||||
});
|
||||
|
||||
return resp;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queries the thumbnail services to check what mime types
|
||||
* are supported for thumbnail generation.
|
||||
* Updates internal state to reflect that.
|
||||
* @returns {Promise<void>} A promise that resolves when the MIME types are updated.
|
||||
*/
|
||||
async query_supported_mime_types_ () {
|
||||
const resp = await axios.request({
|
||||
method: 'get',
|
||||
url: `${this.host_}/supported`,
|
||||
});
|
||||
|
||||
const data = resp.data;
|
||||
|
||||
if ( ! Array.isArray(data) ) {
|
||||
this.log.error('Thumbnail service returned invalid data');
|
||||
return;
|
||||
}
|
||||
|
||||
const mime_set = {};
|
||||
|
||||
for ( const entry of data ) {
|
||||
mime_set[entry.StandardMIMEType] = true;
|
||||
for ( const mime of entry.MIMETypes ) {
|
||||
mime_set[mime] = true;
|
||||
}
|
||||
}
|
||||
|
||||
this.constructor.SUPPORTED_MIMETYPES = Object.keys(mime_set);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
HTTPThumbnailService,
|
||||
};
|
||||
@@ -1,84 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { createTestKernel } from '../../../tools/test.mjs';
|
||||
import { HTTPThumbnailService } from './HTTPThumbnailService.js';
|
||||
|
||||
// We need to access ThumbnailOperation, but it's not exported
|
||||
// Let's recreate it here for testing purposes
|
||||
const { TeePromise } = require('@heyputer/putility').libs.promise;
|
||||
|
||||
class ThumbnailOperation extends TeePromise {
|
||||
static MAX_RECYCLE_COUNT = 3;
|
||||
constructor (file: any) {
|
||||
super();
|
||||
this.file = file;
|
||||
this.recycle_count = 0;
|
||||
}
|
||||
|
||||
recycle () {
|
||||
this.recycle_count++;
|
||||
|
||||
if ( this.recycle_count > this.constructor.MAX_RECYCLE_COUNT ) {
|
||||
this.resolve(undefined);
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
describe('HTTPThumbnailService', () => {
|
||||
it('should handle thumbnail operations correctly', async () => {
|
||||
const testKernel = await createTestKernel({
|
||||
serviceMap: {
|
||||
'thumbs-http': HTTPThumbnailService,
|
||||
},
|
||||
});
|
||||
|
||||
const thumbnailService = testKernel.services!.get('thumbs-http') as HTTPThumbnailService;
|
||||
|
||||
// Mock error reporting and logging
|
||||
thumbnailService.errors.report = () => {
|
||||
};
|
||||
|
||||
thumbnailService.log = {
|
||||
info: () => {
|
||||
},
|
||||
error: () => {
|
||||
},
|
||||
noticeme: () => {
|
||||
},
|
||||
};
|
||||
|
||||
// Thumbnail operation eventually recycles
|
||||
{
|
||||
const thop = new ThumbnailOperation(null);
|
||||
for ( let i = 0 ; i < ThumbnailOperation.MAX_RECYCLE_COUNT ; i++ ) {
|
||||
expect(thop.recycle()).toBe(true);
|
||||
}
|
||||
expect(thop.recycle()).toBe(false);
|
||||
}
|
||||
|
||||
thumbnailService.test_mode = true;
|
||||
|
||||
// Request and await the thumbnailing of a few files
|
||||
for ( let i = 0 ; i < 3 ; i++ ) {
|
||||
const job = new ThumbnailOperation({ behavior: 'ok' });
|
||||
thumbnailService.queue.push(job);
|
||||
}
|
||||
thumbnailService.test_checked_exec = false;
|
||||
await thumbnailService.exec_();
|
||||
expect(thumbnailService.queue.length).toBe(0);
|
||||
expect(thumbnailService.test_checked_exec).toBe(true);
|
||||
|
||||
// test with failed job
|
||||
const job = new ThumbnailOperation({ behavior: 'fail' });
|
||||
thumbnailService.queue.push(job);
|
||||
thumbnailService.test_checked_exec = false;
|
||||
await thumbnailService.exec_();
|
||||
expect(thumbnailService.queue.length).toBe(0);
|
||||
expect(thumbnailService.test_checked_exec).toBe(true);
|
||||
|
||||
thumbnailService.test_mode = false;
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,72 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
const BaseService = require('../BaseService');
|
||||
|
||||
/**
|
||||
* Service class for generating thumbnails using Node API (NAPI)
|
||||
* Extends BaseService to handle thumbnail generation for various image formats
|
||||
* Supports multiple image types (JPEG, PNG, WebP, GIF, AVIF, TIFF, SVG)
|
||||
* Implements size limits and format validation for thumbnail generation
|
||||
* Uses Sharp library for image processing and transformation
|
||||
* @class NAPIThumbnailService
|
||||
* @extends BaseService
|
||||
*/
|
||||
class NAPIThumbnailService extends BaseService {
|
||||
static LIMIT = 400 * 1024 * 1024;
|
||||
static SUPPORTED_MIMETYPES = [
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/webp',
|
||||
'image/gif',
|
||||
'image/avif',
|
||||
'image/tiff',
|
||||
'image/svg+xml',
|
||||
];
|
||||
|
||||
static MODULES = {
|
||||
sharp: () => require('sharp'),
|
||||
};
|
||||
|
||||
is_supported_mimetype (mimetype) {
|
||||
return this.constructor.SUPPORTED_MIMETYPES.includes(mimetype);
|
||||
}
|
||||
|
||||
/**
|
||||
* Checks if a file size is within the supported limit for thumbnail generation
|
||||
* @param {number} size - The file size in bytes to check
|
||||
* @returns {boolean} True if size is less than or equal to the limit, false otherwise
|
||||
*/
|
||||
is_supported_size (size) {
|
||||
return size <= this.constructor.LIMIT;
|
||||
}
|
||||
async thumbify (file) {
|
||||
const transformer = await this.modules.sharp()()
|
||||
.resize(128)
|
||||
.png();
|
||||
file.stream.pipe(transformer);
|
||||
const buffer = await transformer.toBuffer();
|
||||
// .toBuffer();
|
||||
const base64 = buffer.toString('base64');
|
||||
return `data:image/png;base64,${base64}`;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
NAPIThumbnailService,
|
||||
};
|
||||
@@ -1,80 +0,0 @@
|
||||
/*
|
||||
* Copyright (C) 2024-present Puter Technologies Inc.
|
||||
*
|
||||
* This file is part of Puter.
|
||||
*
|
||||
* Puter is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU Affero General Public License as published
|
||||
* by the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU Affero General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU Affero General Public License
|
||||
* along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
*/
|
||||
const Jimp = require('jimp');
|
||||
const BaseService = require('../BaseService');
|
||||
const { stream_to_buffer } = require('../../util/streamutil');
|
||||
|
||||
/**
|
||||
* @class PureJSThumbnailService
|
||||
* @extends BaseService
|
||||
* @description This class represents a thumbnail service that operates entirely in JavaScript without relying on any low-level compiled libraries.
|
||||
* It is designed for development and testing environments due to its CPU-intensive nature, making it less suitable for production deployments.
|
||||
* The service supports various image formats and provides methods to check supported MIME types and file sizes, as well as to generate thumbnails.
|
||||
*
|
||||
* @deprecated as 'sharp' module is now required for app icons anyway
|
||||
*/
|
||||
class PureJSThumbnailService extends BaseService {
|
||||
static DESCRIPTION = `
|
||||
This thumbnail service doesn't depend on any low-level compiled
|
||||
libraries. It is CPU-intensive, so it's not ideal for production
|
||||
deployments, but it's great for development and testing.
|
||||
`;
|
||||
|
||||
static LIMIT = 400 * 1024 * 1024;
|
||||
static SUPPORTED_MIMETYPES = [
|
||||
'image/jpeg',
|
||||
'image/png',
|
||||
'image/bmp',
|
||||
'image/tiff',
|
||||
'image/gif',
|
||||
];
|
||||
|
||||
static MODULES = {
|
||||
jimp: require('jimp'),
|
||||
};
|
||||
|
||||
is_supported_mimetype (mimetype) {
|
||||
return this.constructor.SUPPORTED_MIMETYPES.includes(mimetype);
|
||||
}
|
||||
is_supported_size (size) {
|
||||
return size <= this.constructor.LIMIT;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates a thumbnail for the provided file.
|
||||
*
|
||||
* This method reads the file stream, resizes the image to 128x128 pixels,
|
||||
* and returns the resulting image as a base64 string.
|
||||
*
|
||||
* @param {Object} file - The file object containing the stream.
|
||||
* @param {Stream} file.stream - The stream of the file to be thumbnailed.
|
||||
* @returns {Promise<string>} A promise that resolves to the base64 string of the thumbnail.
|
||||
*/
|
||||
async thumbify (file) {
|
||||
const buffer = await stream_to_buffer(file.stream);
|
||||
const image = await Jimp.read(buffer);
|
||||
image.resize(128, 128);
|
||||
const base64 = await image.getBase64Async(Jimp.MIME_PNG);
|
||||
return base64;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
PureJSThumbnailService,
|
||||
};
|
||||
@@ -23,7 +23,6 @@ import { RuntimeModuleRegistry } from '../src/extension/RuntimeModuleRegistry.js
|
||||
import { Kernel } from '../src/Kernel.js';
|
||||
import { Core2Module } from '../src/modules/core/Core2Module.js';
|
||||
import { Container } from '../src/services/Container.js';
|
||||
import { HTTPThumbnailService } from '../src/services/thumbnails/HTTPThumbnailService.js';
|
||||
import { consoleLogManager } from '../src/util/consolelog.js';
|
||||
import { Context } from '../src/util/context.js';
|
||||
import { TestCoreModule } from '../src/modules/test-core/TestCoreModule.js';
|
||||
@@ -193,12 +192,6 @@ const main = async () => {
|
||||
for ( const mod of EssentialModules ) {
|
||||
k.add_module(new mod());
|
||||
}
|
||||
k.add_module({
|
||||
install: async (context) => {
|
||||
const services = context.get('services');
|
||||
services.registerService('thumbs-http', HTTPThumbnailService);
|
||||
},
|
||||
});
|
||||
k.boot();
|
||||
console.log('awaiting services ready');
|
||||
await k.services.ready;
|
||||
|
||||
Reference in New Issue
Block a user