diff --git a/doc/self-hosters/config.md b/doc/self-hosters/config.md
index 3f84f5aef..6ed403908 100644
--- a/doc/self-hosters/config.md
+++ b/doc/self-hosters/config.md
@@ -64,10 +64,7 @@ something like the following (updated 2025-02-26):
"engine": "sqlite",
"path": "puter-database.sqlite"
},
- "dynamo" :{"path":"./puter-ddb"},
- "thumbnails": {
- "engine": "http"
- }
+ "dynamo" :{"path":"./puter-ddb"}
},
"cookie_name": "...",
"jwt_secret": "...",
diff --git a/src/backend/src/CoreModule.js b/src/backend/src/CoreModule.js
index c5622a69a..9ddd90724 100644
--- a/src/backend/src/CoreModule.js
+++ b/src/backend/src/CoreModule.js
@@ -119,9 +119,6 @@ const install = async ({ context, services, app, useapi, modapi }) => {
// TODO: move these to top level imports or await imports and esm this file
const { CommandService } = require('./services/CommandService');
- const { HTTPThumbnailService } = require('./services/thumbnails/HTTPThumbnailService');
- const { PureJSThumbnailService } = require('./services/thumbnails/PureJSThumbnailService');
- const { NAPIThumbnailService } = require('./services/thumbnails/NAPIThumbnailService');
const { RateLimitService } = require('./services/sla/RateLimitService');
const { AuthService } = require('./services/auth/AuthService');
const { SLAService } = require('./services/sla/SLAService');
@@ -151,7 +148,6 @@ const install = async ({ context, services, app, useapi, modapi }) => {
const { MakeProdDebuggingLessAwfulService } = require('./services/MakeProdDebuggingLessAwfulService');
const { ConfigurableCountingService } = require('./services/ConfigurableCountingService');
const { FSLockService } = require('./services/fs/FSLockService');
- const { StrategizedService } = require('./services/StrategizedService');
const FilesystemAPIService = require('./services/FilesystemAPIService');
const ServeGUIService = require('./services/ServeGUIService');
const PuterAPIService = require('./services/PuterAPIService');
@@ -247,15 +243,6 @@ const install = async ({ context, services, app, useapi, modapi }) => {
services.registerService('identification', IdentificationService);
services.registerService('auth-audit', AuthAuditService);
services.registerService('counting', ConfigurableCountingService);
- services.registerService('thumbnails', StrategizedService, {
- strategy_key: 'engine',
- default_strategy: 'purejs',
- strategies: {
- napi: [NAPIThumbnailService],
- purejs: [PureJSThumbnailService],
- http: [HTTPThumbnailService],
- },
- });
services.registerService('__refresh-assocs', RefreshAssociationsService);
services.registerService('__prod-debugging', MakeProdDebuggingLessAwfulService);
const { EventService } = require('./services/EventService');
diff --git a/src/backend/src/boot/default_config.js b/src/backend/src/boot/default_config.js
index 0cbc17eea..b2ad770ce 100644
--- a/src/backend/src/boot/default_config.js
+++ b/src/backend/src/boot/default_config.js
@@ -34,8 +34,5 @@ module.exports = {
dynamo: {
path: './puter-ddb',
},
- thumbnails: {
- engine: 'purejs',
- },
},
};
diff --git a/src/backend/src/filesystem/hl_operations/hl_write.js b/src/backend/src/filesystem/hl_operations/hl_write.js
index 9cd5bcdf6..448d3b977 100644
--- a/src/backend/src/filesystem/hl_operations/hl_write.js
+++ b/src/backend/src/filesystem/hl_operations/hl_write.js
@@ -24,7 +24,7 @@ const UserParam = require('../../api/filesystem/UserParam');
const config = require('../../config');
const { chkperm, validate_fsentry_name } = require('../../helpers');
const { TeePromise } = require('@heyputer/putility').libs.promise;
-const { pausing_tee, offset_write_stream, stream_to_the_void } = require('../../util/streamutil');
+const { offset_write_stream } = require('../../util/streamutil');
const { TYPE_DIRECTORY } = require('../FSNodeContext');
const { LLRead } = require('../ll_operations/ll_read');
const { RootNodeSelector, NodePathSelector } = require('../node/selectors');
@@ -100,7 +100,7 @@ class HLWrite extends HLFilesystemOperation {
- create missing parent directories
- overwrite existing files
- deduplicate files with the same name
- // - create thumbnails; this will happen in low-level operation for now
+ - accept client-provided thumbnails
- create shortcuts
`;
@@ -136,7 +136,6 @@ class HLWrite extends HLFilesystemOperation {
static MODULES = {
_path: require('path'),
- mime: require('mime-types'),
};
async _run () {
@@ -318,58 +317,13 @@ class HLWrite extends HLFilesystemOperation {
this.checkpoint('before thumbnail');
let thumbnail_promise = new TeePromise();
- if ( await parent.isAppDataDirectory() || values.no_thumbnail ) {
+ if ( await parent.isAppDataDirectory() || values.no_thumbnail || !values.thumbnail ) {
thumbnail_promise.resolve(undefined);
- } else if ( values.thumbnail ) {
- // Use the thumbnail provided by the client (base64 string)
- thumbnail_promise.resolve(values.thumbnail);
} else {
- (async () => {
- const reason = await (async () => {
- const { mime } = this.modules;
- const thumbnails = context.get('services').get('thumbnails');
-
- const content_type = mime.contentType(target_name);
- this.log.debug('CONTENT TYPE', content_type);
- if ( ! content_type ) return 'no content type';
- if ( ! thumbnails.is_supported_mimetype(content_type) ) return 'unsupported content type';
- if ( ! thumbnails.is_supported_size(values.file.size) ) return 'too large';
-
- // Create file object for thumbnail by either using an existing
- // buffer (ex: /download endpoint) or by forking a stream
- // (ex: /write and /batch endpoints).
- const thumb_file = (() => {
- if ( values.file.buffer ) return values.file;
-
- const [replace_stream, thumbnail_stream] =
- pausing_tee(values.file.stream, 2);
-
- values.file.stream = replace_stream;
- return { ...values.file, stream: thumbnail_stream };
- })();
-
- let thumbnail;
- try {
- thumbnail = await thumbnails.thumbify(thumb_file);
- } catch (e) {
- stream_to_the_void(thumb_file.stream);
- return `thumbnail error: ${ e.message}`;
- }
-
- const thumbnailData = { url: thumbnail };
- if ( thumbnailData.url ) {
- await svc_event.emit('thumbnail.created', thumbnailData); // An extension can modify where this thumbnail is stored
- }
-
- thumbnail_promise.resolve(thumbnailData.url);
- })();
- if ( reason ) {
- this.log.debug('REASON', reason);
- thumbnail_promise.resolve(undefined);
-
- // values.file.stream = logging_stream(values.file.stream);
- }
- })();
+ // Allow extensions to transform client-provided thumbnails before DB write.
+ const thumbnailData = { url: values.thumbnail };
+ await svc_event.emit('thumbnail.created', thumbnailData);
+ thumbnail_promise.resolve(thumbnailData.url);
}
this.checkpoint('before delegate');
diff --git a/src/backend/src/services/thumbnails/HTTPThumbnailService.js b/src/backend/src/services/thumbnails/HTTPThumbnailService.js
deleted file mode 100644
index 01f306016..000000000
--- a/src/backend/src/services/thumbnails/HTTPThumbnailService.js
+++ /dev/null
@@ -1,462 +0,0 @@
-/*
- * Copyright (C) 2024-present Puter Technologies Inc.
- *
- * This file is part of Puter.
- *
- * Puter is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published
- * by the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see .
- */
-// TODO: If an RPC protocol is ever used this service can be replaced
-// with a more general RPCService and a model.
-
-const axios = require('axios');
-
-const { TeePromise } = require('@heyputer/putility').libs.promise;
-const FormData = require('form-data');
-const { stream_to_the_void, buffer_to_stream } = require('../../util/streamutil');
-const BaseService = require('../BaseService');
-
-class ThumbnailOperation extends TeePromise {
- // static MAX_RECYCLE_COUNT = 5*3;
- static MAX_RECYCLE_COUNT = 3;
- constructor (file) {
- super();
- this.file = file;
- this.recycle_count = 0;
- }
-
- /**
- * Recycles the ThumbnailOperation instance.
- *
- * Increments the recycle count and checks if the operation can be recycled again.
- * If the recycle count exceeds the maximum allowed, the operation is resolved with undefined.
- *
- * @returns {boolean} Returns true if the operation can be recycled, false otherwise.
- */
- recycle () {
- this.recycle_count++;
-
- if ( this.recycle_count > this.constructor.MAX_RECYCLE_COUNT ) {
- this.resolve(undefined);
- return false;
- }
-
- return true;
- }
-}
-
-/**
-* @class HTTPThumbnailService
-* @extends BaseService
-* @description
-* This class implements a service for generating thumbnails from various file types via HTTP requests.
-* It manages a queue of thumbnail generation operations, handles the execution of these operations,
-* and provides methods to check file support, manage service status, and interact with an external
-* thumbnail generation service. The service can be configured to periodically query supported MIME types
-* and handles file size limitations and recycling of thumbnail generation attempts.
-*/
-class HTTPThumbnailService extends BaseService {
- static STATUS_IDLE = {};
- static STATUS_RUNNING = {};
-
- static LIMIT = 400 * 1024 * 1024;
-
- static MODULES = {
- setTimeout,
- axios,
- };
-
- static SUPPORTED_MIMETYPES = [
- 'audio/ogg',
- 'audio/wave',
- 'audio/mpeg',
- 'application/ogg',
- 'application/pdf',
- // 'image/bmp',
- 'image/gif',
- 'image/jpeg',
- 'image/jpg',
- 'image/png',
- // 'image/tiff',
- 'image/webp',
- 'video/avi',
- 'video/x-msvideo',
- 'video/msvideo',
- 'video/flv',
- 'video/x-flv',
- 'video/mp4',
- 'video/x-matroska',
- 'video/quicktime',
- 'video/webm',
- ];
-
- constructor (cons) {
- const { services, my_config } = cons;
- super(cons);
-
- this.services = services;
- this.log = services.get('log-service').create('thumbnail-service');
- this.errors = services.get('error-service').create(this.log);
- this.config = my_config;
-
- this.queue = [];
- this.status = this.constructor.STATUS_IDLE;
-
- this.LIMIT = my_config?.limit ?? this.constructor.LIMIT;
-
- if ( my_config?.query_supported_types !== false ) {
- /**
- * Periodically queries the thumbnail service for supported MIME types.
- *
- * @memberof HTTPThumbnailService
- * @private
- * @method query_supported_mime_types_
- * @returns {Promise} A promise that resolves when the query is complete.
- * @notes
- * - This method is called every minute if `query_supported_types` in the config is not set to false.
- * - Updates the `SUPPORTED_MIMETYPES` static property of the class with the latest MIME types.
- */
- setInterval(() => {
- this.query_supported_mime_types_();
- }, 60 * 1000);
- }
- }
-
- /**
- * Sets up the HTTP routes for the thumbnail service.
- * This method is called during the installation process of the service.
- *
- * @param {Object} _ - Unused parameter, typically the context or request object.
- * @param {Object} options - An object containing the Express application instance.
- * @param {Object} options.app - The Express application object to mount the routes onto.
- */
- async ['__on_install.routes'] (_, { app }) {
- /**
- * Sets up the routes for the thumbnail service.
- *
- * This method is called when the service is installed to configure the Express application
- * with the necessary routes for handling thumbnail-related HTTP requests.
- *
- * @param {Object} _ - Unused parameter, part of the installation context.
- * @param {Object} context - The context object containing the Express application.
- * @param {Express.Application} context.app - The Express application to configure routes on.
- */
- const r_thumbs = (() => {
- const require = this.require;
- const express = require('express');
- return express.Router();
- })();
-
- app.use('/thumbs', r_thumbs);
-
- r_thumbs.get('/status', (req, res) => {
- /**
- * Get the current status of the thumbnail service.
- * @param {Request} req - Express request object.
- * @param {Response} res - Express response object.
- */
- const status_as_string = (status) => {
- switch ( status ) {
- case this.constructor.STATUS_IDLE:
- return 'idle';
- case this.constructor.STATUS_RUNNING:
- return 'running';
- default:
- return 'unknown';
- }
- };
- res.json({
- status: status_as_string(this.status),
- queue: this.queue.length,
- recycle_counts: this.queue.map(job => job.recycle_count),
- });
- });
- }
-
- /**
- * Initializes the thumbnail service by setting up health checks.
- * This method is called when the service is installed to ensure
- * the thumbnail generation service is responsive.
- *
- * @async
- * @returns {Promise} A promise that resolves when initialization is complete.
- */
- async _init () {
- const services = this.services;
- const svc_serverHealth = services.get('server-health');
-
- /**
- * Initializes the thumbnail service by setting up health checks.
- * @async
- * @method
- * @memberof HTTPThumbnailService
- * @instance
- * @description This method adds a health check for the thumbnail service to ensure it's operational.
- * It uses axios to make a ping request to the thumbnail service.
- */
- svc_serverHealth.add_check('thumbnail-ping', async () => {
- await axios.request({
- method: 'get',
- url: `${this.host_}/ping`,
- });
- });
- }
-
- get host_ () {
- return this.config.host || 'http://127.0.0.1:3101';
- }
-
- is_supported_mimetype (mimetype) {
- return this.constructor.SUPPORTED_MIMETYPES.includes(mimetype);
- }
-
- is_supported_size (size) {
- return size < this.LIMIT;
- }
-
- /**
- * Thumbifies a given file by creating a thumbnail.
- *
- * @param {object} file - An object describing the file in the same format
- * as the file object created by multer. The necessary properties are
- * `buffer`, `filename`, and `mimetype`.
- * @returns {Promise} A Promise that resolves to the base64 encoded thumbnail data URL,
- * or `undefined` if thumbification fails or is not possible.
- * @throws Will log errors if thumbification process encounters issues.
- */
- async thumbify (file) {
- const job = new ThumbnailOperation(file);
- this.queue.push(job);
- this.checkShouldExec_();
- return await job;
- }
-
- /**
- * Checks if the thumbnail generation process should start executing.
- * This method evaluates if the service is in an idle state, has items in the queue,
- * and is not in test mode before initiating the execution.
- *
- * @private
- */
- checkShouldExec_ () {
- if ( this.test_mode ) {
- this.test_checked_exec = true;
- return;
- }
- if ( this.status !== this.constructor.STATUS_IDLE ) return;
- if ( this.queue.length === 0 ) return;
- this.exec_();
- }
-
- /**
- * Executes thumbnail generation for queued files.
- *
- * This method is responsible for processing files in the queue for thumbnail generation.
- * It handles the transition of service status, manages file size limits, and initiates
- * the thumbnail generation process for files within the size limit. If errors occur,
- * it handles the resolution of jobs appropriately and logs errors.
- *
- * @private
- */
- async exec_ () {
- const { setTimeout } = this.modules;
-
- this.status = this.constructor.STATUS_RUNNING;
-
- const LIMIT = this.LIMIT;
-
- // Grab up to 400MB worth of files to send to the thumbnail service.
- // Resolve any jobs as undefined if they're over the limit.
-
- let total_size = 0;
- const queue = [];
- while ( this.queue.length > 0 ) {
- const job = this.queue[0];
- const size = job.file.size;
- if ( size > LIMIT ) {
- job.resolve(undefined);
- if ( job.file.stream ) stream_to_the_void(job.file.stream);
- this.queue.shift();
- continue;
- }
- if ( total_size + size > LIMIT ) break;
- total_size += size;
- queue.push(job);
- this.queue.shift();
- }
-
- if ( queue.length === 0 ) {
- this.status = this.constructor.STATUS_IDLE;
- return;
- }
-
- try {
- return await this.exec_0(queue);
- } catch ( err ) {
- await new Promise(resolve => setTimeout(resolve, 200));
-
- // const new_queue = queue.filter(job => job.recycle());
- // this.queue = new_queue.concat(this.queue);
- this.queue = [];
- for ( const job of queue ) {
- if ( job.file.stream ) stream_to_the_void(job.file.stream);
- job.resolve(undefined);
- }
-
- this.errors.report('thumbnails-exec', {
- source: err,
- trace: true,
- alarm: true,
- });
- } finally {
- this.status = this.constructor.STATUS_IDLE;
- this.checkShouldExec_();
- }
- }
-
- /**
- * Executes the thumbnail generation process for the given queue of jobs.
- *
- * This method attempts to process the provided queue, handling errors gracefully
- * by recycling jobs or resolving them as undefined if they exceed size limits or
- * if an error occurs during the request. After execution, it updates the service
- * status and checks for further executions if needed.
- *
- * @param {Array} queue - An array of ThumbnailOperation objects
- * representing the jobs to be processed.
- * @returns {Promise} - A promise that resolves with the results of the thumbnail
- * generation or undefined if an error occurred.
- */
- async exec_0 (queue) {
- this.log.info('starting thumbnail request');
- const resp = await this.request_({ queue });
- this.log.info('done thumbnail request');
-
- if ( resp.status !== 200 ) {
- this.log.error('Thumbnail service returned non-200 status');
- throw new Error('Thumbnail service returned non-200 status');
- }
-
- const results = resp.data;
-
- if ( results.length !== queue.length ) {
- this.log.error('Thumbnail service returned wrong number of results');
- throw new Error('Thumbnail service returned wrong number of results');
- }
-
- for ( let i = 0 ; i < queue.length ; i++ ) {
- const result = results[i];
- const job = queue[i];
- job.resolve(result.encoded
- && `data:image/png;base64,${result.encoded}`);
- }
- }
-
- /**
- * Handles the thumbnail request process by sending queued files to the thumbnail service,
- * managing the response, and resolving the thumbnail operations accordingly.
- *
- * @param {ThumbnailOperation[]} queue - An array of ThumbnailOperation instances representing files to be thumbnailed.
- * @returns {Promise} A promise that resolves with the thumbnail service response or throws an error if the request fails.
- */
- async request_ ({ queue }) {
- if ( this.test_mode ) {
- const results = [];
- for ( const job of queue ) {
- if ( job.file?.behavior === 'fail' ) {
- throw new Error('test fail');
- }
- results.push({
- encoded: 'data:image/png;base64,' +
- 'iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX' +
- '///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASU' +
- 'VORK5CYII',
- });
- }
- return {
- status: 200,
- data: results,
- };
- }
-
- const form = new FormData();
- for ( const job of queue ) {
-
- /**
- * Prepares and sends a request to the thumbnail service for processing multiple files.
- *
- * @param {Object} options - Options object containing the queue of files.
- * @param {Array} options.queue - An array of ThumbnailOperation objects to be processed.
- * @returns {Promise