diff --git a/src/backend/src/consts/app-icons.js b/src/backend/src/consts/app-icons.js new file mode 100644 index 000000000..9e05df952 --- /dev/null +++ b/src/backend/src/consts/app-icons.js @@ -0,0 +1 @@ +export const APP_ICONS_SUBDOMAIN = 'puter-app-icons'; diff --git a/src/backend/src/filesystem/hl_operations/hl_readdir.js b/src/backend/src/filesystem/hl_operations/hl_readdir.js index 81a57921d..9ee5687e2 100644 --- a/src/backend/src/filesystem/hl_operations/hl_readdir.js +++ b/src/backend/src/filesystem/hl_operations/hl_readdir.js @@ -131,19 +131,19 @@ class HLReadDir extends HLFilesystemOperation { const entry = await child.getSafeEntry(); if ( !no_thumbs && entry.associated_app ) { const svc_appIcon = this.context.get('services').get('app-icon'); - const icon_result = await svc_appIcon.get_icon_stream({ - app_icon: entry.associated_app.icon, - app_uid: entry.associated_app.uid ?? entry.associated_app.uuid, + const iconResult = await svc_appIcon.getIconStream({ + appIcon: entry.associated_app.icon, + appUid: entry.associated_app.uid ?? entry.associated_app.uuid, size: 64, }); - if ( icon_result.data_url ) { - entry.associated_app.icon = icon_result.data_url; + if ( iconResult.dataUrl ?? iconResult.data_url ) { + entry.associated_app.icon = iconResult.dataUrl ?? iconResult.data_url; } else { try { - const buffer = await stream_to_buffer(icon_result.stream); - const resp_data_url = `data:${icon_result.mime};base64,${buffer.toString('base64')}`; - entry.associated_app.icon = resp_data_url; + const buffer = await stream_to_buffer(iconResult.stream); + const respDataUrl = `data:${iconResult.mime};base64,${buffer.toString('base64')}`; + entry.associated_app.icon = respDataUrl; } catch (e) { const svc_error = this.context.get('services').get('error-service'); svc_error.report('hl_readdir:icon-stream', { diff --git a/src/backend/src/helpers.js b/src/backend/src/helpers.js index 48388b003..ab7ffb97d 100644 --- a/src/backend/src/helpers.js +++ b/src/backend/src/helpers.js @@ -29,6 +29,7 @@ const { Context } = require('./util/context'); const { NodeUIDSelector } = require('./filesystem/node/selectors'); const { redisClient } = require('./clients/redis/redisSingleton'); const { kv } = require('./util/kvSingleton'); +const { APP_ICONS_SUBDOMAIN } = require('./consts/app-icons.js'); const identifying_uuid = require('uuid').v4(); @@ -61,14 +62,10 @@ const buildAppIconUrl = (app_uid, size = DEFAULT_APP_ICON_SIZE) => { if ( ! app_uid ) return null; const uid_string = String(app_uid); const normalized_uid = uid_string.startsWith('app-') ? uid_string : `app-${uid_string}`; - const origin = config.origin ?? ( - config.protocol && config.domain - ? `${config.protocol }://${ config.domain }` - : 'https://puter.com' - ); - if ( ! origin ) return null; - const host = origin.replace(/\/$/, ''); - return `${host}/app-icon/${normalized_uid}/${size}`; + const icon_size = Number.isFinite(Number(size)) ? Number(size) : DEFAULT_APP_ICON_SIZE; + const static_hosting_domain = config.static_hosting_domain || config.static_hosting_domain_alt; + if ( ! static_hosting_domain ) return null; + return `https://${APP_ICONS_SUBDOMAIN}.${static_hosting_domain}/${normalized_uid}-${icon_size}.png`; }; const withAppIconUrl = (app) => { @@ -1921,13 +1918,13 @@ async function get_taskbar_items (user, { icon_size, no_icons } = {}) { delete item.icon; } else { const svc_appIcon = _servicesHolder.services.get('app-icon'); - const icon_result = await svc_appIcon.get_icon_stream({ - app_icon: item.icon, - app_uid: item.uid, + const iconResult = await svc_appIcon.getIconStream({ + appIcon: item.icon, + appUid: item.uid, size: icon_size, }); - item.icon = await icon_result.get_data_url(); + item.icon = await iconResult.get_data_url(); } // add to final object diff --git a/src/backend/src/modules/apps/AppIconService.js b/src/backend/src/modules/apps/AppIconService.js index 91273804a..c46e77982 100644 --- a/src/backend/src/modules/apps/AppIconService.js +++ b/src/backend/src/modules/apps/AppIconService.js @@ -17,19 +17,28 @@ * along with this program. If not, see . */ -const { HLWrite } = require('../../filesystem/hl_operations/hl_write'); -const { LLMkdir } = require('../../filesystem/ll_operations/ll_mkdir'); -const { LLRead } = require('../../filesystem/ll_operations/ll_read'); -const { NodePathSelector } = require('../../filesystem/node/selectors'); -const { get_app } = require('../../helpers'); -const { Endpoint } = require('../../util/expressutil'); -const { buffer_to_stream, stream_to_buffer } = require('../../util/streamutil'); -const BaseService = require('../../services/BaseService.js'); +import config from '../../config.js'; +import { createRequire } from 'node:module'; +import { HLWrite } from '../../filesystem/hl_operations/hl_write.js'; +import { LLMkdir } from '../../filesystem/ll_operations/ll_mkdir.js'; +import { LLRead } from '../../filesystem/ll_operations/ll_read.js'; +import { NodePathSelector } from '../../filesystem/node/selectors.js'; +import { APP_ICONS_SUBDOMAIN } from '../../consts/app-icons.js'; +import { get_app, get_user } from '../../helpers.js'; +import BaseService from '../../services/BaseService.js'; +import { DB_WRITE } from '../../services/database/consts.js'; +import { Endpoint } from '../../util/expressutil.js'; +import { buffer_to_stream, stream_to_buffer } from '../../util/streamutil.js'; +import DEFAULT_APP_ICON from './default-app-icon.js'; +import IconResult from './lib/IconResult.js'; + +const require = createRequire(import.meta.url); const ICON_SIZES = [16, 32, 64, 128, 256, 512]; - -const DEFAULT_APP_ICON = require('./default-app-icon.js'); -const IconResult = require('./lib/IconResult.js'); +const LEGACY_ICON_FILENAME = ({ appUid, size }) => `${appUid}-${size}.png`; +const ORIGINAL_ICON_FILENAME = ({ appUid }) => `${appUid}.png`; +const REDIRECT_MAX_AGE_SIZE = 30 * 24 * 60 * 60; // 1 month +const REDIRECT_MAX_AGE_ORIGINAL = 7 * 24 * 60 * 60; // 1 week /** * AppIconService handles icon generation and serving for apps. @@ -44,11 +53,12 @@ const IconResult = require('./lib/IconResult.js'); * UserService emits the `user.system-user-ready` event on the * service container event bus. */ -class AppIconService extends BaseService { +export class AppIconService extends BaseService { static MODULES = { sharp: require('sharp'), bmp: require('sharp-bmp'), ico: require('sharp-ico'), + uuidv4: require('uuid').v4, }; static ICON_SIZES = ICON_SIZES; @@ -64,19 +74,28 @@ class AppIconService extends BaseService { methods: ['GET'], handler: async (req, res) => { // Validate parameters - let { app_uid, size } = req.params; + let { app_uid: appUid, size } = req.params; if ( ! ICON_SIZES.includes(Number(size)) ) { res.status(400).send('Invalid size'); return; } - if ( ! app_uid.startsWith('app-') ) { - app_uid = `app-${app_uid}`; + if ( ! appUid.startsWith('app-') ) { + appUid = `app-${appUid}`; } const { stream, mime, - } = await this.get_icon_stream({ app_uid, size }); + redirectUrl, + redirectCacheControl, + } = await this.getIconStream({ appUid, size, allowRedirect: true }); + + if ( redirectUrl ) { + if ( redirectCacheControl ) { + res.set('Cache-Control', redirectCacheControl); + } + return res.redirect(302, redirectUrl); + } res.set('Content-Type', mime); res.set('Cache-Control', 'public, max-age=3600'); @@ -85,28 +104,28 @@ class AppIconService extends BaseService { }).attach(app); } - get_sizes () { + getSizes () { return this.constructor.ICON_SIZES; } - async iconify_apps ({ apps, size }) { + async iconifyApps ({ apps, size }) { return await Promise.all(apps.map(async app => { - const icon_result = await this.get_icon_stream({ - app_icon: app.icon, - app_uid: app.uid ?? app.uuid, - size: size, + const iconResult = await this.getIconStream({ + appIcon: app.icon, + appUid: app.uid ?? app.uuid, + size, }); - if ( icon_result.data_url ) { - app.icon = icon_result.data_url; + if ( iconResult.dataUrl ?? iconResult.data_url ) { + app.icon = iconResult.dataUrl ?? iconResult.data_url; return app; } try { - const buffer = await stream_to_buffer(icon_result.stream); - const resp_data_url = `data:${icon_result.mime};base64,${buffer.toString('base64')}`; + const buffer = await stream_to_buffer(iconResult.stream); + const respDataUrl = `data:${iconResult.mime};base64,${buffer.toString('base64')}`; - app.icon = resp_data_url; + app.icon = respDataUrl; } catch (e) { this.errors.report('get-launch-apps:icon-stream', { source: e, @@ -116,53 +135,339 @@ class AppIconService extends BaseService { })); } - async get_icon_stream (params) { - const result = await this.get_icon_stream_(params); + async getIconStream (params) { + const result = await this.#getIconStream(params); return new IconResult(result); } - async get_icon_stream_ ({ app_icon, app_uid, size, tries = 0 }) { - const is_data_url = value => ( + normalizeAppUid (appUid) { + if ( typeof appUid !== 'string' ) return appUid; + return appUid.startsWith('app-') ? appUid : `app-${appUid}`; + } + + isDataUrl (value) { + return ( typeof value === 'string' && value.startsWith('data:') && value.includes(',') ); + } - if ( app_icon && !is_data_url(app_icon) ) { - app_icon = null; + parseAppIconEndpointUrl (iconUrl) { + if ( typeof iconUrl !== 'string' || iconUrl.startsWith('data:') ) { + return null; + } + + let pathname; + try { + pathname = new URL(iconUrl, 'http://localhost').pathname; + } catch { + return null; + } + + const match = pathname.match(/^\/app-icon\/([^/]+)\/(\d+)\/?$/); + if ( ! match ) return null; + + return { + appUid: this.normalizeAppUid(match[1]), + size: Number(match[2]), + }; + } + + isAppIconEndpointUrl (iconUrl) { + return !!this.parseAppIconEndpointUrl(iconUrl); + } + + isSameAppIconEndpointUrl ({ iconUrl, appUid, size }) { + const parsed = this.parseAppIconEndpointUrl(iconUrl); + if ( ! parsed ) return false; + return ( + parsed.appUid === this.normalizeAppUid(appUid) && + Number(parsed.size) === Number(size) + ); + } + + extractPuterSubdomainFromUrl (url) { + if ( typeof url !== 'string' ) return null; + + let hostname; + try { + hostname = (new URL(url)).hostname.toLowerCase(); + } catch { + return null; + } + + const hostingDomains = [ + config.static_hosting_domain, + config.static_hosting_domain_alt, + ].filter(Boolean).map(v => v.toLowerCase()); + + for ( const domain of hostingDomains ) { + const suffix = `.${domain}`; + if ( hostname.endsWith(suffix) ) { + const subdomain = hostname.slice(0, hostname.length - suffix.length); + return subdomain || null; + } + } + + return null; + } + + isPuterSubdomainUrl (url) { + return !!this.extractPuterSubdomainFromUrl(url); + } + + getAppIconsBaseUrl () { + if ( this.appIconsBaseUrl !== undefined ) { + return this.appIconsBaseUrl; + } + + const host = config.static_hosting_domain || config.static_hosting_domain_alt; + if ( ! host ) { + this.appIconsBaseUrl = null; + return this.appIconsBaseUrl; + } + + this.appIconsBaseUrl = `https://${APP_ICONS_SUBDOMAIN}.${host}`; + return this.appIconsBaseUrl; + } + + getSizedIconUrl ({ appUid, size }) { + const baseUrl = this.getAppIconsBaseUrl(); + if ( ! baseUrl ) return null; + + const normalizedAppUid = this.normalizeAppUid(appUid); + return `${baseUrl}/${LEGACY_ICON_FILENAME({ + appUid: normalizedAppUid, + size, + })}`; + } + + getOriginalIconUrl ({ appUid }) { + const baseUrl = this.getAppIconsBaseUrl(); + if ( ! baseUrl ) return null; + + const normalizedAppUid = this.normalizeAppUid(appUid); + return `${baseUrl}/${ORIGINAL_ICON_FILENAME({ + appUid: normalizedAppUid, + })}`; + } + + async ensureAppIconsDirectory ({ dirSystem = null } = {}) { + const svcFs = this.services.get('filesystem'); + const svcSu = this.services.get('su'); + const svcUser = this.services.get('user'); + return await svcSu.sudo(async () => { + const dirAppIcons = await svcFs.node(new NodePathSelector('/system/app_icons')); + if ( await dirAppIcons.exists() ) { + this.dir_app_icons = dirAppIcons; + return dirAppIcons; + } + + dirSystem = dirSystem || await svcUser.get_system_dir(); + if ( ! dirSystem ) { + dirSystem = await svcFs.node(new NodePathSelector('/system')); + } + if ( ! await dirSystem.exists() ) { + return dirAppIcons; + } + + const llMkdir = new LLMkdir(); + await llMkdir.run({ + parent: dirSystem, + name: 'app_icons', + actor: await svcSu.get_system_actor(), + }); + + this.dir_app_icons = dirAppIcons; + return dirAppIcons; + }); + } + + async getOriginalIconLookup ({ dirAppIcons, appUid }) { + const normalizedAppUid = this.normalizeAppUid(appUid); + const originalFilename = ORIGINAL_ICON_FILENAME({ appUid: normalizedAppUid }); + const flatOriginalNode = await dirAppIcons.getChild(originalFilename); + if ( await flatOriginalNode.exists() ) { + return { + node: flatOriginalNode, + isFlatOriginal: true, + }; + } + return { + node: null, + isFlatOriginal: false, + }; + } + + async ensureAppIconsSubdomain ({ dirAppIcons }) { + const dbSites = this.services.get('database').get(DB_WRITE, 'sites'); + const existing = await dbSites.read('SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', + [APP_ICONS_SUBDOMAIN]); + if ( existing[0] ) return existing[0]; + + const systemUser = await get_user({ username: 'system' }); + if ( ! systemUser?.id ) return null; + + const rootDirId = await dirAppIcons.get('mysql-id'); + await dbSites.write(`INSERT ${dbSites.case({ + mysql: 'IGNORE', + sqlite: 'OR IGNORE', + })} INTO subdomains (subdomain, user_id, root_dir_id, uuid) VALUES (?, ?, ?, ?)`, [ + APP_ICONS_SUBDOMAIN, + systemUser.id, + rootDirId, + `sd-${this.modules.uuidv4()}`, + ]); + + const rows = await dbSites.read('SELECT * FROM subdomains WHERE subdomain = ? LIMIT 1', + [APP_ICONS_SUBDOMAIN]); + return rows[0] ?? null; + } + + async readIconNodeBuffer ({ node }) { + const svcSu = this.services.get('su'); + const llRead = new LLRead(); + const stream = await llRead.run({ + fsNode: node, + actor: await svcSu.get_system_actor(), + }); + return await stream_to_buffer(stream); + } + + async writePngToDir ({ destination_or_parent, filename, output }) { + const svcSu = this.services.get('su'); + const sysActor = await svcSu.get_system_actor(); + const hlWrite = new HLWrite(); + await hlWrite.run({ + destination_or_parent, + specified_name: filename, + overwrite: true, + actor: sysActor, + user: sysActor.type.user, + no_thumbnail: true, + file: { + size: output.length, + name: filename, + mimetype: 'image/png', + type: 'image/png', + stream: buffer_to_stream(output), + }, + }); + } + + shouldRedirectIconUrl ({ iconUrl, appUid, size }) { + if ( !iconUrl || this.isDataUrl(iconUrl) ) return false; + + const canRedirect = + this.isPuterSubdomainUrl(iconUrl) || + this.isAppIconEndpointUrl(iconUrl); + if ( ! canRedirect ) return false; + + return !this.isSameAppIconEndpointUrl({ + iconUrl, + appUid, + size, + }); + } + + async generateMissingSizeFromOriginal ({ appUid, size }) { + const normalizedAppUid = this.normalizeAppUid(appUid); + const dirAppIcons = await this.ensureAppIconsDirectory(); + if ( ! await dirAppIcons.exists() ) return; + const { node: originalNode } = await this.getOriginalIconLookup({ + dirAppIcons, + appUid: normalizedAppUid, + }); + if ( ! originalNode ) return; + + const sizedFilename = LEGACY_ICON_FILENAME({ + appUid: normalizedAppUid, + size, + }); + const sizedNode = await dirAppIcons.getChild(sizedFilename); + if ( await sizedNode.exists() ) return; + + const originalBuffer = await this.readIconNodeBuffer({ node: originalNode }); + const output = await this.modules.sharp(originalBuffer) + .resize(size) + .png() + .toBuffer(); + + await this.writePngToDir({ + destination_or_parent: dirAppIcons, + filename: sizedFilename, + output, + }); + } + + queueMissingSizeFromOriginal ({ appUid, size }) { + if ( ! this.pendingIconSizeJobs ) { + this.pendingIconSizeJobs = new Set(); + } + + const key = `${this.normalizeAppUid(appUid)}:${size}`; + if ( this.pendingIconSizeJobs.has(key) ) return; + + this.pendingIconSizeJobs.add(key); + Promise.resolve() + .then(async () => { + await this.generateMissingSizeFromOriginal({ appUid, size }); + }) + .catch(error => { + this.errors.report('AppIconService.queueMissingSizeFromOriginal', { + source: error, + appUid, + size, + }); + }) + .finally(() => { + this.pendingIconSizeJobs.delete(key); + }); + } + + async #getIconStream ({ appIcon, appUid, size, tries = 0, allowRedirect = false }) { + appUid = this.normalizeAppUid(appUid); + const appIconOriginal = appIcon; + + if ( appIcon && !this.isDataUrl(appIcon) ) { + appIcon = null; } // If there is an icon provided, and it's an SVG, we'll just return it - if ( app_icon ) { - const [metadata, data] = app_icon.split(','); - const input_mime = metadata.split(';')[0].split(':')[1]; + if ( appIcon ) { + const [metadata, data] = appIcon.split(','); + const inputMime = metadata.split(';')[0].split(':')[1]; // svg icons will be sent as-is - if ( input_mime === 'image/svg+xml' ) { + if ( inputMime === 'image/svg+xml' ) { return { mime: 'image/svg+xml', get stream () { return buffer_to_stream(Buffer.from(data, 'base64')); }, - data_url: app_icon, + dataUrl: appIcon, + data_url: appIcon, }; } } - // Get icon file node - const dir_app_icons = await this.get_app_icons(); - const node = await dir_app_icons.getChild(`${app_uid}-${size}.png`); + let app; + const getAppCached = async () => { + if ( app !== undefined ) return app; + app = await get_app({ uid: appUid }); + return app; + }; - const get_fallback_icon = async () => { - // Use database-stored icon as a fallback - app_icon = app_icon || await (async () => { - const app = await get_app({ uid: app_uid }); + const getFallbackIcon = async () => { + let fallbackIcon = appIcon || await (async () => { + const app = await getAppCached(); return app?.icon || DEFAULT_APP_ICON; })(); - if ( ! is_data_url(app_icon) ) { - app_icon = DEFAULT_APP_ICON; + if ( ! this.isDataUrl(fallbackIcon) ) { + fallbackIcon = DEFAULT_APP_ICON; } - const [metadata, base64] = app_icon.split(','); + const [metadata, base64] = fallbackIcon.split(','); const mime = metadata.split(';')[0].split(':')[1]; const img = Buffer.from(base64, 'base64'); return { @@ -171,59 +476,120 @@ class AppIconService extends BaseService { }; }; - if ( ! await node.exists() ) { - return await get_fallback_icon(); + const getExternalRedirect = async () => { + if ( ! allowRedirect ) return null; + + const appIconUrl = this.shouldRedirectIconUrl({ + iconUrl: appIconOriginal, + appUid, + size, + }) ? appIconOriginal : null; + + let dbIcon; + if ( ! appIconUrl ) { + dbIcon = (await getAppCached())?.icon; + } + + const redirectUrl = [appIconUrl, dbIcon].find(url => this.shouldRedirectIconUrl({ + iconUrl: url, + appUid, + size, + })); + + if ( ! redirectUrl ) return null; + return { redirectUrl }; + }; + + const dirAppIcons = await this.getAppIcons(); + const legacyFilename = LEGACY_ICON_FILENAME({ appUid, size }); + const legacyNode = await dirAppIcons.getChild(legacyFilename); + + if ( await legacyNode.exists() ) { + if ( allowRedirect ) { + const redirectUrl = this.getSizedIconUrl({ appUid, size }); + if ( redirectUrl ) { + return { + redirectUrl, + redirectCacheControl: `public, max-age=${REDIRECT_MAX_AGE_SIZE}`, + }; + } + } + + try { + const output = await this.readIconNodeBuffer({ node: legacyNode }); + return { + mime: 'image/png', + stream: buffer_to_stream(output), + }; + } catch (e) { + this.errors.report('AppIconService.get_icon_stream', { + source: e, + }); + if ( tries < 1 ) { + // Choose the next size up, or 256 if we're already at 512. + const secondSize = size < 512 ? size * 2 : 256; + return await this.#getIconStream({ + appUid, + appIcon: appIconOriginal, + size: secondSize, + tries: tries + 1, + allowRedirect, + }); + } + } } - try { - const svc_su = this.services.get('su'); - const ll_read = new LLRead(); - return { - mime: 'image/png', - stream: await ll_read.run({ - fsNode: node, - actor: await svc_su.get_system_actor(), - }), - }; - } catch (e) { - this.errors.report('AppIconService.get_icon_stream', { - source: e, - }); - if ( tries < 1 ) { - // We can choose the fallback icon in these two ways: + const { + node: originalNode, + isFlatOriginal, + } = await this.getOriginalIconLookup({ dirAppIcons, appUid }); + const hasOriginal = !!originalNode; - // Choose the next size up, or 256 if we're already at 512; - // this prioritizes icon quality over speed and bandwidth. - let second_size = size < 512 ? size * 2 : 256; + if ( hasOriginal ) { + this.queueMissingSizeFromOriginal({ appUid, size }); - // Choose the next size down, or 32 if we're already at 16; - // this prioritizes speed and bandwidth over icon quality. - // let second_size = size > 16 ? size / 2 : 32; + if ( allowRedirect && isFlatOriginal ) { + const redirectUrl = this.getOriginalIconUrl({ appUid }); + if ( redirectUrl ) { + return { + redirectUrl, + redirectCacheControl: `public, max-age=${REDIRECT_MAX_AGE_ORIGINAL}`, + }; + } + } - return await this.get_icon_stream({ - app_uid, size: second_size, tries: tries + 1, + try { + const output = await this.readIconNodeBuffer({ node: originalNode }); + return { + mime: 'image/png', + stream: buffer_to_stream(output), + }; + } catch (e) { + this.errors.report('AppIconService.get_icon_stream:original-read', { + source: e, }); } - return await get_fallback_icon(); } + + return await getExternalRedirect() || await getFallbackIcon(); } /** * Returns an FSNodeContext instance for the app icons * directory. */ - async get_app_icons () { + async getAppIcons () { if ( this.dir_app_icons ) { return this.dir_app_icons; } - const svc_fs = this.services.get('filesystem'); - const dir_app_icons = await svc_fs.node(new NodePathSelector('/system/app_icons')); + const svcFs = this.services.get('filesystem'); + const dirAppIcons = await svcFs.node(new NodePathSelector('/system/app_icons')); - return this.dir_app_icons = dir_app_icons; + return this.dir_app_icons = dirAppIcons; } - get_sharp ({ metadata, input }) { + getSharp ({ metadata, input }) { const type = metadata.split(';')[0].split(':')[1]; if ( type === 'image/bmp' ) { @@ -239,93 +605,113 @@ class AppIconService extends BaseService { return this.modules.sharp(input); } + async loadIconSource ({ iconUrl }) { + if ( typeof iconUrl !== 'string' || !iconUrl ) { + return null; + } + + if ( iconUrl.startsWith('data:') ) { + const [metadata, base64] = iconUrl.split(','); + return { + metadata, + input: Buffer.from(base64, 'base64'), + }; + } + + try { + const response = await fetch(iconUrl); + if ( ! response.ok ) { + throw new Error(`HTTP error! status: ${response.status}`); + } + + return { + input: Buffer.from(await response.arrayBuffer()), + metadata: `data:${response.headers.get('content-type') || 'image/png'};base64`, + }; + } catch ( error ) { + this.errors.report('AppIconService.createAppIcons:fetchUrl', { + source: error, + iconUrl, + }); + return null; + } + } + /** * AppIconService listens to this event to create the * `/system/app_icons` directory if it does not exist, * and then to register the event listener for `app.new-icon`. */ async ['__on_user.system-user-ready'] () { - const svc_su = this.services.get('su'); - const svc_fs = this.services.get('filesystem'); - const svc_user = this.services.get('user'); + const svcSu = this.services.get('su'); + const svcUser = this.services.get('user'); - const dir_system = await svc_user.get_system_dir(); + const dirSystem = await svcUser.get_system_dir(); // Ensure app icons directory exists - await svc_su.sudo(async () => { - const dir_app_icons = await svc_fs.node(new NodePathSelector('/system/app_icons')); - if ( ! await dir_app_icons.exists() ) { - const ll_mkdir = new LLMkdir(); - await ll_mkdir.run({ - parent: dir_system, - name: 'app_icons', - actor: await svc_su.get_system_actor(), - }); - } - this.dir_app_icons = dir_app_icons; + await svcSu.sudo(async () => { + const dirAppIcons = await this.ensureAppIconsDirectory({ dirSystem }); + await this.ensureAppIconsSubdomain({ dirAppIcons }); }); // Listen for new app icons - const svc_event = this.services.get('event'); - svc_event.on('app.new-icon', async (_, data) => { - await this.create_app_icons({ data }); + const svcEvent = this.services.get('event'); + svcEvent.on('app.new-icon', async (_, data) => { + await this.createAppIcons({ data }); }); } - async create_app_icons ({ data }) { - const svc_su = this.services.get('su'); - const dir_app_icons = await this.get_app_icons(); + async createAppIcons ({ data }) { + const svcSu = this.services.get('su'); + const dataUrl = data.dataUrl ?? data.data_url; + const appUid = this.normalizeAppUid(data.appUid ?? data.app_uid); + if ( !dataUrl || !appUid ) return; - // Writing icons as the system user - const icon_jobs = []; - for ( const size of ICON_SIZES ) { - icon_jobs.push((async () => { - await svc_su.sudo(async () => { - const filename = `${data.app_uid}-${size}.png`; - const data_url = data.data_url; - const [metadata, base64] = data_url.split(','); - const input = Buffer.from(base64, 'base64'); + const source = await this.loadIconSource({ iconUrl: dataUrl }); + if ( ! source ) return; - const sharp_instance = this.get_sharp({ - metadata, - input, - }); + const { input, metadata } = source; + const isInputDataUrl = this.isDataUrl(dataUrl); - // NOTE: A stream would be more ideal than a buffer here - // but we have no way of knowing the output size - // before we finish processing the image. - const output = await sharp_instance - .resize(size) - .png() - .toBuffer(); + await svcSu.sudo(async () => { + const dirAppIcons = await this.ensureAppIconsDirectory(); + if ( ! await dirAppIcons.exists() ) { + throw new Error('app icons directory is missing'); + } - const sys_actor = await svc_su.get_system_actor(); - const hl_write = new HLWrite(); - await hl_write.run({ - destination_or_parent: dir_app_icons, - specified_name: filename, - overwrite: true, - actor: sys_actor, - user: sys_actor.type.user, - no_thumbnail: true, - file: { - size: output.length, - name: filename, - mimetype: 'image/png', - type: 'image/png', - stream: buffer_to_stream(output), - }, - }); + const sharpInstance = this.getSharp({ metadata, input }); + + if ( isInputDataUrl ) { + const originalOutput = await sharpInstance.clone() + .png() + .toBuffer(); + await this.writePngToDir({ + destination_or_parent: dirAppIcons, + filename: ORIGINAL_ICON_FILENAME({ appUid }), + output: originalOutput, }); - })()); - } - await Promise.all(icon_jobs); + + const originalUrl = this.getOriginalIconUrl({ appUid }); + if ( originalUrl ) { + data.url = originalUrl; + } + } + + const iconJobs = ICON_SIZES.map(async size => { + const output = await sharpInstance.clone() + .resize(size) + .png() + .toBuffer(); + await this.writePngToDir({ + destination_or_parent: dirAppIcons, + filename: LEGACY_ICON_FILENAME({ appUid, size }), + output, + }); + }); + await Promise.all(iconJobs); + }); } async _init () { } } - -module.exports = { - AppIconService, -}; diff --git a/src/backend/src/modules/apps/AppIconService.test.js b/src/backend/src/modules/apps/AppIconService.test.js new file mode 100644 index 000000000..f9e1eb059 --- /dev/null +++ b/src/backend/src/modules/apps/AppIconService.test.js @@ -0,0 +1,179 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import config from '../../config.js'; +import { AppIconService } from './AppIconService.js'; + +describe('AppIconService', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + }); + + describe('URL helpers', () => { + it('extracts a puter subdomain from a static hosting URL', () => { + const service = Object.create(AppIconService.prototype); + const domain = config.static_hosting_domain; + + const result = service.extractPuterSubdomainFromUrl(`https://dev-center-app-id.${domain}/icon.png`); + + expect(result).toBe('dev-center-app-id'); + }); + + it('does not redirect when URL is the same app-icon endpoint request', () => { + const service = Object.create(AppIconService.prototype); + + const shouldRedirect = service.shouldRedirectIconUrl({ + iconUrl: 'https://api.puter.localhost/app-icon/app-123/64', + appUid: 'app-123', + size: 64, + }); + + expect(shouldRedirect).toBe(false); + }); + }); + + describe('createAppIcons', () => { + it('stores original and resized icons in /system/app_icons for data URLs', async () => { + const sudo = vi.fn(async callback => await callback()); + const dirAppIcons = { + exists: vi.fn().mockResolvedValue(true), + }; + + const service = Object.create(AppIconService.prototype); + service.services = { + get: vi.fn(name => (name === 'su' ? { sudo } : null)), + }; + service.errors = { report: vi.fn() }; + service.ensureAppIconsDirectory = vi.fn().mockResolvedValue(dirAppIcons); + service.getOriginalIconUrl = vi.fn().mockReturnValue('https://puter-app-icons.site.puter.localhost/app-abc.png'); + service.loadIconSource = vi.fn().mockResolvedValue({ + metadata: 'data:image/png;base64', + input: Buffer.from([1, 2, 3]), + }); + service.writePngToDir = vi.fn().mockResolvedValue(undefined); + service.getSharp = vi.fn(() => ({ + clone: vi.fn(() => ({ + resize: vi.fn().mockReturnThis(), + png: vi.fn().mockReturnThis(), + toBuffer: vi.fn().mockResolvedValue(Buffer.from([0x89, 0x50, 0x4e, 0x47])), + })), + })); + + const data = { + appUid: 'app-abc', + dataUrl: 'data:image/png;base64,AA==', + }; + + await service.createAppIcons({ data }); + + expect(service.writePngToDir).toHaveBeenCalledTimes(AppIconService.ICON_SIZES.length + 1); + expect(service.writePngToDir).toHaveBeenCalledWith(expect.objectContaining({ + destination_or_parent: dirAppIcons, + filename: 'app-abc.png', + })); + expect(service.writePngToDir).toHaveBeenCalledWith(expect.objectContaining({ + destination_or_parent: dirAppIcons, + filename: 'app-abc-64.png', + })); + expect(data.url).toBe('https://puter-app-icons.site.puter.localhost/app-abc.png'); + }); + }); + + describe('getIconStream', () => { + const createServiceInstance = () => new AppIconService({ + services: { get: vi.fn() }, + config: {}, + name: 'app-icon', + args: {}, + }); + + it('redirects to puter-app-icons subsite sized file when requested size exists', async () => { + const legacyNode = { + exists: vi.fn().mockResolvedValue(true), + }; + const legacyRoot = { + getChild: vi.fn().mockResolvedValue(legacyNode), + }; + + const service = createServiceInstance(); + service.errors = { report: vi.fn() }; + service.getAppIcons = vi.fn().mockResolvedValue(legacyRoot); + service.getSizedIconUrl = vi.fn().mockReturnValue('https://puter-app-icons.site.puter.localhost/app-abc-64.png'); + + const result = await service.getIconStream({ + appUid: 'app-abc', + size: 64, + allowRedirect: true, + }); + + expect(result.redirectUrl).toBe('https://puter-app-icons.site.puter.localhost/app-abc-64.png'); + expect(result.redirectCacheControl).toContain('max-age=2592000'); + }); + + it('redirects to original and queues resize when requested size is missing', async () => { + const legacyNode = { + exists: vi.fn().mockResolvedValue(false), + }; + const legacyRoot = { + getChild: vi.fn().mockResolvedValue(legacyNode), + }; + const originalNode = { + exists: vi.fn().mockResolvedValue(true), + }; + + const service = createServiceInstance(); + service.errors = { report: vi.fn() }; + service.getAppIcons = vi.fn().mockResolvedValue(legacyRoot); + service.getOriginalIconLookup = vi.fn().mockResolvedValue({ + node: originalNode, + isFlatOriginal: true, + }); + service.getOriginalIconUrl = vi.fn().mockReturnValue('https://puter-app-icons.site.puter.localhost/app-abc.png'); + service.queueMissingSizeFromOriginal = vi.fn(); + + const result = await service.getIconStream({ + appUid: 'app-abc', + size: 128, + allowRedirect: true, + }); + + expect(result.redirectUrl).toBe('https://puter-app-icons.site.puter.localhost/app-abc.png'); + expect(result.redirectCacheControl).toContain('max-age=604800'); + expect(service.queueMissingSizeFromOriginal).toHaveBeenCalledWith({ + appUid: 'app-abc', + size: 128, + }); + }); + + it('redirects to app icon URL when no cached icon exists and URL is eligible', async () => { + const redirectUrl = `https://dev-center-app-id.${config.static_hosting_domain}/raw-icon.png`; + + const legacyNode = { + exists: vi.fn().mockResolvedValue(false), + }; + const legacyRoot = { + getChild: vi.fn().mockResolvedValue(legacyNode), + }; + + const service = createServiceInstance(); + service.errors = { report: vi.fn() }; + service.getAppIcons = vi.fn().mockResolvedValue(legacyRoot); + service.getOriginalIconLookup = vi.fn().mockResolvedValue({ + node: null, + isFlatOriginal: false, + }); + + const result = await service.getIconStream({ + appUid: 'app-abc', + appIcon: redirectUrl, + size: 256, + allowRedirect: true, + }); + + expect(result.redirectUrl).toBe(redirectUrl); + }); + }); +}); diff --git a/src/backend/src/modules/apps/RecommendedAppsService.js b/src/backend/src/modules/apps/RecommendedAppsService.js index ad8c967e2..d87ef79e5 100644 --- a/src/backend/src/modules/apps/RecommendedAppsService.js +++ b/src/backend/src/modules/apps/RecommendedAppsService.js @@ -73,7 +73,7 @@ export default class RecommendedAppsService extends BaseService { const svc_appIcon = this.services.get('app-icon'); const svc_event = this.services.get('event'); svc_event.on('apps.invalidate', async (_, { app }) => { - const sizes = svc_appIcon.get_sizes(); + const sizes = svc_appIcon.getSizes(); // If it's a single-app invalidation, only invalidate if the // app is in the list of recommended apps @@ -122,7 +122,7 @@ export default class RecommendedAppsService extends BaseService { // Iconify apps if ( icon_size ) { - recommended = await svc_appIcon.iconify_apps({ + recommended = await svc_appIcon.iconifyApps({ apps: recommended, size: icon_size, }); diff --git a/src/backend/src/modules/data-access/AppService.js b/src/backend/src/modules/data-access/AppService.js index ec899a891..ae21a9d25 100644 --- a/src/backend/src/modules/data-access/AppService.js +++ b/src/backend/src/modules/data-access/AppService.js @@ -178,13 +178,13 @@ export default class AppService extends BaseService { const icon_size = params.icon_size; const svc_appIcon = this.context.get('services').get('app-icon'); try { - const icon_result = await svc_appIcon.get_icon_stream({ - app_uid: row.uid, - app_icon: row.icon, + const iconResult = await svc_appIcon.getIconStream({ + appUid: row.uid, + appIcon: row.icon, size: icon_size, }); console.log('this is working it looks like'); - app.icon = await icon_result.get_data_url(); + app.icon = await iconResult.get_data_url(); } catch (e) { const svc_error = this.context.get('services').get('error-service'); svc_error.report('AppES:read_transform', { source: e }); @@ -306,12 +306,12 @@ export default class AppService extends BaseService { const icon_size = params.icon_size; const svc_appIcon = this.context.get('services').get('app-icon'); try { - const icon_result = await svc_appIcon.get_icon_stream({ - app_uid: row.uid, - app_icon: row.icon, + const iconResult = await svc_appIcon.getIconStream({ + appUid: row.uid, + appIcon: row.icon, size: icon_size, }); - app.icon = await icon_result.get_data_url(); + app.icon = await iconResult.get_data_url(); } catch (e) { const svc_error = this.context.get('services').get('error-service'); svc_error.report('AppES:read_transform', { source: e }); @@ -383,7 +383,11 @@ export default class AppService extends BaseService { } if ( object.icon !== undefined && object.icon !== null ) { - validate_image_base64(object.icon, { key: 'icon' }); + if ( typeof object.icon === 'string' && object.icon.startsWith('data:') ) { + validate_image_base64(object.icon, { key: 'icon' }); + } else { + validate_url(object.icon, { key: 'icon', maxlen: 3000 }); + } } validate_url(object.index_url, { @@ -453,6 +457,10 @@ export default class AppService extends BaseService { data_url: object.icon, }; await svc_event.emit('app.new-icon', event); + if ( event.url ) { + await this.db_write.write('UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1', + [event.url, uid]); + } } // Return the created app @@ -621,7 +629,11 @@ export default class AppService extends BaseService { } if ( object.icon !== undefined && object.icon !== null ) { - validate_image_base64(object.icon, { key: 'icon' }); + if ( typeof object.icon === 'string' && object.icon.startsWith('data:') ) { + validate_image_base64(object.icon, { key: 'icon' }); + } else { + validate_url(object.icon, { key: 'icon', maxlen: 3000 }); + } } if ( object.index_url !== undefined ) { @@ -691,12 +703,10 @@ export default class AppService extends BaseService { // Check if user has system-wide write permission { - /* eslint-disable */ // We need to fix eslint rule for multi-line calls - const has_permission_to_write_all = await svc_permission.check( - actor, - this.constructor.WRITE_ALL_OWNER_PERMISSION, - ); - /* eslint-enable */ + // We need to fix eslint rule for multi-line calls + const has_permission_to_write_all = await svc_permission.check(actor, + this.constructor.WRITE_ALL_OWNER_PERMISSION); + if ( has_permission_to_write_all ) { return; } @@ -849,6 +859,10 @@ export default class AppService extends BaseService { data_url: object.icon, }; await svc_event.emit('app.new-icon', event); + if ( event.url ) { + await this.db_write.write('UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1', + [event.url, old_app.uid]); + } } // Emit name change event diff --git a/src/backend/src/modules/data-access/AppService.test.js b/src/backend/src/modules/data-access/AppService.test.js index c59a8786e..6915d1089 100644 --- a/src/backend/src/modules/data-access/AppService.test.js +++ b/src/backend/src/modules/data-access/AppService.test.js @@ -316,7 +316,7 @@ describe('AppService', () => { mockDb.read.mockResolvedValue([mockRow]); const mockIconService = { - get_icon_stream: vi.fn().mockResolvedValue({ + getIconStream: vi.fn().mockResolvedValue({ get_data_url: vi.fn().mockResolvedValue('data:image/png;base64,abc123'), }), }; @@ -341,9 +341,9 @@ describe('AppService', () => { params: { icon_size: 64 }, }); - expect(mockIconService.get_icon_stream).toHaveBeenCalledWith({ - app_uid: 'app-uid-123', - app_icon: 'icon.png', + expect(mockIconService.getIconStream).toHaveBeenCalledWith({ + appUid: 'app-uid-123', + appIcon: 'icon.png', size: 64, }); expect(result.icon).toBe('data:image/png;base64,abc123'); @@ -358,7 +358,7 @@ describe('AppService', () => { }; const mockIconService = { - get_icon_stream: vi.fn().mockRejectedValue(new Error('Icon fetch failed')), + getIconStream: vi.fn().mockRejectedValue(new Error('Icon fetch failed')), }; appService.context = { @@ -480,8 +480,8 @@ describe('AppService', () => { mockDb.read.mockResolvedValue(mockRows); const mockIconService = { - get_icon_stream: vi.fn().mockImplementation(({ app_uid }) => ({ - get_data_url: vi.fn().mockResolvedValue(`data:image/png;base64,${app_uid}`), + getIconStream: vi.fn().mockImplementation(({ appUid }) => ({ + get_data_url: vi.fn().mockResolvedValue(`data:image/png;base64,${appUid}`), })), }; @@ -504,7 +504,7 @@ describe('AppService', () => { params: { icon_size: 32 }, }); - expect(mockIconService.get_icon_stream).toHaveBeenCalledTimes(2); + expect(mockIconService.getIconStream).toHaveBeenCalledTimes(2); expect(result[0].icon).toBe('data:image/png;base64,app-1'); expect(result[1].icon).toBe('data:image/png;base64,app-2'); }); diff --git a/src/backend/src/om/entitystorage/AppES.js b/src/backend/src/om/entitystorage/AppES.js index 96598ceac..c86d5a92a 100644 --- a/src/backend/src/om/entitystorage/AppES.js +++ b/src/backend/src/om/entitystorage/AppES.js @@ -178,7 +178,9 @@ class AppES extends BaseES { }; await svc_event.emit('app.new-icon', event); if ( event.url ) { - await entity.set('icon'); + await this.db.write('UPDATE apps SET icon = ? WHERE id = ? LIMIT 1', + [event.url, insert_id]); + await entity.set('icon', event.url); } } @@ -238,6 +240,46 @@ class AppES extends BaseES { return await recurse(predicate); }, + async queueIconMigration (entity) { + if ( ! this.pending_icon_migrations_ ) { + this.pending_icon_migrations_ = new Set(); + } + + const migration_key = entity.private_meta?.mysql_id ?? Symbol('app-icon-migration'); + if ( this.pending_icon_migrations_.has(migration_key) ) { + return; + } + this.pending_icon_migrations_.add(migration_key); + + Promise.resolve().then(async () => { + const icon = await entity.get('icon'); + if ( typeof icon !== 'string' || !icon.startsWith('data:') ) { + return; + } + + const app_uid = await entity.get('uid'); + if ( ! app_uid ) { + return; + } + + const svc_event = this.context.get('services').get('event'); + const event = { + app_uid, + data_url: icon, + }; + await svc_event.emit('app.new-icon', event); + if ( ! event.url ) return; + + await this.db.write('UPDATE apps SET icon = ? WHERE uid = ? LIMIT 1', + [event.url, app_uid]); + }).catch(e => { + const svc_error = this.context.get('services').get('error-service'); + svc_error.report('AppES:queue_icon_migration', { source: e }); + }).finally(() => { + this.pending_icon_migrations_.delete(migration_key); + }); + }, + /** * Transforms app data before reading by adding associations and handling permissions * @param {Object} entity - App entity to transform @@ -252,6 +294,9 @@ class AppES extends BaseES { const stats = await svc_appInformation.get_stats(await entity.get('uid'), { period: Context.get('es_params')?.stats_period, grouping: Context.get('es_params')?.stats_grouping, created_at: await entity.get('created_at') }); entity.set('stats', stats); + // Migrate b64 icons to the filesystem-backed icon flow without blocking reads. + this.queueIconMigration(entity); + entity.set('created_from_origin', await (async () => { const svc_auth = this.context.get('services').get('auth'); try { @@ -291,12 +336,12 @@ class AppES extends BaseES { if ( icon_size ) { const svc_appIcon = this.context.get('services').get('app-icon'); try { - const icon_result = await svc_appIcon.get_icon_stream({ - app_uid: await entity.get('uid'), - app_icon: await entity.get('icon'), + const iconResult = await svc_appIcon.getIconStream({ + appUid: await entity.get('uid'), + appIcon: await entity.get('icon'), size: icon_size, }); - await entity.set('icon', await icon_result.get_data_url()); + await entity.set('icon', await iconResult.get_data_url()); } catch (e) { const svc_error = this.context.get('services').get('error-service'); svc_error.report('AppES:read_transform', { source: e }); diff --git a/src/backend/src/routers/get-launch-apps.js b/src/backend/src/routers/get-launch-apps.js index abf769ff2..2c12bba29 100644 --- a/src/backend/src/routers/get-launch-apps.js +++ b/src/backend/src/routers/get-launch-apps.js @@ -24,13 +24,13 @@ import { DB_READ } from '../services/database/consts.js'; const iconify_apps = async (context, { apps, size }) => { return await Promise.all(apps.map(async app => { const svc_appIcon = context.services.get('app-icon'); - const icon_result = await svc_appIcon.get_icon_stream({ - app_icon: app.icon, - app_uid: app.uid ?? app.uuid, - size: size, + const iconResult = await svc_appIcon.getIconStream({ + appIcon: app.icon, + appUid: app.uid ?? app.uuid, + size, }); - app.icon = await icon_result.get_data_url(); + app.icon = await iconResult.get_data_url(); return app; })); };