diff --git a/src/backend/src/filesystem/FSNodeContext.js b/src/backend/src/filesystem/FSNodeContext.js index b47aeebc3..d6914b65e 100644 --- a/src/backend/src/filesystem/FSNodeContext.js +++ b/src/backend/src/filesystem/FSNodeContext.js @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const { get_user, id2path, id2uuid, is_empty, suggest_app_for_fsentry, get_app } = require('../helpers'); +const { get_user, id2path, id2uuid, is_empty, suggestedAppForFsEntry, get_app } = require('../helpers'); const putility = require('@heyputer/putility'); const config = require('../config'); @@ -548,6 +548,7 @@ module.exports = class FSNodeContext { return this.entry.size; } + /** Avoid using if fetching directory items */ async fetchSuggestedApps (user, force) { if ( this.entry.suggested_apps && !force ) return; @@ -555,7 +556,7 @@ module.exports = class FSNodeContext { if ( ! this.entry ) return; this.entry.suggested_apps = - await suggest_app_for_fsentry(this.entry, { user }); + await suggestedAppForFsEntry(this.entry, { user }); } async fetchIsEmpty () { diff --git a/src/backend/src/filesystem/hl_operations/hl_readdir.js b/src/backend/src/filesystem/hl_operations/hl_readdir.js index 7e8866fff..020bb484c 100644 --- a/src/backend/src/filesystem/hl_operations/hl_readdir.js +++ b/src/backend/src/filesystem/hl_operations/hl_readdir.js @@ -19,13 +19,15 @@ const APIError = require('../../api/APIError'); const { Context } = require('../../util/context'); const { stream_to_buffer } = require('../../util/streamutil'); -const { get_apps } = require('../../helpers'); +const { get_apps, suggestedAppsForFsEntries } = require('../../helpers'); const { ECMAP } = require('../ECMAP'); const { TYPE_DIRECTORY, TYPE_SYMLINK } = require('../FSNodeContext'); const { LLListUsers } = require('../ll_operations/ll_listusers'); const { LLReadDir } = require('../ll_operations/ll_readdir'); const { LLReadShares } = require('../ll_operations/ll_readshares'); const { HLFilesystemOperation } = require('./definitions'); +const { DB_READ } = require('../../services/database/consts'); +const config = require('../../config'); class HLReadDir extends HLFilesystemOperation { static CONCERN = 'filesystem'; @@ -117,13 +119,14 @@ class HLReadDir extends HLFilesystemOperation { } } + if ( ! no_assocs ) { + await Promise.all([ + this.#batchFetchSuggestedApps(children, user), + this.#batchFetchSubdomains(children, user), + ]); + } + return Promise.all(children.map(async child => { - if ( ! no_assocs ) { - await Promise.all([ - child.fetchSuggestedApps(user), - child.fetchSubdomains(user), - ]); - } const entry = await child.getSafeEntry(); if ( !no_thumbs && entry.associated_app ) { const svc_appIcon = this.context.get('services').get('app-icon'); @@ -151,6 +154,59 @@ class HLReadDir extends HLFilesystemOperation { return entry; })); } + + async #batchFetchSubdomains (children, user) { + const dirChildren = []; + const childById = new Map(); + + for ( const child of children ) { + const entry = child.entry; + if ( ! entry?.is_dir ) continue; + entry.subdomains = []; + if ( entry.id == null ) continue; + dirChildren.push(entry.id); + childById.set(entry.id, child); + } + + if ( dirChildren.length === 0 ) return; + + const placeholders = dirChildren.map(() => '?').join(','); + const db = this.context.get('services').get('database').get(DB_READ, 'filesystem'); + const rows = await db.read(`SELECT root_dir_id, subdomain, uuid + FROM subdomains + WHERE root_dir_id IN (${placeholders}) AND user_id = ?`, + [...dirChildren, user.id]); + + for ( const row of rows ) { + const child = childById.get(row.root_dir_id); + if ( ! child ) continue; + child.entry.subdomains.push({ + subdomain: row.subdomain, + address: `${config.protocol }://${ row.subdomain }.puter.site`, + uuid: row.uuid, + }); + child.entry.has_website = true; + } + } + + async #batchFetchSuggestedApps (children, user) { + const entries = []; + const targets = []; + + for ( const child of children ) { + const entry = child.entry; + if ( !entry || entry.suggested_apps ) continue; + entries.push(entry); + targets.push(entry); + } + + if ( entries.length === 0 ) return; + + const suggestedLists = await suggestedAppsForFsEntries(entries, { user }); + for ( let index = 0; index < targets.length; index++ ) { + targets[index].suggested_apps = suggestedLists[index] ?? []; + } + } } module.exports = { diff --git a/src/backend/src/helpers.js b/src/backend/src/helpers.js index 0678c7900..38a26c5ca 100644 --- a/src/backend/src/helpers.js +++ b/src/backend/src/helpers.js @@ -359,198 +359,207 @@ async function get_app (options) { * @param {Object} [options] * @returns {Promise>} */ -async function get_apps (specifiers, options = {}) { - if ( ! Array.isArray(specifiers) ) { - specifiers = [specifiers]; - } +const get_apps = spanify('get_apps', async (specifiers, options = {}) => { + const start = Date.now(); + console.log('Entering Get apps at: ', start); + try { + if ( ! Array.isArray(specifiers) ) { + specifiers = [specifiers]; + } - const cacheApp = (app) => { - if ( ! app ) return; - app = { ...app }; - kv.set(`apps:uid:${app.uid}`, app, { EX: 30 }); - kv.set(`apps:name:${app.name}`, app, { EX: 30 }); - kv.set(`apps:id:${app.id}`, app, { EX: 30 }); - }; + const cacheApp = (app) => { + if ( ! app ) return; + app = { ...app }; + kv.set(`apps:uid:${app.uid}`, app, { EX: 30 }); + kv.set(`apps:name:${app.name}`, app, { EX: 30 }); + kv.set(`apps:id:${app.id}`, app, { EX: 30 }); + }; - const normalized = specifiers.map(spec => spec ? { ...spec } : {}); + const normalized = specifiers.map(spec => spec ? { ...spec } : {}); + + if ( options.follow_old_names ) { + const svc_oldAppName = _servicesHolder.services.get('old-app-name'); + for ( const spec of normalized ) { + if ( spec.uid || !spec.name ) continue; + const old_name = await svc_oldAppName.check_app_name(spec.name); + if ( old_name ) { + spec.uid = old_name.app_uid; + delete spec.name; + } + } + } + + const appByUid = new Map(); + const appByName = new Map(); + const appById = new Map(); + + const addApp = (app) => { + if ( ! app ) return; + appByUid.set(app.uid, app); + appByName.set(app.name, app); + appById.set(app.id, app); + }; + + const pendingLookups = new Map(); + const pendingToResolve = new Map(); + const queryUids = new Set(); + const queryNames = new Set(); + const queryIds = new Set(); + + const queueMissing = (type, value) => { + const queryKey = `${type}:${value}`; + if ( pendingToResolve.has(queryKey) || pendingLookups.has(queryKey) ) { + return; + } + + const pendingKey = `pending_app:${queryKey}`; + const pending = kv.get(pendingKey); + if ( pending ) { + pendingLookups.set(queryKey, pending); + return; + } + + let resolveQuery; + let rejectQuery; + const queryPromise = new Promise((resolve, reject) => { + resolveQuery = resolve; + rejectQuery = reject; + }); + kv.set(pendingKey, queryPromise, { EX: PENDING_QUERY_TTL }); + pendingToResolve.set(queryKey, { resolveQuery, rejectQuery, pendingKey }); + + if ( type === 'uid' ) { + queryUids.add(value); + } else if ( type === 'name' ) { + queryNames.add(value); + } else if ( type === 'id' ) { + queryIds.add(value); + } + }; - if ( options.follow_old_names ) { - const svc_oldAppName = _servicesHolder.services.get('old-app-name'); for ( const spec of normalized ) { - if ( spec.uid || !spec.name ) continue; - const old_name = await svc_oldAppName.check_app_name(spec.name); - if ( old_name ) { - spec.uid = old_name.app_uid; - delete spec.name; + if ( spec.uid ) { + const cached = kv.get(`apps:uid:${spec.uid}`); + if ( cached ) { + addApp(cached); + } else { + queueMissing('uid', spec.uid); + } + continue; + } + if ( spec.name ) { + const cached = kv.get(`apps:name:${spec.name}`); + if ( cached ) { + addApp(cached); + } else { + queueMissing('name', spec.name); + } + continue; + } + if ( spec.id ) { + const cached = kv.get(`apps:id:${spec.id}`); + if ( cached ) { + addApp(cached); + } else { + queueMissing('id', spec.id); + } } } - } - const appByUid = new Map(); - const appByName = new Map(); - const appById = new Map(); + const pendingResultsPromise = pendingLookups.size + ? Promise.all(Array.from(pendingLookups.values())) + : Promise.resolve([]); - const addApp = (app) => { - if ( ! app ) return; - appByUid.set(app.uid, app); - appByName.set(app.name, app); - appById.set(app.id, app); - }; - - const pendingLookups = new Map(); - const pendingToResolve = new Map(); - const queryUids = new Set(); - const queryNames = new Set(); - const queryIds = new Set(); - - const queueMissing = (type, value) => { - const queryKey = `${type}:${value}`; - if ( pendingToResolve.has(queryKey) || pendingLookups.has(queryKey) ) { - return; - } - - const pendingKey = `pending_app:${queryKey}`; - const pending = kv.get(pendingKey); - if ( pending ) { - pendingLookups.set(queryKey, pending); - return; - } - - let resolveQuery; - let rejectQuery; - const queryPromise = new Promise((resolve, reject) => { - resolveQuery = resolve; - rejectQuery = reject; - }); - kv.set(pendingKey, queryPromise, { EX: PENDING_QUERY_TTL }); - pendingToResolve.set(queryKey, { resolveQuery, rejectQuery, pendingKey }); - - if ( type === 'uid' ) { - queryUids.add(value); - } else if ( type === 'name' ) { - queryNames.add(value); - } else if ( type === 'id' ) { - queryIds.add(value); - } - }; - - for ( const spec of normalized ) { - if ( spec.uid ) { - const cached = kv.get(`apps:uid:${spec.uid}`); - if ( cached ) { - addApp(cached); - } else { - queueMissing('uid', spec.uid); - } - continue; - } - if ( spec.name ) { - const cached = kv.get(`apps:name:${spec.name}`); - if ( cached ) { - addApp(cached); - } else { - queueMissing('name', spec.name); - } - continue; - } - if ( spec.id ) { - const cached = kv.get(`apps:id:${spec.id}`); - if ( cached ) { - addApp(cached); - } else { - queueMissing('id', spec.id); - } - } - } - - const pendingResultsPromise = pendingLookups.size - ? Promise.all(Array.from(pendingLookups.values())) - : Promise.resolve([]); - - if ( queryUids.size || queryNames.size || queryIds.size ) { + if ( queryUids.size || queryNames.size || queryIds.size ) { /** @type BaseDatabaseAccessService */ - const db = _servicesHolder.services.get('database').get(DB_READ, 'apps'); + const db = _servicesHolder.services.get('database').get(DB_READ, 'apps'); - const clauses = []; - const params = []; + const clauses = []; + const params = []; - if ( queryUids.size ) { - const uids = Array.from(queryUids); - clauses.push(`uid IN (${uids.map(() => '?').join(', ')})`); - params.push(...uids); - } - if ( queryNames.size ) { - const names = Array.from(queryNames); - clauses.push(`name IN (${names.map(() => '?').join(', ')})`); - params.push(...names); - } - if ( queryIds.size ) { - const ids = Array.from(queryIds); - clauses.push(`id IN (${ids.map(() => '?').join(', ')})`); - params.push(...ids); - } + if ( queryUids.size ) { + const uids = Array.from(queryUids); + clauses.push(`uid IN (${uids.map(() => '?').join(', ')})`); + params.push(...uids); + } + if ( queryNames.size ) { + const names = Array.from(queryNames); + clauses.push(`name IN (${names.map(() => '?').join(', ')})`); + params.push(...names); + } + if ( queryIds.size ) { + const ids = Array.from(queryIds); + clauses.push(`id IN (${ids.map(() => '?').join(', ')})`); + params.push(...ids); + } - let rows = []; - const resolvedKeys = new Set(); - try { - rows = await db.read(`SELECT * FROM \`apps\` WHERE ${clauses.join(' OR ')}`, - params); - for ( const app of rows ) { - cacheApp(app); - addApp(app); + let rows = []; + const resolvedKeys = new Set(); + try { + rows = await db.read(`SELECT * FROM \`apps\` WHERE ${clauses.join(' OR ')}`, + params); + for ( const app of rows ) { + cacheApp(app); + addApp(app); - const uidKey = `uid:${app.uid}`; - const nameKey = `name:${app.name}`; - const idKey = `id:${app.id}`; + const uidKey = `uid:${app.uid}`; + const nameKey = `name:${app.name}`; + const idKey = `id:${app.id}`; - if ( pendingToResolve.has(uidKey) ) { - pendingToResolve.get(uidKey).resolveQuery(app); - resolvedKeys.add(uidKey); + if ( pendingToResolve.has(uidKey) ) { + pendingToResolve.get(uidKey).resolveQuery(app); + resolvedKeys.add(uidKey); + } + if ( pendingToResolve.has(nameKey) ) { + pendingToResolve.get(nameKey).resolveQuery(app); + resolvedKeys.add(nameKey); + } + if ( pendingToResolve.has(idKey) ) { + pendingToResolve.get(idKey).resolveQuery(app); + resolvedKeys.add(idKey); + } } - if ( pendingToResolve.has(nameKey) ) { - pendingToResolve.get(nameKey).resolveQuery(app); - resolvedKeys.add(nameKey); + + for ( const [key, { resolveQuery }] of pendingToResolve.entries() ) { + if ( ! resolvedKeys.has(key) ) { + resolveQuery(null); + } } - if ( pendingToResolve.has(idKey) ) { - pendingToResolve.get(idKey).resolveQuery(app); - resolvedKeys.add(idKey); + } catch ( err ) { + for ( const { rejectQuery } of pendingToResolve.values() ) { + rejectQuery(err); + } + throw err; + } finally { + for ( const { pendingKey } of pendingToResolve.values() ) { + kv.del(pendingKey); } } - for ( const [key, { resolveQuery }] of pendingToResolve.entries() ) { - if ( ! resolvedKeys.has(key) ) { - resolveQuery(null); - } - } - } catch ( err ) { - for ( const { rejectQuery } of pendingToResolve.values() ) { - rejectQuery(err); - } - throw err; - } finally { - for ( const { pendingKey } of pendingToResolve.values() ) { - kv.del(pendingKey); - } } + + const pendingResults = await pendingResultsPromise; + for ( const app of pendingResults ) { + addApp(app); + } + + return normalized.map(spec => { + let app; + if ( spec.uid ) { + app = appByUid.get(spec.uid); + } else if ( spec.name ) { + app = appByName.get(spec.name); + } else if ( spec.id ) { + app = appById.get(spec.id); + } + return app ? { ...app } : null; + }); + } finally { + const end = Date.now(); + console.log('Exiting at: ', end); + console.log('Total time taken for get_apps(): ', end - start); } - - const pendingResults = await pendingResultsPromise; - for ( const app of pendingResults ) { - addApp(app); - } - - return normalized.map(spec => { - let app; - if ( spec.uid ) { - app = appByUid.get(spec.uid); - } else if ( spec.name ) { - app = appByName.get(spec.name); - } else if ( spec.id ) { - app = appById.get(spec.id); - } - return app ? { ...app } : null; - }); -} +}); /** * Checks to see if an app exists @@ -1482,8 +1491,67 @@ function seconds_to_string (seconds) { * @param {*} fsentry * @param {*} options */ -async function suggest_app_for_fsentry (fsentry, options) { - const suggested_apps = []; +const SUGGEST_APP_CODE_EXTS = [ + '.asm', + '.asp', + '.aspx', + '.bash', + '.c', + '.cpp', + '.css', + '.csv', + '.dhtml', + '.f', + '.go', + '.h', + '.htm', + '.html', + '.html5', + '.java', + '.jl', + '.js', + '.jsa', + '.json', + '.jsonld', + '.jsf', + '.jsp', + '.kt', + '.log', + '.lock', + '.lua', + '.md', + '.perl', + '.phar', + '.php', + '.pl', + '.py', + '.r', + '.rb', + '.rdata', + '.rda', + '.rdf', + '.rds', + '.rs', + '.rlib', + '.rpy', + '.scala', + '.sc', + '.scm', + '.sh', + '.sol', + '.sql', + '.ss', + '.svg', + '.swift', + '.toml', + '.ts', + '.wasm', + '.xhtml', + '.xml', + '.yaml', +]; + +const buildSuggestedAppSpecifiers = (fsentry) => { const name_specifiers = []; let content_type = mime.contentType(fsentry.name); @@ -1501,74 +1569,12 @@ async function suggest_app_for_fsentry (fsentry, options) { })(); const file_extension = _path.extname(fsname).toLowerCase(); - const any_of = (list, name) => { - return list.some(v => name.endsWith(v)); - }; + const any_of = (list, name) => list.some(v => name.endsWith(v)); //--------------------------------------------- // Code //--------------------------------------------- - const exts_code = [ - '.asm', - '.asp', - '.aspx', - '.bash', - '.c', - '.cpp', - '.css', - '.csv', - '.dhtml', - '.f', - '.go', - '.h', - '.htm', - '.html', - '.html5', - '.java', - '.jl', - '.js', - '.jsa', - '.json', - '.jsonld', - '.jsf', - '.jsp', - '.kt', - '.log', - '.lock', - '.lua', - '.md', - '.perl', - '.phar', - '.php', - '.pl', - '.py', - '.r', - '.rb', - '.rdata', - '.rda', - '.rdf', - '.rds', - '.rs', - '.rlib', - '.rpy', - '.scala', - '.sc', - '.scm', - '.sh', - '.sol', - '.sql', - '.ss', - '.svg', - '.swift', - '.toml', - '.ts', - '.wasm', - '.xhtml', - '.xml', - '.yaml', - ]; - - if ( any_of(exts_code, fsname) || !fsname.includes('.') ) { + if ( any_of(SUGGEST_APP_CODE_EXTS, fsname) || !fsname.includes('.') ) { name_specifiers.push({ name: 'code' }); name_specifiers.push({ name: 'editor' }); } @@ -1639,42 +1645,113 @@ async function suggest_app_for_fsentry (fsentry, options) { const apps = kv.get(`assocs:${file_extension.slice(1)}:apps`) ?? []; const id_specifiers = apps.map(app_id => ({ id: app_id })); - const specifiers = [...name_specifiers, ...id_specifiers]; - const resolved = specifiers.length > 0 - ? await get_apps(specifiers) - : []; + return { name_specifiers, id_specifiers }; +}; - const name_apps = resolved.slice(0, name_specifiers.length); +const buildSuggestedAppsFromResolved = (resolved, name_specifier_count, options) => { + const suggested_apps = []; + + const name_apps = resolved.slice(0, name_specifier_count); suggested_apps.push(...name_apps); - const third_party_apps = resolved.slice(name_specifiers.length); + const third_party_apps = resolved.slice(name_specifier_count); for ( const third_party_app of third_party_apps ) { if ( ! third_party_app ) continue; if ( third_party_app.approved_for_opening_items || - (options !== undefined && options.user !== undefined && options.user.id === third_party_app.owner_user_id) ) + (options?.user && options.user.id === third_party_app.owner_user_id) ) { suggested_apps.push(third_party_app); } } - // return list - if ( suggested_apps.some(app => app && app.name === 'editor') ) { - const [codeapp] = await get_apps([{ name: 'codeapp' }]); - if ( codeapp ) { - suggested_apps.push(codeapp); - } - } - return suggested_apps.filter((suggested_app, pos, self) => { + const needs_codeapp = suggested_apps.some(app => app && app.name === 'editor'); + return { suggested_apps, needs_codeapp }; +}; + +const normalizeSuggestedApps = (suggested_apps) => ( + suggested_apps.filter((suggested_app, pos, self) => { // Remove any null values caused by calling `get_app()` for apps that don't exist. // This happens on self-host because we don't include `code`, among others. - if ( ! suggested_app ) - { + if ( ! suggested_app ) { return false; } // Remove any duplicate entries return self.indexOf(suggested_app) === pos; - }); + }) +); + +async function suggestedAppsForFsEntries (fsentries, options) { + if ( ! Array.isArray(fsentries) ) { + fsentries = [fsentries]; + } + + const batches = []; + const specifiers = []; + const results = new Array(fsentries.length); + + for ( let index = 0; index < fsentries.length; index++ ) { + const fsentry = fsentries[index]; + if ( ! fsentry ) { + results[index] = []; + continue; + } + + const { name_specifiers, id_specifiers } = buildSuggestedAppSpecifiers(fsentry); + const entry_specifiers = [...name_specifiers, ...id_specifiers]; + + if ( entry_specifiers.length === 0 ) { + results[index] = []; + continue; + } + + const offset = specifiers.length; + specifiers.push(...entry_specifiers); + batches.push({ + index, + offset, + count: entry_specifiers.length, + name_count: name_specifiers.length, + suggested_apps: [], + needs_codeapp: false, + }); + } + + let resolved = []; + if ( specifiers.length > 0 ) { + resolved = await get_apps(specifiers); + } + + let any_needs_codeapp = false; + for ( const batch of batches ) { + const slice = resolved.slice(batch.offset, batch.offset + batch.count); + const { suggested_apps, needs_codeapp } = buildSuggestedAppsFromResolved(slice, + batch.name_count, + options); + batch.suggested_apps = suggested_apps; + batch.needs_codeapp = needs_codeapp; + if ( needs_codeapp ) any_needs_codeapp = true; + } + + let codeapp; + if ( any_needs_codeapp ) { + [codeapp] = await get_apps([{ name: 'codeapp' }]); + } + + for ( const batch of batches ) { + let suggested_apps = batch.suggested_apps; + if ( batch.needs_codeapp && codeapp ) { + suggested_apps = [...suggested_apps, codeapp]; + } + results[batch.index] = normalizeSuggestedApps(suggested_apps); + } + + return results; +} + +async function suggestedAppForFsEntry (fsentry, options) { + const [result] = await suggestedAppsForFsEntries([fsentry], options); + return result; } async function get_taskbar_items (user, { icon_size, no_icons } = {}) { @@ -1898,7 +1975,8 @@ module.exports = { send_email_verification_token, sign_file, subdomain, - suggest_app_for_fsentry, + suggestedAppsForFsEntries, + suggestedAppForFsEntry, df, username_exists, uuid2fsentry,