diff --git a/src/backend/src/filesystem/FSNodeContext.js b/src/backend/src/filesystem/FSNodeContext.js index 14cf2100d..1a27c5312 100644 --- a/src/backend/src/filesystem/FSNodeContext.js +++ b/src/backend/src/filesystem/FSNodeContext.js @@ -46,6 +46,8 @@ const { PermissionUtil } = require("../services/auth/PermissionService"); * @property {string} uid the UUID of the filesystem entry */ module.exports = class FSNodeContext { + static CONCERN = 'filesystem'; + static TYPE_FILE = { label: 'File' }; static TYPE_DIRECTORY = { label: 'Directory' }; static TYPE_SYMLINK = {}; @@ -75,7 +77,9 @@ module.exports = class FSNodeContext { provider, fs }) { - this.log = services.get('log-service').create('fsnode-context'); + this.log = services.get('log-service').create('fsnode-context', { + concern: this.constructor.CONCERN, + }); this.selector_ = null; this.selectors_ = []; this.selector = selector; diff --git a/src/backend/src/filesystem/hl_operations/hl_read.js b/src/backend/src/filesystem/hl_operations/hl_read.js index 963e2655f..0cab3956f 100644 --- a/src/backend/src/filesystem/hl_operations/hl_read.js +++ b/src/backend/src/filesystem/hl_operations/hl_read.js @@ -21,6 +21,7 @@ const { LLRead } = require("../ll_operations/ll_read"); const { HLFilesystemOperation } = require("./definitions"); class HLRead extends HLFilesystemOperation { + static CONCERN = 'filesystem'; static MODULES = { 'stream': require('stream'), } diff --git a/src/backend/src/filesystem/hl_operations/hl_readdir.js b/src/backend/src/filesystem/hl_operations/hl_readdir.js index a06719b71..ebf0be06a 100644 --- a/src/backend/src/filesystem/hl_operations/hl_readdir.js +++ b/src/backend/src/filesystem/hl_operations/hl_readdir.js @@ -25,6 +25,7 @@ const { LLReadShares } = require("../ll_operations/ll_readshares"); const { HLFilesystemOperation } = require("./definitions"); class HLReadDir extends HLFilesystemOperation { + static CONCERN = 'filesystem'; async _run () { const { subject: subject_let, user, no_thumbs, no_assocs, actor } = this.values; let subject = subject_let; diff --git a/src/backend/src/filesystem/ll_operations/ll_mkdir.js b/src/backend/src/filesystem/ll_operations/ll_mkdir.js index 5fc1e415d..a097b2109 100644 --- a/src/backend/src/filesystem/ll_operations/ll_mkdir.js +++ b/src/backend/src/filesystem/ll_operations/ll_mkdir.js @@ -24,6 +24,7 @@ const { RESOURCE_STATUS_PENDING_CREATE } = require("../../modules/puterfs/Resour const { LLFilesystemOperation } = require("./definitions"); class LLMkdir extends LLFilesystemOperation { + static CONCERN = 'filesystem'; static MODULES = { _path: require('path'), uuidv4: require('uuid').v4, diff --git a/src/backend/src/filesystem/ll_operations/ll_read.js b/src/backend/src/filesystem/ll_operations/ll_read.js index 1aad9e047..91abf5acb 100644 --- a/src/backend/src/filesystem/ll_operations/ll_read.js +++ b/src/backend/src/filesystem/ll_operations/ll_read.js @@ -43,6 +43,7 @@ const dry_checks = [ ]; class LLRead extends LLFilesystemOperation { + static CONCERN = 'filesystem'; static METHODS = { _run: new Sequence({ async before_each (a, step) { @@ -98,7 +99,7 @@ class LLRead extends LLFilesystemOperation { const maybe_buffer = await svc_fileCache.try_get(fsNode, a.log); if ( maybe_buffer ) { - a.log.info('cache hit'); + a.log.cache(true, 'll_read'); const { has_range } = a.values(); if ( has_range ) { return a.stop( @@ -110,7 +111,7 @@ class LLRead extends LLFilesystemOperation { ); } - a.log.info('cache miss'); + a.log.cache(false, 'll_read'); }, async function create_S3_read_stream (a) { const context = a.iget('context'); diff --git a/src/backend/src/filesystem/ll_operations/ll_readdir.js b/src/backend/src/filesystem/ll_operations/ll_readdir.js index b4f6c51b7..1c2439723 100644 --- a/src/backend/src/filesystem/ll_operations/ll_readdir.js +++ b/src/backend/src/filesystem/ll_operations/ll_readdir.js @@ -24,6 +24,7 @@ const { NodeUIDSelector, NodeChildSelector } = require("../node/selectors"); const { LLFilesystemOperation } = require("./definitions"); class LLReadDir extends LLFilesystemOperation { + static CONCERN = 'filesystem'; async _run () { const { context } = this; const { subject: subject_let, actor, no_acl } = this.values; diff --git a/src/backend/src/modules/core/LogService.js b/src/backend/src/modules/core/LogService.js index c451f6569..bfd2a8ea9 100644 --- a/src/backend/src/modules/core/LogService.js +++ b/src/backend/src/modules/core/LogService.js @@ -23,8 +23,8 @@ const LOG_LEVEL_WARN = logSeverity(1, 'WARN', '33;1', 'warn'); const LOG_LEVEL_INFO = logSeverity(2, 'INFO', '36;1', 'info'); const LOG_LEVEL_TICK = logSeverity(10, 'TICK', '34;1', 'info'); const LOG_LEVEL_DEBU = logSeverity(4, 'DEBU', '37;1', 'debug'); -const LOG_LEVEL_NOTICEME = logSeverity(4, 'NOTICE_ME', '33;1', 'error'); -const LOG_LEVEL_SYSTEM = logSeverity(4, 'SYSTEM', '33;1', 'system'); +const LOG_LEVEL_NOTICEME = logSeverity(3, 'NOTICE_ME', '33;1', 'error'); +const LOG_LEVEL_SYSTEM = logSeverity(3, 'SYSTEM', '33;1', 'system'); const winston = require('winston'); const { Context } = require('../../util/context'); @@ -192,6 +192,8 @@ class DevLogger { } if ( this.off ) return; + + if ( ! process.env.DEBUG && log_lvl.ordinal >= 4 ) return; const ld = Context.get('logdent', { allow_fallback: true }) const prefix = globalThis.dev_console_indent_on @@ -308,23 +310,44 @@ class CustomLogger { this.delegate = delegate; this.callback = callback; } - onLogMessage (log_lvl, crumbs, message, fields, ...a) { + async onLogMessage (log_lvl, crumbs, message, fields, ...a) { // Logging is allowed to be performed without a context, but we // don't want log functions to be asynchronous which rules out // wrapping with Context.allow_fallback. Instead we provide a // context as a parameter. const context = Context.get(undefined, { allow_fallback: true }); + let ret; + try { + ret = await this.callback({ + context, + log_lvl, crumbs, message, fields, args: a, + }); + } catch (e) { + console.error('error?', e); + } + + if ( ret && ret.skip ) return; + + if ( ! ret ) { + this.delegate.onLogMessage( + log_lvl, + crumbs, + message, + fields, + ...a, + ); + return; + } + const { log_lvl: _log_lvl, crumbs: _crumbs, message: _message, fields: _fields, args, - } = this.callback({ - context, - log_lvl, crumbs, message, fields, args: a, - }); + } = ret; + this.delegate.onLogMessage( _log_lvl ?? log_lvl, _crumbs ?? crumbs, diff --git a/src/backend/src/modules/hostos/ProcessService.js b/src/backend/src/modules/hostos/ProcessService.js index b0f930a07..290387ac9 100644 --- a/src/backend/src/modules/hostos/ProcessService.js +++ b/src/backend/src/modules/hostos/ProcessService.js @@ -26,6 +26,8 @@ class ProxyLogger { } class ProcessService extends BaseService { + static CONCERN = 'workers'; + static MODULES = { path: require('path'), spawn: require('child_process').spawn, diff --git a/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js b/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js index 2a6c5c693..0789a6df3 100644 --- a/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js +++ b/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js @@ -21,6 +21,7 @@ const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSele const BaseService = require("../../services/BaseService"); module.exports = class DatabaseFSEntryFetcher extends BaseService { + static CONCERN = 'filesystem'; _construct () { this.defaultProperties = [ 'id', diff --git a/src/backend/src/modules/puterfs/DatabaseFSEntryService.js b/src/backend/src/modules/puterfs/DatabaseFSEntryService.js index 726225e2c..f1f4f13e2 100644 --- a/src/backend/src/modules/puterfs/DatabaseFSEntryService.js +++ b/src/backend/src/modules/puterfs/DatabaseFSEntryService.js @@ -192,6 +192,8 @@ class DatabaseFSEntryDelete extends AbstractDatabaseFSEntryOperation { class DatabaseFSEntryService extends BaseService { + static CONCERN = 'filesystem'; + static STATUS_READY = {}; static STATUS_RUNNING_JOB = {}; diff --git a/src/backend/src/modules/selfhosted/MinLogService.js b/src/backend/src/modules/selfhosted/MinLogService.js new file mode 100644 index 000000000..7e34987dc --- /dev/null +++ b/src/backend/src/modules/selfhosted/MinLogService.js @@ -0,0 +1,97 @@ +const BaseService = require("../../services/BaseService"); + +class MinLogService extends BaseService { + static DESCRIPTION = ` + MinLogService hides any log messages which specify an area of concern. + A developer can enable particular areas of concern through the console. + ` + + _construct () { + this.on = true; + this.visible = new Set(); + + this.widget_ = null; + } + + _init () { + // Show console widget so developer knows logs are hidden + this.add_dev_console_widget_(); + + // Register log middleware to hide logs + const svc_log = this.services.get('log-service'); + svc_log.register_log_middleware(async log_details => { + if ( ! this.on ) return; + + const { fields } = log_details; + if ( fields.hasOwnProperty('concern') ) { + if ( ! this.visible.has(fields.concern) ) { + return { skip: true }; + } + } + + return; + }); + + this._register_commands(this.services.get('commands')); + } + + add_dev_console_widget_() { + const svc_devConsole = this.services.get('dev-console', { optional: true }); + if ( ! svc_devConsole ) return; + + this.widget_ = () => { + const lines = [ + `\x1B[31;1mSome logs hidden! Type minlog:off to see all logs.\x1B[0m` + ]; + return lines; + } + svc_devConsole.add_widget(this.widget_); + } + + rm_dev_console_widget_() { + const svc_devConsole = this.services.get('dev-console', { optional: true }); + if ( ! svc_devConsole ) return; + + const lines = this.widget_(); + this.log.info(lines[0]); + + svc_devConsole.remove_widget(this.widget_); + this.widget_ = null; + } + + _register_commands (commands) { + commands.registerCommands('minlog', [ + { + id: 'on', + handler: async (args, log) => { + this.on = true; + } + }, + { + id: 'off', + handler: async (args, log) => { + this.rm_dev_console_widget_(); + this.on = false; + } + }, + { + id: 'show', + handler: async (args, log) => { + const [ name ] = args; + + this.visible.add(name); + } + }, + { + id: 'hide', + handler: async (args, log) => { + const [ name ] = args; + + this.visible.delete(name); + } + }, + ]); + } +} + +module.exports = MinLogService; diff --git a/src/backend/src/modules/selfhosted/SelfHostedModule.js b/src/backend/src/modules/selfhosted/SelfHostedModule.js index bcd8c6bda..3f0876beb 100644 --- a/src/backend/src/modules/selfhosted/SelfHostedModule.js +++ b/src/backend/src/modules/selfhosted/SelfHostedModule.js @@ -40,6 +40,9 @@ class SelfHostedModule extends AdvancedBase { const { DBKVService } = require("../../services/DBKVService"); services.registerService('puter-kvstore', DBKVService); + + const MinLogService = require('./MinLogService'); + services.registerService('min-log', MinLogService); // TODO: sucks const RELATIVE_PATH = '../../../../../'; diff --git a/src/backend/src/modules/web/WebServerService.js b/src/backend/src/modules/web/WebServerService.js index 2cce05df0..c1398a6fa 100644 --- a/src/backend/src/modules/web/WebServerService.js +++ b/src/backend/src/modules/web/WebServerService.js @@ -38,6 +38,8 @@ const strutil = require('@heyputer/putility').libs.string; * It also validates the host header and IP addresses to prevent security vulnerabilities. */ class WebServerService extends BaseService { + static CONCERN = 'web'; + static MODULES = { https: require('https'), http: require('http'), @@ -358,7 +360,9 @@ class WebServerService extends BaseService { fields.status, fields.responseTime, ].join(' '); - const log = this.services.get('log-service').create('morgan'); + const log = this.services.get('log-service').create('morgan', { + concern: 'web' + }); try { log.info(message, fields); } catch (e) { @@ -540,7 +544,6 @@ class WebServerService extends BaseService { req.query[k] = undefined; } } - console.log('\x1B[36;1m======= ok???', req.query); next(); }); diff --git a/src/backend/src/om/entitystorage/BaseES.js b/src/backend/src/om/entitystorage/BaseES.js index 6229effb5..f0658890c 100644 --- a/src/backend/src/om/entitystorage/BaseES.js +++ b/src/backend/src/om/entitystorage/BaseES.js @@ -84,7 +84,9 @@ class BaseES extends AdvancedBase { } this.log = Context.get('services').get('log-service') - .create(`ES:${this.entity_name}:${this.constructor.name}`); + .create(`ES:${this.entity_name}:${this.constructor.name}`, { + concern: 'es', + }); } async provide_context ( args ) { diff --git a/src/backend/src/om/entitystorage/SQLES.js b/src/backend/src/om/entitystorage/SQLES.js index 561de9fdd..8af09a66d 100644 --- a/src/backend/src/om/entitystorage/SQLES.js +++ b/src/backend/src/om/entitystorage/SQLES.js @@ -115,10 +115,6 @@ class SQLES extends BaseES { const values = []; if ( predicate ) values.push(...(predicate.values || [])); - if ( this.debug ) { - this.log.info('-> SQL STMT', { stmt, values }); - } - const rows = await this.db.read(stmt, values); const entities = []; diff --git a/src/backend/src/routers/filesystem_api/readdir.js b/src/backend/src/routers/filesystem_api/readdir.js index d08fbe280..34503eca2 100644 --- a/src/backend/src/routers/filesystem_api/readdir.js +++ b/src/backend/src/routers/filesystem_api/readdir.js @@ -43,7 +43,9 @@ module.exports = eggspress('/readdir', { }, async (req, res, next) => { let log; { const x = Context.get(); - log = x.get('services').get('log-service').create('readdir'); + log = x.get('services').get('log-service').create('readdir', { + concern: 'filesystem', + }); log.info(`readdir: ${req.body.path}`); } diff --git a/src/backend/src/routers/get-launch-apps.js b/src/backend/src/routers/get-launch-apps.js index 71ae3da9a..88820832a 100644 --- a/src/backend/src/routers/get-launch-apps.js +++ b/src/backend/src/routers/get-launch-apps.js @@ -81,7 +81,6 @@ module.exports = async (req, res) => { // prepare each app for returning to user by only returning the necessary fields // and adding them to the retobj array result.recent = []; - console.log('\x1B[36;1m -------- RECENT APPS -------- \x1B[0m', apps); for ( const { app_uid: uid } of apps ) { console.log('\x1B[36;1m -------- UID -------- \x1B[0m', uid); const app = await get_app({ uid }); diff --git a/src/backend/src/services/BaseService.js b/src/backend/src/services/BaseService.js index 1582bd959..d48dc8030 100644 --- a/src/backend/src/services/BaseService.js +++ b/src/backend/src/services/BaseService.js @@ -85,7 +85,11 @@ class BaseService extends concepts.Service { */ async init () { const services = this.services; - this.log = services.get('log-service').create(this.service_name); + const log_fields = {}; + if ( this.constructor.CONCERN ) { + log_fields.concern = this.constructor.CONCERN; + } + this.log = services.get('log-service').create(this.service_name, log_fields); this.errors = services.get('error-service').create(this.log); await (this._init || NOOP).call(this, this.args); diff --git a/src/backend/src/services/OperationTraceService.js b/src/backend/src/services/OperationTraceService.js index 60eb9509f..16d789e87 100644 --- a/src/backend/src/services/OperationTraceService.js +++ b/src/backend/src/services/OperationTraceService.js @@ -47,7 +47,8 @@ class OperationFrame { this.id = require('uuid').v4(); this.log = (x ?? Context).get('services').get('log-service').create( - `frame:${this.id}` + `frame:${this.id}`, + { concern: 'filesystem' }, ); } @@ -232,8 +233,12 @@ class OperationFrame { * This service is essential for monitoring and logging the lifecycle of operations within the system. */ class OperationTraceService { + static CONCERN = 'filesystem'; + constructor ({ services }) { - this.log = services.get('log-service').create('operation-trace'); + this.log = services.get('log-service').create('operation-trace', { + concern: this.constructor.CONCERN, + }); // TODO: replace with kv.js set this.ongoing = {}; @@ -334,7 +339,12 @@ class BaseOperation extends AdvancedBase { // let's make the logger for it too this.log = x.get('services').get('log-service').create( - this.constructor.name, { operation: frame.id }); + this.constructor.name, { + operation: frame.id, + ...(this.constructor.CONCERN ? { + concern: this.constructor.CONCERN, + } : {}) + }); // Run operation in new context try { diff --git a/src/backend/src/services/auth/PermissionService.js b/src/backend/src/services/auth/PermissionService.js index ea375fe9e..6bcc24ff2 100644 --- a/src/backend/src/services/auth/PermissionService.js +++ b/src/backend/src/services/auth/PermissionService.js @@ -264,6 +264,7 @@ class PermissionUtil { * This service interacts with the database to manage permissions and logs actions for auditing purposes. */ class PermissionService extends BaseService { + static CONCERN = 'permissions'; /** * Initializes the PermissionService by setting up internal arrays for permission handling. * diff --git a/src/backend/src/services/drivers/DriverService.js b/src/backend/src/services/drivers/DriverService.js index 8a42d3547..bee469a51 100644 --- a/src/backend/src/services/drivers/DriverService.js +++ b/src/backend/src/services/drivers/DriverService.js @@ -35,6 +35,8 @@ const strutil = require('@heyputer/putility').libs.string; * It provides methods for registering drivers, calling driver methods, and handling driver errors. */ class DriverService extends BaseService { + static CONCERN = 'drivers'; + static MODULES = { types: require('./types'), }