dev: reduce visible log output by default

- introduce concern-based log categories
- ensure debug-level logs aren't shown without DEBUG env variable
- remove some unnecessary/outdated logs
- add concern fields to services and log instantiations

src/backend/src/filesystem/FSNodeContext.js

@@ -46,6 +46,8 @@ const { PermissionUtil } = require("../services/auth/PermissionService");
  * @property {string} uid the UUID of the filesystem entry
  */
 module.exports = class FSNodeContext {
+    static CONCERN = 'filesystem';
+
     static TYPE_FILE = { label: 'File' };
     static TYPE_DIRECTORY = { label: 'Directory' };
     static TYPE_SYMLINK = {};
@@ -75,7 +77,9 @@ module.exports = class FSNodeContext {
         provider,
         fs
     }) {
-        this.log = services.get('log-service').create('fsnode-context');
+        this.log = services.get('log-service').create('fsnode-context', {
+            concern: this.constructor.CONCERN,
+        });
         this.selector_ = null;
         this.selectors_ = [];
         this.selector = selector;

src/backend/src/filesystem/hl_operations/hl_read.js

@@ -21,6 +21,7 @@ const { LLRead } = require("../ll_operations/ll_read");
 const { HLFilesystemOperation } = require("./definitions");
 
 class HLRead extends HLFilesystemOperation {
+    static CONCERN = 'filesystem';
     static MODULES = {
         'stream': require('stream'),
     }

src/backend/src/filesystem/hl_operations/hl_readdir.js

@@ -25,6 +25,7 @@ const { LLReadShares } = require("../ll_operations/ll_readshares");
 const { HLFilesystemOperation } = require("./definitions");
 
 class HLReadDir extends HLFilesystemOperation {
+    static CONCERN = 'filesystem';
     async _run () {
         const { subject: subject_let, user, no_thumbs, no_assocs, actor } = this.values;
         let subject = subject_let;

src/backend/src/filesystem/ll_operations/ll_mkdir.js

@@ -24,6 +24,7 @@ const { RESOURCE_STATUS_PENDING_CREATE } = require("../../modules/puterfs/Resour
 const { LLFilesystemOperation } = require("./definitions");
 
 class LLMkdir extends LLFilesystemOperation {
+    static CONCERN = 'filesystem';
     static MODULES = {
         _path: require('path'),
         uuidv4: require('uuid').v4,

src/backend/src/filesystem/ll_operations/ll_read.js

@@ -43,6 +43,7 @@ const dry_checks = [
 ];
 
 class LLRead extends LLFilesystemOperation {
+    static CONCERN = 'filesystem';
     static METHODS = {
         _run: new Sequence({
             async before_each (a, step) {
@@ -98,7 +99,7 @@ class LLRead extends LLFilesystemOperation {
 
                 const maybe_buffer = await svc_fileCache.try_get(fsNode, a.log);
                 if ( maybe_buffer ) {
-                    a.log.info('cache hit');
+                    a.log.cache(true, 'll_read');
                     const { has_range } = a.values();
                     if ( has_range ) {
                         return a.stop(
@@ -110,7 +111,7 @@ class LLRead extends LLFilesystemOperation {
                     );
                 }
 
-                a.log.info('cache miss');
+                a.log.cache(false, 'll_read');
             },
             async function create_S3_read_stream (a) {
                 const context = a.iget('context');

src/backend/src/filesystem/ll_operations/ll_readdir.js

@@ -24,6 +24,7 @@ const { NodeUIDSelector, NodeChildSelector } = require("../node/selectors");
 const { LLFilesystemOperation } = require("./definitions");
 
 class LLReadDir extends LLFilesystemOperation {
+    static CONCERN = 'filesystem';
     async _run () {
         const { context } = this;
         const { subject: subject_let, actor, no_acl } = this.values;

src/backend/src/modules/core/LogService.js

@@ -23,8 +23,8 @@ const LOG_LEVEL_WARN = logSeverity(1, 'WARN', '33;1', 'warn');
 const LOG_LEVEL_INFO = logSeverity(2, 'INFO', '36;1', 'info');
 const LOG_LEVEL_TICK = logSeverity(10, 'TICK', '34;1', 'info');
 const LOG_LEVEL_DEBU = logSeverity(4, 'DEBU', '37;1', 'debug');
-const LOG_LEVEL_NOTICEME = logSeverity(4, 'NOTICE_ME', '33;1', 'error');
-const LOG_LEVEL_SYSTEM = logSeverity(4, 'SYSTEM', '33;1', 'system');
+const LOG_LEVEL_NOTICEME = logSeverity(3, 'NOTICE_ME', '33;1', 'error');
+const LOG_LEVEL_SYSTEM = logSeverity(3, 'SYSTEM', '33;1', 'system');
 
 const winston = require('winston');
 const { Context } = require('../../util/context');
@@ -192,6 +192,8 @@ class DevLogger {
         }
 
         if ( this.off ) return;
+        
+        if ( ! process.env.DEBUG && log_lvl.ordinal >= 4 ) return;
 
         const ld = Context.get('logdent', { allow_fallback: true })
         const prefix = globalThis.dev_console_indent_on
@@ -308,23 +310,44 @@ class CustomLogger {
         this.delegate = delegate;
         this.callback = callback;
     }
-    onLogMessage (log_lvl, crumbs, message, fields, ...a) {
+    async onLogMessage (log_lvl, crumbs, message, fields, ...a) {
         // Logging is allowed to be performed without a context, but we
         // don't want log functions to be asynchronous which rules out
         // wrapping with Context.allow_fallback. Instead we provide a
         // context as a parameter.
         const context = Context.get(undefined, { allow_fallback: true });
 
+        let ret;
+        try {
+            ret = await this.callback({
+                context,
+                log_lvl, crumbs, message, fields, args: a,
+            });
+        } catch (e) {
+            console.error('error?', e);
+        }
+        
+        if ( ret && ret.skip ) return;
+
+        if ( ! ret ) {
+            this.delegate.onLogMessage(
+                log_lvl,
+                crumbs,
+                message,
+                fields,
+                ...a,
+            );
+            return;
+        }
+        
         const {
             log_lvl: _log_lvl,
             crumbs: _crumbs,
             message: _message,
             fields: _fields,
             args,
-        } = this.callback({
-            context,
-            log_lvl, crumbs, message, fields, args: a,
-        });
+        } = ret;
+
         this.delegate.onLogMessage(
             _log_lvl ?? log_lvl,
             _crumbs ?? crumbs,

src/backend/src/modules/hostos/ProcessService.js

@@ -26,6 +26,8 @@ class ProxyLogger {
 }
 
 class ProcessService extends BaseService {
+    static CONCERN = 'workers';
+
     static MODULES = {
         path: require('path'),
         spawn: require('child_process').spawn,

src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js

@@ -21,6 +21,7 @@ const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSele
 const BaseService = require("../../services/BaseService");
 
 module.exports = class DatabaseFSEntryFetcher extends BaseService {
+    static CONCERN = 'filesystem';
     _construct () {
         this.defaultProperties = [
             'id',

src/backend/src/modules/puterfs/DatabaseFSEntryService.js

@@ -192,6 +192,8 @@ class DatabaseFSEntryDelete extends AbstractDatabaseFSEntryOperation {
 
 
 class DatabaseFSEntryService extends BaseService {
+    static CONCERN = 'filesystem';
+
     static STATUS_READY = {};
     static STATUS_RUNNING_JOB = {};
 

src/backend/src/modules/selfhosted/MinLogService.js

@@ -0,0 +1,97 @@
+const BaseService = require("../../services/BaseService");
+
+class MinLogService extends BaseService {
+    static DESCRIPTION = `
+        MinLogService hides any log messages which specify an area of concern.
+        A developer can enable particular areas of concern through the console.
+    `
+    
+    _construct () {
+        this.on = true;
+        this.visible = new Set();
+        
+        this.widget_ = null;
+    }
+    
+    _init () {
+        // Show console widget so developer knows logs are hidden
+        this.add_dev_console_widget_();
+
+        // Register log middleware to hide logs
+        const svc_log = this.services.get('log-service');
+        svc_log.register_log_middleware(async log_details => {
+            if ( ! this.on ) return;
+
+            const { fields } = log_details;
+            if ( fields.hasOwnProperty('concern') ) {
+                if ( ! this.visible.has(fields.concern) ) {
+                    return { skip: true };
+                }
+            }
+            
+            return;
+        });
+        
+        this._register_commands(this.services.get('commands'));
+    }
+    
+    add_dev_console_widget_() {
+        const svc_devConsole = this.services.get('dev-console', { optional: true });
+        if ( ! svc_devConsole ) return;
+        
+        this.widget_ = () => {
+            const lines = [
+                `\x1B[31;1mSome logs hidden! Type minlog:off to see all logs.\x1B[0m`
+            ];
+            return lines;
+        }
+        svc_devConsole.add_widget(this.widget_);
+    }
+
+    rm_dev_console_widget_() {
+        const svc_devConsole = this.services.get('dev-console', { optional: true });
+        if ( ! svc_devConsole ) return;
+        
+        const lines = this.widget_();
+        this.log.info(lines[0]);
+
+        svc_devConsole.remove_widget(this.widget_);
+        this.widget_ = null;
+    }
+
+    _register_commands (commands) {
+        commands.registerCommands('minlog', [
+            {
+                id: 'on',
+                handler: async (args, log) => {
+                    this.on = true;
+                }
+            },
+            {
+                id: 'off',
+                handler: async (args, log) => {
+                    this.rm_dev_console_widget_();
+                    this.on = false;
+                }
+            },
+            {
+                id: 'show',
+                handler: async (args, log) => {
+                    const [ name ] = args;
+
+                    this.visible.add(name);
+                }
+            },
+            {
+                id: 'hide',
+                handler: async (args, log) => {
+                    const [ name ] = args;
+
+                    this.visible.delete(name);
+                }
+            },
+        ]);
+    }
+}
+
+module.exports = MinLogService;

src/backend/src/modules/selfhosted/SelfHostedModule.js

@@ -40,6 +40,9 @@ class SelfHostedModule extends AdvancedBase {
         
         const { DBKVService } = require("../../services/DBKVService");
         services.registerService('puter-kvstore', DBKVService);
+        
+        const MinLogService = require('./MinLogService');
+        services.registerService('min-log', MinLogService);
 
         // TODO: sucks
         const RELATIVE_PATH = '../../../../../';

src/backend/src/modules/web/WebServerService.js

@@ -38,6 +38,8 @@ const strutil = require('@heyputer/putility').libs.string;
 * It also validates the host header and IP addresses to prevent security vulnerabilities.
 */
 class WebServerService extends BaseService {
+    static CONCERN = 'web';
+
     static MODULES = {
         https: require('https'),
         http: require('http'),
@@ -358,7 +360,9 @@ class WebServerService extends BaseService {
                     fields.status, fields.responseTime,
                 ].join(' ');
 
-                const log = this.services.get('log-service').create('morgan');
+                const log = this.services.get('log-service').create('morgan', {
+                    concern: 'web'
+                });
                 try {
                     log.info(message, fields);
                 } catch (e) {
@@ -540,7 +544,6 @@ class WebServerService extends BaseService {
                     req.query[k] = undefined;
                 }
             }
-            console.log('\x1B[36;1m======= ok???', req.query);
             next();
         });
         

src/backend/src/om/entitystorage/BaseES.js

@@ -84,7 +84,9 @@ class BaseES extends AdvancedBase {
         }
 
         this.log = Context.get('services').get('log-service')
-            .create(`ES:${this.entity_name}:${this.constructor.name}`);
+            .create(`ES:${this.entity_name}:${this.constructor.name}`, {
+                concern: 'es',
+            });
     }
 
     async provide_context ( args ) {

src/backend/src/om/entitystorage/SQLES.js

@@ -115,10 +115,6 @@ class SQLES extends BaseES {
             const values = [];
             if ( predicate ) values.push(...(predicate.values || []));
 
-            if ( this.debug ) {
-                this.log.info('-> SQL STMT', { stmt, values });
-            }
-
             const rows = await this.db.read(stmt, values);
 
             const entities = [];

src/backend/src/routers/filesystem_api/readdir.js

@@ -43,7 +43,9 @@ module.exports = eggspress('/readdir', {
 }, async (req, res, next) => {
     let log; {
         const x = Context.get();
-        log = x.get('services').get('log-service').create('readdir');
+        log = x.get('services').get('log-service').create('readdir', {
+            concern: 'filesystem',
+        });
         log.info(`readdir: ${req.body.path}`);
     }
 

src/backend/src/routers/get-launch-apps.js

@@ -81,7 +81,6 @@ module.exports = async (req, res) => {
     // prepare each app for returning to user by only returning the necessary fields
     // and adding them to the retobj array
     result.recent = [];
-    console.log('\x1B[36;1m -------- RECENT APPS -------- \x1B[0m', apps);
     for ( const { app_uid: uid } of apps ) {
         console.log('\x1B[36;1m -------- UID -------- \x1B[0m', uid);
         const app = await get_app({ uid });

src/backend/src/services/BaseService.js

@@ -85,7 +85,11 @@ class BaseService extends concepts.Service {
     */
     async init () {
         const services = this.services;
-        this.log = services.get('log-service').create(this.service_name);
+        const log_fields = {};
+        if ( this.constructor.CONCERN ) {
+            log_fields.concern = this.constructor.CONCERN;
+        }
+        this.log = services.get('log-service').create(this.service_name, log_fields);
         this.errors = services.get('error-service').create(this.log);
 
         await (this._init || NOOP).call(this, this.args);

src/backend/src/services/OperationTraceService.js

@@ -47,7 +47,8 @@ class OperationFrame {
         this.id = require('uuid').v4();
 
         this.log = (x ?? Context).get('services').get('log-service').create(
-            `frame:${this.id}`
+            `frame:${this.id}`,
+            { concern: 'filesystem' },
         );
     }
 
@@ -232,8 +233,12 @@ class OperationFrame {
 * This service is essential for monitoring and logging the lifecycle of operations within the system.
 */
 class OperationTraceService {
+    static CONCERN = 'filesystem';
+
     constructor ({ services }) {
-        this.log = services.get('log-service').create('operation-trace');
+        this.log = services.get('log-service').create('operation-trace', {
+            concern: this.constructor.CONCERN,
+        });
 
         // TODO: replace with kv.js set
         this.ongoing = {};
@@ -334,7 +339,12 @@ class BaseOperation extends AdvancedBase {
 
         // let's make the logger for it too
         this.log = x.get('services').get('log-service').create(
-            this.constructor.name, { operation: frame.id });
+            this.constructor.name, {
+                operation: frame.id,
+                ...(this.constructor.CONCERN ? {
+                    concern: this.constructor.CONCERN,
+                } : {})
+            });
 
         // Run operation in new context
         try {

src/backend/src/services/auth/PermissionService.js

@@ -264,6 +264,7 @@ class PermissionUtil {
 * This service interacts with the database to manage permissions and logs actions for auditing purposes.
 */
 class PermissionService extends BaseService {
+    static CONCERN = 'permissions';
     /**
     * Initializes the PermissionService by setting up internal arrays for permission handling.
     * 

src/backend/src/services/drivers/DriverService.js

@@ -35,6 +35,8 @@ const strutil = require('@heyputer/putility').libs.string;
  * It provides methods for registering drivers, calling driver methods, and handling driver errors.
  */
 class DriverService extends BaseService {
+    static CONCERN = 'drivers';
+
     static MODULES = {
         types: require('./types'),
     }