perf: improve tel + decrease logs (#2309)

* perfmon: lower healthcheck status cache, add kv health signal

* perf: improve tel + decrease logs

* logs

extensions/extensionController/src/ExtensionController.ts

@@ -96,8 +96,10 @@ export class HttpError extends Error {
 
 // Registers all routes from a decorated controller instance to an Express router
 export class ExtensionController {
+    logger?: Console;
     // TODO DS: make this work with other express-like routers
     registerRoutes () {
+        const logger = this.logger || console;
         const prefix = Object.getPrototypeOf(this).__controllerPrefix || '';
         const adminsForController = Object.getPrototypeOf(this).__adminUsernames as
             | string[]
@@ -126,7 +128,7 @@ export class ExtensionController {
             if ( ! extension[route.method] ) {
                 throw new Error(`Unsupported HTTP method: ${route.method}`);
             } else {
-                console.log(`Registering route: [${route.method.toUpperCase()}] ${fullPath}`);
+                logger.log(`Registering route: [${route.method.toUpperCase()}] ${fullPath}`);
 
                 (extension[route.method] as RouterMethods[HttpMethod])(
                                 fullPath,
@@ -154,20 +156,16 @@ export class ExtensionController {
                                     } catch ( error ) {
                                         if ( error instanceof HttpError ) {
                                             res.status(error.statusCode).send({ error: error.message });
-                                            console.error('httpError:', error);
+                                            logger.warn('httpError:', error);
                                             return;
                                         }
                                         if ( error instanceof Error ) {
-                                            res
-                                                .status(StatusCodes.INTERNAL_SERVER_ERROR)
-                                                .send({ error: error.message });
-                                            console.error('Non-http error:', error);
+                                            res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: error.message });
+                                            logger.error('Non-http error:', error);
                                             return;
                                         }
-                                        res
-                                            .status(StatusCodes.INTERNAL_SERVER_ERROR)
-                                            .send({ error: 'An unknown error occurred' });
-                                        console.error('An unknown error occurred:', error);
+                                        res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: 'An unknown error occurred' });
+                                        logger.error('An unknown error occurred:', error);
                                     }
                                 });
             }

src/backend/exports.js

@@ -35,7 +35,6 @@ import { EntityStoreModule } from './src/modules/entitystore/EntityStoreModule.j
 import { HostOSModule } from './src/modules/hostos/HostOSModule.js';
 import { InternetModule } from './src/modules/internet/InternetModule.js';
 import { KVStoreModule } from './src/modules/kvstore/KVStoreModule.js';
-import { PerfMonModule } from './src/modules/perfmon/PerfMonModule.js';
 import { PuterFSModule } from './src/modules/puterfs/PuterFSModule.js';
 import SelfHostedModule from './src/modules/selfhosted/SelfHostedModule.js';
 import { TestConfigModule } from './src/modules/test-config/TestConfigModule.js';
@@ -89,6 +88,5 @@ export default {
     DataAccessModule,
 
     // Development modules
-    PerfMonModule,
     DevelopmentModule,
 };

src/backend/src/CoreModule.js

@@ -35,6 +35,10 @@ const query = require('./om/query/query');
  */
 const install = async ({ context, services, app, useapi, modapi }) => {
     const config = require('./config');
+    const { TelemetryService } = require('./modules/perfmon/TelemetryService');
+    if ( ! services.has('telemetry') ) {
+        services.registerService('telemetry', TelemetryService);
+    }
 
     // === LIBRARIES ===
 

src/backend/src/helpers.js

@@ -276,9 +276,7 @@ async function refresh_apps_cache (options, override) {
 async function refresh_associations_cache () {
     /** @type BaseDatabaseAccessService */
     const db = _servicesHolder.services.get('database').get(DB_READ, 'apps');
-
-    const log = _servicesHolder.services.get('log-service').create('helpers.js');
-    log.tick('refresh file associations');
+    console.debug('refresh file associations');
     const associations = await db.read('SELECT * FROM app_filetype_association');
     const lists = {};
     for ( const association of associations ) {
@@ -356,6 +354,7 @@ async function get_app (options) {
     const pendingKey = `pending_app:${queryKey}`;
     const pending = kv.get(pendingKey);
     if ( pending ) {
+        // Reuse the existing pending query
         const result = await pending;
         // shallow clone the result
         return result ? { ...result } : null;

src/backend/src/modules/broadcast/BroadcastService.js

@@ -90,7 +90,7 @@ class BroadcastService extends BaseService {
                 }
 
                 if ( key === 'test' ) {
-                    this.log.noticeme(`test message: ${
+                    console.debug(`test message: ${
                         JSON.stringify(data)}`);
                 }
 
@@ -108,7 +108,7 @@ class BroadcastService extends BaseService {
             {
                 id: 'test',
                 description: 'send a test message',
-                handler: async (args, ctx) => {
+                handler: async () => {
                     this.on_event('test', {
                         contents: 'I am a test message',
                     }, {});

src/backend/src/modules/captcha/services/CaptchaService.js

@@ -404,16 +404,15 @@ class CaptchaService extends BaseService {
      * @returns {boolean} Whether the answer is valid
      */
     verifyCaptcha (token, userAnswer) {
-        console.log('====== CAPTCHA SERVICE VERIFICATION DIAGNOSTIC ======');
-        console.log('TOKENS_TRACKING: verifyCaptcha called. Service ID:', this.serviceId);
-        console.log('TOKENS_TRACKING: Request counter during verification:', this.requestCounter);
-        console.log('TOKENS_TRACKING: Static test token exists:', this.captchaTokens.has('test-static-token'));
-        console.log('TOKENS_TRACKING: Trying to verify token:', token ? `${token.substring(0, 8) }...` : 'undefined');
-
-        console.log('verifyCaptcha called with token:', token ? `${token.substring(0, 8) }...` : 'undefined');
-        console.log('userAnswer:', userAnswer);
-        console.log('Service enabled:', this.enabled);
-        console.log('Number of tokens in captchaTokens:', this.captchaTokens.size);
+        console.debug('====== CAPTCHA SERVICE VERIFICATION DIAGNOSTIC ======');
+        console.debug('TOKENS_TRACKING: verifyCaptcha called. Service ID:', this.serviceId);
+        console.debug('TOKENS_TRACKING: Request counter during verification:', this.requestCounter);
+        console.debug('TOKENS_TRACKING: Static test token exists:', this.captchaTokens.has('test-static-token'));
+        console.debug('TOKENS_TRACKING: Trying to verify token:', token ? `${token.substring(0, 8) }...` : 'undefined');
+        console.debug('verifyCaptcha called with token:', token ? `${token.substring(0, 8) }...` : 'undefined');
+        console.debug('userAnswer:', userAnswer);
+        console.debug('Service enabled:', this.enabled);
+        console.debug('Number of tokens in captchaTokens:', this.captchaTokens.size);
 
         // Service health check
         this._checkServiceHealth();

src/backend/src/modules/core/LogService.js

@@ -255,7 +255,7 @@ class WinstonLogger {
     constructor (winst) {
         this.winst = winst;
     }
-    onLogMessage (log_lvl, crumbs, message, fields, objects) {
+    onLogMessage (log_lvl, crumbs, message, fields) {
         this.winst.log({
             ...fields,
             label: crumbs.join('.'),
@@ -336,7 +336,7 @@ class CustomLogger {
                 args: a,
             });
         } catch (e) {
-            console.error('error?', e);
+            console.error(e);
         }
 
         if ( ret && ret.skip ) return;
@@ -404,7 +404,7 @@ class LogService extends BaseService {
             {
                 id: 'show',
                 description: 'toggle log output',
-                handler: async (args, log) => {
+                handler: async () => {
                     this.devlogger && (this.devlogger.off = !this.devlogger.off);
                 },
             },
@@ -423,7 +423,7 @@ class LogService extends BaseService {
             {
                 id: 'stop',
                 description: 'stop recording to a file via dev logger',
-                handler: async ([name], log) => {
+                handler: async ([_name], log) => {
                     if ( ! this.devlogger ) {
                         log('no dev logger; what are you doing?');
                     }
@@ -433,7 +433,7 @@ class LogService extends BaseService {
             {
                 id: 'indent',
                 description: 'toggle log indentation',
-                handler: async (args, log) => {
+                handler: async () => {
                     globalThis.dev_console_indent_on =
                         !globalThis.dev_console_indent_on;
                 },
@@ -481,21 +481,21 @@ class LogService extends BaseService {
                         filename: `${this.log_directory}/%DATE%.log`,
                         datePattern: 'YYYY-MM-DD',
                         maxSize: '20m',
-                        maxFiles: '14d',
+                        maxFiles: '2d',
                     }),
                     new winston.transports.DailyRotateFile({
                         level: 'error',
                         filename: `${this.log_directory}/error-%DATE%.log`,
                         datePattern: 'YYYY-MM-DD',
                         maxSize: '20m',
-                        maxFiles: '14d',
+                        maxFiles: '2d',
                     }),
                     new winston.transports.DailyRotateFile({
                         level: 'system',
                         filename: `${this.log_directory}/system-%DATE%.log`,
                         datePattern: 'YYYY-MM-DD',
                         maxSize: '20m',
-                        maxFiles: '14d',
+                        maxFiles: '2d',
                     }),
                 ],
             }));

src/backend/src/modules/core/ServerHealthService.js

@@ -17,6 +17,7 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 const BaseService = require('../../services/BaseService');
+const { kv } = require('../../util/kvSingleton');
 const { promise } = require('@heyputer/putility').libs;
 const SECOND = 1000;
 
@@ -202,10 +203,10 @@ class ServerHealthService extends BaseService {
     add_check (name, fn) {
         const chainable = {
             on_fail_: [],
-        };
-        chainable.on_fail = (fn) => {
-            chainable.on_fail_.push(fn);
-            return chainable;
+            on_fail: (fn) => {
+                chainable.on_fail_.push(fn);
+                return chainable;
+            },
         };
         this.checks_.push({ name, fn, chainable });
         return chainable;
@@ -221,27 +222,27 @@ class ServerHealthService extends BaseService {
     */
     async get_status () {
         const cache_key = 'server-health:status';
-        
+
         // Check cache first
-        if ( globalThis.kv ) {
-            const cached = globalThis.kv.get(cache_key);
+        if ( kv ) {
+            const cached = kv.get(cache_key);
             if ( cached ) {
                 return cached;
             }
         }
-        
+
         // Compute status
         const failures = this.failures_.map(v => v.name);
         const status = {
             ok: failures.length === 0,
             ...(failures.length ? { failed: failures } : {}),
         };
-        
+
         // Cache with 30 second TTL
-        if ( globalThis.kv ) {
-            globalThis.kv.set(cache_key, status, { EX: 30 });
+        if ( kv ) {
+            kv.set(cache_key, status, { EX: 5 });
         }
-        
+
         return status;
     }
 }

src/backend/src/modules/development/LocalTerminalService.js

@@ -101,7 +101,7 @@ class LocalTerminalService extends BaseService {
                     const svc_socketio = req.services.get('socketio');
                     proc.stdout.on('data', data => {
                         const base64 = data.toString('base64');
-                        console.log('---------------------- CHUNK?', base64);
+                        console.debug('---------------------- CHUNK?', base64);
                         svc_socketio.send({ room: req.user.id },
                                         'local-terminal.stdout',
                                         {
@@ -111,7 +111,7 @@ class LocalTerminalService extends BaseService {
                     });
                     proc.stderr.on('data', data => {
                         const base64 = data.toString('base64');
-                        console.log('---------------------- CHUNK?', base64);
+                        console.debug('---------------------- CHUNK?', base64);
                         svc_socketio.send({ room: req.user.id },
                                         'local-terminal.stderr',
                                         {

src/backend/src/modules/perfmon/PerfMonModule.js

@@ -1,39 +0,0 @@
-/*
- * Copyright (C) 2024-present Puter Technologies Inc.
- *
- * This file is part of Puter.
- *
- * Puter is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published
- * by the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-const { AdvancedBase } = require('@heyputer/putility');
-
-/**
- * Enable this module when you want performance monitoring.
- *
- * Performance monitoring requires additional setup. Jaegar should be installed
- * and running.
- */
-class PerfMonModule extends AdvancedBase {
-    async install (context) {
-        const services = context.get('services');
-
-        const { TelemetryService } = require('./TelemetryService');
-        services.registerService('telemetry', TelemetryService);
-    }
-}
-
-module.exports = {
-    PerfMonModule,
-};

src/backend/src/modules/perfmon/TelemetryService.js

@@ -24,51 +24,36 @@ import { Resource } from '@opentelemetry/resources';
 import { ConsoleMetricExporter, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { NodeSDK } from '@opentelemetry/sdk-node';
 import { ConsoleSpanExporter } from '@opentelemetry/sdk-trace-base';
-import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
+import { SemanticAttributes, SemanticResourceAttributes } from '@opentelemetry/semantic-conventions';
 import config from '../../config.js';
 import BaseService from '../../services/BaseService.js';
 
 export class TelemetryService extends BaseService {
+    static TRACER_NAME = 'puter-tracer';
+    static #sharedSdk = null;
+    static #sharedTracer = null;
+    static #telemetryStarted = false;
+
     /** @type {import('@opentelemetry/api').Tracer} */
     #tracer = null;
-    _construct () {
 
-        const traceExporter = this.#getConfiguredExporter();
-        const metricExporter = this.#getMetricExporter();
-
-        if ( !traceExporter && !metricExporter ) {
-            console.log('TelemetryService not configured, skipping initialization.');
-            return;
-        }
-
-        const resource = Resource.default().merge(
-                        new Resource({
-                            [SemanticResourceAttributes.SERVICE_NAME]: 'puter-backend',
-                            [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0',
-                        }));
-
-        const sdk = new NodeSDK({
-            resource,
-            traceExporter: traceExporter,
-            metricReader: new PeriodicExportingMetricReader({
-                exporter: metricExporter,
-            }),
-            instrumentations: [getNodeAutoInstrumentations()],
+    constructor (service_resources, ...args) {
+        super(service_resources, ...args);
+        const { sdk, tracer } = TelemetryService.#startTelemetry({
+            serviceConfig: this.config,
         });
-
         this.sdk = sdk;
-
-        this.sdk.start();
-
-        this.#tracer = trace.getTracer('puter-tracer');
-
+        this.#tracer = tracer;
     }
 
     _init () {
         if ( ! this.#tracer ) {
             return;
         }
-        const svc_context = this.services.get('context');
+        const svc_context = this.services.get('context', { optional: true });
+        if ( ! svc_context ) {
+            return;
+        }
         svc_context.register_context_hook('pre_arun', ({ hints, trace_name, callback, replace_callback }) => {
             if ( ! trace_name ) return;
             if ( ! hints.trace ) return;
@@ -87,21 +72,138 @@ export class TelemetryService extends BaseService {
         });
     }
 
-    #getConfiguredExporter () {
-        if ( config.jaeger ?? this.config.jaeger ) {
-            return new OTLPTraceExporter(config.jaeger ?? this.config.jaeger);
+    static #normalizeRoute (route) {
+        if ( Array.isArray(route) ) {
+            for ( const entry of route ) {
+                if ( typeof entry === 'string' ) {
+                    return entry;
+                }
+            }
+            return undefined;
         }
-        if ( this.config.console ) {
+        if ( typeof route === 'string' ) {
+            return route;
+        }
+        if ( route instanceof RegExp ) {
+            return route.toString();
+        }
+    }
+
+    static #buildRoute (req, route) {
+        const normalized = TelemetryService.#normalizeRoute(route);
+        if ( ! normalized ) {
+            return undefined;
+        }
+        const baseUrl = typeof req?.baseUrl === 'string' ? req.baseUrl : '';
+        const combined = `${baseUrl}${normalized}`;
+        return combined || normalized;
+    }
+
+    static #applyRouteToSpan (span, req, route) {
+        if ( ! route ) {
+            return;
+        }
+        span.setAttribute(SemanticAttributes.HTTP_ROUTE, route);
+        if ( typeof span.updateName === 'function' && req?.method ) {
+            span.updateName(`HTTP ${req.method} ${route}`);
+        }
+    }
+
+    static #buildInstrumentationConfig () {
+        return {
+            '@opentelemetry/instrumentation-http': {
+                responseHook: (span, response) => {
+                    const req = response?.req;
+                    const route = TelemetryService.#buildRoute(req, req?.route?.path);
+                    TelemetryService.#applyRouteToSpan(span, req, route);
+                },
+            },
+            '@opentelemetry/instrumentation-express': {
+                spanNameHook: (info, defaultName) => {
+                    if ( info.layerType !== 'request_handler' ) {
+                        return defaultName;
+                    }
+                    const route = TelemetryService.#buildRoute(info.request, info.route);
+                    if ( !route || !info.request?.method ) {
+                        return defaultName;
+                    }
+                    return `HTTP ${info.request.method} ${route}`;
+                },
+                requestHook: (span, info) => {
+                    const route = TelemetryService.#buildRoute(info.request, info.route);
+                    if ( route ) {
+                        span.setAttribute(SemanticAttributes.HTTP_ROUTE, route);
+                    }
+                },
+            },
+        };
+    }
+
+    static #resolveExporterConfig (serviceConfig) {
+        return config.jaeger ?? serviceConfig?.jaeger;
+    }
+
+    static #getConfiguredExporter (serviceConfig) {
+        const exporterConfig = TelemetryService.#resolveExporterConfig(serviceConfig);
+        if ( exporterConfig ) {
+            return new OTLPTraceExporter(exporterConfig);
+        }
+        if ( serviceConfig?.console ) {
             return new ConsoleSpanExporter();
         }
     }
 
-    #getMetricExporter () {
-        if ( config.jaeger ?? this.config.jaeger ) {
-            return new OTLPMetricExporter(config.jaeger ?? this.config.jaeger);
+    static #getMetricExporter (serviceConfig) {
+        const exporterConfig = TelemetryService.#resolveExporterConfig(serviceConfig);
+        if ( exporterConfig ) {
+            return new OTLPMetricExporter(exporterConfig);
         }
-        if ( this.config.console ) {
+        if ( serviceConfig?.console ) {
             return new ConsoleMetricExporter();
         }
     }
-}
+
+    static #startTelemetry ({ serviceConfig } = {}) {
+        if ( TelemetryService.#telemetryStarted ) {
+            return { sdk: TelemetryService.#sharedSdk, tracer: TelemetryService.#sharedTracer };
+        }
+        TelemetryService.#telemetryStarted = true;
+
+        const effectiveConfig = serviceConfig ?? config.services?.telemetry ?? {};
+        const traceExporter = TelemetryService.#getConfiguredExporter(effectiveConfig);
+        const metricExporter = TelemetryService.#getMetricExporter(effectiveConfig);
+
+        if ( !traceExporter && !metricExporter ) {
+            console.log('TelemetryService not configured, skipping initialization.');
+            return { sdk: null, tracer: null };
+        }
+
+        const resource = Resource.default().merge(
+                        new Resource({
+                            [SemanticResourceAttributes.SERVICE_NAME]: 'puter-backend',
+                            [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0',
+                        }));
+
+        const sdkConfig = {
+            resource,
+            instrumentations: [
+                getNodeAutoInstrumentations(TelemetryService.#buildInstrumentationConfig()),
+            ],
+        };
+
+        if ( traceExporter ) {
+            sdkConfig.traceExporter = traceExporter;
+        }
+        if ( metricExporter ) {
+            sdkConfig.metricReader = new PeriodicExportingMetricReader({
+                exporter: metricExporter,
+            });
+        }
+
+        TelemetryService.#sharedSdk = new NodeSDK(sdkConfig);
+        TelemetryService.#sharedSdk.start();
+        TelemetryService.#sharedTracer = trace.getTracer(TelemetryService.TRACER_NAME);
+
+        return { sdk: TelemetryService.#sharedSdk, tracer: TelemetryService.#sharedTracer };
+    }
+}

src/backend/src/services/BaseService.d.ts

@@ -1,14 +1,19 @@
+import type { ServerHealthService } from '../modules/core/ServerHealthService';
 import { SqliteDatabaseAccessService } from './database/SqliteDatabaseAccessService';
-import type { MeteringService } from './MeteringService/MeteringService';
 import { MeteringServiceWrapper } from './MeteringService/MeteringServiceWrapper.mjs';
+import { DDBClient } from './repositories/DDBClient';
 import { DynamoKVStore } from './repositories/DynamoKVStore/DynamoKVStore';
+import type { SUService } from './SUService';
 
 export interface ServiceResources {
     services: {
         get (name: 'meteringService'): MeteringServiceWrapper;
         get (name: 'puter-kvstore'): DynamoKVStore;
-        get (name: 'database'): SqliteDatabaseAccessService
-        get (name: string): any
+        get (name: 'database'): SqliteDatabaseAccessService;
+        get (name: 'server-health'): ServerHealthService;
+        get (name: 'su'): SUService;
+        get (name: 'dynamo'): DDBClient;
+        get (name: string): any;
     };
     config: Record<string, any> & { services?: Record<string, any>; server_id?: string };
     name?: string;

src/backend/src/services/abuse-prevention/IdentificationService.js

@@ -20,7 +20,11 @@ const { AdvancedBase } = require('@heyputer/putility');
 const BaseService = require('../BaseService');
 const { Context } = require('../../util/context');
 const config = require('../../config');
-
+const isBot = require('isbot');
+const IGNORED_BOT_UAS = new Set([
+    'Amazon-Route53-Health-Check-Service (ref 7599f3a9-c2af-43ac-a4b6-299da2e3861c; report http://amzn.to/1vsZADi)',
+    'Better Stack Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36',
+]);
 /**
 * @class Requester
 * @classdesc This class represents a requester in the system. It encapsulates
@@ -127,9 +131,6 @@ class Requester {
 * The class uses the 'isbot' module to determine if the requester is a bot.
 */
 class RequesterIdentificationExpressMiddleware extends AdvancedBase {
-    static MODULES = {
-        isbot: require('isbot'),
-    };
     register_initializer (initializer) {
         this.value_initializers_.push(initializer);
     }
@@ -140,14 +141,18 @@ class RequesterIdentificationExpressMiddleware extends AdvancedBase {
         const x = Context.get();
 
         const requester = Requester.from_request(req);
-        const is_bot = this.modules.isbot(requester.ua);
+        const is_bot = isBot(requester.ua);
         requester.is_bot = is_bot;
 
         x.set('requester', requester);
         req.requester = requester;
 
         if ( requester.is_bot ) {
-            this.log.info('bot detected', requester.serialize());
+            if ( IGNORED_BOT_UAS.has(requester.ua) ) {
+                this.log.info('healthcheck bot:', requester.serialize());
+            } else {
+                this.log.info('bot detected', requester.serialize());
+            }
         }
 
         next();

src/backend/src/services/repositories/DDBClient.ts

@@ -16,14 +16,14 @@ interface DBClientConfig {
 }
 
 export class DDBClient {
-    ddbClient: Promise<DynamoDBClient>;
+    ddbClientPromise: Promise<DynamoDBClient>;
     #documentClient!: DynamoDBDocumentClient;
     config?: DBClientConfig;
 
     constructor (config?: DBClientConfig) {
         this.config = config;
-        this.ddbClient = this.#getClient();
-        this.ddbClient.then(client => {
+        this.ddbClientPromise = this.#getClient();
+        this.ddbClientPromise.then(client => {
             this.#documentClient = DynamoDBDocumentClient.from(client, {
                 marshallOptions: {
                     removeUndefinedValues: true,
@@ -31,6 +31,14 @@ export class DDBClient {
         });
     }
 
+    async recreateClient () {
+        this.ddbClientPromise = this.#getClient();
+        this.#documentClient = DynamoDBDocumentClient.from(await this.ddbClientPromise, {
+            marshallOptions: {
+                removeUndefinedValues: true,
+            } });
+    }
+
     async #getClient () {
         if ( ! this.config?.aws ) {
             console.warn('No config for DynamoDB, will fall back on local dynalite');

src/backend/src/services/repositories/DDBClientWrapper.ts

@@ -7,7 +7,7 @@ class DDBClientServiceWrapper extends BaseService {
     async _construct () {
         this.ddbClient = new DDBClient(this.config as unknown as ConstructorParameters<typeof DDBClient>[0]);
 
-        await this.ddbClient.ddbClient; // ensure client is ready
+        await this.ddbClient.ddbClientPromise; // ensure client is ready
 
         Object.getOwnPropertyNames(DDBClient.prototype).forEach(fn => {
             if ( fn === 'constructor' ) return;

src/backend/src/services/repositories/DynamoKVStore/DynamoKVStoreWrapper.ts

@@ -20,6 +20,30 @@ class DynamoKVStoreServiceWrapper extends BaseService {
             this[fn] = (...args: unknown[]) => this.kvStore[fn](...args);
         });
     }
+
+    async registerHealthcheck () {
+        const healthcheckService = this.services.get('server-health');
+
+        healthcheckService.add_check('kv-store', async () => {
+            try {
+                const passed = await this.services.get('su').sudo(async () => {
+                    const rand = Math.floor(Math.random() * 1000000);
+                    await this.kvStore.set({ key: 'healthTestKey', value: rand });
+                    const setRight = await this.kvStore.get({ key: 'healthTestKey' }) === rand;
+                    await this.kvStore.del({ key: 'healthTestKey' });
+                    return setRight;
+                });
+                if ( ! passed ) {
+                    throw new Error('KV Store healthcheck failed: set/get mismatch');
+                }
+            } catch (e) {
+                throw new Error(`KV Store healthcheck failed: ${(e as Error).message}`);
+            }
+        }).on_fail(async () => {
+            await this.services.get('dynamo').recreateClient();
+        });
+    }
+
     static IMPLEMENTS = {
         ['puter-kvstore']: Object.getOwnPropertyNames(DynamoKVStore.prototype)
             .filter(n => n !== 'constructor')

src/backend/src/services/thumbnails/HTTPThumbnailService.js

@@ -349,9 +349,6 @@ class HTTPThumbnailService extends BaseService {
 
         const results = resp.data;
 
-        console.debug('response?', { resp });
-        console.debug('data?', { data: resp.data });
-
         if ( results.length !== queue.length ) {
             this.log.error('Thumbnail service returned wrong number of results');
             throw new Error('Thumbnail service returned wrong number of results');
@@ -393,9 +390,7 @@ class HTTPThumbnailService extends BaseService {
         }
 
         const form = new FormData();
-        let expected = 0;
         for ( const job of queue ) {
-            expected++;
 
             /**
             * Prepares and sends a request to the thumbnail service for processing multiple files.

tools/run-selfhosted.js

@@ -98,7 +98,6 @@ const main = async () => {
         InternetModule,
         DevelopmentModule,
         DNSModule,
-        PerfMonModule,
         DataAccessModule,
     } = (await import('@heyputer/backend')).default;
 
@@ -118,7 +117,6 @@ const main = async () => {
     k.add_module(new PuterAIModule());
     k.add_module(new InternetModule());
     k.add_module(new DNSModule());
-    k.add_module(new PerfMonModule());
     if ( process.env.UNSAFE_PUTER_DEV ) {
         k.add_module(new DevelopmentModule());
     }