diff --git a/extensions/extensionController/src/ExtensionController.ts b/extensions/extensionController/src/ExtensionController.ts index f28f711b4..70340d67d 100644 --- a/extensions/extensionController/src/ExtensionController.ts +++ b/extensions/extensionController/src/ExtensionController.ts @@ -96,8 +96,10 @@ export class HttpError extends Error { // Registers all routes from a decorated controller instance to an Express router export class ExtensionController { + logger?: Console; // TODO DS: make this work with other express-like routers registerRoutes () { + const logger = this.logger || console; const prefix = Object.getPrototypeOf(this).__controllerPrefix || ''; const adminsForController = Object.getPrototypeOf(this).__adminUsernames as | string[] @@ -126,7 +128,7 @@ export class ExtensionController { if ( ! extension[route.method] ) { throw new Error(`Unsupported HTTP method: ${route.method}`); } else { - console.log(`Registering route: [${route.method.toUpperCase()}] ${fullPath}`); + logger.log(`Registering route: [${route.method.toUpperCase()}] ${fullPath}`); (extension[route.method] as RouterMethods[HttpMethod])( fullPath, @@ -154,20 +156,16 @@ export class ExtensionController { } catch ( error ) { if ( error instanceof HttpError ) { res.status(error.statusCode).send({ error: error.message }); - console.error('httpError:', error); + logger.warn('httpError:', error); return; } if ( error instanceof Error ) { - res - .status(StatusCodes.INTERNAL_SERVER_ERROR) - .send({ error: error.message }); - console.error('Non-http error:', error); + res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: error.message }); + logger.error('Non-http error:', error); return; } - res - .status(StatusCodes.INTERNAL_SERVER_ERROR) - .send({ error: 'An unknown error occurred' }); - console.error('An unknown error occurred:', error); + res.status(StatusCodes.INTERNAL_SERVER_ERROR).send({ error: 'An unknown error occurred' }); + logger.error('An unknown error occurred:', error); } }); } diff --git a/src/backend/exports.js b/src/backend/exports.js index ca03ebe1e..5f191db06 100644 --- a/src/backend/exports.js +++ b/src/backend/exports.js @@ -35,7 +35,6 @@ import { EntityStoreModule } from './src/modules/entitystore/EntityStoreModule.j import { HostOSModule } from './src/modules/hostos/HostOSModule.js'; import { InternetModule } from './src/modules/internet/InternetModule.js'; import { KVStoreModule } from './src/modules/kvstore/KVStoreModule.js'; -import { PerfMonModule } from './src/modules/perfmon/PerfMonModule.js'; import { PuterFSModule } from './src/modules/puterfs/PuterFSModule.js'; import SelfHostedModule from './src/modules/selfhosted/SelfHostedModule.js'; import { TestConfigModule } from './src/modules/test-config/TestConfigModule.js'; @@ -89,6 +88,5 @@ export default { DataAccessModule, // Development modules - PerfMonModule, DevelopmentModule, }; diff --git a/src/backend/src/CoreModule.js b/src/backend/src/CoreModule.js index dae143fd0..a6e52cd05 100644 --- a/src/backend/src/CoreModule.js +++ b/src/backend/src/CoreModule.js @@ -35,6 +35,10 @@ const query = require('./om/query/query'); */ const install = async ({ context, services, app, useapi, modapi }) => { const config = require('./config'); + const { TelemetryService } = require('./modules/perfmon/TelemetryService'); + if ( ! services.has('telemetry') ) { + services.registerService('telemetry', TelemetryService); + } // === LIBRARIES === diff --git a/src/backend/src/helpers.js b/src/backend/src/helpers.js index d5313c329..8d2203d57 100644 --- a/src/backend/src/helpers.js +++ b/src/backend/src/helpers.js @@ -276,9 +276,7 @@ async function refresh_apps_cache (options, override) { async function refresh_associations_cache () { /** @type BaseDatabaseAccessService */ const db = _servicesHolder.services.get('database').get(DB_READ, 'apps'); - - const log = _servicesHolder.services.get('log-service').create('helpers.js'); - log.tick('refresh file associations'); + console.debug('refresh file associations'); const associations = await db.read('SELECT * FROM app_filetype_association'); const lists = {}; for ( const association of associations ) { @@ -356,6 +354,7 @@ async function get_app (options) { const pendingKey = `pending_app:${queryKey}`; const pending = kv.get(pendingKey); if ( pending ) { + // Reuse the existing pending query const result = await pending; // shallow clone the result return result ? { ...result } : null; diff --git a/src/backend/src/modules/broadcast/BroadcastService.js b/src/backend/src/modules/broadcast/BroadcastService.js index a7fd23c43..0ae3dc853 100644 --- a/src/backend/src/modules/broadcast/BroadcastService.js +++ b/src/backend/src/modules/broadcast/BroadcastService.js @@ -90,7 +90,7 @@ class BroadcastService extends BaseService { } if ( key === 'test' ) { - this.log.noticeme(`test message: ${ + console.debug(`test message: ${ JSON.stringify(data)}`); } @@ -108,7 +108,7 @@ class BroadcastService extends BaseService { { id: 'test', description: 'send a test message', - handler: async (args, ctx) => { + handler: async () => { this.on_event('test', { contents: 'I am a test message', }, {}); diff --git a/src/backend/src/modules/captcha/services/CaptchaService.js b/src/backend/src/modules/captcha/services/CaptchaService.js index 84129a035..3919f50ce 100644 --- a/src/backend/src/modules/captcha/services/CaptchaService.js +++ b/src/backend/src/modules/captcha/services/CaptchaService.js @@ -404,16 +404,15 @@ class CaptchaService extends BaseService { * @returns {boolean} Whether the answer is valid */ verifyCaptcha (token, userAnswer) { - console.log('====== CAPTCHA SERVICE VERIFICATION DIAGNOSTIC ======'); - console.log('TOKENS_TRACKING: verifyCaptcha called. Service ID:', this.serviceId); - console.log('TOKENS_TRACKING: Request counter during verification:', this.requestCounter); - console.log('TOKENS_TRACKING: Static test token exists:', this.captchaTokens.has('test-static-token')); - console.log('TOKENS_TRACKING: Trying to verify token:', token ? `${token.substring(0, 8) }...` : 'undefined'); - - console.log('verifyCaptcha called with token:', token ? `${token.substring(0, 8) }...` : 'undefined'); - console.log('userAnswer:', userAnswer); - console.log('Service enabled:', this.enabled); - console.log('Number of tokens in captchaTokens:', this.captchaTokens.size); + console.debug('====== CAPTCHA SERVICE VERIFICATION DIAGNOSTIC ======'); + console.debug('TOKENS_TRACKING: verifyCaptcha called. Service ID:', this.serviceId); + console.debug('TOKENS_TRACKING: Request counter during verification:', this.requestCounter); + console.debug('TOKENS_TRACKING: Static test token exists:', this.captchaTokens.has('test-static-token')); + console.debug('TOKENS_TRACKING: Trying to verify token:', token ? `${token.substring(0, 8) }...` : 'undefined'); + console.debug('verifyCaptcha called with token:', token ? `${token.substring(0, 8) }...` : 'undefined'); + console.debug('userAnswer:', userAnswer); + console.debug('Service enabled:', this.enabled); + console.debug('Number of tokens in captchaTokens:', this.captchaTokens.size); // Service health check this._checkServiceHealth(); diff --git a/src/backend/src/modules/core/LogService.js b/src/backend/src/modules/core/LogService.js index fabb645db..86980640d 100644 --- a/src/backend/src/modules/core/LogService.js +++ b/src/backend/src/modules/core/LogService.js @@ -255,7 +255,7 @@ class WinstonLogger { constructor (winst) { this.winst = winst; } - onLogMessage (log_lvl, crumbs, message, fields, objects) { + onLogMessage (log_lvl, crumbs, message, fields) { this.winst.log({ ...fields, label: crumbs.join('.'), @@ -336,7 +336,7 @@ class CustomLogger { args: a, }); } catch (e) { - console.error('error?', e); + console.error(e); } if ( ret && ret.skip ) return; @@ -404,7 +404,7 @@ class LogService extends BaseService { { id: 'show', description: 'toggle log output', - handler: async (args, log) => { + handler: async () => { this.devlogger && (this.devlogger.off = !this.devlogger.off); }, }, @@ -423,7 +423,7 @@ class LogService extends BaseService { { id: 'stop', description: 'stop recording to a file via dev logger', - handler: async ([name], log) => { + handler: async ([_name], log) => { if ( ! this.devlogger ) { log('no dev logger; what are you doing?'); } @@ -433,7 +433,7 @@ class LogService extends BaseService { { id: 'indent', description: 'toggle log indentation', - handler: async (args, log) => { + handler: async () => { globalThis.dev_console_indent_on = !globalThis.dev_console_indent_on; }, @@ -481,21 +481,21 @@ class LogService extends BaseService { filename: `${this.log_directory}/%DATE%.log`, datePattern: 'YYYY-MM-DD', maxSize: '20m', - maxFiles: '14d', + maxFiles: '2d', }), new winston.transports.DailyRotateFile({ level: 'error', filename: `${this.log_directory}/error-%DATE%.log`, datePattern: 'YYYY-MM-DD', maxSize: '20m', - maxFiles: '14d', + maxFiles: '2d', }), new winston.transports.DailyRotateFile({ level: 'system', filename: `${this.log_directory}/system-%DATE%.log`, datePattern: 'YYYY-MM-DD', maxSize: '20m', - maxFiles: '14d', + maxFiles: '2d', }), ], })); diff --git a/src/backend/src/modules/core/ServerHealthService.js b/src/backend/src/modules/core/ServerHealthService.js index 64f32da01..de14204e5 100644 --- a/src/backend/src/modules/core/ServerHealthService.js +++ b/src/backend/src/modules/core/ServerHealthService.js @@ -17,6 +17,7 @@ * along with this program. If not, see . */ const BaseService = require('../../services/BaseService'); +const { kv } = require('../../util/kvSingleton'); const { promise } = require('@heyputer/putility').libs; const SECOND = 1000; @@ -202,10 +203,10 @@ class ServerHealthService extends BaseService { add_check (name, fn) { const chainable = { on_fail_: [], - }; - chainable.on_fail = (fn) => { - chainable.on_fail_.push(fn); - return chainable; + on_fail: (fn) => { + chainable.on_fail_.push(fn); + return chainable; + }, }; this.checks_.push({ name, fn, chainable }); return chainable; @@ -221,27 +222,27 @@ class ServerHealthService extends BaseService { */ async get_status () { const cache_key = 'server-health:status'; - + // Check cache first - if ( globalThis.kv ) { - const cached = globalThis.kv.get(cache_key); + if ( kv ) { + const cached = kv.get(cache_key); if ( cached ) { return cached; } } - + // Compute status const failures = this.failures_.map(v => v.name); const status = { ok: failures.length === 0, ...(failures.length ? { failed: failures } : {}), }; - + // Cache with 30 second TTL - if ( globalThis.kv ) { - globalThis.kv.set(cache_key, status, { EX: 30 }); + if ( kv ) { + kv.set(cache_key, status, { EX: 5 }); } - + return status; } } diff --git a/src/backend/src/modules/development/LocalTerminalService.js b/src/backend/src/modules/development/LocalTerminalService.js index ec5ff81ee..8a1f93cf2 100644 --- a/src/backend/src/modules/development/LocalTerminalService.js +++ b/src/backend/src/modules/development/LocalTerminalService.js @@ -101,7 +101,7 @@ class LocalTerminalService extends BaseService { const svc_socketio = req.services.get('socketio'); proc.stdout.on('data', data => { const base64 = data.toString('base64'); - console.log('---------------------- CHUNK?', base64); + console.debug('---------------------- CHUNK?', base64); svc_socketio.send({ room: req.user.id }, 'local-terminal.stdout', { @@ -111,7 +111,7 @@ class LocalTerminalService extends BaseService { }); proc.stderr.on('data', data => { const base64 = data.toString('base64'); - console.log('---------------------- CHUNK?', base64); + console.debug('---------------------- CHUNK?', base64); svc_socketio.send({ room: req.user.id }, 'local-terminal.stderr', { diff --git a/src/backend/src/modules/perfmon/PerfMonModule.js b/src/backend/src/modules/perfmon/PerfMonModule.js deleted file mode 100644 index 8c53eaa32..000000000 --- a/src/backend/src/modules/perfmon/PerfMonModule.js +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -const { AdvancedBase } = require('@heyputer/putility'); - -/** - * Enable this module when you want performance monitoring. - * - * Performance monitoring requires additional setup. Jaegar should be installed - * and running. - */ -class PerfMonModule extends AdvancedBase { - async install (context) { - const services = context.get('services'); - - const { TelemetryService } = require('./TelemetryService'); - services.registerService('telemetry', TelemetryService); - } -} - -module.exports = { - PerfMonModule, -}; diff --git a/src/backend/src/modules/perfmon/TelemetryService.js b/src/backend/src/modules/perfmon/TelemetryService.js index 3af18867e..2dfa1fc13 100644 --- a/src/backend/src/modules/perfmon/TelemetryService.js +++ b/src/backend/src/modules/perfmon/TelemetryService.js @@ -24,51 +24,36 @@ import { Resource } from '@opentelemetry/resources'; import { ConsoleMetricExporter, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; import { NodeSDK } from '@opentelemetry/sdk-node'; import { ConsoleSpanExporter } from '@opentelemetry/sdk-trace-base'; -import { SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; +import { SemanticAttributes, SemanticResourceAttributes } from '@opentelemetry/semantic-conventions'; import config from '../../config.js'; import BaseService from '../../services/BaseService.js'; export class TelemetryService extends BaseService { + static TRACER_NAME = 'puter-tracer'; + static #sharedSdk = null; + static #sharedTracer = null; + static #telemetryStarted = false; + /** @type {import('@opentelemetry/api').Tracer} */ #tracer = null; - _construct () { - const traceExporter = this.#getConfiguredExporter(); - const metricExporter = this.#getMetricExporter(); - - if ( !traceExporter && !metricExporter ) { - console.log('TelemetryService not configured, skipping initialization.'); - return; - } - - const resource = Resource.default().merge( - new Resource({ - [SemanticResourceAttributes.SERVICE_NAME]: 'puter-backend', - [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0', - })); - - const sdk = new NodeSDK({ - resource, - traceExporter: traceExporter, - metricReader: new PeriodicExportingMetricReader({ - exporter: metricExporter, - }), - instrumentations: [getNodeAutoInstrumentations()], + constructor (service_resources, ...args) { + super(service_resources, ...args); + const { sdk, tracer } = TelemetryService.#startTelemetry({ + serviceConfig: this.config, }); - this.sdk = sdk; - - this.sdk.start(); - - this.#tracer = trace.getTracer('puter-tracer'); - + this.#tracer = tracer; } _init () { if ( ! this.#tracer ) { return; } - const svc_context = this.services.get('context'); + const svc_context = this.services.get('context', { optional: true }); + if ( ! svc_context ) { + return; + } svc_context.register_context_hook('pre_arun', ({ hints, trace_name, callback, replace_callback }) => { if ( ! trace_name ) return; if ( ! hints.trace ) return; @@ -87,21 +72,138 @@ export class TelemetryService extends BaseService { }); } - #getConfiguredExporter () { - if ( config.jaeger ?? this.config.jaeger ) { - return new OTLPTraceExporter(config.jaeger ?? this.config.jaeger); + static #normalizeRoute (route) { + if ( Array.isArray(route) ) { + for ( const entry of route ) { + if ( typeof entry === 'string' ) { + return entry; + } + } + return undefined; } - if ( this.config.console ) { + if ( typeof route === 'string' ) { + return route; + } + if ( route instanceof RegExp ) { + return route.toString(); + } + } + + static #buildRoute (req, route) { + const normalized = TelemetryService.#normalizeRoute(route); + if ( ! normalized ) { + return undefined; + } + const baseUrl = typeof req?.baseUrl === 'string' ? req.baseUrl : ''; + const combined = `${baseUrl}${normalized}`; + return combined || normalized; + } + + static #applyRouteToSpan (span, req, route) { + if ( ! route ) { + return; + } + span.setAttribute(SemanticAttributes.HTTP_ROUTE, route); + if ( typeof span.updateName === 'function' && req?.method ) { + span.updateName(`HTTP ${req.method} ${route}`); + } + } + + static #buildInstrumentationConfig () { + return { + '@opentelemetry/instrumentation-http': { + responseHook: (span, response) => { + const req = response?.req; + const route = TelemetryService.#buildRoute(req, req?.route?.path); + TelemetryService.#applyRouteToSpan(span, req, route); + }, + }, + '@opentelemetry/instrumentation-express': { + spanNameHook: (info, defaultName) => { + if ( info.layerType !== 'request_handler' ) { + return defaultName; + } + const route = TelemetryService.#buildRoute(info.request, info.route); + if ( !route || !info.request?.method ) { + return defaultName; + } + return `HTTP ${info.request.method} ${route}`; + }, + requestHook: (span, info) => { + const route = TelemetryService.#buildRoute(info.request, info.route); + if ( route ) { + span.setAttribute(SemanticAttributes.HTTP_ROUTE, route); + } + }, + }, + }; + } + + static #resolveExporterConfig (serviceConfig) { + return config.jaeger ?? serviceConfig?.jaeger; + } + + static #getConfiguredExporter (serviceConfig) { + const exporterConfig = TelemetryService.#resolveExporterConfig(serviceConfig); + if ( exporterConfig ) { + return new OTLPTraceExporter(exporterConfig); + } + if ( serviceConfig?.console ) { return new ConsoleSpanExporter(); } } - #getMetricExporter () { - if ( config.jaeger ?? this.config.jaeger ) { - return new OTLPMetricExporter(config.jaeger ?? this.config.jaeger); + static #getMetricExporter (serviceConfig) { + const exporterConfig = TelemetryService.#resolveExporterConfig(serviceConfig); + if ( exporterConfig ) { + return new OTLPMetricExporter(exporterConfig); } - if ( this.config.console ) { + if ( serviceConfig?.console ) { return new ConsoleMetricExporter(); } } -} \ No newline at end of file + + static #startTelemetry ({ serviceConfig } = {}) { + if ( TelemetryService.#telemetryStarted ) { + return { sdk: TelemetryService.#sharedSdk, tracer: TelemetryService.#sharedTracer }; + } + TelemetryService.#telemetryStarted = true; + + const effectiveConfig = serviceConfig ?? config.services?.telemetry ?? {}; + const traceExporter = TelemetryService.#getConfiguredExporter(effectiveConfig); + const metricExporter = TelemetryService.#getMetricExporter(effectiveConfig); + + if ( !traceExporter && !metricExporter ) { + console.log('TelemetryService not configured, skipping initialization.'); + return { sdk: null, tracer: null }; + } + + const resource = Resource.default().merge( + new Resource({ + [SemanticResourceAttributes.SERVICE_NAME]: 'puter-backend', + [SemanticResourceAttributes.SERVICE_VERSION]: '0.1.0', + })); + + const sdkConfig = { + resource, + instrumentations: [ + getNodeAutoInstrumentations(TelemetryService.#buildInstrumentationConfig()), + ], + }; + + if ( traceExporter ) { + sdkConfig.traceExporter = traceExporter; + } + if ( metricExporter ) { + sdkConfig.metricReader = new PeriodicExportingMetricReader({ + exporter: metricExporter, + }); + } + + TelemetryService.#sharedSdk = new NodeSDK(sdkConfig); + TelemetryService.#sharedSdk.start(); + TelemetryService.#sharedTracer = trace.getTracer(TelemetryService.TRACER_NAME); + + return { sdk: TelemetryService.#sharedSdk, tracer: TelemetryService.#sharedTracer }; + } +} diff --git a/src/backend/src/services/BaseService.d.ts b/src/backend/src/services/BaseService.d.ts index 649a9e989..cb0c6e30d 100644 --- a/src/backend/src/services/BaseService.d.ts +++ b/src/backend/src/services/BaseService.d.ts @@ -1,14 +1,19 @@ +import type { ServerHealthService } from '../modules/core/ServerHealthService'; import { SqliteDatabaseAccessService } from './database/SqliteDatabaseAccessService'; -import type { MeteringService } from './MeteringService/MeteringService'; import { MeteringServiceWrapper } from './MeteringService/MeteringServiceWrapper.mjs'; +import { DDBClient } from './repositories/DDBClient'; import { DynamoKVStore } from './repositories/DynamoKVStore/DynamoKVStore'; +import type { SUService } from './SUService'; export interface ServiceResources { services: { get (name: 'meteringService'): MeteringServiceWrapper; get (name: 'puter-kvstore'): DynamoKVStore; - get (name: 'database'): SqliteDatabaseAccessService - get (name: string): any + get (name: 'database'): SqliteDatabaseAccessService; + get (name: 'server-health'): ServerHealthService; + get (name: 'su'): SUService; + get (name: 'dynamo'): DDBClient; + get (name: string): any; }; config: Record & { services?: Record; server_id?: string }; name?: string; diff --git a/src/backend/src/services/abuse-prevention/IdentificationService.js b/src/backend/src/services/abuse-prevention/IdentificationService.js index aea41d288..50f5ab9a5 100644 --- a/src/backend/src/services/abuse-prevention/IdentificationService.js +++ b/src/backend/src/services/abuse-prevention/IdentificationService.js @@ -20,7 +20,11 @@ const { AdvancedBase } = require('@heyputer/putility'); const BaseService = require('../BaseService'); const { Context } = require('../../util/context'); const config = require('../../config'); - +const isBot = require('isbot'); +const IGNORED_BOT_UAS = new Set([ + 'Amazon-Route53-Health-Check-Service (ref 7599f3a9-c2af-43ac-a4b6-299da2e3861c; report http://amzn.to/1vsZADi)', + 'Better Stack Better Uptime Bot Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36', +]); /** * @class Requester * @classdesc This class represents a requester in the system. It encapsulates @@ -127,9 +131,6 @@ class Requester { * The class uses the 'isbot' module to determine if the requester is a bot. */ class RequesterIdentificationExpressMiddleware extends AdvancedBase { - static MODULES = { - isbot: require('isbot'), - }; register_initializer (initializer) { this.value_initializers_.push(initializer); } @@ -140,14 +141,18 @@ class RequesterIdentificationExpressMiddleware extends AdvancedBase { const x = Context.get(); const requester = Requester.from_request(req); - const is_bot = this.modules.isbot(requester.ua); + const is_bot = isBot(requester.ua); requester.is_bot = is_bot; x.set('requester', requester); req.requester = requester; if ( requester.is_bot ) { - this.log.info('bot detected', requester.serialize()); + if ( IGNORED_BOT_UAS.has(requester.ua) ) { + this.log.info('healthcheck bot:', requester.serialize()); + } else { + this.log.info('bot detected', requester.serialize()); + } } next(); diff --git a/src/backend/src/services/repositories/DDBClient.ts b/src/backend/src/services/repositories/DDBClient.ts index 3daea797b..6c1a7e05c 100644 --- a/src/backend/src/services/repositories/DDBClient.ts +++ b/src/backend/src/services/repositories/DDBClient.ts @@ -16,14 +16,14 @@ interface DBClientConfig { } export class DDBClient { - ddbClient: Promise; + ddbClientPromise: Promise; #documentClient!: DynamoDBDocumentClient; config?: DBClientConfig; constructor (config?: DBClientConfig) { this.config = config; - this.ddbClient = this.#getClient(); - this.ddbClient.then(client => { + this.ddbClientPromise = this.#getClient(); + this.ddbClientPromise.then(client => { this.#documentClient = DynamoDBDocumentClient.from(client, { marshallOptions: { removeUndefinedValues: true, @@ -31,6 +31,14 @@ export class DDBClient { }); } + async recreateClient () { + this.ddbClientPromise = this.#getClient(); + this.#documentClient = DynamoDBDocumentClient.from(await this.ddbClientPromise, { + marshallOptions: { + removeUndefinedValues: true, + } }); + } + async #getClient () { if ( ! this.config?.aws ) { console.warn('No config for DynamoDB, will fall back on local dynalite'); diff --git a/src/backend/src/services/repositories/DDBClientWrapper.ts b/src/backend/src/services/repositories/DDBClientWrapper.ts index 3d48f3440..ed8825783 100644 --- a/src/backend/src/services/repositories/DDBClientWrapper.ts +++ b/src/backend/src/services/repositories/DDBClientWrapper.ts @@ -7,7 +7,7 @@ class DDBClientServiceWrapper extends BaseService { async _construct () { this.ddbClient = new DDBClient(this.config as unknown as ConstructorParameters[0]); - await this.ddbClient.ddbClient; // ensure client is ready + await this.ddbClient.ddbClientPromise; // ensure client is ready Object.getOwnPropertyNames(DDBClient.prototype).forEach(fn => { if ( fn === 'constructor' ) return; diff --git a/src/backend/src/services/repositories/DynamoKVStore/DynamoKVStoreWrapper.ts b/src/backend/src/services/repositories/DynamoKVStore/DynamoKVStoreWrapper.ts index 8008402fc..4c5e22259 100644 --- a/src/backend/src/services/repositories/DynamoKVStore/DynamoKVStoreWrapper.ts +++ b/src/backend/src/services/repositories/DynamoKVStore/DynamoKVStoreWrapper.ts @@ -20,6 +20,30 @@ class DynamoKVStoreServiceWrapper extends BaseService { this[fn] = (...args: unknown[]) => this.kvStore[fn](...args); }); } + + async registerHealthcheck () { + const healthcheckService = this.services.get('server-health'); + + healthcheckService.add_check('kv-store', async () => { + try { + const passed = await this.services.get('su').sudo(async () => { + const rand = Math.floor(Math.random() * 1000000); + await this.kvStore.set({ key: 'healthTestKey', value: rand }); + const setRight = await this.kvStore.get({ key: 'healthTestKey' }) === rand; + await this.kvStore.del({ key: 'healthTestKey' }); + return setRight; + }); + if ( ! passed ) { + throw new Error('KV Store healthcheck failed: set/get mismatch'); + } + } catch (e) { + throw new Error(`KV Store healthcheck failed: ${(e as Error).message}`); + } + }).on_fail(async () => { + await this.services.get('dynamo').recreateClient(); + }); + } + static IMPLEMENTS = { ['puter-kvstore']: Object.getOwnPropertyNames(DynamoKVStore.prototype) .filter(n => n !== 'constructor') diff --git a/src/backend/src/services/thumbnails/HTTPThumbnailService.js b/src/backend/src/services/thumbnails/HTTPThumbnailService.js index 15368daaa..01f306016 100644 --- a/src/backend/src/services/thumbnails/HTTPThumbnailService.js +++ b/src/backend/src/services/thumbnails/HTTPThumbnailService.js @@ -349,9 +349,6 @@ class HTTPThumbnailService extends BaseService { const results = resp.data; - console.debug('response?', { resp }); - console.debug('data?', { data: resp.data }); - if ( results.length !== queue.length ) { this.log.error('Thumbnail service returned wrong number of results'); throw new Error('Thumbnail service returned wrong number of results'); @@ -393,9 +390,7 @@ class HTTPThumbnailService extends BaseService { } const form = new FormData(); - let expected = 0; for ( const job of queue ) { - expected++; /** * Prepares and sends a request to the thumbnail service for processing multiple files. diff --git a/tools/run-selfhosted.js b/tools/run-selfhosted.js index dce5b7003..fdeb55347 100644 --- a/tools/run-selfhosted.js +++ b/tools/run-selfhosted.js @@ -98,7 +98,6 @@ const main = async () => { InternetModule, DevelopmentModule, DNSModule, - PerfMonModule, DataAccessModule, } = (await import('@heyputer/backend')).default; @@ -118,7 +117,6 @@ const main = async () => { k.add_module(new PuterAIModule()); k.add_module(new InternetModule()); k.add_module(new DNSModule()); - k.add_module(new PerfMonModule()); if ( process.env.UNSAFE_PUTER_DEV ) { k.add_module(new DevelopmentModule()); }