fix: handle closing server nicer on shutdown signal (#2741)

* fix: handle closing server nicer on shutdown signal * fix: bad check
2026-05-04 08:30:39 +00:00 · 2026-03-27 17:24:33 -07:00
parent 89a67a4a6f
commit b3656fdaa1
13 changed files with 209 additions and 101 deletions
@@ -60,15 +60,6 @@ interface DriverInterface {

 export type HttpMethod = 'get' | 'post' | 'put' | 'delete' | 'patch';

-export type ExtensionRequestHandler = RequestHandler<
-    Record<string, string | undefined>,
-    unknown,
-    unknown
->;
-export type ExtensionRequest = Parameters<ExtensionRequestHandler>[0];
-export type ExtensionResponse = Parameters<ExtensionRequestHandler>[1];
-export type ExtensionNextFunction = Parameters<ExtensionRequestHandler>[2];
-
 export type AddRouteFunction = (
    path: string,
    options: EndpointOptions,
@@ -1,4 +1,4 @@
-import type { RequestHandler } from 'express';
+import type { NextFunction, Request, Response } from 'express';
 import { StatusCodes } from 'http-status-codes';
 import type {
    EndpointOptions,
@@ -31,7 +31,7 @@ interface RouteMeta {
    method: HttpMethod;
    path: string;
    options?: EndpointOptions | undefined;
-    handler: RequestHandler;
+    handler: (req: Request, res: Response, next: NextFunction) => void | Promise<void>;
    adminUsernames?: string[];
    allowedAppIds?: string[];
 }
@@ -44,13 +44,13 @@ const createMethodDecorator = (method: HttpMethod) => {
    ) => {
        const { allowedAppIds, ...options } = routeOptions ?? {};
        return (
-            target: RequestHandler<any, any, any, any, any>,
+            target: (req: Request, res: Response, next: NextFunction) => void | Promise<void>,
            _context: ClassMethodDecoratorContext<
                This,
                (
                    this: This,
-                    ...args: Parameters<RequestHandler<any, any, any, any, any>>
-                ) => ReturnType<RequestHandler<any, any, any, any, any>>
+                    ...args: [req: Request, res: Response, next: NextFunction]
+                ) => void | Promise<void>
            >,
        ) => {
            _context.addInitializer(function () {
@@ -2,9 +2,9 @@
 import type { BaseDatabaseAccessService } from '@heyputer/backend/src/services/database/BaseDatabaseAccessService.js';
 import type { MeteringService } from '@heyputer/backend/src/services/MeteringService/MeteringService.js';
 import type {
-    ExtensionRequest,
-    ExtensionResponse,
-} from '../../api.d.ts';
+    Request,
+    Response,
+} from 'express';

 const { Controller, Get, ExtensionController } = extension.import('extensionController');

@@ -23,7 +23,7 @@ export class UsageController extends ExtensionController {
    }

    @Get('usage', { subdomain: 'api' })
-    async getUsage (req: ExtensionRequest, res: ExtensionResponse) {
+    async getUsage (req: Request, res: Response) {
        const actor = req.actor;
        if ( ! actor ) {
            throw Error('actor not found in context');
@@ -40,7 +40,7 @@ export class UsageController extends ExtensionController {
    }

    @Get('usage/:appIdOrName', { subdomain: 'api' })
-    async getUsageByApp (req: ExtensionRequest, res: ExtensionResponse) {
+    async getUsageByApp (req: Request, res: Response) {
        const actor = req.actor;
        if ( ! actor ) {
            throw Error('actor not found in context');
@@ -83,7 +83,7 @@ export class UsageController extends ExtensionController {
    }

    @Get('globalUsage', { subdomain: 'api' }, extension.config.allowedGlobalUsageUsers || [])
-    async getGlobalUsage (req: ExtensionRequest, res: ExtensionResponse) {
+    async getGlobalUsage (req: Request, res: Response) {
        const actor = req.actor;
        if ( ! actor ) {
            throw Error('actor not found in context');
@@ -1,16 +1,16 @@
 /* global config, extension */
+import type {
+    Request,
+    Response,
+} from 'express';
 import fs from 'fs/promises';
 import os from 'os';
-import type {
-    ExtensionRequest,
-    ExtensionResponse,
-} from '../api.d.ts';
 const { Controller, Get, ExtensionController } = extension.import('extensionController');

@Controller('/serverInfo', [...config.allowedUsernames])
 class ServerInfoController extends ExtensionController {
    @Get('', { subdomain: 'api' })
-    async getServerInfo (_req: ExtensionRequest, res: ExtensionResponse) {
+    async getServerInfo (_req: Request, res: Response) {
        const osData = {
            platform: os.platform(),
            type: os.type(),
@@ -27,6 +27,7 @@
        "ai": "^6.0.73",
        "dedent": "^1.5.3",
        "dynalite": "^4.0.0",
+        "express": "^4.18.2",
        "express-xml-bodyparser": "^0.4.1",
        "file-type": "21.3.3",
        "javascript-time-ago": "^2.5.11",
@@ -41,6 +42,7 @@
        "@eslint/js": "^9.35.0",
        "@playwright/test": "^1.56.1",
        "@stylistic/eslint-plugin": "^5.3.1",
+        "@types/express": "^4.17.21",
        "@types/mime-types": "^3.0.1",
        "@types/uuid": "^10.0.0",
        "@typescript-eslint/eslint-plugin": "^8.46.1",
@@ -52,7 +54,6 @@
        "dotenv": "^16.4.5",
        "eslint": "^9.35.0",
        "eslint-rule-composer": "^0.3.0",
-        "express": "^4.18.2",
        "globals": "^15.15.0",
        "html-entities": "^2.3.3",
        "html-webpack-plugin": "^5.6.0",
@@ -7965,20 +7966,21 @@
      "license": "MIT"
    },
    "node_modules/@types/express": {
-      "version": "5.0.6",
-      "resolved": "https://registry.npmjs.org/@types/express/-/express-5.0.6.tgz",
-      "integrity": "sha512-sKYVuV7Sv9fbPIt/442koC7+IIwK5olP1KWeD88e/idgoJqDm3JV/YUiPwkoKK92ylff2MGxSz1CSjsXelx0YA==",
+      "version": "4.17.25",
+      "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz",
+      "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==",
      "license": "MIT",
      "dependencies": {
        "@types/body-parser": "*",
-        "@types/express-serve-static-core": "^5.0.0",
-        "@types/serve-static": "^2"
+        "@types/express-serve-static-core": "^4.17.33",
+        "@types/qs": "*",
+        "@types/serve-static": "^1"
      }
    },
    "node_modules/@types/express-serve-static-core": {
-      "version": "5.1.1",
-      "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-5.1.1.tgz",
-      "integrity": "sha512-v4zIMr/cX7/d2BpAEX3KNKL/JrT1s43s96lLvvdTmza1oEvDudCqK9aF/djc/SWgy8Yh0h30TZx5VpzqFCxk5A==",
+      "version": "4.19.8",
+      "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz",
+      "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==",
      "license": "MIT",
      "dependencies": {
        "@types/node": "*",
@@ -8267,12 +8269,23 @@
      }
    },
    "node_modules/@types/serve-static": {
-      "version": "2.2.0",
-      "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-2.2.0.tgz",
-      "integrity": "sha512-8mam4H1NHLtu7nmtalF7eyBH14QyOASmcxHhSfEoRyr0nP/YdoesEtU+uSRvMe96TW/HPTtkoKqQLl53N7UXMQ==",
+      "version": "1.15.10",
+      "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz",
+      "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==",
      "license": "MIT",
      "dependencies": {
        "@types/http-errors": "*",
+        "@types/node": "*",
+        "@types/send": "<1"
+      }
+    },
+    "node_modules/@types/serve-static/node_modules/@types/send": {
+      "version": "0.17.6",
+      "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz",
+      "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==",
+      "license": "MIT",
+      "dependencies": {
+        "@types/mime": "^1",
        "@types/node": "*"
      }
    },
@@ -15568,51 +15581,6 @@
        "node": ">=10 < 13 || >=14"
      }
    },
-    "node_modules/jwks-rsa/node_modules/@types/express": {
-      "version": "4.17.25",
-      "resolved": "https://registry.npmjs.org/@types/express/-/express-4.17.25.tgz",
-      "integrity": "sha512-dVd04UKsfpINUnK0yBoYHDF3xu7xVH4BuDotC/xGuycx4CgbP48X/KF/586bcObxT0HENHXEU8Nqtu6NR+eKhw==",
-      "license": "MIT",
-      "dependencies": {
-        "@types/body-parser": "*",
-        "@types/express-serve-static-core": "^4.17.33",
-        "@types/qs": "*",
-        "@types/serve-static": "^1"
-      }
-    },
-    "node_modules/jwks-rsa/node_modules/@types/express-serve-static-core": {
-      "version": "4.19.8",
-      "resolved": "https://registry.npmjs.org/@types/express-serve-static-core/-/express-serve-static-core-4.19.8.tgz",
-      "integrity": "sha512-02S5fmqeoKzVZCHPZid4b8JH2eM5HzQLZWN2FohQEy/0eXTq8VXZfSN6Pcr3F6N9R/vNrj7cpgbhjie6m/1tCA==",
-      "license": "MIT",
-      "dependencies": {
-        "@types/node": "*",
-        "@types/qs": "*",
-        "@types/range-parser": "*",
-        "@types/send": "*"
-      }
-    },
-    "node_modules/jwks-rsa/node_modules/@types/send": {
-      "version": "0.17.6",
-      "resolved": "https://registry.npmjs.org/@types/send/-/send-0.17.6.tgz",
-      "integrity": "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og==",
-      "license": "MIT",
-      "dependencies": {
-        "@types/mime": "^1",
-        "@types/node": "*"
-      }
-    },
-    "node_modules/jwks-rsa/node_modules/@types/serve-static": {
-      "version": "1.15.10",
-      "resolved": "https://registry.npmjs.org/@types/serve-static/-/serve-static-1.15.10.tgz",
-      "integrity": "sha512-tRs1dB+g8Itk72rlSI2ZrW6vZg0YrLI81iQSTkMmOqnqCaNr/8Ek4VwWcN5vZgCYWbg/JJSGBlUaYGAOP73qBw==",
-      "license": "MIT",
-      "dependencies": {
-        "@types/http-errors": "*",
-        "@types/node": "*",
-        "@types/send": "<1"
-      }
-    },
    "node_modules/jws": {
      "version": "4.0.1",
      "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz",
@@ -25,7 +25,7 @@
    "dotenv": "^16.4.5",
    "eslint": "^9.35.0",
    "eslint-rule-composer": "^0.3.0",
-    "express": "^4.18.2",
+    "@types/express": "^4.17.21",
    "globals": "^15.15.0",
    "html-entities": "^2.3.3",
    "html-webpack-plugin": "^5.6.0",
@@ -92,7 +92,8 @@
    "open": "^10.1.0",
    "parse-domain": "^8.2.2",
    "string-template": "^1.0.0",
-    "uuid": "^9.0.1"
+    "uuid": "^9.0.1",
+    "express": "^4.18.2"
  },
  "optionalDependencies": {
    "sharp": "^0.34.4",
@@ -102,4 +103,4 @@
  "engines": {
    "node": ">=24.0.0"
  }
-}
+}
@@ -43,6 +43,8 @@ class AlarmService extends BaseService {
        this.alarm_aliases = {};

        this.known_errors = [];
+        this.isDraining = false;
+        this.drainSuppressionLogged = false;
    }
    /**
    * Method to initialize AlarmService. Sets the known errors and registers commands.
@@ -76,6 +78,12 @@ class AlarmService extends BaseService {
        return id;
    }

+    beginDrain (reason = 'shutdown') {
+        if ( this.isDraining ) return;
+        this.isDraining = true;
+        this.log.info(`alarm service entering drain mode: ${reason}`);
+    }
+
    /**
     * Method to create an alarm with the given ID, message, and fields.
     * If the ID already exists, it will be updated with the new fields
@@ -86,6 +94,14 @@ class AlarmService extends BaseService {
     * @param {object} fields - Additional information about the alarm.
     */
    create (id, message, fields) {
+        if ( this.isDraining ) {
+            if ( ! this.drainSuppressionLogged ) {
+                this.drainSuppressionLogged = true;
+                this.log.info('suppressing alarm create/pager dispatch while draining');
+            }
+            return;
+        }
+
        if ( this.config.log_upcoming_alarms ) {
            this.log.error(`upcoming alarm: ${id}: ${message}`);
        }
@@ -232,8 +248,10 @@ class AlarmService extends BaseService {
    }

    handle_alarm_repeat_ (alarm) {
-        this.log.warn(`REPEAT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,
-                        alarm.fields);
+        this.log.warn(
+            `REPEAT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,
+            alarm.fields,
+        );

        this.apply_known_errors_(alarm);

@@ -260,8 +278,10 @@ class AlarmService extends BaseService {
    }

    handle_alarm_on_ (alarm) {
-        this.log.error(`ACTIVE ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,
-                        alarm.fields);
+        this.log.error(
+            `ACTIVE ${alarm.id_string} :: ${alarm.message} (${alarm.count})`,
+            alarm.fields,
+        );

        this.apply_known_errors_(alarm);

@@ -326,8 +346,10 @@ class AlarmService extends BaseService {
    }

    handle_alarm_off_ (alarm) {
-        this.log.info(`CLEAR ${alarm.id} :: ${alarm.message} (${alarm.count})`,
-                        alarm.fields);
+        this.log.info(
+            `CLEAR ${alarm.id} :: ${alarm.message} (${alarm.count})`,
+            alarm.fields,
+        );
    }

    /**
@@ -54,6 +54,7 @@ class ServerHealthService extends BaseService {
        this.last_check_cycle_started_at_ = 0;
        this.last_check_cycle_completed_at_ = 0;
        this.web_checks_registered_ = false;
+        this.isDraining_ = false;
    }

    async _init () {
@@ -67,6 +68,18 @@ class ServerHealthService extends BaseService {
        this.#registerWebChecks();
    }

+    beginDrain (reason = 'shutdown') {
+        if ( this.isDraining_ ) return;
+        this.isDraining_ = true;
+        this.failures_ = [];
+        this.last_check_cycle_completed_at_ = Date.now();
+        this.stats_ = this.stats_ ?? {};
+        this.stats_.last_check_cycle_completed_at = this.last_check_cycle_completed_at_;
+        this.stats_.check_durations_ms = {};
+        this.stats_.failed_checks = [];
+        this.log.info(`server health entering drain mode: ${reason}`);
+    }
+
    #initDefaultChecks () {
        const dbService = this.#getServiceIfAvailable('database');
        if ( dbService && typeof dbService.read === 'function' ) {
@@ -148,9 +161,14 @@ class ServerHealthService extends BaseService {
        * @returns {void}
        */
        promise.asyncSafeSetInterval(async () => {
-            this.last_check_cycle_started_at_ = Date.now();
-            this.stats_.last_check_cycle_started_at = this.last_check_cycle_started_at_;
-            this.log.tick('service checks');
+            if ( this.isDraining_ ) {
+                this.last_check_cycle_completed_at_ = Date.now();
+                this.stats_.last_check_cycle_completed_at = this.last_check_cycle_completed_at_;
+                this.stats_.check_durations_ms = {};
+                this.stats_.failed_checks = [];
+                return;
+            }
+
            const check_failures = [];
            const check_durations_ms = {};
            for ( const { name, fn, chainable } of this.checks_ ) {
@@ -245,6 +263,13 @@ class ServerHealthService extends BaseService {
    * - `failed` {Array<string>}: An array of names of failed health checks, if any.
    */
    async get_status () {
+        if ( this.isDraining_ ) {
+            return {
+                ok: false,
+                failed: ['draining'],
+            };
+        }
+
        const cacheKey = ServerHealthRedisCacheKeys.status;

        // Check cache first
@@ -66,6 +66,11 @@ class WebServerService extends BaseService {
    };

    allowedRoutesWithUndefinedOrigins = [];
+    isDraining = false;
+    shutdownStarted = false;
+    shutdownForceExitTimer = null;
+    shutdownCloseTimer = null;
+    gracefulShutdownHandlersInstalled = false;

    allow_undefined_origin (route) {
        this.allowedRoutesWithUndefinedOrigins.push(route);
@@ -252,7 +257,8 @@ class WebServerService extends BaseService {
        server.timeout = 1000 * 60 * 60 * 2; // 2 hours
        server.requestTimeout = 1000 * 60 * 60 * 2; // 2 hours
        server.headersTimeout = 1000 * 60 * 60 * 2; // 2 hours
-        // server.keepAliveTimeout = 1000 * 60 * 60 * 2; // 2 hours
+        const albIdleTimeoutMs = 1000 * 60 * 5;
+        server.keepAliveTimeout = albIdleTimeoutMs + (1000 * 15);

        // Socket.io server instance
        // const socketio = require('../../socketio.js').init(server);
@@ -318,6 +324,7 @@ class WebServerService extends BaseService {
        });

        this.server_ = server;
+        this.registerGracefulShutdownHandlers();
        await this.services.emit('install.websockets');
    }

@@ -331,6 +338,87 @@ class WebServerService extends BaseService {
        return this.server_;
    }

+    registerGracefulShutdownHandlers () {
+        if ( this.gracefulShutdownHandlersInstalled ) return;
+        this.gracefulShutdownHandlersInstalled = true;
+
+        process.on('SIGTERM', () => {
+            this.beginGracefulShutdown('SIGTERM');
+        });
+        process.on('SIGINT', () => {
+            this.beginGracefulShutdown('SIGINT');
+        });
+    }
+
+    beginGracefulShutdown (signal) {
+        if ( this.shutdownStarted ) return;
+        this.shutdownStarted = true;
+        this.isDraining = true;
+        this.app?.set('isDraining', true);
+        this.drainCoreServicesForShutdown(signal);
+        const albFailoverDelayMs = 15 * 1000;
+
+        this.log.info(
+            `received ${signal}; beginning graceful shutdown with ${albFailoverDelayMs}ms ALB failover delay`,
+        );
+        const server = this.server_;
+        if ( ! server ) {
+            process.exit(0);
+            return;
+        }
+
+        this.shutdownForceExitTimer = setTimeout(() => {
+            this.log.error('graceful shutdown timed out; forcing process exit');
+            process.exit(1);
+        }, 110 * 1000);
+        if ( typeof this.shutdownForceExitTimer.unref === 'function' ) {
+            this.shutdownForceExitTimer.unref();
+        }
+
+        this.shutdownCloseTimer = setTimeout(() => {
+            this.shutdownCloseTimer = null;
+            if ( typeof server.closeIdleConnections === 'function' ) {
+                server.closeIdleConnections();
+            }
+
+            server.close((error) => {
+                if ( this.shutdownForceExitTimer ) {
+                    clearTimeout(this.shutdownForceExitTimer);
+                    this.shutdownForceExitTimer = null;
+                }
+
+                if ( error ) {
+                    this.log.error('error while closing HTTP server during shutdown', error);
+                    process.exit(1);
+                    return;
+                }
+
+                this.log.info('graceful shutdown completed');
+                process.exit(0);
+            });
+        }, albFailoverDelayMs);
+        if ( typeof this.shutdownCloseTimer.unref === 'function' ) {
+            this.shutdownCloseTimer.unref();
+        }
+    }
+
+    drainCoreServicesForShutdown (signal) {
+        const reason = `signal:${signal}`;
+        for ( const serviceName of ['alarm', 'server-health'] ) {
+            try {
+                const service = this.services.get(serviceName);
+                if ( typeof service?.beginDrain === 'function' ) {
+                    service.beginDrain(reason);
+                }
+            } catch ( error ) {
+                this.log.error(
+                    `failed to drain ${serviceName} during shutdown`,
+                    error,
+                );
+            }
+        }
+    }
+
    /**
    * Handles starting and managing the Puter web server.
    *
@@ -341,6 +429,7 @@ class WebServerService extends BaseService {
        this.app = app;

        app.set('services', this.services);
+        app.set('isDraining', false);

        this.middlewares = { auth };

@@ -479,6 +568,7 @@ class WebServerService extends BaseService {
                onFinished(res, () => {
                    if ( res.statusCode !== 500 ) return;
                    if ( req.__error_handled ) return;
+                    if ( req.path === '/healthcheck' ) return;
                    const alarm = this.services.get('alarm');
                    alarm.create('responded-500', 'server sent a 500 response', {
                        error: req.__error_source,
@@ -79,6 +79,14 @@ const send_health_status = async (req, res, { fail_with_http_error = false }) =>
 // GET /healthcheck
 // -----------------------------------------------------------------------//
 router.get('/healthcheck', async (req, res, next) => {
+    if ( req.app?.get('isDraining') ) {
+        res.status(503).json({
+            ok: false,
+            failed: ['draining'],
+        });
+        return;
+    }
+
    if ( isHostedDomainRequest(req) ) {
        next();
        return;
@@ -18,8 +18,9 @@ import type { MeteringService } from './MeteringService/MeteringService';
 import type { MeteringServiceWrapper } from './MeteringService/MeteringServiceWrapper.mjs';
 import type { SUService } from './SUService';
 import type { UserService } from './UserService';
-import { TokenService } from './auth/TokenService';
-import { SessionService } from './SessionService';
+import type { TokenService } from './auth/TokenService';
+import type { SessionService } from './SessionService';
+import type { PermissionService } from './auth/PermissionService';

 export interface ServicesMap {
    su: SUService;
@@ -44,6 +45,7 @@ export interface ServicesMap {
    driver: DriverService;
    'token': TokenService;
    'session': SessionService;
+    'permission': PermissionService;
 }

 export interface ServiceResources {
@@ -29,9 +29,10 @@ class DynamoKVStoreServiceWrapper extends BaseService {
            try {
                const passed = await this.services.get('su').sudo(async () => {
                    const rand = Math.floor(Math.random() * 1000000);
-                    await this.kvStore.set({ key: 'healthTestKey', value: rand });
-                    const setRight = await this.kvStore.get({ key: 'healthTestKey' }) === rand;
-                    await this.kvStore.del({ key: 'healthTestKey' });
+                    const randKey = `healthcheck-${Date.now()}-${rand}`;
+                    await this.kvStore.set({ key: randKey, value: rand });
+                    const setRight = await this.kvStore.get({ key: randKey }) === rand;
+                    await this.kvStore.del({ key: randKey });
                    return setRight;
                });
                if ( ! passed ) {
@@ -23,7 +23,7 @@ const { get_user } = require('../../helpers');
 const configurable_auth = require('../../middleware/configurable_auth');
 const { Context } = require('../../util/context');
 const { Endpoint } = require('../../util/expressutil');
-const BaseService = require('../BaseService');
+const { BaseService } = require('../BaseService');
 const { AppUnderUserActorType, UserActorType, Actor, SystemActorType, AccessTokenActorType } = require('./Actor');
 const { DB_READ } = require('../database/consts');
 const { MANAGE_PERM_PREFIX } = require('./permissionConts.mjs');