From 57f46e46f72a62d41c466e27c2a30e76bd80bf66 Mon Sep 17 00:00:00 2001 From: Neal Shah Date: Fri, 10 Jul 2026 16:38:16 -0400 Subject: [PATCH] add mark-degraded to healthcheck --- .../controllers/system/SystemController.js | 29 +++-- .../system/SystemController.test.ts | 104 ++++++++++++++---- .../services/health/ServerHealthService.ts | 69 +++++++----- 3 files changed, 145 insertions(+), 57 deletions(-) diff --git a/src/backend/controllers/system/SystemController.js b/src/backend/controllers/system/SystemController.js index d7ce74d4d..ab57da149 100644 --- a/src/backend/controllers/system/SystemController.js +++ b/src/backend/controllers/system/SystemController.js @@ -37,24 +37,33 @@ export class SystemController extends PuterController { ) { // -- Healthcheck --------------------------------------------- // Delegates to ServerHealthService for the real check-based - // status. Returns `{ ok: true }` when all registered checks pass, - // or `{ ok: false, failed: [...] }` + 503 when any fail or the + // status. Returns `{ ok: true }` + 200 when all registered checks + // pass, or `{ ok: false, failed: [...] }` + 503 when any fail or the // server is draining. + // + // `?ignore=a,b` disregards the named checks for this request only. + // `?marked-degraded=a,b` demotes the named checks to a non-fatal + // `degraded` list: `ok` stays true but the response is 207 so the + // caller can tell the node is running in a degraded state. + const parseNames = (value) => + typeof value === 'string' + ? value + .split(',') + .map((name) => name.trim()) + .filter(Boolean) + : []; router.get('/healthcheck', { subdomain: '*' }, async (req, res) => { const health = this.services.health; if (!health || typeof health.getStatus !== 'function') { // Fallback for boot ordering / missing service. return res.send('ok'); } - const ignore = - typeof req.query.ignore === 'string' - ? req.query.ignore - .split(',') - .map((name) => name.trim()) - .filter(Boolean) - : []; - const status = await health.getStatus(ignore); + const status = await health.getStatus({ + ignore: parseNames(req.query.ignore), + degrade: parseNames(req.query['marked-degraded']), + }); if (!status.ok) return res.status(503).json(status); + if (status.degraded?.length) return res.status(207).json(status); return res.json(status); }); diff --git a/src/backend/controllers/system/SystemController.test.ts b/src/backend/controllers/system/SystemController.test.ts index baa422d10..638cfeb63 100644 --- a/src/backend/controllers/system/SystemController.test.ts +++ b/src/backend/controllers/system/SystemController.test.ts @@ -141,7 +141,7 @@ describe('SystemController GET /healthcheck', () => { expect(captured.statusCode).toBe(200); }); - it('parses ?ignore into a trimmed name list passed to getStatus', async () => { + it('parses ?ignore and ?marked-degraded into trimmed name lists', async () => { const spy = vi .spyOn(server.services.health, 'getStatus') .mockResolvedValue({ ok: true }); @@ -150,13 +150,18 @@ describe('SystemController GET /healthcheck', () => { await callRoute( 'get', '/healthcheck', - makeReq({ query: { ignore: 'database-liveness, thumbnailer' } }), + makeReq({ + query: { + ignore: 'database-liveness, thumbnailer', + 'marked-degraded': ' socket-initialized ', + }, + }), res, ); - expect(spy).toHaveBeenCalledWith([ - 'database-liveness', - 'thumbnailer', - ]); + expect(spy).toHaveBeenCalledWith({ + ignore: ['database-liveness', 'thumbnailer'], + degrade: ['socket-initialized'], + }); } finally { spy.mockRestore(); } @@ -165,7 +170,7 @@ describe('SystemController GET /healthcheck', () => { it('returns ok:true + 200 when the only failures are ignored', async () => { const spy = vi .spyOn(server.services.health, 'getStatus') - .mockImplementation(async (ignore: string[] = []) => { + .mockImplementation(async ({ ignore = [] } = {}) => { const failed = ['database-liveness'].filter( (name) => !ignore.includes(name), ); @@ -188,10 +193,32 @@ describe('SystemController GET /healthcheck', () => { } }); + it('returns ok:true + 207 when the only failures are marked degraded', async () => { + const spy = vi + .spyOn(server.services.health, 'getStatus') + .mockResolvedValue({ ok: true, degraded: ['database-liveness'] }); + try { + const { res, captured } = makeRes(); + await callRoute( + 'get', + '/healthcheck', + makeReq({ query: { 'marked-degraded': 'database-liveness' } }), + res, + ); + expect(captured.body).toEqual({ + ok: true, + degraded: ['database-liveness'], + }); + expect(captured.statusCode).toBe(207); + } finally { + spy.mockRestore(); + } + }); + it('still 503s when a non-ignored failure remains', async () => { const spy = vi .spyOn(server.services.health, 'getStatus') - .mockImplementation(async (ignore: string[] = []) => { + .mockImplementation(async ({ ignore = [] } = {}) => { const failed = ['database-liveness', 'socket-initialized'].filter( (name) => !ignore.includes(name), ); @@ -218,13 +245,13 @@ describe('SystemController GET /healthcheck', () => { }); }); -// ── ServerHealthService.getStatus ignore filtering ────────────────── +// ── ServerHealthService.getStatus ignore / degrade filtering ──────── // // Exercises the real service against the live (mock) redis client by // seeding the status cache the service reads from, so the actual -// per-request ignore filtering runs — not a stubbed getStatus. +// per-request classification runs — not a stubbed getStatus. -describe('ServerHealthService.getStatus ignore filtering', () => { +describe('ServerHealthService.getStatus ignore/degrade filtering', () => { const STATUS_CACHE_KEY = 'server-health:status'; const seedStatus = async (status: unknown) => { @@ -238,34 +265,67 @@ describe('ServerHealthService.getStatus ignore filtering', () => { it('collapses to ok:true when every failure is ignored', async () => { await seedStatus({ ok: false, failed: ['database-liveness', 'thumbnailer'] }); - const status = await server.services.health.getStatus([ - 'database-liveness', - 'thumbnailer', - ]); + const status = await server.services.health.getStatus({ + ignore: ['database-liveness', 'thumbnailer'], + }); expect(status).toEqual({ ok: true }); }); it('keeps the non-ignored failures', async () => { await seedStatus({ ok: false, failed: ['database-liveness', 'thumbnailer'] }); - const status = await server.services.health.getStatus([ - 'database-liveness', - ]); + const status = await server.services.health.getStatus({ + ignore: ['database-liveness'], + }); expect(status).toEqual({ ok: false, failed: ['thumbnailer'] }); }); it('is a no-op for a healthy status', async () => { await seedStatus({ ok: true }); - const status = await server.services.health.getStatus([ - 'database-liveness', - ]); + const status = await server.services.health.getStatus({ + ignore: ['database-liveness'], + }); expect(status).toEqual({ ok: true }); }); it('ignores unknown names without affecting real failures', async () => { await seedStatus({ ok: false, failed: ['database-liveness'] }); - const status = await server.services.health.getStatus(['not-a-check']); + const status = await server.services.health.getStatus({ + ignore: ['not-a-check'], + }); expect(status).toEqual({ ok: false, failed: ['database-liveness'] }); }); + + it('demotes marked failures to degraded and stays ok:true', async () => { + await seedStatus({ ok: false, failed: ['database-liveness'] }); + const status = await server.services.health.getStatus({ + degrade: ['database-liveness'], + }); + expect(status).toEqual({ ok: true, degraded: ['database-liveness'] }); + }); + + it('reports degraded alongside remaining hard failures (ok:false)', async () => { + await seedStatus({ + ok: false, + failed: ['database-liveness', 'socket-initialized'], + }); + const status = await server.services.health.getStatus({ + degrade: ['database-liveness'], + }); + expect(status).toEqual({ + ok: false, + failed: ['socket-initialized'], + degraded: ['database-liveness'], + }); + }); + + it('lets ignore take precedence over degrade for the same name', async () => { + await seedStatus({ ok: false, failed: ['database-liveness'] }); + const status = await server.services.health.getStatus({ + ignore: ['database-liveness'], + degrade: ['database-liveness'], + }); + expect(status).toEqual({ ok: true }); + }); }); // ── /version ──────────────────────────────────────────────────────── diff --git a/src/backend/services/health/ServerHealthService.ts b/src/backend/services/health/ServerHealthService.ts index 0085db387..662abea0f 100644 --- a/src/backend/services/health/ServerHealthService.ts +++ b/src/backend/services/health/ServerHealthService.ts @@ -69,6 +69,17 @@ interface HealthStats { export interface HealthStatus { ok: boolean; failed?: string[]; + degraded?: string[]; +} + +export interface GetStatusOptions { + /** Failing check names to drop entirely (healthy if all failures ignored). */ + ignore?: string[]; + /** + * Failing check names to demote to non-fatal `degraded`. They don't make + * `ok` false, but their presence signals partial health to the caller. + */ + degrade?: string[]; } export class ServerHealthService extends PuterService { @@ -131,24 +142,21 @@ export class ServerHealthService extends PuterService { * Current health status. Results are cached in Redis for 5 seconds * so a busy /healthcheck endpoint doesn't hammer the DB on every hit. * - * `ignore` names a set of failing states to disregard for this request - * only, letting an orchestrator poll `/healthcheck` while tolerating - * specific known-failing checks. Any failure name may be ignored, - * including the `draining` lifecycle state. When the remaining failures - * are all ignored the status collapses back to `{ ok: true }`. The - * cached status is always the full, unfiltered set — filtering is - * applied per-request after the cache read so it never leaks across - * callers. + * `ignore` names failing states to disregard for this request only, + * letting an orchestrator poll `/healthcheck` while tolerating specific + * known-failing checks; when the remaining failures are all ignored the + * status collapses back to `{ ok: true }`. `degrade` instead demotes + * named failures to a non-fatal `degraded` list — `ok` stays true but + * the caller can see the partial state. Any failure name may be filtered + * this way, including the `draining` lifecycle state. The cached status + * is always the full, unfiltered set — filtering is applied per-request + * after the cache read so it never leaks across callers. */ - async getStatus(ignore: string[] = []): Promise { - if (this.#draining) { - return this.#applyIgnore( - { ok: false, failed: ['draining'] }, - ignore, - ); - } - const status = await this.#getCachedStatus(); - return this.#applyIgnore(status, ignore); + async getStatus(opts: GetStatusOptions = {}): Promise { + const base = this.#draining + ? { ok: false, failed: ['draining'] } + : await this.#getCachedStatus(); + return this.#applyFilters(base, opts.ignore ?? [], opts.degrade ?? []); } async #getCachedStatus(): Promise { @@ -192,18 +200,29 @@ export class ServerHealthService extends PuterService { } /** - * Drop `ignore`d check names from a status. If every failure is - * ignored the status becomes healthy again; otherwise the remaining - * failures are reported as usual. A healthy status is returned as-is. + * Reclassify a status against the per-request `ignore`/`degrade` sets. + * `ignore`d failures are dropped; `degrade`d failures move to a + * non-fatal `degraded` list; anything left stays a hard failure. `ok` + * is false only while hard failures remain. A healthy status is + * returned as-is. */ - #applyIgnore(status: HealthStatus, ignore: string[]): HealthStatus { - if (status.ok || ignore.length === 0 || !status.failed) return status; + #applyFilters( + status: HealthStatus, + ignore: string[], + degrade: string[], + ): HealthStatus { + if (status.ok || !status.failed) return status; + const remaining = status.failed.filter( (name) => !ignore.includes(name), ); - return remaining.length === 0 - ? { ok: true } - : { ok: false, failed: remaining }; + const degraded = remaining.filter((name) => degrade.includes(name)); + const failed = remaining.filter((name) => !degrade.includes(name)); + + const result: HealthStatus = { ok: failed.length === 0 }; + if (failed.length > 0) result.failed = failed; + if (degraded.length > 0) result.degraded = degraded; + return result; } #registerDefaultChecks(): void {