add mark-degraded to healthcheck

src/backend/controllers/system/SystemController.js

@@ -37,24 +37,33 @@ export class SystemController extends PuterController {
     ) {
         // -- Healthcheck ---------------------------------------------
         // Delegates to ServerHealthService for the real check-based
-        // status. Returns `{ ok: true }` when all registered checks pass,
-        // or `{ ok: false, failed: [...] }` + 503 when any fail or the
+        // status. Returns `{ ok: true }` + 200 when all registered checks
+        // pass, or `{ ok: false, failed: [...] }` + 503 when any fail or the
         // server is draining.
+        //
+        // `?ignore=a,b` disregards the named checks for this request only.
+        // `?marked-degraded=a,b` demotes the named checks to a non-fatal
+        // `degraded` list: `ok` stays true but the response is 207 so the
+        // caller can tell the node is running in a degraded state.
+        const parseNames = (value) =>
+            typeof value === 'string'
+                ? value
+                      .split(',')
+                      .map((name) => name.trim())
+                      .filter(Boolean)
+                : [];
         router.get('/healthcheck', { subdomain: '*' }, async (req, res) => {
             const health = this.services.health;
             if (!health || typeof health.getStatus !== 'function') {
                 // Fallback for boot ordering / missing service.
                 return res.send('ok');
             }
-            const ignore =
-                typeof req.query.ignore === 'string'
-                    ? req.query.ignore
-                          .split(',')
-                          .map((name) => name.trim())
-                          .filter(Boolean)
-                    : [];
-            const status = await health.getStatus(ignore);
+            const status = await health.getStatus({
+                ignore: parseNames(req.query.ignore),
+                degrade: parseNames(req.query['marked-degraded']),
+            });
             if (!status.ok) return res.status(503).json(status);
+            if (status.degraded?.length) return res.status(207).json(status);
             return res.json(status);
         });
 

src/backend/controllers/system/SystemController.test.ts

@@ -141,7 +141,7 @@ describe('SystemController GET /healthcheck', () => {
         expect(captured.statusCode).toBe(200);
     });
 
-    it('parses ?ignore into a trimmed name list passed to getStatus', async () => {
+    it('parses ?ignore and ?marked-degraded into trimmed name lists', async () => {
         const spy = vi
             .spyOn(server.services.health, 'getStatus')
             .mockResolvedValue({ ok: true });
@@ -150,13 +150,18 @@ describe('SystemController GET /healthcheck', () => {
             await callRoute(
                 'get',
                 '/healthcheck',
-                makeReq({ query: { ignore: 'database-liveness, thumbnailer' } }),
+                makeReq({
+                    query: {
+                        ignore: 'database-liveness, thumbnailer',
+                        'marked-degraded': ' socket-initialized ',
+                    },
+                }),
                 res,
             );
-            expect(spy).toHaveBeenCalledWith([
-                'database-liveness',
-                'thumbnailer',
-            ]);
+            expect(spy).toHaveBeenCalledWith({
+                ignore: ['database-liveness', 'thumbnailer'],
+                degrade: ['socket-initialized'],
+            });
         } finally {
             spy.mockRestore();
         }
@@ -165,7 +170,7 @@ describe('SystemController GET /healthcheck', () => {
     it('returns ok:true + 200 when the only failures are ignored', async () => {
         const spy = vi
             .spyOn(server.services.health, 'getStatus')
-            .mockImplementation(async (ignore: string[] = []) => {
+            .mockImplementation(async ({ ignore = [] } = {}) => {
                 const failed = ['database-liveness'].filter(
                     (name) => !ignore.includes(name),
                 );
@@ -188,10 +193,32 @@ describe('SystemController GET /healthcheck', () => {
         }
     });
 
+    it('returns ok:true + 207 when the only failures are marked degraded', async () => {
+        const spy = vi
+            .spyOn(server.services.health, 'getStatus')
+            .mockResolvedValue({ ok: true, degraded: ['database-liveness'] });
+        try {
+            const { res, captured } = makeRes();
+            await callRoute(
+                'get',
+                '/healthcheck',
+                makeReq({ query: { 'marked-degraded': 'database-liveness' } }),
+                res,
+            );
+            expect(captured.body).toEqual({
+                ok: true,
+                degraded: ['database-liveness'],
+            });
+            expect(captured.statusCode).toBe(207);
+        } finally {
+            spy.mockRestore();
+        }
+    });
+
     it('still 503s when a non-ignored failure remains', async () => {
         const spy = vi
             .spyOn(server.services.health, 'getStatus')
-            .mockImplementation(async (ignore: string[] = []) => {
+            .mockImplementation(async ({ ignore = [] } = {}) => {
                 const failed = ['database-liveness', 'socket-initialized'].filter(
                     (name) => !ignore.includes(name),
                 );
@@ -218,13 +245,13 @@ describe('SystemController GET /healthcheck', () => {
     });
 });
 
-// ── ServerHealthService.getStatus ignore filtering ──────────────────
+// ── ServerHealthService.getStatus ignore / degrade filtering ────────
 //
 // Exercises the real service against the live (mock) redis client by
 // seeding the status cache the service reads from, so the actual
-// per-request ignore filtering runs — not a stubbed getStatus.
+// per-request classification runs — not a stubbed getStatus.
 
-describe('ServerHealthService.getStatus ignore filtering', () => {
+describe('ServerHealthService.getStatus ignore/degrade filtering', () => {
     const STATUS_CACHE_KEY = 'server-health:status';
 
     const seedStatus = async (status: unknown) => {
@@ -238,34 +265,67 @@ describe('ServerHealthService.getStatus ignore filtering', () => {
 
     it('collapses to ok:true when every failure is ignored', async () => {
         await seedStatus({ ok: false, failed: ['database-liveness', 'thumbnailer'] });
-        const status = await server.services.health.getStatus([
-            'database-liveness',
-            'thumbnailer',
-        ]);
+        const status = await server.services.health.getStatus({
+            ignore: ['database-liveness', 'thumbnailer'],
+        });
         expect(status).toEqual({ ok: true });
     });
 
     it('keeps the non-ignored failures', async () => {
         await seedStatus({ ok: false, failed: ['database-liveness', 'thumbnailer'] });
-        const status = await server.services.health.getStatus([
-            'database-liveness',
-        ]);
+        const status = await server.services.health.getStatus({
+            ignore: ['database-liveness'],
+        });
         expect(status).toEqual({ ok: false, failed: ['thumbnailer'] });
     });
 
     it('is a no-op for a healthy status', async () => {
         await seedStatus({ ok: true });
-        const status = await server.services.health.getStatus([
-            'database-liveness',
-        ]);
+        const status = await server.services.health.getStatus({
+            ignore: ['database-liveness'],
+        });
         expect(status).toEqual({ ok: true });
     });
 
     it('ignores unknown names without affecting real failures', async () => {
         await seedStatus({ ok: false, failed: ['database-liveness'] });
-        const status = await server.services.health.getStatus(['not-a-check']);
+        const status = await server.services.health.getStatus({
+            ignore: ['not-a-check'],
+        });
         expect(status).toEqual({ ok: false, failed: ['database-liveness'] });
     });
+
+    it('demotes marked failures to degraded and stays ok:true', async () => {
+        await seedStatus({ ok: false, failed: ['database-liveness'] });
+        const status = await server.services.health.getStatus({
+            degrade: ['database-liveness'],
+        });
+        expect(status).toEqual({ ok: true, degraded: ['database-liveness'] });
+    });
+
+    it('reports degraded alongside remaining hard failures (ok:false)', async () => {
+        await seedStatus({
+            ok: false,
+            failed: ['database-liveness', 'socket-initialized'],
+        });
+        const status = await server.services.health.getStatus({
+            degrade: ['database-liveness'],
+        });
+        expect(status).toEqual({
+            ok: false,
+            failed: ['socket-initialized'],
+            degraded: ['database-liveness'],
+        });
+    });
+
+    it('lets ignore take precedence over degrade for the same name', async () => {
+        await seedStatus({ ok: false, failed: ['database-liveness'] });
+        const status = await server.services.health.getStatus({
+            ignore: ['database-liveness'],
+            degrade: ['database-liveness'],
+        });
+        expect(status).toEqual({ ok: true });
+    });
 });
 
 // ── /version ────────────────────────────────────────────────────────

src/backend/services/health/ServerHealthService.ts

@@ -69,6 +69,17 @@ interface HealthStats {
 export interface HealthStatus {
     ok: boolean;
     failed?: string[];
+    degraded?: string[];
+}
+
+export interface GetStatusOptions {
+    /** Failing check names to drop entirely (healthy if all failures ignored). */
+    ignore?: string[];
+    /**
+     * Failing check names to demote to non-fatal `degraded`. They don't make
+     * `ok` false, but their presence signals partial health to the caller.
+     */
+    degrade?: string[];
 }
 
 export class ServerHealthService extends PuterService {
@@ -131,24 +142,21 @@ export class ServerHealthService extends PuterService {
      * Current health status. Results are cached in Redis for 5 seconds
      * so a busy /healthcheck endpoint doesn't hammer the DB on every hit.
      *
-     * `ignore` names a set of failing states to disregard for this request
-     * only, letting an orchestrator poll `/healthcheck` while tolerating
-     * specific known-failing checks. Any failure name may be ignored,
-     * including the `draining` lifecycle state. When the remaining failures
-     * are all ignored the status collapses back to `{ ok: true }`. The
-     * cached status is always the full, unfiltered set — filtering is
-     * applied per-request after the cache read so it never leaks across
-     * callers.
+     * `ignore` names failing states to disregard for this request only,
+     * letting an orchestrator poll `/healthcheck` while tolerating specific
+     * known-failing checks; when the remaining failures are all ignored the
+     * status collapses back to `{ ok: true }`. `degrade` instead demotes
+     * named failures to a non-fatal `degraded` list — `ok` stays true but
+     * the caller can see the partial state. Any failure name may be filtered
+     * this way, including the `draining` lifecycle state. The cached status
+     * is always the full, unfiltered set — filtering is applied per-request
+     * after the cache read so it never leaks across callers.
      */
-    async getStatus(ignore: string[] = []): Promise<HealthStatus> {
-        if (this.#draining) {
-            return this.#applyIgnore(
-                { ok: false, failed: ['draining'] },
-                ignore,
-            );
-        }
-        const status = await this.#getCachedStatus();
-        return this.#applyIgnore(status, ignore);
+    async getStatus(opts: GetStatusOptions = {}): Promise<HealthStatus> {
+        const base = this.#draining
+            ? { ok: false, failed: ['draining'] }
+            : await this.#getCachedStatus();
+        return this.#applyFilters(base, opts.ignore ?? [], opts.degrade ?? []);
     }
 
     async #getCachedStatus(): Promise<HealthStatus> {
@@ -192,18 +200,29 @@ export class ServerHealthService extends PuterService {
     }
 
     /**
-     * Drop `ignore`d check names from a status. If every failure is
-     * ignored the status becomes healthy again; otherwise the remaining
-     * failures are reported as usual. A healthy status is returned as-is.
+     * Reclassify a status against the per-request `ignore`/`degrade` sets.
+     * `ignore`d failures are dropped; `degrade`d failures move to a
+     * non-fatal `degraded` list; anything left stays a hard failure. `ok`
+     * is false only while hard failures remain. A healthy status is
+     * returned as-is.
      */
-    #applyIgnore(status: HealthStatus, ignore: string[]): HealthStatus {
-        if (status.ok || ignore.length === 0 || !status.failed) return status;
+    #applyFilters(
+        status: HealthStatus,
+        ignore: string[],
+        degrade: string[],
+    ): HealthStatus {
+        if (status.ok || !status.failed) return status;
+
         const remaining = status.failed.filter(
             (name) => !ignore.includes(name),
         );
-        return remaining.length === 0
-            ? { ok: true }
-            : { ok: false, failed: remaining };
+        const degraded = remaining.filter((name) => degrade.includes(name));
+        const failed = remaining.filter((name) => !degrade.includes(name));
+
+        const result: HealthStatus = { ok: failed.length === 0 };
+        if (failed.length > 0) result.failed = failed;
+        if (degraded.length > 0) result.degraded = degraded;
+        return result;
     }
 
     #registerDefaultChecks(): void {