feat: slack alarms (#3489)

AGENTS.md

@@ -10,6 +10,7 @@ Use these as the source of truth before exploring further:
 - [doc/architecture.md](doc/architecture.md) — backend layered stack (controllers → drivers → services → stores → clients), `PuterServer` wiring, `Context` (ALS), and extensions.
 - [doc/contributing-apis.md](doc/contributing-apis.md) — adding and maintaining public APIs end to end (backend surface → puter.js → types → docs → tests). Follow it for any API work.
 - [doc/pagination.md](doc/pagination.md) — the one pagination convention for list APIs (limit/cursor/offset/includeTotal, envelope shape, cursor semantics).
+- [doc/alarms.md](doc/alarms.md) — raising alarms and picking a severity (what pages, what only gets recorded), plus the config that routes them.
 - [doc/self-hosting.md](doc/self-hosting.md) — running Puter outside hosted infra.
 - [CONTRIBUTING.md](CONTRIBUTING.md) — testing, security, AI-assisted code, PR conventions, Boy Scout Rule.
 - [SECURITY.md](SECURITY.md) — how to report vulnerabilities (do not file them publicly).

config.template.jsonc

@@ -251,6 +251,56 @@
         }
     },
 
+    // ── Alarms / alerting ───────────────────────────────────────────────
+    // Where system alarms go. Severity is the routing decision — each
+    // transport takes everything at or above its own `minSeverity`:
+    //
+    //   critical — an unhandled server error; pages on-call.
+    //   error    — pages as well; prefer critical or warning.
+    //   warning  — look at it today; no page.
+    //   info     — a record in the chat channel only.
+    //
+    // Both transports are off unless enabled, so a self-hosted node just
+    // logs its alarms to the console.
+    "pager": {
+        // Severity for call sites that don't pick one. Default "critical".
+        "defaultSeverity": "critical",
+
+        // Retier or silence an alarm without a deploy. Keys are alarm ids,
+        // or a prefix ending in `*`; the exact id wins over a pattern, and
+        // the longest matching pattern wins among patterns. Values are a
+        // severity or "mute". This is applied last, so it overrides both the
+        // call site and any known-error rule.
+        "severityOverrides": {
+            // "cronMonitor:*": "info",
+            // "http_500:GET:/some/flapping/route:*": "mute"
+        },
+
+        "pagerduty": {
+            "enabled": false,
+            "routingKey": "",
+            // Lowest severity that reaches PagerDuty. Default "warning",
+            // which keeps `info` out of the paging system entirely.
+            "minSeverity": "warning"
+        },
+
+        // Slack incoming webhook — the low-noise destination for everything
+        // that shouldn't page.
+        "slack": {
+            "enabled": false,
+            "webhookUrl": "",
+            // Optional; defaults to the channel the webhook was created for.
+            "channel": "#alerts",
+            "username": "puter-alarms",
+            // Lowest severity posted. Default "info" (everything).
+            "minSeverity": "info",
+            // Don't repost the same alarm id within this window. The first
+            // occurrence always posts; the next post that gets through
+            // reports how many occurrences piled up. 0 disables throttling.
+            "repeatThrottleMs": 900000
+        }
+    },
+
     // ── Rate limiting ───────────────────────────────────────────────────
     // `memory` for single-node, `redis` for multi-node (default), `kv` for
     // dynamo-backed counters.

doc/alarms.md

@@ -0,0 +1,83 @@
+# Alarms
+
+`clients.alarm` is the one way code reports that something is wrong. It
+de-dupes by alarm id, counts occurrences, and routes each alarm to alert
+transports by **severity**.
+
+```ts
+this.clients.alarm.create(
+    `driver_rate_limit_hit:${iface}:${method}`, // de-dupe key
+    `Driver rate limit hit on ${iface}:${method}`, // what a human reads
+    { iface, method, userUuid }, // context fields
+    'info', // severity
+);
+```
+
+## Severity is the routing decision
+
+| Severity   | Meaning                                                | Goes to            |
+| ---------- | ------------------------------------------------------ | ------------------ |
+| `critical` | An unhandled server error. Someone gets woken up.      | Pager + chat       |
+| `error`    | Same urgency as critical; prefer one of the other two. | Pager + chat       |
+| `warning`  | Worth a look today. Nobody is paged.                   | Pager (low) + chat |
+| `info`     | A record of something expected-but-notable.            | Chat only          |
+
+Each transport declares the lowest severity it accepts, so the value a call
+site passes is what decides where the alarm lands. The bar for `critical` is
+deliberately high: an unhandled 5xx out of the HTTP error handler is the main
+thing that still pages. Anything a human can look at tomorrow is `warning`,
+and anything that's just worth recording is `info`.
+
+Omitting the severity takes `pager.defaultSeverity` (itself `critical`), so
+pass one explicitly unless you really mean "page someone".
+
+### Choosing one
+
+- Did the server fail to do its job in a way nobody expected? → `critical`
+- Is a background job, rate, or dependency degraded? → `warning`
+- Is this a user doing something notable (hitting a limit, tripping an abuse
+  heuristic, overspending)? → `info`
+
+An extension whose signals are all one tier can default its own local
+`raiseAlarm` helper to that tier instead of repeating it at every call site —
+see [extensions/cronMonitor](../../../extensions/cronMonitor/index.js).
+
+## Configuration
+
+Everything lives under `pager` in config (see
+[config.template.jsonc](../config.template.jsonc) for the annotated version).
+Both transports are off unless enabled, so a self-hosted node just logs
+alarms to the console.
+
+```jsonc
+"pager": {
+    "defaultSeverity": "critical",
+    "severityOverrides": { "cronMonitor:*": "info" },
+    "pagerduty": { "enabled": true, "routingKey": "…", "minSeverity": "warning" },
+    "slack": {
+        "enabled": true,
+        "webhookUrl": "…",
+        "channel": "#alerts",
+        "minSeverity": "info",
+        "repeatThrottleMs": 900000,
+    },
+}
+```
+
+### Retiering without a deploy
+
+`severityOverrides` is the escape hatch for an alarm that turns out to be
+noisier or more serious than its call site assumed. Keys are alarm ids or a
+prefix ending in `*`; the exact id beats a pattern, and the longest matching
+prefix wins among patterns. Values are a severity, or `mute` to drop the
+alarm before any transport sees it.
+
+It is applied *after* the call site's severity and any known-error rule, so
+config always has the last word.
+
+### Repeat throttling
+
+The chat transport won't repost the same alarm id within
+`repeatThrottleMs` (default 15 minutes). The first occurrence always posts,
+and the next one that gets through reports how many piled up in between —
+so a hot loop reads as one message with a count, not a wall of them.

src/backend/clients/alarm/AlarmClient.test.ts

@@ -0,0 +1,219 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest';
+import { AlarmClient } from './AlarmClient';
+import type { IConfig, IPagerConfig } from '../../types';
+import type { AlertPayload } from './types';
+
+const pdEvent = vi.hoisted(() => vi.fn(async () => ({})));
+vi.mock('@pagerduty/pdjs', () => ({ event: pdEvent }));
+
+const makeClient = (pager: IPagerConfig = {}) =>
+    new AlarmClient({ serverId: 'test-node', pager } as unknown as IConfig);
+
+/** Register a capturing handler in place of a real transport. */
+const capture = (
+    client: AlarmClient,
+    minSeverity?: 'critical' | 'error' | 'warning' | 'info',
+) => {
+    const seen: AlertPayload[] = [];
+    client.addAlertHandler(
+        async (alert) => {
+            seen.push(alert);
+        },
+        { name: 'capture', minSeverity },
+    );
+    return seen;
+};
+
+describe('AlarmClient severity routing', () => {
+    beforeEach(() => {
+        vi.spyOn(console, 'error').mockImplementation(() => {});
+        vi.spyOn(console, 'warn').mockImplementation(() => {});
+        vi.spyOn(console, 'log').mockImplementation(() => {});
+        pdEvent.mockClear();
+    });
+
+    it('defaults to critical when the call site says nothing', () => {
+        const client = makeClient();
+        const seen = capture(client);
+
+        client.create('boom', 'everything is on fire');
+
+        expect(seen).toHaveLength(1);
+        expect(seen[0].severity).toBe('critical');
+    });
+
+    it('honours a configured default severity', () => {
+        const client = makeClient({ defaultSeverity: 'info' });
+        const seen = capture(client);
+
+        client.create('boom', 'everything is on fire');
+
+        expect(seen[0].severity).toBe('info');
+    });
+
+    it('skips handlers whose floor the alarm does not reach', () => {
+        const client = makeClient();
+        const paging = capture(client, 'warning');
+        const chat = capture(client, 'info');
+
+        client.create('rate-limit', 'a user hit a limit', {}, 'info');
+
+        expect(paging).toHaveLength(0);
+        expect(chat).toHaveLength(1);
+    });
+
+    it('retiers an alarm from config', () => {
+        const client = makeClient({
+            severityOverrides: { 'noisy:*': 'info' },
+        });
+        const paging = capture(client, 'warning');
+        const chat = capture(client, 'info');
+
+        client.create('noisy:thing', 'used to page', {}, 'critical');
+
+        expect(paging).toHaveLength(0);
+        expect(chat[0].severity).toBe('info');
+    });
+
+    it('mutes an alarm from config', () => {
+        const client = makeClient({
+            severityOverrides: { 'noisy:thing': 'mute' },
+        });
+        const chat = capture(client, 'info');
+
+        client.create('noisy:thing', 'not worth reporting');
+        client.create('noisy:thing', 'still not worth reporting');
+
+        expect(chat).toHaveLength(0);
+    });
+
+    it('lets config override a known-error rule', () => {
+        const client = makeClient({
+            severityOverrides: { 'known:thing': 'critical' },
+        });
+        client.setKnownErrors([
+            {
+                match: { id: 'known:thing' },
+                action: { type: 'severity', value: 'info' },
+            },
+        ]);
+        const paging = capture(client, 'warning');
+
+        client.create('known:thing', 'known but escalated');
+
+        expect(paging[0].severity).toBe('critical');
+    });
+
+    it('still respects a no-alert known-error rule', () => {
+        const client = makeClient();
+        client.setKnownErrors([
+            { match: { id: 'known:quiet' }, action: { type: 'no-alert' } },
+        ]);
+        const chat = capture(client, 'info');
+
+        client.create('known:quiet', 'suppressed');
+
+        expect(chat).toHaveLength(0);
+    });
+});
+
+describe('AlarmClient alert payload', () => {
+    beforeEach(() => {
+        vi.spyOn(console, 'error').mockImplementation(() => {});
+        vi.spyOn(console, 'warn').mockImplementation(() => {});
+    });
+
+    it('reports occurrence counts across repeats', () => {
+        const client = makeClient();
+        const seen = capture(client);
+
+        client.create('flap', 'first');
+        client.create('flap', 'second');
+
+        expect(seen[0]).toMatchObject({ repeatCount: 1, isRepeat: false });
+        expect(seen[1]).toMatchObject({ repeatCount: 2, isRepeat: true });
+    });
+
+    it('renders fields as strings and lifts the stack out of the error', () => {
+        const client = makeClient();
+        const seen = capture(client);
+        const error = new Error('kaboom');
+
+        client.create('boom', 'failed', { error, status: 500 }, 'critical');
+
+        expect(seen[0].fields.status).toBe('500');
+        expect(seen[0].trace).toBe(error.stack);
+        expect(seen[0].shortId).toMatch(/^[a-z]+-[a-z]+-[a-z]+$/);
+    });
+
+    it('keeps one failing handler from starving the others', async () => {
+        const client = makeClient();
+        client.addAlertHandler(
+            async () => {
+                throw new Error('transport down');
+            },
+            { name: 'broken' },
+        );
+        const seen = capture(client);
+
+        client.create('boom', 'failed');
+        await Promise.resolve();
+
+        expect(seen).toHaveLength(1);
+    });
+
+    it('suppresses alarms once draining', () => {
+        const client = makeClient();
+        const seen = capture(client);
+
+        client.onServerPrepareShutdown();
+        client.create('boom', 'too late');
+
+        expect(seen).toHaveLength(0);
+    });
+});
+
+describe('AlarmClient transport registration', () => {
+    beforeEach(() => {
+        vi.spyOn(console, 'error').mockImplementation(() => {});
+        vi.spyOn(console, 'warn').mockImplementation(() => {});
+        vi.spyOn(console, 'log').mockImplementation(() => {});
+        pdEvent.mockClear();
+    });
+
+    it('keeps info alarms out of PagerDuty but sends them to Slack', async () => {
+        const fetchMock = vi.fn(async () => ({ ok: true, status: 200 }));
+        vi.stubGlobal('fetch', fetchMock);
+
+        const client = makeClient({
+            pagerduty: { enabled: true, routingKey: 'rk' },
+            slack: { enabled: true, webhookUrl: 'https://hooks.example/abc' },
+        });
+        await client.onServerStart();
+
+        client.create('quiet:thing', 'informational', {}, 'info');
+        await vi.waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1));
+        expect(pdEvent).not.toHaveBeenCalled();
+
+        client.create('loud:thing', 'paging', {}, 'critical');
+        await vi.waitFor(() => expect(pdEvent).toHaveBeenCalledTimes(1));
+        expect(fetchMock).toHaveBeenCalledTimes(2);
+
+        vi.unstubAllGlobals();
+    });
+
+    it('skips transports that are enabled but not configured', async () => {
+        const client = makeClient({
+            pagerduty: { enabled: true },
+            slack: { enabled: true },
+        });
+        await client.onServerStart();
+
+        const seen = capture(client);
+        client.create('boom', 'nowhere to send this');
+
+        // Only the capturing handler is registered.
+        expect(seen).toHaveLength(1);
+        expect(pdEvent).not.toHaveBeenCalled();
+    });
+});

src/backend/clients/alarm/AlarmClient.ts

@@ -3,81 +3,64 @@
  *
  * This file is part of Puter.
  *
- * Puter is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published
- * by the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
+ * Puter is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
  *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
  *
  * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ * along with this program. If not, see
+ * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/).
  */
 
 import { event as pdEvent } from '@pagerduty/pdjs';
 import { inspect } from 'node:util';
 import { createHash } from 'node:crypto';
-import type { IConfig } from '../../types';
+import type { IConfig, PagerSeverity, SeverityRule } from '../../types';
 import { PuterClient } from '../types';
+import { meetsMinSeverity, resolveSeverityOverride } from './severity';
+import { createSlackAlertHandler } from './slack';
+import type {
+    Alarm,
+    AlarmFields,
+    AlertHandler,
+    AlertPayload,
+    KnownErrorRule,
+} from './types';
+
+export type {
+    Alarm,
+    AlarmFields,
+    AlertHandler,
+    AlertPayload,
+    KnownErrorRule,
+} from './types';
+export type { PagerSeverity } from '../../types';
 
 // -- Types ------------------------------------------------------------
 
-export interface AlarmFields {
-    error?: Error;
-    [key: string]: unknown;
+interface RegisteredHandler {
+    name: string;
+    /** Lowest severity this transport accepts. */
+    minSeverity: PagerSeverity;
+    handler: AlertHandler;
 }
 
-interface AlarmOccurrence {
-    message: string;
-    fields: AlarmFields;
-    timestamp: number;
-}
-
-interface Alarm {
-    id: string;
-    shortId: string;
-    message: string;
-    fields: AlarmFields;
-    error?: Error;
-    started: number;
-    timestamps: number[];
-    occurrences: AlarmOccurrence[];
-    severity?: PagerSeverity;
-    noAlert?: boolean;
-}
-
-export type PagerSeverity = 'critical' | 'error' | 'warning' | 'info';
-
-export interface AlertPayload {
-    id: string;
-    message: string;
-    source: string;
-    severity: PagerSeverity;
-    custom?: Record<string, unknown>;
-}
-
-type AlertHandler = (alert: AlertPayload) => Promise<void>;
-
-interface KnownErrorRule {
-    match: {
-        id: string;
-        message?: string;
-        fields?: Record<string, unknown>;
-    };
-    action: {
-        type: 'no-alert' | 'severity';
-        value?: PagerSeverity;
-    };
-}
+/** Severity used when neither the call site nor config picks one. */
+const FALLBACK_SEVERITY: PagerSeverity = 'critical';
+/** Keeps `info` alarms out of the paging system unless config says otherwise. */
+const DEFAULT_PAGERDUTY_MIN_SEVERITY: PagerSeverity = 'warning';
 
 // -- Helpers ----------------------------------------------------------
 
 /**
- * Deterministic short identifier derived from an alarm ID.
- * Produces a readable 3-word slug like "amber-delta-fox".
+ * Deterministic short identifier derived from an alarm ID. Produces a readable
+ * 3-word slug like "amber-delta-fox".
  */
 const WORD_POOL = [
     'alpha',
@@ -169,13 +152,17 @@ function cleanFields(fields: AlarmFields): Record<string, string> {
 // -- AlarmClient ------------------------------------------------------
 
 /**
- * Manages system alarms and dispatches alerts to external paging
- * services (PagerDuty, or any registered handler).
+ * Manages system alarms and routes them to alert transports by severity.
+ *
+ * Severity is the routing decision: each transport declares the lowest severity
+ * it accepts, so `critical` pages on-call while `info` only reaches the chat
+ * channel. Config can retier or mute any alarm id after the fact — see
+ * `pager.severityOverrides` in {@link IConfig}.
  */
 export class AlarmClient extends PuterClient {
     private alarms = new Map<string, Alarm>();
     private aliases = new Map<string, Alarm>();
-    private alertHandlers: AlertHandler[] = [];
+    private alertHandlers: RegisteredHandler[] = [];
     private knownErrors: KnownErrorRule[] = [];
     private draining = false;
     private drainLogged = false;
@@ -187,39 +174,8 @@ export class AlarmClient extends PuterClient {
     // -- Lifecycle ----------------------------------------------------
 
     override async onServerStart(): Promise<void> {
-        const pagerConf = this.config.pager;
-        if (!pagerConf?.pagerduty?.enabled) return;
-
-        const routingKey = pagerConf.pagerduty.routingKey;
-        if (!routingKey) {
-            console.warn(
-                '[alarm] PagerDuty enabled but no routingKey configured',
-            );
-            return;
-        }
-
-        const serverId = this.config.serverId;
-
-        this.alertHandlers.push(async (alert) => {
-            await pdEvent({
-                data: {
-                    routing_key: routingKey,
-                    event_action: 'trigger',
-                    dedup_key: alert.id,
-                    payload: {
-                        summary: alert.message,
-                        source: alert.source,
-                        severity: alert.severity,
-                        custom_details: {
-                            ...alert.custom,
-                            server_id: serverId,
-                        },
-                    },
-                },
-            });
-        });
-
-        console.log('[alarm] PagerDuty handler registered');
+        this.registerPagerDuty();
+        this.registerSlack();
     }
 
     override onServerPrepareShutdown(): void {
@@ -228,16 +184,88 @@ export class AlarmClient extends PuterClient {
         console.log('[alarm] entering drain mode — suppressing new alarms');
     }
 
+    private registerPagerDuty(): void {
+        const pagerDutyConf = this.config.pager?.pagerduty;
+        if (!pagerDutyConf?.enabled) return;
+
+        const routingKey = pagerDutyConf.routingKey;
+        if (!routingKey) {
+            console.warn(
+                '[alarm] PagerDuty enabled but no routingKey configured',
+            );
+            return;
+        }
+
+        const serverId = this.config.serverId;
+        const minSeverity =
+            pagerDutyConf.minSeverity ?? DEFAULT_PAGERDUTY_MIN_SEVERITY;
+
+        this.addAlertHandler(
+            async (alert) => {
+                await pdEvent({
+                    data: {
+                        routing_key: routingKey,
+                        event_action: 'trigger',
+                        dedup_key: alert.id,
+                        payload: {
+                            summary: alert.message,
+                            source: alert.source,
+                            severity: alert.severity,
+                            custom_details: {
+                                ...alert.custom,
+                                server_id: serverId,
+                            },
+                        },
+                    },
+                });
+            },
+            { name: 'pagerduty', minSeverity },
+        );
+
+        console.log(
+            `[alarm] PagerDuty handler registered (min severity: ${minSeverity})`,
+        );
+    }
+
+    private registerSlack(): void {
+        const slackConf = this.config.pager?.slack;
+        if (!slackConf?.enabled) return;
+
+        if (!slackConf.webhookUrl) {
+            console.warn('[alarm] Slack enabled but no webhookUrl configured');
+            return;
+        }
+
+        const minSeverity = slackConf.minSeverity ?? 'info';
+
+        this.addAlertHandler(
+            createSlackAlertHandler(slackConf, {
+                serverId: this.config.serverId,
+            }),
+            { name: 'slack', minSeverity },
+        );
+
+        console.log(
+            `[alarm] Slack handler registered (min severity: ${minSeverity})`,
+        );
+    }
+
     // -- Public API ---------------------------------------------------
 
     /**
-     * Create or update an alarm. If the alarm ID already exists, the
-     * occurrence count is incremented and a repeat alert is dispatched.
+     * Create or update an alarm. If the alarm ID already exists, the occurrence
+     * count is incremented and a repeat alert is dispatched.
      *
-     * `severity` is the PagerDuty severity for this alarm; omit for the
-     * default 'critical'. Use 'info' / 'warning' for expected-but-worth-
-     * tracking signals (e.g. a user hitting a rate limit) so they record
-     * and de-dupe like any other alarm but don't page on-call.
+     * `severity` decides where the alarm lands:
+     *
+     * Critical — a real outage; pages on-call. Reserve it for unhandled server
+     * errors. error — pages on-call as well; prefer `critical` or `warning`.
+     * warning — worth a look soon, but nobody gets woken up. info — a record in
+     * the chat channel; never pages.
+     *
+     * Omit it to take `pager.defaultSeverity` (itself defaulting to
+     * 'critical'). Operators can retier or mute any alarm id from config
+     * afterwards, so the value here is the starting point, not the last word.
      */
     create(
         id: string,
@@ -268,7 +296,9 @@ export class AlarmClient extends PuterClient {
             fields,
             severity,
             started: Date.now(),
-            timestamps: [Date.now()],
+            // `recordOccurrence` below stamps the first occurrence; seeding one
+            // here too would report every alarm as one occurrence ahead.
+            timestamps: [],
             occurrences: [],
         };
         if (fields.error) alarm.error = fields.error;
@@ -295,16 +325,22 @@ export class AlarmClient extends PuterClient {
     }
 
     /**
-     * Register an additional alert handler. Handlers are called for
-     * every alarm that isn't suppressed by a known-error rule.
+     * Register an additional alert handler. Handlers are called for every alarm
+     * that isn't suppressed by a known-error rule or muted by config, and that
+     * meets the handler's own `minSeverity` (default: everything).
      */
-    addAlertHandler(handler: AlertHandler): void {
-        this.alertHandlers.push(handler);
+    addAlertHandler(
+        handler: AlertHandler,
+        opts: { name?: string; minSeverity?: PagerSeverity } = {},
+    ): void {
+        this.alertHandlers.push({
+            name: opts.name ?? `handler-${this.alertHandlers.length}`,
+            minSeverity: opts.minSeverity ?? 'info',
+            handler,
+        });
     }
 
-    /**
-     * Add rules that can suppress or adjust severity of known errors.
-     */
+    /** Add rules that can suppress or adjust severity of known errors. */
     setKnownErrors(rules: KnownErrorRule[]): void {
         this.knownErrors = rules;
     }
@@ -381,25 +417,61 @@ export class AlarmClient extends PuterClient {
         this.dispatchAlert(alarm);
     }
 
+    /**
+     * Call-site severity, then any known-error rule (both already on the
+     * alarm), then the config override — so an operator always has the last
+     * word over what the code asked for.
+     */
+    private resolveSeverity(alarm: Alarm): SeverityRule {
+        const base =
+            alarm.severity ??
+            this.config.pager?.defaultSeverity ??
+            FALLBACK_SEVERITY;
+        return (
+            resolveSeverityOverride(
+                alarm.id,
+                this.config.pager?.severityOverrides,
+            ) ?? base
+        );
+    }
+
     private dispatchAlert(alarm: Alarm): void {
-        const severity = alarm.severity ?? 'critical';
+        const resolved = this.resolveSeverity(alarm);
+        if (resolved === 'mute') {
+            if (!alarm.muteLogged) {
+                alarm.muteLogged = true;
+                console.log(`[alarm] MUTED by config ${displayId(alarm)}`);
+            }
+            return;
+        }
+        alarm.severity = resolved;
+
         const fieldsClean = cleanFields(alarm.fields);
+        const repeatCount = alarm.timestamps.length;
 
         const payload: AlertPayload = {
             id: alarm.id || 'something-bad',
+            shortId: alarm.shortId,
             message: alarm.message || alarm.id || 'something bad happened',
             source: 'alarm',
-            severity,
+            severity: resolved,
+            fields: fieldsClean,
+            trace: alarm.error?.stack,
+            repeatCount,
+            isRepeat: repeatCount > 1,
             custom: {
                 fields: fieldsClean,
                 trace: alarm.error?.stack,
-                repeat_count: alarm.timestamps.length,
+                repeat_count: repeatCount,
             },
         };
 
-        for (const handler of this.alertHandlers) {
+        for (const { name, minSeverity, handler } of this.alertHandlers) {
+            if (!meetsMinSeverity(resolved, minSeverity)) continue;
             handler(payload).catch((err) => {
-                console.error(`[alarm] alert handler failed: ${err?.message}`);
+                console.error(
+                    `[alarm] ${name} alert handler failed: ${err?.message}`,
+                );
             });
         }
     }

src/backend/clients/alarm/severity.test.ts

@@ -0,0 +1,76 @@
+import { describe, expect, it, vi } from 'vitest';
+import { meetsMinSeverity, resolveSeverityOverride } from './severity';
+
+describe('meetsMinSeverity', () => {
+    it('accepts everything at or above the floor', () => {
+        expect(meetsMinSeverity('critical', 'warning')).toBe(true);
+        expect(meetsMinSeverity('error', 'warning')).toBe(true);
+        expect(meetsMinSeverity('warning', 'warning')).toBe(true);
+        expect(meetsMinSeverity('info', 'warning')).toBe(false);
+    });
+
+    it('lets an info floor take every severity', () => {
+        for (const severity of [
+            'critical',
+            'error',
+            'warning',
+            'info',
+        ] as const) {
+            expect(meetsMinSeverity(severity, 'info')).toBe(true);
+        }
+    });
+});
+
+describe('resolveSeverityOverride', () => {
+    it('returns undefined without overrides or on no match', () => {
+        expect(resolveSeverityOverride('a:b', undefined)).toBeUndefined();
+        expect(
+            resolveSeverityOverride('a:b', { 'c:*': 'info' }),
+        ).toBeUndefined();
+    });
+
+    it('matches an exact id', () => {
+        expect(
+            resolveSeverityOverride('cronMonitor:lowSignupRate', {
+                'cronMonitor:lowSignupRate': 'info',
+            }),
+        ).toBe('info');
+    });
+
+    it('matches a prefix pattern', () => {
+        expect(
+            resolveSeverityOverride('cronMonitor:high_aiLogEntries', {
+                'cronMonitor:*': 'warning',
+            }),
+        ).toBe('warning');
+    });
+
+    it('prefers the exact id over a prefix pattern', () => {
+        expect(
+            resolveSeverityOverride('cronMonitor:lowSignupRate', {
+                'cronMonitor:*': 'warning',
+                'cronMonitor:lowSignupRate': 'mute',
+            }),
+        ).toBe('mute');
+    });
+
+    it('prefers the longest matching prefix', () => {
+        expect(
+            resolveSeverityOverride('abuse:card-verification:setup-failed', {
+                'abuse:*': 'info',
+                'abuse:card-verification:*': 'mute',
+            }),
+        ).toBe('mute');
+    });
+
+    it('drops an unrecognized rule rather than guessing', () => {
+        const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+        expect(
+            resolveSeverityOverride('a:b', {
+                'a:b': 'silent' as unknown as 'mute',
+            }),
+        ).toBeUndefined();
+        expect(warn).toHaveBeenCalled();
+        warn.mockRestore();
+    });
+});

src/backend/clients/alarm/severity.ts

@@ -0,0 +1,83 @@
+/**
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+ * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/).
+ */
+
+import type { PagerSeverity, SeverityRule } from '../../types';
+
+/** Ascending urgency. A transport takes everything at or above its floor. */
+const RANK: Record<PagerSeverity, number> = {
+    info: 0,
+    warning: 1,
+    error: 2,
+    critical: 3,
+};
+
+const SEVERITIES = Object.keys(RANK) as PagerSeverity[];
+
+export function isPagerSeverity(value: unknown): value is PagerSeverity {
+    return typeof value === 'string' && value in RANK;
+}
+
+/** True when `severity` is urgent enough for a transport whose floor is `min`. */
+export function meetsMinSeverity(
+    severity: PagerSeverity,
+    min: PagerSeverity,
+): boolean {
+    return RANK[severity] >= RANK[min];
+}
+
+/**
+ * Look up the operator override for an alarm id. Exact ids win over prefix
+ * patterns (`cronMonitor:*`); among patterns the longest prefix wins, so a
+ * specific rule can carve an exception out of a broad one.
+ */
+export function resolveSeverityOverride(
+    id: string,
+    overrides: Record<string, SeverityRule> | undefined,
+): SeverityRule | undefined {
+    if (!overrides) return undefined;
+
+    const exact = overrides[id];
+    if (exact !== undefined) return validRule(id, exact);
+
+    let bestLength = -1;
+    let best: SeverityRule | undefined;
+    for (const [pattern, rule] of Object.entries(overrides)) {
+        if (!pattern.endsWith('*')) continue;
+        const prefix = pattern.slice(0, -1);
+        if (!id.startsWith(prefix)) continue;
+        if (prefix.length <= bestLength) continue;
+        bestLength = prefix.length;
+        best = rule;
+    }
+    return best === undefined ? undefined : validRule(id, best);
+}
+
+/**
+ * A typo in the override map would otherwise silently mute or escalate an
+ * alarm, so an unrecognized value is dropped with a warning instead.
+ */
+function validRule(id: string, rule: SeverityRule): SeverityRule | undefined {
+    if (rule === 'mute' || isPagerSeverity(rule)) return rule;
+    console.warn(
+        `[alarm] ignoring invalid severity override "${rule}" for ${id} ` +
+            `(expected mute or one of ${SEVERITIES.join(', ')})`,
+    );
+    return undefined;
+}

src/backend/clients/alarm/slack.test.ts

@@ -0,0 +1,151 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+import { buildSlackMessage, createSlackAlertHandler } from './slack';
+import type { AlertPayload } from './types';
+
+const alert = (over: Partial<AlertPayload> = {}): AlertPayload => ({
+    id: 'cronMonitor:high_aiLogEntries',
+    shortId: 'amber-delta-fox',
+    message: 'High AI log entries: 1200 in the last 10 minutes',
+    source: 'alarm',
+    severity: 'warning',
+    fields: { count: '1200', threshold: '1000' },
+    repeatCount: 1,
+    isRepeat: false,
+    ...over,
+});
+
+describe('buildSlackMessage', () => {
+    it('renders severity, message and fields', () => {
+        const msg = buildSlackMessage(alert(), {
+            channel: '#alerts',
+            username: 'puter-alarms',
+            serverId: 'oregon',
+        });
+
+        expect(msg.text).toContain('[WARNING]');
+        expect(msg.text).toContain('High AI log entries');
+        expect(msg.channel).toBe('#alerts');
+        expect(msg.username).toBe('puter-alarms');
+        expect(msg.attachments[0].fields).toEqual([
+            { title: 'count', value: '1200', short: true },
+            { title: 'threshold', value: '1000', short: true },
+        ]);
+        expect(msg.attachments[0].footer).toBe(
+            'amber-delta-fox • cronMonitor:high_aiLogEntries • oregon',
+        );
+    });
+
+    it('marks repeats with an occurrence count', () => {
+        const msg = buildSlackMessage(
+            alert({ repeatCount: 7, isRepeat: true }),
+        );
+        expect(msg.text).toContain('(x7)');
+    });
+
+    it('colours each severity differently', () => {
+        const colors = (['critical', 'error', 'warning', 'info'] as const).map(
+            (severity) =>
+                buildSlackMessage(alert({ severity })).attachments[0].color,
+        );
+        expect(new Set(colors).size).toBe(4);
+    });
+
+    it('puts the stack in a code block and keeps it out of the fields', () => {
+        const msg = buildSlackMessage(
+            alert({
+                fields: { error: 'Error: boom', path: '/api/x' },
+                trace: 'Error: boom\n    at handler',
+            }),
+        );
+        expect(msg.attachments[0].text).toBe(
+            '```Error: boom\n    at handler```',
+        );
+        expect(msg.attachments[0].fields.map((f) => f.title)).toEqual(['path']);
+    });
+
+    it('truncates long values and caps the field count', () => {
+        const fields: Record<string, string> = { long: 'x'.repeat(1000) };
+        for (let i = 0; i < 30; i++) fields[`f${i}`] = String(i);
+
+        const msg = buildSlackMessage(alert({ fields }));
+        expect(msg.attachments[0].fields.length).toBe(12);
+        expect(msg.attachments[0].fields[0].value).toHaveLength(400);
+        expect(msg.attachments[0].fields[0].value.endsWith('…')).toBe(true);
+    });
+
+    it('omits channel and username when unset', () => {
+        const msg = buildSlackMessage(alert());
+        expect(msg.channel).toBeUndefined();
+        expect(msg.username).toBeUndefined();
+    });
+});
+
+describe('createSlackAlertHandler', () => {
+    let fetchMock: ReturnType<typeof vi.fn>;
+
+    beforeEach(() => {
+        vi.useFakeTimers();
+        fetchMock = vi.fn(async () => ({ ok: true, status: 200 }));
+        vi.stubGlobal('fetch', fetchMock);
+    });
+
+    afterEach(() => {
+        vi.useRealTimers();
+        vi.unstubAllGlobals();
+    });
+
+    it('posts the payload to the webhook', async () => {
+        const handler = createSlackAlertHandler({
+            webhookUrl: 'https://hooks.example/abc',
+            channel: '#alerts',
+        });
+        await handler(alert());
+
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+        const [url, init] = fetchMock.mock.calls[0];
+        expect(url).toBe('https://hooks.example/abc');
+        expect(init.method).toBe('POST');
+        expect(JSON.parse(init.body).channel).toBe('#alerts');
+    });
+
+    it('throttles repeats of the same alarm and lets other ids through', async () => {
+        const handler = createSlackAlertHandler({
+            webhookUrl: 'https://hooks.example/abc',
+            repeatThrottleMs: 60_000,
+        });
+
+        await handler(alert());
+        await handler(alert({ repeatCount: 2, isRepeat: true }));
+        expect(fetchMock).toHaveBeenCalledTimes(1);
+
+        await handler(alert({ id: 'other:alarm' }));
+        expect(fetchMock).toHaveBeenCalledTimes(2);
+
+        vi.advanceTimersByTime(60_000);
+        await handler(alert({ repeatCount: 3, isRepeat: true }));
+        expect(fetchMock).toHaveBeenCalledTimes(3);
+    });
+
+    it('posts every occurrence when throttling is disabled', async () => {
+        const handler = createSlackAlertHandler({
+            webhookUrl: 'https://hooks.example/abc',
+            repeatThrottleMs: 0,
+        });
+
+        await handler(alert());
+        await handler(alert({ repeatCount: 2, isRepeat: true }));
+        expect(fetchMock).toHaveBeenCalledTimes(2);
+    });
+
+    it('does not let a failed post consume the throttle slot', async () => {
+        fetchMock.mockResolvedValueOnce({ ok: false, status: 500 });
+        const handler = createSlackAlertHandler({
+            webhookUrl: 'https://hooks.example/abc',
+            repeatThrottleMs: 60_000,
+        });
+
+        await expect(handler(alert())).rejects.toThrow('500');
+        await handler(alert());
+        expect(fetchMock).toHaveBeenCalledTimes(2);
+    });
+});

src/backend/clients/alarm/slack.ts

@@ -0,0 +1,179 @@
+/**
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+ * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/).
+ */
+
+import type { ISlackAlertConfig, PagerSeverity } from '../../types';
+import type { AlertHandler, AlertPayload } from './types';
+
+const REQUEST_TIMEOUT_MS = 5000;
+export const DEFAULT_REPEAT_THROTTLE_MS = 15 * 60 * 1000;
+
+/** Beyond this many tracked alarm ids, throttle state is pruned. */
+const MAX_THROTTLE_ENTRIES = 5000;
+
+const MAX_FIELDS = 12;
+const MAX_FIELD_VALUE = 400;
+const MAX_TRACE = 1500;
+/** Values under this length sit two-per-row in the Slack attachment. */
+const SHORT_FIELD_LENGTH = 40;
+
+const STYLE: Record<PagerSeverity, { emoji: string; color: string }> = {
+    critical: { emoji: ':rotating_light:', color: '#d64545' },
+    error: { emoji: ':red_circle:', color: '#e08d4c' },
+    warning: { emoji: ':warning:', color: '#e0b84c' },
+    info: { emoji: ':information_source:', color: '#4c8de0' },
+};
+
+interface SlackField {
+    title: string;
+    value: string;
+    short: boolean;
+}
+
+export interface SlackMessage {
+    text: string;
+    channel?: string;
+    username?: string;
+    icon_emoji?: string;
+    attachments: Array<{
+        color: string;
+        fallback: string;
+        fields: SlackField[];
+        footer: string;
+        mrkdwn_in: string[];
+        text?: string;
+    }>;
+}
+
+function truncate(value: string, max: number): string {
+    return value.length <= max ? value : `${value.slice(0, max - 1)}…`;
+}
+
+/**
+ * Render an alert as an incoming-webhook payload: severity-coloured attachment,
+ * the alarm's fields as a table, and the stack (when there is one) as a code
+ * block. Split out from the handler so the formatting is testable on its own.
+ */
+export function buildSlackMessage(
+    alert: AlertPayload,
+    opts: { channel?: string; username?: string; serverId?: string } = {},
+): SlackMessage {
+    const style = STYLE[alert.severity] ?? STYLE.info;
+    const repeat = alert.isRepeat ? ` (x${alert.repeatCount})` : '';
+    const headline = `${style.emoji} *[${alert.severity.toUpperCase()}]* ${alert.message}${repeat}`;
+
+    const fields: SlackField[] = [];
+    for (const [key, value] of Object.entries(alert.fields)) {
+        if (fields.length >= MAX_FIELDS) break;
+        // `error` is already rendered as the trace block below.
+        if (key === 'error') continue;
+        const rendered = truncate(value, MAX_FIELD_VALUE);
+        fields.push({
+            title: key,
+            value: rendered,
+            short: rendered.length <= SHORT_FIELD_LENGTH,
+        });
+    }
+
+    const footerParts = [alert.shortId, alert.id];
+    if (opts.serverId) footerParts.push(opts.serverId);
+
+    return {
+        text: headline,
+        ...(opts.channel ? { channel: opts.channel } : {}),
+        ...(opts.username ? { username: opts.username } : {}),
+        attachments: [
+            {
+                color: style.color,
+                fallback: `[${alert.severity}] ${alert.message}`,
+                fields,
+                footer: footerParts.join(' • '),
+                mrkdwn_in: ['text'],
+                ...(alert.trace
+                    ? { text: '```' + truncate(alert.trace, MAX_TRACE) + '```' }
+                    : {}),
+            },
+        ],
+    };
+}
+
+/**
+ * Post alerts to a Slack incoming webhook. Repeats of the same alarm id are
+ * throttled so a hot loop doesn't flood the channel — the occurrence count on
+ * the next post that gets through tells the reader what they missed.
+ */
+export function createSlackAlertHandler(
+    conf: ISlackAlertConfig,
+    opts: { serverId?: string } = {},
+): AlertHandler {
+    const webhookUrl = conf.webhookUrl as string;
+    const throttleMs = conf.repeatThrottleMs ?? DEFAULT_REPEAT_THROTTLE_MS;
+    const lastPosted = new Map<string, number>();
+
+    const shouldPost = (alert: AlertPayload): boolean => {
+        if (throttleMs <= 0) return true;
+        const now = Date.now();
+        const previous = lastPosted.get(alert.id);
+        if (previous !== undefined && now - previous < throttleMs) return false;
+
+        if (lastPosted.size >= MAX_THROTTLE_ENTRIES) {
+            for (const [id, at] of lastPosted) {
+                if (now - at >= throttleMs) lastPosted.delete(id);
+            }
+            // Still full of live entries — drop the oldest to stay bounded.
+            if (lastPosted.size >= MAX_THROTTLE_ENTRIES) {
+                const oldest = lastPosted.keys().next().value;
+                if (oldest !== undefined) lastPosted.delete(oldest);
+            }
+        }
+        lastPosted.set(alert.id, now);
+        return true;
+    };
+
+    return async (alert) => {
+        if (!shouldPost(alert)) return;
+
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+        try {
+            const res = await fetch(webhookUrl, {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify(
+                    buildSlackMessage(alert, {
+                        channel: conf.channel,
+                        username: conf.username,
+                        serverId: opts.serverId,
+                    }),
+                ),
+                signal: controller.signal,
+            });
+            if (!res.ok) {
+                // A rejected post shouldn't leave the id marked as delivered.
+                lastPosted.delete(alert.id);
+                throw new Error(`Slack webhook returned ${res.status}`);
+            }
+        } catch (err) {
+            lastPosted.delete(alert.id);
+            throw err;
+        } finally {
+            clearTimeout(timer);
+        }
+    };
+}

src/backend/clients/alarm/types.ts

@@ -0,0 +1,80 @@
+/**
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify it under the
+ * terms of the GNU Affero General Public License as published by the Free
+ * Software Foundation, either version 3 of the License, or (at your option) any
+ * later version.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more
+ * details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+ * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/).
+ */
+
+import type { PagerSeverity } from '../../types';
+
+export interface AlarmFields {
+    error?: Error;
+    [key: string]: unknown;
+}
+
+export interface AlarmOccurrence {
+    message: string;
+    fields: AlarmFields;
+    timestamp: number;
+}
+
+export interface Alarm {
+    id: string;
+    shortId: string;
+    message: string;
+    fields: AlarmFields;
+    error?: Error;
+    started: number;
+    timestamps: number[];
+    occurrences: AlarmOccurrence[];
+    severity?: PagerSeverity;
+    noAlert?: boolean;
+    /** Set once a config mute has been logged, so it's reported only once. */
+    muteLogged?: boolean;
+}
+
+export interface AlertPayload {
+    id: string;
+    /** Readable slug for the same alarm, for humans quoting it back. */
+    shortId: string;
+    message: string;
+    source: string;
+    severity: PagerSeverity;
+    /** Field values rendered as strings, ready to display. */
+    fields: Record<string, string>;
+    /** Stack of the attached error, when the alarm carried one. */
+    trace?: string;
+    /** Total occurrences of this alarm so far, including this one. */
+    repeatCount: number;
+    /** False for the first occurrence of an alarm id. */
+    isRepeat: boolean;
+    /** PagerDuty `custom_details` payload. */
+    custom?: Record<string, unknown>;
+}
+
+export type AlertHandler = (alert: AlertPayload) => Promise<void>;
+
+export interface KnownErrorRule {
+    match: {
+        id: string;
+        message?: string;
+        fields?: Record<string, unknown>;
+    };
+    action: {
+        type: 'no-alert' | 'severity';
+        value?: PagerSeverity;
+    };
+}

src/backend/server.ts

@@ -87,6 +87,7 @@ import { puterStores } from './stores';
 import type {
     IConfig,
     LayerInstances,
+    PagerSeverity,
     WithControllerRegistration,
     WithLifecycle,
 } from './types';
@@ -729,25 +730,29 @@ export class PuterServer {
                     // instead of N pages.
                     //
                     // FORCED_ALERT_CODES override the 5xx-only rule: a
-                    // status < 500 still pages if its legacyCode is in
-                    // the set. Use this for things we want to know about
+                    // status < 500 still alarms if its legacyCode is in
+                    // the map. Use this for things we want to know about
                     // even though we expose them as 4xx to users (e.g.
-                    // sustained upstream provider rate limits).
+                    // sustained upstream provider rate limits). They are
+                    // not our own crashes, so each maps to the severity it
+                    // deserves rather than paging.
                     //
                     // SKIP_ALERT_PREFIXES override the 5xx rule the other
                     // direction: an error tagged as caused by an upstream
                     // provider or a misbehaving client gets exposed to
-                    // the user but does not page.
-                    const FORCED_ALERT_CODES = new Set([
-                        'upstream_rate_limited',
-                        'upstream_auth_failed',
+                    // the user but does not alarm at all.
+                    const FORCED_ALERT_CODES = new Map<string, PagerSeverity>([
+                        ['upstream_rate_limited', 'info'],
+                        // Our credentials for a provider stopped working —
+                        // everything through it fails until someone looks.
+                        ['upstream_auth_failed', 'warning'],
                     ]);
                     const SKIP_ALERT_PREFIXES = /^(upstream_|client_)/;
                     const isHttp = isHttpError(err);
                     const status = isHttp ? err.statusCode : 500;
                     const legacyCode = isHttp ? (err.legacyCode ?? '') : '';
-                    const forced = FORCED_ALERT_CODES.has(legacyCode);
-                    if (!forced) {
+                    const forcedSeverity = FORCED_ALERT_CODES.get(legacyCode);
+                    if (!forcedSeverity) {
                         if (status < 500) return;
                         if (SKIP_ALERT_PREFIXES.test(legacyCode)) return;
                     }
@@ -774,6 +779,9 @@ export class PuterServer {
                             route: routePath,
                             actor: req.actor,
                         },
+                        // An unhandled server error is the one thing that
+                        // still pages on-call.
+                        forcedSeverity ?? 'critical',
                     );
                 },
             }),

src/backend/services/metering/MeteringService.test.ts

@@ -271,8 +271,9 @@ describe('MeteringService', () => {
             expect(result['kv:read']).toMatchObject({ cost: 1, units: 1 });
             expect(alarmSpy).toHaveBeenCalledWith(
                 expect.stringContaining('negative cost'),
-                expect.any(String),
+                expect.stringContaining(actor.user!.email!),
                 expect.objectContaining({ usageType: 'kv:read' }),
+                'info',
             );
             alarmSpy.mockRestore();
         });
@@ -392,11 +393,13 @@ describe('MeteringService', () => {
             );
 
             expect(alarmSpy).toHaveBeenCalledWith(
-                expect.stringContaining('usage exceeded'),
+                // The account is named by email — what someone reading the
+                // alert needs to look it up.
+                expect.stringContaining(overActor.user!.email!),
                 expect.stringContaining('exceeded their usage allowance'),
                 expect.objectContaining({ totalUsage: expect.any(Number) }),
-                // Non-paging severity — records and de-dupes but doesn't page on-call.
-                'warning',
+                // Chat-only severity — records and de-dupes but doesn't page.
+                'info',
             );
             alarmSpy.mockRestore();
         });
@@ -518,7 +521,7 @@ describe('MeteringService', () => {
                 expect.stringContaining('usage exceeded'),
                 expect.stringContaining('exceeded their usage allowance'),
                 expect.objectContaining({ purchasedCredits: credit }),
-                'warning',
+                'info',
             );
             alarmSpy.mockRestore();
         });
@@ -588,8 +591,9 @@ describe('MeteringService', () => {
             ]);
             expect(alarmSpy).toHaveBeenCalledWith(
                 expect.stringContaining('negative cost'),
-                expect.any(String),
+                expect.stringContaining(actor.user!.email!),
                 expect.objectContaining({ usageType: 'kv:read' }),
+                'info',
             );
             alarmSpy.mockRestore();
         });

src/backend/services/metering/MeteringService.ts

@@ -51,6 +51,22 @@ interface UsageInput {
     costOverride?: number;
 }
 
+// -- Helpers ----------------------------------------------------------
+
+/**
+ * How an actor is named in metering alarms. Email is what someone reading the
+ * alert actually needs to find the account; username and uuid are fallbacks for
+ * actors that have no email (temp accounts).
+ */
+function actorLabel(actor: Actor): string {
+    return (
+        actor.user?.email ??
+        actor.user?.username ??
+        actor.user?.uuid ??
+        'unknown-user'
+    );
+}
+
 // -- MeteringService --------------------------------------------------
 
 /**
@@ -158,7 +174,7 @@ export class MeteringService extends PuterService {
         if (costOverrideRaw && costOverrideRaw < 0) {
             this.clients.alarm.create(
                 `metering unexpected negative cost access to: ${usageType}`,
-                'negative cost abuse vector!',
+                `negative cost abuse vector! (${actorLabel(actor)})`,
                 {
                     userId: actor.user?.uuid,
                     username: actor.user?.username,
@@ -168,6 +184,7 @@ export class MeteringService extends PuterService {
                     usageAmount,
                     costOverride,
                 },
+                'info',
             );
         }
 
@@ -275,7 +292,7 @@ export class MeteringService extends PuterService {
                 error: e,
             });
             this.clients.alarm.create(
-                `metering service error for user: ${actor.user?.username} app: ${actor.app?.uid}`,
+                `metering service error for user: ${actorLabel(actor)} app: ${actor.app?.uid}`,
                 (e as Error).message,
                 {
                     userId: actor.user?.uuid,
@@ -287,6 +304,7 @@ export class MeteringService extends PuterService {
                     usageAmount,
                     costOverride,
                 },
+                'info',
             );
             return { total: 0 } as UsageByType;
         }
@@ -325,7 +343,7 @@ export class MeteringService extends PuterService {
                 if (costOverrideRaw && costOverrideRaw < 0) {
                     this.clients.alarm.create(
                         `metering unexpected negative cost access to: ${usageType}`,
-                        'negative cost abuse vector!',
+                        `negative cost abuse vector! (${actorLabel(actor)})`,
                         {
                             userId: actor.user?.uuid,
                             username: actor.user?.username,
@@ -336,6 +354,7 @@ export class MeteringService extends PuterService {
                             costOverride,
                             costOverrideRaw,
                         },
+                        'info',
                     );
                 }
 
@@ -433,7 +452,7 @@ export class MeteringService extends PuterService {
                 error: e,
             });
             this.clients.alarm.create(
-                `metering service error for user: ${actor.user?.username} app: ${actor.app?.uid}`,
+                `metering service error for user: ${actorLabel(actor)} app: ${actor.app?.uid}`,
                 (e as Error).message,
                 {
                     userId: actor.user?.uuid,
@@ -444,6 +463,7 @@ export class MeteringService extends PuterService {
                     actor,
                     batchUsages: usages,
                 },
+                'info',
             );
             return { total: 0 } as UsageByType;
         }
@@ -922,8 +942,8 @@ export class MeteringService extends PuterService {
         if (!(wasAlreadyOverLimit && crossedMultiple)) return;
 
         this.clients.alarm.create(
-            `metering usage exceeded by user: ${actor.user?.username}`,
-            `Actor ${userId} has exceeded their usage allowance significantly`,
+            `metering usage exceeded by user: ${actorLabel(actor)}`,
+            `${actorLabel(actor)} (${userId}) has exceeded their usage allowance significantly`,
             {
                 userId: actor.user?.uuid,
                 username: actor.user?.username,
@@ -938,8 +958,9 @@ export class MeteringService extends PuterService {
                 purchasedCredits,
                 consumedPurchaseCredits,
             },
-            // Expected-but-worth-tracking signal — record/de-dupe it but don't page on-call.
-            'warning',
+            // One account outspending its allowance is a thing to look at, not
+            // an incident — a record in the alerts channel is enough.
+            'info',
         );
     }
 
@@ -973,6 +994,9 @@ export class MeteringService extends PuterService {
                         maxAllowedPerMinute:
                             MeteringService.MAX_GLOBAL_USAGE_PER_MINUTE,
                     },
+                    // Fleet-wide spend running away — worth someone's attention
+                    // the same day, but it isn't an outage.
+                    'warning',
                 );
             }
         }

src/backend/types.ts

@@ -70,11 +70,57 @@ export interface IRedisConfig {
     useMock?: boolean;
 }
 
+/**
+ * Alert severity. Ordered `info` < `warning` < `error` < `critical`; each alert
+ * transport takes everything at or above its own `minSeverity`, so the severity
+ * a call site picks is what decides where the alarm lands.
+ */
+export type PagerSeverity = 'critical' | 'error' | 'warning' | 'info';
+
+/** A severity, or `mute` to drop the alarm before any transport sees it. */
+export type SeverityRule = PagerSeverity | 'mute';
+
+export interface IPagerDutyConfig {
+    enabled?: boolean;
+    routingKey?: string;
+    /**
+     * Lowest severity that reaches PagerDuty. Default `warning`, which keeps
+     * `info` alarms out of the paging system entirely.
+     */
+    minSeverity?: PagerSeverity;
+}
+
+export interface ISlackAlertConfig {
+    enabled?: boolean;
+    /** Incoming-webhook URL to post alerts to. */
+    webhookUrl?: string;
+    /** Channel override (e.g. `#alerts`). Defaults to the webhook's own. */
+    channel?: string;
+    /** Bot display name on the posted message. */
+    username?: string;
+    /** Lowest severity posted to Slack. Default `info` (everything). */
+    minSeverity?: PagerSeverity;
+    /**
+     * Don't repost the same alarm id within this window. The first occurrence
+     * always posts; repeats inside the window only bump the occurrence count
+     * that the next post reports. Default 15 minutes; `0` disables throttling.
+     */
+    repeatThrottleMs?: number;
+}
+
 export interface IPagerConfig {
-    pagerduty?: {
-        enabled?: boolean;
-        routingKey?: string;
-    };
+    /** Severity used when a call site doesn't pass one. Default `critical`. */
+    defaultSeverity?: PagerSeverity;
+    /**
+     * Operator overrides keyed by alarm id, or by prefix with a trailing `*`
+     * (`cronMonitor:*`). Exact ids beat patterns and the longest matching
+     * prefix wins. Applied after the call site's severity and any known-error
+     * rule, so this is the final say — it can retier or mute a noisy alarm
+     * without a deploy.
+     */
+    severityOverrides?: Record<string, SeverityRule>;
+    pagerduty?: IPagerDutyConfig;
+    slack?: ISlackAlertConfig;
 }
 
 export interface ICfFileCacheConfig {
@@ -144,7 +190,12 @@ export interface IPreludeConfig {
      * an RCS agent provisioned in the Prelude account to actually use RCS.
      */
     preferredChannel?:
-        'sms' | 'rcs' | 'whatsapp' | 'viber' | 'zalo' | 'telegram';
+        | 'sms'
+        | 'rcs'
+        | 'whatsapp'
+        | 'viber'
+        | 'zalo'
+        | 'telegram';
 }
 
 /**
@@ -863,8 +914,7 @@ export interface WithLifecycle extends Object {
 }
 
 export interface WithCostsReporting extends WithLifecycle {
-    getReportedCosts?: () =>
-        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    getReportedCosts?: () => // eslint-disable-next-line @typescript-eslint/no-explicit-any
         | Promise<Record<string, any>[]>
         // eslint-disable-next-line @typescript-eslint/no-explicit-any
         | Record<string, any>[];