From 79d4201f12cca93ff0e43e7fb167c2711957ca9c Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Mon, 10 Aug 2026 19:09:47 -0700 Subject: [PATCH] fix: rate limits, AI routing, and a type-check gate (#3529) - declare rate + concurrency limits on every route and driver that lacked one - add acquireConcurrent for websocket connections and the DAV mount - bucket AI models by identity key only; keep resold duplicates of any vendor - skip recently-failed provider routes; cap the fallback chain at 3 attempts - let full-access access tokens bind a worker to an app their own user owns - cache resolved subscriptions so tiered limits don't add a round trip --- .github/workflows/backend-tests.yaml | 32 + config.template.jsonc | 9 +- doc/alarms.md | 16 +- extensions/appTelemetry.ts | 41 +- extensions/installedApps.ts | 12 +- extensions/metering.ts | 28 +- extensions/whoami.ts | 19 +- package.json | 2 + src/backend/clients/alarm/AlarmClient.test.ts | 18 + src/backend/clients/alarm/AlarmClient.ts | 35 +- src/backend/clients/alarm/types.ts | 10 + src/backend/clients/database/SQLBatcher.js | 15 +- .../clients/database/SQLBatcher.test.ts | 30 + .../clients/database/retriableErrors.ts | 30 + src/backend/clients/event/types.ts | 8 + src/backend/clients/s3/S3Client.ts | 14 + src/backend/controllers/apps/AppController.js | 54 +- .../controllers/apps/AppController.test.ts | 22 + .../controllers/auth/AuthController.test.ts | 21 + .../controllers/auth/AuthController.ts | 207 +++++- .../controllers/desktop/DesktopController.js | 17 + .../DriverController.concurrent.test.ts | 6 +- .../drivers/DriverController.errors.test.ts | 9 +- .../drivers/DriverController.test.ts | 16 + .../controllers/drivers/DriverController.ts | 66 +- src/backend/controllers/fs/FSController.ts | 139 +++- .../controllers/fs/LegacyFSController.ts | 297 +++++--- src/backend/controllers/fs/limits.test.ts | 163 ++++ src/backend/controllers/fs/limits.ts | 243 ++++++ .../controllers/hosting/HostingController.js | 16 + .../notification/NotificationController.ts | 16 + .../controllers/oidc/OIDCController.test.ts | 32 + .../controllers/oidc/OIDCController.ts | 36 +- .../controllers/peer/PeerController.ts | 51 +- .../puterai/PuterAIController.test.ts | 9 + .../controllers/puterai/PuterAIController.ts | 36 +- .../static/StaticPagesController.ts | 294 ++++---- .../controllers/system/SystemController.js | 167 ++++- .../system/SystemController.test.ts | 59 ++ .../webdav/WebDAVController.test.ts | 20 + .../controllers/webdav/WebDAVController.ts | 48 ++ .../controllers/wisp/WispController.ts | 35 +- src/backend/core/http/middleware/rateLimit.js | 173 ++++- .../core/http/middleware/rateLimit.test.js | 273 ++++++- .../ChatCompletionDriver.routing.test.ts | 148 +++- .../drivers/ai-chat/ChatCompletionDriver.ts | 201 ++--- .../ai-chat/utils/modelRouting.test.ts | 75 +- .../drivers/ai-chat/utils/modelRouting.ts | 31 +- .../ai-chat/utils/providerHealth.test.ts | 84 +++ .../drivers/ai-chat/utils/providerHealth.ts | 60 ++ src/backend/drivers/apps/AppDriver.js | 53 ++ src/backend/drivers/driverPolicies.test.ts | 192 ++++- src/backend/drivers/kv/KVStoreDriver.ts | 39 +- .../notification/NotificationDriver.ts | 12 +- .../drivers/subdomain/SubdomainDriver.test.ts | 56 +- .../drivers/subdomain/SubdomainDriver.ts | 58 +- src/backend/drivers/workers/WorkerDriver.ts | 69 ++ src/backend/services/auth/AuthService.test.ts | 66 +- src/backend/services/auth/AuthService.ts | 29 +- .../health/ServerHealthService.test.ts | 253 ++++++- .../services/health/ServerHealthService.ts | 336 +++++++-- .../dependencyProbes.integration.test.ts | 122 +++ .../services/metering/MeteringService.test.ts | 109 ++- .../services/metering/MeteringService.ts | 96 ++- .../services/socket/SocketService.test.ts | 97 +++ src/backend/services/socket/SocketService.ts | 189 ++++- src/backend/stores/systemKv/SystemKVStore.ts | 7 +- .../stores/systemKv/tableDefinition.ts | 4 +- src/backend/types.ts | 26 + src/backend/util/identifier.js | 32 +- src/backend/util/identifier.test.js | 8 +- src/gui/src/helpers.js | 13 +- src/puter-js/src/lib/networkUtils.js | 697 +++++++++++------- src/puter-js/src/lib/networkUtils.test.js | 509 ++++++++++--- tools/typecheck-baseline.json | 58 ++ tools/typecheck.mjs | 136 ++++ tsconfig.build.json | 2 + 77 files changed, 5659 insertions(+), 1052 deletions(-) create mode 100644 src/backend/controllers/fs/limits.test.ts create mode 100644 src/backend/controllers/fs/limits.ts create mode 100644 src/backend/drivers/ai-chat/utils/providerHealth.test.ts create mode 100644 src/backend/drivers/ai-chat/utils/providerHealth.ts create mode 100644 src/backend/services/health/dependencyProbes.integration.test.ts create mode 100644 tools/typecheck-baseline.json create mode 100644 tools/typecheck.mjs diff --git a/.github/workflows/backend-tests.yaml b/.github/workflows/backend-tests.yaml index 0edba6bf6..5792edbf5 100644 --- a/.github/workflows/backend-tests.yaml +++ b/.github/workflows/backend-tests.yaml @@ -13,6 +13,9 @@ on: - 'tools/**' - 'package.json' - 'package-lock.json' + # The type-check job below reads these. + - 'tsconfig.json' + - 'tsconfig.build.json' - '.github/workflows/backend-tests.yaml' permissions: @@ -20,6 +23,35 @@ permissions: pull-requests: write jobs: + # `tsconfig.build.json` builds with `noCheck: true`, so nothing else in CI + # runs the type checker and a missing export compiles to `undefined`, + # surfacing only when the call is finally reached at runtime. This diffs + # against tools/typecheck-baseline.json and fails only on *new* errors. + # + # The consuming repo runs an equivalent gate, but only once someone bumps the + # submodule pointer — too late to keep the error off this repo's default + # branch. Hence a gate here, on this repo's own pull requests. + # + # Its own job rather than a step in `test`: that one runs a base/PR matrix for + # coverage comparison, and the check only needs the PR ref, once. + typecheck: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: '24' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Type check (new errors only) + run: npm run typecheck + test: runs-on: ubuntu-latest strategy: diff --git a/config.template.jsonc b/config.template.jsonc index d80a2d5d7..f64215def 100644 --- a/config.template.jsonc +++ b/config.template.jsonc @@ -438,5 +438,12 @@ // ── Metering ──────────────────────────────────────────────────────── // When true, all metering checks pass — no per-actor limits enforced. - "unlimitedMetering": false + "unlimitedMetering": false, + + // Fleet-wide spend rate, in micro-cents per minute, past which metering + // raises the `metering:excessiveGlobalUsageRate` alarm. Omit it (the + // default) to leave the check off: the only useful value is a multiple of + // what this deployment's normal traffic costs, so it has to be measured + // rather than guessed, and a stale number here alarms on healthy growth. + // "maxGlobalUsagePerMinute": 200000000 } diff --git a/doc/alarms.md b/doc/alarms.md index 76a620722..fa01b9ec2 100644 --- a/doc/alarms.md +++ b/doc/alarms.md @@ -6,9 +6,9 @@ transports by **severity**. ```ts this.clients.alarm.create( - `driver_rate_limit_hit:${iface}:${method}`, // de-dupe key - `Driver rate limit hit on ${iface}:${method}`, // what a human reads - { iface, method, userUuid }, // context fields + `metering_write_failed:${userUuid}`, // de-dupe key + 'Metering write failed', // what a human reads + { userUuid, appId, error }, // context fields 'info', // severity ); ``` @@ -37,8 +37,14 @@ pass one explicitly unless you really mean "page someone". - Did the server fail to do its job in a way nobody expected? → `critical` - Is a background job, rate, or dependency degraded? → `warning` -- Is this a user doing something notable (hitting a limit, tripping an abuse - heuristic, overspending)? → `info` +- Is this a user doing something notable (tripping an abuse heuristic, + overspending)? → `info` +- Did one of *our own* limits reject a caller — a rate limit, a concurrency + cap, a quota? → don't alarm at all. The limit doing its job is not an + event; the 429 is the whole signal, and alarming on it only produces noise + proportional to traffic. An *upstream provider* rate-limiting us is the + opposite case and still alarms (`upstream_rate_limited`, `info`) — that one + is not something we chose. An extension whose signals are all one tier can default its own local `raiseAlarm` helper to that tier instead of repeating it at every call site — diff --git a/extensions/appTelemetry.ts b/extensions/appTelemetry.ts index 07b7a4e85..43f62050e 100644 --- a/extensions/appTelemetry.ts +++ b/extensions/appTelemetry.ts @@ -2,6 +2,14 @@ import { Context } from '@heyputer/backend/src/core'; import type { Actor } from '@heyputer/backend/src/core/actor'; import { HttpError } from '@heyputer/backend/src/core/http'; import { PuterDriver } from '@heyputer/backend/src/drivers/types'; +import type { + DriverConcurrentConfig, + DriverRateLimitConfig, +} from '@heyputer/backend/src/drivers/meta'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '@heyputer/backend/src/services/metering/consts'; import { extension } from '@heyputer/backend/src/extensions'; // App-telemetry lets an app owner enumerate the users who have @@ -50,17 +58,42 @@ const parseIntParam = ( * Driver exposing the `app-telemetry` interface. * * The `/drivers/call` permission gate checks - * `service:app-telemetry:ii:app-telemetry`, which every actor already holds - * via the blanket `service` grant (hardcoded-permissions.js + + * `service:app-telemetry:ii:app-telemetry`, which every actor already holds via + * the blanket `service` grant (hardcoded-permissions.js + * `default_implicit_user_app_permissions`). The real authorization — "is the - * caller the app owner?" — is enforced inside `get_users` below, exactly as - * v1 did. + * caller the app owner?" — is enforced inside `get_users` below, exactly as v1 + * did. */ export class AppTelemetryDriver extends PuterDriver { readonly driverInterface = 'app-telemetry'; readonly driverName = 'app-telemetry'; readonly isDefault = true; + // Declaring nothing here would leave both methods on the generic + // 600/minute driver default, which does not fit a paginated scan that + // can ask for MAX_LIMIT rows at MAX_OFFSET. This is a dashboard read — + // nobody calls it in a loop. + readonly rateLimit: DriverRateLimitConfig = { + default: { + limit: 60, + window: 60_000, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 30, + [DEFAULT_TEMP_SUBSCRIPTION]: 10, + }, + }, + }; + + readonly concurrent: DriverConcurrentConfig = { + default: { + limit: 5, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 2, + [DEFAULT_TEMP_SUBSCRIPTION]: 2, + }, + }, + }; + /** Users who have authenticated into the given app (owner-only). */ async get_users({ app_uuid, diff --git a/extensions/installedApps.ts b/extensions/installedApps.ts index 8cd75213b..8ef7625c8 100644 --- a/extensions/installedApps.ts +++ b/extensions/installedApps.ts @@ -81,6 +81,16 @@ export const handleInstalledApps = async ( extension.get( '/installedApps', - { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true }, + { + subdomain: 'api', + requireUserActor: true, + allowFullAccessToken: true, + rateLimit: { + scope: 'installed-apps', + limit: 120, + window: 60_000, + key: 'user', + }, + }, handleInstalledApps, ); diff --git a/extensions/metering.ts b/extensions/metering.ts index 16295860a..da61337dc 100644 --- a/extensions/metering.ts +++ b/extensions/metering.ts @@ -110,26 +110,46 @@ export const handleMeteringAllCosts = async ( res.json({ costs: cachedAllCosts }); }; +/** Dashboard reads over the per-actor KV aggregates. */ +const USAGE_READ_LIMIT = { + scope: 'metering-usage', + limit: 120, + window: 60_000, + key: 'user' as const, +}; + extension.get( '/metering/usage', - { subdomain: 'api', requireAuth: true }, + { subdomain: 'api', requireAuth: true, rateLimit: USAGE_READ_LIMIT }, handleMeteringUsage, ); extension.get( '/metering/usage/:appIdOrName', - { subdomain: 'api', requireAuth: true }, + { subdomain: 'api', requireAuth: true, rateLimit: USAGE_READ_LIMIT }, handleMeteringUsageForApp, ); extension.get( '/metering/globalUsage', - { subdomain: 'api', adminOnly: true }, + { + subdomain: 'api', + adminOnly: true, + // Sums across every shard of the global aggregate. Admin-gated, so + // this is loop protection — but one accidental poll is an + // expensive minute. + rateLimit: { + scope: 'metering-global-usage', + limit: 10, + window: 60_000, + key: 'user', + }, + }, handleMeteringGlobalUsage, ); extension.get( '/metering/allCosts', - { subdomain: 'api', requireAuth: true }, + { subdomain: 'api', requireAuth: true, rateLimit: USAGE_READ_LIMIT }, handleMeteringAllCosts, ); diff --git a/extensions/whoami.ts b/extensions/whoami.ts index 81eedb675..2d44f8042 100644 --- a/extensions/whoami.ts +++ b/extensions/whoami.ts @@ -201,6 +201,23 @@ export const handleWhoami = async ( extension.get( '/whoami', - { subdomain: 'api', requireAuth: true, allowUnconfirmed: true }, + { + subdomain: 'api', + requireAuth: true, + allowUnconfirmed: true, + // The GUI polls this, and each call fans out to every `whoami` + // event listener — so it costs more than the response suggests. + // + // It is also the call everything else leans on to find out who it is + // talking to, so it rides along with unrelated work rather than + // arriving at its own pace: the ceiling has to clear whatever the + // busiest session is doing, not what a person clicks. + rateLimit: { + scope: 'whoami', + limit: 1_800, + window: 60_000, + key: 'user', + }, + }, handleWhoami, ); diff --git a/package.json b/package.json index c6e7f0e15..493842e9b 100644 --- a/package.json +++ b/package.json @@ -67,6 +67,8 @@ "check-translations": "node tools/check-translations.js", "prepare": "husky", "build:ts": "tsc -p tsconfig.build.json && node ./tools/write-dist-package-json.mjs", + "typecheck": "node tools/typecheck.mjs", + "typecheck:update": "node tools/typecheck.mjs --update", "setupExtensions": "node ./tools/extensionSetup.mjs" }, "workspaces": [ diff --git a/src/backend/clients/alarm/AlarmClient.test.ts b/src/backend/clients/alarm/AlarmClient.test.ts index fb23e728b..82bdf4ac4 100644 --- a/src/backend/clients/alarm/AlarmClient.test.ts +++ b/src/backend/clients/alarm/AlarmClient.test.ts @@ -371,6 +371,24 @@ describe('AlarmClient alarm registry', () => { expect(client.get('flap')?.occurrences).toHaveLength(2); }); + it('caps retained occurrences while still counting every repeat', () => { + const client = makeClient(); + const seen = capture(client); + + for (let i = 0; i < 50; i++) { + client.create('hot', `occurrence ${i}`, { i }); + } + + const alarm = client.get('hot')!; + expect(alarm.count).toBe(50); + expect(alarm.occurrences).toHaveLength(20); + expect(alarm.timestamps).toHaveLength(20); + // The window kept is the most recent one, not the oldest. + expect(alarm.occurrences[19].message).toBe('occurrence 49'); + // Trimming history must not rewind what the transports are told. + expect(seen[49]).toMatchObject({ repeatCount: 50, isRepeat: true }); + }); + it('names anonymous handlers by their registration order', () => { const client = makeClient(); client.addAlertHandler(async () => { diff --git a/src/backend/clients/alarm/AlarmClient.ts b/src/backend/clients/alarm/AlarmClient.ts index ea245f966..179c3afbd 100644 --- a/src/backend/clients/alarm/AlarmClient.ts +++ b/src/backend/clients/alarm/AlarmClient.ts @@ -60,6 +60,15 @@ interface RegisteredHandler { /** Severity used when neither the call site nor config picks one. */ const FALLBACK_SEVERITY: PagerSeverity = 'critical'; +/** + * How many recent occurrences an alarm keeps. An alarm is never cleared unless + * something calls `clear`, and a hot one repeats for as long as the fault lasts + * — so retaining every occurrence means retaining every message and field set + * it was ever raised with, request bodies and actors included, for the life of + * the process. The last few are what a human reads; the rest is only a count, + * and `count` keeps that. + */ +const OCCURRENCE_HISTORY_LIMIT = 20; /** Keeps `info` alarms out of the paging system unless config says otherwise. */ const DEFAULT_PAGERDUTY_MIN_SEVERITY: PagerSeverity = 'warning'; /** Slack's ceiling once a pager exists: chat gets what doesn't page. */ @@ -321,6 +330,7 @@ export class AlarmClient extends PuterClient { started: Date.now(), // `recordOccurrence` below stamps the first occurrence; seeding one // here too would report every alarm as one occurrence ahead. + count: 0, timestamps: [], occurrences: [], }; @@ -381,16 +391,25 @@ export class AlarmClient extends PuterClient { message: string, fields: AlarmFields, ): void { + const now = Date.now(); alarm.message = message; alarm.fields = { ...alarm.fields, ...fields }; - alarm.timestamps.push(Date.now()); + alarm.count++; if (fields.error) alarm.error = fields.error; - alarm.occurrences.push({ - message, - fields, - timestamp: Date.now(), - }); + alarm.timestamps.push(now); + alarm.occurrences.push({ message, fields, timestamp: now }); + + if (alarm.timestamps.length > OCCURRENCE_HISTORY_LIMIT) { + alarm.timestamps.splice( + 0, + alarm.timestamps.length - OCCURRENCE_HISTORY_LIMIT, + ); + alarm.occurrences.splice( + 0, + alarm.occurrences.length - OCCURRENCE_HISTORY_LIMIT, + ); + } } private applyKnownErrors(alarm: Alarm): void { @@ -438,7 +457,7 @@ export class AlarmClient extends PuterClient { this.applyKnownErrors(alarm); console.warn( - `[alarm] REPEAT ${displayId(alarm)} :: ${alarm.message} (${alarm.timestamps.length})`, + `[alarm] REPEAT ${displayId(alarm)} :: ${alarm.message} (${alarm.count})`, ); if (alarm.noAlert) return; @@ -476,7 +495,7 @@ export class AlarmClient extends PuterClient { alarm.severity = resolved; const fieldsClean = cleanFields(alarm.fields); - const repeatCount = alarm.timestamps.length; + const repeatCount = alarm.count; const id = alarm.id || 'something-bad'; diff --git a/src/backend/clients/alarm/types.ts b/src/backend/clients/alarm/types.ts index 66a9c113e..9437b4a5c 100644 --- a/src/backend/clients/alarm/types.ts +++ b/src/backend/clients/alarm/types.ts @@ -48,7 +48,17 @@ export interface Alarm extends AlarmOptions { fields: AlarmFields; error?: Error; started: number; + /** + * Every occurrence ever counted, including those aged out of the two lists + * below. + */ + count: number; + /** + * Timestamps of the most recent occurrences only — see + * `OCCURRENCE_HISTORY_LIMIT`. + */ timestamps: number[]; + /** The most recent occurrences only — see `OCCURRENCE_HISTORY_LIMIT`. */ occurrences: AlarmOccurrence[]; severity?: PagerSeverity; noAlert?: boolean; diff --git a/src/backend/clients/database/SQLBatcher.js b/src/backend/clients/database/SQLBatcher.js index 5089863f7..2b313ea57 100644 --- a/src/backend/clients/database/SQLBatcher.js +++ b/src/backend/clients/database/SQLBatcher.js @@ -22,6 +22,7 @@ import { POOL_ACQUIRE_TIMEOUT, isNeverSentError, isRetriableError, + isRolledBackError, } from './retriableErrors.js'; const DEFAULT_MAX_QUEUE_SIZE = 1000; @@ -361,8 +362,14 @@ export class SQLBatcher { // Run one fallback item, retrying transient failures with backoff. // A read-only batcher may retry anything transient; a batcher that // carries writes only retries failures where the statement provably - // never reached the server — a write that died mid-flight may have - // committed, and re-running it would double-apply. + // did not apply — either it never reached the server, or the server + // rolled it back itself. A write that died mid-flight may have + // committed, and re-running that one would double-apply. + // + // Lock contention lands in the second group and is worth retrying rather + // than surfacing: an item is a single statement, so a deadlock victim has + // been fully undone, and the caller sees an unhandled 500 for what the + // database is telling us to just run again. async #runFallbackItem(b) { let attempt = 0; while (true) { @@ -379,8 +386,8 @@ export class SQLBatcher { }; } catch (error) { const canRetry = this.readOnly - ? isRetriableError(error) - : isNeverSentError(error); + ? isRetriableError(error) || isRolledBackError(error) + : isNeverSentError(error) || isRolledBackError(error); if (!canRetry || attempt >= ITEM_RETRY_ATTEMPTS) { return { ok: false, error }; } diff --git a/src/backend/clients/database/SQLBatcher.test.ts b/src/backend/clients/database/SQLBatcher.test.ts index c35162cc6..14c416bd4 100644 --- a/src/backend/clients/database/SQLBatcher.test.ts +++ b/src/backend/clients/database/SQLBatcher.test.ts @@ -208,6 +208,36 @@ describe('SQLBatcher', () => { expect(fallbackAttempts).toBe(2); }); + it('retries a deadlocked write instead of surfacing it', async () => { + let fallbackAttempts = 0; + const conn = makeConnection((sql) => { + if (isBatchQuery(sql)) throw makeError('ER_LOCK_DEADLOCK'); + fallbackAttempts++; + if (fallbackAttempts === 1) throw makeError('ER_LOCK_DEADLOCK'); + return [[{ ok: 1 }], undefined]; + }); + const { pool } = makePool(conn); + const batcher = new SQLBatcher(pool, { maxTimeInQueue: 5 }); + + // The victim statement was rolled back by the server, so re-running it + // can't double-apply — the caller should never see the deadlock. + const result = await batcher.query('UPDATE notification SET x', []); + expect(result[0]).toEqual([{ ok: 1 }]); + expect(fallbackAttempts).toBe(2); + }); + + it('gives up on a write that deadlocks past the retry budget', async () => { + const conn = makeConnection(() => { + throw makeError('ER_LOCK_DEADLOCK'); + }); + const { pool } = makePool(conn); + const batcher = new SQLBatcher(pool, { maxTimeInQueue: 5 }); + + await expect(batcher.query('UPDATE hot SET x', [])).rejects.toMatchObject( + { code: 'ER_LOCK_DEADLOCK' }, + ); + }); + it('never retries deterministic row-level errors and does not escalate the breaker', async () => { let fallbackAttempts = 0; const conn = makeConnection((sql) => { diff --git a/src/backend/clients/database/retriableErrors.ts b/src/backend/clients/database/retriableErrors.ts index ad8e129b2..12fee9ef6 100644 --- a/src/backend/clients/database/retriableErrors.ts +++ b/src/backend/clients/database/retriableErrors.ts @@ -54,6 +54,26 @@ const NEVER_SENT_ERROR_CODES = new Set([ POOL_ACQUIRE_TIMEOUT, ]); +/** + * Failures the server itself rolled back before returning. The statement did + * reach the database, so this is not `NEVER_SENT`, but InnoDB guarantees it + * left no effect — which makes a retry just as safe for writes. + * + * Only true for a _single_ statement: under autocommit each statement is its + * own transaction, so "the transaction was rolled back" means "this statement + * was rolled back". Retrying a multi-statement string on one of these would + * re-run the statements that already committed ahead of the failure. + */ +const ROLLED_BACK_ERROR_CODES = new Set([ + // Deadlock — InnoDB picked this transaction as the victim and undid it. + // MySQL's own message for it is "try restarting transaction". + 'ER_LOCK_DEADLOCK', + // Lock wait timeout. Rolls back the statement rather than the transaction + // unless innodb_rollback_on_timeout is set — the same thing when the + // transaction is one statement. + 'ER_LOCK_WAIT_TIMEOUT', +]); + const errorCode = (error: unknown): string | undefined => (error as { code?: string } | null)?.code; @@ -74,3 +94,13 @@ export const isNeverSentError = (error: unknown): boolean => { const code = errorCode(error); return Boolean(code && NEVER_SENT_ERROR_CODES.has(code)); }; + +/** + * Lock-contention failures the server rolled back on its own. Safe to retry a + * single statement on, writes included — see `ROLLED_BACK_ERROR_CODES` for the + * one-statement precondition. + */ +export const isRolledBackError = (error: unknown): boolean => { + const code = errorCode(error); + return Boolean(code && ROLLED_BACK_ERROR_CODES.has(code)); +}; diff --git a/src/backend/clients/event/types.ts b/src/backend/clients/event/types.ts index c0ca89a89..d5349c331 100644 --- a/src/backend/clients/event/types.ts +++ b/src/backend/clients/event/types.ts @@ -435,6 +435,14 @@ export type EventMap = { [K in `route.${string}`]: RouteLifecycleEvent; } & { [K in `pubsub.login.${string}`]: { authtoken: string }; +} & { + /** + * A user's subscription now resolves to a different policy. Carried on the + * `outer.pubsub.*` channel so it reaches sibling nodes and peer clusters, + * not just the one that handled the change — every node caches the resolved + * policy, so a purchase is only live once they have all dropped theirs. + */ + 'outer.pubsub.metering.subscription-changed': { userUuid: string }; }; /** diff --git a/src/backend/clients/s3/S3Client.ts b/src/backend/clients/s3/S3Client.ts index d2e8f5b83..e682c9d5f 100644 --- a/src/backend/clients/s3/S3Client.ts +++ b/src/backend/clients/s3/S3Client.ts @@ -21,6 +21,7 @@ import { AbortMultipartUploadCommand, CompleteMultipartUploadCommand, CreateMultipartUploadCommand, + HeadBucketCommand, PutObjectCommand, S3Client as AwsS3Client, type S3ClientConfig, @@ -244,6 +245,19 @@ export class S3Client extends PuterClient { return client; } + /** + * Cheapest round-trip that proves the object store answers for a bucket: no + * object data, no listing, just a HEAD. Throws on any failure (missing + * bucket, bad credentials, unreachable endpoint) so callers can treat it as + * a liveness probe. Reuses the pooled per-region client. + */ + async headBucket( + bucket = this.config.s3_bucket || LEGACY_STORAGE_BUCKET, + region?: string, + ): Promise { + await this.get(region).send(new HeadBucketCommand({ Bucket: bucket })); + } + // ------------------------------------------------------------------ // Legacy storage migration // ------------------------------------------------------------------ diff --git a/src/backend/controllers/apps/AppController.js b/src/backend/controllers/apps/AppController.js index 8d8e6280f..002f5912e 100644 --- a/src/backend/controllers/apps/AppController.js +++ b/src/backend/controllers/apps/AppController.js @@ -35,6 +35,39 @@ import DEFAULT_APP_ICON from './default-app-icon.js'; * are just thin shape adapters that translate REST conventions into driver * calls. */ +/** + * Desktop boot reads the app list and individual app records repeatedly, so the + * ceiling is set well above normal boot traffic and exists to catch a runaway + * client rather than to pace one. + * + * "Repeatedly" is the operative word: an app record is read on launch, on + * permission checks, and again by anything resolving an app by name, so these + * accumulate against whatever else a session is doing rather than arriving on + * their own. Sized for a session working hard, not for a person clicking. + */ +const APP_READ_LIMIT = { + scope: 'app-read', + limit: 1_800, + window: 60_000, + key: 'user', +}; + +/** + * Unauthenticated icon serving; no actor to key on, so the bucket is the + * address — and an address is a NAT, a campus or a carrier gateway that can + * hold hundreds of desktops. Each desktop boot pulls the taskbar's icons at + * once, so a single burst from one network is already thousands of requests. + * Responses are publicly cacheable and usually a redirect, so the ceiling is + * not protecting bandwidth; it is there so a client looping on a broken icon + * can't spin unbounded. + */ +const APP_ICON_LIMIT = { + scope: 'app-icon', + limit: 12_000, + window: 60_000, + key: 'ip', +}; + export class AppController extends PuterController { get appStore() { return this.stores.app; @@ -110,6 +143,7 @@ export class AppController extends PuterController { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true, + rateLimit: APP_READ_LIMIT, }, async (req, res) => { const apps = await this.appDriver.select({ @@ -125,6 +159,15 @@ export class AppController extends PuterController { { subdomain: 'api', requireAuth: true, + // Answers "does this name exist?" for any name, so it is a + // name-enumeration oracle however cheap it is to serve. + // Mirrors the `isNameAvailable` budget on AppDriver. + rateLimit: { + scope: 'app-name-available', + limit: 60, + window: 60_000, + key: 'user', + }, }, async (req, res) => { const name = req.query?.name; @@ -159,6 +202,7 @@ export class AppController extends PuterController { { subdomain: 'api', requireAuth: true, + rateLimit: APP_READ_LIMIT, }, async (req, res) => { const actor = req.actor; @@ -217,6 +261,7 @@ export class AppController extends PuterController { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true, + rateLimit: APP_READ_LIMIT, }, async (req, res) => { const raw = req.params.name; @@ -290,6 +335,7 @@ export class AppController extends PuterController { { subdomain: 'api', requireAuth: true, + rateLimit: APP_READ_LIMIT, }, async (req, res) => { const appList = Array.isArray(req.body) ? req.body : []; @@ -498,10 +544,14 @@ export class AppController extends PuterController { // Icons are targets from the GUI (root) AND resolved via // api_base_url in taskbar payloads. Register on both so either origin // works without a cross-subdomain redirect. - router.get('/app-icon/:app_uid', { subdomain: ['api', ''] }, serveIcon); + router.get( + '/app-icon/:app_uid', + { subdomain: ['api', ''], rateLimit: APP_ICON_LIMIT }, + serveIcon, + ); router.get( '/app-icon/:app_uid/:size', - { subdomain: ['api', ''] }, + { subdomain: ['api', ''], rateLimit: APP_ICON_LIMIT }, serveIcon, ); } diff --git a/src/backend/controllers/apps/AppController.test.ts b/src/backend/controllers/apps/AppController.test.ts index 950764e07..b9e06e4de 100644 --- a/src/backend/controllers/apps/AppController.test.ts +++ b/src/backend/controllers/apps/AppController.test.ts @@ -1018,3 +1018,25 @@ describe('AppController GET /app-icon remote icons', () => { expect(captured.headers['content-type']).toContain('image/svg+xml'); }); }); + +// ── app-icon rate limit ───────────────────────────────────────────── + +describe('AppController app-icon rate limit', () => { + it('sizes the icon bucket for a whole network rather than one desktop', () => { + // Icons are unauthenticated `` targets, so the only key is + // the address — which one NAT shares across every desktop behind it, + // and each boot pulls the taskbar's icons in a burst. + for (const path of ['/app-icon/:app_uid', '/app-icon/:app_uid/:size']) { + const route = router.routes.find( + (r) => r.method === 'get' && r.path === path, + ); + if (!route) throw new Error(`No GET ${path} route`); + expect(route.options.rateLimit).toEqual({ + scope: 'app-icon', + limit: 12_000, + window: 60_000, + key: 'ip', + }); + } + }); +}); diff --git a/src/backend/controllers/auth/AuthController.test.ts b/src/backend/controllers/auth/AuthController.test.ts index 9967f6b75..b9f698da4 100644 --- a/src/backend/controllers/auth/AuthController.test.ts +++ b/src/backend/controllers/auth/AuthController.test.ts @@ -42,6 +42,7 @@ import type { TokenSource } from '../../core/http/types.js'; import { PuterServer } from '../../server.js'; import { FULL_API_ACCESS } from '../../services/permission/consts.js'; import { setupTestServer } from '../../testUtil.js'; +import { FS_READ_LIMIT } from '../fs/limits.js'; // ── Test harness ──────────────────────────────────────────────────── @@ -1390,6 +1391,26 @@ describe('AuthController account-lifecycle route gating', () => { expect(opts.requireUserActor).toBe(true); }); + // A fresh token is minted per protected mutation and nothing caches them, + // so issuance has to outrun the COMBINED rate of everything that spends + // one. The session-authenticated download path dominates — a multi-select + // download spends a token per file — with logout and the handful of + // session-management writes behind it. + it('GET /get-anticsrf-token clears the budgets that consume tokens', () => { + type Window = { limit: number; window: number }; + const issuance = routeOptions('get', '/get-anticsrf-token') + .rateLimit as Window; + const logout = routeOptions('post', '/logout').rateLimit as Window; + + expect(issuance.window).toBe(60_000); + expect(logout.window).toBe(60_000); + expect(FS_READ_LIMIT.window).toBe(60_000); + + expect(issuance.limit).toBeGreaterThan( + FS_READ_LIMIT.limit + logout.limit, + ); + }); + it('requireUserActorGate rejects app-under-user and access-token actors', () => { const gate = requireUserActorGate(); const run = (actor: Partial) => diff --git a/src/backend/controllers/auth/AuthController.ts b/src/backend/controllers/auth/AuthController.ts index c5d2c0992..072c2d556 100644 --- a/src/backend/controllers/auth/AuthController.ts +++ b/src/backend/controllers/auth/AuthController.ts @@ -83,6 +83,94 @@ const DEFAULT_CARD_FALLBACK_ATTEMPTS = 2; // crossed. const SEND_PHONE_RATE_LIMIT = 10; const SEND_PHONE_RATE_WINDOW_MS = 60 * 60_000; + +// -- Post-login route limits ----------------------------------------- +// +// The credential legs above (login, signup, recovery, confirmation) each +// carry their own limit. Everything a session can reach *after* signing +// in shares the four shapes below, keyed on the actor rather than the +// network — a per-account ceiling is the meaningful one once we know who +// is calling. + +/** + * Mints or reconfigures a credential. Deliberately an hour-scale window: these + * are human actions taken a handful of times, and an unbounded rate turns one + * compromised session into a durable foothold. + */ +const CREDENTIAL_MINT_LIMIT = { + scope: 'auth-credential-mint', + limit: 20, + window: 60 * 60_000, + key: 'user', +} as const; + +/** + * Second-factor configuration, including the verify leg. Shorter window than + * the mint limit because enabling 2FA legitimately involves a few attempts in a + * row, but unbounded verification is a TOTP brute force. + */ +const TWO_FACTOR_LIMIT = { + scope: 'auth-2fa-configure', + limit: 30, + window: 15 * 60_000, + key: 'user', +} as const; + +/** Permission and membership writes. Never called in a loop by a client. */ +const GRANT_LIMIT = { + scope: 'auth-grant', + limit: 60, + window: 60_000, + key: 'user', +} as const; + +/** + * Read-only checks the GUI makes on nearly every interaction. The ceiling is + * high enough that only a runaway loop reaches it. + */ +const AUTH_CHECK_LIMIT = { + scope: 'auth-check', + limit: 300, + window: 60_000, + key: 'user', +} as const; + +/** + * Anti-CSRF token issuance. Clients mint a fresh token per protected mutation + * and cache nothing, so this ceiling has to clear the SUM of the budgets that + * spend tokens — matching any single one of them guarantees the gate fires + * before the mutation it guards does. + * + * What spends them: the session-authenticated download path, one token per + * file, at the read budget of 600/min — a multi-selection download burns tokens + * exactly the way a bulk delete burns its own budget, so this tracks the bulk + * figure used for filesystem mutations; logout at 60/min; and the + * session-management writes (revoke, rename), which a person triggers a handful + * of times. Call it ~700/min of real demand, and leave enough on top that a + * bulk operation runs out of files before it runs out of tokens. + */ +const ANTI_CSRF_MINT_LIMIT = { + scope: 'anticsrf', + limit: 1200, + window: 60_000, + key: 'user', +} as const; + +/** Settings-page reads — enumerating sessions, permissions, groups. */ +const AUTH_LIST_LIMIT = { + scope: 'auth-list', + limit: 120, + window: 60_000, + key: 'user', +} as const; + +/** Session plumbing: logout, GUI token, cookie sync. */ +const SESSION_LIMIT = { + scope: 'auth-session', + limit: 60, + window: 60_000, + key: 'user', +} as const; // Once the threshold is crossed the fallback stays open this long, so the // user can finish the card flow without racing the attempt counter's expiry. const CARD_FALLBACK_OPEN_TTL_SECONDS = 24 * 60 * 60; @@ -1013,7 +1101,8 @@ export class AuthController extends PuterController { is_temp: user!.password === null && user!.email === null, ip: (req?.headers?.['x-forwarded-for'] as - string | undefined) || + | string + | undefined) || ( req as unknown as { connection?: { remoteAddress?: string }; @@ -1049,6 +1138,7 @@ export class AuthController extends PuterController { requireUserActor: true, allowUnconfirmed: true, antiCsrf: true, + rateLimit: SESSION_LIMIT, }) async handleLogout(req: Request, res: Response): Promise { // Clear the session cookie + `puter_token_v2`. Nothing issues the @@ -2712,7 +2802,25 @@ export class AuthController extends PuterController { // -- Captcha generation ------------------------------------------- - @Get('/api/captcha/generate', { subdomain: '*' }) + @Get('/api/captcha/generate', { + subdomain: '*', + // Unauthenticated, renders an image per call, and is the gate + // protecting /login and /signup — so bulk pre-generation is + // directly useful to an attacker. Per-fingerprint for fairness on + // shared IPs, plus a per-IP backstop against header rotation. + // + // The fingerprint bucket is the one sized for a person: a handful of + // refreshes while getting a captcha right. The IP bucket is not — one + // address is a whole office, campus or carrier gateway, and everyone + // behind it is signing in through the same counter, so sizing it for + // a browser would deny the captcha to a network rather than to an + // attacker. It stays wide enough for that population and narrow + // enough that header rotation still runs out. + rateLimit: [ + { scope: 'captcha', limit: 30, window: 60_000 }, + { scope: 'captcha-ip', limit: 3_000, window: 60_000, key: 'ip' }, + ], + }) async handleCaptchaGenerate(_req: Request, res: Response): Promise { const difficulty = (this.config as { captcha?: { difficulty?: string } }).captcha @@ -2724,6 +2832,7 @@ export class AuthController extends PuterController { // -- Anti-CSRF token generation ---------------------------------- @Get('/get-anticsrf-token', { + rateLimit: ANTI_CSRF_MINT_LIMIT, // Anti-CSRF tokens are only consumed by `requireUserActor` routes, // so issuance is scoped to the same actor kind for consistency. requireUserActor: true, @@ -2744,6 +2853,7 @@ export class AuthController extends PuterController { @Post('/auth/grant-user-user', { subdomain: 'api', requireUserActor: true, + rateLimit: GRANT_LIMIT, }) async handleGrantUserUser(req: Request, res: Response): Promise { const { target_username, permission, extra, meta } = req.body; @@ -2939,6 +3049,7 @@ export class AuthController extends PuterController { @Post('/auth/grant-user-app', { subdomain: 'api', requireUserActor: true, + rateLimit: GRANT_LIMIT, }) async handleGrantUserApp(req: Request, res: Response): Promise { let { app_uid } = req.body; @@ -2986,6 +3097,7 @@ export class AuthController extends PuterController { @Post('/auth/grant-user-group', { subdomain: 'api', requireUserActor: true, + rateLimit: GRANT_LIMIT, }) async handleGrantUserGroup(req: Request, res: Response): Promise { const { group_uid, permission, extra, meta } = req.body; @@ -3014,6 +3126,7 @@ export class AuthController extends PuterController { @Post('/auth/revoke-user-user', { subdomain: 'api', requireUserActor: true, + rateLimit: GRANT_LIMIT, }) async handleRevokeUserUser(req: Request, res: Response): Promise { const { target_username, permission, meta } = req.body; @@ -3036,6 +3149,7 @@ export class AuthController extends PuterController { @Post('/auth/revoke-user-app', { subdomain: 'api', requireUserActor: true, + rateLimit: GRANT_LIMIT, }) async handleRevokeUserApp(req: Request, res: Response): Promise { let { app_uid } = req.body; @@ -3079,6 +3193,7 @@ export class AuthController extends PuterController { @Post('/auth/revoke-user-group', { subdomain: 'api', requireUserActor: true, + rateLimit: GRANT_LIMIT, }) async handleRevokeUserGroup(req: Request, res: Response): Promise { const { group_uid, permission, meta } = req.body; @@ -3098,7 +3213,11 @@ export class AuthController extends PuterController { // -- Permission checks ------------------------------------------- - @Post('/auth/check-permissions', { subdomain: 'api', requireAuth: true }) + @Post('/auth/check-permissions', { + subdomain: 'api', + requireAuth: true, + rateLimit: AUTH_CHECK_LIMIT, + }) async handleCheckPermissions(req: Request, res: Response): Promise { const { permissions } = req.body; if (!Array.isArray(permissions)) { @@ -3126,7 +3245,11 @@ export class AuthController extends PuterController { // -- Session management ------------------------------------------ - @Get('/auth/list-sessions', { subdomain: 'api', requireUserActor: true }) + @Get('/auth/list-sessions', { + subdomain: 'api', + requireUserActor: true, + rateLimit: AUTH_LIST_LIMIT, + }) async handleListSessions(req: Request, res: Response): Promise { const sessions = await this.services.auth.listSessions(req.actor!); res.json(sessions); @@ -3208,7 +3331,11 @@ export class AuthController extends PuterController { // -- Dev app permissions ----------------------------------------- - @Post('/auth/grant-dev-app', { subdomain: 'api', requireUserActor: true }) + @Post('/auth/grant-dev-app', { + subdomain: 'api', + requireUserActor: true, + rateLimit: GRANT_LIMIT, + }) async handleGrantDevApp(req: Request, res: Response): Promise { let { app_uid } = req.body; const { origin, permission, extra, meta } = req.body; @@ -3241,6 +3368,7 @@ export class AuthController extends PuterController { @Post('/auth/revoke-dev-app', { subdomain: 'api', requireUserActor: true, + rateLimit: GRANT_LIMIT, }) async handleRevokeDevApp(req: Request, res: Response): Promise { let { app_uid } = req.body; @@ -3276,6 +3404,7 @@ export class AuthController extends PuterController { @Get('/auth/list-permissions', { subdomain: 'api', requireUserActor: true, + rateLimit: AUTH_LIST_LIMIT, }) async handleListPermissions(req: Request, res: Response): Promise { const userId = req.actor!.user.id; @@ -3340,7 +3469,11 @@ export class AuthController extends PuterController { // -- App origin resolution --------------------------------------- - @Post('/auth/app-uid-from-origin', { subdomain: 'api', requireAuth: true }) + @Post('/auth/app-uid-from-origin', { + subdomain: 'api', + requireAuth: true, + rateLimit: AUTH_CHECK_LIMIT, + }) async handleAppUidFromOrigin(req: Request, res: Response): Promise { const origin = req.body?.origin || req.query?.origin; if (!origin) @@ -3356,6 +3489,9 @@ export class AuthController extends PuterController { @Post('/auth/get-user-app-token', { subdomain: 'api', requireUserActor: true, + // Called once per app launch, and the GUI can legitimately launch + // several in quick succession. + rateLimit: { ...AUTH_CHECK_LIMIT, scope: 'app-token', limit: 120 }, }) async handleGetUserAppToken(req: Request, res: Response): Promise { let { app_uid } = req.body; @@ -3474,7 +3610,11 @@ export class AuthController extends PuterController { res.json({ token, app_uid }); } - @Post('/auth/check-app', { subdomain: 'api', requireUserActor: true }) + @Post('/auth/check-app', { + subdomain: 'api', + requireUserActor: true, + rateLimit: AUTH_CHECK_LIMIT, + }) async handleCheckApp(req: Request, res: Response): Promise { let { app_uid } = req.body; const { origin } = req.body; @@ -3513,6 +3653,7 @@ export class AuthController extends PuterController { @Post('/auth/create-access-token', { subdomain: 'api', requireAuth: true, + rateLimit: CREDENTIAL_MINT_LIMIT, }) async handleCreateAccessToken(req: Request, res: Response): Promise { const { permissions, expiresIn, label } = req.body; @@ -3581,6 +3722,7 @@ export class AuthController extends PuterController { @Post('/auth/configure-2fa/:action', { subdomain: 'api', requireUserActor: true, + rateLimit: TWO_FACTOR_LIMIT, }) async handleConfigure2fa(req: Request, res: Response): Promise { const action = req.params.action; @@ -3711,7 +3853,11 @@ export class AuthController extends PuterController { // -- Developer profile ------------------------------------------- - @Get('/get-dev-profile', { subdomain: 'api', requireUserActor: true }) + @Get('/get-dev-profile', { + subdomain: 'api', + requireUserActor: true, + rateLimit: AUTH_LIST_LIMIT, + }) async handleGetDevProfile(req: Request, res: Response): Promise { const user = await this.stores.user.getById(req.actor!.user.id!, { force: true, @@ -3741,7 +3887,13 @@ export class AuthController extends PuterController { // -- Group management -------------------------------------------- - @Post('/group/create', { subdomain: 'api', requireUserActor: true }) + @Post('/group/create', { + subdomain: 'api', + requireUserActor: true, + // Creates a persistent row per call with no quota behind it, so it + // sits on the hour-scale budget rather than the grant one. + rateLimit: { ...CREDENTIAL_MINT_LIMIT, scope: 'group-create' }, + }) async handleGroupCreate(req: Request, res: Response): Promise { const extra = req.body.extra ?? {}; const metadata = req.body.metadata ?? {}; @@ -3762,7 +3914,11 @@ export class AuthController extends PuterController { res.json({ uid }); } - @Post('/group/add-users', { subdomain: 'api', requireUserActor: true }) + @Post('/group/add-users', { + subdomain: 'api', + requireUserActor: true, + rateLimit: GRANT_LIMIT, + }) async handleGroupAddUsers(req: Request, res: Response): Promise { const { uid, users } = req.body ?? {}; if (!uid) @@ -3794,7 +3950,11 @@ export class AuthController extends PuterController { res.json({}); } - @Post('/group/remove-users', { subdomain: 'api', requireUserActor: true }) + @Post('/group/remove-users', { + subdomain: 'api', + requireUserActor: true, + rateLimit: GRANT_LIMIT, + }) async handleGroupRemoveUsers(req: Request, res: Response): Promise { const { uid, users } = req.body ?? {}; if (!uid) @@ -3826,7 +3986,11 @@ export class AuthController extends PuterController { res.json({}); } - @Get('/group/list', { subdomain: 'api', requireUserActor: true }) + @Get('/group/list', { + subdomain: 'api', + requireUserActor: true, + rateLimit: AUTH_LIST_LIMIT, + }) async handleGroupList(req: Request, res: Response): Promise { const userId = req.actor!.user.id!; const [owned, member] = await Promise.all([ @@ -3839,7 +4003,22 @@ export class AuthController extends PuterController { }); } - @Get('/group/public-groups', { subdomain: 'api' }) + @Get('/group/public-groups', { + subdomain: 'api', + // The only unauthenticated route in the group set, so IP is the + // only key available — and that makes the bucket an aggregate: + // one office, campus or carrier gateway is a single key for + // everybody behind it, and each of them reads this once while + // bootstrapping. Sized for that population of real people rather + // than one browser, and no wider: this sits next to the sign-in + // surface, so it stays a real bound on enumeration. + rateLimit: { + scope: 'public-groups', + limit: 1_200, + window: 60_000, + key: 'ip', + }, + }) async handleGroupPublicGroups(_req: Request, res: Response): Promise { res.json({ user: this.config.default_user_group ?? null, @@ -3852,6 +4031,7 @@ export class AuthController extends PuterController { @Get('/get-gui-token', { requireUserActor: true, allowUnconfirmed: true, + rateLimit: SESSION_LIMIT, }) async handleGetGuiToken(req: Request, res: Response): Promise { if (!req.actor?.session?.uid) @@ -3871,6 +4051,7 @@ export class AuthController extends PuterController { } @Get('/session/sync-cookie', { + rateLimit: SESSION_LIMIT, // Installs the session cookie. Only page script on our own origin // should be able to ask for that (the `tokenSource` check below is the // companion rule: the token has to come from an Authorization header, diff --git a/src/backend/controllers/desktop/DesktopController.js b/src/backend/controllers/desktop/DesktopController.js index 88229eeaa..4f2b2a5dc 100644 --- a/src/backend/controllers/desktop/DesktopController.js +++ b/src/backend/controllers/desktop/DesktopController.js @@ -32,6 +32,19 @@ const ALLOWED_SORT_ORDER = ['asc', 'desc']; * - User-level: desktop background, taskbar items (UserStore) * - Folder-level: layout, sort_by/sort_order (fsentries table) */ +/** + * Desktop preference writes — background, taskbar, layout, sort order. All four + * persist to the user row on every call, and the GUI fires them on direct user + * action, so a per-minute ceiling well above human speed is enough to catch a + * stuck client. + */ +const PREFERENCE_WRITE_LIMIT = { + scope: 'desktop-preference', + limit: 120, + window: 60_000, + key: 'user', +}; + export class DesktopController extends PuterController { constructor(config, clients, stores, services) { super(config, clients, stores, services); @@ -53,6 +66,7 @@ export class DesktopController extends PuterController { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true, + rateLimit: PREFERENCE_WRITE_LIMIT, }, async (req, res) => { const { url, color, fit } = req.body ?? {}; @@ -108,6 +122,7 @@ export class DesktopController extends PuterController { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true, + rateLimit: PREFERENCE_WRITE_LIMIT, }, async (req, res) => { const { items } = req.body ?? {}; @@ -134,6 +149,7 @@ export class DesktopController extends PuterController { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true, + rateLimit: PREFERENCE_WRITE_LIMIT, }, async (req, res) => { const { item_uid, item_path, layout } = req.body ?? {}; @@ -161,6 +177,7 @@ export class DesktopController extends PuterController { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true, + rateLimit: PREFERENCE_WRITE_LIMIT, }, async (req, res) => { const { item_uid, item_path, sort_by, sort_order } = diff --git a/src/backend/controllers/drivers/DriverController.concurrent.test.ts b/src/backend/controllers/drivers/DriverController.concurrent.test.ts index d380cd209..0a053ad93 100644 --- a/src/backend/controllers/drivers/DriverController.concurrent.test.ts +++ b/src/backend/controllers/drivers/DriverController.concurrent.test.ts @@ -124,9 +124,9 @@ const makeSyntheticDriver = () => ({ const buildController = (driver: ReturnType) => { // The controller reads `this.services?.permission` only when an actor - // is on the request; otherwise the services bag is unused. The - // alarm client is invoked on the rate-limit / concurrency rejection - // paths, so stub a no-op `create`. + // is on the request; otherwise the services bag is unused. The rejection + // paths no longer alarm, but other paths still reach the client, so keep + // a no-op `create` stubbed. const clients = { alarm: { create: () => {} } }; return new DriverController( {} as any, diff --git a/src/backend/controllers/drivers/DriverController.errors.test.ts b/src/backend/controllers/drivers/DriverController.errors.test.ts index 3974de2d7..812b01e83 100644 --- a/src/backend/controllers/drivers/DriverController.errors.test.ts +++ b/src/backend/controllers/drivers/DriverController.errors.test.ts @@ -409,7 +409,7 @@ describe('DriverController per-method rate limiting', () => { configureRateLimit({ disabled: false } as never); }); - it('answers 429 and raises a de-duped info alarm once the per-method budget is spent', async () => { + it('answers 429 without alarming once the per-method budget is spent', async () => { const { handler, alarms, iface } = build({ run: () => ({ ok: true }), rateLimit: { default: { limit: 1, window: 60_000 } }, @@ -430,9 +430,8 @@ describe('DriverController per-method rate limiting', () => { legacyCode: 'too_many_requests', }); - expect(alarms).toContainEqual({ - id: 'driver_rate_limit_hit:rate-limited-iface:run', - severity: 'info', - }); + // Spending your own budget is the limit working as designed, so it + // must not raise anything — the 429 is the whole signal. + expect(alarms).toEqual([]); }); }); diff --git a/src/backend/controllers/drivers/DriverController.test.ts b/src/backend/controllers/drivers/DriverController.test.ts index daa339cf3..156da29b1 100644 --- a/src/backend/controllers/drivers/DriverController.test.ts +++ b/src/backend/controllers/drivers/DriverController.test.ts @@ -158,11 +158,14 @@ interface MockRes { sentBody: string | undefined; contentType: string | undefined; pipedFrom: Readable | undefined; + listeners: Record void>>; status(code: number): MockRes; json(body: unknown): MockRes; setHeader(key: string, value: string): MockRes; type(t: string): MockRes; send(body: string): MockRes; + once(event: string, fn: () => void): MockRes; + emit(event: string): void; } const makeRes = (): MockRes => { const res: MockRes = { @@ -172,6 +175,19 @@ const makeRes = (): MockRes => { sentBody: undefined, contentType: undefined, pipedFrom: undefined, + // `#handleCall` releases a driver's concurrency slot on `finish` / + // `close`, so the stub has to behave like an emitter for any driver + // that declares a `concurrent` policy. + listeners: {}, + once(event: string, fn: () => void) { + (this.listeners[event] ??= []).push(fn); + return this; + }, + emit(event: string) { + const fns = this.listeners[event] ?? []; + this.listeners[event] = []; + for (const fn of fns) fn(); + }, status(code: number) { this.statusCode = code; return this; diff --git a/src/backend/controllers/drivers/DriverController.ts b/src/backend/controllers/drivers/DriverController.ts index 9bad62181..d91cb7c17 100644 --- a/src/backend/controllers/drivers/DriverController.ts +++ b/src/backend/controllers/drivers/DriverController.ts @@ -45,6 +45,27 @@ import { PuterController } from '../types.js'; type DriverInstance = WithLifecycle & Record; +/** + * Coarse envelope over the whole `/call` surface, so that spreading calls + * across many interfaces can't dodge every individual bucket. Per-driver limits + * are what actually shape traffic. + * + * "Coarse" is a constraint, not a description: for this to be an envelope it + * has to sit _above_ every per-driver budget, or it silently becomes the real + * limit for the widest ones and overrides the tier policy they declare. + * `driverPolicies.test.ts` asserts that ordering against every registered + * driver, so raising a driver's budget past this number fails there rather than + * in production. The headroom above the widest driver (notifications, at + * 3000/30s) is what leaves room for one caller to be busy on two interfaces at + * once. + */ +export const DRIVERS_CALL_LIMIT = { + scope: 'drivers-call', + limit: 8000, + window: 60_000, + key: 'user' as const, +}; + // Every driver call is timed here already, for the lifecycle events below. // Recording the same number as a histogram makes the per-interface latency // distribution available downstream; which interfaces are worth keeping is a @@ -180,12 +201,26 @@ export class DriverController extends PuterController { registerRoutes(router: PuterRouter): void { router.post( '/call', - { subdomain: 'api', requireAuth: true }, + { + subdomain: 'api', + requireAuth: true, + rateLimit: DRIVERS_CALL_LIMIT, + }, this.#handleCall, ); router.get( '/list-interfaces', - { subdomain: 'api', requireAuth: true }, + { + subdomain: 'api', + requireAuth: true, + // Static introspection output, read once at boot. + rateLimit: { + scope: 'drivers-list-interfaces', + limit: 60, + window: 60_000, + key: 'user', + }, + }, this.#handleListInterfaces, ); } @@ -305,19 +340,8 @@ export class DriverController extends PuterController { if ( !(await checkDriverRateLimit(req, ifaceName, method, rateLimitSpec)) ) { - // De-dupe on (iface, method) so a hot loop across many users - // aggregates as occurrences on a single low-severity alarm - // instead of fanning out one per user. - this.clients.alarm.create( - `driver_rate_limit_hit:${ifaceName}:${method}`, - `Driver rate limit hit on ${ifaceName}:${method}`, - { - iface: ifaceName, - method, - userUuid: req.actor?.user?.uuid, - }, - 'info', - ); + // Deliberately unalarmed: a caller spending its own budget is + // the limit working, not an incident. The 429 is the signal. throw new HttpError(429, 'Too many requests.', { legacyCode: 'too_many_requests', }); @@ -340,16 +364,8 @@ export class DriverController extends PuterController { concurrentSpec, ); if (!handle.ok) { - this.clients.alarm.create( - `driver_concurrent_limit_hit:${ifaceName}:${method}`, - `Driver concurrency limit hit on ${ifaceName}:${method}`, - { - iface: ifaceName, - method, - userUuid: req.actor?.user?.uuid, - }, - 'info', - ); + // Unalarmed for the same reason as the rate-limit rejection + // above: hitting a declared cap is the cap doing its job. throw new HttpError(429, 'Too many concurrent requests.', { legacyCode: 'too_many_requests', }); diff --git a/src/backend/controllers/fs/FSController.ts b/src/backend/controllers/fs/FSController.ts index f9d2ac1d2..a80b529be 100644 --- a/src/backend/controllers/fs/FSController.ts +++ b/src/backend/controllers/fs/FSController.ts @@ -39,6 +39,18 @@ import { import { applyInlineContentSecurity } from '../../util/inlineContentSecurity.js'; import { PuterController } from '../types.js'; import { FS_COSTS } from './costs.js'; +import { + FS_MULTIPART_LIMIT, + FS_MUTATE_LIMIT, + FS_READ_CONCURRENT, + FS_READ_LIMIT, + FS_READDIR_LIMIT, + FS_SEARCH_CONCURRENT, + FS_SEARCH_LIMIT, + FS_STAT_LIMIT, + FS_WRITE_CONCURRENT, + FS_WRITE_LIMIT, +} from './limits.js'; import { assertAccess as assertLegacyAccess, fsEntryMimeType, @@ -114,7 +126,11 @@ export class FSController extends PuterController { })); } - @Post('/startWrite', { subdomain: 'api', requireVerified: true }) + @Post('/startWrite', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MULTIPART_LIMIT, + }) async startWrite( req: Request, res: Response, @@ -172,7 +188,11 @@ export class FSController extends PuterController { ); } - @Post('/startBatchWrite', { subdomain: 'api', requireVerified: true }) + @Post('/startBatchWrite', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MULTIPART_LIMIT, + }) async startBatchWrites( req: Request, res: Response, @@ -272,7 +292,11 @@ export class FSController extends PuterController { ); } - @Post('/completeWrite', { subdomain: 'api', requireVerified: true }) + @Post('/completeWrite', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MULTIPART_LIMIT, + }) async completeWrite( req: Request, res: Response, @@ -303,7 +327,11 @@ export class FSController extends PuterController { ); } - @Post('/completeBatchWrite', { subdomain: 'api', requireVerified: true }) + @Post('/completeBatchWrite', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MULTIPART_LIMIT, + }) async completeBatchWrites( req: Request, res: Response, @@ -346,7 +374,11 @@ export class FSController extends PuterController { ); } - @Post('/abortWrite', { subdomain: 'api', requireVerified: true }) + @Post('/abortWrite', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MULTIPART_LIMIT, + }) async abortWrite( req: Request, res: Response<{ ok: true }>, @@ -362,7 +394,11 @@ export class FSController extends PuterController { res.json({ ok: true }); } - @Post('/signMultipartParts', { subdomain: 'api', requireVerified: true }) + @Post('/signMultipartParts', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MULTIPART_LIMIT, + }) async signMultipartParts( req: Request, res: Response, @@ -375,7 +411,12 @@ export class FSController extends PuterController { res.json(this.#withoutStorageInternals(response)); } - @Post('/write', { subdomain: 'api', requireVerified: true }) + @Post('/write', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_WRITE_LIMIT, + concurrent: FS_WRITE_CONCURRENT, + }) async write( req: Request, res: Response, @@ -421,7 +462,12 @@ export class FSController extends PuterController { res.json(this.#withRequiredClientFsEntry(updatedResponse)); } - @Post('/batchWrite', { subdomain: 'api', requireVerified: true }) + @Post('/batchWrite', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_WRITE_LIMIT, + concurrent: FS_WRITE_CONCURRENT, + }) async batchWrites( req: Request, res: Response, @@ -888,7 +934,11 @@ export class FSController extends PuterController { // -- Read-side routes ------------------------------------------------ - @Post('/stat', { subdomain: 'api', requireVerified: true }) + @Post('/stat', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_STAT_LIMIT, + }) async statEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const userId = this.#getActorUserId(req); @@ -1002,12 +1052,20 @@ export class FSController extends PuterController { * without a JSON body. Every value arrives as a string, so parameter * parsing goes through the same coercion helpers the POST path uses. */ - @Get('/readdir', { subdomain: 'api', requireVerified: true }) + @Get('/readdir', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_READDIR_LIMIT, + }) async readdirEntriesViaGet(req: Request, res: Response) { return this.readdirEntries(req, res); } - @Post('/readdir', { subdomain: 'api', requireVerified: true }) + @Post('/readdir', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_READDIR_LIMIT, + }) async readdirEntries(req: Request, res: Response) { const actor = this.#requireActor(req); // GET carries its parameters in the query string; POST in the body. @@ -1203,7 +1261,12 @@ export class FSController extends PuterController { } } - @Post('/search', { subdomain: 'api', requireVerified: true }) + @Post('/search', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_SEARCH_LIMIT, + concurrent: FS_SEARCH_CONCURRENT, + }) async searchEntries(req: Request, res: Response) { const actor = this.#requireActor(req); const userId = this.#getActorUserId(req); @@ -1229,7 +1292,12 @@ export class FSController extends PuterController { res.json(results); } - @Get('/read', { subdomain: 'api', requireVerified: true }) + @Get('/read', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_READ_LIMIT, + concurrent: FS_READ_CONCURRENT, + }) async readEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const query = this.#toObjectRecord(req.query); @@ -1310,7 +1378,11 @@ export class FSController extends PuterController { // -- Mutation routes ------------------------------------------------ - @Post('/mkdir', { subdomain: 'api', requireVerified: true }) + @Post('/mkdir', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MUTATE_LIMIT, + }) async mkdirEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const userId = this.#getActorUserId(req); @@ -1351,7 +1423,11 @@ export class FSController extends PuterController { res.json(this.#toClientEntry(entry)); } - @Post('/touch', { subdomain: 'api', requireVerified: true }) + @Post('/touch', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MUTATE_LIMIT, + }) async touchEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const userId = this.#getActorUserId(req); @@ -1387,7 +1463,11 @@ export class FSController extends PuterController { res.json(this.#toClientEntry(entry)); } - @Post('/rename', { subdomain: 'api', requireVerified: true }) + @Post('/rename', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MUTATE_LIMIT, + }) async renameEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const body = this.#toObjectRecord(req.body); @@ -1405,7 +1485,11 @@ export class FSController extends PuterController { res.json(this.#toClientEntry(renamed)); } - @Post('/delete', { subdomain: 'api', requireVerified: true }) + @Post('/delete', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MUTATE_LIMIT, + }) async deleteEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const userId = this.#getActorUserId(req); @@ -1423,7 +1507,11 @@ export class FSController extends PuterController { res.json({ ok: true }); } - @Post('/move', { subdomain: 'api', requireVerified: true }) + @Post('/move', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MUTATE_LIMIT, + }) async moveEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const userId = this.#getActorUserId(req); @@ -1451,7 +1539,11 @@ export class FSController extends PuterController { res.json(this.#toClientEntry(moved)); } - @Post('/copy', { subdomain: 'api', requireVerified: true }) + @Post('/copy', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MUTATE_LIMIT, + }) async copyEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const userId = this.#getActorUserId(req); @@ -1479,7 +1571,11 @@ export class FSController extends PuterController { res.json(this.#toClientEntry(copy)); } - @Post('/mkshortcut', { subdomain: 'api', requireVerified: true }) + @Post('/mkshortcut', { + subdomain: 'api', + requireVerified: true, + rateLimit: FS_MUTATE_LIMIT, + }) async mkshortcutEntry(req: Request, res: Response) { const actor = this.#requireActor(req); const userId = this.#getActorUserId(req); @@ -2090,7 +2186,8 @@ export class FSController extends PuterController { // the ActorUser type. Access via the escape hatch until a proper // storage-quota mechanism is in place. const actorUser = req.actor?.user as - Record | undefined; + | Record + | undefined; const candidates = [ this.#toStorageCapacityCandidate(actorUser?.free_storage), diff --git a/src/backend/controllers/fs/LegacyFSController.ts b/src/backend/controllers/fs/LegacyFSController.ts index 562bd555a..82c7b3d45 100644 --- a/src/backend/controllers/fs/LegacyFSController.ts +++ b/src/backend/controllers/fs/LegacyFSController.ts @@ -43,6 +43,24 @@ import { } from '../../util/hostedAppBacking.js'; import { applyInlineContentSecurity } from '../../util/inlineContentSecurity.js'; import { PuterController } from '../types.js'; +import { + FS_BATCH_CONCURRENT, + FS_BATCH_LIMIT, + FS_DF_LIMIT, + FS_HELPER_LIMIT, + FS_MUTATE_LIMIT, + FS_POLL_LIMIT, + FS_READ_CONCURRENT, + FS_READ_LIMIT, + FS_READDIR_LIMIT, + FS_SEARCH_CONCURRENT, + FS_SEARCH_LIMIT, + FS_SIGN_LIMIT, + FS_SIGNED_CONCURRENT, + FS_SIGNED_READ_LIMIT, + FS_SIGNED_WRITE_LIMIT, + FS_STAT_LIMIT, +} from './limits.js'; import { FS_COSTS } from './costs.js'; import { asRecord, @@ -106,40 +124,107 @@ export class LegacyFSController extends PuterController { } as RouteOptions; // Core filesystem_api routes — direct handlers over the FS service. - router.post('/stat', apiOptions, this.stat); - router.post('/readdir', apiOptions, this.readdir); - router.post('/mkdir', apiOptions, this.mkdir); - router.post('/copy', apiOptions, this.copy); - router.post('/move', apiOptions, this.move); - router.post('/delete', apiOptions, this.delete); - router.post('/rename', apiOptions, this.rename); - router.post('/touch', apiOptions, this.touch); - router.post('/search', apiOptions, this.search); - router.get('/read', apiOptions, this.read); + // Limits come from `./limits` and carry an explicit `scope`, so these + // draw from the same per-user budget as their v2 counterparts rather + // than handing a caller a second allowance for the same operation. + const mutate = { ...apiOptions, rateLimit: FS_MUTATE_LIMIT }; + router.post( + '/stat', + { ...apiOptions, rateLimit: FS_STAT_LIMIT }, + this.stat, + ); + router.post( + '/readdir', + { ...apiOptions, rateLimit: FS_READDIR_LIMIT }, + this.readdir, + ); + router.post('/mkdir', mutate, this.mkdir); + router.post('/copy', mutate, this.copy); + router.post('/move', mutate, this.move); + router.post('/delete', mutate, this.delete); + router.post('/rename', mutate, this.rename); + router.post('/touch', mutate, this.touch); + router.post( + '/search', + { + ...apiOptions, + rateLimit: FS_SEARCH_LIMIT, + concurrent: FS_SEARCH_CONCURRENT, + }, + this.search, + ); + router.get( + '/read', + { + ...apiOptions, + rateLimit: FS_READ_LIMIT, + concurrent: FS_READ_CONCURRENT, + }, + this.read, + ); router.get( '/token-read', { subdomain: 'api', requireVerified: false, allowAccessToken: true, + // An access token may or may not carry a user, so this shares + // the network-keyed budget the other signed routes use. + rateLimit: FS_SIGNED_READ_LIMIT, }, this.tokenRead, ); - router.post('/batch', apiOptions, this.batch); + router.post( + '/batch', + { + ...apiOptions, + rateLimit: FS_BATCH_LIMIT, + concurrent: FS_BATCH_CONCURRENT, + }, + this.batch, + ); // Signed-URL + meta routes. - router.post('/sign', apiOptions, this.sign); - router.post('/writeFile', signedOptions, this.writeFile); - router.get('/file', signedOptions, this.file); - router.all('/df', apiOptions, this.df); - router.post('/open_item', apiOptions, this.openItem); + router.post( + '/sign', + { ...apiOptions, rateLimit: FS_SIGN_LIMIT }, + this.sign, + ); + router.post( + '/writeFile', + { + ...signedOptions, + rateLimit: FS_SIGNED_WRITE_LIMIT, + concurrent: FS_SIGNED_CONCURRENT, + }, + this.writeFile, + ); + router.get( + '/file', + { + ...signedOptions, + rateLimit: FS_SIGNED_READ_LIMIT, + concurrent: FS_SIGNED_CONCURRENT, + }, + this.file, + ); + router.all('/df', { ...apiOptions, rateLimit: FS_DF_LIMIT }, this.df); + router.post( + '/open_item', + { ...apiOptions, rateLimit: FS_HELPER_LIMIT }, + this.openItem, + ); router.post( '/auth/request-app-root-dir', - apiOptions, + { ...apiOptions, rateLimit: FS_SIGN_LIMIT }, this.requestAppRootDir, ); - router.post('/auth/check-app-acl', apiOptions, this.checkAppAcl); + router.post( + '/auth/check-app-acl', + { ...apiOptions, rateLimit: FS_SIGN_LIMIT }, + this.checkAppAcl, + ); // `/down` — session-auth'd file download. Unlike `/file` (signed URL) // this accepts a path on the user's behalf and streams as attachment. @@ -157,6 +242,8 @@ export class LegacyFSController extends PuterController { allowFullAccessToken: true, requireVerified: true, antiCsrf: true, + rateLimit: FS_READ_LIMIT, + concurrent: FS_READ_CONCURRENT, }, this.down, ); @@ -167,92 +254,102 @@ export class LegacyFSController extends PuterController { }); }); - router.get('/get-launch-apps', apiOptions, async (req, res) => { - const recommendedSvc = this.services.recommendedApps as unknown as - { getRecommendedApps?: () => Promise } | undefined; - const recommended = recommendedSvc?.getRecommendedApps - ? await recommendedSvc.getRecommendedApps() - : []; + router.get( + '/get-launch-apps', + { ...apiOptions, rateLimit: FS_HELPER_LIMIT }, + async (req, res) => { + const recommendedSvc = this.services + .recommendedApps as unknown as + | { getRecommendedApps?: () => Promise } + | undefined; + const recommended = recommendedSvc?.getRecommendedApps + ? await recommendedSvc.getRecommendedApps() + : []; - let recent: unknown[] = []; - const userId = req.actor?.user?.id; - if (userId) { - const recentUids = - (await ( + let recent: unknown[] = []; + const userId = req.actor?.user?.id; + if (userId) { + const recentUids = + (await ( + this.stores.app as unknown as { + getRecentAppOpens?: ( + id: number, + opts?: { limit?: number }, + ) => Promise; + } + ).getRecentAppOpens?.(userId, { limit: 10 })) ?? []; + // One batched read for the rows, then the backing checks + // concurrently. Serially awaiting a lookup per uid put ~2 + // round trips of latency on every desktop boot. + const appsByUid = await ( this.stores.app as unknown as { - getRecentAppOpens?: ( - id: number, - opts?: { limit?: number }, - ) => Promise; + getByUids: ( + uids: string[], + ) => Promise>>; } - ).getRecentAppOpens?.(userId, { limit: 10 })) ?? []; - // One batched read for the rows, then the backing checks - // concurrently. Serially awaiting a lookup per uid put ~2 - // round trips of latency on every desktop boot. - const appsByUid = await ( - this.stores.app as unknown as { - getByUids: ( - uids: string[], - ) => Promise>>; - } - ).getByUids(recentUids); + ).getByUids(recentUids); - // `recentUids` is ordered most-recent-first; preserve it. - const orderedApps = recentUids - .map((uid) => appsByUid.get(uid)) - .filter((app): app is Record => - Boolean(app), + // `recentUids` is ordered most-recent-first; preserve it. + const orderedApps = recentUids + .map((uid) => appsByUid.get(uid)) + .filter((app): app is Record => + Boolean(app), + ); + + // Don't hand out an index_url whose puter-hosted backing is + // gone or reclaimed. The taskbar launches recents by name (so + // AppDriver's guard applies), but this list is a + // launch-metadata producer like any other — a future consumer + // reading index_url straight off it shouldn't inherit a stale + // origin. + const backingGoneFlags = await Promise.all( + orderedApps.map((app) => + hostedIndexUrlBackingIsUnavailable({ + app, + subdomainStore: this.stores.subdomain, + config: this.config, + }).catch(() => true), + ), ); - // Don't hand out an index_url whose puter-hosted backing is - // gone or reclaimed. The taskbar launches recents by name (so - // AppDriver's guard applies), but this list is a - // launch-metadata producer like any other — a future consumer - // reading index_url straight off it shouldn't inherit a stale - // origin. - const backingGoneFlags = await Promise.all( - orderedApps.map((app) => - hostedIndexUrlBackingIsUnavailable({ - app, - subdomainStore: this.stores.subdomain, - config: this.config, - }).catch(() => true), - ), - ); + recent = orderedApps.map((app, index) => { + const backingGone = backingGoneFlags[index]; + return { + uuid: app.uid, + name: app.name, + title: app.title, + icon: app.icon ?? null, + godmode: Boolean(app.godmode), + maximize_on_start: Boolean(app.maximize_on_start), + index_url: backingGone ? null : app.index_url, + ...(backingGone + ? { privateAccess: buildHostedBackingDenial() } + : {}), + // An app with no owner isn't owned by a Puter user — + // it's an "external" (origin-bootstrapped) app. + external: + app.owner_user_id == null || + app.owner_user_id === '', + }; + }); + } - recent = orderedApps.map((app, index) => { - const backingGone = backingGoneFlags[index]; - return { - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon ?? null, - godmode: Boolean(app.godmode), - maximize_on_start: Boolean(app.maximize_on_start), - index_url: backingGone ? null : app.index_url, - ...(backingGone - ? { privateAccess: buildHostedBackingDenial() } - : {}), - // An app with no owner isn't owned by a Puter user — - // it's an "external" (origin-bootstrapped) app. - external: - app.owner_user_id == null || - app.owner_user_id === '', - }; - }); - } + res.json({ recommended, recent }); + }, + ); - res.json({ recommended, recent }); - }); - - router.post('/suggest_apps', apiOptions, this.suggestApps); + router.post( + '/suggest_apps', + { ...apiOptions, rateLimit: FS_HELPER_LIMIT }, + this.suggestApps, + ); // puter-js polls this to decide whether to purge its in-memory FS // cache. SocketService bumps a per-user Redis key on every // `outer.gui.item.*` mutation — read it back here. router.get( '/cache/last-change-timestamp', - apiOptions, + { ...apiOptions, rateLimit: FS_POLL_LIMIT }, async (req, res) => { const userId = req.actor?.user?.id; if (!userId) { @@ -273,10 +370,14 @@ export class LegacyFSController extends PuterController { }, ); - router.post('/readdir-subdomains', apiOptions, this.readdirSubdomains); + router.post( + '/readdir-subdomains', + { ...apiOptions, rateLimit: FS_HELPER_LIMIT }, + this.readdirSubdomains, + ); router.post( '/update-fsentry-thumbnail', - apiOptions, + { ...apiOptions, rateLimit: FS_HELPER_LIMIT }, this.updateFsentryThumbnail, ); @@ -707,7 +808,9 @@ export class LegacyFSController extends PuterController { // Trash, and `null`/`{}` when restoring. See // `src/gui/src/helpers.js` → `window.move_items`. newMetadata: (body.new_metadata ?? undefined) as - Record | null | undefined, + | Record + | null + | undefined, }); const oldPath = source.path; await this.#emitGuiEvent('outer.gui.item.moved', moved, { @@ -1168,7 +1271,8 @@ export class LegacyFSController extends PuterController { } type SignedOrEmpty = - (SignedFile & { path?: string }) | Record; + | (SignedFile & { path?: string }) + | Record; const result: { signatures: SignedOrEmpty[]; token?: string } = { signatures: [], }; @@ -1708,7 +1812,10 @@ export class LegacyFSController extends PuterController { const subjectRef = body.subject; const appRef = body.app; const mode = (getString(body, 'mode') ?? 'read') as - 'see' | 'list' | 'read' | 'write'; + | 'see' + | 'list' + | 'read' + | 'write'; if (!subjectRef || !appRef) throw new HttpError(400, '`subject` and `app` are required', { legacyCode: 'bad_request', diff --git a/src/backend/controllers/fs/limits.test.ts b/src/backend/controllers/fs/limits.test.ts new file mode 100644 index 000000000..53b71171b --- /dev/null +++ b/src/backend/controllers/fs/limits.test.ts @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { describe, expect, it } from 'vitest'; + +import * as limits from './limits.js'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '../../services/metering/consts.js'; +import type { RouteOptions, RouteRateLimit } from '../../core/http/types'; + +type Spec = RouteRateLimit | NonNullable; + +const all = Object.entries(limits) as Array<[string, Spec | RouteRateLimit[]]>; +const flat: Array<[string, Spec]> = all.flatMap(([name, value]) => + Array.isArray(value) + ? value.map((v, i): [string, Spec] => [`${name}[${i}]`, v]) + : [[name, value] as [string, Spec]], +); + +const windows = flat.filter(([, s]) => 'window' in s) as Array< + [string, RouteRateLimit] +>; +const concurrents = flat.filter(([, s]) => !('window' in s)); + +describe('filesystem limit specs', () => { + // The whole point of this module is that the legacy and v2 controllers + // import the same specs. That only ties the counters together if every + // spec carries an explicit scope — without one the gate falls back to + // the route path, and the two controllers get separate budgets. + it.each(flat)('%s pins an explicit scope', (_name, spec) => { + expect(spec.scope).toBeTruthy(); + }); + + it.each(windows)('%s has a positive limit and window', (_name, spec) => { + expect(spec.limit).toBeGreaterThan(0); + expect(spec.window).toBeGreaterThan(0); + }); + + // Base is the paid value; the free tiers are carved out beneath it. + // A free tier above the base would mean paying made you worse off. + it.each(windows)('%s never lets a free tier exceed paid', (_name, spec) => { + for (const n of Object.values(spec.bySubscription ?? {})) { + expect(n).toBeLessThanOrEqual(spec.limit); + } + }); + + it.each(windows)( + '%s caps temp at or below registered-free', + (_name, spec) => { + const free = spec.bySubscription?.[DEFAULT_FREE_SUBSCRIPTION]; + const temp = spec.bySubscription?.[DEFAULT_TEMP_SUBSCRIPTION]; + if (free === undefined || temp === undefined) return; + expect(temp).toBeLessThanOrEqual(free); + }, + ); + + // A single in-flight slot turns incidental client parallelism into a + // spurious 429; paid tiers keep room to actually parallelise. + it.each(concurrents)( + '%s keeps concurrency at 5+ paid and 2+ for every tier', + (_name, spec) => { + expect(spec.limit).toBeGreaterThanOrEqual(5); + for (const n of Object.values(spec.bySubscription ?? {})) { + expect(n).toBeGreaterThanOrEqual(2); + } + }, + ); + + it('gives search the tightest window of the read paths', () => { + expect(limits.FS_SEARCH_LIMIT.limit).toBeLessThan( + limits.FS_STAT_LIMIT.limit, + ); + expect(limits.FS_SEARCH_LIMIT.limit).toBeLessThan( + limits.FS_READ_LIMIT.limit, + ); + }); + + // The desktop deletes/moves one item per request with no batching and no + // pacing, so a single "empty trash" or "select all, delete" has to fit + // inside the minute window on every tier. The hour window is the abuse + // ceiling that a minute window this wide can no longer be. + it('gives mutations a bulk-sized minute window plus an hourly backstop', () => { + const [minute, hourly] = limits.FS_MUTATE_LIMIT; + expect(limits.FS_MUTATE_LIMIT).toHaveLength(2); + expect(minute.window).toBe(60_000); + expect(hourly.window).toBe(60 * 60_000); + + const tiers = (spec: RouteRateLimit) => [ + spec.limit, + spec.bySubscription![DEFAULT_FREE_SUBSCRIPTION], + spec.bySubscription![DEFAULT_TEMP_SUBSCRIPTION], + ]; + + // A few hundred items clears in one pass, anonymous included. + for (const n of tiers(minute)) expect(n).toBeGreaterThanOrEqual(500); + // Anonymous stays meaningfully tighter than paid on both windows. + expect(tiers(minute)[2]).toBeLessThanOrEqual(minute.limit / 2); + expect(tiers(hourly)[2]).toBeLessThanOrEqual(hourly.limit / 2); + + // The hour window has to bind rather than decorate: it allows a + // handful of bulk passes an hour, not sixty minutes' worth. + for (const [perMinute, perHour] of tiers(minute).map( + (n, i): [number, number] => [n, tiers(hourly)[i]], + )) { + expect(perHour).toBeGreaterThan(perMinute); + expect(perHour).toBeLessThanOrEqual(perMinute * 10); + } + }); + + // The DAV gate is consumed imperatively, before the request has an actor: + // it keys on the network fingerprint and reads only `limit` / `window`. A + // `key: 'user'` or a `bySubscription` map here would describe tiering that + // never happens. + it('shapes the DAV specs the way the DAV gate consumes them', () => { + expect(limits.DAV_LIMIT.key).toBe('fingerprint'); + expect(limits.DAV_CONCURRENT.key).toBe('fingerprint'); + expect(limits.DAV_LIMIT.bySubscription).toBeUndefined(); + expect(limits.DAV_CONCURRENT.bySubscription).toBeUndefined(); + }); + + // With no session to key on, the alternative is the bare address — and an + // address is a household, an office or a carrier gateway, so keying there + // makes one bucket serve everyone behind it and tighten as more real users + // arrive. The fingerprint separates clients within a network while still + // being something one client can't vary per request. + it('keys the signed-URL routes on the network fingerprint', () => { + expect(limits.FS_SIGNED_READ_LIMIT.key).toBe('fingerprint'); + expect(limits.FS_SIGNED_WRITE_LIMIT.key).toBe('fingerprint'); + expect(limits.FS_SIGNED_CONCURRENT.key).toBe('fingerprint'); + }); + + // These serve page subresources — a gallery, an app's own assets — so the + // in-flight cap has to sit above what a browser opens to one origin at + // once, or a normal page load is what trips it. + it('leaves the signed-URL in-flight cap above a browser`s own parallelism', () => { + expect(limits.FS_SIGNED_CONCURRENT.limit).toBeGreaterThan(30); + }); + + it('uses distinct scopes so counters cannot collide', () => { + const scopes = flat + .filter(([, s]) => 'window' in s) + .map(([, s]) => s.scope); + expect(new Set(scopes).size).toBe(scopes.length); + }); +}); diff --git a/src/backend/controllers/fs/limits.ts b/src/backend/controllers/fs/limits.ts new file mode 100644 index 000000000..70c3f9f74 --- /dev/null +++ b/src/backend/controllers/fs/limits.ts @@ -0,0 +1,243 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '../../services/metering/consts.js'; +import type { RouteOptions, RouteRateLimit } from '../../core/http/types'; + +// -- Shared filesystem limits ---------------------------------------- +// +// The v2 controller and the legacy controller expose the same operations +// on separate route tables. Both import the specs below so a caller can't +// get two budgets for one operation by switching endpoints — the `scope` +// is what ties the counters together, and it only works if both sides +// pass the same one. +// +// The base `limit` is what subscribed tiers see; `bySubscription` carves +// out the free tiers beneath it. Any plan id not enumerated (a Stripe +// plan, the dev-only `unlimited`) falls through to the base, so new plans +// are generous by default rather than accidentally throttled. +// +// Storage quota already bounds total bytes. These bound request *count*, +// which quota does not: small-file spam still costs object-store writes +// and fsentry rows, and metadata reads cost database time while being +// entirely free to the caller. + +/** Per-user sliding window. Free tiers are carved out of the paid base. */ +const userWindow = ( + scope: string, + paid: number, + free: number, + temp: number, + window = 60_000, +): RouteRateLimit => ({ + scope, + limit: paid, + window, + key: 'user', + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: free, + [DEFAULT_TEMP_SUBSCRIPTION]: temp, + }, +}); + +/** + * In-flight cap. Nothing drops below 2 — a single slot turns any incidental + * parallelism in a client (two tabs, a prefetch alongside a user action) into a + * spurious 429 — and subscribed tiers keep enough headroom to actually + * parallelise. + */ +const userConcurrent = ( + scope: string, + paid: number, + free: number, + temp: number, +): NonNullable => ({ + scope, + limit: paid, + key: 'user', + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: free, + [DEFAULT_TEMP_SUBSCRIPTION]: temp, + }, +}); + +/** + * Per-network window, for the signature-authenticated routes with no session. + * + * `fingerprint` rather than `ip`: without an actor the only alternative is the + * bare address, and an address is an aggregate — a household, an office, a + * mobile carrier's gateway. Keying on it means one bucket for everyone behind + * it, which is a shared cap that gets tighter the more real users are present. + * The fingerprint folds in the headers a client can't vary per request without + * also looking like a different client, so separate browsers behind one NAT get + * separate buckets while a single client still can't mint fresh ones per call. + */ +const networkWindow = ( + scope: string, + limit: number, + window = 60_000, +): RouteRateLimit => ({ scope, limit, window, key: 'fingerprint' }); + +// -- Metadata reads -------------------------------------------------- + +/** Chattiest call in the GUI; high enough to only ever catch a loop. */ +export const FS_STAT_LIMIT = userWindow('fs:stat', 1200, 600, 300); + +/** + * Desktop boot fans out hard, so the minute budget is generous — the short + * second window is what actually catches a runaway loop before it has spent the + * whole minute's allowance. + */ +export const FS_READDIR_LIMIT: RouteRateLimit[] = [ + userWindow('fs:readdir', 600, 300, 120), + userWindow('fs:readdir-burst', 120, 60, 30, 10_000), +]; + +/** + * Unindexed scan across the user's tree, and unmetered — `FS_COSTS` prices + * egress bytes only, so this is the cheapest way to occupy a database + * connection. Tightest limit in the file. + */ +export const FS_SEARCH_LIMIT = userWindow('fs:search', 60, 30, 10); +export const FS_SEARCH_CONCURRENT = userConcurrent('fs:search', 5, 2, 2); + +/** Aggregation over the whole tree; the GUI needs it rarely. */ +export const FS_DF_LIMIT = userWindow('fs:df', 60, 30, 15); + +// -- Content transfer ------------------------------------------------ + +/** + * Egress is billed, so cost is already bounded — the cap here is against + * connection exhaustion, which is why the concurrency slot matters more than + * the window. + */ +export const FS_READ_LIMIT = userWindow('fs:read', 600, 300, 120); +export const FS_READ_CONCURRENT = userConcurrent('fs:read', 10, 5, 3); + +export const FS_WRITE_LIMIT = userWindow('fs:write', 300, 120, 30); +export const FS_WRITE_CONCURRENT = userConcurrent('fs:write', 15, 6, 3); + +/** + * Multipart handshake — several calls per upload, so a multiple of the write + * budget rather than a peer of it. Signing is the half that costs us + * object-store calls. + * + * The multiple is what matters: a per-minute ceiling here divides down into a + * much smaller number of files, and selecting a folder's worth of them is one + * gesture. `FS_WRITE_LIMIT` and storage quota already bound what actually gets + * written, so this only has to stay out of the way of a real upload. + */ +export const FS_MULTIPART_LIMIT = userWindow('fs:multipart', 2400, 1200, 600); + +// -- Metadata mutations ---------------------------------------------- + +/** + * Mkdir / touch / rename / delete / move / copy / mkshortcut. + * + * The desktop issues one call per item and does not pace them: emptying the + * trash, deleting a multi-selection, or dragging a folder's worth of files + * fires the whole set back to back. A minute budget in the low hundreds turns + * an ordinary "select all, delete" into a partial failure, so the minute window + * is sized to clear a bulk pass over a few hundred items on every tier — an + * anonymous session gets a real desktop, so it needs a real bulk allowance too, + * just a smaller one. + * + * The hour window is where the abuse ceiling actually lives. It allows a few of + * those bulk passes per hour, which is more than a person driving a file + * manager will ever need and well under what a script grinding metadata writes + * would want. + */ +export const FS_MUTATE_LIMIT: RouteRateLimit[] = [ + userWindow('fs:mutate', 1200, 900, 600), + userWindow('fs:mutate-sustained', 6000, 3000, 1800, 60 * 60_000), +]; + +/** Mints a URL that outlives the request, so worth its own budget. */ +export const FS_SIGN_LIMIT = userWindow('fs:sign', 300, 150, 60); + +/** Low-frequency GUI helpers. */ +export const FS_HELPER_LIMIT = userWindow('fs:helper', 120, 60, 30); + +/** + * Puter.js polls this on a timer to decide whether to purge its FS cache. Same + * ceiling for free and paid — it is a single cache read. + */ +export const FS_POLL_LIMIT = userWindow('fs:poll', 240, 240, 120); + +// -- Legacy multipart upload ----------------------------------------- + +/** + * `/batch` buffers every file fully into memory before any quota or storage + * check runs, up to BATCH_MAX_FILES × BATCH_MAX_FILE_SIZE. The concurrency slot + * is doing the real work here; the window is secondary, and sized to say so — + * one upload is one call, so a per-minute ceiling in the tens is a cap on how + * many files someone may upload rather than a bound on cost. What actually + * bounds the memory this route can tie up is how many run at once. + */ +export const FS_BATCH_LIMIT = userWindow('fs:batch', 600, 300, 300); +export const FS_BATCH_CONCURRENT = userConcurrent('fs:batch', 5, 2, 2); + +// -- Signed-URL routes (no session to key on) ------------------------ +// +// The URL's own signature is what authorizes these; the limits below only +// bound what an unsigned flood can cost. Sized for what the routes are +// actually used for, which is content the browser fetches as a subresource: +// a gallery, a document's images, an app loading its own assets. A page +// opening a few dozen of those at once is ordinary, and every rejection here +// is a broken image rather than a slow one — so the ceiling clears a burst of +// real page loads and only catches something looping. + +export const FS_SIGNED_READ_LIMIT = networkWindow('fs:signed-read', 3_000); +export const FS_SIGNED_WRITE_LIMIT = networkWindow('fs:signed-write', 600); +export const FS_SIGNED_CONCURRENT: NonNullable = { + scope: 'fs:signed', + // Above what a browser will open to one origin at once, so the in-flight + // cap never decides the outcome for a single client — it is there for a + // client that opens connections without closing them. + limit: 60, + key: 'fingerprint', +}; + +// -- WebDAV ---------------------------------------------------------- + +/** + * One `router.use` fronts the whole DAV surface, so a single gate there covers + * every verb. Desktop DAV clients are bursty — a lower ceiling shows up as + * spurious failures in Finder / Explorer. + * + * Unlike everything above, the DAV gate runs before the request is + * authenticated: there is no actor yet, so there is no subscription to resolve + * a tier against and one ceiling applies to every caller. These are shaped to + * say that — keyed on the network fingerprint the gate buckets on, with no + * `bySubscription` map to imply a tiering that never happens. + */ +export const DAV_LIMIT: RouteRateLimit = { + scope: 'dav', + limit: 600, + window: 60_000, + key: 'fingerprint', +}; +export const DAV_CONCURRENT: NonNullable = { + scope: 'dav', + limit: 10, + key: 'fingerprint', +}; diff --git a/src/backend/controllers/hosting/HostingController.js b/src/backend/controllers/hosting/HostingController.js index 9ce59d63c..d1ac44378 100644 --- a/src/backend/controllers/hosting/HostingController.js +++ b/src/backend/controllers/hosting/HostingController.js @@ -19,6 +19,10 @@ import { HttpError } from '../../core/http/HttpError.js'; import { PuterController } from '../types.js'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '../../services/metering/consts.js'; /** * Site hosting endpoints. Listing and create/update are not exposed as @@ -46,6 +50,18 @@ export class HostingController extends PuterController { requireUserActor: true, allowFullAccessToken: true, requireVerified: true, + // Destructive, and pairs with the `subdomains:create` + // budget on the driver side. + rateLimit: { + scope: 'delete-site', + limit: 60, + window: 60_000, + key: 'user', + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 30, + [DEFAULT_TEMP_SUBSCRIPTION]: 10, + }, + }, }, async (req, res) => { const { site_uuid } = req.body ?? {}; diff --git a/src/backend/controllers/notification/NotificationController.ts b/src/backend/controllers/notification/NotificationController.ts index 1ffbfce59..ca5342def 100644 --- a/src/backend/controllers/notification/NotificationController.ts +++ b/src/backend/controllers/notification/NotificationController.ts @@ -41,6 +41,14 @@ export class NotificationController extends PuterController { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true, + // Fires per notification interaction, so the ceiling stays + // generous — it is here to catch a loop, not to pace a user. + rateLimit: { + scope: 'notification-mark', + limit: 300, + window: 60_000, + key: 'user', + }, }) async markAck(req: Request, res: Response): Promise { const uid = req.body?.uid; @@ -84,6 +92,14 @@ export class NotificationController extends PuterController { subdomain: 'api', requireUserActor: true, allowFullAccessToken: true, + // Fires per notification interaction, so the ceiling stays + // generous — it is here to catch a loop, not to pace a user. + rateLimit: { + scope: 'notification-mark', + limit: 300, + window: 60_000, + key: 'user', + }, }) async markRead(req: Request, res: Response): Promise { const uid = req.body?.uid; diff --git a/src/backend/controllers/oidc/OIDCController.test.ts b/src/backend/controllers/oidc/OIDCController.test.ts index f5c768c35..2f2a25cfc 100644 --- a/src/backend/controllers/oidc/OIDCController.test.ts +++ b/src/backend/controllers/oidc/OIDCController.test.ts @@ -1746,3 +1746,35 @@ describe('OIDCController POST /auth/oidc/verify-popup-return', () => { expect((await redeem(proof)).body).toMatchObject({ oidc_login: false }); }); }); + +// ── rate-limit scopes ─────────────────────────────────────────────── + +describe('OIDCController rate limits', () => { + const rateLimitOf = (method: string, path: string) => { + const route = router.routes.find( + (r) => r.method === method && r.path === path, + ); + if (!route) throw new Error(`No ${method.toUpperCase()} ${path} route`); + return route.options.rateLimit as { scope: string; limit: number }; + }; + + it('keeps the revalidate landing page off the identity-provider bucket', () => { + // `/auth/revalidate-done` serves a constant HTML page; the start and + // callback routes exchange codes with an identity provider. Sharing + // a scope made the cheap page inherit the flow routes' tight ceiling, + // which one shared address can exhaust on its own. + const done = rateLimitOf('get', '/auth/revalidate-done'); + const start = rateLimitOf('get', '/auth/oidc/:provider/start'); + expect(done.scope).not.toBe(start.scope); + expect(done.scope).toBe('oidc-revalidate-done'); + expect(done.limit).toBeGreaterThan(start.limit); + }); + + it('sizes the public provider list for a shared address, not a fleet', () => { + // Unauthenticated and keyed on IP, so the bucket covers every client + // behind one NAT or campus, not one browser. It is still login + // surface, so it stays bounded well below the ceilings given to + // static reads like icons or version info. + expect(rateLimitOf('get', '/auth/oidc/providers').limit).toBe(1_200); + }); +}); diff --git a/src/backend/controllers/oidc/OIDCController.ts b/src/backend/controllers/oidc/OIDCController.ts index ae194265c..60df9d145 100644 --- a/src/backend/controllers/oidc/OIDCController.ts +++ b/src/backend/controllers/oidc/OIDCController.ts @@ -228,10 +228,26 @@ export class OIDCController extends PuterController { // -- GET /auth/oidc/providers -------------------------------- // Public — list enabled provider IDs for the frontend. + // + // Every render of a login form reads this, and with no actor the + // bucket is the address: one corporate egress, campus or carrier + // gateway stands for every person behind it. Size it for that — a + // large shared address is thousands of people, and a shift or class + // change bunches their sign-ins into the same minute — but no + // further. This is login surface, so the ceiling should still bound + // someone enumerating which identity providers a deployment accepts. router.get( '/auth/oidc/providers', - { subdomain: 'api' }, + { + subdomain: 'api', + rateLimit: { + scope: 'oidc-providers', + limit: 1_200, + window: 60_000, + key: 'ip', + }, + }, async (_req: Request, res: Response) => { const providers = await this.services.oidc.getEnabledProviderIds(); @@ -584,10 +600,26 @@ export class OIDCController extends PuterController { // -- GET /auth/revalidate-done ------------------------------- // Landing page after revalidation; posts to opener for popup flow. + // + // Deliberately not on the `oidc-general` bucket the start/callback + // routes share: those exchange codes with an identity provider, this + // one is a constant HTML page that touches nothing. Sharing a bucket + // meant everyone reachable through one address — an office, a school, + // a carrier gateway — competed for the same 30 popup closes a minute. + // The replacement is sized for the humans behind one such address + // re-validating at once, not for a fleet: it is still auth surface, + // and one page view per revalidation is a low-volume event. router.get( '/auth/revalidate-done', - { subdomain: '' }, + { + subdomain: '', + rateLimit: { + scope: 'oidc-revalidate-done', + limit: 600, + window: 60_000, + }, + }, (_req: Request, res: Response) => { const origin = this.config.origin ?? ''; res.set('Content-Type', 'text/html; charset=utf-8'); diff --git a/src/backend/controllers/peer/PeerController.ts b/src/backend/controllers/peer/PeerController.ts index b35ecb53e..08f0dcf5f 100644 --- a/src/backend/controllers/peer/PeerController.ts +++ b/src/backend/controllers/peer/PeerController.ts @@ -24,6 +24,10 @@ import { HttpError } from '../../core/http/HttpError.js'; import type { PuterRouter } from '../../core/http/PuterRouter.js'; import { PuterController } from '../types.js'; import { PEER_COSTS } from './costs.js'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '../../services/metering/consts.js'; /** * Constant-time secret comparison for the internal-auth header. HMAC both sides @@ -103,17 +107,58 @@ export class PeerController extends PuterController { registerRoutes(router: PuterRouter): void { router.get( '/peer/signaller-info', - { subdomain: 'api' }, + { + subdomain: 'api', + // Public config read — the signaller URL and the fallback + // ICE list, both deploy constants. Unauthenticated, so the + // key is the address, and one address covers every client + // on that network; a peer session starts with this call, so + // the bucket has to hold a whole network's sessions. Nothing + // here is secret or expensive, so the ceiling only bounds a + // client stuck re-reading it. + rateLimit: { + scope: 'peer-signaller-info', + limit: 3_000, + window: 60_000, + key: 'ip', + }, + }, this.#signallerInfo, ); router.post( '/peer/generate-turn', - { subdomain: 'api', requireAuth: true }, + { + subdomain: 'api', + requireAuth: true, + // Every call reaches the upstream TURN API and mints + // credentials against a paid allocation, so this is a + // spend limit as much as an abuse limit. + rateLimit: { + scope: 'peer-generate-turn', + limit: 30, + window: 60_000, + key: 'user', + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 10, + [DEFAULT_TEMP_SUBSCRIPTION]: 5, + }, + }, + }, this.#generateTurn, ); router.post( '/turn/ingest-usage', - { subdomain: 'api' }, + { + subdomain: 'api', + // Shared-secret authenticated, so this only bounds how + // fast someone can guess the secret. + rateLimit: { + scope: 'turn-ingest', + limit: 60, + window: 60_000, + key: 'ip', + }, + }, this.#ingestUsage, ); } diff --git a/src/backend/controllers/puterai/PuterAIController.test.ts b/src/backend/controllers/puterai/PuterAIController.test.ts index 147d4238f..503489ab5 100644 --- a/src/backend/controllers/puterai/PuterAIController.test.ts +++ b/src/backend/controllers/puterai/PuterAIController.test.ts @@ -229,9 +229,18 @@ describe('PuterAIController.registerRoutes', () => { const modelsRoute = calls.find( (c) => c.path === '/puterai/chat/models', ); + // Unauthenticated, so the limit keys on IP rather than an actor — + // which makes the bucket an aggregate over every client behind that + // address, hence a ceiling sized for a network rather than a browser. expect(modelsRoute?.opts).toEqual({ subdomain: 'api', requireAuth: false, + rateLimit: { + scope: 'puterai-models', + limit: 3_000, + window: 60_000, + key: 'ip', + }, }); }); diff --git a/src/backend/controllers/puterai/PuterAIController.ts b/src/backend/controllers/puterai/PuterAIController.ts index f1ee14343..8d7be3ba1 100644 --- a/src/backend/controllers/puterai/PuterAIController.ts +++ b/src/backend/controllers/puterai/PuterAIController.ts @@ -90,7 +90,23 @@ export class PuterAIController extends PuterController { key: aiPolicyKey, }, } as RouteOptions; - const publicOpts = { subdomain: 'api', requireAuth: false } as const; + // Model listings are unauthenticated, so the only key available is + // the address — which is an aggregate, not a user: a NAT, a school, + // a mobile carrier gateway or a server-side renderer all arrive as + // one address, and the SDK and GUI both fetch the catalogue on + // startup. The ceiling therefore has to cover a whole network's page + // loads. What it still protects is the serialisation cost of the + // catalogue under a client stuck in a fetch loop. + const publicOpts = { + subdomain: 'api', + requireAuth: false, + rateLimit: { + scope: 'puterai-models', + limit: 3_000, + window: 60_000, + key: 'ip', + }, + } as RouteOptions; // Every route below carries the `/puterai` prefix for wire // compatibility with puter-js and existing API tests. @@ -153,7 +169,23 @@ export class PuterAIController extends PuterController { // URL itself is HMAC-signed, so no additional auth gate. router.get( '/puterai/video/proxy', - { subdomain: 'api' }, + { + subdomain: 'api', + // HMAC-signed but unauthenticated, and it streams provider + // bandwidth through us — so the in-flight cap matters as + // much as the window. + rateLimit: { + scope: 'puterai-video-proxy', + limit: 60, + window: 60_000, + key: 'ip', + }, + concurrent: { + scope: 'puterai-video-proxy', + limit: 5, + key: 'ip', + }, + }, this.#videoProxy, ); } diff --git a/src/backend/controllers/static/StaticPagesController.ts b/src/backend/controllers/static/StaticPagesController.ts index d181d2441..036cbd4f4 100644 --- a/src/backend/controllers/static/StaticPagesController.ts +++ b/src/backend/controllers/static/StaticPagesController.ts @@ -32,6 +32,17 @@ import { promoteToVerifiedGroup } from '../../util/userProvisioning.js'; * All root-subdomain-only, all unauthenticated (the confirm/unsubscribe tokens * in the query string are the auth). */ +/** + * Unauthenticated pages reached from a token-bearing email link. Matches what + * the emailSend extension already applies to its own link pages. + */ +const TOKEN_LINK_LIMIT = { + scope: 'token-link-page', + limit: 30, + window: 60_000, + key: 'ip' as const, +}; + export class StaticPagesController extends PuterController { registerRoutes(router: PuterRouter) { const origin = this.config.origin ?? ''; @@ -159,151 +170,178 @@ export class StaticPagesController extends PuterController { }); // -- /sitemap.xml -------------------------------------------- - router.get('/sitemap.xml', {}, async (req, res) => { - const domain = this.config.domain ?? req.hostname; - const origin = `${req.protocol}://${domain}`; - const apps = (await this.clients.db.read( - `SELECT \`name\` FROM \`apps\` WHERE \`approved_for_listing\` = ${this.clients.db.booleanLiteral(true)}`, - )) as Array<{ name: string }>; - const urls = [ - `${req.protocol}://docs.${domain}/`, - ...apps.map( - (a) => `${origin}/app/${a.name}`, - ), - ]; - const body = - '' + - '' + - urls.join('') + - ''; - res.type('application/xml').send(body); - }); + router.get( + '/sitemap.xml', + { + // Unauthenticated and runs a full-table scan over approved + // apps on every request, with no response cache in front. + rateLimit: { + scope: 'sitemap', + limit: 10, + window: 60_000, + key: 'ip', + }, + }, + async (req, res) => { + const domain = this.config.domain ?? req.hostname; + const origin = `${req.protocol}://${domain}`; + const apps = (await this.clients.db.read( + `SELECT \`name\` FROM \`apps\` WHERE \`approved_for_listing\` = ${this.clients.db.booleanLiteral(true)}`, + )) as Array<{ name: string }>; + const urls = [ + `${req.protocol}://docs.${domain}/`, + ...apps.map( + (a) => `${origin}/app/${a.name}`, + ), + ]; + const body = + '' + + '' + + urls.join('') + + ''; + res.type('application/xml').send(body); + }, + ); // -- /unsubscribe -------------------------------------------- - router.get('/unsubscribe', {}, async (req, res) => { - const userUuid = - typeof req.query.user_uuid === 'string' - ? req.query.user_uuid - : undefined; - if (!userUuid) { - res.send(err('user_uuid is required')); - return; - } + router.get( + '/unsubscribe', + { rateLimit: TOKEN_LINK_LIMIT }, + async (req, res) => { + const userUuid = + typeof req.query.user_uuid === 'string' + ? req.query.user_uuid + : undefined; + if (!userUuid) { + res.send(err('user_uuid is required')); + return; + } - const user = await this.stores.user.getByUuid(userUuid); - if (!user) { - res.send(err('User not found.')); - return; - } - if (user.unsubscribed) { - res.send(ok('You are already unsubscribed.')); - return; - } + const user = await this.stores.user.getByUuid(userUuid); + if (!user) { + res.send(err('User not found.')); + return; + } + if (user.unsubscribed) { + res.send(ok('You are already unsubscribed.')); + return; + } - await this.stores.user.update(user.id, { unsubscribed: 1 }); - res.send(ok('You have successfully unsubscribed from all emails.')); - }); + await this.stores.user.update(user.id, { unsubscribed: 1 }); + res.send( + ok('You have successfully unsubscribed from all emails.'), + ); + }, + ); // -- /confirm-email-by-token --------------------------------- - router.get('/confirm-email-by-token', {}, async (req, res) => { - const userUuid = - typeof req.query.user_uuid === 'string' - ? req.query.user_uuid - : undefined; - const token = - typeof req.query.token === 'string' - ? req.query.token - : undefined; - if (!userUuid) { - res.send(err('user_uuid is required')); - return; - } - if (!token) { - res.send(err('token is required')); - return; - } + router.get( + '/confirm-email-by-token', + { rateLimit: TOKEN_LINK_LIMIT }, + async (req, res) => { + const userUuid = + typeof req.query.user_uuid === 'string' + ? req.query.user_uuid + : undefined; + const token = + typeof req.query.token === 'string' + ? req.query.token + : undefined; + if (!userUuid) { + res.send(err('user_uuid is required')); + return; + } + if (!token) { + res.send(err('token is required')); + return; + } - const user = await this.stores.user.getByProperty( - 'uuid', - userUuid, - { force: true }, - ); - if (!user) { - res.send(err('user not found.')); - return; - } - if (user.email_confirmed) { - res.send(ok('Email already confirmed.')); - return; - } - if (user.email_confirm_token !== token) { - res.send(err('invalid token.')); - return; - } + const user = await this.stores.user.getByProperty( + 'uuid', + userUuid, + { force: true }, + ); + if (!user) { + res.send(err('user not found.')); + return; + } + if (user.email_confirmed) { + res.send(ok('Email already confirmed.')); + return; + } + if (user.email_confirm_token !== token) { + res.send(err('invalid token.')); + return; + } - // v2 writes `clean_email` at signup (lowercased email). Older rows - // that predate that may be null — fall back to email.lower(). - const cleanEmail = - (user.clean_email as string | null | undefined) ?? - String(user.email ?? '').toLowerCase(); + // v2 writes `clean_email` at signup (lowercased email). Older rows + // that predate that may be null — fall back to email.lower(). + const cleanEmail = + (user.clean_email as string | null | undefined) ?? + String(user.email ?? '').toLowerCase(); - const [dupe] = (await this.clients.db.read( - `SELECT EXISTS( + const [dupe] = (await this.clients.db.read( + `SELECT EXISTS( SELECT 1 FROM \`user\` WHERE (\`email\` = ? OR \`clean_email\` = ?) AND \`email_confirmed\` = ${this.clients.db.booleanLiteral(true)} AND \`password\` IS NOT NULL ) AS email_exists`, - [user.email, cleanEmail], - )) as Array<{ email_exists: number }>; - if (dupe?.email_exists) { - res.send( - err('This email was confirmed on a different account.'), + [user.email, cleanEmail], + )) as Array<{ email_exists: number }>; + if (dupe?.email_exists) { + res.send( + err('This email was confirmed on a different account.'), + ); + return; + } + + // Revoke any other accounts' pending change-email slots targeting + // this address — they're no longer valid once someone confirms it. + await this.clients.db.write( + 'UPDATE `user` SET `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL WHERE `unconfirmed_change_email` = ?', + [user.email], ); - return; - } - // Revoke any other accounts' pending change-email slots targeting - // this address — they're no longer valid once someone confirms it. - await this.clients.db.write( - 'UPDATE `user` SET `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL WHERE `unconfirmed_change_email` = ?', - [user.email], - ); + await this.stores.user.update(user.id, { + email_confirmed: 1, + requires_email_confirmation: 0, + email_confirm_code: null, + email_confirm_token: null, + }); - await this.stores.user.update(user.id, { - email_confirmed: 1, - requires_email_confirmation: 0, - email_confirm_code: null, - email_confirm_token: null, - }); - - await promoteToVerifiedGroup(this.stores.group, this.config, user); - - // Best-effort side-channels — don't fail the user-visible response - // if sockets or the event bus are unavailable. - try { - await this.services.socket.send( - { room: user.id }, - 'user.email_confirmed', - {}, + await promoteToVerifiedGroup( + this.stores.group, + this.config, + user, ); - } catch { - /* ignore */ - } - try { - this.clients.event?.emit( - 'user.email-confirmed', - { - user_id: user.id, - user_uid: user.uuid, - email: user.email, - }, - {}, - ); - } catch { - /* ignore */ - } - res.send(ok('Your email has been successfully confirmed.')); - }); + // Best-effort side-channels — don't fail the user-visible response + // if sockets or the event bus are unavailable. + try { + await this.services.socket.send( + { room: user.id }, + 'user.email_confirmed', + {}, + ); + } catch { + /* ignore */ + } + try { + this.clients.event?.emit( + 'user.email-confirmed', + { + user_id: user.id, + user_uid: user.uuid, + email: user.email, + }, + {}, + ); + } catch { + /* ignore */ + } + + res.send(ok('Your email has been successfully confirmed.')); + }, + ); } } diff --git a/src/backend/controllers/system/SystemController.js b/src/backend/controllers/system/SystemController.js index 6b168965e..95d542ff9 100644 --- a/src/backend/controllers/system/SystemController.js +++ b/src/backend/controllers/system/SystemController.js @@ -25,6 +25,75 @@ import { PuterController } from '../types.js'; * * These are all low-risk, authenticated or not, and mostly stateless. */ +/** + * Liveness polling. The callers here are infrastructure, not people: a load + * balancer, an orchestrator and any external uptime prober all poll this, and + * they typically egress from a small set of addresses. A 429 here is read as an + * unhealthy node and takes the node out of rotation, so the ceiling is set + * where only a runaway loop can reach it. The handler itself reads a status + * snapshot refreshed on a background timer, so the per-request cost is close to + * nil. + * + * `memory` rather than the shared default, and that choice is load-bearing: + * + * - This route decides whether a node stays in rotation, so it must not depend on + * anything it isn't already reporting on. The default backend is redis, and + * the cluster is configured with an offline queue and no per-command timeout + * — so while redis is unreachable a gated request waits on it rather than + * failing fast. The gate does fail open, but only once the call rejects, and + * the ALB gives a target 4s per probe and evicts after two. A redis + * degradation could therefore empty every target group in every region, which + * is the outcome the `@dependencies` degrade rules in the health check query + * exist to prevent. Keeping the counter in-process removes redis from the + * liveness path entirely. + * - Per-node counting is also the more honest bucket here. The ceiling only ever + * needs to cover the pollers hitting _this_ node, not (pollers x fleet size) + * as a shared counter does. + */ +const HEALTHCHECK_LIMIT = { + scope: 'healthcheck', + limit: 30_000, + window: 60_000, + key: 'ip', + backend: 'memory', +}; + +/** + * Deploy-constant build info, polled by clients. One address is a NAT, a + * campus, a proxy or a server-side renderer, so this bucket aggregates every + * client behind it — sizing it for a single browser would throttle a whole + * office. The response is cached per-client for a minute, which bounds each + * client to roughly one hit per window; the ceiling is what is left to catch a + * client that ignores the cache. + */ +const VERSION_LIMIT = { + scope: 'version', + limit: 6_000, + window: 60_000, + key: 'ip', +}; + +/** + * Deploy-constant deployment identity, read once per page load to decide + * whether to offer signup. Same aggregation as `/version` — the bucket is a + * whole network's worth of clients — and the payload is four constants, so the + * limit only guards against an unbounded client loop. + */ +const WHOAREWE_LIMIT = { + scope: 'whoarewe', + limit: 6_000, + window: 60_000, + key: 'ip', +}; + +/** Static introspection output, read once at boot rather than in a loop. */ +const LSMOD_LIMIT = { + scope: 'lsmod', + limit: 60, + window: 60_000, + key: 'user', +}; + export class SystemController extends PuterController { constructor(config, clients, stores, services, drivers) { super(config, clients, stores, services, drivers); @@ -44,7 +113,10 @@ export class SystemController extends PuterController { // `?ignore=a,b` disregards the named checks for this request only. // `?marked-degraded=a,b` demotes the named checks to a non-fatal // `degraded` list: `ok` stays true but the response is 207 so the - // caller can tell the node is running in a degraded state. + // caller can tell the node is running in a degraded state. Either list + // accepts `@` to stand for every check in a group — notably + // `@dependencies` for the backing-service probes — so a caller polling + // this route doesn't have to enumerate them. const parseNames = (value) => typeof value === 'string' ? value @@ -52,44 +124,53 @@ export class SystemController extends PuterController { .map((name) => name.trim()) .filter(Boolean) : []; - router.get('/healthcheck', { subdomain: '*' }, async (req, res) => { - const health = this.services.health; - if (!health || typeof health.getStatus !== 'function') { - // Fallback for boot ordering / missing service. - return res.send('ok'); - } - const status = await health.getStatus({ - ignore: parseNames(req.query.ignore), - degrade: parseNames(req.query['marked-degraded']), - }); - if (!status.ok) return res.status(503).json(status); - if (status.degraded?.length) return res.status(207).json(status); - return res.json(status); - }); + router.get( + '/healthcheck', + { subdomain: '*', rateLimit: HEALTHCHECK_LIMIT }, + async (req, res) => { + const health = this.services.health; + if (!health || typeof health.getStatus !== 'function') { + // Fallback for boot ordering / missing service. + return res.send('ok'); + } + const status = await health.getStatus({ + ignore: parseNames(req.query.ignore), + degrade: parseNames(req.query['marked-degraded']), + }); + if (!status.ok) return res.status(503).json(status); + if (status.degraded?.length) + return res.status(207).json(status); + return res.json(status); + }, + ); // -- Version ------------------------------------------------- - router.get('/version', { subdomain: '*' }, (_req, res) => { - const version = - this.config.version ?? - process.env.npm_package_version ?? - 'unknown'; - const parts = String(version).split('.'); - // Deploy-constant, and callers poll it. Cache per-client only: - // a shared cache could pin one region's `location` for everyone, - // and the short window still bounds how long a client can miss a - // new deploy. - res.setHeader('Cache-Control', 'private, max-age=60'); - res.json({ - version, - major: parts[0] ? Number(parts[0]) : null, - minor: parts[1] ? Number(parts[1]) : null, - patch: parts[2] ? Number(parts[2]) : null, - environment: this.config.env ?? 'prod', - location: this.config.serverId ?? null, - deploy_timestamp: this.bootTime, - }); - }); + router.get( + '/version', + { subdomain: '*', rateLimit: VERSION_LIMIT }, + (_req, res) => { + const version = + this.config.version ?? + process.env.npm_package_version ?? + 'unknown'; + const parts = String(version).split('.'); + // Deploy-constant, and callers poll it. Cache per-client only: + // a shared cache could pin one region's `location` for everyone, + // and the short window still bounds how long a client can miss a + // new deploy. + res.setHeader('Cache-Control', 'private, max-age=60'); + res.json({ + version, + major: parts[0] ? Number(parts[0]) : null, + minor: parts[1] ? Number(parts[1]) : null, + patch: parts[2] ? Number(parts[2]) : null, + environment: this.config.env ?? 'prod', + location: this.config.serverId ?? null, + deploy_timestamp: this.bootTime, + }); + }, + ); // -- Contact us ---------------------------------------------- @@ -153,7 +234,7 @@ export class SystemController extends PuterController { // -- GET /whoarewe ------------------------------------------- - router.get('/whoarewe', {}, (_req, res) => { + router.get('/whoarewe', { rateLimit: WHOAREWE_LIMIT }, (_req, res) => { res.json({ name: 'Puter', version: this.config.version ?? null, @@ -181,8 +262,16 @@ export class SystemController extends PuterController { } res.json({ interfaces }); }; - router.get('/lsmod', { subdomain: 'api', requireAuth: true }, lsmod); - router.post('/lsmod', { subdomain: 'api', requireAuth: true }, lsmod); + router.get( + '/lsmod', + { subdomain: 'api', requireAuth: true, rateLimit: LSMOD_LIMIT }, + lsmod, + ); + router.post( + '/lsmod', + { subdomain: 'api', requireAuth: true, rateLimit: LSMOD_LIMIT }, + lsmod, + ); } onServerStart() {} diff --git a/src/backend/controllers/system/SystemController.test.ts b/src/backend/controllers/system/SystemController.test.ts index 4ba0e63e0..142b323ec 100644 --- a/src/backend/controllers/system/SystemController.test.ts +++ b/src/backend/controllers/system/SystemController.test.ts @@ -141,6 +141,19 @@ const callRoute = async ( // ── /healthcheck ──────────────────────────────────────────────────── describe('SystemController GET /healthcheck', () => { + // This route decides whether a node stays in rotation, so its rate limit + // must not reach for a backing service. The default backend is redis, + // whose client queues rather than fails fast while it is unreachable — + // enough to push a probe past the 4s the load balancer allows and evict + // every target during a redis degradation. In-process counting keeps the + // liveness path free of anything it is itself reporting on. + it('counts in-process, so liveness never waits on redis', () => { + const route = router.routes.find( + (r) => r.method === 'get' && r.path === '/healthcheck', + ); + expect(route?.options.rateLimit?.backend).toBe('memory'); + }); + it('returns the live ServerHealthService status payload', async () => { const { res, captured } = makeRes(); await callRoute('get', '/healthcheck', makeReq({}), res); @@ -461,6 +474,52 @@ describe('SystemController GET /lsmod', () => { }); }); +// ── rate-limit scopes ─────────────────────────────────────────────── + +describe('SystemController public route rate limits', () => { + const rateLimitOf = (path: string) => { + const route = router.routes.find( + (r) => r.method === 'get' && r.path === path, + ); + if (!route) throw new Error(`No GET ${path} route`); + return route.options.rateLimit as { + scope: string; + limit: number; + window: number; + key: string; + }; + }; + + it('gives /healthcheck, /version and /whoarewe separate buckets', () => { + // These once shared one scope, which meant clients polling /version + // could exhaust the budget that liveness probes depend on. Keep them + // apart: a 429 on /healthcheck is read as an unhealthy node. + const scopes = [ + rateLimitOf('/healthcheck').scope, + rateLimitOf('/version').scope, + rateLimitOf('/whoarewe').scope, + ]; + expect(new Set(scopes).size).toBe(3); + expect(scopes).toEqual(['healthcheck', 'version', 'whoarewe']); + }); + + it('sizes the unauthenticated buckets for a shared address, not one client', () => { + // All three key on IP, and an IP is a NAT, a campus or a carrier + // gateway — the bucket aggregates everyone behind it, and the + // limiter counts region-wide rather than per process. + for (const path of ['/healthcheck', '/version', '/whoarewe']) { + const limit = rateLimitOf(path); + expect(limit.key).toBe('ip'); + expect(limit.window).toBe(60_000); + expect(limit.limit).toBeGreaterThanOrEqual(6_000); + } + // Liveness polling is the most generous of the three by design. + expect(rateLimitOf('/healthcheck').limit).toBe(30_000); + expect(rateLimitOf('/version').limit).toBe(6_000); + expect(rateLimitOf('/whoarewe').limit).toBe(6_000); + }); +}); + // ── lifecycle ─────────────────────────────────────────────────────── describe('SystemController.onServerPrepareShutdown', () => { diff --git a/src/backend/controllers/webdav/WebDAVController.test.ts b/src/backend/controllers/webdav/WebDAVController.test.ts index 89b3446ac..6c3235022 100644 --- a/src/backend/controllers/webdav/WebDAVController.test.ts +++ b/src/backend/controllers/webdav/WebDAVController.test.ts @@ -64,7 +64,20 @@ const makeRes = () => { headers: {}, ended: false, }; + const listeners: Record void>> = {}; const res = { + // The DAV mount holds a concurrency slot for the life of the request + // and releases it on `finish` / `close`, so the stub has to behave + // like an emitter or every dispatch throws. + once: vi.fn((event: string, fn: () => void) => { + (listeners[event] ??= []).push(fn); + return res; + }), + emit: vi.fn((event: string) => { + const fns = listeners[event] ?? []; + listeners[event] = []; + for (const fn of fns) fn(); + }), json: vi.fn((value: unknown) => { captured.body = value; return res; @@ -89,10 +102,12 @@ const makeRes = () => { }), send: vi.fn((value: unknown) => { captured.body = value; + res.emit('finish'); return res; }), end: vi.fn(() => { captured.ended = true; + res.emit('finish'); return res; }), headersSent: false, @@ -962,6 +977,11 @@ describe('WebDAVController verbs', () => { send: (value: unknown) => { captured.body = value; captured.ended = true; + // End the underlying Writable so `finish` fires, as it does + // on a real response. The DAV mount releases its concurrency + // slot on that event — without it every `send()` path would + // leak a slot and later requests would 429. + if (!sink.writableEnded) sink.end(); return res; }, headersSent: false, diff --git a/src/backend/controllers/webdav/WebDAVController.ts b/src/backend/controllers/webdav/WebDAVController.ts index 7f072a530..c36be3b6c 100644 --- a/src/backend/controllers/webdav/WebDAVController.ts +++ b/src/backend/controllers/webdav/WebDAVController.ts @@ -42,6 +42,12 @@ import { hasWritePermission, refreshLock, } from './locks.js'; +import { DAV_CONCURRENT, DAV_LIMIT } from '../fs/limits.js'; +import { + acquireConcurrent, + checkRateLimit, + computeNetworkFingerprint, +} from '../../core/http/middleware/rateLimit.js'; const DAV_HEADERS = { DAV: '1, 2, ordered-collections', @@ -70,10 +76,17 @@ export class WebDAVController extends PuterController { // Single catch-all on the `dav` subdomain. We dispatch by req.method // inside the handler because WebDAV uses non-standard HTTP verbs that // Express doesn't have first-class router methods for in all versions. + // + // The rate limit is applied inside the handler rather than through + // `RouteOptions`. For a `use` mount the subdomain check lives in the + // handler wrapper, not in the middleware chain — so a `rateLimit` + // here would run for every request on every subdomain and count + // non-DAV traffic against the DAV budget. router.use( { subdomain: 'dav' }, async (req: Request, res: Response, _next) => { try { + if (!(await this.#admit(req, res))) return; await this.#dispatch(req, res); } catch (err) { if (err instanceof HttpError) { @@ -88,6 +101,41 @@ export class WebDAVController extends PuterController { ); } + /** + * Rate + concurrency gate for the whole DAV surface. Returns false when the + * request was rejected (429 already sent). + * + * Runs before `#dispatch` authenticates, so it keys on the network + * fingerprint rather than an actor. That is the coarser bucket, but a DAV + * client sends credentials on every request anyway — there is no + * unauthenticated browsing phase to protect a per-user key from. + */ + async #admit(req: Request, res: Response): Promise { + const key = computeNetworkFingerprint(req); + if ( + !(await checkRateLimit( + `${DAV_LIMIT.scope}:${key}`, + DAV_LIMIT.limit, + DAV_LIMIT.window, + )) + ) { + res.status(429).send('Too many requests.'); + return false; + } + const slot = await acquireConcurrent( + `${DAV_CONCURRENT.scope}:${key}`, + DAV_CONCURRENT.limit, + ); + if (!slot.ok) { + res.status(429).send('Too many concurrent requests.'); + return false; + } + // `finish` and `close` can both fire; release is once-only. + res.once('finish', () => void slot.release()); + res.once('close', () => void slot.release()); + return true; + } + async #dispatch(req: Request, res: Response): Promise { // Authenticate const actor = await this.#resolveActor(req, res); diff --git a/src/backend/controllers/wisp/WispController.ts b/src/backend/controllers/wisp/WispController.ts index 9761e76b5..c7a044131 100644 --- a/src/backend/controllers/wisp/WispController.ts +++ b/src/backend/controllers/wisp/WispController.ts @@ -21,6 +21,10 @@ import type { Request, Response } from 'express'; import { HttpError } from '../../core/http/HttpError.js'; import type { PuterRouter } from '../../core/http/PuterRouter.js'; import { PuterController } from '../types.js'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '../../services/metering/consts.js'; /** * WISP relay token controller — create and verify short-lived JWT tokens for @@ -32,12 +36,39 @@ export class WispController extends PuterController { registerRoutes(router: PuterRouter): void { router.post( '/wisp/relay-token/create', - { subdomain: 'api', requireAuth: true }, + { + subdomain: 'api', + requireAuth: true, + // Auth is optional in practice (the handler tolerates an + // anonymous actor), so the key falls back to a fingerprint + // when there is no user to key on. + rateLimit: { + scope: 'wisp-token-create', + limit: 60, + window: 60_000, + key: 'user', + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 30, + [DEFAULT_TEMP_SUBSCRIPTION]: 10, + }, + }, + }, this.#create, ); router.post( '/wisp/relay-token/verify', - { subdomain: 'api', requireAuth: false }, + { + subdomain: 'api', + requireAuth: false, + // Unauthenticated by design, which makes it a token-guessing + // oracle without a ceiling. + rateLimit: { + scope: 'wisp-token-verify', + limit: 300, + window: 60_000, + key: 'ip', + }, + }, this.#verify, ); } diff --git a/src/backend/core/http/middleware/rateLimit.js b/src/backend/core/http/middleware/rateLimit.js index e6e26935f..a479ab318 100644 --- a/src/backend/core/http/middleware/rateLimit.js +++ b/src/backend/core/http/middleware/rateLimit.js @@ -57,32 +57,57 @@ export const RATE_LIMIT_BACKENDS = ['memory', 'redis', 'kv']; const MEMORY_MAX_KEYS = 10_000; const MEMORY_MAX_RETAIN_MS = 60 * 60_000; +/** + * Key → `{ ts, windowMs }`. The window is kept alongside the timestamps because + * the sweep has to know it: collecting on `MEMORY_MAX_RETAIN_MS` alone would + * drop the state of any limit whose window is longer than the retention floor, + * which silently shortens that limit to the floor. Day-scale windows are a real + * shape — a "few per day" grant, for one — and under `memory` they were being + * reset every hour. Retention is therefore whichever is longer, and the key cap + * below is what actually bounds memory. + */ const memoryWindows = new Map(); + +/** + * Drop memory buckets that can no longer affect a decision. Runs on a timer + * below; exported as a test seam because the timer isn't drivable from a test + * (it's created at module load, before any fake clock is installed). + */ +export function sweepMemoryWindows() { + const now = Date.now(); + for (const [k, entry] of memoryWindows) { + const retainMs = Math.max(MEMORY_MAX_RETAIN_MS, entry.windowMs); + if ( + entry.ts.length === 0 || + entry.ts[entry.ts.length - 1] < now - retainMs + ) + memoryWindows.delete(k); + } +} + { - const sweep = setInterval(() => { - const cutoff = Date.now() - MEMORY_MAX_RETAIN_MS; - for (const [k, ts] of memoryWindows) { - if (ts.length === 0 || ts[ts.length - 1] < cutoff) - memoryWindows.delete(k); - } - }, 60_000); + const sweep = setInterval(sweepMemoryWindows, 60_000); sweep.unref?.(); } async function checkMemory(key, limit, windowMs) { const now = Date.now(); const cutoff = now - windowMs; - let timestamps = memoryWindows.get(key); - if (!timestamps) { + let entry = memoryWindows.get(key); + if (!entry) { // Map preserves insertion order; FIFO-evict before adding so a // unique-key flood between sweep ticks can't blow up memory. if (memoryWindows.size >= MEMORY_MAX_KEYS) { const oldest = memoryWindows.keys().next().value; memoryWindows.delete(oldest); } - timestamps = []; - memoryWindows.set(key, timestamps); + entry = { ts: [], windowMs }; + memoryWindows.set(key, entry); + } else { + // A scope's window can change across a deploy; the live value wins. + entry.windowMs = windowMs; } + const timestamps = entry.ts; while (timestamps.length > 0 && timestamps[0] < cutoff) timestamps.shift(); if (timestamps.length >= limit) return false; timestamps.push(now); @@ -185,37 +210,57 @@ async function acquireMemoryConcurrent(key, limit) { if (c <= 1) memoryConcurrentCounts.delete(key); else memoryConcurrentCounts.set(key, c - 1); }, + // Nothing expires a memory slot but the process holding it, so there + // is no staleness to renew away. + renew: async () => {}, }; } async function acquireRedisConcurrent(redis, key, limit) { const redisKey = `concurrent:${key}`; - // Atomic INCR + EXPIRE. If the new count exceeds the limit we DECR - // ourselves back out; the brief over-count is invisible to other - // callers because INCR is atomic per-key. EXPIRE is a safety net for - // process death between acquire and release — slots eventually clear - // on their own so a crashed worker can't pin the bucket. + const member = `${Date.now()}-${crypto.randomUUID()}`; + // One sorted-set member per held slot, scored by acquire time — the same + // shape `checkRedis` uses for windows, and for the same reason: expiry has + // to be per-slot, not per-key. + // + // A counter with a key-wide TTL cannot express that. Whoever touches the + // key last decides when *every* slot on it expires, so a rejected acquire + // extends the life of the slots that rejected it — and a client that + // retries on rejection (a websocket reconnect loop is the pointed case) + // holds a leaked bucket open forever, locking its owner out of a resource + // nobody is actually using. Here the sweep below drops each slot on its own + // age, so a leak drains on schedule no matter how hard anyone retries. + const now = Date.now(); const results = await redis .multi() - .incr(redisKey) + // Slots older than the orphan window belonged to a process that died + // before releasing; drop them before counting. + .zremrangebyscore(redisKey, 0, now - ORPHAN_SAFETY_TTL_MS) + .zadd(redisKey, now, member) + .zcard(redisKey) + // Key-level TTL is only garbage collection for a bucket that goes + // quiet — the per-member sweep above is what bounds a live one. .expire(redisKey, ORPHAN_SAFETY_TTL_SEC) .exec(); const count = Number( - Array.isArray(results[0]) ? results[0][1] : results[0], + Array.isArray(results[2]) ? results[2][1] : results[2], ); if (count > limit) { - await redis.decr(redisKey); + await redis.zrem(redisKey, member); return { ok: false }; } return { ok: true, release: async () => { - // DECR can race below zero if the TTL fired between acquire - // and release (slot cleared, counter resets, we DECR to -1). - // Clamp on the next observation; it costs an extra read only - // on the rare TTL race. - const after = await redis.decr(redisKey); - if (after < 0) await redis.set(redisKey, 0); + await redis.zrem(redisKey, member); + }, + renew: async () => { + // Re-score in place so a slot held longer than the orphan window + // isn't mistaken for one whose owner died. `ZADD XX` only touches + // a member that's still there, so renewing after release (or after + // a sweep) can't resurrect the slot. + await redis.zadd(redisKey, 'XX', Date.now(), member); + await redis.expire(redisKey, ORPHAN_SAFETY_TTL_SEC); }, }; } @@ -246,6 +291,15 @@ async function acquireKvConcurrent(kv, key, limit) { release: async () => { await kv.del({ key: slotKey }); }, + renew: async () => { + // Push the row's own TTL out; a slot that outlives the orphan + // window is held, not abandoned. + await kv.set({ + key: slotKey, + value: 1, + expireAt: Math.ceil((Date.now() + ORPHAN_SAFETY_TTL_MS) / 1000), + }); + }, }; } @@ -559,6 +613,75 @@ export async function checkRateLimit(key, limit, windowMs, backend) { } } +/** + * Imperative concurrency acquire — the `acquire` twin to `checkRateLimit`, for + * long-lived things that aren't a request/response pair and so can't use + * `concurrencyGate`. The websocket handshake is the motivating case: the slot + * has to be held for the life of the connection, not the life of a response. + * + * Caller MUST invoke `release()` exactly once when the thing being counted ends + * (`ok: false` still returns a no-op `release`, so callers can release + * unconditionally). Fails open on backend error. + * + * `release()` returns a promise that settles once the slot is actually back — + * await it when the next observation has to see the freed slot. Fire-and-forget + * is fine for the usual case (an event handler on connection close), which is + * why it never rejects. + * + * A holder that can outlive `ORPHAN_SAFETY_TTL_MS` must call `renew()` on a + * timer, or the orphan sweep will reclaim its slot as abandoned and the cap + * stops counting it. Anything that finishes in seconds can ignore it. + */ +export async function acquireConcurrent(key, limit, backend) { + const bk = resolveBackend(backend); + try { + const result = await bk.acquire(key, limit); + if (!result.ok) + return { + ok: false, + release: async () => {}, + renew: async () => {}, + }; + let released = false; + return { + ok: true, + release: async () => { + if (released) return; + released = true; + try { + await result.release(); + } catch (err) { + console.error( + '[concurrent] imperative release failed:', + err, + ); + } + }, + renew: async () => { + if (released) return; + try { + await result.renew?.(); + } catch (err) { + console.error('[concurrent] imperative renew failed:', err); + } + }, + }; + } catch (err) { + console.error( + '[concurrent] imperative acquire failed, failing open:', + err, + ); + return { ok: true, release: async () => {}, renew: async () => {} }; + } +} + +/** + * How long a held slot stays valid without a `renew()`. Exported so a + * long-lived holder can pick a renewal cadence from it rather than hardcoding + * one that drifts out of step. + */ +export const CONCURRENT_SLOT_TTL_MS = ORPHAN_SAFETY_TTL_MS; + // -- Subscription-aware limit resolution ----------------------------- /** diff --git a/src/backend/core/http/middleware/rateLimit.test.js b/src/backend/core/http/middleware/rateLimit.test.js index 8d7257b3b..ad4a7b206 100644 --- a/src/backend/core/http/middleware/rateLimit.test.js +++ b/src/backend/core/http/middleware/rateLimit.test.js @@ -32,6 +32,8 @@ import { EventEmitter } from 'node:events'; import { isHttpError } from '../HttpError.js'; import { setupTestServer } from '../../../testUtil.ts'; import { + CONCURRENT_SLOT_TTL_MS, + acquireConcurrent, acquireDriverConcurrent, checkDriverRateLimit, checkRateLimit, @@ -39,6 +41,7 @@ import { configureRateLimit, listConfiguredRateLimitBackends, rateLimitGate, + sweepMemoryWindows, } from './rateLimit.js'; // The rate-limit module is configured once at boot in production. In tests @@ -848,7 +851,7 @@ describe('concurrencyGate — redis backend', () => { await redis.flushall(); }); - it('uses INCR/EXPIRE to coordinate slots and releases on finish', async () => { + it('coordinates slots across callers and releases on finish', async () => { const opts = { limit: 1, key: 'ip', @@ -879,6 +882,85 @@ describe('concurrencyGate — redis backend', () => { }); }); +// ── concurrency: orphaned slots ───────────────────────────────────── +// +// A slot whose holder died without releasing has to age out on its own. The +// pointed case is a caller that retries on rejection — a reconnecting socket — +// where a per-key expiry gets refreshed by the very attempts it is rejecting +// and the bucket never drains. + +describe('acquireConcurrent — orphan recovery (redis)', () => { + let redis; + beforeAll(() => { + redis = new RedisMock(); + configureRateLimit({ default: 'redis', redis }); + }); + afterAll(async () => { + vi.useRealTimers(); + await redis?.quit?.(); + configureRateLimit(); + }); + beforeEach(async () => { + vi.useFakeTimers(); + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')); + await redis.flushall(); + }); + + it('frees a leaked slot after the orphan window even while callers retry', async () => { + const key = 'orphan-retry'; + const start = Date.now(); + const step = 5 * 60_000; + + // Holder takes the only slot and dies — never releases. + expect((await acquireConcurrent(key, 1)).ok).toBe(true); + expect((await acquireConcurrent(key, 1)).ok).toBe(false); + + // Retry throughout the orphan window, the way a reconnecting client + // does. Each rejection must leave the leaked slot's own age untouched. + for ( + let elapsed = step; + elapsed < CONCURRENT_SLOT_TTL_MS; + elapsed += step + ) { + vi.setSystemTime(new Date(start + elapsed)); + expect((await acquireConcurrent(key, 1)).ok).toBe(false); + } + + vi.setSystemTime(new Date(start + CONCURRENT_SLOT_TTL_MS + step)); + expect((await acquireConcurrent(key, 1)).ok).toBe(true); + }); + + it('keeps a renewed slot held past the orphan window', async () => { + const key = 'orphan-renew'; + const slot = await acquireConcurrent(key, 1); + expect(slot.ok).toBe(true); + + // A live long-lived holder renews rather than letting the sweep + // mistake it for an abandoned one. + for ( + let elapsed = 0; + elapsed < 2 * CONCURRENT_SLOT_TTL_MS; + elapsed += 10 * 60_000 + ) { + vi.setSystemTime(new Date(Date.now() + 10 * 60_000)); + await slot.renew(); + } + + expect((await acquireConcurrent(key, 1)).ok).toBe(false); + await slot.release(); + expect((await acquireConcurrent(key, 1)).ok).toBe(true); + }); + + it('does not resurrect a slot renewed after release', async () => { + const key = 'orphan-renew-after-release'; + const slot = await acquireConcurrent(key, 1); + await slot.release(); + await slot.renew(); + + expect((await acquireConcurrent(key, 1)).ok).toBe(true); + }); +}); + // ── concurrency: subscription-based limits ────────────────────────── describe('concurrencyGate — bySubscription overrides', () => { @@ -1184,6 +1266,31 @@ describe('driver helpers — backend failures', () => { const req = { ip: '7.7.7.7', headers: {}, socket: {} }; + // Minimal redis whose concurrent-acquire always reports one held slot, so + // a test can substitute its own failing `zrem` and exercise release alone. + const stubConcurrentRedis = () => ({ + multi: () => ({ + zremrangebyscore: function () { + return this; + }, + zadd: function () { + return this; + }, + zcard: function () { + return this; + }, + expire: function () { + return this; + }, + exec: async () => [ + [null, 0], + [null, 1], + [null, 1], + [null, 1], + ], + }), + }); + it('checkDriverRateLimit fails open when the backend throws', async () => { configureRateLimit({ default: 'redis', @@ -1225,20 +1332,11 @@ describe('driver helpers — backend failures', () => { configureRateLimit({ default: 'redis', redis: { - multi: () => ({ - incr: function () { - return this; - }, - expire: function () { - return this; - }, - exec: async () => [[null, 1]], - }), - decr: async () => { - if (failRelease) throw new Error('decr exploded'); - return 0; + ...stubConcurrentRedis(), + zrem: async () => { + if (failRelease) throw new Error('zrem exploded'); + return 1; }, - set: async () => 'OK', }, }); const handle = await acquireDriverConcurrent(req, 'iface', 'm', { @@ -1261,19 +1359,10 @@ describe('driver helpers — backend failures', () => { configureRateLimit({ default: 'redis', redis: { - multi: () => ({ - incr: function () { - return this; - }, - expire: function () { - return this; - }, - exec: async () => [[null, 1]], - }), - decr: async () => { - throw new Error('decr exploded'); + ...stubConcurrentRedis(), + zrem: async () => { + throw new Error('zrem exploded'); }, - set: async () => 'OK', }, }); const res = new EventEmitter(); @@ -1295,25 +1384,26 @@ describe('driver helpers — backend failures', () => { spy.mockRestore(); }); - it('clamps a redis concurrent counter that went negative after a TTL race', async () => { + it('absorbs a release whose slot was already swept away', async () => { const redis = new RedisMock(); await redis.flushall(); configureRateLimit({ default: 'redis', redis }); - const handle = await acquireDriverConcurrent(req, 'iface', 'clamp', { - limit: 2, + const handle = await acquireDriverConcurrent(req, 'iface', 'sweep', { + limit: 1, }); expect(handle.ok).toBe(true); - // Simulate the orphan TTL firing between acquire and release. - const keys = await redis.keys('concurrent:*'); - for (const k of keys) await redis.del(k); + // The orphan sweep collected the whole bucket before release ran. + for (const k of await redis.keys('concurrent:*')) await redis.del(k); + await expect(handle.release()).resolves.toBeUndefined(); - await handle.release(); - const remaining = await redis.keys('concurrent:*'); - for (const k of remaining) { - expect(Number(await redis.get(k))).toBe(0); - } + // Releasing a slot that is already gone must not leave the bucket + // owing anything — the next caller gets a clean one. + const next = await acquireDriverConcurrent(req, 'iface', 'sweep', { + limit: 1, + }); + expect(next.ok).toBe(true); await redis.quit?.(); }); }); @@ -1342,3 +1432,112 @@ describe('memory backend key cap', () => { expect(await checkRateLimit(victim, 1, 60_000, 'memory')).toBe(true); }); }); + +describe('memory backend sweep retention', () => { + beforeAll(() => configureRateLimit()); + afterAll(() => { + vi.useRealTimers(); + configureRateLimit(); + }); + + // The sweep's retention floor is an hour. A window longer than that has + // to survive it, or the limit is silently shortened to the floor — a + // day-scale "few per day" grant would reset hourly. + it('keeps a bucket whose window outlives the retention floor', async () => { + vi.useFakeTimers(); + const day = 24 * 60 * 60_000; + const key = `long-window-${Math.random()}`; + + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')); + expect(await checkRateLimit(key, 1, day, 'memory')).toBe(true); + expect(await checkRateLimit(key, 1, day, 'memory')).toBe(false); + + // Two hours on: past the retention floor, far short of the window. + vi.setSystemTime(new Date('2026-01-01T02:00:00Z')); + sweepMemoryWindows(); + expect(await checkRateLimit(key, 1, day, 'memory')).toBe(false); + + // Past the window itself, the bucket is collectable again. + vi.setSystemTime(new Date('2026-01-02T01:00:00Z')); + sweepMemoryWindows(); + expect(await checkRateLimit(key, 1, day, 'memory')).toBe(true); + }); + + it('still collects a short-window bucket at the retention floor', async () => { + vi.useFakeTimers(); + const key = `short-window-${Math.random()}`; + + vi.setSystemTime(new Date('2026-01-01T00:00:00Z')); + expect(await checkRateLimit(key, 1, 60_000, 'memory')).toBe(true); + + vi.setSystemTime(new Date('2026-01-01T02:00:00Z')); + sweepMemoryWindows(); + // Nothing asserts the delete directly; a fresh bucket is the + // observable consequence of having been collected. + expect(await checkRateLimit(key, 1, 60_000, 'memory')).toBe(true); + }); +}); + +describe('acquireConcurrent (imperative)', () => { + beforeAll(() => configureRateLimit()); + afterAll(() => configureRateLimit()); + + // The websocket handshake is the motivating case: the slot is held for + // the life of the connection, not the life of a response, so it cannot + // go through `concurrencyGate`. + it('admits up to the limit and rejects past it', async () => { + const key = `imperative-${Math.random()}`; + const a = await acquireConcurrent(key, 2, 'memory'); + const b = await acquireConcurrent(key, 2, 'memory'); + const c = await acquireConcurrent(key, 2, 'memory'); + + expect(a.ok).toBe(true); + expect(b.ok).toBe(true); + expect(c.ok).toBe(false); + + await a.release(); + expect((await acquireConcurrent(key, 2, 'memory')).ok).toBe(true); + }); + + it('returns a no-op release on rejection so callers can release blindly', async () => { + const key = `imperative-noop-${Math.random()}`; + const held = await acquireConcurrent(key, 1, 'memory'); + const denied = await acquireConcurrent(key, 1, 'memory'); + + expect(denied.ok).toBe(false); + // Releasing a slot we never got must not free the one we did. + await denied.release(); + expect((await acquireConcurrent(key, 1, 'memory')).ok).toBe(false); + + await held.release(); + expect((await acquireConcurrent(key, 1, 'memory')).ok).toBe(true); + }); + + it('releases at most once even if called repeatedly', async () => { + const key = `imperative-once-${Math.random()}`; + const a = await acquireConcurrent(key, 1, 'memory'); + await a.release(); + await a.release(); + await a.release(); + + // A double release would have driven the counter negative and + // handed out more slots than the limit allows. + expect((await acquireConcurrent(key, 1, 'memory')).ok).toBe(true); + expect((await acquireConcurrent(key, 1, 'memory')).ok).toBe(false); + }); + + it('fails open when the backend throws', async () => { + const err = new Error('backend down'); + const redis = { + multi: () => ({ + incr: () => { + throw err; + }, + }), + }; + configureRateLimit({ redis }); + const result = await acquireConcurrent('any', 1, 'redis'); + expect(result.ok).toBe(true); + configureRateLimit(); + }); +}); diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts index db07d7161..25330bc3f 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.routing.test.ts @@ -31,6 +31,7 @@ import { afterAll, beforeAll, + beforeEach, describe, expect, it, @@ -44,6 +45,10 @@ import { setupTestServer } from '../../testUtil.js'; import { kv } from '../../util/kvSingleton.js'; import { withTestActor } from '../integrationTestUtil.js'; import { ChatCompletionDriver } from './ChatCompletionDriver.js'; +import { + clearUnhealthyRoutes, + markRouteUnhealthy, +} from './utils/providerHealth.js'; // -- OpenAI SDK mock ------------------------------------------------ // Gemini reaches Google through `new openai.OpenAI()` (default export) and @@ -75,6 +80,7 @@ let server: PuterServer; let driver: ChatCompletionDriver; const INFRON_KV_KEY = 'infronChat:models'; +const OPENROUTER_KV_KEY = 'openrouterChat:models'; // Google lists gemini-2.5-flash input at $0.30/MTok. The gateway quotes a // floor price across its upstream routes, so it undercuts — which is exactly @@ -101,12 +107,47 @@ const GATEWAY_CATALOG = [ min_prompt_price: 0.3, min_completion_price: 2.5, }, + { + // A vendor we integrate with directly that isn't Google — the case + // resold duplicates used to be dropped for. + id: 'deepseek/deepseek-v4-pro', + display_name: 'DeepSeek: V4 Pro', + category_type: 'LLM', + supported_endpoint_types: ['openai'], + context_length: 1_000_000, + max_output_tokens: 65_536, + min_prompt_price: 0.1, + min_completion_price: 0.5, + }, ]; +// OpenRouter's catalog is shaped differently, and it carries the same model +// under two upstream orgs — four routes total for deepseek-v4-pro once the +// vendor and Infron are counted. +const OPENROUTER_CATALOG = [ + 'deepseek/deepseek-v4-pro', + 'deepseek-ai/deepseek-v4-pro', + 'google/gemini-2.5-flash', +].map((id) => ({ + id, + name: `${id} (via OpenRouter)`, + pricing: { prompt: '0.0000001', completion: '0.0000005' }, + context_length: 1_000_000, + top_provider: { max_completion_tokens: 65_536 }, + created: 1_700_000_000, +})); + beforeAll(async () => { server = await setupTestServer(); kv.del?.(INFRON_KV_KEY); - axiosRequestMock.mockResolvedValue({ data: { data: GATEWAY_CATALOG } }); + kv.del?.(OPENROUTER_KV_KEY); + axiosRequestMock.mockImplementation(({ url }: { url: string }) => ({ + data: { + data: url.includes('openrouter') + ? OPENROUTER_CATALOG + : GATEWAY_CATALOG, + }, + })); // Built once, not per-test: `#buildModelMap` mutates the catalogs // providers hand back (lowercasing ids, pushing `puterId` onto the @@ -116,7 +157,9 @@ beforeAll(async () => { { providers: { gemini: { apiKey: 'test-key' }, + deepseek: { apiKey: 'test-key' }, infron: { apiKey: 'test-key' }, + openrouter: { apiKey: 'test-key' }, ollama: { enabled: false }, }, } as never, @@ -126,16 +169,26 @@ beforeAll(async () => { ); driver.onServerStart(); // `onServerStart` doesn't await `#buildModelMap`, and the gateway - // catalog resolves on a microtask — poll until both providers land. + // catalogs resolve on a microtask — poll until both gateways land. for (let i = 0; i < 200; i++) { const ids = await driver.list(); - if (ids.some((id) => id.startsWith('infron:'))) break; + if ( + ids.some((id) => id.startsWith('infron:')) && + ids.some((id) => id.startsWith('openrouter:')) + ) { + break; + } await new Promise((r) => setTimeout(r, 5)); } }); +// Every failure in this file marks the route it hit. Without this the first +// test would decide where the second one starts. +beforeEach(() => clearUnhealthyRoutes()); + afterAll(async () => { await server?.shutdown(); + clearUnhealthyRoutes(); }); /** @@ -157,8 +210,11 @@ const attemptsFor = async (model: string) => { caught = e as HttpError; } expect(caught).toBeInstanceOf(HttpError); - return (caught as unknown as { fields: { attempts: { model: string; provider: string }[] } }) - .fields.attempts; + return ( + caught as unknown as { + fields: { attempts: { model: string; provider: string }[] }; + } + ).fields.attempts; }; describe('ChatCompletionDriver gemini routing', () => { @@ -186,7 +242,87 @@ describe('ChatCompletionDriver gemini routing', () => { }); it('still routes models only the gateway carries to the gateway', async () => { - const attempts = await attemptsFor('google/gemini-2.5-flash-image-preview'); + const attempts = await attemptsFor( + 'google/gemini-2.5-flash-image-preview', + ); + + expect(attempts[0]).toMatchObject({ provider: 'infron' }); + }); +}); + +describe('ChatCompletionDriver duplicate-model fallback', () => { + // deepseek-v4-pro is served directly by DeepSeek, by Infron, and twice by + // OpenRouter (two upstream orgs) — four routes in one bucket. + const SHARED = 'deepseek-v4-pro'; + + it('keeps a reseller duplicate of any vendor, not just Google', async () => { + const attempts = await attemptsFor(SHARED); + + expect(attempts[0]).toMatchObject({ provider: 'deepseek' }); + expect(attempts.map((a) => a.provider)).toContain('infron'); + }); + + it('leaves openrouter below the other resellers in the chain', async () => { + const attempts = await attemptsFor(SHARED); + const providers = attempts.map((a) => a.provider); + + expect(providers.indexOf('infron')).toBeLessThan( + providers.indexOf('openrouter'), + ); + }); + + it('stops after three attempts even with a fourth route available', async () => { + const attempts = await attemptsFor(SHARED); + expect(attempts).toHaveLength(3); + + // Proof the cap is what stopped the chain rather than the bucket + // running dry: take the three just tried out of contention and a + // fourth route is still there to be served. + const burned = attempts.map((a) => `${a.provider}:${a.model}`); + for (const a of attempts) markRouteUnhealthy(a.provider, a.model); + + const next = await attemptsFor(SHARED); + expect(burned).not.toContain(`${next[0].provider}:${next[0].model}`); + }); + + it('never tries the same provider-and-model pair twice', async () => { + const attempts = await attemptsFor(SHARED); + const routes = attempts.map((a) => `${a.provider}:${a.model}`); + + expect(new Set(routes).size).toBe(routes.length); + }); +}); + +describe('ChatCompletionDriver unhealthy-route skipping', () => { + it('skips a route marked by an earlier failure and serves the next one', async () => { + markRouteUnhealthy('deepseek', 'deepseek-v4-pro'); + + const attempts = await attemptsFor('deepseek-v4-pro'); + + expect(attempts[0].provider).toBe('infron'); + expect(attempts.map((a) => a.provider)).not.toContain('deepseek'); + }); + + it('marks the routes a failing request burned through', async () => { + await attemptsFor('deepseek-v4-pro'); + + // The marks the first request left behind push the second one past + // everything that just failed. + const next = await attemptsFor('deepseek-v4-pro'); + expect(next[0].provider).not.toBe('deepseek'); + }); + + it('still serves a marked route when it is the only one left', async () => { + // gemini-2.5-flash-image-preview has a single route; marking it must + // degrade to trying it anyway rather than failing with no attempt. + markRouteUnhealthy( + 'infron', + 'infron:google/gemini-2.5-flash-image-preview', + ); + + const attempts = await attemptsFor( + 'google/gemini-2.5-flash-image-preview', + ); expect(attempts[0]).toMatchObject({ provider: 'infron' }); }); diff --git a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts index 6df950988..233636523 100644 --- a/src/backend/drivers/ai-chat/ChatCompletionDriver.ts +++ b/src/backend/drivers/ai-chat/ChatCompletionDriver.ts @@ -62,12 +62,17 @@ import { normalize_single_message, } from './utils/Messages.js'; import { - AGGREGATOR_PROVIDERS, compareModelPreference, + isIdentityKey, + normalizeModelKey, } from './utils/modelRouting.js'; +import { + isRouteUnhealthy, + markRouteUnhealthy, +} from './utils/providerHealth.js'; import { AIChatStream } from './utils/Streaming.js'; -const MAX_FALLBACKS = 4; // includes first attempt +const MAX_ATTEMPTS = 3; // the first attempt plus two fallbacks type ProviderAttempt = { model: string; @@ -128,6 +133,25 @@ const isUpstream5xx = (a: ProviderAttempt) => a.error, ); +/** + * Whether a failure indicts the route rather than the request. + * + * Outages, rate limits and bad credentials will hit the next caller too, so the + * route is worth marking. A 4xx the upstream returned on the request's own + * merits (malformed tools, oversized prompt) says nothing about the route and + * must not take it out of rotation for everyone else. Attempts with no status + * at all are transport failures — treat them as route problems. + */ +const isRouteLevelFailure = (a: ProviderAttempt) => + a.status === undefined || + isRateLimit(a) || + isAuthFailure(a) || + isUpstream5xx(a); + +// One bucket can hold the same model id under several providers *and* several +// ids under one provider, so only the pair identifies an attempt. +const routeId = (provider: string, modelId: string) => `${provider}:${modelId}`; + /** * Map an exhausted fallback chain to a single user-facing HttpError. * @@ -450,6 +474,20 @@ export class ChatCompletionDriver extends PuterDriver { const attempts: ProviderAttempt[] = []; let res: IChatCompleteResult | undefined; + // A failed route is remembered briefly so the next request skips it + // rather than paying its timeout again. + const recordFailure = ( + modelId: string, + providerId: string, + err: unknown, + ) => { + const attempt = toAttempt(modelId, providerId, err); + attempts.push(attempt); + if (isRouteLevelFailure(attempt)) { + markRouteUnhealthy(providerId, modelId); + } + }; + try { res = await provider.complete({ ...args, @@ -457,19 +495,17 @@ export class ChatCompletionDriver extends PuterDriver { provider: model.provider, }); } catch (e) { - attempts.push(toAttempt(model.id, model.provider!, e)); + recordFailure(model.id, model.provider!, e); - // Fallback loop - const tried = [model.id]; - const triedProviders = [model.provider!]; + // Fallback loop — the bucket holds every provider that serves this + // model, ranked by `compareModelPreference`, so each miss walks one + // step down that order. + const bucketKey = model.id; + const tried = new Set([routeId(model.provider!, model.id)]); let lastError: Error | null = e as Error; - while (lastError && tried.length < MAX_FALLBACKS) { - const fallback = this.#findFallback( - model.id, - tried, - triedProviders, - ); + while (lastError && attempts.length < MAX_ATTEMPTS) { + const fallback = this.#findFallback(bucketKey, tried); if (!fallback) break; const fbProvider = this.#providers[fallback.provider!]; @@ -486,8 +522,7 @@ export class ChatCompletionDriver extends PuterDriver { }); } - tried.push(fallback.id); - triedProviders.push(fallback.provider!); + tried.add(routeId(fallback.provider!, fallback.id)); try { res = await fbProvider.complete({ @@ -499,9 +534,7 @@ export class ChatCompletionDriver extends PuterDriver { lastError = null; } catch (fbErr) { lastError = fbErr as Error; - attempts.push( - toAttempt(fallback.id, fallback.provider!, fbErr), - ); + recordFailure(fallback.id, fallback.provider!, fbErr); } } } @@ -1030,107 +1063,79 @@ export class ChatCompletionDriver extends PuterDriver { // -- Model map --------------------------------------------------- + /** + * Group every provider's catalog into per-model buckets. + * + * A bucket is the set of routes to one model: the vendor we integrate with + * directly plus every reseller carrying it. They are deliberately _not_ + * deduplicated — the duplicates are what the fallback loop walks when a + * route fails. `compareModelPreference` decides who serves first, so a + * reseller only takes traffic once the vendor has actually failed. + * + * Entries join a bucket by identity key (see `isIdentityKey`); display + * names remain addressable but never merge two providers' entries. + */ async #buildModelMap() { for (const providerName in this.#providers) { const provider = this.#providers[providerName]; - const isAggregator = AGGREGATOR_PROVIDERS.has(providerName); for (const model of await provider.models()) { - model.id = model.id.trim().toLowerCase(); - if (!this.#modelIdMap[model.id]) { - this.#modelIdMap[model.id] = []; - } - this.#modelIdMap[model.id].push({ - ...model, - provider: providerName, - }); - + model.id = normalizeModelKey(model.id); if (model.puterId) { - if (model.aliases) { - model.aliases.push(model.puterId); - } else { - model.aliases = [model.puterId]; - } + model.aliases = model.aliases + ? [...model.aliases, model.puterId] + : [model.puterId]; } - if (isAggregator && model.aliases) { - let skip = false; - for (const rawAlias of model.aliases) { - const alias = rawAlias.trim().toLowerCase(); - const existing = this.#modelIdMap[alias]; - if ( - existing && - existing !== this.#modelIdMap[model.id] - ) { - if (existing.some((m) => m.provider === 'gemini')) { - // Gemini is the one vendor whose resold - // duplicates we keep, so a Google outage has - // somewhere to fall back to. Ranking (see - // `compareModelPreference`) keeps the direct - // provider ahead of them. - continue; - } - skip = true; - break; - } - } - if (skip) { - // Remove the entry we just pushed; leave the bucket - // intact for other providers. - const bucket = this.#modelIdMap[model.id]; - bucket.pop(); - if (bucket.length === 0) { - delete this.#modelIdMap[model.id]; - } - continue; - } + // Catalogs derive an alias by stripping the vendor org off the + // id, which yields '' for ids that carry no org. Drop those — + // an empty key would pool unrelated models together. + const keys = [model.id, ...(model.aliases ?? [])] + .map(normalizeModelKey) + .filter((key) => key.length > 0); + + const bucket = + keys + .filter(isIdentityKey) + .map((key) => this.#modelIdMap[key]) + .find(Boolean) ?? []; + bucket.push({ ...model, provider: providerName }); + + // First registration owns a key: a name already claimed by + // another model keeps pointing where it did. + for (const key of keys) { + this.#modelIdMap[key] ??= bucket; } - if (model.aliases) { - for (let alias of model.aliases) { - alias = alias.trim().toLowerCase(); - if (!this.#modelIdMap[alias]) { - this.#modelIdMap[alias] = - this.#modelIdMap[model.id]; - } else if ( - this.#modelIdMap[alias] !== - this.#modelIdMap[model.id] - ) { - this.#modelIdMap[alias].push({ - ...model, - provider: providerName, - }); - this.#modelIdMap[model.id] = - this.#modelIdMap[alias]; - } - } - } - - this.#modelIdMap[model.id].sort(compareModelPreference); + bucket.sort(compareModelPreference); } } } #resolveModel(modelId: string, provider?: string): IChatModel | null { - const models = this.#modelIdMap[modelId?.trim().toLowerCase()]; + const models = this.#modelIdMap[normalizeModelKey(modelId ?? '')]; if (!models || models.length === 0) return null; - if (!provider) return models[0]; - return models.find((m) => m.provider === provider) ?? models[0]; + // An explicitly requested provider is honoured even if its route is + // marked — the caller asked for that one, not for the cheapest hop. + if (provider) { + const pinned = models.find((m) => m.provider === provider); + if (pinned) return pinned; + } + return this.#preferHealthy(models) ?? models[0]; } - #findFallback( - modelId: string, - tried: string[], - triedProviders: string[], - ): IChatModel | null { + #findFallback(modelId: string, tried: Set): IChatModel | null { const models = this.#modelIdMap[modelId]; if (!models) return null; - return ( - models.find( - (m) => - !tried.includes(m.id) || - !triedProviders.includes(m.provider!), - ) ?? null + const untried = models.filter( + (m) => !tried.has(routeId(m.provider!, m.id)), ); + // Degrade to a marked route rather than to no route at all: the marks + // are a hint about recent failures, not a quota. + return this.#preferHealthy(untried) ?? untried[0] ?? null; + } + + #preferHealthy(models: IChatModel[]): IChatModel | undefined { + return models.find((m) => !isRouteUnhealthy(m.provider!, m.id)); } } diff --git a/src/backend/drivers/ai-chat/utils/modelRouting.test.ts b/src/backend/drivers/ai-chat/utils/modelRouting.test.ts index 2073364d4..018bcc000 100644 --- a/src/backend/drivers/ai-chat/utils/modelRouting.test.ts +++ b/src/backend/drivers/ai-chat/utils/modelRouting.test.ts @@ -3,24 +3,29 @@ * * This file is part of Puter. * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * Puter is free software: you can redistribute it and/or modify it under the + * terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. * * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . + * along with this program. If not, see + * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/). */ import { describe, expect, it } from 'vitest'; import { GEMINI_MODELS } from '../providers/gemini/models.js'; import type { IChatModel } from '../types.js'; -import { compareModelPreference } from './modelRouting.js'; +import { + compareModelPreference, + isIdentityKey, + normalizeModelKey, +} from './modelRouting.js'; // `#buildModelMap` mutates the catalogs providers hand back, and // `GeminiChatProvider.models()` returns the module-level `GEMINI_MODELS` by @@ -97,6 +102,21 @@ describe('compareModelPreference', () => { expect(winner(together, openrouter).provider).toBe('openrouter'); }); + it('puts openrouter then together-ai at the very bottom of the chain', () => { + // Both quote well under the other resellers — price must not lift + // either of them out of the last two slots. + const bucket = [ + resoldModel('together-ai', 'meta/llama-4', 1), + resoldModel('openrouter', 'meta/llama-4', 2), + resoldModel('neuralwatt', 'meta/llama-4', 400), + resoldModel('infron', 'meta/llama-4', 500), + ]; + + expect( + bucket.sort(compareModelPreference).map((m) => m.provider), + ).toEqual(['neuralwatt', 'infron', 'openrouter', 'together-ai']); + }); + it('still orders two direct providers by cheapest input cost', () => { const cheap = geminiModel('gemini-2.5-flash-lite'); const pricey = geminiModel('gemini-2.5-pro'); @@ -122,7 +142,9 @@ describe('compareModelPreference', () => { // The image-preview models are absent from GEMINI_MODELS, so the // gateway is the only route and must stay the winner. expect( - GEMINI_MODELS.some((m) => m.id === 'gemini-2.5-flash-image-preview'), + GEMINI_MODELS.some( + (m) => m.id === 'gemini-2.5-flash-image-preview', + ), ).toBe(false); const onlyRoute = resoldModel( @@ -133,3 +155,34 @@ describe('compareModelPreference', () => { expect(winner(onlyRoute).provider).toBe('infron'); }); }); + +describe('isIdentityKey', () => { + it('accepts the machine ids a catalog uses to name a model', () => { + for (const key of [ + 'claude-sonnet-4', + 'anthropic/claude-sonnet-4', + 'openrouter:anthropic/claude-sonnet-4', + 'gpt-4o', + ]) { + expect(isIdentityKey(key)).toBe(true); + } + }); + + it('rejects display names, so a shared label cannot merge two providers', () => { + // Gateways carry these alongside the machine ids. Merging on one + // would be merging two providers on a human-readable string. + for (const key of [ + normalizeModelKey('Google: Gemini 2.5 Flash'), + normalizeModelKey('Anthropic: Claude Sonnet 4'), + normalizeModelKey('Meta Llama 3.1 8B Instruct Turbo'), + ]) { + expect(isIdentityKey(key)).toBe(false); + } + }); + + it('rejects the empty key catalogs produce for ids carrying no vendor org', () => { + // `'gpt-4o'.split('/').slice(1).join('/')` is '' — pooling models + // under that key would put unrelated models in one bucket. + expect(isIdentityKey('')).toBe(false); + }); +}); diff --git a/src/backend/drivers/ai-chat/utils/modelRouting.ts b/src/backend/drivers/ai-chat/utils/modelRouting.ts index 3c1635ddb..a8bb849f9 100644 --- a/src/backend/drivers/ai-chat/utils/modelRouting.ts +++ b/src/backend/drivers/ai-chat/utils/modelRouting.ts @@ -32,14 +32,39 @@ export const AGGREGATOR_PROVIDERS = new Set([ 'neuralwatt', ]); -// Lower rank is served first. `together-ai` sits behind the other resellers — -// a pre-existing guarantee this ranking extends rather than replaces. +// Lower rank is served first. `openrouter` and `together-ai` sit at the very +// bottom, in that order, behind the other resellers. const providerRank = (provider?: string): number => { - if (provider === 'together-ai') return 2; + if (provider === 'together-ai') return 3; + if (provider === 'openrouter') return 2; if (provider && AGGREGATOR_PROVIDERS.has(provider)) return 1; return 0; }; +/** + * Lookup form for a model id or alias: the model map is keyed case- and + * whitespace-insensitively. + */ +export const normalizeModelKey = (key: string): string => + key.trim().toLowerCase(); + +/** + * Whether a key asserts _which model this is_, rather than merely being another + * way to name it. + * + * Catalogs mix both into `aliases`: machine ids (`anthropic/claude-sonnet-4`, + * `claude-sonnet-4`) alongside human labels (`Anthropic: Claude Sonnet 4`). + * Only the former may pull an entry into another provider's bucket — two + * gateways agreeing on a display string is not evidence they serve the same + * weights, and a label collision would otherwise silently reroute traffic. + * Labels stay usable for lookup; they just don't merge anything. + * + * Vendor model ids never contain whitespace, and every display name in the + * catalogs we consume does — that separation is the whole test. + */ +export const isIdentityKey = (key: string): boolean => + key.length > 0 && !/\s/.test(key); + /** * Orders the candidates that share a model bucket; the first one gets served. * diff --git a/src/backend/drivers/ai-chat/utils/providerHealth.test.ts b/src/backend/drivers/ai-chat/utils/providerHealth.test.ts new file mode 100644 index 000000000..f895f6c6d --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/providerHealth.test.ts @@ -0,0 +1,84 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify it under the + * terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see + * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/). + */ + +import { afterEach, describe, expect, it } from 'vitest'; +import { kv } from '../../../util/kvSingleton.js'; +import { + clearUnhealthyRoutes, + isRouteUnhealthy, + markRouteUnhealthy, + UNHEALTHY_TTL_SEC, +} from './providerHealth.js'; + +afterEach(() => clearUnhealthyRoutes()); + +describe('providerHealth', () => { + it('reports an unmarked route as healthy', () => { + expect(isRouteUnhealthy('gemini', 'gemini-2.5-flash')).toBe(false); + }); + + it('marks one route without touching the same model elsewhere', () => { + markRouteUnhealthy('gemini', 'gemini-2.5-flash'); + + expect(isRouteUnhealthy('gemini', 'gemini-2.5-flash')).toBe(true); + // The whole point of the fallback chain: another provider still + // serves this model. + expect( + isRouteUnhealthy('infron', 'infron:google/gemini-2.5-flash'), + ).toBe(false); + }); + + it('marks one model without taking the rest of the provider out', () => { + markRouteUnhealthy('openai-completion', 'gpt-4o'); + + expect(isRouteUnhealthy('openai-completion', 'gpt-4o')).toBe(true); + expect(isRouteUnhealthy('openai-completion', 'gpt-4o-mini')).toBe( + false, + ); + }); + + it('expires the mark rather than needing a reset path', () => { + expect(UNHEALTHY_TTL_SEC).toBeGreaterThanOrEqual(5 * 60); + expect(UNHEALTHY_TTL_SEC).toBeLessThanOrEqual(15 * 60); + + markRouteUnhealthy('groq', 'llama-3.3-70b'); + const ttl = kv.ttl('aiChat:unhealthyRoute:groq:llama-3.3-70b'); + expect(ttl).toBeGreaterThan(0); + expect(ttl).toBeLessThanOrEqual(UNHEALTHY_TTL_SEC); + }); + + it('forgets the route once the mark has expired', () => { + markRouteUnhealthy('xai', 'grok-4'); + expect(isRouteUnhealthy('xai', 'grok-4')).toBe(true); + + kv.expire('aiChat:unhealthyRoute:xai:grok-4', -1); + expect(isRouteUnhealthy('xai', 'grok-4')).toBe(false); + }); + + it('clears every mark at once', () => { + markRouteUnhealthy('gemini', 'gemini-2.5-flash'); + markRouteUnhealthy('claude', 'claude-sonnet-4'); + + clearUnhealthyRoutes(); + + expect(isRouteUnhealthy('gemini', 'gemini-2.5-flash')).toBe(false); + expect(isRouteUnhealthy('claude', 'claude-sonnet-4')).toBe(false); + }); +}); diff --git a/src/backend/drivers/ai-chat/utils/providerHealth.ts b/src/backend/drivers/ai-chat/utils/providerHealth.ts new file mode 100644 index 000000000..a5ca5a418 --- /dev/null +++ b/src/backend/drivers/ai-chat/utils/providerHealth.ts @@ -0,0 +1,60 @@ +/** + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify it under the + * terms of the GNU Affero General Public License as published by the Free + * Software Foundation, either version 3 of the License, or (at your option) any + * later version. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS + * FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more + * details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see + * [https://www.gnu.org/licenses/](https://www.gnu.org/licenses/). + */ + +/** + * Short-lived memory of which (provider, model) routes are currently failing, + * so requests skip past a route that just broke instead of paying its timeout + * again on every call. + * + * Deliberately process-local and short-lived: it is a latency optimisation, not + * a circuit breaker. Nothing is ever hard-blocked — routing only _prefers_ + * healthy routes, and an entry expires on its own without any probe or reset + * path to get wrong. + */ + +import { kv } from '../../../util/kvSingleton.js'; + +/** How long a route stays marked after a route-level failure. */ +export const UNHEALTHY_TTL_SEC = 10 * 60; + +const routeKey = (provider: string, modelId: string) => + `aiChat:unhealthyRoute:${provider}:${modelId}`; + +/** + * Mark `modelId` as currently unserveable by `provider`. + * + * Only for failures that say something about the route itself — upstream 5xx, + * rate limits, bad credentials, transport errors. A request the upstream + * refused on its merits (a 400, an oversized prompt) says nothing about whether + * the next caller will be served, and must not mark anything. + */ +export const markRouteUnhealthy = (provider: string, modelId: string): void => { + kv.set(routeKey(provider, modelId), 1, { EX: UNHEALTHY_TTL_SEC }); +}; + +export const isRouteUnhealthy = (provider: string, modelId: string): boolean => + kv.get(routeKey(provider, modelId)) !== undefined; + +/** Test seam — drops every mark. */ +export const clearUnhealthyRoutes = (): void => { + for (const key of kv.keys('aiChat:unhealthyRoute:*')) { + kv.del(key); + } +}; diff --git a/src/backend/drivers/apps/AppDriver.js b/src/backend/drivers/apps/AppDriver.js index 62db4808d..70ce9e5f7 100644 --- a/src/backend/drivers/apps/AppDriver.js +++ b/src/backend/drivers/apps/AppDriver.js @@ -50,6 +50,27 @@ import { } from '../../util/validation.js'; import { PuterDriver } from '../types.js'; +/** + * Shared by every method that writes an app row. Each one also allocates or + * releases an app directory and a subdomain, so the ceiling is set by what the + * write costs rather than by the row itself. + * + * Not only by what a developer does by hand, though — app creation is also a + * programmatic step: deploying a worker creates an app to sandbox it under, so + * a script that provisions several in a row is ordinary rather than abusive, + * and a ceiling in the tens turns that into a partial deploy. + * + * @type {import('../meta.js').DriverRateLimitSpec} + */ +const APP_WRITE_LIMIT = { + limit: 240, + window: 60_000, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 120, + [DEFAULT_TEMP_SUBSCRIPTION]: 60, + }, +}; + const APP_NAME_REGEX = /^[a-zA-Z0-9_-]+$/; const APP_NAME_MAX_LEN = 100; const APP_TITLE_MAX_LEN = 100; @@ -118,6 +139,7 @@ export class AppDriver extends PuterDriver { // on permission grants in `hardcoded-permissions.js`. Re-expressed // here as subscription-tier overrides — the metering service maps // anonymous users to `temp_free` and registered users to `user_free`. + /** @type {import('../meta.js').DriverRateLimitConfig} */ rateLimit = { default: { limit: 100, @@ -127,6 +149,37 @@ export class AppDriver extends PuterDriver { [DEFAULT_TEMP_SUBSCRIPTION]: 50, }, }, + methods: { + // The blanket envelope above is sized for `read`/`select`, + // which desktop boot calls repeatedly. Writing an app row also + // allocates an app directory and a subdomain, so it does not + // belong on a read-shaped budget. + create: APP_WRITE_LIMIT, + update: APP_WRITE_LIMIT, + upsert: APP_WRITE_LIMIT, + delete: APP_WRITE_LIMIT, + // Answers "does this name exist?" for any name, so it is a + // name-enumeration oracle regardless of how cheap it is. + isNameAvailable: { + limit: 60, + window: 60_000, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 30, + [DEFAULT_TEMP_SUBSCRIPTION]: 10, + }, + }, + }, + }; + + /** @type {import('../meta.js').DriverConcurrentConfig} */ + concurrent = { + default: { + limit: 20, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 10, + [DEFAULT_TEMP_SUBSCRIPTION]: 5, + }, + }, }; get appStore() { diff --git a/src/backend/drivers/driverPolicies.test.ts b/src/backend/drivers/driverPolicies.test.ts index c2ce7017b..4ac4de13e 100644 --- a/src/backend/drivers/driverPolicies.test.ts +++ b/src/backend/drivers/driverPolicies.test.ts @@ -31,7 +31,9 @@ import { AppDriver } from './apps/AppDriver.js'; import { KVStoreDriver } from './kv/KVStoreDriver.js'; import { NotificationDriver } from './notification/NotificationDriver.js'; import { SubdomainDriver } from './subdomain/SubdomainDriver.js'; +import { WorkerDriver } from './workers/WorkerDriver.js'; +import { DRIVERS_CALL_LIMIT } from '../controllers/drivers/DriverController.js'; import { resolveDriverMeta } from './meta.js'; import { DEFAULT_FREE_SUBSCRIPTION, @@ -74,8 +76,44 @@ describe('KVStoreDriver — rate-limit policy', () => { }); }); - it('declares no concurrent cap (intentional — kv calls are cheap)', () => { - expect(m.concurrent).toBeUndefined(); + // Asserted as a relationship rather than as literals: what has to hold is + // that a scan is charged more than a point read and that every tier still + // clears an app rendering a view, not that the numbers are any particular + // pair. + it('gives `list` its own budget — a prefix scan, not a point read', () => { + const list = m.rateLimit?.methods?.list; + const dflt = m.rateLimit?.default; + expect(list).toBeDefined(); + + // Per-second, since the two use different windows. + const perSecond = (spec: { limit: number; window?: number }): number => + spec.limit / ((spec.window ?? 60_000) / 1000); + expect(perSecond(list!)).toBeLessThan(perSecond(dflt!)); + + for (const tier of [ + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, + ]) { + expect(list!.bySubscription?.[tier]).toBeLessThanOrEqual( + list!.limit, + ); + // A view that lists on open shouldn't run out mid-session. + expect(list!.bySubscription?.[tier]).toBeGreaterThanOrEqual(30); + } + }); + + // An individual kv call is cheap, which is what the window is sized for. + // The concurrent cap is a different axis: it bounds how many can be in + // flight at once from a caller that never waits for a response. + it('caps in-flight calls, with `list` tighter than the default', () => { + expect(m.concurrent?.default).toEqual({ + limit: 30, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 15, + [DEFAULT_TEMP_SUBSCRIPTION]: 8, + }, + }); + expect(m.concurrent?.methods?.list?.limit).toBe(5); }); }); @@ -92,6 +130,72 @@ describe('AppDriver — rate-limit policy', () => { }, }); }); + + // The blanket envelope above is sized for the reads desktop boot makes. + // Writing an app row also allocates an app directory and a subdomain. + it('puts the write methods on a tighter budget than the reads', () => { + const perSecond = (spec: { limit: number; window?: number }): number => + spec.limit / ((spec.window ?? 60_000) / 1000); + const readRate = perSecond(m.rateLimit!.default!); + + for (const method of ['create', 'update', 'upsert', 'delete']) { + const spec = m.rateLimit?.methods?.[method]; + expect(spec).toBeDefined(); + expect(perSecond(spec!)).toBeLessThan(readRate); + + // Tighter than the reads, but not so tight that provisioning a + // few apps in a row — which deploying a worker does on the + // user's behalf — runs out partway through. + for (const tier of [ + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, + ]) { + expect(spec!.bySubscription?.[tier]).toBeGreaterThanOrEqual(30); + } + } + }); + + it('rate-limits the name-availability oracle separately', () => { + expect(m.rateLimit?.methods?.isNameAvailable?.limit).toBe(60); + }); +}); + +describe('WorkerDriver — rate-limit policy', () => { + const m = meta(new WorkerDriver(...fake())); + + // Without a declared policy this driver fell back to the generic + // 600/minute default, which does not fit a method that deploys code. + it('pins `create` below the driver`s own read budget', () => { + const create = m.rateLimit?.methods?.create; + expect(create).toBeDefined(); + expect(create!.limit).toBeLessThan(m.rateLimit!.default!.limit); + + // Developing against workers means redeploying on every change, so + // the floor has to clear a working session rather than a few tries. + for (const tier of [ + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, + ]) { + expect(create!.bySubscription?.[tier]).toBeGreaterThanOrEqual(20); + expect(create!.bySubscription?.[tier]).toBeLessThanOrEqual( + create!.limit, + ); + } + }); + + it('never drops a concurrency slot below 2', () => { + const specs = [ + m.concurrent?.default, + ...Object.values(m.concurrent?.methods ?? {}), + ].filter(Boolean); + expect(specs.length).toBeGreaterThan(0); + for (const spec of specs) { + expect(spec!.limit).toBeGreaterThanOrEqual(2); + for (const n of Object.values(spec!.bySubscription ?? {})) { + expect(n).toBeGreaterThanOrEqual(2); + } + } + }); }); describe('SubdomainDriver — rate-limit policy', () => { @@ -183,3 +287,87 @@ describe('puter-speech2txt — one driver covers every provider', () => { ); }); }); + +// ── Cross-driver invariants ──────────────────────────────────────── + +describe('every registered driver', () => { + const drivers = [ + ['kvStore', KVStoreDriver], + ['aiChat', ChatCompletionDriver], + ['aiImage', ImageGenerationDriver], + ['aiTts', TTSDriver], + ['aiVideo', VideoGenerationDriver], + ['aiSpeech2Speech', VoiceChangerDriver], + ['aiSpeech2Txt', SpeechToTextDriver], + ['aiOcr', OCRDriver], + ['apps', AppDriver], + ['subdomains', SubdomainDriver], + ['notifications', NotificationDriver], + ['workers', WorkerDriver], + ] as const; + + // A driver that declares nothing silently inherits the generic + // 600/minute fallback in `checkDriverRateLimit`, which is far too loose + // for anything that writes or spends. Declaring is the point. + it.each(drivers)('%s declares a rate-limit policy', (_name, Driver) => { + const m = meta(new (Driver as any)(...fake())); + expect(m.rateLimit?.default ?? m.rateLimit?.methods).toBeTruthy(); + }); + + // Concurrency has no fallback at all — undeclared means unbounded. + it.each(drivers)('%s declares a concurrency cap', (_name, Driver) => { + const m = meta(new (Driver as any)(...fake())); + expect(m.concurrent?.default).toBeTruthy(); + }); + + // A single slot turns incidental client parallelism — two tabs, a + // prefetch alongside a user action — into a spurious 429. Paid tiers + // keep enough headroom to actually parallelise. + it.each(drivers)( + '%s keeps every concurrency slot at 2 or more, and 5+ when paid', + (_name, Driver) => { + const m = meta(new (Driver as any)(...fake())); + const specs = [ + m.concurrent?.default, + ...Object.values(m.concurrent?.methods ?? {}), + ].filter(Boolean); + for (const spec of specs) { + expect(spec!.limit).toBeGreaterThanOrEqual(5); + for (const n of Object.values(spec!.bySubscription ?? {})) { + expect(n).toBeGreaterThanOrEqual(2); + } + } + }, + ); + + // The `/call` route carries its own limit across the whole driver + // surface. It is meant to catch fan-out across many interfaces, which + // only works if it sits above what any single driver already allows — + // otherwise it quietly becomes the operative limit for the widest + // drivers and overrides the tier policy they declare, while their own + // assertions above keep passing because those check the declaration + // rather than the ceiling a caller actually meets. + // + // Windows differ per driver (10s, 30s, 60s), so compare rates. + const perMinute = (spec: { limit: number; window: number }) => + (spec.limit / spec.window) * 60_000; + + const envelopePerMinute = perMinute(DRIVERS_CALL_LIMIT); + + it.each(drivers)( + '%s declares no budget wider than the /call envelope', + (_name, Driver) => { + const m = meta(new (Driver as any)(...fake())); + const specs = [ + m.rateLimit?.default, + ...Object.values(m.rateLimit?.methods ?? {}), + ].filter(Boolean); + expect(specs.length).toBeGreaterThan(0); + for (const spec of specs) { + // `bySubscription` only ever carves *tighter* caps out of + // `limit`, so the base is the widest value in the spec. + expect(perMinute(spec!)).toBeLessThanOrEqual(envelopePerMinute); + } + }, + ); +}); diff --git a/src/backend/drivers/kv/KVStoreDriver.ts b/src/backend/drivers/kv/KVStoreDriver.ts index bc642a3c0..9e4c16663 100644 --- a/src/backend/drivers/kv/KVStoreDriver.ts +++ b/src/backend/drivers/kv/KVStoreDriver.ts @@ -25,7 +25,7 @@ import { } from '../../services/metering/consts.js'; import { PuterDriver } from '../types.js'; import type { Actor } from '../../core/actor.js'; -import type { DriverRateLimitConfig } from '../meta.js'; +import type { DriverConcurrentConfig, DriverRateLimitConfig } from '../meta.js'; import { APP_DATA_KV_METHOD_OPS, APP_DATA_KV_TTL_PARAMS, @@ -74,6 +74,43 @@ export class KVStoreDriver extends PuterDriver { [DEFAULT_TEMP_SUBSCRIPTION]: 200, }, }, + methods: { + // `list` is a prefix scan, not a point read — it does not + // belong on the same budget as `get`/`set`. It is still a + // foreground call an app makes to render a view, though, so the + // window has to clear a session's worth of those; the in-flight + // cap below is what keeps the scans from piling up. + list: { + limit: 240, + window: 60_000, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 120, + [DEFAULT_TEMP_SUBSCRIPTION]: 60, + }, + }, + }, + }; + + // The rate window above is well-tuned; what was missing is an in-flight + // bound. This is the driver most likely to be called from a tight loop + // inside a worker, where the caller never waits for a response. + readonly concurrent: DriverConcurrentConfig = { + default: { + limit: 30, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 15, + [DEFAULT_TEMP_SUBSCRIPTION]: 8, + }, + }, + methods: { + list: { + limit: 5, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 3, + [DEFAULT_TEMP_SUBSCRIPTION]: 2, + }, + }, + }, }; override getReportedCosts(): Record[] { diff --git a/src/backend/drivers/notification/NotificationDriver.ts b/src/backend/drivers/notification/NotificationDriver.ts index 676031c66..80d6ae9f0 100644 --- a/src/backend/drivers/notification/NotificationDriver.ts +++ b/src/backend/drivers/notification/NotificationDriver.ts @@ -25,7 +25,7 @@ import { } from '../../services/metering/consts.js'; import { PuterDriver } from '../types.js'; import type { Actor } from '../../core/actor.js'; -import type { DriverRateLimitConfig } from '../meta.js'; +import type { DriverConcurrentConfig, DriverRateLimitConfig } from '../meta.js'; const MAX_SELECT_LIMIT = 200; @@ -71,6 +71,16 @@ export class NotificationDriver extends PuterDriver { }, }; + readonly concurrent: DriverConcurrentConfig = { + default: { + limit: 20, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 10, + [DEFAULT_TEMP_SUBSCRIPTION]: 5, + }, + }, + }; + // -- Driver methods ---------------------------------------------- async create(args: Record): Promise { diff --git a/src/backend/drivers/subdomain/SubdomainDriver.test.ts b/src/backend/drivers/subdomain/SubdomainDriver.test.ts index 001f6b771..45ba572c7 100644 --- a/src/backend/drivers/subdomain/SubdomainDriver.test.ts +++ b/src/backend/drivers/subdomain/SubdomainDriver.test.ts @@ -17,7 +17,7 @@ * along with this program. If not, see . */ -import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { afterAll, beforeAll, describe, expect, it, vi } from 'vitest'; import { v4 as uuidv4 } from 'uuid'; import type { Actor } from '../../core/actor.js'; import { runWithContext } from '../../core/context.js'; @@ -179,6 +179,60 @@ describe('SubdomainDriver.create', () => { ).rejects.toMatchObject({ statusCode: 409 }); }); + it('reports a lost uniqueness race as 409, not a 500', async () => { + const { actor } = await makeUser(); + + // The uniqueness check and the insert are two statements, so a name can + // be claimed in between and only the index catches it. The in-memory + // sqlite schema has no unique index on `subdomain` (mysql and postgres + // do), so the losing insert is what gets stubbed here. + const dup = Object.assign(new Error('Duplicate entry'), { + code: 'ER_DUP_ENTRY', + errno: 1062, + }); + const create = vi + .spyOn(server.stores.subdomain, 'create') + .mockRejectedValueOnce(dup); + + try { + await expect( + withActor(actor, () => + driver.create({ + object: { + subdomain: uniqueSubdomain('race'), + root_dir: `/${actor.user!.username}/Public`, + }, + }), + ), + ).rejects.toMatchObject({ statusCode: 409 }); + } finally { + create.mockRestore(); + } + }); + + it('lets a non-uniqueness insert failure surface as a server error', async () => { + const { actor } = await makeUser(); + + const create = vi + .spyOn(server.stores.subdomain, 'create') + .mockRejectedValueOnce(new Error('connection lost')); + + try { + await expect( + withActor(actor, () => + driver.create({ + object: { + subdomain: uniqueSubdomain('boom'), + root_dir: `/${actor.user!.username}/Public`, + }, + }), + ), + ).rejects.toThrow('connection lost'); + } finally { + create.mockRestore(); + } + }); + it('rejects when root_dir does not exist', async () => { const { actor } = await makeUser(); await expect( diff --git a/src/backend/drivers/subdomain/SubdomainDriver.ts b/src/backend/drivers/subdomain/SubdomainDriver.ts index 6a4ea4c53..8ec5bc547 100644 --- a/src/backend/drivers/subdomain/SubdomainDriver.ts +++ b/src/backend/drivers/subdomain/SubdomainDriver.ts @@ -27,10 +27,11 @@ import { } from '../../services/metering/consts.js'; import { PuterDriver } from '../types.js'; import type { Actor } from '../../core/actor.js'; -import type { DriverRateLimitConfig } from '../meta.js'; +import type { DriverConcurrentConfig, DriverRateLimitConfig } from '../meta.js'; import type { FSEntry } from '../../stores/fs/FSEntry.js'; import type { UserRow } from '../../stores/user/UserStore.js'; import { expandTildePath } from '../../services/fs/resolveNode.js'; +import { isUniqueViolation } from '../../util/dbError.js'; import { buildHostedSubdomainIndexUrlCandidates } from '../../util/hostedAppBacking.js'; import { WORKER_SUBDOMAIN_PREFIX } from '../../stores/subdomain/SubdomainStore.js'; import { @@ -95,6 +96,32 @@ export class SubdomainDriver extends PuterDriver { [DEFAULT_TEMP_SUBSCRIPTION]: 100, }, }, + methods: { + // Unlike the reads this shares an envelope with, `create` + // consumes a name out of a global namespace nobody gets back. + // A known abuse target, so it keeps its own tighter budget — + // but publishing a site is also something the platform does on + // the user's behalf (an app gets one, a worker gets one), so the + // floor still has to clear a handful of those back to back. + create: { + limit: 120, + window: 60_000, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 60, + [DEFAULT_TEMP_SUBSCRIPTION]: 30, + }, + }, + }, + }; + + readonly concurrent: DriverConcurrentConfig = { + default: { + limit: 20, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 10, + [DEFAULT_TEMP_SUBSCRIPTION]: 5, + }, + }, }; // -- Driver methods ---------------------------------------------- @@ -178,13 +205,28 @@ export class SubdomainDriver extends PuterDriver { // `apps.owner_user_id = subdomain.user_id` + `index_url` match // (see `#hydrateRows`), so a subdomain row can never assert an // association with an app the caller doesn't own. - const created = await this.stores.subdomain.create({ - userId: actor.user.id, - subdomain, - rootDirId, - associatedAppId: null, - appOwner: actor.app?.id ?? null, - }); + // + // The uniqueness answer above is a check-then-insert, so two callers + // racing on the same name both pass it and the second one loses to the + // unique index. That is the same conflict, learned a moment later — + // report it the same way instead of letting the driver escape as a 500. + let created; + try { + created = await this.stores.subdomain.create({ + userId: actor.user.id, + subdomain, + rootDirId, + associatedAppId: null, + appOwner: actor.app?.id ?? null, + }); + } catch (err) { + if (!isUniqueViolation(err)) throw err; + throw new HttpError( + 409, + 'A site with this subdomain already exists', + { legacyCode: 'conflict' }, + ); + } const [shaped] = await this.#hydrateRows( created ? [created as Record] : [], ); diff --git a/src/backend/drivers/workers/WorkerDriver.ts b/src/backend/drivers/workers/WorkerDriver.ts index 03fea6d31..0f22d925f 100644 --- a/src/backend/drivers/workers/WorkerDriver.ts +++ b/src/backend/drivers/workers/WorkerDriver.ts @@ -30,6 +30,11 @@ import { WORKER_SUBDOMAIN_PREFIX, type SubdomainRow, } from '../../stores/subdomain/SubdomainStore.js'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '../../services/metering/consts.js'; +import type { DriverConcurrentConfig, DriverRateLimitConfig } from '../meta.js'; import { PuterDriver } from '../types.js'; import { loadFileInput } from '../util/fileInput.js'; import { @@ -131,6 +136,70 @@ export class WorkerDriver extends PuterDriver { readonly driverName = 'worker-service'; readonly isDefault = true; + // Without this the driver falls back to the generic 600/minute default, + // which is far too loose for `create` — every call reads the source out + // of the user's FS, bundles it, and provisions upstream. + // + // Deploys still get their own tighter budget, but not a single-digit one: + // developing against workers means redeploying on every change, and a + // tooling client that deploys a set of them does it back to back. The + // in-flight cap below is what bounds the concurrent bundling work; this + // window is only here to stop a loop. + readonly rateLimit: DriverRateLimitConfig = { + // Everything that isn't `create`/`destroy` is a metadata read — + // listing workers, resolving one by name, enumerating its files — and + // a client walks several of those per deploy and again per page of a + // listing. Cheap to serve, so the budget only catches a loop. + default: { + limit: 600, + window: 60_000, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 300, + [DEFAULT_TEMP_SUBSCRIPTION]: 150, + }, + }, + methods: { + create: { + limit: 120, + window: 60_000, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 80, + [DEFAULT_TEMP_SUBSCRIPTION]: 40, + }, + }, + destroy: { + limit: 30, + window: 60_000, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 20, + [DEFAULT_TEMP_SUBSCRIPTION]: 10, + }, + }, + }, + }; + + readonly concurrent: DriverConcurrentConfig = { + default: { + limit: 10, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 5, + [DEFAULT_TEMP_SUBSCRIPTION]: 3, + }, + }, + methods: { + // Deploys are the expensive path; the floor stays at 2 so a + // client that kicks off a second deploy while the first is + // still settling doesn't get a spurious rejection. + create: { + limit: 5, + bySubscription: { + [DEFAULT_FREE_SUBSCRIPTION]: 2, + [DEFAULT_TEMP_SUBSCRIPTION]: 2, + }, + }, + }, + }; + #cfBaseUrl = ''; #hotReloadSubscribed = false; diff --git a/src/backend/services/auth/AuthService.test.ts b/src/backend/services/auth/AuthService.test.ts index b5ba35fd9..267981536 100644 --- a/src/backend/services/auth/AuthService.test.ts +++ b/src/backend/services/auth/AuthService.test.ts @@ -1336,10 +1336,11 @@ describe('AuthService (integration)', () => { ).rejects.toMatchObject({ statusCode: 403 }); }); - it('still refuses an access-token actor (403)', async () => { - const user = await makeUser(); - const target = await makeApp('target', user.id); - const actor = { + const tokenActorFor = ( + user: { id: number; uuid: string; username: string }, + fullAccess: boolean, + ) => + ({ user: { id: user.id, uuid: user.uuid, @@ -1348,15 +1349,64 @@ describe('AuthService (integration)', () => { accessToken: { uid: uuidv4(), issuer: { user }, - fullAccess: true, + fullAccess, }, - } as unknown as Actor; + }) as unknown as Actor; + + // The credential AuthMe hands the MCP connector and the CLI. Its + // reach is the user's own, and `puter.workers.create` binds every + // deploy to a `sandbox-` app it creates under that user. + it('mints a token for a full-access token actor on its own app', async () => { + const user = await makeUser(); + const target = await makeApp('target', user.id); + + const token = await authService.createWorkerAppToken( + tokenActorFor(user, true), + target.uid, + 'wk-token', + ); + + const decoded = decodeAuth(token); + expect(decoded.app_uid).toBe(target.uid); + expect(decoded.user_uid).toBe(user.uuid); + expect(decoded.worker).toBe(true); + }); + + it('refuses a full-access token actor on another user’s app (403)', async () => { + const user = await makeUser(); + const stranger = await makeUser(); + const strangersApp = await makeApp('strangers', stranger.id); await expect( authService.createWorkerAppToken( - actor, + tokenActorFor(user, true), + strangersApp.uid, + 'wk-stranger-token', + ), + ).rejects.toMatchObject({ statusCode: 403 }); + }); + + it('refuses a full-access token actor naming an unknown app (403)', async () => { + const user = await makeUser(); + + await expect( + authService.createWorkerAppToken( + tokenActorFor(user, true), + `app-${uuidv4()}`, + 'wk-unknown-token', + ), + ).rejects.toMatchObject({ statusCode: 403 }); + }); + + it('still refuses a scoped access-token actor (403)', async () => { + const user = await makeUser(); + const target = await makeApp('target', user.id); + + await expect( + authService.createWorkerAppToken( + tokenActorFor(user, false), target.uid, - 'wk-token', + 'wk-scoped-token', ), ).rejects.toMatchObject({ statusCode: 403 }); }); diff --git a/src/backend/services/auth/AuthService.ts b/src/backend/services/auth/AuthService.ts index 35fdc34bc..b35528fb0 100644 --- a/src/backend/services/auth/AuthService.ts +++ b/src/backend/services/auth/AuthService.ts @@ -410,8 +410,18 @@ export class AuthService extends PuterService { * is what stamps `app_owner`, and an app that owns another app already has * full write access to it (`AppDriver.#checkWriteAccess`) including its * `index_url`. Everything else stays as strict as interactive delegation — - * access-token actors still can't delegate at all, and an app can never + * a scoped access token still can't delegate at all, and an app can never * name an app it didn't create. + * + * A full-access ("personal access token") actor is the third shape: it + * carries the issuing user's own API reach, which is exactly what + * `puter.workers.create` needs — its default sandbox binds the worker to a + * `sandbox-` app the same call just created under that user. Blanket- + * refusing it broke every worker deploy from a credential minted through + * AuthMe (the MCP connector, the CLI). It stays narrower than a root + * session: the app must exist and be owned by the same user, and the + * resulting token carries an `app`, so account-management gates + * (`requireUserActor`) still reject it. */ async #assertWorkerAppDelegationAllowed( actor: Actor, @@ -426,8 +436,16 @@ export class AuthService extends PuterService { // Root user session: unchanged: may bind a worker to any app. if (!actor.app && !actor.accessToken) return; - // Access tokens are bound to their issuing identity; no delegation. - if (!actor.app) throw forbidden(); + if (!actor.app) { + // Scoped access tokens are bound to their issuing identity and + // never delegate; full-access ones may name an app of their user's. + if (!actor.accessToken?.fullAccess) throw forbidden(); + const ownApp = await this.stores.app.getByUid(appUid); + if (!ownApp) throw forbidden(); + if (Number(ownApp.owner_user_id) !== Number(actor.user.id)) + throw forbidden(); + return; + } const app = await this.stores.app.getByUid(appUid); if (!app) throw forbidden(); @@ -594,8 +612,9 @@ export class AuthService extends PuterService { async revokeSession(uuid: string): Promise { // Read first — after the cascade these rows carry `revoked_at` and // no longer count as active. - const tokenUids = - await this.stores.session.accessTokenUidsForCascade(uuid); + const tokenUids = (await this.stores.session.accessTokenUidsForCascade( + uuid, + )) as string[]; await this.stores.session.revokeCascade(uuid); for (const tokenUid of tokenUids) { await this.#dropAccessTokenGrants(tokenUid); diff --git a/src/backend/services/health/ServerHealthService.test.ts b/src/backend/services/health/ServerHealthService.test.ts index be7204df1..cb2b225ef 100644 --- a/src/backend/services/health/ServerHealthService.test.ts +++ b/src/backend/services/health/ServerHealthService.test.ts @@ -24,19 +24,36 @@ import { ServerHealthService } from './ServerHealthService.js'; const STATUS_CACHE_KEY = 'server-health:status'; const CHECK_INTERVAL_MS = 5000; +const DEPENDENCY_INTERVAL_MS = 30_000; + interface Harness { service: ServerHealthService; dbRead: ReturnType; + dbPread: ReturnType; hasIO: ReturnType; + ping: ReturnType; + dynamoGet: ReturnType; + headBucket: ReturnType; } const makeService = ( config: Record = {}, - opts: { db?: boolean; socket?: boolean } = {}, + opts: { db?: boolean; socket?: boolean; deps?: boolean } = {}, ): Harness => { const dbRead = vi.fn().mockResolvedValue([{ ok: 1 }]); + const dbPread = vi.fn().mockResolvedValue([{ ok: 1 }]); const hasIO = vi.fn().mockReturnValue(true); - const clients = opts.db === false ? {} : { db: { read: dbRead } }; + const ping = vi.fn().mockResolvedValue('PONG'); + const dynamoGet = vi.fn().mockResolvedValue({ Item: undefined }); + const headBucket = vi.fn().mockResolvedValue(undefined); + + const clients: Record = + opts.db === false ? {} : { db: { read: dbRead, pread: dbPread } }; + if (opts.deps) { + clients.redis = { ping }; + clients.dynamo = { get: dynamoGet }; + clients.s3 = { headBucket }; + } const services = opts.socket === false ? {} : { socket: { hasIO } }; const args = [ config, @@ -44,7 +61,15 @@ const makeService = ( {}, services, ] as unknown as ConstructorParameters; - return { service: new ServerHealthService(...args), dbRead, hasIO }; + return { + service: new ServerHealthService(...args), + dbRead, + dbPread, + hasIO, + ping, + dynamoGet, + headBucket, + }; }; /** Run one full check cycle by advancing past the loop interval. */ @@ -227,6 +252,196 @@ describe('ServerHealthService — default checks', () => { }); }); +describe('ServerHealthService — dependency checks', () => { + it('probes every wired-up backing service', async () => { + const { service, ping, dynamoGet, headBucket } = makeService( + {}, + { deps: true }, + ); + service.onServerStart(); + await runCycle(); + + expect(ping).toHaveBeenCalledTimes(1); + expect(headBucket).toHaveBeenCalledTimes(1); + expect(dynamoGet).toHaveBeenCalledWith('store-kv-v1', { + namespace: 'server-health', + key: 'liveness-probe', + }); + expect(await service.getStatus()).toEqual({ ok: true }); + service.onServerShutdown(); + }); + + it('skips the probes for dependencies that are not wired up', async () => { + const { service } = makeService({}, { socket: false }); + service.onServerStart(); + await runCycle(); + + expect(Object.keys(service.getStats().check_durations_ms)).toEqual([ + 'database-liveness', + ]); + service.onServerShutdown(); + }); + + it('reads the primary directly, but only when a replica is in play', async () => { + const withoutReplica = makeService({}, {}); + withoutReplica.service.onServerStart(); + await runCycle(); + expect(withoutReplica.dbPread).not.toHaveBeenCalled(); + withoutReplica.service.onServerShutdown(); + + kv.del(STATUS_CACHE_KEY); + const withReplica = makeService({ + database: { engine: 'mysql', replica: { host: 'replica.local' } }, + }); + withReplica.service.onServerStart(); + await runCycle(); + expect(withReplica.dbPread).toHaveBeenCalledWith('SELECT 1 AS ok'); + expect(await withReplica.service.getStatus()).toEqual({ ok: true }); + withReplica.service.onServerShutdown(); + }); + + it('fails the primary check when the primary returns no rows', async () => { + const { service, dbPread } = makeService({ + database: { engine: 'mysql', replica: { host: 'replica.local' } }, + }); + dbPread.mockResolvedValue([]); + service.onServerStart(); + await runCycle(); + expect(await service.getStatus()).toEqual({ + ok: false, + failed: ['database-primary-liveness'], + }); + service.onServerShutdown(); + }); + + it('fails redis on a reply that is not PONG', async () => { + const { service, ping } = makeService({}, { deps: true }); + ping.mockResolvedValue('LOADING'); + service.onServerStart(); + await runCycle(); + expect(await service.getStatus()).toEqual({ + ok: false, + failed: ['redis-liveness'], + }); + service.onServerShutdown(); + }); + + it('fails a dependency that answers slower than its threshold', async () => { + const { service, headBucket } = makeService( + { server_health: { s3_liveness_latency_fail_ms: 10 } }, + { deps: true }, + ); + headBucket.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 50); + }); + service.onServerStart(); + await runCycle(); + expect(await service.getStatus()).toEqual({ + ok: false, + failed: ['s3-liveness'], + }); + service.onServerShutdown(); + }); + + it('fails a dependency whose probe rejects', async () => { + const { service, dynamoGet } = makeService({}, { deps: true }); + dynamoGet.mockRejectedValue(new Error('ResourceNotFoundException')); + service.onServerStart(); + await runCycle(); + expect(await service.getStatus()).toEqual({ + ok: false, + failed: ['dynamo-liveness'], + }); + service.onServerShutdown(); + }); + + it('runs the probes on their own slower cadence, holding the last result', async () => { + const { service, ping, dbRead } = makeService({}, { deps: true }); + service.onServerStart(); + + await runCycle(); + expect(ping).toHaveBeenCalledTimes(1); + expect(dbRead).toHaveBeenCalledTimes(1); + ping.mockRejectedValue(new Error('down')); + + // Several cycles inside the dependency interval: the cheap check keeps + // running, the probe does not, and its passing result stands. + await vi.advanceTimersByTimeAsync(CHECK_INTERVAL_MS * 4); + expect(ping).toHaveBeenCalledTimes(1); + expect(dbRead).toHaveBeenCalledTimes(5); + kv.del(STATUS_CACHE_KEY); + expect(await service.getStatus()).toEqual({ ok: true }); + + // Past the interval it runs again and the failure lands. + await vi.advanceTimersByTimeAsync(DEPENDENCY_INTERVAL_MS); + expect(ping).toHaveBeenCalledTimes(2); + kv.del(STATUS_CACHE_KEY); + expect(await service.getStatus()).toEqual({ + ok: false, + failed: ['redis-liveness'], + }); + + // And keeps standing on the cycles where it is skipped. + await vi.advanceTimersByTimeAsync(CHECK_INTERVAL_MS); + expect(ping).toHaveBeenCalledTimes(2); + kv.del(STATUS_CACHE_KEY); + expect(await service.getStatus()).toEqual({ + ok: false, + failed: ['redis-liveness'], + }); + + service.onServerShutdown(); + }); + + it('honours a configured dependency cadence', async () => { + const { service, ping } = makeService( + { server_health: { dependency_check_interval_ms: 60_000 } }, + { deps: true }, + ); + service.onServerStart(); + await runCycle(); + expect(ping).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(DEPENDENCY_INTERVAL_MS); + expect(ping).toHaveBeenCalledTimes(1); + await vi.advanceTimersByTimeAsync(DEPENDENCY_INTERVAL_MS); + expect(ping).toHaveBeenCalledTimes(2); + + service.onServerShutdown(); + }); + + it('drops checks named in disabled_checks', async () => { + const { service, ping, dynamoGet } = makeService( + { server_health: { disabled_checks: ['redis-liveness'] } }, + { deps: true }, + ); + service.onServerStart(); + await runCycle(); + expect(ping).not.toHaveBeenCalled(); + expect(dynamoGet).toHaveBeenCalledTimes(1); + expect( + service.getStats().check_durations_ms, + ).not.toHaveProperty('redis-liveness'); + service.onServerShutdown(); + }); + + it('runs checks concurrently so their timeouts do not stack', async () => { + const { service } = makeService({}, { db: false, socket: false }); + service.addCheck('hangs-a', () => new Promise(() => {})); + service.addCheck('hangs-b', () => new Promise(() => {})); + service.onServerStart(); + + // One 4s timeout window, not two. + await vi.advanceTimersByTimeAsync(CHECK_INTERVAL_MS + 1); + await vi.advanceTimersByTimeAsync(4001); + expect(await service.getStatus()).toEqual({ + ok: false, + failed: ['hangs-a', 'hangs-b'], + }); + service.onServerShutdown(); + }); +}); + describe('ServerHealthService.getStatus — filtering', () => { const failingService = async () => { const { service } = makeService({}, { db: false, socket: false }); @@ -276,6 +491,38 @@ describe('ServerHealthService.getStatus — filtering', () => { service.onServerShutdown(); }); + it('expands an @group token to every check in that group', async () => { + const { service, ping, dynamoGet } = makeService({}, { deps: true }); + ping.mockRejectedValue(new Error('down')); + dynamoGet.mockRejectedValue(new Error('down')); + service.addCheck('unrelated', () => { + throw new Error('u'); + }); + service.onServerStart(); + await runCycle(); + + expect(await service.getStatus({ degrade: ['@dependencies'] })).toEqual({ + ok: false, + failed: ['unrelated'], + degraded: ['redis-liveness', 'dynamo-liveness'], + }); + expect( + await service.getStatus({ + ignore: ['@dependencies', 'unrelated'], + }), + ).toEqual({ ok: true }); + service.onServerShutdown(); + }); + + it('treats an unknown @group as matching nothing', async () => { + const service = await failingService(); + expect(await service.getStatus({ degrade: ['@nope'] })).toEqual({ + ok: false, + failed: ['alpha', 'beta'], + }); + service.onServerShutdown(); + }); + it('caches the unfiltered status so filters never leak between callers', async () => { const service = await failingService(); expect(await service.getStatus({ ignore: ['alpha', 'beta'] })).toEqual({ diff --git a/src/backend/services/health/ServerHealthService.ts b/src/backend/services/health/ServerHealthService.ts index 107469ade..9c079b7c0 100644 --- a/src/backend/services/health/ServerHealthService.ts +++ b/src/backend/services/health/ServerHealthService.ts @@ -20,6 +20,7 @@ import { PuterService } from '../types'; import type { SocketService } from '../socket/SocketService'; import { kv } from '../../util/kvSingleton'; +import { PUTER_KV_STORE_TABLE_NAME } from '../../stores/systemKv/tableDefinition'; /** * Periodic liveness monitor for the backend. Other services register checks via @@ -29,11 +30,31 @@ import { kv } from '../../util/kvSingleton'; * * Default checks registered on server start: * - * - `database-liveness` — `SELECT 1 AS ok` latency-gated against + * - `database-liveness` — `SELECT 1 AS ok` through the normal read path (a + * read-replica where one is configured), latency-gated against * `config.server_health.db_liveness_latency_fail_ms` (default 1500ms). * - `socket-initialized` — socket.io must be attached. Only registered when * SocketService is present (skipped for API-only deployments). * + * Plus one probe per backing service this node can't serve traffic without, + * each in the `dependencies` group (see `addCheck`) and each registered only + * when that dependency is actually wired up: + * + * - `database-primary-liveness` — `SELECT 1 AS ok` pinned to the primary. Only + * registered when a read-replica exists, since without one it would just + * re-probe the connection `database-liveness` already covers. + * - `redis-liveness` — `PING`. + * - `dynamo-liveness` — point read of a key that is never written, so the probe + * exercises the data plane without depending on any stored state. + * - `s3-liveness` — `HEAD` on the default storage bucket. + * + * These four are deliberately cheap and run on their own slower cadence + * (`config.server_health.dependency_check_interval_ms`, default 30s) rather + * than the 5s loop: they cross the network to metered services, and detecting a + * dependency outage seconds sooner isn't worth a standing request stream from + * every node. Any of them can be turned off with + * `config.server_health.disabled_checks`. + * * Draining mode: `onServerPrepareShutdown` flips the service into drain and * clears failure state. `/healthcheck` returns 503 so load balancers route * traffic away before the process exits. @@ -44,9 +65,25 @@ const CHECK_INTERVAL_MS = 5 * SECOND; const CHECK_TIMEOUT_MS = 4 * SECOND; const HEALTH_LOOP_STALE_MULTIPLIER = 3; const DEFAULT_DB_LIVENESS_LATENCY_FAIL_MS = 1500; +const DEFAULT_DEPENDENCY_CHECK_INTERVAL_MS = 30 * SECOND; +const DEFAULT_REDIS_LATENCY_FAIL_MS = 1 * SECOND; +const DEFAULT_DYNAMO_LATENCY_FAIL_MS = 1500; +const DEFAULT_S3_LATENCY_FAIL_MS = 2 * SECOND; const STATUS_CACHE_TTL_SECONDS = 5; const STATUS_CACHE_KEY = 'server-health:status'; +/** Group name covering every backing-service probe. */ +const DEPENDENCY_GROUP = 'dependencies'; + +/** + * Key the dynamo probe reads. Nothing ever writes it — a point read that misses + * still proves the round-trip, and costs the same minimum as one that hits. + */ +const DYNAMO_PROBE_KEY = { + namespace: 'server-health', + key: 'liveness-probe', +}; + type CheckFn = () => Promise | unknown; type FailHandler = (err: unknown) => Promise | void; @@ -54,10 +91,30 @@ interface Chainable { onFail(handler: FailHandler): Chainable; } +export interface AddCheckOptions { + /** + * Minimum gap between runs. Defaults to 0 — every loop cycle. A check with + * a real cost (network hop, metered service) should set this; the loop + * skips it until it's due and keeps reporting its last result meanwhile. + */ + intervalMs?: number; + /** + * Group names this check also answers to, so `ignore`/`degrade` callers can + * name a whole class of checks as `@` instead of enumerating them. + */ + groups?: string[]; +} + interface RegisteredCheck { name: string; fn: CheckFn; onFailHandlers: FailHandler[]; + groups: string[]; + minIntervalMs: number; + lastRunAt: number; + lastDurationMs: number; + hasRun: boolean; + failing: boolean; } interface HealthStats { @@ -85,7 +142,6 @@ export interface GetStatusOptions { export class ServerHealthService extends PuterService { #checks: RegisteredCheck[] = []; - #failures: { name: string }[] = []; #healthStartedAt = Date.now(); #lastCycleCompletedAt = 0; #stats: HealthStats = { @@ -105,7 +161,7 @@ export class ServerHealthService extends PuterService { override onServerPrepareShutdown(): void { if (this.#draining) return; this.#draining = true; - this.#failures = []; + for (const check of this.#checks) check.failing = false; this.#lastCycleCompletedAt = Date.now(); this.#stats = { last_check_cycle_completed_at: this.#lastCycleCompletedAt, @@ -126,10 +182,26 @@ export class ServerHealthService extends PuterService { * Register a named health check. The returned chainable exposes * `onFail(fn)` so callers can hook self-heal logic (e.g., recreating a * pooled DB client after a liveness drop). + * + * A check named in `config.server_health.disabled_checks` is dropped here + * and never runs — the chainable still works, its handlers just never + * fire. */ - addCheck(name: string, fn: CheckFn): Chainable { - const registered: RegisteredCheck = { name, fn, onFailHandlers: [] }; - this.#checks.push(registered); + addCheck(name: string, fn: CheckFn, opts: AddCheckOptions = {}): Chainable { + const registered: RegisteredCheck = { + name, + fn, + onFailHandlers: [], + groups: opts.groups ?? [], + minIntervalMs: opts.intervalMs ?? 0, + lastRunAt: 0, + lastDurationMs: 0, + hasRun: false, + failing: false, + }; + const disabled = this.config.server_health?.disabled_checks ?? []; + if (!disabled.includes(name)) this.#checks.push(registered); + const chainable: Chainable = { onFail: (handler) => { registered.onFailHandlers.push(handler); @@ -151,9 +223,12 @@ export class ServerHealthService extends PuterService { * status collapses back to `{ ok: true }`. `degrade` instead demotes named * failures to a non-fatal `degraded` list — `ok` stays true but the caller * can see the partial state. Any failure name may be filtered this way, - * including the `draining` lifecycle state. The cached status is always the - * full, unfiltered set — filtering is applied per-request after the cache - * read so it never leaks across callers. + * including the `draining` lifecycle state. A name of the form `@` + * stands for every check registered in that group, so a caller can tolerate + * a whole class of checks — `@dependencies` for the backing-service probes + * — without having to be redeployed each time one is added. The cached + * status is always the full, unfiltered set — filtering is applied + * per-request after the cache read so it never leaks across callers. */ async getStatus(opts: GetStatusOptions = {}): Promise { const base = this.#draining @@ -189,11 +264,14 @@ export class ServerHealthService extends PuterService { ): HealthStatus { if (status.ok || !status.failed) return status; + const ignoredNames = this.#expandNames(ignore); + const degradedNames = this.#expandNames(degrade); + const remaining = status.failed.filter( - (name) => !ignore.includes(name), + (name) => !ignoredNames.has(name), ); - const degraded = remaining.filter((name) => degrade.includes(name)); - const failed = remaining.filter((name) => !degrade.includes(name)); + const degraded = remaining.filter((name) => degradedNames.has(name)); + const failed = remaining.filter((name) => !degradedNames.has(name)); const result: HealthStatus = { ok: failed.length === 0 }; if (failed.length > 0) result.failed = failed; @@ -201,6 +279,22 @@ export class ServerHealthService extends PuterService { return result; } + /** Resolve `@` tokens to the names of the checks in that group. */ + #expandNames(names: string[]): Set { + const resolved = new Set(); + for (const name of names) { + if (!name.startsWith('@')) { + resolved.add(name); + continue; + } + const group = name.slice(1); + for (const check of this.#checks) { + if (check.groups.includes(group)) resolved.add(check.name); + } + } + return resolved; + } + #registerDefaultChecks(): void { const latencyFailMs = Number(this.config.server_health?.db_liveness_latency_fail_ms) || @@ -237,6 +331,116 @@ export class ServerHealthService extends PuterService { } }); } + + this.#registerDependencyChecks(); + } + + /** + * Probes for the backing services a node needs to serve traffic. Each is + * the cheapest round-trip that still proves the data path works, runs on + * the slow dependency cadence, and is skipped when the dependency isn't + * wired up (self-hosted subsets, partially-stubbed tests). + */ + #registerDependencyChecks(): void { + const db = this.clients.db; + if ( + this.config.database?.replica && + db && + typeof db.pread === 'function' + ) { + // `read()` above goes to the replica when one exists, so a primary + // that is gone (or lagging behind a failover) looks healthy there. + this.#addDependencyCheck( + 'database-primary-liveness', + this.config.server_health?.db_liveness_latency_fail_ms, + DEFAULT_DB_LIVENESS_LATENCY_FAIL_MS, + async () => { + const rows = (await db.pread( + 'SELECT 1 AS ok', + )) as unknown[]; + if (!Array.isArray(rows) || rows.length === 0) { + throw new Error( + 'primary database liveness query returned no rows', + ); + } + }, + ); + } + + const redis = this.clients.redis; + if (redis && typeof redis.ping === 'function') { + this.#addDependencyCheck( + 'redis-liveness', + this.config.server_health?.redis_liveness_latency_fail_ms, + DEFAULT_REDIS_LATENCY_FAIL_MS, + async () => { + const reply = await redis.ping(); + if (String(reply).toUpperCase() !== 'PONG') { + throw new Error(`unexpected ping reply: ${reply}`); + } + }, + ); + } + + const dynamo = this.clients.dynamo; + if (dynamo && typeof dynamo.get === 'function') { + this.#addDependencyCheck( + 'dynamo-liveness', + this.config.server_health?.dynamo_liveness_latency_fail_ms, + DEFAULT_DYNAMO_LATENCY_FAIL_MS, + async () => { + await dynamo.get( + PUTER_KV_STORE_TABLE_NAME, + DYNAMO_PROBE_KEY, + ); + }, + ); + } + + const s3 = this.clients.s3; + if (s3 && typeof s3.headBucket === 'function') { + this.#addDependencyCheck( + 's3-liveness', + this.config.server_health?.s3_liveness_latency_fail_ms, + DEFAULT_S3_LATENCY_FAIL_MS, + async () => { + await s3.headBucket(); + }, + ); + } + } + + /** + * Wrap a dependency probe with a latency gate and register it on the slow + * cadence, in the group `ignore`/`degrade` callers address as + * `@dependencies`. + */ + #addDependencyCheck( + name: string, + configuredLatencyFailMs: number | undefined, + defaultLatencyFailMs: number, + probe: () => Promise, + ): void { + const latencyFailMs = + Number(configuredLatencyFailMs) || defaultLatencyFailMs; + const intervalMs = + Number(this.config.server_health?.dependency_check_interval_ms) || + DEFAULT_DEPENDENCY_CHECK_INTERVAL_MS; + + this.addCheck( + name, + async () => { + const startedAt = Date.now(); + await probe(); + const durationMs = Date.now() - startedAt; + if (durationMs > latencyFailMs) { + throw new Error( + `${name} latency ${durationMs}ms > threshold ${latencyFailMs}ms`, + ); + } + }, + { intervalMs, groups: [DEPENDENCY_GROUP] }, + ); } #startLoop(): void { @@ -261,66 +465,84 @@ export class ServerHealthService extends PuterService { return; } - const newFailures: { name: string }[] = []; - const durations: Record = {}; + // Concurrently, not one after another: checks are all I/O waits, and + // serially they'd stack their timeouts into a cycle long enough to trip + // the loop-staleness check. + const due = this.#checks.filter((check) => this.#isDue(check)); + await Promise.all(due.map((check) => this.#runCheck(check))); + const durations: Record = {}; for (const check of this.#checks) { - const startedAt = Date.now(); - let timeoutHandle: NodeJS.Timeout | null = null; - try { - await new Promise((resolve, reject) => { - timeoutHandle = setTimeout( - () => reject(new Error('Health check timed out')), - CHECK_TIMEOUT_MS, - ); - Promise.resolve(check.fn()).then(() => resolve(), reject); - }); - } catch (err) { - newFailures.push({ name: check.name }); - const alreadyFailing = this.#failures.some( - (f) => f.name === check.name, - ); - if (!alreadyFailing) { - // Intentionally do not page PagerDuty for health-check - // failures — external uptime monitors cover this and the - // internal threshold flaps under normal load. Failures - // are still logged below and still trigger self-heal - // onFail handlers. - for (const handler of check.onFailHandlers) { - try { - await handler(err); - } catch (hErr) { - console.error( - `[server-health] onFail handler for ${check.name} threw:`, - hErr, - ); - } - } - } - console.error( - `[server-health] check "${check.name}" failed:`, - err, - ); - } finally { - if (timeoutHandle) clearTimeout(timeoutHandle); - durations[check.name] = Date.now() - startedAt; - } + if (check.hasRun) durations[check.name] = check.lastDurationMs; } - this.#failures = newFailures; this.#lastCycleCompletedAt = Date.now(); this.#stats.last_check_cycle_completed_at = this.#lastCycleCompletedAt; this.#stats.check_durations_ms = durations; - this.#stats.failed_checks = newFailures.map((f) => f.name); + this.#stats.failed_checks = this.#collectCheckFailures(); + } + + /** Every cycle unless the check asked for a slower cadence. */ + #isDue(check: RegisteredCheck): boolean { + if (!check.hasRun || check.minIntervalMs === 0) return true; + return Date.now() - check.lastRunAt >= check.minIntervalMs; + } + + async #runCheck(check: RegisteredCheck): Promise { + const startedAt = Date.now(); + check.lastRunAt = startedAt; + check.hasRun = true; + + let timeoutHandle: NodeJS.Timeout | null = null; + try { + await new Promise((resolve, reject) => { + timeoutHandle = setTimeout( + () => reject(new Error('Health check timed out')), + CHECK_TIMEOUT_MS, + ); + Promise.resolve(check.fn()).then(() => resolve(), reject); + }); + check.failing = false; + } catch (err) { + const alreadyFailing = check.failing; + check.failing = true; + if (!alreadyFailing) { + // Intentionally do not page PagerDuty for health-check + // failures — external uptime monitors cover this and the + // internal threshold flaps under normal load. Failures + // are still logged below and still trigger self-heal + // onFail handlers. + for (const handler of check.onFailHandlers) { + try { + await handler(err); + } catch (hErr) { + console.error( + `[server-health] onFail handler for ${check.name} threw:`, + hErr, + ); + } + } + } + console.error(`[server-health] check "${check.name}" failed:`, err); + } finally { + if (timeoutHandle) clearTimeout(timeoutHandle); + check.lastDurationMs = Date.now() - startedAt; + } } #collectFailures(): string[] { - const names = this.#failures.map((f) => f.name); + const names = this.#collectCheckFailures(); const stale = this.#staleLoopFailure(); if (stale) names.push(stale); return names; } + #collectCheckFailures(): string[] { + return this.#checks + .filter((check) => check.failing) + .map((check) => check.name); + } + #staleLoopFailure(): string | null { const staleAfterMs = Number(this.config.server_health?.stale_health_loop_fail_ms) || diff --git a/src/backend/services/health/dependencyProbes.integration.test.ts b/src/backend/services/health/dependencyProbes.integration.test.ts new file mode 100644 index 000000000..9249500fa --- /dev/null +++ b/src/backend/services/health/dependencyProbes.integration.test.ts @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { DDBClient } from '../../clients/dynamodb/DDBClient'; +import { RedisClient } from '../../clients/redis/RedisClient'; +import { S3Client } from '../../clients/s3/S3Client'; +import { PUTER_KV_STORE_TABLE_DEFINITION } from '../../stores/systemKv/tableDefinition'; +import type { IConfig } from '../../types'; +import { ServerHealthService } from './ServerHealthService'; + +/** + * The dependency probes against real client implementations rather than mocks — + * a probe that passes a stubbed `get`/`ping`/`headBucket` proves nothing about + * whether the underlying protocol call is one the backing service accepts. + * Runs fully in-process: dynalite, fauxqs, and the redis mock. + */ + +const config = { + dynamo: { inMemory: true }, + redis: { useMock: true }, + s3: { localConfig: { inMemory: true } }, + s3_bucket: 'puter-local', +} as unknown as IConfig; + +let dynamo: DDBClient; +let redis: RedisClient; +let s3: S3Client; + +const makeService = (): ServerHealthService => { + const args = [ + config, + { dynamo, redis, s3 }, + {}, + {}, + ] as unknown as ConstructorParameters; + return new ServerHealthService(...args); +}; + +beforeAll(async () => { + dynamo = new DDBClient(config); + await dynamo.createTableIfNotExists(PUTER_KV_STORE_TABLE_DEFINITION, 'ttl'); + + redis = new RedisClient(config); + + s3 = new S3Client(config); + await s3.onServerStart(); +}, 60_000); + +afterAll(async () => { + await s3.onServerShutdown(); + await redis.onServerShutdown?.(); +}); + +describe('dependency probes against real clients', () => { + it('the health loop registers and passes every probe', async () => { + const service = makeService(); + service.onServerStart(); + + // Real timers, and the loop's first cycle only fires once the 5s + // interval elapses — so poll rather than sleeping a fixed span. + // + // The deadline is in wall-clock time but the loop it waits on is not: + // a worker sharing a busy machine can burn tens of seconds of + // wall-clock while its 5s interval gets almost no turns, and a budget + // sized for an idle machine then reports zero probes rather than slow + // ones. Generous enough to survive that; on an idle machine it still + // falls through in about one cycle. + const expected = ['dynamo-liveness', 'redis-liveness', 's3-liveness']; + const deadline = Date.now() + 45_000; + while (Date.now() < deadline) { + const ran = Object.keys(service.getStats().check_durations_ms); + if (expected.every((name) => ran.includes(name))) break; + await new Promise((resolve) => setTimeout(resolve, 250)); + } + + expect( + Object.keys(service.getStats().check_durations_ms).sort(), + ).toEqual(expected); + expect(service.getStats().failed_checks).toEqual([]); + expect(await service.getStatus()).toEqual({ ok: true }); + + service.onServerShutdown(); + }, 60_000); + + it('dynamo answers the liveness point read', async () => { + const response = await dynamo.get('store-kv-v1', { + namespace: 'server-health', + key: 'liveness-probe', + }); + expect(response.Item).toBeUndefined(); + expect(response.$metadata.httpStatusCode).toBe(200); + }); + + it('redis answers PING', async () => { + await expect(redis.ping()).resolves.toBe('PONG'); + }); + + it('the object store answers HEAD on the default bucket', async () => { + await expect(s3.headBucket()).resolves.toBeUndefined(); + }); + + it('the object store probe rejects for a bucket that is not there', async () => { + await expect(s3.headBucket('definitely-not-a-bucket')).rejects.toThrow(); + }); +}); diff --git a/src/backend/services/metering/MeteringService.test.ts b/src/backend/services/metering/MeteringService.test.ts index 33c20fb7b..c11289d0a 100644 --- a/src/backend/services/metering/MeteringService.test.ts +++ b/src/backend/services/metering/MeteringService.test.ts @@ -158,6 +158,110 @@ describe('MeteringService', () => { expect(policy.id).toBe('custom-default'); }); + // Rate and concurrency gates resolve the subscription on every gated + // request, and a resolver may reach a remote store to answer. Without + // the cache, adding a tiered limit to a hot route would add a round + // trip to that route. + it('resolves once per actor within the cache window', async () => { + const stub = vi.fn(async () => null); + target.registerSubscriptionResolver(stub); + + await target.getActorSubscription(actor); + await target.getActorSubscription(actor); + await target.getActorSubscription(actor); + + expect(stub).toHaveBeenCalledTimes(1); + }); + + it('caches per actor, not globally', async () => { + const other: Actor = { user: makeUser({ email: null }) }; + const stub = vi.fn(async () => null); + target.registerSubscriptionResolver(stub); + + expect((await target.getActorSubscription(actor)).id).toBe( + DEFAULT_FREE_SUBSCRIPTION, + ); + expect((await target.getActorSubscription(other)).id).toBe( + DEFAULT_TEMP_SUBSCRIPTION, + ); + expect(stub).toHaveBeenCalledTimes(2); + }); + + it('re-resolves after the entry is invalidated', async () => { + const stub = vi.fn(async () => null); + target.registerSubscriptionResolver(stub); + + await target.getActorSubscription(actor); + expect(stub).toHaveBeenCalledTimes(1); + + // What a purchase or cancellation calls, so a new plan applies + // to the very next request rather than at the end of the window. + target.invalidateActorSubscription(actor.user!.uuid as string); + + await target.getActorSubscription(actor); + expect(stub).toHaveBeenCalledTimes(2); + }); + + it('announces an invalidation so other nodes drop their copy too', async () => { + const seen = vi.fn(); + server.clients.event.on( + 'outer.pubsub.metering.subscription-changed', + seen, + ); + + target.invalidateActorSubscription('some-user-uuid'); + + // `outer.pubsub.*` is the channel that reaches sibling nodes and + // peer clusters — a local-only drop would leave every other node + // serving the old tier until its entry expired. + expect(seen).toHaveBeenCalledWith( + 'outer.pubsub.metering.subscription-changed', + { userUuid: 'some-user-uuid' }, + expect.anything(), + ); + server.clients.event.off( + 'outer.pubsub.metering.subscription-changed', + seen, + ); + }); + + it('drops its own copy when another node announces a change', async () => { + const stub = vi.fn(async () => null); + target.registerSubscriptionResolver(stub); + + await target.getActorSubscription(actor); + expect(stub).toHaveBeenCalledTimes(1); + + // What arrives on a node that did not handle the purchase. + server.clients.event.emit( + 'outer.pubsub.metering.subscription-changed', + { userUuid: actor.user!.uuid as string }, + {}, + ); + await vi.waitFor(async () => { + await target.getActorSubscription(actor); + expect(stub).toHaveBeenCalledTimes(2); + }); + }); + + it('re-resolves once the cache window has passed', async () => { + const stub = vi.fn(async () => null); + target.registerSubscriptionResolver(stub); + + await target.getActorSubscription(actor); + const cacheMs = ( + target.constructor as unknown as { + SUBSCRIPTION_CACHE_MS: number; + } + ).SUBSCRIPTION_CACHE_MS; + const now = Date.now(); + vi.spyOn(Date, 'now').mockReturnValue(now + cacheMs + 1); + await target.getActorSubscription(actor); + vi.mocked(Date.now).mockRestore(); + + expect(stub).toHaveBeenCalledTimes(2); + }); + it('rejects an actor with no user uuid', async () => { await expect( target.getActorSubscription({ @@ -1407,9 +1511,8 @@ describe('MeteringService', () => { ); expect(global.total).toBe(700); - const { usage } = await target.getActorCurrentMonthUsageDetails( - actor, - ); + const { usage } = + await target.getActorCurrentMonthUsageDetails(actor); expect(usage.total).toBe(750); }); diff --git a/src/backend/services/metering/MeteringService.ts b/src/backend/services/metering/MeteringService.ts index 546ad6464..b230e5fa1 100644 --- a/src/backend/services/metering/MeteringService.ts +++ b/src/backend/services/metering/MeteringService.ts @@ -39,7 +39,6 @@ import type { UsageInput, UsageRecord, } from './types'; -import { toMicroCents } from './utils'; import { SUB_POLICIES } from '../../data/subPolicies/index.js'; @@ -88,8 +87,6 @@ export class MeteringService extends PuterService { static GLOBAL_SHARD_COUNT = 10000; static APP_SHARD_COUNT = 10000; - static MAX_GLOBAL_USAGE_PER_MINUTE = toMicroCents(0.2); - /** * Share of the allowance past which an approximate running total is no * longer good enough to decide on. @@ -104,11 +101,33 @@ export class MeteringService extends PuterService { */ static MONTHLY_CHARGE_MEMO_LIMIT = 100_000; + /** + * How long a resolved subscription is reused before asking the resolvers + * again, and how many actors are remembered at once. Rate/concurrency gates + * resolve the subscription on every gated request, and a resolver may reach + * a remote store to answer — without this, adding a tiered limit to a hot + * route would add a round trip to that route. + * + * This is the backstop, not the mechanism: a change we know about is + * announced to every node by `invalidateActorSubscription` and applies at + * once. The window only bounds staleness for changes nobody told us about — + * a resolver reading state that moved underneath it, or a node that missed + * the announcement. + */ + static SUBSCRIPTION_CACHE_MS = 60_000; + static SUBSCRIPTION_CACHE_LIMIT = 50_000; + private rateCheckTimer: ReturnType | null = null; private extraPolicies: SubscriptionPolicy[] = []; private subscriptionResolvers: SubscriptionResolver[] = []; private defaultSubscriptionResolvers: SubscriptionResolver[] = []; + /** Uuid → resolved policy + expiry. See SUBSCRIPTION_CACHE_MS. */ + private subscriptionCache = new Map< + string, + { policy: SubscriptionPolicy; expiresAt: number } + >(); + /** Actors settled for `settledMonth`; see MONTHLY_CHARGE_MEMO_LIMIT. */ private settledMonth: string | null = null; private settledActors = new Set(); @@ -123,6 +142,15 @@ export class MeteringService extends PuterService { // -- Lifecycle ---------------------------------------------------- override onServerStart(): void { + // Applied, not re-announced: the sender already fanned this out, and + // echoing it would put every node's drop back on the wire. + this.clients.event.on( + 'outer.pubsub.metering.subscription-changed', + (_key, data) => { + if (data?.userUuid) this.#dropCachedSubscription(data.userUuid); + }, + ); + this.rateCheckTimer = setInterval( () => { this.checkRateOfChange().catch((e) => { @@ -747,15 +775,66 @@ export class MeteringService extends PuterService { return (await this.getRemainingUsage(actor)) >= amount; } + /** + * Drop the cached subscription for an actor. Call after anything that + * changes which policy they resolve to (a purchase landing, a cancellation, + * an admin edit) so the new plan applies immediately rather than at the end + * of the cache window. + * + * Announced as well as applied. Only one node handles the write that + * changed the plan, but every node has its own cache, so dropping locally + * fixes the tier for one node and leaves the rest serving the old one until + * their entries expire. The event goes out on the `outer.pubsub.*` channel, + * which reaches sibling nodes and peer clusters alike — a user who upgrades + * shouldn't get their old limits back by being routed elsewhere. + */ + invalidateActorSubscription(userUuid: string): void { + this.#dropCachedSubscription(userUuid); + this.clients.event.emit( + 'outer.pubsub.metering.subscription-changed', + { userUuid }, + {}, + ); + } + + /** Local-only drop. The announcement path is `invalidateActorSubscription`. */ + #dropCachedSubscription(userUuid: string): void { + this.subscriptionCache.delete(userUuid); + } + async getActorSubscription(actor: Actor): Promise { if (!actor.user?.uuid) throw new HttpError(403, 'Actor must be a user to get policy', { legacyCode: 'forbidden', }); + const uuid = actor.user.uuid; + const now = Date.now(); + const cached = this.subscriptionCache.get(uuid); + if (cached && cached.expiresAt > now) return cached.policy; + + const policy = await this.#resolveActorSubscription(actor); + + // Map preserves insertion order; FIFO-evict so a flood of one-shot + // actors can't grow this without bound. + if ( + this.subscriptionCache.size >= + MeteringService.SUBSCRIPTION_CACHE_LIMIT + ) { + const oldest = this.subscriptionCache.keys().next().value; + if (oldest !== undefined) this.subscriptionCache.delete(oldest); + } + this.subscriptionCache.set(uuid, { + policy, + expiresAt: now + MeteringService.SUBSCRIPTION_CACHE_MS, + }); + return policy; + } + + async #resolveActorSubscription(actor: Actor): Promise { const fallbackDefault = this.config.unlimitedMetering ? UNLIMITED_SUBSCRIPTION - : actor.user.email + : actor.user?.email ? DEFAULT_FREE_SUBSCRIPTION : DEFAULT_TEMP_SUBSCRIPTION; @@ -1192,19 +1271,20 @@ export class MeteringService extends PuterService { const globalUsage = await this.getGlobalUsage(); const currTotal = globalUsage.total; - if (lastChange) { + const maxPerMinute = this.config.maxGlobalUsagePerMinute; + + if (lastChange && maxPerMinute && maxPerMinute > 0) { const timeDelta = now - lastChange.timestamp; const usageDelta = currTotal - lastChange.total; const usagePerMinute = usageDelta / (timeDelta / 60000); - if (usagePerMinute > MeteringService.MAX_GLOBAL_USAGE_PER_MINUTE) { + if (usagePerMinute > maxPerMinute) { this.clients.alarm.create( 'metering:excessiveGlobalUsageRate', `Global usage rate is excessive: ${usagePerMinute} micro-cents per minute`, { usagePerMinute, - maxAllowedPerMinute: - MeteringService.MAX_GLOBAL_USAGE_PER_MINUTE, + maxAllowedPerMinute: maxPerMinute, }, // Fleet-wide spend running away — worth someone's attention // the same day, but it isn't an outage. diff --git a/src/backend/services/socket/SocketService.test.ts b/src/backend/services/socket/SocketService.test.ts index e7853b205..b82d34991 100644 --- a/src/backend/services/socket/SocketService.test.ts +++ b/src/backend/services/socket/SocketService.test.ts @@ -327,6 +327,103 @@ describe('SocketService (live socket.io)', () => { }), ).rejects.toThrow(); }); + + // -- Connection caps ---------------------------------------------- + // + // The handshake succeeds and the cap is applied after, so an over-cap + // client sees `connect` followed by a server-side `disconnect`. + + /** + * Whether the server dropped this socket shortly after connect. Settled by + * polling `connected` rather than by listening for `disconnect`: the + * rejection can land before a listener attached post-`connect()` is in + * place, and a missed event would read as "admitted". + */ + const wasDropped = async (socket: ClientSocket): Promise => { + await new Promise((r) => setTimeout(r, 500)); + return !socket.connected; + }; + + const withLimits = async ( + limits: { perOrigin: number; perUser: number }, + body: () => Promise, + ) => { + const prevOrigin = SocketService.MAX_SOCKETS_PER_ORIGIN; + const prevUser = SocketService.MAX_SOCKETS_PER_USER; + const prevTiers = SocketService.MAX_SOCKETS_BY_SUBSCRIPTION; + SocketService.MAX_SOCKETS_PER_ORIGIN = limits.perOrigin; + SocketService.MAX_SOCKETS_PER_USER = limits.perUser; + // Empty the tier map so the base above applies whatever tier the + // test user resolves to. + SocketService.MAX_SOCKETS_BY_SUBSCRIPTION = {}; + try { + await body(); + } finally { + SocketService.MAX_SOCKETS_PER_ORIGIN = prevOrigin; + SocketService.MAX_SOCKETS_PER_USER = prevUser; + SocketService.MAX_SOCKETS_BY_SUBSCRIPTION = prevTiers; + } + }; + + const connectFrom = (origin: string | undefined) => + connect( + { auth_token: `Bearer ${user.token}` }, + origin ? { extraHeaders: { Origin: origin } } : {}, + ); + + it('caps connections per origin', async () => { + await withLimits({ perOrigin: 1, perUser: 100 }, async () => { + const first = await connectFrom('https://one.example'); + expect(await wasDropped(first)).toBe(false); + + const second = await connectFrom('https://one.example'); + expect(await wasDropped(second)).toBe(true); + + first.disconnect(); + }); + }); + + it('lets a second origin through while the account has room', async () => { + await withLimits({ perOrigin: 1, perUser: 100 }, async () => { + const first = await connectFrom('https://a.example'); + const second = await connectFrom('https://b.example'); + + expect(await wasDropped(first)).toBe(false); + expect(await wasDropped(second)).toBe(false); + + first.disconnect(); + second.disconnect(); + }); + }); + + it('still bounds the account once origins are exhausted', async () => { + await withLimits({ perOrigin: 5, perUser: 1 }, async () => { + const first = await connectFrom('https://c.example'); + expect(await wasDropped(first)).toBe(false); + + // Fresh origin, so the per-origin bucket is empty — the account + // total is the only thing left to say no. + const second = await connectFrom('https://d.example'); + expect(await wasDropped(second)).toBe(true); + + first.disconnect(); + }); + }); + + it('gives a slot back when the connection closes', async () => { + await withLimits({ perOrigin: 1, perUser: 100 }, async () => { + const first = await connectFrom('https://e.example'); + expect(await wasDropped(first)).toBe(false); + first.disconnect(); + + await vi.waitFor(async () => { + const next = await connectFrom('https://e.example'); + const dropped = await wasDropped(next); + next.disconnect(); + expect(dropped).toBe(false); + }); + }); + }); }); // -- Event-bus fan-out ------------------------------------------------ diff --git a/src/backend/services/socket/SocketService.ts b/src/backend/services/socket/SocketService.ts index 1a687bb4e..8aa8b50c8 100644 --- a/src/backend/services/socket/SocketService.ts +++ b/src/backend/services/socket/SocketService.ts @@ -22,6 +22,15 @@ import type { Server as HttpServer } from 'node:http'; import { Server as SocketIOServer, type Socket } from 'socket.io'; import type { Actor } from '../../core/actor.js'; import { isAccessTokenActor, isAppActor } from '../../core/actor.js'; +import { + CONCURRENT_SLOT_TTL_MS, + acquireConcurrent, + checkRateLimit, +} from '../../core/http/middleware/rateLimit.js'; +import { + DEFAULT_FREE_SUBSCRIPTION, + DEFAULT_TEMP_SUBSCRIPTION, +} from '../metering/consts.js'; import type { AuthResult, AuthService } from '../auth/AuthService.js'; import { PuterService } from '../types.js'; @@ -352,32 +361,192 @@ export class SocketService extends PuterService { }); } + /** + * Client events don't pass through the HTTP middleware chain, so the route + * gates never see them. Both handlers below fan out to other sockets or + * onto the event bus, and a client can emit as fast as the connection + * allows — so each one gets its own window via the imperative helper. Per + * (user, event), matching how the route gates bucket by actor. + */ + static SOCKET_EVENT_LIMIT = 60; + static SOCKET_EVENT_WINDOW_MS = 60_000; + + /** + * Simultaneous connections per user, across every node. A connection costs + * an adapter room membership and a slot on whichever node terminates it, + * and nothing bounded how many a single account could hold open. + * + * Sized for an account, not a browser. One person is routinely several + * windows across several machines, a phone that reconnects on every + * foreground, and anything embedding the SDK against their session — and + * the cost of one connection is small enough that being generous here is + * cheaper than being wrong. This is the backstop against an account opening + * connections without bound; `MAX_SOCKETS_PER_ORIGIN` is what keeps any one + * page from spending the whole account allowance. + */ + static MAX_SOCKETS_PER_USER = 400; + static MAX_SOCKETS_BY_SUBSCRIPTION: Record = { + [DEFAULT_FREE_SUBSCRIPTION]: 200, + [DEFAULT_TEMP_SUBSCRIPTION]: 100, + }; + + /** + * Simultaneous connections per (user, origin). + * + * The natural split would be per app, but there isn't one to key on: + * `decideSocketAuth` accepts only plain user actors, so an app-token actor + * never reaches this code and every socket here belongs to a session. The + * requesting origin is the next-best proxy — it separates our own pages + * from a third-party site embedding the SDK against the same session, which + * is the split that matters. Without it a single looping page consumes the + * account's whole allowance and takes every other window offline with it. + * + * A browser sets `Origin` itself, so a page can't lie about its own; a + * non-browser client can put anything there, which is exactly why the + * per-user total above still applies and is the real bound. + */ + static MAX_SOCKETS_PER_ORIGIN = 150; + + async #socketLimitFor(actor: Actor): Promise { + const base = SocketService.MAX_SOCKETS_PER_USER; + try { + const sub = + await this.services.metering.getActorSubscription(actor); + return SocketService.MAX_SOCKETS_BY_SUBSCRIPTION[sub.id] ?? base; + } catch { + // Same policy as the route gates: a failure to resolve the tier + // falls through to the base rather than tightening. + return base; + } + } + + /** + * Bucket a handshake by requesting origin. Everything without one — a + * non-browser client, a same-origin request that omits the header — shares + * a single bucket rather than each getting a private allowance. + */ + static socketOriginKey(socket: AuthenticatedSocket): string { + const raw = socket.handshake?.headers?.origin; + const origin = Array.isArray(raw) ? raw[0] : raw; + return typeof origin === 'string' && origin.length > 0 + ? origin.slice(0, 128) + : 'none'; + } + + /** + * Take a per-origin and a per-account slot for one connection, and hold + * both until it closes. + * + * Connections routinely outlive `CONCURRENT_SLOT_TTL_MS` — a desktop left + * open all day is the normal case, not the exception — and a slot that old + * is indistinguishable from one a dead process abandoned. Renewing on a + * timer is what tells the two apart; without it the sweep reclaims live + * connections and the cap quietly stops counting exactly the long-lived + * ones it exists for. + */ + async #admitConnection( + socket: AuthenticatedSocket, + actor: Actor, + userId: number, + ): Promise { + const originKey = SocketService.socketOriginKey(socket); + const slots: { + release: () => Promise; + renew: () => Promise; + }[] = []; + + const reject = async () => { + await Promise.all(slots.map((s) => s.release())); + socket.disconnect(true); + }; + + const perOrigin = await acquireConcurrent( + `socket:conn:${userId}:${originKey}`, + SocketService.MAX_SOCKETS_PER_ORIGIN, + ); + if (!perOrigin.ok) return void (await reject()); + slots.push(perOrigin); + + const perUser = await acquireConcurrent( + `socket:conn:${userId}`, + await this.#socketLimitFor(actor), + ); + if (!perUser.ok) return void (await reject()); + slots.push(perUser); + + // A third of the window: two renewals may be missed (a paused timer, a + // slow backend) before a live slot looks abandoned. + const renewTimer = setInterval( + () => void Promise.all(slots.map((s) => s.renew())), + Math.floor(CONCURRENT_SLOT_TTL_MS / 3), + ); + renewTimer.unref?.(); + + const finish = () => { + clearInterval(renewTimer); + void Promise.all(slots.map((s) => s.release())); + }; + socket.once('disconnect', finish); + // The socket may already be gone by the time the tier lookup resolved; + // don't strand the slots until they age out. + if (socket.disconnected) finish(); + } + + async #allowSocketEvent(userId: number, event: string): Promise { + return checkRateLimit( + `socket:${event}:${userId}`, + SocketService.SOCKET_EVENT_LIMIT, + SocketService.SOCKET_EVENT_WINDOW_MS, + ); + } + #installConnectionHandler(): void { if (!this.#io) return; this.#io.on('connection', (socket: AuthenticatedSocket) => { const actor = socket.actor; - if (!actor || !actor.user) return; + // The id is what both limits below bucket on, so a user without + // one has nothing to key against. + if (!actor || actor.user?.id === undefined) return; const userId = actor.user.id; const userRoom = String(userId); + // Hold slots for the life of the connection. Released on + // `disconnect`, which socket.io fires for clean closes, transport + // errors, and server-side disconnects alike — so an abandoned + // connection gives its slots back the same way a closed one does. + void this.#admitConnection(socket, actor, userId); + // Peer-echo: one tab notifies others that trash is empty. socket.on('trash.is_empty', (msg: unknown) => { - socket.broadcast.to(userRoom).emit('trash.is_empty', msg); + void this.#allowSocketEvent(userId, 'trash.is_empty').then( + (ok) => { + if (!ok) return; + socket.broadcast + .to(userRoom) + .emit('trash.is_empty', msg); + }, + ); }); // Legacy probe some frontends use to signal "the UI is // really up, not just a health-check connection". Extensions // sometimes listen for the follow-up event. socket.on('puter_is_actually_open', () => { - this.clients.event.emit( - 'web.socket.user-connected', - { - socket, - user: actor.user, - }, - {}, - ); + void this.#allowSocketEvent( + userId, + 'puter_is_actually_open', + ).then((ok) => { + if (!ok) return; + this.clients.event.emit( + 'web.socket.user-connected', + { + socket, + user: actor.user, + }, + {}, + ); + }); }); // Fire-and-forget connect event. diff --git a/src/backend/stores/systemKv/SystemKVStore.ts b/src/backend/stores/systemKv/SystemKVStore.ts index 580add67f..ae8a3254a 100644 --- a/src/backend/stores/systemKv/SystemKVStore.ts +++ b/src/backend/stores/systemKv/SystemKVStore.ts @@ -24,7 +24,10 @@ import { SYSTEM_ACTOR, SYSTEM_ACTOR_UUID, } from '../../core/actor'; -import { PUTER_KV_STORE_TABLE_DEFINITION } from './tableDefinition'; +import { + PUTER_KV_STORE_TABLE_DEFINITION, + PUTER_KV_STORE_TABLE_NAME, +} from './tableDefinition'; import { HttpError } from '../../core/http'; import { decodeCursor, @@ -285,7 +288,7 @@ const cleanAttrName = (chunk: string): string => * If `opts.actor` is omitted, operations are scoped to the system namespace. */ export class SystemKVStore extends PuterStore { - private tableName = 'store-kv-v1'; + private tableName = PUTER_KV_STORE_TABLE_NAME; private initialized: Promise | null = null; override async onServerStart(): Promise { diff --git a/src/backend/stores/systemKv/tableDefinition.ts b/src/backend/stores/systemKv/tableDefinition.ts index ac0121067..588197a95 100644 --- a/src/backend/stores/systemKv/tableDefinition.ts +++ b/src/backend/stores/systemKv/tableDefinition.ts @@ -19,8 +19,10 @@ import type { CreateTableCommandInput } from '@aws-sdk/client-dynamodb'; +export const PUTER_KV_STORE_TABLE_NAME = 'store-kv-v1'; + export const PUTER_KV_STORE_TABLE_DEFINITION: CreateTableCommandInput = { - TableName: 'store-kv-v1', + TableName: PUTER_KV_STORE_TABLE_NAME, BillingMode: 'PAY_PER_REQUEST', AttributeDefinitions: [ { AttributeName: 'namespace', AttributeType: 'S' }, diff --git a/src/backend/types.ts b/src/backend/types.ts index 825a61a1f..24b8193e6 100644 --- a/src/backend/types.ts +++ b/src/backend/types.ts @@ -383,6 +383,24 @@ export interface IServerHealthConfig { db_liveness_latency_fail_ms?: number; /** Staleness threshold for the health-check loop itself (ms). */ stale_health_loop_fail_ms?: number; + /** + * Cadence for the external-dependency probes (redis, dynamo, object store, + * primary database). Deliberately slower than the 5s check loop so probing + * a paid, rate-limited backing service stays a rounding error against real + * traffic. Default 30000. + */ + dependency_check_interval_ms?: number; + /** Redis liveness latency threshold (ms). Default 1000. */ + redis_liveness_latency_fail_ms?: number; + /** Dynamo liveness latency threshold (ms). Default 1500. */ + dynamo_liveness_latency_fail_ms?: number; + /** Object-store liveness latency threshold (ms). Default 2000. */ + s3_liveness_latency_fail_ms?: number; + /** + * Check names to skip registering entirely — an operator kill switch for a + * probe that turns out to be noisy, without waiting on a deploy. + */ + disabled_checks?: string[]; } export interface IS3LocalConfig { @@ -881,6 +899,14 @@ interface IConfigOptional { //Metering unlimitedMetering?: boolean; + /** + * Fleet-wide spend rate, in micro-cents per minute, past which the metering + * service raises an alarm. There is no defensible default here — the right + * number is a multiple of what this deployment's own traffic normally + * costs, so set it from observed rate and revisit it as traffic grows. An + * unset or non-positive value turns the check off rather than guessing. + */ + maxGlobalUsagePerMinute?: number; } /** diff --git a/src/backend/util/identifier.js b/src/backend/util/identifier.js index c6308645d..d39ac5205 100644 --- a/src/backend/util/identifier.js +++ b/src/backend/util/identifier.js @@ -232,27 +232,35 @@ const nouns = [ const randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)]; +// Size of the numeric suffix's range. The word lists alone only reach ~11k +// combinations, so the number carries most of the entropy: every caller feeds +// these into a UNIQUE column (usernames, subdomains), where the collision rate +// scales with rows-already-taken over total combinations. Six digits keeps the +// name readable while putting the space comfortably ahead of that growth. +const SUFFIX_RANGE = 1_000_000; + /** - * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999). - * The result is returned as a string with components separated by the specified separator. - * It is useful when you need to create unique identifiers that are also human-friendly. - * - * @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided. - * @returns {string} A unique, human-friendly identifier. + * A function that generates a unique identifier by combining a random + * adjective, a random noun, and a random number. The result is returned as a + * string with components separated by the specified separator. It is useful + * when you need to create unique identifiers that are also human-friendly. * * @example + * let identifier = window.generate_identifier(); + * // identifier would be something like 'clever-idea-483920' * - * let identifier = window.generate_identifier(); - * // identifier would be something like 'clever-idea-123' - * + * @param {string} [separator='_'] - The character used to separate the + * adjective, noun, and number. Defaults to '_' if not provided. Default is + * `'_'` + * @returns {string} A unique, human-friendly identifier. */ function generate_identifier(separator = '_', rng = Math.random) { - // return a random combination of first_adj + noun + number (between 0 and 9999) - // e.g. clever-idea-123 + // return a random combination of first_adj + noun + number + // e.g. clever-idea-483920 return [ randomItem(adjectives, rng), randomItem(nouns, rng), - Math.floor(rng() * 10000), + Math.floor(rng() * SUFFIX_RANGE), ].join(separator); } diff --git a/src/backend/util/identifier.test.js b/src/backend/util/identifier.test.js index 03ccb1e4c..c6e9c8447 100644 --- a/src/backend/util/identifier.test.js +++ b/src/backend/util/identifier.test.js @@ -23,14 +23,14 @@ import identifier from './identifier.js'; const { generate_identifier, generate_random_code } = identifier; describe('generate_identifier', () => { - it('joins adjective, noun and a 0-9999 number with the default separator', () => { + it('joins adjective, noun and a 0-999999 number with the default separator', () => { const value = generate_identifier(); const parts = value.split('_'); expect(parts).toHaveLength(3); expect(parts[0]).toMatch(/^[a-z]+$/); expect(parts[1]).toMatch(/^[a-z]+$/); expect(Number(parts[2])).toBeGreaterThanOrEqual(0); - expect(Number(parts[2])).toBeLessThan(10000); + expect(Number(parts[2])).toBeLessThan(1000000); }); it('honours a custom separator', () => { @@ -47,9 +47,9 @@ describe('generate_identifier', () => { }); it('reaches the last entry of each list at the top of the range', () => { - const almostOne = () => 0.999999; + const almostOne = () => 0.9999999; const value = generate_identifier('|', almostOne); - expect(value.split('|')[2]).toBe('9999'); + expect(value.split('|')[2]).toBe('999999'); }); }); diff --git a/src/gui/src/helpers.js b/src/gui/src/helpers.js index 0c5fb6b20..c3e18226e 100644 --- a/src/gui/src/helpers.js +++ b/src/gui/src/helpers.js @@ -335,7 +335,7 @@ window.validate_fsentry_name = function (name) { }; /** - * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999). + * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number. * The result is returned as a string with components separated by hyphens. * It is useful when you need to create unique identifiers that are also human-friendly. * @@ -344,7 +344,7 @@ window.validate_fsentry_name = function (name) { * @example * * let identifier = window.generate_identifier(); - * // identifier would be something like 'clever-idea-123' + * // identifier would be something like 'clever-idea-483920' * */ window.generate_identifier = function () { @@ -361,9 +361,12 @@ window.generate_identifier = function () { 'ladybug', 'snail', 'camel', 'kangaroo', 'koala', 'panda', 'piglet', 'sheep', 'wolf', 'fox', 'deer', 'mouse', 'seal', 'chicken', 'cow', 'dinosaur', 'puppy', 'kitten', 'circle', 'square', 'garden', 'otter', 'bunny', 'meerkat', 'harp']; - // return a random combination of first_adj + noun + number (between 0 and 9999) - // e.g. clever-idea-123 - return `${first_adj[Math.floor(Math.random() * first_adj.length)] }-${ nouns[Math.floor(Math.random() * nouns.length)] }-${ Math.floor(Math.random() * 10000)}`; + // return a random combination of first_adj + noun + number + // e.g. clever-idea-483920. The word lists only reach ~3.6k pairs, so the + // number carries the entropy: this name is offered as a subdomain, and + // subdomains are globally unique, so a narrow suffix makes the taken-name + // error a routine part of publishing rather than a rarity. + return `${first_adj[Math.floor(Math.random() * first_adj.length)] }-${ nouns[Math.floor(Math.random() * nouns.length)] }-${ Math.floor(Math.random() * 1000000)}`; }; /** diff --git a/src/puter-js/src/lib/networkUtils.js b/src/puter-js/src/lib/networkUtils.js index 4edc74cb1..57345d6ac 100644 --- a/src/puter-js/src/lib/networkUtils.js +++ b/src/puter-js/src/lib/networkUtils.js @@ -38,9 +38,9 @@ const createDeferred = () => { * * @param {Object} resp - The parsed response body. * @param {string} [sentToken] - The token the failed request carried. - * @returns {{action: 'reject', error: Object}} + * @returns {{ action: 'reject'; error: Object }} */ -function resolveBackgroundReauth (resp, sentToken) { +function resolveBackgroundReauth(resp, sentToken) { puter.dropStaleAuthToken({ reason: resp.reason, auth_id: resp.auth_id, @@ -65,33 +65,36 @@ function resolveBackgroundReauth (resp, sentToken) { * utils.js) apply the exact same policy. * * Recognised backend signals: - * - `reauth_required` (`authProbe`): retired v1 tokens, revoked sessions, - * and expired sessions beyond the silent re-mint window. - * - `token_auth_failed` (legacy `APIError.create('token_auth_failed')`): - * token no longer valid, prompt re-login (web env only). + * + * - `reauth_required` (`authProbe`): retired v1 tokens, revoked sessions, and + * expired sessions beyond the silent re-mint window. + * - `token_auth_failed` (legacy `APIError.create('token_auth_failed')`): token no + * longer valid, prompt re-login (web env only). * * @param {Object} resp - The parsed response body. * @param {Object} [opts] * @param {boolean} [opts.interactive=true] - Whether this request may raise * sign-in UI. False for requests the user didn't ask for (see - * `resolveBackgroundReauth`). + * `resolveBackgroundReauth`). Default is `true` * @param {string} [opts.sentToken] - The token the failed request carried, so a * background reauth only discards a token that is still the current one. - * @returns {Promise<{action: 'replay'}|{action: 'reject', error: Object}|null>} + * @returns {Promise< + * { action: 'replay' } | { action: 'reject'; error: Object } | null + * >} * `replay` when the caller should re-issue the request once with the fresh * token, `reject` with the error to surface, or `null` when this is not a * reauth-recoverable 401 and the caller should handle it normally. */ -async function resolveReauth (resp, { interactive = true, sentToken } = {}) { - if ( resp?.code === 'reauth_required' ) { - if ( ! interactive ) return resolveBackgroundReauth(resp, sentToken); +async function resolveReauth(resp, { interactive = true, sentToken } = {}) { + if (resp?.code === 'reauth_required') { + if (!interactive) return resolveBackgroundReauth(resp, sentToken); try { await puter.triggerReauth({ reason: resp.reason, auth_id: resp.auth_id, }); return { action: 'replay' }; - } catch ( e ) { + } catch (e) { return { action: 'reject', error: { @@ -104,8 +107,8 @@ async function resolveReauth (resp, { interactive = true, sentToken } = {}) { }; } } - if ( resp?.code === 'token_auth_failed' && puter.env === 'web' ) { - if ( ! interactive ) return resolveBackgroundReauth(resp, sentToken); + if (resp?.code === 'token_auth_failed' && puter.env === 'web') { + if (!interactive) return resolveBackgroundReauth(resp, sentToken); try { puter.resetAuthToken(); await puter.ui.authenticateWithPuter(); @@ -114,7 +117,8 @@ async function resolveReauth (resp, { interactive = true, sentToken } = {}) { action: 'reject', error: { error: { - code: 'auth_canceled', message: 'Authentication canceled', + code: 'auth_canceled', + message: 'Authentication canceled', }, }, }; @@ -128,22 +132,24 @@ async function resolveReauth (resp, { interactive = true, sentToken } = {}) { * request, applies headers/credentials/responseType, and stashes the whole * `spec` on `xhr._puterReq` as the single replay representation — any attempt * (reauth, permission, transient) rebuilds the request by calling - * `buildXhr(spec)` again, which re-reads the live token when `includePuterAuth`. + * `buildXhr(spec)` again, which re-reads the live token when + * `includePuterAuth`. * * @param {Object} spec * @param {string} spec.url - Full request URL. - * @param {string} [spec.method='GET'] + * @param {string} [spec.method='GET'] Default is `'GET'` * @param {Object} [spec.headers] - Extra headers (nullish values skipped). - * @param {boolean} [spec.includePuterAuth=false] - Add a fresh `Authorization: Bearer`. - * @param {string} [spec.authToken] - The token to send when there is no live one - * to read — during construction, before `globalThis.puter` is assigned — or, - * without `includePuterAuth`, a token that isn't the live one at all. - * @param {boolean} [spec.withCredentials=true] - * @param {string} [spec.responseType=''] + * @param {boolean} [spec.includePuterAuth=false] - Add a fresh `Authorization: + * Bearer`. Default is `false` + * @param {string} [spec.authToken] - The token to send when there is no live + * one to read — during construction, before `globalThis.puter` is assigned — + * or, without `includePuterAuth`, a token that isn't the live one at all. + * @param {boolean} [spec.withCredentials=true] Default is `true` + * @param {string} [spec.responseType=''] Default is `''` * @param {Object} [spec.logId] - Pre-built apiCallLogger request id. * @returns {XMLHttpRequest} */ -function buildXhr (spec) { +function buildXhr(spec) { const { url, method = 'GET', @@ -160,17 +166,17 @@ function buildXhr (spec) { xhr.responseType = responseType ?? ''; const bearer = includePuterAuth - ? ( globalThis.puter?.authToken ?? authToken ) + ? (globalThis.puter?.authToken ?? authToken) : authToken; - if ( bearer ) { + if (bearer) { // Recorded per attempt: a background 401 only discards the token it was // actually sent with, so a reauth that landed in the meantime keeps the // fresh one it installed. spec._sentAuthToken = bearer; xhr.setRequestHeader('Authorization', `Bearer ${bearer}`); } - for ( const [ name, value ] of Object.entries(headers) ) { - if ( value !== undefined && value !== null ) { + for (const [name, value] of Object.entries(headers)) { + if (value !== undefined && value !== null) { xhr.setRequestHeader(name, value); } } @@ -182,7 +188,7 @@ function buildXhr (spec) { return origSend(body); }; - if ( globalThis.puter?.apiCallLogger?.isEnabled() ) { + if (globalThis.puter?.apiCallLogger?.isEnabled()) { xhr._puterRequestId = spec.logId ?? { method, service: 'xhr', @@ -208,47 +214,54 @@ function buildXhr (spec) { */ /** - * @typedef {Object} PuterResponse - * A `fetch`-Response-like view over a completed (or streaming) XHR. - * @property {boolean} ok - status in the 200-299 range. + * @typedef {Object} PuterResponse A `fetch`-Response-like view over a + * completed (or streaming) XHR. + * @property {boolean} ok - Status in the 200-299 range. * @property {number} status * @property {string} statusText - * @property {string} url - final response URL. - * @property {{ get(name: string): (string|null) }} headers + * @property {string} url - Final response URL. + * @property {{ get(name: string): string | null }} headers * @property {() => Promise} json * @property {() => Promise} text * @property {() => Promise} blob * @property {() => Promise} arrayBuffer - * @property {() => AsyncGenerator} stream - parsed NDJSON lines; only + * @property {() => AsyncGenerator} stream - Parsed NDJSON lines; only * meaningful for `application/x-ndjson` responses. */ -const isNdjson = contentType => (contentType || '').includes('application/x-ndjson'); +const isNdjson = (contentType) => + (contentType || '').includes('application/x-ndjson'); /** Read the XHR body as text regardless of the responseType it was sent with. */ -async function bodyText (xhr) { - switch ( xhr.responseType ) { - case 'blob': return await xhr.response.text(); - case 'arraybuffer': return new TextDecoder().decode(xhr.response); - case 'json': return JSON.stringify(xhr.response); - default: return xhr.responseText; // '' | 'text' +async function bodyText(xhr) { + switch (xhr.responseType) { + case 'blob': + return await xhr.response.text(); + case 'arraybuffer': + return new TextDecoder().decode(xhr.response); + case 'json': + return JSON.stringify(xhr.response); + default: + return xhr.responseText; // '' | 'text' } } -async function bodyBlob (xhr) { - if ( xhr.responseType === 'blob' ) return xhr.response; - const type = xhr.getResponseHeader('content-type') || 'application/octet-stream'; - if ( xhr.responseType === 'arraybuffer' ) return new Blob([xhr.response], { type }); +async function bodyBlob(xhr) { + if (xhr.responseType === 'blob') return xhr.response; + const type = + xhr.getResponseHeader('content-type') || 'application/octet-stream'; + if (xhr.responseType === 'arraybuffer') + return new Blob([xhr.response], { type }); return new Blob([await bodyText(xhr)], { type }); } -async function bodyArrayBuffer (xhr) { - if ( xhr.responseType === 'arraybuffer' ) return xhr.response; +async function bodyArrayBuffer(xhr) { + if (xhr.responseType === 'arraybuffer') return xhr.response; return await (await bodyBlob(xhr)).arrayBuffer(); } -async function bodyJson (xhr) { - if ( xhr.responseType === 'json' ) return xhr.response; +async function bodyJson(xhr) { + if (xhr.responseType === 'json') return xhr.response; return JSON.parse(await bodyText(xhr)); } @@ -261,62 +274,77 @@ async function bodyJson (xhr) { * @param {XMLHttpRequest} xhr * @returns {Promise} */ -async function parseResponse (xhr) { - if ( xhr.responseType !== 'blob' ) { +async function parseResponse(xhr) { + if (xhr.responseType !== 'blob') { try { return JSON.parse(xhr.responseText); - } catch ( e ) { + } catch (e) { return xhr.responseText; } } const contentType = xhr.getResponseHeader('content-type'); - if ( contentType.startsWith('application/json') ) { + if (contentType.startsWith('application/json')) { const text = await xhr.response.text(); try { return JSON.parse(text); - } catch ( e ) { + } catch (e) { return text; } } - if ( contentType.startsWith('application/octet-stream') ) { + if (contentType.startsWith('application/octet-stream')) { return xhr.response; } return { success: true, result: xhr.response }; } -function makeResponse (xhr, stream) { +function makeResponse(xhr, stream) { const status = xhr.status; return { ok: status >= 200 && status < 300, status, statusText: xhr.statusText, url: xhr.responseURL || '', - headers: { get: name => xhr.getResponseHeader(name) }, + headers: { get: (name) => xhr.getResponseHeader(name) }, text: () => bodyText(xhr), json: () => bodyJson(xhr), blob: () => bodyBlob(xhr), arrayBuffer: () => bodyArrayBuffer(xhr), stream: () => { - if ( ! stream ) { - throw new Error('stream() is only available for application/x-ndjson responses'); + if (!stream) { + throw new Error( + 'stream() is only available for application/x-ndjson responses', + ); } return stream; }, }; } -function logRequest (logId, { result = null, error = null } = {}) { - if ( ! logId || ! globalThis.puter?.apiCallLogger?.isEnabled() ) return; +function logRequest(logId, { result = null, error = null } = {}) { + if (!logId || !globalThis.puter?.apiCallLogger?.isEnabled()) return; globalThis.puter.apiCallLogger.logRequest({ ...logId, result, error }); } -/** Best-effort body for logging — parsed JSON where sensible, else a placeholder. */ -async function bodyForLog (xhr) { +/** + * Best-effort body for logging — parsed JSON where sensible, else a + * placeholder. + */ +async function bodyForLog(xhr) { const contentType = xhr.getResponseHeader('content-type') || ''; - if ( xhr.responseType === '' || xhr.responseType === 'text' || contentType.includes('json') ) { - try { return await bodyJson(xhr); } catch ( e ) { - try { return await bodyText(xhr); } catch ( e2 ) { return null; } + if ( + xhr.responseType === '' || + xhr.responseType === 'text' || + contentType.includes('json') + ) { + try { + return await bodyJson(xhr); + } catch (e) { + try { + return await bodyText(xhr); + } catch (e2) { + return null; + } } } return `[${contentType || 'binary'}]`; @@ -328,13 +356,22 @@ async function bodyForLog (xhr) { // hand the result to the caller's shaper. A replay just rebuilds from the same // spec, so there are no hand-listed argument lists to get wrong. -const RETRYABLE_STATUS = new Set([ 429, 502, 503, 504 ]); +// Transient statuses that may or may not have run the handler. A 502/503/504 +// can mean the request was half-applied upstream, so only a read replays. +const RETRYABLE_STATUS = new Set([502, 503, 504]); + +// A rate/concurrency gate rejects in middleware, before the handler that would +// have done the work — so nothing happened, and replaying is safe for a write +// exactly as it is for a read. Treating this like the statuses above meant a +// multi-file upload or a burst of driver writes surfaced the gate as a hard +// failure to the app, when waiting a moment is the whole remedy. +const GATE_REJECT_STATUS = 429; // Fixed retry backoff: a quick ramp to a 2s ceiling, then hold at 2s. Index i is // the wait (ms) after attempt i+1 fails. The array length caps the retries — 8 // delays ⇒ 9 attempts total, the 2s ceiling used 5 times — after which the // request is failed. -const RETRY_DELAYS_MS = [ 250, 500, 1000, 2000, 2000, 2000, 2000, 2000 ]; +const RETRY_DELAYS_MS = [250, 500, 1000, 2000, 2000, 2000, 2000, 2000]; const RETRY_CEILING_MS = 2000; // If a ceiling-length wait overruns real time by more than this, the clock // jumped (e.g. the laptop slept mid-wait); the request is stale, so give up @@ -344,47 +381,66 @@ const MAX_SLEEP_DRIFT_MS = 2000; // Kill-switch seam: puter.configure() (deferred) will drive this. Default on. const autoRetryEnabled = () => globalThis.puter?.config?.autoRetry ?? true; -const sleep = (ms, signal) => new Promise((resolve, reject) => { - if ( signal?.aborted ) return reject(signal.reason ?? new DOMException('Aborted', 'AbortError')); - const t = setTimeout(resolve, ms); - signal?.addEventListener('abort', () => { - clearTimeout(t); - reject(signal.reason ?? new DOMException('Aborted', 'AbortError')); - }, { once: true }); -}); +const sleep = (ms, signal) => + new Promise((resolve, reject) => { + if (signal?.aborted) + return reject( + signal.reason ?? new DOMException('Aborted', 'AbortError'), + ); + const t = setTimeout(resolve, ms); + signal?.addEventListener( + 'abort', + () => { + clearTimeout(t); + reject( + signal.reason ?? new DOMException('Aborted', 'AbortError'), + ); + }, + { once: true }, + ); + }); -const retryDelay = attempt => RETRY_DELAYS_MS[attempt - 1]; +const retryDelay = (attempt) => RETRY_DELAYS_MS[attempt - 1]; -const transientRetry = ctx => { - if ( ! (ctx.retrySafe && autoRetryEnabled()) ) return null; +const transientRetry = (ctx) => { + if (!(ctx.retrySafe && autoRetryEnabled())) return null; + const delayMs = retryDelay(ctx.attempt); + return delayMs === undefined ? null : { delayMs }; +}; + +// Same backoff schedule as `transientRetry`, but not gated on read-safety — +// see GATE_REJECT_STATUS. An explicit `retry: false` still wins: that is the +// caller saying "never replay this one", and it means it. +const gateRetry = (ctx) => { + if (!(ctx.retryGated && autoRetryEnabled())) return null; const delayMs = retryDelay(ctx.attempt); return delayMs === undefined ? null : { delayMs }; }; /** * Drive the env-specific permission prompt for a denied driver call. - * @returns {Promise<{granted: boolean}>} + * + * @returns {Promise<{ granted: boolean }>} */ -async function resolvePermission (permission) { +async function resolvePermission(permission) { try { // requestPermission resolves to a boolean; the legacy `{granted}` // object shape is also tolerated for safety. const perm = await puter.ui.requestPermission({ permission }); return { granted: perm === true || perm?.granted === true }; - } catch ( e ) { + } catch (e) { return { granted: false }; } } /** - * Send one attempt. Resolves with a terminal outcome: - * { streamed: true, xhr, lineStream } — NDJSON, resolved at HEADERS_RECEIVED - * { xhr, status } — buffered response (any HTTP status) - * { networkError: true, xhr } — transport error - * Rejects only on abort. Per-line semantics (usage/email prompts, `toString`) - * belong to the caller's `shapeStream`. + * Send one attempt. Resolves with a terminal outcome: { streamed: true, xhr, + * lineStream } — NDJSON, resolved at HEADERS_RECEIVED { xhr, status } — + * buffered response (any HTTP status) { networkError: true, xhr } — transport + * error Rejects only on abort. Per-line semantics (usage/email prompts, + * `toString`) belong to the caller's `shapeStream`. */ -function sendOnce (spec) { +function sendOnce(spec) { return new Promise((resolve, reject) => { const xhr = buildXhr(spec); @@ -396,13 +452,13 @@ function sendOnce (spec) { let consumed = 0; const lineStream = (async function* () { - while ( true ) { - while ( lines.length > 0 ) { + while (true) { + while (lines.length > 0) { const line = lines.shift(); - if ( line.trim() === '' ) continue; + if (line.trim() === '') continue; yield JSON.parse(line); } - if ( responseComplete ) break; + if (responseComplete) break; const sig = createDeferred(); signalStreamUpdate = sig.resolve; await sig.promise; @@ -410,25 +466,31 @@ function sendOnce (spec) { })(); xhr.onreadystatechange = () => { - if ( xhr.readyState === 2 && isNdjson(xhr.getResponseHeader('Content-Type')) ) { + if ( + xhr.readyState === 2 && + isNdjson(xhr.getResponseHeader('Content-Type')) + ) { streamed = true; resolve({ streamed: true, xhr, lineStream }); } - if ( xhr.readyState === 4 && streamed ) { - if ( carry.length > 0 ) { lines.push(carry); carry = ''; } + if (xhr.readyState === 4 && streamed) { + if (carry.length > 0) { + lines.push(carry); + carry = ''; + } responseComplete = true; signalStreamUpdate?.(); } }; xhr.onprogress = () => { - if ( ! streamed ) return; + if (!streamed) return; const fresh = xhr.responseText.slice(consumed); consumed = xhr.responseText.length; - if ( ! fresh ) return; + if (!fresh) return; carry += fresh; let nl; - while ( (nl = carry.indexOf('\n')) !== -1 ) { + while ((nl = carry.indexOf('\n')) !== -1) { lines.push(carry.slice(0, nl)); carry = carry.slice(nl + 1); } @@ -436,18 +498,32 @@ function sendOnce (spec) { }; xhr.addEventListener('load', () => { - if ( streamed ) return; + if (streamed) return; resolve({ xhr, status: xhr.status }); }); - xhr.addEventListener('error', () => resolve({ networkError: true, xhr })); - xhr.addEventListener('abort', () => reject(spec.signal?.reason ?? new DOMException('Aborted', 'AbortError'))); + xhr.addEventListener('error', () => + resolve({ networkError: true, xhr }), + ); + xhr.addEventListener('abort', () => + reject( + spec.signal?.reason ?? + new DOMException('Aborted', 'AbortError'), + ), + ); - if ( spec.signal ) { - if ( spec.signal.aborted ) return reject(spec.signal.reason ?? new DOMException('Aborted', 'AbortError')); - spec.signal.addEventListener('abort', () => xhr.abort(), { once: true }); + if (spec.signal) { + if (spec.signal.aborted) + return reject( + spec.signal.reason ?? + new DOMException('Aborted', 'AbortError'), + ); + spec.signal.addEventListener('abort', () => xhr.abort(), { + once: true, + }); } - const body = typeof spec.buildBody === 'function' ? spec.buildBody() : spec.body; + const body = + typeof spec.buildBody === 'function' ? spec.buildBody() : spec.body; xhr.send(body ?? null); }); } @@ -455,48 +531,62 @@ function sendOnce (spec) { /** * Classify a completed attempt into a retry decision. Reauth and permission are * one-shot (tracked in `ctx.done`) and apply to any request; transient backoff - * applies only to `ctx.retrySafe` requests and honors the autoRetry kill switch. - * Memoizes the parsed body on `outcome.parsed` and stashes any reauth error on - * `outcome.reauthError` for the shaper. + * applies only to `ctx.retrySafe` requests and honors the autoRetry kill + * switch. Memoizes the parsed body on `outcome.parsed` and stashes any reauth + * error on `outcome.reauthError` for the shaper. * - * @returns {Promise<{delayMs:number}|null>} a delay to retry after, or null to stop. + * @returns {Promise<{ delayMs: number } | null>} A delay to retry after, or + * null to stop. */ -async function classifyRetry (outcome, ctx) { - if ( outcome.streamed ) return null; // committed stream — never retried +async function classifyRetry(outcome, ctx) { + if (outcome.streamed) return null; // committed stream — never retried - if ( outcome.networkError ) return transientRetry(ctx); + if (outcome.networkError) return transientRetry(ctx); const { xhr, status } = outcome; - if ( outcome.parsed === undefined ) { + if (outcome.parsed === undefined) { outcome.parsed = await bodyJson(xhr).catch(() => null); } const parsed = outcome.parsed; // reauth (401 / token_auth_failed) — one-shot, any method, no backoff. - if ( status === 401 || parsed?.code === 'token_auth_failed' ) { - if ( ! ctx.done.has('reauth') ) { + if (status === 401 || parsed?.code === 'token_auth_failed') { + if (!ctx.done.has('reauth')) { const spec = xhr?._puterReq; const reauth = await resolveReauth(parsed, { interactive: spec?.interactiveReauth !== false, sentToken: spec?._sentAuthToken, }); - if ( reauth?.action === 'replay' ) { ctx.done.add('reauth'); return { delayMs: 0 }; } - if ( reauth?.action === 'reject' ) outcome.reauthError = reauth.error; + if (reauth?.action === 'replay') { + ctx.done.add('reauth'); + return { delayMs: 0 }; + } + if (reauth?.action === 'reject') outcome.reauthError = reauth.error; } return null; } // permission denied (200 success:false) — one-shot, any method, no backoff. - if ( ctx.permission && parsed?.success === false && parsed?.error?.code === 'permission_denied' ) { - if ( ! ctx.done.has('permission') ) { + if ( + ctx.permission && + parsed?.success === false && + parsed?.error?.code === 'permission_denied' + ) { + if (!ctx.done.has('permission')) { const perm = await resolvePermission(ctx.permission); - if ( perm.granted ) { ctx.done.add('permission'); return { delayMs: 0 }; } + if (perm.granted) { + ctx.done.add('permission'); + return { delayMs: 0 }; + } } return null; } + // gate rejection — any method, honors kill switch, fixed schedule. + if (status === GATE_REJECT_STATUS) return gateRetry(ctx); + // transient status — read-safe only, honors kill switch, fixed schedule. - if ( RETRYABLE_STATUS.has(status) ) return transientRetry(ctx); + if (RETRYABLE_STATUS.has(status)) return transientRetry(ctx); return null; } @@ -506,25 +596,48 @@ async function classifyRetry (outcome, ctx) { * outcome, and retries on reauth / permission / transient causes; otherwise * hands the outcome to `shape`. * - * @param {Object} spec - buildXhr spec (+ optional buildBody, signal). + * @param {Object} spec - BuildXhr spec (+ optional buildBody, signal). * @param {Object} opts - * @param {boolean} [opts.retrySafe=false] - eligible for transient backoff retry. - * @param {string|null} [opts.permission] - `driver::` enables the permission cause. - * @param {(lineStream, xhr) => any} opts.shapeStream - wrap an NDJSON stream. - * @param {(outcome) => any} opts.shape - shape a buffered outcome (may throw). + * @param {boolean} [opts.retrySafe=false] - Eligible for transient backoff + * retry. Default is `false` + * @param {boolean} [opts.retryGated=true] - Eligible for 429 backoff retry, + * regardless of method (the gate rejects before the handler runs). Default is + * `true` + * @param {string | null} [opts.permission] - `driver::` enables + * the permission cause. + * @param {(lineStream, xhr) => any} opts.shapeStream - Wrap an NDJSON stream. + * @param {(outcome) => any} opts.shape - Shape a buffered outcome (may throw). */ -async function sendWithRetry (spec, { retrySafe = false, permission = null, shapeStream, shape }) { - const ctx = { attempt: 0, retrySafe, permission, done: new Set() }; - while ( true ) { +async function sendWithRetry( + spec, + { + retrySafe = false, + retryGated = true, + permission = null, + shapeStream, + shape, + }, +) { + const ctx = { + attempt: 0, + retrySafe, + retryGated, + permission, + done: new Set(), + }; + while (true) { ctx.attempt++; const outcome = await sendOnce(spec); - if ( outcome.streamed ) return shapeStream(outcome.lineStream, outcome.xhr); + if (outcome.streamed) + return shapeStream(outcome.lineStream, outcome.xhr); const decision = await classifyRetry(outcome, ctx); - if ( decision ) { + if (decision) { const before = Date.now(); await sleep(decision.delayMs, spec.signal); - if ( decision.delayMs >= RETRY_CEILING_MS - && (Date.now() - before) - decision.delayMs > MAX_SLEEP_DRIFT_MS ) { + if ( + decision.delayMs >= RETRY_CEILING_MS && + Date.now() - before - decision.delayMs > MAX_SLEEP_DRIFT_MS + ) { return shape(outcome); } continue; @@ -541,22 +654,24 @@ async function sendWithRetry (spec, { retrySafe = false, permission = null, shap const inflightRequests = new Map(); /** - * @param {string} key - fully-qualified request key (namespace it yourself, e.g. - * `${method}:${url}:${bodyKey}`). - * @param {() => Promise} factory - runs the request; called only on a miss. - * @param {{windowMs?: number}} [opts] - * @returns {Promise} shared promise (resolved value shared by reference). + * @param {string} key - Fully-qualified request key (namespace it yourself, + * e.g. `${method}:${url}:${bodyKey}`). + * @param {() => Promise} factory - Runs the request; called only on a + * miss. + * @param {{ windowMs?: number }} [opts] + * @returns {Promise} Shared promise (resolved value shared by reference). */ -function dedupe (key, factory, { windowMs = 2000 } = {}) { +function dedupe(key, factory, { windowMs = 2000 } = {}) { const existing = inflightRequests.get(key); - if ( existing ) { - if ( Date.now() - existing.timestamp < windowMs ) return existing.promise; + if (existing) { + if (Date.now() - existing.timestamp < windowMs) return existing.promise; inflightRequests.delete(key); // stale — fall through and re-issue } const promise = factory(); inflightRequests.set(key, { promise, timestamp: Date.now() }); const cleanup = () => { - if ( inflightRequests.get(key)?.promise === promise ) inflightRequests.delete(key); + if (inflightRequests.get(key)?.promise === promise) + inflightRequests.delete(key); }; promise.then(cleanup, cleanup); return promise; @@ -565,7 +680,7 @@ function dedupe (key, factory, { windowMs = 2000 } = {}) { /** * XHR-based `fetch()` replacement. Returns a `fetch`-Response-like object. * - * fetch semantics: the promise resolves for any HTTP status (`ok` reflects + * Fetch semantics: the promise resolves for any HTTP status (`ok` reflects * 2xx); it rejects only on network/abort errors. The one exception is a 401 * carrying a reauth signal — the reauth flow is driven first and, on success, * the request is replayed once with the fresh token (transparent recovery). A @@ -573,32 +688,36 @@ function dedupe (key, factory, { windowMs = 2000 } = {}) { * * @param {string} url - Full request URL (callers own origin composition). * @param {Object} [opts] - * @param {boolean} [opts.includePuterAuth=false] - Add `Authorization: Bearer `. - * @param {string} [opts.authToken] - The Bearer credential to send when the live - * `puter.authToken` can't be read yet (boot-time calls, before the global is - * assigned) or, without `includePuterAuth`, a token that isn't the live one at - * all (one being exchanged for another). - * @param {string} [opts.method='GET'] - * @param {Object} [opts.headers] - Extra request headers (undefined/null values skipped). - * @param {string|Blob|ArrayBuffer|FormData|null} [opts.body] - * @param {''|'text'|'json'|'blob'|'arraybuffer'} [opts.responseType=''] - * @param {boolean} [opts.withCredentials=true] + * @param {boolean} [opts.includePuterAuth=false] - Add `Authorization: Bearer + * `. Default is `false` + * @param {string} [opts.authToken] - The Bearer credential to send when the + * live `puter.authToken` can't be read yet (boot-time calls, before the + * global is assigned) or, without `includePuterAuth`, a token that isn't the + * live one at all (one being exchanged for another). + * @param {string} [opts.method='GET'] Default is `'GET'` + * @param {Object} [opts.headers] - Extra request headers (undefined/null values + * skipped). + * @param {string | Blob | ArrayBuffer | FormData | null} [opts.body] + * @param {'' | 'text' | 'json' | 'blob' | 'arraybuffer'} [opts.responseType=''] + * Default is `''` + * @param {boolean} [opts.withCredentials=true] Default is `true` * @param {AbortSignal} [opts.signal] - * @param {{service: string, operation: string, params?: Object}} [opts.logContext] + * @param {{ service: string; operation: string; params?: Object }} [opts.logContext] * Semantic context for the centralized API-call log. Omit to log generically. * @param {boolean} [opts.retry] - Force-enable (`true`) or disable (`false`) * transient-failure auto-retry for this request; omit for the default * (idempotent methods retry, others don't). Never retries a write. - * @param {boolean|string} [opts.dedupe] - Coalesce concurrent identical in-flight - * requests (reads only): `true` auto-keys by method+url+body, or pass a key. + * @param {boolean | string} [opts.dedupe] - Coalesce concurrent identical + * in-flight requests (reads only): `true` auto-keys by method+url+body, or + * pass a key. * @param {boolean} [opts.interactiveReauth=true] - Whether a reauth-recoverable * 401 may raise sign-in UI. Pass `false` for requests the user didn't ask for - * (boot telemetry, cache warmers): the stale token is dropped silently and the - * 401 surfaces to the caller instead. + * (boot telemetry, cache warmers): the stale token is dropped silently and + * the 401 surfaces to the caller instead. Default is `true` * @param {Object} [opts.paginate] - Reserved for a later sprint step (ignored). * @returns {Promise} */ -function fetchUrl (url, opts = {}) { +function fetchUrl(url, opts = {}) { const { includePuterAuth = false, authToken, @@ -614,42 +733,77 @@ function fetchUrl (url, opts = {}) { interactiveReauth = true, } = opts; - const logId = logContext ?? { service: 'fetchUrl', operation: `${method} ${url}`, params: { url, method } }; - const spec = { url, method, headers, includePuterAuth, authToken, withCredentials, responseType, body, signal, logId, interactiveReauth }; + const logId = logContext ?? { + service: 'fetchUrl', + operation: `${method} ${url}`, + params: { url, method }, + }; + const spec = { + url, + method, + headers, + includePuterAuth, + authToken, + withCredentials, + responseType, + body, + signal, + logId, + interactiveReauth, + }; // Read-safety: idempotent methods auto-retry; a POST read opts in with // `retry:true`; nothing retries when `retry:false` (writes/uploads). const idempotent = method === 'GET' || method === 'HEAD'; - const retrySafe = retry === false ? false : ( retry === true || idempotent ); + const retrySafe = retry === false ? false : retry === true || idempotent; + // A 429 is the exception: the request never ran, so a write replays too. + const retryGated = retry !== false; const loggingOn = () => globalThis.puter?.apiCallLogger?.isEnabled(); - const run = () => sendWithRetry(spec, { - retrySafe, - shapeStream: (lineStream, xhr) => { - if ( loggingOn() ) logRequest(logId, { result: '[stream]' }); - return makeResponse(xhr, lineStream); - }, - shape: async (outcome) => { - if ( outcome.networkError ) { - if ( loggingOn() ) logRequest(logId, { error: { message: 'Network error occurred' } }); - throw new TypeError(`Network request to ${url} failed`); - } - const { xhr } = outcome; - const resp = makeResponse(xhr); - if ( loggingOn() ) { - const logged = await bodyForLog(xhr); - logRequest(logId, xhr.status >= 400 - ? { error: logged ?? { message: xhr.statusText, status: xhr.status } } - : { result: logged }); - } - return resp; - }, - }); + const run = () => + sendWithRetry(spec, { + retrySafe, + retryGated, + shapeStream: (lineStream, xhr) => { + if (loggingOn()) logRequest(logId, { result: '[stream]' }); + return makeResponse(xhr, lineStream); + }, + shape: async (outcome) => { + if (outcome.networkError) { + if (loggingOn()) + logRequest(logId, { + error: { message: 'Network error occurred' }, + }); + throw new TypeError(`Network request to ${url} failed`); + } + const { xhr } = outcome; + const resp = makeResponse(xhr); + if (loggingOn()) { + const logged = await bodyForLog(xhr); + logRequest( + logId, + xhr.status >= 400 + ? { + error: logged ?? { + message: xhr.statusText, + status: xhr.status, + }, + } + : { result: logged }, + ); + } + return resp; + }, + }); - if ( dedupeOpt ) { - const bodyKey = body == null ? '' : ( typeof body === 'string' ? body : '[body]' ); - const key = typeof dedupeOpt === 'string' ? dedupeOpt : `${method}:${url}:${bodyKey}`; + if (dedupeOpt) { + const bodyKey = + body == null ? '' : typeof body === 'string' ? body : '[body]'; + const key = + typeof dedupeOpt === 'string' + ? dedupeOpt + : `${method}:${url}:${bodyKey}`; return dedupe(key, run); } return run(); @@ -666,33 +820,34 @@ const DRIVER_CONTENT_TYPE = 'text/plain;actually=json'; /** * @typedef {{ - * iface: string, - * method: string, - * args?: unknown, - * driver?: string, - * testMode?: boolean, - * puter?: unknown, + * iface: string; + * method: string; + * args?: unknown; + * driver?: string; + * testMode?: boolean; + * puter?: unknown; * }} DriverCall - * A driver method to invoke. `iface` is the interface name and `driver` the - * concrete implementation behind it, which the backend resolves to the - * interface's default when omitted. `puter` is the SDK instance the call runs - * against; it falls back to the global instance. + * A driver method to invoke. `iface` is the interface name and `driver` the + * concrete implementation behind it, which the backend resolves to the + * interface's default when omitted. `puter` is the SDK instance the call runs + * against; it falls back to the global instance. */ -const callInstance = call => call.puter ?? globalThis.puter; +const callInstance = (call) => call.puter ?? globalThis.puter; /** The wire body. Fields left `undefined` drop out of the JSON. */ -const callBody = (call, puter) => JSON.stringify({ - interface: call.iface, - driver: call.driver, - test_mode: call.testMode, - method: call.method, - args: call.args, - auth_token: puter.authToken, -}); +const callBody = (call, puter) => + JSON.stringify({ + interface: call.iface, + driver: call.driver, + test_mode: call.testMode, + method: call.method, + args: call.args, + auth_token: puter.authToken, + }); const logCall = (call, fields) => { - if ( ! globalThis.puter?.apiCallLogger?.isEnabled() ) return; + if (!globalThis.puter?.apiCallLogger?.isEnabled()) return; globalThis.puter.apiCallLogger.logRequest({ service: 'drivers', operation: `${call.iface}::${call.method}`, @@ -707,18 +862,21 @@ const logCall = (call, fields) => { }; /** Prompt for funding, or hand off to the app's upgrade flow. */ -async function promptUpgrade (puter, message) { - if ( puter.env === 'web' ) { +async function promptUpgrade(puter, message) { + if (puter.env === 'web') { showUsageLimitDialog(message); - } else if ( puter.env === 'app' ) { + } else if (puter.env === 'app') { await puter.ui.requestUpgrade(); } } -function promptEmailConfirmation (puter, error) { - if ( error?.code !== 'email_must_be_confirmed' || puter.env !== 'web' ) return; - showEmailConfirmationDialog(error.message - || 'Email confirmation required. Go to Puter.com to confirm your email address.'); +function promptEmailConfirmation(puter, error) { + if (error?.code !== 'email_must_be_confirmed' || puter.env !== 'web') + return; + showEmailConfirmationDialog( + error.message || + 'Email confirmation required. Go to Puter.com to confirm your email address.', + ); } /** @@ -726,14 +884,20 @@ function promptEmailConfirmation (puter, error) { * per-line usage/email prompts, `toString()` on text parts, and the `start` * adapter that lets the stream feed a `ReadableStream` controller. */ -function driverLineStream (lineStream, puter) { +function driverLineStream(lineStream, puter) { const stream = (async function* () { - for await ( const line of lineStream ) { - if ( line?.error?.code === 'insufficient_funds' || line?.metadata?.usage_limited === true ) { - await promptUpgrade(puter, 'You have reached your usage limit for this account.
Please upgrade to continue.'); + for await (const line of lineStream) { + if ( + line?.error?.code === 'insufficient_funds' || + line?.metadata?.usage_limited === true + ) { + await promptUpgrade( + puter, + 'You have reached your usage limit for this account.
Please upgrade to continue.', + ); } promptEmailConfirmation(puter, line?.error); - if ( typeof line.text === 'string' ) { + if (typeof line.text === 'string') { Object.defineProperty(line, 'toString', { enumerable: false, value: () => line.text, @@ -747,7 +911,7 @@ function driverLineStream (lineStream, puter) { enumerable: false, value: async (controller) => { const encoder = new TextEncoder(); - for await ( const part of stream ) { + for await (const part of stream) { controller.enqueue(encoder.encode(part)); } controller.close(); @@ -765,30 +929,35 @@ function driverLineStream (lineStream, puter) { * * @param {DriverCall} call * @param {{ - * responseType?: '' | 'text' | 'blob', - * readonly?: boolean, - * transform?: (result: unknown) => unknown, - * onError?: (error: unknown) => void, - * }} [opts] `readonly` marks the method retry-safe on transient failures, - * `transform` post-processes a successful result, and `onError` is the - * legacy error callback the module APIs accept alongside the promise. + * responseType?: '' | 'text' | 'blob'; + * readonly?: boolean; + * transform?: (result: unknown) => unknown; + * onError?: (error: unknown) => void; + * }} [opts] + * `readonly` marks the method retry-safe on transient failures (a + * rate/concurrency 429 replays either way — see GATE_REJECT_STATUS), + * `transform` post-processes a successful result, and `onError` is the legacy + * error callback the module APIs accept alongside the promise. * @returns {Promise} */ -async function driverCall (call, opts = {}) { +async function driverCall(call, opts = {}) { const { responseType = '', readonly = false, transform, onError } = opts; const puter = callInstance(call); const fail = (error) => { - if ( typeof onError === 'function' ) onError(error); + if (typeof onError === 'function') onError(error); throw error; }; // A signed-out visitor on a third-party page gets the sign-in flow first. - if ( ! puter.authToken && puter.env === 'web' ) { + if (!puter.authToken && puter.env === 'web') { try { await puter.ui.authenticateWithPuter(); - } catch ( e ) { - const canceled = { code: 'auth_canceled', message: 'Authentication canceled' }; + } catch (e) { + const canceled = { + code: 'auth_canceled', + message: 'Authentication canceled', + }; logCall(call, { error: canceled }); throw { error: canceled }; } @@ -807,11 +976,11 @@ async function driverCall (call, opts = {}) { return await sendWithRetry(spec, { retrySafe: readonly, permission: `driver:${call.iface}:${call.method}`, - shapeStream: lineStream => driverLineStream(lineStream, puter), + shapeStream: (lineStream) => driverLineStream(lineStream, puter), // Reauth, permission grants, and transient retries are already spent by // the time the engine hands the outcome over, so this is terminal. shape: async (outcome) => { - if ( outcome.networkError ) { + if (outcome.networkError) { logCall(call, { error: { message: 'Network error occurred' } }); return fail(outcome.xhr); } @@ -819,19 +988,29 @@ async function driverCall (call, opts = {}) { const { status } = outcome.xhr; const resp = await parseResponse(outcome.xhr); const failed = status >= 400 || resp?.success === false; - logCall(call, { result: failed ? null : resp, error: failed ? resp : null }); + logCall(call, { + result: failed ? null : resp, + error: failed ? resp : null, + }); - if ( status === 402 || resp?.error?.code === 'insufficient_funds' - || resp?.error?.status === 402 || resp?.metadata?.usage_limited === true ) { - await promptUpgrade(puter, 'Your account has not enough funding to complete this request.
Please upgrade to continue.'); + if ( + status === 402 || + resp?.error?.code === 'insufficient_funds' || + resp?.error?.status === 402 || + resp?.metadata?.usage_limited === true + ) { + await promptUpgrade( + puter, + 'Your account has not enough funding to complete this request.
Please upgrade to continue.', + ); } promptEmailConfirmation(puter, resp?.error); - if ( status === 401 || resp?.code === 'token_auth_failed' ) { + if (status === 401 || resp?.code === 'token_auth_failed') { return fail({ status: 401, message: 'Unauthorized' }); } - if ( status && status !== 200 ) return fail(resp); - if ( resp.success === false ) return fail(resp); + if (status && status !== 200) return fail(resp); + if (resp.success === false) return fail(resp); const result = resp.result !== undefined ? resp.result : resp; return transform ? await transform(result) : result; @@ -851,7 +1030,7 @@ async function driverCall (call, opts = {}) { * @param {DriverCall} call * @returns {Promise} */ -async function driverCallEnvelope (call) { +async function driverCallEnvelope(call) { const puter = callInstance(call); try { const resp = await fetchUrl(`${puter.APIOrigin}/drivers/call`, { @@ -861,22 +1040,34 @@ async function driverCallEnvelope (call) { }); // TODO: parser for Content-Type - const contentType = (resp.headers.get('content-type') ?? '').split(';')[0].trim(); + const contentType = (resp.headers.get('content-type') ?? '') + .split(';')[0] + .trim(); const result = await (() => { - switch ( contentType ) { - case 'application/x-ndjson': return resp.stream(); - case 'application/octet-stream': return resp.blob(); - // A response that declares no type at all is JSON, the API's default. - case 'application/json': case '': return resp.json(); - default: throw new Error(`unrecognized content type: ${contentType}`); + switch (contentType) { + case 'application/x-ndjson': + return resp.stream(); + case 'application/octet-stream': + return resp.blob(); + // A response that declares no type at all is JSON, the API's default. + case 'application/json': + case '': + return resp.json(); + default: + throw new Error( + `unrecognized content type: ${contentType}`, + ); } })(); logCall(call, { result }); return result; - } catch ( error ) { + } catch (error) { logCall(call, { - error: { message: error?.message ?? String(error), stack: error?.stack }, + error: { + message: error?.message ?? String(error), + stack: error?.stack, + }, }); throw error; } diff --git a/src/puter-js/src/lib/networkUtils.test.js b/src/puter-js/src/lib/networkUtils.test.js index 76fb1545b..31e7f01ca 100644 --- a/src/puter-js/src/lib/networkUtils.test.js +++ b/src/puter-js/src/lib/networkUtils.test.js @@ -1,13 +1,19 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { dedupe, driverCall, driverCallEnvelope, fetchUrl, sendWithRetry } from './networkUtils.js'; +import { + dedupe, + driverCall, + driverCallEnvelope, + fetchUrl, + sendWithRetry, +} from './networkUtils.js'; // -- Controllable fake XMLHttpRequest -- // Drives fetchUrl's event handlers deterministically. Each instance plays the // `program` set on the class, which the test uses to script the response. -function installFakeXHR (program) { +function installFakeXHR(program) { const instances = []; class FakeXHR extends EventTarget { - constructor () { + constructor() { super(); this.readyState = 0; this.status = 0; @@ -23,39 +29,67 @@ function installFakeXHR (program) { this.onprogress = null; instances.push(this); } - open (method, url) { this.method = method; this.url = url; } - setRequestHeader (name, value) { this._reqHeaders[name.toLowerCase()] = String(value); } - getResponseHeader (name) { return this._respHeaders[name.toLowerCase()] ?? null; } - abort () { this.dispatchEvent(new Event('abort')); } - send (body) { + open(method, url) { + this.method = method; + this.url = url; + } + setRequestHeader(name, value) { + this._reqHeaders[name.toLowerCase()] = String(value); + } + getResponseHeader(name) { + return this._respHeaders[name.toLowerCase()] ?? null; + } + abort() { + this.dispatchEvent(new Event('abort')); + } + send(body) { this.reqBody = body; queueMicrotask(() => program(this)); } // Test helpers the program uses to emit response phases. - _setHeaders (status, headers = {}) { + _setHeaders(status, headers = {}) { this.status = status; this.statusText = String(status); - for ( const [ k, v ] of Object.entries(headers) ) this._respHeaders[k.toLowerCase()] = v; + for (const [k, v] of Object.entries(headers)) + this._respHeaders[k.toLowerCase()] = v; + } + _headersReceived() { + this.readyState = 2; + this.onreadystatechange?.(); + } + _progress(chunk) { + this.responseText += chunk; + this.readyState = 3; + this.onprogress?.(); + } + _done() { + this.readyState = 4; + this.onreadystatechange?.(); + this.dispatchEvent(new Event('load')); + } + _networkError() { + this.dispatchEvent(new Event('error')); } - _headersReceived () { this.readyState = 2; this.onreadystatechange?.(); } - _progress (chunk) { this.responseText += chunk; this.readyState = 3; this.onprogress?.(); } - _done () { this.readyState = 4; this.onreadystatechange?.(); this.dispatchEvent(new Event('load')); } - _networkError () { this.dispatchEvent(new Event('error')); } } globalThis.XMLHttpRequest = FakeXHR; return instances; } // A simple buffered JSON/text response. -const respond = ({ status = 200, contentType = 'application/json', body = '' }) => xhr => { - xhr._setHeaders(status, { 'content-type': contentType }); - xhr._headersReceived(); - xhr.responseText = typeof body === 'string' ? body : JSON.stringify(body); - xhr._done(); -}; +const respond = + ({ status = 200, contentType = 'application/json', body = '' }) => + (xhr) => { + xhr._setHeaders(status, { 'content-type': contentType }); + xhr._headersReceived(); + xhr.responseText = + typeof body === 'string' ? body : JSON.stringify(body); + xhr._done(); + }; let savedXHR; -beforeEach(() => { savedXHR = globalThis.XMLHttpRequest; }); +beforeEach(() => { + savedXHR = globalThis.XMLHttpRequest; +}); afterEach(() => { globalThis.XMLHttpRequest = savedXHR; delete globalThis.puter; @@ -66,7 +100,9 @@ describe('fetchUrl', () => { it('adds a Bearer header from the live puter.authToken when includePuterAuth', async () => { globalThis.puter = { authToken: 'tok-123' }; const xhrs = installFakeXHR(respond({ body: { ok: true } })); - await fetchUrl('https://api.example/whoami', { includePuterAuth: true }); + await fetchUrl('https://api.example/whoami', { + includePuterAuth: true, + }); expect(xhrs[0]._reqHeaders['authorization']).toBe('Bearer tok-123'); }); @@ -74,7 +110,10 @@ describe('fetchUrl', () => { // The migration endpoint sends a token that is about to be replaced. globalThis.puter = { authToken: 'live' }; const xhrs = installFakeXHR(respond({ body: {} })); - await fetchUrl('https://api.example/migrate', { method: 'POST', authToken: 'explicit' }); + await fetchUrl('https://api.example/migrate', { + method: 'POST', + authToken: 'explicit', + }); expect(xhrs[0]._reqHeaders['authorization']).toBe('Bearer explicit'); }); @@ -85,7 +124,9 @@ describe('fetchUrl', () => { includePuterAuth: true, authToken: 'from-instance', }); - expect(xhrs[0]._reqHeaders['authorization']).toBe('Bearer from-instance'); + expect(xhrs[0]._reqHeaders['authorization']).toBe( + 'Bearer from-instance', + ); }); it('prefers the live token over the fallback when both are available', async () => { @@ -127,7 +168,7 @@ describe('fetchUrl', () => { }); it('resolves (not rejects) with ok:false on 404 and 500', async () => { - for ( const status of [ 404, 500 ] ) { + for (const status of [404, 500]) { installFakeXHR(respond({ status, body: { error: 'nope' } })); const resp = await fetchUrl('https://api.example/x'); expect(resp.ok).toBe(false); @@ -139,12 +180,19 @@ describe('fetchUrl', () => { it('rejects on a network error (write — no retry)', async () => { // A write never auto-retries, so the network error surfaces immediately. // Read retry-then-reject is covered in the transient-retry suite. - installFakeXHR(xhr => xhr._networkError()); - await expect(fetchUrl('https://api.example/x', { method: 'POST' })).rejects.toThrow(/failed/); + installFakeXHR((xhr) => xhr._networkError()); + await expect( + fetchUrl('https://api.example/x', { method: 'POST' }), + ).rejects.toThrow(/failed/); }); it('exposes text(), json(), and blob() accessors', async () => { - installFakeXHR(respond({ contentType: 'application/json', body: { hello: 'world' } })); + installFakeXHR( + respond({ + contentType: 'application/json', + body: { hello: 'world' }, + }), + ); const resp = await fetchUrl('https://api.example/x'); expect(await resp.text()).toBe('{"hello":"world"}'); expect(await resp.json()).toEqual({ hello: 'world' }); @@ -155,7 +203,7 @@ describe('fetchUrl', () => { }); it('streams parsed NDJSON objects across chunk boundaries', async () => { - installFakeXHR(xhr => { + installFakeXHR((xhr) => { xhr._setHeaders(200, { 'content-type': 'application/x-ndjson' }); xhr._headersReceived(); // A JSON object split across two progress deltas, plus a full line. @@ -165,28 +213,43 @@ describe('fetchUrl', () => { }); const resp = await fetchUrl('https://api.example/stream'); const got = []; - for await ( const obj of resp.stream() ) got.push(obj); - expect(got).toEqual([ { n: 1 }, { n: 2 }, { n: 3 } ]); + for await (const obj of resp.stream()) got.push(obj); + expect(got).toEqual([{ n: 1 }, { n: 2 }, { n: 3 }]); }); describe('401 reauth', () => { it('triggers reauth once and replays with the fresh token', async () => { - const triggerReauth = vi.fn(async () => { globalThis.puter.authToken = 'fresh'; }); - globalThis.puter = { authToken: 'stale', env: 'web', triggerReauth }; + const triggerReauth = vi.fn(async () => { + globalThis.puter.authToken = 'fresh'; + }); + globalThis.puter = { + authToken: 'stale', + env: 'web', + triggerReauth, + }; let call = 0; - installFakeXHR(xhr => { + installFakeXHR((xhr) => { call++; - if ( call === 1 ) { + if (call === 1) { // first attempt: 401 reauth_required - return respond({ status: 401, body: { code: 'reauth_required', reason: 'x', auth_id: 'a' } })(xhr); + return respond({ + status: 401, + body: { + code: 'reauth_required', + reason: 'x', + auth_id: 'a', + }, + })(xhr); } // replay carries the fresh token and succeeds expect(xhr._reqHeaders['authorization']).toBe('Bearer fresh'); return respond({ status: 200, body: { ok: true } })(xhr); }); - const resp = await fetchUrl('https://api.example/x', { includePuterAuth: true }); + const resp = await fetchUrl('https://api.example/x', { + includePuterAuth: true, + }); expect(triggerReauth).toHaveBeenCalledTimes(1); expect(resp.ok).toBe(true); expect(await resp.json()).toEqual({ ok: true }); @@ -194,10 +257,18 @@ describe('fetchUrl', () => { it('does not loop: a second 401 after replay surfaces as ok:false', async () => { const triggerReauth = vi.fn(async () => {}); - globalThis.puter = { authToken: 'stale', env: 'web', triggerReauth }; + globalThis.puter = { + authToken: 'stale', + env: 'web', + triggerReauth, + }; - installFakeXHR(respond({ status: 401, body: { code: 'reauth_required' } })); - const resp = await fetchUrl('https://api.example/x', { includePuterAuth: true }); + installFakeXHR( + respond({ status: 401, body: { code: 'reauth_required' } }), + ); + const resp = await fetchUrl('https://api.example/x', { + includePuterAuth: true, + }); // reauth attempted exactly once; replayed request's 401 is returned. expect(triggerReauth).toHaveBeenCalledTimes(1); expect(resp.ok).toBe(false); @@ -207,8 +278,12 @@ describe('fetchUrl', () => { it('plain 401 (no reauth code) resolves ok:false without triggering reauth', async () => { const triggerReauth = vi.fn(); globalThis.puter = { authToken: 't', env: 'web', triggerReauth }; - installFakeXHR(respond({ status: 401, body: { message: 'Unauthorized' } })); - const resp = await fetchUrl('https://api.example/x', { includePuterAuth: true }); + installFakeXHR( + respond({ status: 401, body: { message: 'Unauthorized' } }), + ); + const resp = await fetchUrl('https://api.example/x', { + includePuterAuth: true, + }); expect(triggerReauth).not.toHaveBeenCalled(); expect(resp.ok).toBe(false); }); @@ -228,10 +303,16 @@ describe('fetchUrl', () => { it('drops the stale token without raising UI on reauth_required', async () => { globalThis.puter = makePuter(); - installFakeXHR(respond({ - status: 401, - body: { code: 'reauth_required', reason: 'token_v1', auth_id: 'a' }, - })); + installFakeXHR( + respond({ + status: 401, + body: { + code: 'reauth_required', + reason: 'token_v1', + auth_id: 'a', + }, + }), + ); const resp = await fetchUrl('https://api.example/rao', { method: 'POST', @@ -240,10 +321,14 @@ describe('fetchUrl', () => { }); expect(globalThis.puter.triggerReauth).not.toHaveBeenCalled(); - expect(globalThis.puter.ui.authenticateWithPuter).not.toHaveBeenCalled(); + expect( + globalThis.puter.ui.authenticateWithPuter, + ).not.toHaveBeenCalled(); // Reported with the token it was sent with, so a reauth that // completed meanwhile keeps the token it installed. - expect(globalThis.puter.dropStaleAuthToken).toHaveBeenCalledWith({ + expect( + globalThis.puter.dropStaleAuthToken, + ).toHaveBeenCalledWith({ reason: 'token_v1', auth_id: 'a', sentToken: 'stale', @@ -254,16 +339,25 @@ describe('fetchUrl', () => { it('drops the stale token without raising UI on token_auth_failed', async () => { globalThis.puter = makePuter(); - installFakeXHR(respond({ status: 401, body: { code: 'token_auth_failed' } })); + installFakeXHR( + respond({ + status: 401, + body: { code: 'token_auth_failed' }, + }), + ); const resp = await fetchUrl('https://api.example/whoami', { includePuterAuth: true, interactiveReauth: false, }); - expect(globalThis.puter.ui.authenticateWithPuter).not.toHaveBeenCalled(); + expect( + globalThis.puter.ui.authenticateWithPuter, + ).not.toHaveBeenCalled(); expect(globalThis.puter.resetAuthToken).not.toHaveBeenCalled(); - expect(globalThis.puter.dropStaleAuthToken).toHaveBeenCalledTimes(1); + expect( + globalThis.puter.dropStaleAuthToken, + ).toHaveBeenCalledTimes(1); expect(resp.ok).toBe(false); }); @@ -277,13 +371,18 @@ describe('fetchUrl', () => { interactiveReauth: false, }); expect(resp.ok).toBe(true); - expect(globalThis.puter.dropStaleAuthToken).not.toHaveBeenCalled(); + expect( + globalThis.puter.dropStaleAuthToken, + ).not.toHaveBeenCalled(); }); }); }); describe('API call logging', () => { - const makeLogger = () => ({ isEnabled: () => true, logRequest: vi.fn() }); + const makeLogger = () => ({ + isEnabled: () => true, + logRequest: vi.fn(), + }); it('logs on success when the logger is enabled', async () => { const apiCallLogger = makeLogger(); @@ -297,7 +396,9 @@ describe('fetchUrl', () => { it('logs an error entry on a 4xx', async () => { const apiCallLogger = makeLogger(); globalThis.puter = { apiCallLogger }; - installFakeXHR(respond({ status: 404, body: { code: 'not_found' } })); + installFakeXHR( + respond({ status: 404, body: { code: 'not_found' } }), + ); await fetchUrl('https://api.example/x'); expect(apiCallLogger.logRequest).toHaveBeenCalledTimes(1); const entry = apiCallLogger.logRequest.mock.calls[0][0]; @@ -310,10 +411,17 @@ describe('fetchUrl', () => { globalThis.puter = { apiCallLogger }; installFakeXHR(respond({ status: 200, body: { u: 1 } })); await fetchUrl('https://api.example/whoami', { - logContext: { service: 'auth', operation: 'whoami', params: {} }, + logContext: { + service: 'auth', + operation: 'whoami', + params: {}, + }, }); const entry = apiCallLogger.logRequest.mock.calls[0][0]; - expect(entry).toMatchObject({ service: 'auth', operation: 'whoami' }); + expect(entry).toMatchObject({ + service: 'auth', + operation: 'whoami', + }); expect(entry.result).toEqual({ u: 1 }); }); }); @@ -322,19 +430,23 @@ describe('fetchUrl', () => { // Play a scripted response per attempt (retries create fresh XHR instances). const sequence = (...steps) => { let i = 0; - return xhr => steps[Math.min(i++, steps.length - 1)](xhr); + return (xhr) => steps[Math.min(i++, steps.length - 1)](xhr); }; -const netError = () => xhr => xhr._networkError(); +const netError = () => (xhr) => xhr._networkError(); describe('transient retry', () => { - beforeEach(() => { globalThis.puter = {}; }); + beforeEach(() => { + globalThis.puter = {}; + }); it('retries a GET on 503 then resolves the success', async () => { vi.useFakeTimers(); - const xhrs = installFakeXHR(sequence( - respond({ status: 503, body: {} }), - respond({ status: 200, body: { ok: 1 } }), - )); + const xhrs = installFakeXHR( + sequence( + respond({ status: 503, body: {} }), + respond({ status: 200, body: { ok: 1 } }), + ), + ); const p = fetchUrl('https://api.example/x'); // GET → retry-safe await vi.advanceTimersByTimeAsync(60_000); const resp = await p; @@ -344,19 +456,28 @@ describe('transient retry', () => { }); it('does not retry a POST by default', async () => { - const xhrs = installFakeXHR(sequence(respond({ status: 503, body: {} }))); - const resp = await fetchUrl('https://api.example/x', { method: 'POST' }); + const xhrs = installFakeXHR( + sequence(respond({ status: 503, body: {} })), + ); + const resp = await fetchUrl('https://api.example/x', { + method: 'POST', + }); expect(resp.status).toBe(503); expect(xhrs.length).toBe(1); }); it('retries a POST when retry:true (read-style opt-in)', async () => { vi.useFakeTimers(); - const xhrs = installFakeXHR(sequence( - respond({ status: 503, body: {} }), - respond({ status: 200, body: { ok: 1 } }), - )); - const p = fetchUrl('https://api.example/x', { method: 'POST', retry: true }); + const xhrs = installFakeXHR( + sequence( + respond({ status: 503, body: {} }), + respond({ status: 200, body: { ok: 1 } }), + ), + ); + const p = fetchUrl('https://api.example/x', { + method: 'POST', + retry: true, + }); await vi.advanceTimersByTimeAsync(60_000); expect((await p).status).toBe(200); expect(xhrs.length).toBe(2); @@ -364,22 +485,80 @@ describe('transient retry', () => { }); it('retry:false disables retry even for a GET', async () => { - const xhrs = installFakeXHR(sequence(respond({ status: 503, body: {} }))); + const xhrs = installFakeXHR( + sequence(respond({ status: 503, body: {} })), + ); const resp = await fetchUrl('https://api.example/x', { retry: false }); expect(resp.status).toBe(503); expect(xhrs.length).toBe(1); }); it('does not retry a non-retryable status (400)', async () => { - const xhrs = installFakeXHR(sequence(respond({ status: 400, body: {} }))); + const xhrs = installFakeXHR( + sequence(respond({ status: 400, body: {} })), + ); const resp = await fetchUrl('https://api.example/x'); // GET expect(resp.status).toBe(400); expect(xhrs.length).toBe(1); }); + // A 429 comes from a gate that runs before the handler, so nothing was + // applied and a write is as safe to replay as a read. Uploads and bursts + // of driver writes are the callers this matters to. + it('retries a POST on 429 even though 503 would not', async () => { + vi.useFakeTimers(); + const xhrs = installFakeXHR( + sequence( + respond({ status: 429, body: {} }), + respond({ status: 200, body: { ok: 1 } }), + ), + ); + const p = fetchUrl('https://api.example/x', { method: 'POST' }); + await vi.advanceTimersByTimeAsync(60_000); + expect((await p).status).toBe(200); + expect(xhrs.length).toBe(2); + vi.useRealTimers(); + }); + + it('gives up on 429 after the retry schedule is spent', async () => { + vi.useFakeTimers(); + const xhrs = installFakeXHR(respond({ status: 429, body: {} })); + const p = fetchUrl('https://api.example/x', { method: 'POST' }); + await vi.advanceTimersByTimeAsync(60_000); + expect((await p).status).toBe(429); + expect(xhrs.length).toBe(9); // 1 initial + 8 scheduled retries + vi.useRealTimers(); + }); + + it('honors retry:false on a 429', async () => { + const xhrs = installFakeXHR( + sequence(respond({ status: 429, body: {} })), + ); + const resp = await fetchUrl('https://api.example/x', { + method: 'POST', + retry: false, + }); + expect(resp.status).toBe(429); + expect(xhrs.length).toBe(1); + }); + + it('honors the autoRetry kill switch on a 429', async () => { + globalThis.puter = { config: { autoRetry: false } }; + const xhrs = installFakeXHR( + sequence(respond({ status: 429, body: {} })), + ); + const resp = await fetchUrl('https://api.example/x', { + method: 'POST', + }); + expect(resp.status).toBe(429); + expect(xhrs.length).toBe(1); + }); + it('respects the autoRetry kill switch', async () => { globalThis.puter = { config: { autoRetry: false } }; - const xhrs = installFakeXHR(sequence(respond({ status: 503, body: {} }))); + const xhrs = installFakeXHR( + sequence(respond({ status: 503, body: {} })), + ); const resp = await fetchUrl('https://api.example/x'); // GET, but retry off expect(resp.status).toBe(503); expect(xhrs.length).toBe(1); @@ -388,7 +567,7 @@ describe('transient retry', () => { it('retries a network error for a read, then rejects after the cap', async () => { vi.useFakeTimers(); const xhrs = installFakeXHR(netError()); // every attempt fails - const p = fetchUrl('https://api.example/x').catch(e => e); // GET + const p = fetchUrl('https://api.example/x').catch((e) => e); // GET await vi.advanceTimersByTimeAsync(60_000); // clears the ~11.75s schedule const err = await p; expect(err).toBeInstanceOf(TypeError); @@ -398,14 +577,16 @@ describe('transient retry', () => { it('rejects a write network error immediately (no retry)', async () => { const xhrs = installFakeXHR(netError()); - await expect(fetchUrl('https://api.example/x', { method: 'POST' })).rejects.toThrow(/failed/); + await expect( + fetchUrl('https://api.example/x', { method: 'POST' }), + ).rejects.toThrow(/failed/); expect(xhrs.length).toBe(1); }); it('gives up on a 2s retry when the clock jumps (sleep/drift guard)', async () => { // Fake only the timers, not Date — a manual `clock` drives Date.now so we // can simulate the machine sleeping during a 2s ceiling wait. - vi.useFakeTimers({ toFake: [ 'setTimeout', 'clearTimeout' ] }); + vi.useFakeTimers({ toFake: ['setTimeout', 'clearTimeout'] }); let clock = 0; vi.spyOn(Date, 'now').mockImplementation(() => clock); const xhrs = installFakeXHR(respond({ status: 503, body: {} })); // always retryable @@ -420,7 +601,7 @@ describe('transient retry', () => { const resp = await p; expect(resp.status).toBe(503); // failed with the last outcome — no further retry - expect(xhrs.length).toBe(4); // stopped after the drifted 2s wait, before attempt 5 + expect(xhrs.length).toBe(4); // stopped after the drifted 2s wait, before attempt 5 vi.useRealTimers(); }); }); @@ -428,8 +609,14 @@ describe('transient retry', () => { describe('dedupe', () => { it('coalesces concurrent identical requests into one call', async () => { let calls = 0; - const factory = () => { calls++; return new Promise(r => setTimeout(() => r({ v: calls }), 5)); }; - const [ a, b ] = await Promise.all([ dedupe('k', factory), dedupe('k', factory) ]); + const factory = () => { + calls++; + return new Promise((r) => setTimeout(() => r({ v: calls }), 5)); + }; + const [a, b] = await Promise.all([ + dedupe('k', factory), + dedupe('k', factory), + ]); expect(calls).toBe(1); expect(a).toBe(b); // shared resolved value by reference }); @@ -444,8 +631,9 @@ describe('dedupe', () => { it('does not collide across distinct keys', async () => { let calls = 0; - const factory = () => new Promise(r => setTimeout(() => r(++calls), 5)); - await Promise.all([ dedupe('a', factory), dedupe('b', factory) ]); + const factory = () => + new Promise((r) => setTimeout(() => r(++calls), 5)); + await Promise.all([dedupe('a', factory), dedupe('b', factory)]); expect(calls).toBe(2); }); }); @@ -457,25 +645,42 @@ describe('driver permission-grant replay (regression)', () => { // requestPermission resolves to a boolean (its real contract). const requestPermission = vi.fn(async () => true); globalThis.puter = { ui: { requestPermission } }; - const xhrs = installFakeXHR(sequence( - respond({ status: 200, body: { success: false, error: { code: 'permission_denied' } } }), - respond({ status: 200, body: { success: true, result: 'ok' } }), - )); + const xhrs = installFakeXHR( + sequence( + respond({ + status: 200, + body: { + success: false, + error: { code: 'permission_denied' }, + }, + }), + respond({ status: 200, body: { success: true, result: 'ok' } }), + ), + ); const spec = { url: 'https://api.example/drivers/call', method: 'POST', headers: { 'Content-Type': 'text/plain;actually=json' }, - buildBody: () => JSON.stringify({ interface: 'iface', method: 'm', args: { a: 1 } }), + buildBody: () => + JSON.stringify({ + interface: 'iface', + method: 'm', + args: { a: 1 }, + }), }; const result = await sendWithRetry(spec, { permission: 'driver:iface:m', shapeStream: () => {}, - shape: outcome => outcome.parsed, + shape: (outcome) => outcome.parsed, }); expect(requestPermission).toHaveBeenCalledTimes(1); - expect(requestPermission).toHaveBeenCalledWith({ permission: 'driver:iface:m' }); + expect(requestPermission).toHaveBeenCalledWith({ + permission: 'driver:iface:m', + }); expect(xhrs.length).toBe(2); - expect(xhrs[1].reqBody).toBe(JSON.stringify({ interface: 'iface', method: 'm', args: { a: 1 } })); + expect(xhrs[1].reqBody).toBe( + JSON.stringify({ interface: 'iface', method: 'm', args: { a: 1 } }), + ); expect(result).toEqual({ success: true, result: 'ok' }); }); @@ -483,13 +688,21 @@ describe('driver permission-grant replay (regression)', () => { // Legacy `{granted}` object shape is still tolerated. const requestPermission = vi.fn(async () => ({ granted: true })); globalThis.puter = { ui: { requestPermission } }; - const denied = respond({ status: 200, body: { success: false, error: { code: 'permission_denied' } } }); + const denied = respond({ + status: 200, + body: { success: false, error: { code: 'permission_denied' } }, + }); const xhrs = installFakeXHR(sequence(denied, denied, denied)); - const spec = { url: 'https://api.example/drivers/call', method: 'POST', headers: {}, buildBody: () => '{}' }; + const spec = { + url: 'https://api.example/drivers/call', + method: 'POST', + headers: {}, + buildBody: () => '{}', + }; const result = await sendWithRetry(spec, { permission: 'driver:iface:m', shapeStream: () => {}, - shape: outcome => outcome.parsed, + shape: (outcome) => outcome.parsed, }); expect(requestPermission).toHaveBeenCalledTimes(1); // one-shot expect(xhrs.length).toBe(2); @@ -501,16 +714,24 @@ describe('driverCall', () => { const call = { iface: 'puter-kvstore', method: 'get', args: { key: 'k' } }; beforeEach(() => { - globalThis.puter = { authToken: 'tok', APIOrigin: 'https://api.example', env: 'nodejs' }; + globalThis.puter = { + authToken: 'tok', + APIOrigin: 'https://api.example', + env: 'nodejs', + }; }); it('posts the driver envelope and resolves the unwrapped result', async () => { - const xhrs = installFakeXHR(respond({ body: { success: true, result: 'v' } })); + const xhrs = installFakeXHR( + respond({ body: { success: true, result: 'v' } }), + ); const result = await driverCall(call); expect(result).toBe('v'); expect(xhrs[0].method).toBe('POST'); expect(xhrs[0].url).toBe('https://api.example/drivers/call'); - expect(xhrs[0]._reqHeaders['content-type']).toBe('text/plain;actually=json'); + expect(xhrs[0]._reqHeaders['content-type']).toBe( + 'text/plain;actually=json', + ); expect(JSON.parse(xhrs[0].reqBody)).toEqual({ interface: 'puter-kvstore', method: 'get', @@ -520,9 +741,14 @@ describe('driverCall', () => { }); it('sends driver and test_mode only when the caller sets them', async () => { - const xhrs = installFakeXHR(respond({ body: { success: true, result: {} } })); + const xhrs = installFakeXHR( + respond({ body: { success: true, result: {} } }), + ); await driverCall({ ...call, driver: 'ai-chat', testMode: false }); - expect(JSON.parse(xhrs[0].reqBody)).toMatchObject({ driver: 'ai-chat', test_mode: false }); + expect(JSON.parse(xhrs[0].reqBody)).toMatchObject({ + driver: 'ai-chat', + test_mode: false, + }); }); it('resolves the whole response when the driver returns no result field', async () => { @@ -532,22 +758,35 @@ describe('driverCall', () => { it('applies transform to a successful result', async () => { installFakeXHR(respond({ body: { success: true, result: 2 } })); - const result = await driverCall(call, { transform: async n => n * 21 }); + const result = await driverCall(call, { + transform: async (n) => n * 21, + }); expect(result).toBe(42); }); it('rejects the driver error payload and notifies onError', async () => { - installFakeXHR(respond({ body: { success: false, error: { code: 'key_too_large' } } })); + installFakeXHR( + respond({ + body: { success: false, error: { code: 'key_too_large' } }, + }), + ); const onError = vi.fn(); await expect(driverCall(call, { onError })).rejects.toEqual({ - success: false, error: { code: 'key_too_large' }, + success: false, + error: { code: 'key_too_large' }, + }); + expect(onError).toHaveBeenCalledWith({ + success: false, + error: { code: 'key_too_large' }, }); - expect(onError).toHaveBeenCalledWith({ success: false, error: { code: 'key_too_large' } }); }); it('rejects a leftover 401 as Unauthorized', async () => { installFakeXHR(respond({ status: 401, body: {} })); - await expect(driverCall(call)).rejects.toEqual({ status: 401, message: 'Unauthorized' }); + await expect(driverCall(call)).rejects.toEqual({ + status: 401, + message: 'Unauthorized', + }); }); it('rejects auth_canceled when a signed-out visitor dismisses the prompt', async () => { @@ -555,33 +794,44 @@ describe('driverCall', () => { authToken: null, APIOrigin: 'https://api.example', env: 'web', - ui: { authenticateWithPuter: async () => { throw new Error('dismissed'); } }, + ui: { + authenticateWithPuter: async () => { + throw new Error('dismissed'); + }, + }, }; - const xhrs = installFakeXHR(respond({ body: { success: true, result: 'v' } })); + const xhrs = installFakeXHR( + respond({ body: { success: true, result: 'v' } }), + ); await expect(driverCall(call)).rejects.toEqual({ - error: { code: 'auth_canceled', message: 'Authentication canceled' }, + error: { + code: 'auth_canceled', + message: 'Authentication canceled', + }, }); expect(xhrs.length).toBe(0); // no request without a token }); it('resolves an NDJSON response as an iterator of lines that stringify to their text', async () => { - installFakeXHR(xhr => { + installFakeXHR((xhr) => { xhr._setHeaders(200, { 'content-type': 'application/x-ndjson' }); xhr._headersReceived(); xhr._progress('{"text":"he"}\n{"text":"llo"}\n'); xhr._done(); }); const parts = []; - for await ( const part of await driverCall(call) ) parts.push(`${part}`); - expect(parts).toEqual([ 'he', 'llo' ]); + for await (const part of await driverCall(call)) parts.push(`${part}`); + expect(parts).toEqual(['he', 'llo']); }); it('retries a readonly method on a transient failure', async () => { vi.useFakeTimers(); - const xhrs = installFakeXHR(sequence( - respond({ status: 503, body: {} }), - respond({ body: { success: true, result: 'v' } }), - )); + const xhrs = installFakeXHR( + sequence( + respond({ status: 503, body: {} }), + respond({ body: { success: true, result: 'v' } }), + ), + ); const p = driverCall(call, { readonly: true }); await vi.advanceTimersByTimeAsync(60_000); await expect(p).resolves.toBe('v'); @@ -594,31 +844,50 @@ describe('driverCallEnvelope', () => { const call = { iface: 'ipgeo', method: 'ipgeo', args: { ip: '1.2.3.4' } }; beforeEach(() => { - globalThis.puter = { authToken: 'tok', APIOrigin: 'https://api.example', env: 'nodejs' }; + globalThis.puter = { + authToken: 'tok', + APIOrigin: 'https://api.example', + env: 'nodejs', + }; }); it('resolves the envelope as the backend sent it', async () => { - installFakeXHR(respond({ body: { success: true, result: { country: 'US' } } })); - expect(await driverCallEnvelope(call)).toEqual({ success: true, result: { country: 'US' } }); + installFakeXHR( + respond({ body: { success: true, result: { country: 'US' } } }), + ); + expect(await driverCallEnvelope(call)).toEqual({ + success: true, + result: { country: 'US' }, + }); }); it('resolves rather than rejects on a driver-level failure', async () => { - installFakeXHR(respond({ body: { success: false, error: { code: 'not_found' } } })); - expect(await driverCallEnvelope(call)).toEqual({ success: false, error: { code: 'not_found' } }); + installFakeXHR( + respond({ body: { success: false, error: { code: 'not_found' } } }), + ); + expect(await driverCallEnvelope(call)).toEqual({ + success: false, + error: { code: 'not_found' }, + }); }); it('reads a response with no declared content type as JSON', async () => { - installFakeXHR(xhr => { + installFakeXHR((xhr) => { xhr._setHeaders(200); xhr._headersReceived(); xhr.responseText = '{"success":true,"result":[1,2]}'; xhr._done(); }); - expect(await driverCallEnvelope(call)).toEqual({ success: true, result: [ 1, 2 ] }); + expect(await driverCallEnvelope(call)).toEqual({ + success: true, + result: [1, 2], + }); }); it('throws on a content type it cannot read', async () => { installFakeXHR(respond({ contentType: 'text/html', body: '' })); - await expect(driverCallEnvelope(call)).rejects.toThrow('unrecognized content type: text/html'); + await expect(driverCallEnvelope(call)).rejects.toThrow( + 'unrecognized content type: text/html', + ); }); }); diff --git a/tools/typecheck-baseline.json b/tools/typecheck-baseline.json new file mode 100644 index 000000000..3e9ae9a21 --- /dev/null +++ b/tools/typecheck-baseline.json @@ -0,0 +1,58 @@ +{ + "extensions/whoami.ts | TS2353": 1, + "src/backend/clients/database/MySQLDatabaseClient.ts | TS2769": 1, + "src/backend/clients/database/PostgresDatabaseClient.ts | TS2322": 1, + "src/backend/clients/s3/S3Client.ts | TS2307": 2, + "src/backend/controllers/auth/AuthController.ts | TS2345": 1, + "src/backend/controllers/auth/AuthController.ts | TS7018": 2, + "src/backend/controllers/fs/FSController.ts | TS7011": 3, + "src/backend/controllers/homepage/HomepageController.ts | TS2352": 1, + "src/backend/controllers/wisp/WispController.ts | TS2345": 1, + "src/backend/core/http/__typecheck__.ts | TS2578": 1, + "src/backend/core/http/middleware/errorHandler.ts | TS2345": 1, + "src/backend/drivers/ai-chat/ChatCompletionDriver.ts | TS2322": 10, + "src/backend/drivers/ai-chat/ChatCompletionDriver.ts | TS2345": 1, + "src/backend/drivers/ai-chat/providers/ChatProvider.ts | TS2420": 1, + "src/backend/drivers/ai-chat/providers/FakeChatProvider.ts | TS2416": 1, + "src/backend/drivers/ai-chat/providers/FakeChatProvider.ts | TS7018": 1, + "src/backend/drivers/ai-chat/providers/alibaba/AlibabaProvider.ts | TS2416": 1, + "src/backend/drivers/ai-chat/providers/azure/AzureChatProvider.ts | TS2352": 1, + "src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts | TS2322": 1, + "src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts | TS2352": 1, + "src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts | TS2353": 1, + "src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts | TS2416": 1, + "src/backend/drivers/ai-chat/providers/azure/AzureResponsesProvider.ts | TS7006": 1, + "src/backend/drivers/ai-chat/providers/claude/ClaudeProvider.ts | TS7011": 1, + "src/backend/drivers/ai-chat/providers/deepseek/DeepSeekProvider.ts | TS2416": 1, + "src/backend/drivers/ai-chat/providers/groq/GroqAIProvider.ts | TS7006": 1, + "src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts | TS7006": 3, + "src/backend/drivers/ai-chat/providers/mistral/MistralAiProvider.ts | TS7053": 1, + "src/backend/drivers/ai-chat/providers/ollama/OllamaProvider.ts | TS2416": 1, + "src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts | TS2352": 1, + "src/backend/drivers/ai-chat/providers/openai/OpenAiChatCompletionsProvider.ts | TS2416": 1, + "src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts | TS2322": 1, + "src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts | TS2352": 1, + "src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts | TS2353": 1, + "src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts | TS2416": 1, + "src/backend/drivers/ai-chat/providers/openai/OpenAiChatResponsesProvider.ts | TS7006": 2, + "src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts | TS7031": 6, + "src/backend/drivers/ai-chat/providers/openrouter/OpenRouterProvider.ts | TS7053": 1, + "src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts | TS2416": 1, + "src/backend/drivers/ai-chat/providers/together/TogetherAIProvider.ts | TS7053": 1, + "src/backend/drivers/ai-speech2txt/providers/openai/OpenAISpeechToTextProvider.ts | TS2352": 2, + "src/backend/drivers/ai-speech2txt/providers/xai/XAISpeechToTextProvider.ts | TS2322": 1, + "src/backend/drivers/integrationTestUtil.ts | TS7011": 1, + "src/backend/services/apps/AppPermissionService.ts | TS2345": 2, + "src/backend/services/auth/OIDCService.ts | TS2322": 2, + "src/backend/services/broadcast/BroadcastService.ts | TS2345": 2, + "src/backend/services/fs/FSService.ts | TS2345": 1, + "src/backend/services/fs/cacheInvalidation.ts | TS2345": 1, + "src/backend/services/homepage/PuterHomepageService.ts | TS2345": 1, + "src/backend/services/localworker/LocalWorkerService.ts | TS7018": 3, + "src/backend/services/notification/NotificationService.ts | TS2322": 1, + "src/backend/stores/fs/S3ObjectStore.ts | TS7011": 1, + "src/backend/testUtil.ts | TS1470": 1, + "src/backend/testUtil.ts | TS2345": 1, + "src/backend/util/privateLaunchAccess.ts | TS2345": 1, + "src/backend/vitest.config.ts | TS7018": 1 +} diff --git a/tools/typecheck.mjs b/tools/typecheck.mjs new file mode 100644 index 000000000..2792b3ef2 --- /dev/null +++ b/tools/typecheck.mjs @@ -0,0 +1,136 @@ +#!/usr/bin/env node +/** + * Type-checks the backend and fails on errors that aren't already known. + * + * `tsconfig.build.json` sets `noCheck: true`, so `tsc` emits without ever + * checking types — which is how a call to a function that no longer existed + * (`effectiveActorApp`) shipped to production and crashed every node that + * served an AI prompt. The compiler had the error the whole time; nothing ran + * it. + * + * The consuming repo (heyputer) runs an equivalent gate over this project plus + * its extensions. That one only fires once someone bumps the submodule pointer, + * which is too late to keep a broken export out of this repo's default branch — + * so the same check runs here, on this repo's own pull requests. Keep the two + * scripts behaving the same way; each keeps its own baseline, since the error + * sets differ with how the project is resolved. + * + * Turning checking on wholesale isn't possible yet: there is a real backlog of + * pre-existing errors (see the baseline). So this runs the check with + * `--noCheck false`, diffs against that recorded backlog, and fails only on + * errors that are *new*. The backlog can then be burned down without blocking + * anyone, and the day it hits zero this becomes a plain `tsc` gate and the + * `noCheck` flag comes out of the tsconfig. + * + * Usage: + * node tools/typecheck.mjs # check; exit 1 on new errors + * node tools/typecheck.mjs --update # rewrite the baseline + */ + +import { execFileSync } from 'node:child_process'; +import { readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const BASELINE_PATH = join(ROOT, 'tools', 'typecheck-baseline.json'); +const CONFIG = 'tsconfig.build.json'; + +// `file(line,col): error TSxxxx: message` +const ERROR_RE = /^(?[^(]+)\((?\d+),(?\d+)\): error (?TS\d+): (?.*)$/; + +const runTsc = () => { + try { + execFileSync( + 'npx', + ['tsc', '-p', CONFIG, '--noCheck', 'false', '--noEmit'], + { cwd: ROOT, encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] }, + ); + return ''; + } catch (e) { + // tsc exits non-zero when it reports errors; that's the normal path. + if (e.stdout === undefined && e.stderr === undefined) throw e; + return `${e.stdout ?? ''}${e.stderr ?? ''}`; + } +}; + +/** + * Key an error by file and code — deliberately *not* by line number, so + * unrelated edits above an existing error don't churn the baseline. The message + * is dropped too: it embeds inferred type text that shifts whenever a nearby + * signature changes, which would otherwise read as a new error. + */ +const SEP = ' | '; +const keyOf = (file, code) => [file, code].join(SEP); + +const collect = () => { + const counts = new Map(); + const samples = new Map(); + for (const line of runTsc().split('\n')) { + const m = ERROR_RE.exec(line.trim()); + if (!m) continue; + const { file, code, message } = m.groups; + const key = keyOf(file, code); + counts.set(key, (counts.get(key) ?? 0) + 1); + if (!samples.has(key)) { + samples.set(key, `${file}:${m.groups.line} ${code}: ${message}`); + } + } + return { counts, samples }; +}; + +const { counts, samples } = collect(); +const update = process.argv.includes('--update'); + +if (update) { + const baseline = Object.fromEntries([...counts.entries()].sort()); + writeFileSync(BASELINE_PATH, `${JSON.stringify(baseline, null, 2)}\n`); + const total = [...counts.values()].reduce((a, b) => a + b, 0); + console.log( + `Baseline written: ${total} known errors across ${counts.size} file/code pairs.`, + ); + process.exit(0); +} + +if (!existsSync(BASELINE_PATH)) { + console.error('No baseline found. Run: node tools/typecheck.mjs --update'); + process.exit(1); +} + +const baseline = JSON.parse(readFileSync(BASELINE_PATH, 'utf8')); + +const regressions = []; +for (const [key, count] of counts) { + const known = baseline[key] ?? 0; + if (count > known) regressions.push({ key, count, known }); +} + +const fixed = []; +for (const [key, known] of Object.entries(baseline)) { + const count = counts.get(key) ?? 0; + if (count < known) fixed.push({ key, count, known }); +} + +if (fixed.length) { + const net = fixed.reduce((a, f) => a + (f.known - f.count), 0); + console.log( + `${net} baselined error(s) fixed. Run \`npm run typecheck:update\` to lock that in.\n`, + ); +} + +if (!regressions.length) { + const total = [...counts.values()].reduce((a, b) => a + b, 0); + console.log(`Type check passed — no new errors (${total} known, baselined).`); + process.exit(0); +} + +console.error('New type errors (not in the baseline):\n'); +for (const { key, count, known } of regressions) { + const extra = known ? ` (${known} known, ${count} now)` : ''; + console.error(` ${samples.get(key)}${extra}`); +} +console.error( + `\n${regressions.length} new error(s). Fix them, or if they are genuinely` + + ' pre-existing, run `npm run typecheck:update` and say so in review.', +); +process.exit(1); diff --git a/tsconfig.build.json b/tsconfig.build.json index 292eb7403..bb995b279 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -2,6 +2,8 @@ "extends": "./tsconfig.json", "compilerOptions": { "strict": false, + // Emit-only: this build transpiles, it does not type check. Run the + // checker with `tsc -p tsconfig.build.json --noCheck false --noEmit`. "noCheck": true, "noImplicitAny": true }