From c815dc82a1a10385bf2f5a0477039f15514184ca Mon Sep 17 00:00:00 2001 From: Daniel Salazar Date: Sun, 11 Oct 2026 21:53:33 -0700 Subject: [PATCH] refactor(core-util): shared bounded map, cheaper route lifecycle, DDB/DB/rate-limit/gate dedup (#4185) * refactor(util): add BoundedTtlMap for core caches and log throttles One size-capped, LRU-evicting map with an optional TTL and a once-per-window shouldEmit helper. Replaces the hand-rolled maps in the DynamoDB throttle and repair loggers (the repair one was unbounded), the Slack repeat throttle, the auth probe's reauth log throttle, and the card-verification status memo. * perf(http): skip route lifecycle work when nothing listens The per-route lifecycle middleware built a payload, opened an event.emitAndWait span and hooked finish/close on every request, listeners or not. Each phase now checks hasListeners first. EventClient's three copies of the key-prefix walk become one memoized #matchKeys. * perf(ddb): write batch chunks concurrently and retry the TTL sweep batchPut/batchDel sent their 25-item chunks one after another; they now run up to four at a time through runWithConcurrencyLimit, and a failed chunk stops the ones not yet started. batchGet and the batch write share one #sendWithUnprocessedRetry instead of two copies of the backoff loop, and the TTL sweep goes through #batchWrite, so the unprocessed deletes it used to drop are retried. The AWS and local clients are built by one #bindClient. * refactor(db): share migrations and tryHardRead in the base client MySQL and Postgres each carried a copy of the migration-path runner and of the replica-racing tryHardRead; both now live on AbstractDatabaseClient (applyMigrationPaths, and tryHardRead gated on hasReadReplica), and SQLite drops its own tryHardRead override. Removes MySQL reinitPrimary, reinitReplica, isRetriableError and readWithRetry and the base requireRead, which had no callers outside tests. SQLite prepared statements and the Postgres SQL rewrite are memoized per query text in a bounded map. * refactor(rate-limit): route every entry point through one charge() The nine rate/peek/acquire entry points each repeated key and limit resolution, the backend call and the fail-open catch. They now share charge(), routeKey() and guardSlot(); log lines are unchanged. Driver concurrency slots release at most once like the other slot handles, and checkRateLimit takes { failOpen: false } for limits that must hold when the backend is down. * refactor(util): add secretsEqual for constant-time secret comparison One HMAC-then-timingSafeEqual helper, safe for inputs of any length, so callers stop hand-rolling the length guard. File signature verification uses it. * refactor(fs): infer signed-file MIME types with mime-types mimeFromName looks extensions up in mime-types instead of a 35-entry inline table, keeping the two types the table reported differently (ico, wav). The purpose-labelled signing key is derived once per secret instead of on every signature, where signFile derived it twice. * refactor(http): build route gates from one gate() helper Each gate repeated a try/catch -> next(err) wrapper or a .then(next, next) chain around an assertion. gate(check) does that once: a throw or rejection goes to next(err), and a synchronous check still calls next synchronously. The credits, subscription and reputation gate files held nothing but such a wrapper; their factories move into gates.ts and the files go. * perf(hosting): build the puter-site hosting config once createPuterSiteMiddleware rebuilt the normalized hosting-domain config on every hosted-site request; it is derived from static config, like the domain lists the middleware already builds at construction. * fix(db): tryHardRead's replica leg doesn't fail over to the primary MySQL's read() retries on the primary after a load-shed or transient replica error, so routing tryHardRead through it read the primary twice on that path. The replica leg now reads the replica alone. * fix(ddb): settle in-flight batch chunks before rethrowing A failed chunk stopped unstarted chunks but rejected while others were still retrying, so writes could land after the caller had handled the error and invalidated its cache. --- src/backend/clients/alarm/slack.ts | 29 +- .../clients/database/DatabaseClient.test.ts | 25 +- .../clients/database/DatabaseClient.ts | 153 ++++++- .../database/MySQLDatabaseClient.test.ts | 160 +------ .../clients/database/MySQLDatabaseClient.ts | 215 +--------- .../database/PostgresDatabaseClient.test.ts | 31 ++ .../database/PostgresDatabaseClient.ts | 117 +---- .../database/SqliteDatabaseClient.test.ts | 57 ++- .../clients/database/SqliteDatabaseClient.ts | 52 +-- .../clients/dynamodb/DDBClient.test.ts | 172 ++++++++ src/backend/clients/dynamodb/DDBClient.ts | 394 ++++++++--------- src/backend/clients/dynamodb/throttleLog.ts | 13 +- src/backend/clients/event/EventClient.test.ts | 24 ++ src/backend/clients/event/EventClient.ts | 72 ++-- src/backend/core/http/middleware/authProbe.ts | 24 +- src/backend/core/http/middleware/credits.ts | 50 --- ...ation.test.ts => gates.reputation.test.ts} | 2 +- ...ion.test.ts => gates.subscription.test.ts} | 2 +- .../core/http/middleware/gates.test.ts | 54 +++ src/backend/core/http/middleware/gates.ts | 320 +++++++------- src/backend/core/http/middleware/puterSite.ts | 3 +- src/backend/core/http/middleware/rateLimit.js | 399 +++++++++--------- .../core/http/middleware/rateLimit.test.js | 30 ++ .../core/http/middleware/reputation.ts | 48 --- .../core/http/middleware/subscription.ts | 58 --- src/backend/core/http/routeLifecycle.test.ts | 79 +++- src/backend/core/http/routeLifecycle.ts | 21 +- src/backend/server.ts | 6 +- src/backend/util/boundedTtlMap.test.ts | 133 ++++++ src/backend/util/boundedTtlMap.ts | 102 +++++ src/backend/util/cardFallback.ts | 16 +- src/backend/util/fileSigning.test.ts | 48 +++ src/backend/util/fileSigning.ts | 86 ++-- src/backend/util/secureCompare.test.ts | 46 ++ src/backend/util/secureCompare.ts | 36 ++ 35 files changed, 1677 insertions(+), 1400 deletions(-) delete mode 100644 src/backend/core/http/middleware/credits.ts rename src/backend/core/http/middleware/{reputation.test.ts => gates.reputation.test.ts} (97%) rename src/backend/core/http/middleware/{subscription.test.ts => gates.subscription.test.ts} (98%) delete mode 100644 src/backend/core/http/middleware/reputation.ts delete mode 100644 src/backend/core/http/middleware/subscription.ts create mode 100644 src/backend/util/boundedTtlMap.test.ts create mode 100644 src/backend/util/boundedTtlMap.ts create mode 100644 src/backend/util/secureCompare.test.ts create mode 100644 src/backend/util/secureCompare.ts diff --git a/src/backend/clients/alarm/slack.ts b/src/backend/clients/alarm/slack.ts index 3a371f1bb..487048154 100644 --- a/src/backend/clients/alarm/slack.ts +++ b/src/backend/clients/alarm/slack.ts @@ -18,6 +18,7 @@ */ import type { ISlackAlertConfig, PagerSeverity } from '../../types'; +import { BoundedTtlMap } from '../../util/boundedTtlMap'; import type { AlertHandler, AlertPayload } from './types'; const REQUEST_TIMEOUT_MS = 5000; @@ -123,30 +124,14 @@ export function createSlackAlertHandler( ): AlertHandler { const webhookUrl = conf.webhookUrl as string; const throttleMs = conf.repeatThrottleMs ?? DEFAULT_REPEAT_THROTTLE_MS; - const lastPosted = new Map(); - - const shouldPost = (alert: AlertPayload): boolean => { - if (throttleMs <= 0) return true; - const now = Date.now(); - const previous = lastPosted.get(alert.id); - if (previous !== undefined && now - previous < throttleMs) return false; - - if (lastPosted.size >= MAX_THROTTLE_ENTRIES) { - for (const [id, at] of lastPosted) { - if (now - at >= throttleMs) lastPosted.delete(id); - } - // Still full of live entries — drop the oldest to stay bounded. - if (lastPosted.size >= MAX_THROTTLE_ENTRIES) { - const oldest = lastPosted.keys().next().value; - if (oldest !== undefined) lastPosted.delete(oldest); - } - } - lastPosted.set(alert.id, now); - return true; - }; + // A zero or negative throttle expires every entry on arrival. + const lastPosted = new BoundedTtlMap({ + maxEntries: MAX_THROTTLE_ENTRIES, + ttlMs: throttleMs, + }); return async (alert) => { - if (!shouldPost(alert)) return; + if (!lastPosted.shouldEmit(alert.id)) return; const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS); diff --git a/src/backend/clients/database/DatabaseClient.test.ts b/src/backend/clients/database/DatabaseClient.test.ts index f8f53fe87..9776d8a42 100644 --- a/src/backend/clients/database/DatabaseClient.test.ts +++ b/src/backend/clients/database/DatabaseClient.test.ts @@ -45,10 +45,15 @@ class FixtureDatabaseClient extends AbstractDatabaseClient { constructor( private readonly replicaRows: Record[] | Error, private readonly primaryRows: Record[] = [], + private readonly replica = true, ) { super(config('fixture')); } + protected override hasReadReplica(): boolean { + return this.replica; + } + override async read(query: string): Promise[]> { this.reads.push(query); if (this.replicaRows instanceof Error) throw this.replicaRows; @@ -127,14 +132,22 @@ describe('AbstractDatabaseClient — replica-aware reads', () => { ]); }); - it('names the failing query when a required read finds nothing', async () => { - const client = new FixtureDatabaseClient([], []); - await expect( - client.requireRead('SELECT * FROM `user` WHERE `id` = ?'), - ).rejects.toThrow( - 'required read returned no rows: SELECT * FROM `user` WHERE `id` = ?', + it('surfaces the primary error when both reads fail', async () => { + const client = new FixtureDatabaseClient(new Error('replica down')); + client.pread = async () => { + throw new Error('primary down'); + }; + await expect(client.tryHardRead('SELECT 1')).rejects.toThrow( + 'primary down', ); }); + + it('issues a single read when there is no replica', async () => { + const client = new FixtureDatabaseClient([], [{ id: 2 }], false); + await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([]); + expect(client.reads).toEqual(['SELECT 1']); + expect(client.preads).toEqual([]); + }); }); describe('AbstractDatabaseClient — SQL generation', () => { diff --git a/src/backend/clients/database/DatabaseClient.ts b/src/backend/clients/database/DatabaseClient.ts index ad8965820..f666059a6 100644 --- a/src/backend/clients/database/DatabaseClient.ts +++ b/src/backend/clients/database/DatabaseClient.ts @@ -17,8 +17,12 @@ * along with this program. If not, see . */ +import { readdirSync, readFileSync } from 'fs'; +import { isAbsolute, resolve as resolvePath } from 'path'; import type { IConfig } from '../../types'; +import { Span } from '../../util/span.js'; import { PuterClient } from '../types'; +import { compareMigrationFilenames } from './migrationFilenames.js'; export interface WriteResult { insertId: number | bigint; @@ -33,6 +37,12 @@ export interface BatchEntry { type SqlJsonPath = readonly [string, ...string[]]; +/** The slice of a pooled connection that migrations need. */ +export interface MigrationConnection { + query(sql: string): Promise; + release(): void; +} + /** * Base database client. Subclasses must override every method that throws here. * @@ -104,39 +114,133 @@ export class AbstractDatabaseClient extends PuterClient { return this.write(sql, values); } + /** Whether `read()` goes to a separate replica rather than the primary. */ + protected hasReadReplica(): boolean { + return false; + } + /** - * Like `read()` but falls back to the primary when read-replicas are in - * use. Subclasses may override with replica-aware logic; the default - * delegates to `pread()`. + * The replica leg of `tryHardRead`. Override when `read()` falls back to + * the primary by itself, or a replica failure reads the primary twice. */ + protected replicaRead( + query: string, + params: unknown[], + ): Promise[]> { + return this.read(query, params); + } + + /** + * Like `read()`, but an empty or failed replica read falls back to the + * primary, so replication lag can't hide a row that was just written. The + * primary read starts alongside the replica one rather than after it. + * Without a replica this is a single `read()`. + */ + @Span('db.tryHardRead', (query: string) => ({ 'db.statement': query })) async tryHardRead( query: string, params: unknown[] = [], ): Promise[]> { - const primary = this.pread(query, params); - primary.catch(() => {}); + if (!this.hasReadReplica()) return this.read(query, params); + const primary = settle(this.pread(query, params)); try { - const rows = await this.read(query, params); - if (rows.length > 0) { - return rows; - } + const rows = await this.replicaRead(query, params); + if (rows.length > 0) return rows; } catch { // replica failed — fall through to primary } - return primary; + const result = await primary; + if ('error' in result) throw result.error; + return result.value; } - /** Like `tryHardRead()` but throws when the result set is empty. */ - async requireRead( - query: string, - params: unknown[] = [], - ): Promise[]> { - const rows = await this.tryHardRead(query, params); - if (rows.length === 0) { - throw new Error(`required read returned no rows: ${query}`); + /** + * Apply the `.sql` files named `*` in each configured migration + * directory, in filename order. Files must be idempotent: nothing records + * which ones already ran. A failure aborts startup. `transactional` wraps + * each file in BEGIN/COMMIT, for engines whose DDL can roll back. + */ + protected async applyMigrationPaths( + connect: () => Promise, + splitStatements: (contents: string) => string[], + { transactional }: { transactional: boolean }, + ): Promise { + const paths = this.config.database?.migrationPaths; + if (!paths || paths.length === 0) return; + + const tag = `[${this.engineName}]`; + const conn = await connect(); + try { + for (const rawPath of paths) { + const dir = isAbsolute(rawPath) + ? rawPath + : resolvePath(process.cwd(), rawPath); + + let files: string[]; + try { + files = readdirSync(dir) + .filter( + (f) => + f.endsWith('.sql') && + f.startsWith(this.engineName), + ) + .sort(compareMigrationFilenames); + } catch (e) { + throw new Error( + `${tag} migration path is unreadable: ${dir}`, + { + cause: e, + }, + ); + } + + if (files.length === 0) { + console.log(`${tag} no migrations in ${dir}`); + continue; + } + + console.log( + `${tag} running migrations from ${dir}: ${files.length} file(s)`, + ); + + for (const file of files) { + const contents = readFileSync( + resolvePath(dir, file), + 'utf8', + ); + const statements = splitStatements(contents); + if (transactional) await conn.query('BEGIN'); + try { + for (let i = 0; i < statements.length; i++) { + try { + await conn.query(statements[i]); + } catch (e) { + throw new Error( + `${tag} failed to apply ${file} at statement ${i}`, + { cause: e }, + ); + } + } + if (transactional) await conn.query('COMMIT'); + } catch (e) { + if (transactional) { + try { + await conn.query('ROLLBACK'); + } catch { + // the original failure is the one worth surfacing + } + } + throw e; + } + console.log( + `${tag} applied ${file} (${statements.length} statements)`, + ); + } + } + } finally { + conn.release(); } - return rows; } /** @@ -269,3 +373,14 @@ export class AbstractDatabaseClient extends PuterClient { return `'${value.replaceAll("'", "''")}'`; } } + +/** Await later without an unhandled rejection in the meantime. */ +const settle = async ( + promise: Promise, +): Promise<{ value: T } | { error: unknown }> => { + try { + return { value: await promise }; + } catch (error) { + return { error }; + } +}; diff --git a/src/backend/clients/database/MySQLDatabaseClient.test.ts b/src/backend/clients/database/MySQLDatabaseClient.test.ts index 8b15c953c..ae2608121 100644 --- a/src/backend/clients/database/MySQLDatabaseClient.test.ts +++ b/src/backend/clients/database/MySQLDatabaseClient.test.ts @@ -619,6 +619,21 @@ describe('MySQLDatabaseClient — tryHardRead', () => { { from: 'primary' }, ]); }); + + it('reads the primary once when the replica fails in a way read() fails over on', async () => { + const replica = { + execute: vi.fn().mockRejectedValue(codedError('dbBatchFailed')), + }; + const primary = { + execute: vi.fn().mockResolvedValue([[{ from: 'primary' }]]), + }; + const client = makeClient({ replica, primary }); + + await expect(client.tryHardRead('SELECT 1')).resolves.toEqual([ + { from: 'primary' }, + ]); + expect(primary.execute).toHaveBeenCalledTimes(1); + }); }); describe('MySQLDatabaseClient — migrations', () => { @@ -696,46 +711,6 @@ describe('MySQLDatabaseClient — migrations', () => { }); describe('MySQLDatabaseClient — pool lifecycle', () => { - it('replaces and closes the previous primary pool on reinit', async () => { - const client = await startClient(); - const original = createdPools[0]; - - client.reinitPrimary(); - - expect(createdPools).toHaveLength(2); - expect(original.ended).toBe(true); - // Single-node: the replica batcher must follow the new primary. - createdPools[1].respond = () => rowSets([{ from: 'new-primary' }]); - await expect(client.read('SELECT 1')).resolves.toEqual([ - { from: 'new-primary' }, - ]); - }); - - it('replaces only the replica pool on replica reinit', async () => { - const client = await startClient({ replica: { host: 'replica' } }); - const [primary, originalReplica] = createdPools; - - client.reinitReplica(); - - expect(createdPools).toHaveLength(3); - expect(originalReplica.ended).toBe(true); - expect(primary.ended).toBe(false); - }); - - it('ignores a replica reinit when no replica is configured', async () => { - const client = await startClient(); - client.reinitReplica(); - expect(createdPools).toHaveLength(1); - }); - - it('ignores reinit once shutdown has started', async () => { - const client = await startClient(); - await client.onServerPrepareShutdown(); - client.reinitPrimary(); - expect(createdPools).toHaveLength(1); - await client.onServerShutdown(); - }); - it('closes both pools on shutdown when a replica is configured', async () => { const client = await startClient({ replica: { host: 'replica' } }); await client.onServerShutdown(); @@ -779,108 +754,3 @@ describe('MySQLDatabaseClient — pool lifecycle', () => { await expect(client.onServerShutdown()).rejects.toThrow('end exploded'); }); }); - -describe('MySQLDatabaseClient.readWithRetry', () => { - const codedFailure = (code: string) => { - const err = new Error(code) as Error & { code: string }; - err.code = code; - return err; - }; - - it('classifies transient connection failures as retriable', () => { - expect( - MySQLDatabaseClient.isRetriableError(codedFailure('ECONNRESET')), - ).toBe(true); - expect( - MySQLDatabaseClient.isRetriableError(codedFailure('ER_DUP_ENTRY')), - ).toBe(false); - expect( - MySQLDatabaseClient.isRetriableError(new Error('Connection lost')), - ).toBe(true); - }); - - it('retries a transient failure and returns the eventual result', async () => { - const client = await startClient(); - let attempts = 0; - const operation = vi.fn(async () => { - attempts += 1; - if (attempts < 3) throw codedFailure('PROTOCOL_CONNECTION_LOST'); - return [{ ok: 1 }]; - }); - - await expect( - client.readWithRetry('health', operation, { baseBackoffMs: 0 }), - ).resolves.toEqual([{ ok: 1 }]); - expect(operation).toHaveBeenCalledTimes(3); - }); - - it('gives up after the attempt budget is spent', async () => { - const client = await startClient(); - const operation = vi.fn(async () => { - throw codedFailure('ETIMEDOUT'); - }); - - await expect( - client.readWithRetry('health', operation, { - maxAttempts: 2, - baseBackoffMs: 0, - }), - ).rejects.toMatchObject({ code: 'ETIMEDOUT' }); - expect(operation).toHaveBeenCalledTimes(2); - }); - - it('never retries a deterministic SQL error', async () => { - const client = await startClient(); - const operation = vi.fn(async () => { - throw codedFailure('ER_PARSE_ERROR'); - }); - - await expect( - client.readWithRetry('health', operation, { baseBackoffMs: 0 }), - ).rejects.toMatchObject({ code: 'ER_PARSE_ERROR' }); - expect(operation).toHaveBeenCalledTimes(1); - }); - - it('stops retrying once shutdown has started', async () => { - const client = await startClient(); - await client.onServerPrepareShutdown(); - const operation = vi.fn(async () => { - throw codedFailure('ECONNRESET'); - }); - - await expect( - client.readWithRetry('health', operation, { baseBackoffMs: 0 }), - ).rejects.toMatchObject({ code: 'ECONNRESET' }); - expect(operation).toHaveBeenCalledTimes(1); - await client.onServerShutdown(); - }); - - it('applies jittered exponential backoff capped at maxBackoffMs', async () => { - const client = await startClient(); - const delays: number[] = []; - const realSetTimeout = globalThis.setTimeout; - const timeoutSpy = vi - .spyOn(globalThis, 'setTimeout') - .mockImplementation(((fn: () => void, ms?: number) => { - delays.push(ms ?? 0); - return realSetTimeout(fn, 0); - }) as typeof setTimeout); - - try { - const operation = vi.fn(async () => { - throw codedFailure('ECONNRESET'); - }); - await expect( - client.readWithRetry('health', operation, { - maxAttempts: 4, - baseBackoffMs: 100, - maxBackoffMs: 150, - jitterRatio: 0, - }), - ).rejects.toMatchObject({ code: 'ECONNRESET' }); - expect(delays).toEqual([100, 150, 150]); - } finally { - timeoutSpy.mockRestore(); - } - }); -}); diff --git a/src/backend/clients/database/MySQLDatabaseClient.ts b/src/backend/clients/database/MySQLDatabaseClient.ts index f5946bb8b..6a5b9d934 100644 --- a/src/backend/clients/database/MySQLDatabaseClient.ts +++ b/src/backend/clients/database/MySQLDatabaseClient.ts @@ -17,8 +17,6 @@ * along with this program. If not, see . */ -import { readdirSync, readFileSync } from 'fs'; -import { isAbsolute, resolve as resolvePath } from 'path'; import { metrics } from '@opentelemetry/api'; import { createPool, ExecuteValues, type Pool } from 'mysql2'; import { Span } from '../../util/span.js'; @@ -26,7 +24,6 @@ import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient'; import { SQLBatcher } from './SQLBatcher.js'; import { isRetriableError } from './retriableErrors.js'; import { splitMysqlStatements } from './splitMysqlStatements.js'; -import { compareMigrationFilenames } from './migrationFilenames.js'; import type { IConfig } from '../../types'; const DEFAULT_SELECT_TIMEOUT_MS = 30_000; @@ -40,7 +37,7 @@ const replicaFailoverCounter = metrics 'Reads that failed on the replica batcher and were retried on the primary', }); -export { compareMigrationFilenames }; +export { compareMigrationFilenames } from './migrationFilenames.js'; type PoolConfig = Parameters[0]; @@ -95,15 +92,19 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient { this.dbReplica = this.createReplicaBatcher(this.replicaPool); - await this.runMigrations(); + await this.applyMigrationPaths( + () => this.primaryPool.promise().getConnection(), + splitMysqlStatements, + { transactional: false }, + ); } override async onServerPrepareShutdown(): Promise { if (this.shutdownStarted) return; this.shutdownStarted = true; - // Blocks reinitPrimary/reinitReplica from here on. Pools stay open; - // onServerShutdown closes them after the layers above have drained. + // Pools stay open; onServerShutdown closes them after the layers + // above have drained. console.log('[mysql] entering drain mode'); } @@ -200,101 +201,18 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient { } } - @Span('db.tryHardRead', (query: string) => ({ 'db.statement': query })) - override async tryHardRead( - query: string, - params: unknown[] = [], - ): Promise[]> { - if (this.configuration === Configuration.SINGLE) { - return this.read(query, params); - } - - // Run both reads in parallel — prefer replica when it returns rows, - // otherwise fall back to primary to handle replication lag. - const primaryPromise = this.dbPrimaryRead.execute(query, params); - try { - const replicaResult = await this.dbReplica.execute(query, params); - if ( - Array.isArray(replicaResult?.[0]) && - (replicaResult[0] as unknown[]).length > 0 - ) { - primaryPromise.catch(() => {}); // suppress unhandled rejection - return replicaResult[0] as Record[]; - } - } catch { - // fall through to primary - } - - const primaryResult = await primaryPromise; - return (primaryResult?.[0] as Record[]) ?? []; + protected override hasReadReplica(): boolean { + return this.configuration === Configuration.REPLICA; } - // ------------------------------------------------------------------ - // Migrations - // ------------------------------------------------------------------ - - /** - * Apply `.sql` files from each configured migration directory in order. - * Files within a directory are sorted lexically. Files MUST be idempotent — - * there is no per-file applied-state tracking. Failures abort startup so - * operators see schema problems loud. - */ - private async runMigrations(): Promise { - const paths = this.config.database?.migrationPaths; - if (!paths || paths.length === 0) return; - - const conn = await this.primaryPool.promise().getConnection(); - try { - for (const rawPath of paths) { - const dir = isAbsolute(rawPath) - ? rawPath - : resolvePath(process.cwd(), rawPath); - - let files: string[]; - try { - files = readdirSync(dir) - .filter( - (f) => f.endsWith('.sql') && f.startsWith('mysql'), - ) - .sort(compareMigrationFilenames); - } catch (e) { - throw new Error( - `[mysql] migration path is unreadable: ${dir}`, - { cause: e }, - ); - } - - if (files.length === 0) { - console.log(`[mysql] no migrations in ${dir}`); - continue; - } - - console.log( - `[mysql] running migrations from ${dir}: ${files.length} file(s)`, - ); - - for (const file of files) { - const filePath = resolvePath(dir, file); - const contents = readFileSync(filePath, 'utf8'); - const statements = splitMysqlStatements(contents); - for (let i = 0; i < statements.length; i++) { - try { - await conn.query(statements[i]); - } catch (e) { - throw new Error( - `[mysql] failed to apply ${file} at statement ${i}`, - { cause: e }, - ); - } - } - console.log( - `[mysql] applied ${file} (${statements.length} statements)`, - ); - } - } - } finally { - conn.release(); - } + // No failover: `tryHardRead` already has a primary read in flight. + protected override async replicaRead( + query: string, + params: unknown[], + ): Promise[]> { + const result = await this.dbReplica.execute(query, params); + if (!result) return []; + return (result[0] as Record[]) ?? []; } // ------------------------------------------------------------------ @@ -376,57 +294,6 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient { }); } - /** Reinitialize the primary pool (e.g. after a health-check failure). */ - reinitPrimary(): void { - if (this.shutdownStarted) return; - - const dbConf = this.config.database!; - const previous = this.primaryPool; - this.primaryPool = this.createPool({ - host: dbConf.host ?? '127.0.0.1', - port: dbConf.port ?? 3306, - user: dbConf.user ?? 'root', - password: dbConf.password ?? '', - database: dbConf.database ?? 'puter', - }); - this.db = this.createPrimaryBatcher(this.primaryPool); - this.dbPrimaryRead = this.createPrimaryReadBatcher(this.primaryPool); - - if (this.configuration === Configuration.SINGLE) { - this.replicaPool = this.primaryPool; - this.dbReplica = this.createReplicaBatcher(this.primaryPool); - } - - if (previous && previous !== this.primaryPool) { - this.closePool(previous, 'reinit:primary').catch(() => {}); - } - } - - /** Reinitialize the replica pool. */ - reinitReplica(): void { - if (this.shutdownStarted || !this.config.database?.replica) return; - - const previous = this.replicaPool; - this.replicaPool = this.createPool(this.config.database.replica); - this.dbReplica = this.createReplicaBatcher(this.replicaPool); - - if ( - previous && - previous !== this.replicaPool && - previous !== this.primaryPool - ) { - this.closePool(previous, 'reinit:replica').catch(() => {}); - } - } - - // ------------------------------------------------------------------ - // Retry helpers (for health checks or resilient reads) - // ------------------------------------------------------------------ - - static isRetriableError(error: unknown): boolean { - return isRetriableError(error); - } - /** * Replica failures worth retrying on the primary: batcher load-shed or * transient connection errors — never deterministic SQL errors. @@ -436,52 +303,6 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient { return code === 'dbBatchFailed' || isRetriableError(error); } - async readWithRetry( - label: string, - operation: () => Promise, - opts?: { - maxAttempts?: number; - baseBackoffMs?: number; - maxBackoffMs?: number; - jitterRatio?: number; - }, - ): Promise { - const maxAttempts = opts?.maxAttempts ?? 3; - const baseBackoffMs = opts?.baseBackoffMs ?? 100; - const maxBackoffMs = opts?.maxBackoffMs ?? 500; - const jitterRatio = opts?.jitterRatio ?? 0.2; - - let attempt = 1; - - while (true) { - try { - return await operation(); - } catch (error) { - if (this.shutdownStarted) throw error; - if ( - attempt >= maxAttempts || - !MySQLDatabaseClient.isRetriableError(error) - ) - throw error; - - const raw = baseBackoffMs * 2 ** (attempt - 1); - const capped = Math.min(maxBackoffMs, raw); - const window = Math.round(capped * jitterRatio); - const jitter = - window === 0 - ? 0 - : Math.floor(Math.random() * (window * 2 + 1)) - window; - const delay = Math.max(0, capped + jitter); - - console.warn( - `[${label}] transient mysql error (${(error as { code?: string })?.code ?? 'unknown'}); retry ${attempt + 1}/${maxAttempts} in ${delay}ms`, - ); - await new Promise((r) => setTimeout(r, delay)); - attempt++; - } - } - } - // ------------------------------------------------------------------ // Internal pool lifecycle // ------------------------------------------------------------------ diff --git a/src/backend/clients/database/PostgresDatabaseClient.test.ts b/src/backend/clients/database/PostgresDatabaseClient.test.ts index b63e0fc53..380a47691 100644 --- a/src/backend/clients/database/PostgresDatabaseClient.test.ts +++ b/src/backend/clients/database/PostgresDatabaseClient.test.ts @@ -28,6 +28,20 @@ import { type PostgresPoolClient, } from './PostgresDatabaseClient.js'; +// Pass-through spy: the rewrite is pure, so its call count shows the memo. +const { rewriteSpy } = vi.hoisted(() => ({ rewriteSpy: vi.fn() })); +vi.mock('./preparePostgresSql.js', async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + preparePostgresSql: (sql: string) => { + rewriteSpy(sql); + return actual.preparePostgresSql(sql); + }, + }; +}); + type QueryCall = { text: string; values?: unknown[]; @@ -167,6 +181,23 @@ describe('PostgresDatabaseClient', () => { }); }); + it('rewrites a repeated query once', async () => { + const pool = new RecordingPool(); + const client = new PostgresDatabaseClient(postgresConfig(), () => pool); + await client.onServerStart(); + const query = 'SELECT `name` FROM `apps` WHERE `id` = ?'; + rewriteSpy.mockClear(); + + await client.read(query, [1]); + await client.pread(query, [2]); + await client.write(query, [3]); + + expect(rewriteSpy).toHaveBeenCalledTimes(1); + expect(pool.calls.slice(-3).map((c) => c.text)).toEqual( + Array(3).fill('SELECT "name" FROM "apps" WHERE "id" = $1'), + ); + }); + it('normalizes int8 fields on read and primary read rows', async () => { const pool = new RecordingPool( new RecordingPoolClient(), diff --git a/src/backend/clients/database/PostgresDatabaseClient.ts b/src/backend/clients/database/PostgresDatabaseClient.ts index 5ffa688d8..235db418c 100644 --- a/src/backend/clients/database/PostgresDatabaseClient.ts +++ b/src/backend/clients/database/PostgresDatabaseClient.ts @@ -17,16 +17,14 @@ * along with this program. If not, see . */ -import { readdirSync, readFileSync } from 'fs'; -import { isAbsolute, resolve as resolvePath } from 'path'; import { Pool, type PoolConfig, type QueryResult } from 'pg'; +import { BoundedTtlMap } from '../../util/boundedTtlMap.js'; import { Span } from '../../util/span.js'; import { AbstractDatabaseClient, type BatchEntry, type WriteResult, } from './DatabaseClient'; -import { compareMigrationFilenames } from './migrationFilenames.js'; import { preparePostgresSql } from './preparePostgresSql.js'; import { splitPostgresStatements } from './splitPostgresStatements.js'; import type { IConfig } from '../../types'; @@ -62,6 +60,7 @@ enum Configuration { } const POSTGRES_INT8_OID = 20; +const PREPARED_SQL_CACHE_SIZE = 1000; const INTEGER_TEXT_PATTERN = /^-?\d+$/u; const normalizePostgresInt8 = ( @@ -138,6 +137,10 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient { private replicaPool!: PostgresPool; private configuration = Configuration.SINGLE; private shutdownStarted = false; + /** Query text as written → its Postgres rewrite, which is pure. */ + private readonly preparedSql = new BoundedTtlMap({ + maxEntries: PREPARED_SQL_CACHE_SIZE, + }); constructor( config: IConfig, @@ -164,7 +167,11 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient { this.configuration = Configuration.SINGLE; } - await this.runMigrations(); + await this.applyMigrationPaths( + () => this.primaryPool.connect(), + splitPostgresStatements, + { transactional: true }, + ); } override async onServerPrepareShutdown(): Promise { @@ -248,98 +255,8 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient { } } - @Span('db.tryHardRead', (query: string) => ({ 'db.statement': query })) - override async tryHardRead( - query: string, - params: unknown[] = [], - ): Promise[]> { - if (this.configuration === Configuration.SINGLE) { - return this.read(query, params); - } - - const primaryPromise = this.query(this.primaryPool, query, params); - try { - const replicaResult = await this.query( - this.replicaPool, - query, - params, - ); - if (replicaResult.rows.length > 0) { - primaryPromise.catch(() => {}); - return normalizePostgresRows(replicaResult); - } - } catch { - // fall through to primary - } - - const primaryResult = await primaryPromise; - return normalizePostgresRows(primaryResult); - } - - private async runMigrations(): Promise { - const paths = this.config.database?.migrationPaths; - if (!paths || paths.length === 0) return; - - const conn = await this.primaryPool.connect(); - try { - for (const rawPath of paths) { - const dir = isAbsolute(rawPath) - ? rawPath - : resolvePath(process.cwd(), rawPath); - - let files: string[]; - try { - files = readdirSync(dir) - .filter( - (f) => - f.endsWith('.sql') && f.startsWith('postgres'), - ) - .sort(compareMigrationFilenames); - } catch (e) { - throw new Error( - `[postgres] migration path is unreadable: ${dir}`, - { cause: e }, - ); - } - - if (files.length === 0) { - console.log(`[postgres] no migrations in ${dir}`); - continue; - } - - console.log( - `[postgres] running migrations from ${dir}: ${files.length} file(s)`, - ); - - for (const file of files) { - const filePath = resolvePath(dir, file); - const contents = readFileSync(filePath, 'utf8'); - const statements = splitPostgresStatements(contents); - await conn.query('BEGIN'); - try { - for (let i = 0; i < statements.length; i++) { - try { - await conn.query(statements[i]); - } catch (e) { - throw new Error( - `[postgres] failed to apply ${file} at statement ${i}`, - { cause: e }, - ); - } - } - await conn.query('COMMIT'); - } catch (e) { - await conn.query('ROLLBACK').catch(() => {}); - throw e; - } - console.log( - `[postgres] applied ${file} (${statements.length} statements)`, - ); - } - } - } finally { - conn.release(); - } + protected override hasReadReplica(): boolean { + return this.configuration === Configuration.REPLICA; } private createPool(dbConf: PostgresEndpointConfig): PostgresPool { @@ -366,8 +283,12 @@ export class PostgresDatabaseClient extends AbstractDatabaseClient { query: string, params: unknown[] = [], ): Promise { - const prepared = preparePostgresSql(query); - return target.query(prepared.text, params); + let text = this.preparedSql.get(query); + if (text === undefined) { + text = preparePostgresSql(query).text; + this.preparedSql.set(query, text); + } + return target.query(text, params); } private async closeCurrentPools(): Promise { diff --git a/src/backend/clients/database/SqliteDatabaseClient.test.ts b/src/backend/clients/database/SqliteDatabaseClient.test.ts index 5cc6657a8..3a6a85765 100644 --- a/src/backend/clients/database/SqliteDatabaseClient.test.ts +++ b/src/backend/clients/database/SqliteDatabaseClient.test.ts @@ -1038,8 +1038,7 @@ describe('SqliteDatabaseClient — query interface', () => { }); // Single-node engines have no replica to race, so `tryHardRead` must not -// issue the query twice — the base-class default fires `pread()` and -// `read()` in parallel, which on sqlite is the same connection. +// issue the query twice on the one connection. describe('SqliteDatabaseClient — tryHardRead', () => { let client: SqliteDatabaseClient; @@ -1078,20 +1077,50 @@ describe('SqliteDatabaseClient — tryHardRead', () => { expect(readSpy).toHaveBeenCalledTimes(1); expect(preadSpy).not.toHaveBeenCalled(); }); +}); - it('throws from requireRead when nothing matches', async () => { - await expect( - client.requireRead( - 'SELECT `name` FROM `widget` WHERE `id` = ?', - [42], - ), - ).rejects.toThrow('required read returned no rows'); +describe('SqliteDatabaseClient — prepared statements', () => { + let client: SqliteDatabaseClient; + let prepare: ReturnType; + + beforeEach(async () => { + client = await bootClient(); + await client.write( + 'CREATE TABLE `widget` (`id` INTEGER PRIMARY KEY, `name` TEXT)', + ); + // better-sqlite3 is the driver boundary; count compilations there. + const db = (client as unknown as { db: Database.Database }).db; + prepare = vi.spyOn(db, 'prepare'); }); - it('returns the rows from requireRead when something matches', async () => { - await client.write('INSERT INTO `widget` (`name`) VALUES (?)', ['ok']); - await expect( - client.requireRead('SELECT `name` FROM `widget`'), - ).resolves.toEqual([{ name: 'ok' }]); + afterEach(() => { + client.onServerShutdown(); + }); + + it('compiles a repeated query once', async () => { + const insert = 'INSERT INTO `widget` (`name`) VALUES (?)'; + await client.write(insert, ['a']); + await client.write(insert, ['b']); + await client.batchWrite([{ statement: insert, values: ['c'] }]); + const select = 'SELECT `name` FROM `widget` ORDER BY `id`'; + await client.read(select); + + await expect(client.read(select)).resolves.toEqual([ + { name: 'a' }, + { name: 'b' }, + { name: 'c' }, + ]); + expect(prepare).toHaveBeenCalledTimes(2); + }); + + it('keeps serving a cached query after the schema changes', async () => { + const select = 'SELECT * FROM `widget`'; + await client.write('INSERT INTO `widget` (`name`) VALUES (?)', ['a']); + await client.read(select); + await client.write('ALTER TABLE `widget` ADD COLUMN `size` INTEGER'); + + await expect(client.read(select)).resolves.toEqual([ + { id: 1, name: 'a', size: null }, + ]); }); }); diff --git a/src/backend/clients/database/SqliteDatabaseClient.ts b/src/backend/clients/database/SqliteDatabaseClient.ts index 71dc44ace..d0b5c7a3a 100644 --- a/src/backend/clients/database/SqliteDatabaseClient.ts +++ b/src/backend/clients/database/SqliteDatabaseClient.ts @@ -21,10 +21,15 @@ import { existsSync, mkdirSync, readFileSync } from 'fs'; import { basename, dirname, extname, join, resolve } from 'path'; import { createContext, runInContext } from 'vm'; import type { IConfig } from '../../types'; +import { BoundedTtlMap } from '../../util/boundedTtlMap.js'; import { Span } from '../../util/span.js'; import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient'; const MIGRATIONS_DIR = resolve(__dirname, './migrations/sqlite'); +const STATEMENT_CACHE_SIZE = 1000; + +type SqliteDatabase = InstanceType; +type SqliteStatement = import('better-sqlite3').Statement; /** * Ordered list of [threshold_version, files[]] pairs. A database whose @@ -128,7 +133,11 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient { override readonly engineName = 'sqlite'; // better-sqlite3 instance — set during onServerStart - private db!: InstanceType; + private db!: SqliteDatabase; + /** Prepared statements by query text; SQLite re-prepares on schema change. */ + private readonly statements = new BoundedTtlMap({ + maxEntries: STATEMENT_CACHE_SIZE, + }); constructor(config: IConfig) { super(config); @@ -150,12 +159,14 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient { mkdirSync(dirname(dbPath), { recursive: true }); } + this.statements.clear(); this.db = new Database(dbPath); await this.runMigrations(isNew); } override onServerShutdown(): void { + this.statements.clear(); if (this.db) { this.db.close(); } @@ -170,12 +181,9 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient { query: string, params: unknown[] = [], ): Promise[]> { - query = this.transformQuery(query); - params = this.transformParams(params); - return this.db.prepare(query).all(...params) as Record< - string, - unknown - >[]; + return this.prepare(query).all( + ...this.transformParams(params), + ) as Record[]; } override async pread( @@ -186,25 +194,12 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient { return this.read(query, params); } - override async tryHardRead( - query: string, - params: unknown[] = [], - ): Promise[]> { - // No replica to race, so the base class's parallel - // primary-plus-replica read would run the same statement twice on - // the one connection. - return this.read(query, params); - } - @Span('db.write', (query: string) => ({ 'db.statement': query })) override async write( query: string, params: unknown[] = [], ): Promise { - query = this.transformQuery(query); - params = this.transformParams(params); - - const info = this.db.prepare(query).run(...params); + const info = this.prepare(query).run(...this.transformParams(params)); return { insertId: info.lastInsertRowid, @@ -220,10 +215,8 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient { entries: { statement: string; values: unknown[] }[], ): Promise { this.db.transaction(() => { - for (let { statement, values } of entries) { - statement = this.transformQuery(statement); - values = this.transformParams(values); - this.db.prepare(statement).run(...values); + for (const { statement, values } of entries) { + this.prepare(statement).run(...this.transformParams(values)); } })(); } @@ -232,6 +225,15 @@ export class SqliteDatabaseClient extends AbstractDatabaseClient { // SQLite-specific transforms // ------------------------------------------------------------------ + private prepare(query: string): SqliteStatement { + let statement = this.statements.get(query); + if (!statement) { + statement = this.db.prepare(this.transformQuery(query)); + this.statements.set(query, statement); + } + return statement; + } + private transformQuery(query: string): string { return query.replace(/now\(\)/gi, "datetime('now')"); } diff --git a/src/backend/clients/dynamodb/DDBClient.test.ts b/src/backend/clients/dynamodb/DDBClient.test.ts index c9a672f3f..c1c522b59 100644 --- a/src/backend/clients/dynamodb/DDBClient.test.ts +++ b/src/backend/clients/dynamodb/DDBClient.test.ts @@ -774,6 +774,178 @@ describe('DDBClient — unprocessed batch keys', () => { }); }); +// Answers by operation (X-Amz-Target) and records how many requests overlap, so +// chunk concurrency and the TTL sweep's retries can be asserted. +describe('DDBClient — batch write chunks and TTL sweep', () => { + let server: Server; + let endpoint: string; + let handlers: Record) => unknown>; + let calls: string[]; + let inFlight = 0; + let peakInFlight = 0; + + beforeAll(async () => { + server = createServer((req, res) => { + let body = ''; + req.on('data', (chunk) => { + body += chunk; + }); + req.on('end', () => { + const op = String(req.headers['x-amz-target']).split('.')[1]; + calls.push(op); + inFlight += 1; + peakInFlight = Math.max(peakInFlight, inFlight); + setTimeout(() => { + inFlight -= 1; + const reply = handlers[op]?.(JSON.parse(body)) ?? {}; + const { status = 200, ...payload } = reply as { + status?: number; + }; + res.writeHead(status, { + 'content-type': 'application/x-amz-json-1.0', + }); + res.end(JSON.stringify(payload)); + }, 20); + }); + }); + await new Promise((resolve) => + server.listen(0, '127.0.0.1', resolve), + ); + endpoint = `http://127.0.0.1:${(server.address() as AddressInfo).port}`; + }); + + afterAll(async () => { + await new Promise((resolve) => server.close(() => resolve())); + }); + + beforeEach(() => { + calls = []; + inFlight = 0; + peakInFlight = 0; + }); + + const stubClient = (extra: Record = {}) => + new DDBClient({ + port: 0, + extensions: [], + dynamo: { + aws: { access_key: 'a', secret_key: 'b', region: 'us-west-2' }, + endpoint, + ...extra, + }, + } as unknown as IConfig); + + it('writes chunks concurrently, a bounded number at a time', async () => { + handlers = { BatchWriteItem: () => ({ UnprocessedItems: {} }) }; + + await stubClient().batchPut( + Array.from({ length: 200 }, (_, i) => ({ + table: TABLE, + item: { pk: `item-${i}` }, + })), + ); + + expect(calls).toHaveLength(8); + expect(peakInFlight).toBeGreaterThan(1); + expect(peakInFlight).toBeLessThanOrEqual(4); + }); + + it('stops starting chunks once one fails', async () => { + handlers = { + BatchWriteItem: () => ({ + status: 400, + __type: 'com.amazon.coral.validate#ValidationException', + message: 'bad item', + }), + }; + + await expect( + stubClient().batchPut( + Array.from({ length: 250 }, (_, i) => ({ + table: TABLE, + item: { pk: `item-${i}` }, + })), + ), + ).rejects.toThrow('bad item'); + // The first wave fails; none of the six chunks queued behind it run. + expect(calls.length).toBeLessThanOrEqual(4); + }); + + it('lets chunks already in flight finish before rejecting', async () => { + let replies = 0; + handlers = { + BatchWriteItem: (body) => { + replies += 1; + if (replies === 1) { + return { + status: 400, + __type: 'com.amazon.coral.validate#ValidationException', + message: 'bad item', + }; + } + // The rest need one retry, so they are still writing when the + // failure comes back. + return replies <= 4 + ? { UnprocessedItems: body.RequestItems } + : { UnprocessedItems: {} }; + }, + }; + + await expect( + stubClient().batchPut( + Array.from({ length: 100 }, (_, i) => ({ + table: TABLE, + item: { pk: `item-${i}` }, + })), + ), + ).rejects.toThrow('bad item'); + const sentBeforeReject = calls.length; + await new Promise((resolve) => setTimeout(resolve, 200)); + + // Nothing is written after the caller has seen the error. + expect(calls.length).toBe(sentBeforeReject); + expect(inFlight).toBe(0); + }); + + it('retries the unprocessed deletes of the TTL sweep', async () => { + const expired = [{ pk: { S: 'old' } }]; + let batchWrites = 0; + handlers = { + CreateTable: () => ({ + status: 400, + __type: 'com.amazonaws.dynamodb.v20120810#ResourceInUseException', + message: 'Table already exists', + }), + Scan: () => ({ Items: expired }), + BatchWriteItem: (body) => { + batchWrites += 1; + return batchWrites === 1 + ? { UnprocessedItems: body.RequestItems } + : { UnprocessedItems: {} }; + }, + }; + + await stubClient({ bootstrapTables: true }).createTableIfNotExists( + { + TableName: TABLE, + KeySchema: [{ AttributeName: 'pk', KeyType: 'HASH' }], + AttributeDefinitions: [ + { AttributeName: 'pk', AttributeType: 'S' }, + ], + BillingMode: 'PAY_PER_REQUEST', + }, + 'ttl', + ); + + expect(calls).toEqual([ + 'CreateTable', + 'Scan', + 'BatchWriteItem', + 'BatchWriteItem', + ]); + }); +}); + // Throttles are a service-side signal the emulator never produces, so these // drive the client against a stub answering in DynamoDB's error shape. describe('DDBClient — throttle logging', () => { diff --git a/src/backend/clients/dynamodb/DDBClient.ts b/src/backend/clients/dynamodb/DDBClient.ts index feabff3ad..50e3cad7b 100644 --- a/src/backend/clients/dynamodb/DDBClient.ts +++ b/src/backend/clients/dynamodb/DDBClient.ts @@ -43,6 +43,8 @@ import { once } from 'node:events'; import { Agent as httpsAgent } from 'node:https'; import { HttpError } from '../../core/http'; import type { IConfig, IDynamoConfig } from '../../types'; +import { BoundedTtlMap } from '../../util/boundedTtlMap.js'; +import { runWithConcurrencyLimitSettled } from '../../util/concurrency.js'; import { Span } from '../../util/span.js'; import { PuterClient } from '../types'; import { @@ -55,10 +57,10 @@ import { attachThrottleLogging } from './throttleLog.js'; const LOCAL_DYNAMO_MEMORY_PREFIX = ':memory:'; const localDynaliteEndpointPromises = new Map>(); const MAX_BATCH_WRITE_ITEMS = 25; -const MAX_BATCH_WRITE_RETRIES = 8; -const BATCH_WRITE_RETRY_BASE_MS = 25; -const MAX_BATCH_GET_RETRIES = 8; -const BATCH_GET_RETRY_BASE_MS = 25; +// Chunks of one batch write in flight at once. +const BATCH_WRITE_CONCURRENCY = 4; +const MAX_BATCH_RETRIES = 8; +const BATCH_RETRY_BASE_MS = 25; // In-memory mode gives each DDBClient its own unique key, which keys // into a fresh dynalite server. This is what test parallelism needs: @@ -118,21 +120,45 @@ const chunkValues = (values: T[], size: number): T[][] => { return chunks; }; +type BatchWriteItems = NonNullable; +type WriteRequest = BatchWriteItems[string][number]; + +/** Consumed capacity summed per table across the requests of one batch. */ +class ConsumedCapacityTally { + #byTable = new Map(); + + add(entries: { TableName?: string; CapacityUnits?: number }[] = []) { + for (const { TableName, CapacityUnits } of entries) { + if (!TableName) continue; + this.#byTable.set( + TableName, + (this.#byTable.get(TableName) ?? 0) + + Number(CapacityUnits ?? 0), + ); + } + } + + entries() { + return Array.from(this.#byTable, ([TableName, CapacityUnits]) => ({ + TableName, + CapacityUnits, + })); + } +} + const sleep = async (ms: number) => { await new Promise((resolve) => setTimeout(resolve, ms)); }; // One caller writing an out-of-range number usually keeps doing it, so the // warning is per-target and throttled rather than one line per write. -const REPAIR_WARNING_INTERVAL_MS = 60_000; -const lastRepairWarningAt = new Map(); +const repairWarnings = new BoundedTtlMap({ + maxEntries: 1_000, + ttlMs: 60_000, +}); const warnRepaired = (target: string, message: string): void => { - const now = Date.now(); - const lastWarnedAt = lastRepairWarningAt.get(target) ?? 0; - if (now - lastWarnedAt < REPAIR_WARNING_INTERVAL_MS) return; - - lastRepairWarningAt.set(target, now); + if (!repairWarnings.shouldEmit(target)) return; console.warn( `[ddb] clamped an out-of-range value in ${target}: ${message}`, ); @@ -257,9 +283,7 @@ export class DDBClient extends PuterClient { {} as Record[]>, ); - let requestItems: BatchGetCommandInput['RequestItems'] = Object.entries( - allRequestItemsPerTable, - ).reduce( + const requestItems = Object.entries(allRequestItemsPerTable).reduce( (acc, [table, keyList]) => { acc[table] = { Keys: keyList, @@ -272,53 +296,30 @@ export class DDBClient extends PuterClient { const client = await this.#getDocumentClient(); const responsesByTable = new Map[]>(); - const consumedCapacityByTable = new Map(); + const capacity = new ConsumedCapacityTally(); - for (let attempt = 0; attempt <= MAX_BATCH_GET_RETRIES; attempt++) { - if (Object.keys(requestItems).length === 0) break; - - const response = await client.send( - new BatchGetCommand({ - RequestItems: requestItems, - ReturnConsumedCapacity: 'TOTAL', - }), - ); - - for (const [table, items] of Object.entries( - response.Responses ?? {}, - )) { - const existing = responsesByTable.get(table) ?? []; - existing.push(...items); - responsesByTable.set(table, existing); - } - for (const entry of response.ConsumedCapacity ?? []) { - if (!entry.TableName) continue; - consumedCapacityByTable.set( - entry.TableName, - (consumedCapacityByTable.get(entry.TableName) ?? 0) + - Number(entry.CapacityUnits ?? 0), + const drained = await this.#sendWithUnprocessedRetry( + requestItems, + async (pending) => { + const response = await client.send( + new BatchGetCommand({ + RequestItems: pending, + ReturnConsumedCapacity: 'TOTAL', + }), ); - } + for (const [table, items] of Object.entries( + response.Responses ?? {}, + )) { + const existing = responsesByTable.get(table) ?? []; + existing.push(...items); + responsesByTable.set(table, existing); + } + capacity.add(response.ConsumedCapacity); + return response.UnprocessedKeys as typeof pending | undefined; + }, + ); - const unprocessedKeys = response.UnprocessedKeys ?? {}; - if (Object.keys(unprocessedKeys).length === 0) { - requestItems = {}; - break; - } - - requestItems = unprocessedKeys as NonNullable< - BatchGetCommandInput['RequestItems'] - >; - if (attempt < MAX_BATCH_GET_RETRIES) { - const delayMs = Math.min( - 1000, - BATCH_GET_RETRY_BASE_MS * 2 ** attempt, - ); - await sleep(delayMs); - } - } - - if (Object.keys(requestItems).length > 0) { + if (!drained) { throw new HttpError( 400, 'Failed to batch get all items from DynamoDB', @@ -328,12 +329,7 @@ export class DDBClient extends PuterClient { return { Responses: Object.fromEntries(responsesByTable), - ConsumedCapacity: Array.from(consumedCapacityByTable.entries()).map( - ([TableName, CapacityUnits]) => ({ - TableName, - CapacityUnits, - }), - ), + ConsumedCapacity: capacity.entries(), }; } @@ -361,122 +357,96 @@ export class DDBClient extends PuterClient { ); } - // Shared BatchWriteItem plumbing for batchPut/batchDel: 25-item chunks, - // UnprocessedItems retried with capped exponential backoff, consumed - // capacity accumulated per table across every request. - async #batchWrite( - params: { - table: string; - request: NonNullable< - BatchWriteCommandInput['RequestItems'] - >[string][number]; - }[], - ) { - const consumedCapacityByTable = new Map(); + // Shared BatchWriteItem plumbing for batchPut/batchDel and the TTL sweep: + // 25-item chunks a few at a time, UnprocessedItems retried, consumed + // capacity summed per table across every request. + async #batchWrite(params: { table: string; request: WriteRequest }[]) { if (params.length === 0) { return { ConsumedCapacity: [] }; } - const accumulateConsumedCapacity = ( - consumedCapacityEntries: - | Array<{ TableName?: string; CapacityUnits?: number }> - | undefined, - ) => { - if (!consumedCapacityEntries) { - return; - } - - for (const consumedCapacityEntry of consumedCapacityEntries) { - const table = consumedCapacityEntry.TableName; - if (!table) { - continue; - } - - const existingUsage = consumedCapacityByTable.get(table) ?? 0; - consumedCapacityByTable.set( - table, - existingUsage + - Number(consumedCapacityEntry.CapacityUnits ?? 0), - ); - } - }; - const client = await this.#getDocumentClient(); - const chunks = chunkValues(params, MAX_BATCH_WRITE_ITEMS); + const capacity = new ConsumedCapacityTally(); + // Once a chunk fails, the rest aren't started. Those in flight settle + // before the error is thrown, so no write lands after the caller's + // cleanup. + let failed = false; - for (const chunk of chunks) { - let requestItems = chunk.reduce( - (acc, curr) => { - const tableRequests = acc[curr.table] ?? []; - tableRequests.push(curr.request); - acc[curr.table] = tableRequests; - return acc; - }, - {} as NonNullable, - ); - - for ( - let attempt = 0; - attempt <= MAX_BATCH_WRITE_RETRIES; - attempt++ - ) { - if (Object.keys(requestItems).length === 0) { - break; + const outcomes = await runWithConcurrencyLimitSettled( + chunkValues(params, MAX_BATCH_WRITE_ITEMS), + BATCH_WRITE_CONCURRENCY, + async (chunk) => { + if (failed) return; + const requestItems: BatchWriteItems = {}; + for (const { table, request } of chunk) { + (requestItems[table] ??= []).push(request); } - - const response = await sendRepairingValues( - requestItems, - (itemsToWrite) => - client.send( - new BatchWriteCommand({ - RequestItems: itemsToWrite, - ReturnConsumedCapacity: 'TOTAL', - }), - ), - () => - `a batch write to ${Object.keys(requestItems).join(', ')}`, - ); - accumulateConsumedCapacity( - response.ConsumedCapacity as - | Array<{ TableName?: string; CapacityUnits?: number }> - | undefined, - ); - - const unprocessedItems = response.UnprocessedItems ?? {}; - if (Object.keys(unprocessedItems).length === 0) { - requestItems = {}; - break; - } - - requestItems = unprocessedItems as NonNullable< - BatchWriteCommandInput['RequestItems'] - >; - if (attempt < MAX_BATCH_WRITE_RETRIES) { - const delayMs = Math.min( - 1000, - BATCH_WRITE_RETRY_BASE_MS * 2 ** attempt, + try { + const drained = await this.#sendWithUnprocessedRetry( + requestItems, + async (pending) => { + const response = await sendRepairingValues( + pending, + (itemsToWrite) => + client.send( + new BatchWriteCommand({ + RequestItems: itemsToWrite, + ReturnConsumedCapacity: 'TOTAL', + }), + ), + () => + `a batch write to ${Object.keys(pending).join(', ')}`, + ); + capacity.add(response.ConsumedCapacity); + return response.UnprocessedItems as + | BatchWriteItems + | undefined; + }, ); - await sleep(delayMs); + if (!drained) { + throw new HttpError( + 400, + 'Failed to batch write all items to DynamoDB', + { legacyCode: 'bad_request' }, + ); + } + } catch (error) { + failed = true; + throw error; } - } + }, + ); + const rejected = outcomes.find( + (outcome): outcome is PromiseRejectedResult => + outcome.status === 'rejected', + ); + if (rejected) throw rejected.reason; - if (Object.keys(requestItems).length > 0) { - throw new HttpError( - 400, - 'Failed to batch write all items to DynamoDB', - { legacyCode: 'bad_request' }, - ); + return { ConsumedCapacity: capacity.entries() }; + } + + /** + * Send a batch request and resend whatever comes back unprocessed, with + * capped exponential backoff. `send` returns the unprocessed remainder. + * Resolves false when items are still left after the last retry. + */ + async #sendWithUnprocessedRetry( + requestItems: TItems, + send: (pending: TItems) => Promise, + ): Promise { + let pending: TItems | undefined = requestItems; + for (let attempt = 0; attempt <= MAX_BATCH_RETRIES; attempt++) { + if (!pending || Object.keys(pending).length === 0) return true; + pending = await send(pending); + if ( + attempt < MAX_BATCH_RETRIES && + pending && + Object.keys(pending).length > 0 + ) { + await sleep(Math.min(1000, BATCH_RETRY_BASE_MS * 2 ** attempt)); } } - - return { - ConsumedCapacity: Array.from(consumedCapacityByTable.entries()).map( - ([TableName, CapacityUnits]) => ({ - TableName, - CapacityUnits, - }), - ), - }; + return !pending || Object.keys(pending).length === 0; } @Span('ddb.del', (table: string) => ({ 'db.table': table })) @@ -728,34 +698,10 @@ export class DDBClient extends PuterClient { ); } - const region = this.#ddbConfig.aws?.region || 'us-west-2'; - const ddbClient = new DynamoDBClient({ - credentials: { - accessKeyId, - secretAccessKey, - }, - maxAttempts: 3, - requestHandler: new NodeHttpHandler({ - connectionTimeout: 5000, - requestTimeout: 5000, - httpsAgent: new httpsAgent({ keepAlive: true }), - }), - ...(this.#ddbConfig.endpoint - ? { endpoint: this.#ddbConfig.endpoint } - : {}), - region, - }); - attachThrottleLogging(ddbClient, region, (table) => - this.#keyNamesByTable.get(table), - ); - - this.#documentClient = DynamoDBDocumentClient.from(ddbClient, { - marshallOptions: { - removeUndefinedValues: true, - }, - unmarshallOptions: { - wrapNumbers: clampStoredNumber, - }, + this.#bindClient({ + credentials: { accessKeyId, secretAccessKey }, + region: this.#ddbConfig.aws?.region || 'us-west-2', + endpoint: this.#ddbConfig.endpoint, }); } @@ -763,21 +709,36 @@ export class DDBClient extends PuterClient { const endpoint = await getOrCreateLocalDynaliteEndpoint( this.#localPathKey, ); + this.#bindClient({ + credentials: { accessKeyId: 'fake', secretAccessKey: 'fake' }, + region: 'us-west-2', + endpoint, + }); + } + #bindClient({ + credentials, + region, + endpoint, + }: { + credentials: { accessKeyId: string; secretAccessKey: string }; + region: string; + endpoint?: string; + }) { const ddbClient = new DynamoDBClient({ - credentials: { - accessKeyId: 'fake', - secretAccessKey: 'fake', - }, + credentials, maxAttempts: 3, requestHandler: new NodeHttpHandler({ connectionTimeout: 5000, requestTimeout: 5000, httpsAgent: new httpsAgent({ keepAlive: true }), }), - endpoint, - region: 'us-west-2', + ...(endpoint ? { endpoint } : {}), + region, }); + attachThrottleLogging(ddbClient, region, (table) => + this.#keyNamesByTable.get(table), + ); this.#documentClient = DynamoDBDocumentClient.from(ddbClient, { marshallOptions: { @@ -823,26 +784,23 @@ export class DDBClient extends PuterClient { ); lastEvaluatedKey = scan.LastEvaluatedKey as - Record | undefined; + | Record + | undefined; const items = scan.Items; if (!items || items.length === 0) continue; - const chunks = chunkValues(items, MAX_BATCH_WRITE_ITEMS); - for (const chunk of chunks) { - await client.send( - new BatchWriteCommand({ - RequestItems: { - [table]: chunk.map((item) => ({ - DeleteRequest: { - Key: Object.fromEntries( - keyNames.map((key) => [key, item[key]]), - ), - }, - })), + await this.#batchWrite( + items.map((item) => ({ + table, + request: { + DeleteRequest: { + Key: Object.fromEntries( + keyNames.map((key) => [key, item[key]]), + ), }, - }), - ); - } + }, + })), + ); } while (lastEvaluatedKey); } } diff --git a/src/backend/clients/dynamodb/throttleLog.ts b/src/backend/clients/dynamodb/throttleLog.ts index eee6bde07..d93b04b56 100644 --- a/src/backend/clients/dynamodb/throttleLog.ts +++ b/src/backend/clients/dynamodb/throttleLog.ts @@ -18,6 +18,7 @@ */ import type { DynamoDBClient } from '@aws-sdk/client-dynamodb'; +import { BoundedTtlMap } from '../../util/boundedTtlMap.js'; const THROTTLE_ERROR_NAMES = new Set([ 'ProvisionedThroughputExceededException', @@ -39,7 +40,10 @@ interface Target { keys: string; } -const lastLogged = new Map(); +// No TTL: an entry past its interval still carries the count to report. +const lastLogged = new BoundedTtlMap({ + maxEntries: MAX_TRACKED_TARGETS, +}); const logOnce = (target: string, detail?: string): void => { const now = Date.now(); @@ -48,13 +52,6 @@ const logOnce = (target: string, detail?: string): void => { previous.folded += 1; return; } - - if (!previous && lastLogged.size >= MAX_TRACKED_TARGETS) { - for (const [key, entry] of lastLogged) { - if (now - entry.at >= LOG_INTERVAL_MS) lastLogged.delete(key); - } - if (lastLogged.size >= MAX_TRACKED_TARGETS) lastLogged.clear(); - } lastLogged.set(target, { at: now, folded: 0 }); const folded = previous?.folded diff --git a/src/backend/clients/event/EventClient.test.ts b/src/backend/clients/event/EventClient.test.ts index 4e09043ab..81f6d619a 100644 --- a/src/backend/clients/event/EventClient.test.ts +++ b/src/backend/clients/event/EventClient.test.ts @@ -136,6 +136,30 @@ describe('EventClient', () => { }); }); + describe('repeat emits', () => { + it('reaches listeners added after the key was first emitted', () => { + target.emit(`${key}.a.b`, {}, {}); + const wildcard = vi.fn(); + const exact = vi.fn(); + target.on(`${key}.a.*`, wildcard); + target.on(`${key}.a.b`, exact); + target.emit(`${key}.a.b`, {}, {}); + expect(wildcard).toHaveBeenCalledTimes(1); + expect(exact).toHaveBeenCalledTimes(1); + expect(target.hasListeners(`${key}.a.b`)).toBe(true); + }); + + it('does not skip a listener when an earlier one unsubscribes mid-dispatch', () => { + const second = vi.fn(); + const first = vi.fn(() => target.off(key, first)); + target.on(key, first); + target.on(key, second); + target.emit(key, {}, {}); + expect(first).toHaveBeenCalledTimes(1); + expect(second).toHaveBeenCalledTimes(1); + }); + }); + describe('emitAndWait', () => { it('awaits async listeners before resolving', async () => { let resolved = false; diff --git a/src/backend/clients/event/EventClient.ts b/src/backend/clients/event/EventClient.ts index a3591279d..1f71d5658 100644 --- a/src/backend/clients/event/EventClient.ts +++ b/src/backend/clients/event/EventClient.ts @@ -18,6 +18,7 @@ */ import { extensionStore } from '../../extensions'; +import { BoundedTtlMap } from '../../util/boundedTtlMap.js'; import { withSpan } from '../../util/span.js'; import { PuterClient } from '../types'; import { @@ -28,8 +29,13 @@ import { MatchingEvents, } from './types'; +const NO_LISTENERS: readonly EventListener[] = []; + export class EventClient extends PuterClient { #eventListeners: Partial> = {}; + #matchKeyMemo = new BoundedTtlMap({ + maxEntries: 1024, + }); onServerStart() { this.emit('serverStart', {}, {}); @@ -63,19 +69,8 @@ export class EventClient extends PuterClient { data: EventMap[T], meta: EventMetadata, ) { - const parts = key.split('.'); - for (let i = 0; i < parts.length; i++) { - const matchKey = ( - i === parts.length - 1 - ? key - : `${parts.slice(0, i + 1).join('.')}.*` - ) as ListenKey; - const extensionListeners = extensionStore.events[matchKey]; - const listeners = (this.#eventListeners[matchKey] || []).concat( - extensionListeners || [], - ); - if (!listeners) continue; - for (const listener of listeners) { + for (const matchKey of this.#matchKeys(key)) { + for (const listener of this.#listenersFor(matchKey)) { this.#emitEvent(listener, key, data, meta); } } @@ -102,21 +97,10 @@ export class EventClient extends PuterClient { // Spanned because callers block on listeners — this is where time // spent in extension hooks (e.g. `ip.validate`) gets attributed. return withSpan('event.emitAndWait', { 'event.key': key }, async () => { - const parts = key.split('.'); - for (let i = 0; i < parts.length; i++) { - const matchKey = ( - i === parts.length - 1 - ? key - : `${parts.slice(0, i + 1).join('.')}.*` - ) as ListenKey; - const extensionListeners = extensionStore.events[matchKey]; - const listeners = (this.#eventListeners[matchKey] || []).concat( - extensionListeners || [], - ); - if (!listeners) continue; + for (const matchKey of this.#matchKeys(key)) { // A wildcard listener observes the event; it does not implement it. const isExact = matchKey === key; - for (const listener of listeners) { + for (const listener of this.#listenersFor(matchKey)) { try { await listener(key, data, meta); } catch (e) { @@ -142,13 +126,7 @@ export class EventClient extends PuterClient { * otherwise perfectly cheap and doesn't need this. */ hasListeners(key: T): boolean { - const parts = key.split('.'); - for (let i = 0; i < parts.length; i++) { - const matchKey = ( - i === parts.length - 1 - ? key - : `${parts.slice(0, i + 1).join('.')}.*` - ) as ListenKey; + for (const matchKey of this.#matchKeys(key)) { if (this.#eventListeners[matchKey]?.length) return true; if (extensionStore.events[matchKey]?.length) return true; } @@ -191,6 +169,34 @@ export class EventClient extends PuterClient { const idx = listeners.indexOf(callback as EventListener); if (idx !== -1) listeners.splice(idx, 1); } + /** + * The keys a listener can be registered under to hear `key`: each + * `.*` shorter than the key, then the key itself. Memoized, since + * the same few keys are emitted on every request. + */ + #matchKeys(key: string): readonly ListenKey[] { + const cached = this.#matchKeyMemo.get(key); + if (cached) return cached; + const parts = key.split('.'); + const matchKeys = parts.map( + (_, i) => + (i === parts.length - 1 + ? key + : `${parts.slice(0, i + 1).join('.')}.*`) as ListenKey, + ); + this.#matchKeyMemo.set(key, matchKeys); + return matchKeys; + } + + #listenersFor(matchKey: ListenKey): readonly EventListener[] { + const own = this.#eventListeners[matchKey]; + const fromExtensions = extensionStore.events[matchKey]; + if (!own?.length && !fromExtensions?.length) return NO_LISTENERS; + // A copy, so a listener that unsubscribes mid-dispatch can't make the + // loop skip its neighbour. + return (own ?? []).concat(fromExtensions ?? []); + } + async #emitEvent( listener: EventListener, key: T, diff --git a/src/backend/core/http/middleware/authProbe.ts b/src/backend/core/http/middleware/authProbe.ts index b58d9df5f..040ab6310 100644 --- a/src/backend/core/http/middleware/authProbe.ts +++ b/src/backend/core/http/middleware/authProbe.ts @@ -22,6 +22,7 @@ import type { AuthService, ReauthReason, } from '../../../services/auth/AuthService'; +import { BoundedTtlMap } from '../../../util/boundedTtlMap'; import { assertResolvedActor } from '../../actor'; import type { TokenSource } from '../types'; @@ -68,23 +69,10 @@ const REAUTH_LOG_MAX_KEYS = 1024; export const createAuthProbe = (opts: AuthProbeOptions): RequestHandler => { const { authService, cookieName } = opts; - const reauthLoggedAt = new Map(); - const shouldLogReauth = (key: string): boolean => { - const now = Date.now(); - const last = reauthLoggedAt.get(key); - if (last !== undefined && now - last < REAUTH_LOG_WINDOW_MS) { - return false; - } - if (reauthLoggedAt.size >= REAUTH_LOG_MAX_KEYS) { - // Map iterates in insertion order and every log re-inserts, so - // the first key is the least recently logged. - const oldest = reauthLoggedAt.keys().next().value; - if (oldest !== undefined) reauthLoggedAt.delete(oldest); - } - reauthLoggedAt.delete(key); - reauthLoggedAt.set(key, now); - return true; - }; + const reauthLogged = new BoundedTtlMap({ + maxEntries: REAUTH_LOG_MAX_KEYS, + ttlMs: REAUTH_LOG_WINDOW_MS, + }); return async (req, _res, next): Promise => { // If something upstream already attached an actor, respect it. @@ -144,7 +132,7 @@ export const createAuthProbe = (opts: AuthProbeOptions): RequestHandler => { return signedToken; }, }; - if (shouldLogReauth(`${reason}:${auth_id ?? '-'}`)) { + if (reauthLogged.shouldEmit(`${reason}:${auth_id ?? '-'}`)) { console.info( `[auth-v2] reauth reason=${reason} auth_id=${auth_id ?? '-'}`, ); diff --git a/src/backend/core/http/middleware/credits.ts b/src/backend/core/http/middleware/credits.ts deleted file mode 100644 index cfc54ff10..000000000 --- a/src/backend/core/http/middleware/credits.ts +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import type { RequestHandler } from 'express'; -import type { IConfig } from '../../../types'; -import { - assertActorHasCredits, - type CreditMetering, -} from '../../../services/metering/enforcement.js'; -import '../expressAugmentation'; - -/** - * Reject an authenticated caller with nothing left of their budget, for routes - * that opt in with `requireCredits`. - * - * Anonymous callers pass: the signed-URL routes authorize on the URL rather - * than a session, and there is no account to charge or turn away. So do worker - * sessions, unless configured otherwise — see `creditEnforcementExempt`. - * - * The answer comes from a short-lived per-actor cache in the metering service, - * so this normally costs nothing beyond a map lookup. That is what makes it - * affordable on routes that are called hundreds of times a minute. - */ -export const requireCreditsGate = ( - metering: CreditMetering | undefined, - config: IConfig, -): RequestHandler => { - return (req, _res, next) => { - assertActorHasCredits(metering, req.actor, config).then( - () => next(), - (err) => next(err), - ); - }; -}; diff --git a/src/backend/core/http/middleware/reputation.test.ts b/src/backend/core/http/middleware/gates.reputation.test.ts similarity index 97% rename from src/backend/core/http/middleware/reputation.test.ts rename to src/backend/core/http/middleware/gates.reputation.test.ts index d297a13cb..f422c0434 100644 --- a/src/backend/core/http/middleware/reputation.test.ts +++ b/src/backend/core/http/middleware/gates.reputation.test.ts @@ -21,7 +21,7 @@ import type { Request, Response } from 'express'; import { describe, expect, it } from 'vitest'; import type { IConfig } from '../../../types'; import { isHttpError } from '../HttpError'; -import { requireReputationGate } from './reputation'; +import { requireReputationGate } from './gates'; // The decision itself is covered in `core/reputation.test.ts`; what matters // here is that the gate resolves it and hands the outcome to `next()` rather diff --git a/src/backend/core/http/middleware/subscription.test.ts b/src/backend/core/http/middleware/gates.subscription.test.ts similarity index 98% rename from src/backend/core/http/middleware/subscription.test.ts rename to src/backend/core/http/middleware/gates.subscription.test.ts index 829b76573..e041b2408 100644 --- a/src/backend/core/http/middleware/subscription.test.ts +++ b/src/backend/core/http/middleware/gates.subscription.test.ts @@ -22,7 +22,7 @@ import { describe, expect, it, vi } from 'vitest'; import { EventClient } from '../../../clients/event/EventClient'; import type { IConfig } from '../../../types'; import { isHttpError } from '../HttpError'; -import { requireSubscriptionGate } from './subscription'; +import { requireSubscriptionGate } from './gates'; // The decision itself is covered in `services/metering/enforcement.test.ts`; // what matters here is that the gate resolves it and hands the outcome to diff --git a/src/backend/core/http/middleware/gates.test.ts b/src/backend/core/http/middleware/gates.test.ts index 804c86d8f..ec339c8b4 100644 --- a/src/backend/core/http/middleware/gates.test.ts +++ b/src/backend/core/http/middleware/gates.test.ts @@ -28,6 +28,7 @@ import { adminOnlyGate, allowedAppIdsGate, assertNotUserSession, + gate, noUserSessionGate, requireAnyVerifiedGate, type AnyVerifiedDeps, @@ -117,6 +118,59 @@ const expectHttpError = (got: NextArg, status: number, legacyCode?: string) => { if (legacyCode) expect(err.legacyCode).toBe(legacyCode); }; +// ── gate ──────────────────────────────────────────────────────────── + +describe('gate', () => { + const denied = () => + new HttpError(403, 'nope', { legacyCode: 'forbidden' }); + + it('calls next() synchronously when a sync check passes', () => { + expect(runGate(gate(() => {}), {})).toBeUndefined(); + }); + + it('hands a sync throw to next(err) synchronously', () => { + const err = denied(); + expect( + runGate( + gate(() => { + throw err; + }), + {}, + ), + ).toBe(err); + }); + + it('calls next() once an async check resolves', async () => { + const got = await runGateAsync( + gate(async () => {}), + {}, + ); + expect(got).toBeUndefined(); + }); + + it('hands an async rejection to next(err)', async () => { + const err = denied(); + const got = await runGateAsync( + gate(async () => { + throw err; + }), + {}, + ); + expect(got).toBe(err); + }); + + it('passes the request to the check', () => { + let seen: unknown; + runGate( + gate((req) => { + seen = req.headers; + }), + { headers: { host: 'x' } }, + ); + expect(seen).toEqual({ host: 'x' }); + }); +}); + // ── subdomainGate ─────────────────────────────────────────────────── describe('subdomainGate', () => { diff --git a/src/backend/core/http/middleware/gates.ts b/src/backend/core/http/middleware/gates.ts index 9b2e33eb3..aae513c23 100644 --- a/src/backend/core/http/middleware/gates.ts +++ b/src/backend/core/http/middleware/gates.ts @@ -17,7 +17,16 @@ * along with this program. If not, see . */ -import type { Request, RequestHandler } from 'express'; +import type { NextFunction, Request, RequestHandler } from 'express'; +import { + assertActorHasCredits, + assertActorHasSubscription, + type CreditMetering, + type SubscriptionGateHook, + type SubscriptionMetering, + type SubscriptionRequirement, +} from '../../../services/metering/enforcement.js'; +import type { IConfig } from '../../../types'; import { isCardVerificationEnabled, type CardFallbackDeps, @@ -29,6 +38,10 @@ import { isPlainUserActor, type Actor, } from '../../actor'; +import { + assertActorMeetsReputation, + type ReputationRequirement, +} from '../../reputation.js'; import { HttpError } from '../HttpError'; import type { AccountGateUser, VerificationFactor } from '../types'; import { assertVerifiedEmail } from '../verifiedEmail'; @@ -62,6 +75,43 @@ const rejectAuth = (req: Request): HttpError => { }); }; +/** + * Route middleware from an assertion: a throw or a rejection goes to + * `next(err)`, anything else to `next()`. A synchronous check calls `next` + * synchronously. + */ +export const gate = ( + check: (req: Request) => void | Promise, +): RequestHandler => { + return (req, _res, next) => { + let pending: void | Promise; + try { + pending = check(req); + } catch (err) { + next(err); + return; + } + if (!pending) { + next(); + return; + } + return settleGate(pending, next); + }; +}; + +const settleGate = async ( + pending: Promise, + next: NextFunction, +): Promise => { + try { + await pending; + } catch (err) { + next(err); + return; + } + next(); +}; + /** * Skip this route entirely (via `next('route')`) when the request's leftmost * subdomain doesn't match. This _isn't_ a rejection — it lets a different route @@ -82,31 +132,18 @@ export const subdomainGate = (allowed: string | string[]): RequestHandler => { }; /** 401 for anonymous requests, 403 for suspended accounts. */ -export const requireAuthGate = (): RequestHandler => { - return (req, _res, next) => { +export const requireAuthGate = (): RequestHandler => + gate((req) => { if (req.appBlocked) { - next( - new HttpError( - 403, - 'This app is not allowed to access Puter resources', - { legacyCode: 'app_blocked' }, - ), + throw new HttpError( + 403, + 'This app is not allowed to access Puter resources', + { legacyCode: 'app_blocked' }, ); - return; } - if (!req.actor) { - next(rejectAuth(req)); - return; - } - try { - assertNotSuspended(req.actor.user); - } catch (err) { - next(err); - return; - } - next(); - }; -}; + if (!req.actor) throw rejectAuth(req); + assertNotSuspended(req.actor.user); + }); /** * Reject app-under-user and access-token actors with 403. `allowFullAccess` @@ -116,14 +153,11 @@ export const requireAuthGate = (): RequestHandler => { */ export const requireUserActorGate = ( opts: { allowFullAccess?: boolean } = {}, -): RequestHandler => { - return (req, _res, next) => { +): RequestHandler => + gate((req) => { const actor = req.actor; // requireAuth runs first; this gate just narrows the actor type. - if (!actor) { - next(rejectAuth(req)); - return; - } + if (!actor) throw rejectAuth(req); const appBlocked = isAppActor(actor); // `isAccountContext`, not the `fullAccess` flag: an app anywhere in the // chain, or an actor that skipped `makeActor`, answers no. @@ -131,18 +165,13 @@ export const requireUserActorGate = ( !!actor.accessToken && !(opts.allowFullAccess && isAccountContext(actor)); if (appBlocked || tokenBlocked) { - next( - new HttpError( - 403, - 'This endpoint is only available to user sessions', - { legacyCode: 'forbidden' }, - ), + throw new HttpError( + 403, + 'This endpoint is only available to user sessions', + { legacyCode: 'forbidden' }, ); - return; } - next(); - }; -}; + }); /** * Reject bare account-session actors (no app, no access token) so a session @@ -168,45 +197,26 @@ export const assertNotUserSession = ( }; /** Route-option form of {@link assertNotUserSession} (`noUserSession: true`). */ -export const noUserSessionGate = (): RequestHandler => { - return (req, _res, next) => { - const actor = req.actor; - if (!actor) { - next(rejectAuth(req)); - return; - } - try { - assertNotUserSession(actor); - } catch (err) { - next(err); - return; - } - next(); - }; -}; +export const noUserSessionGate = (): RequestHandler => + gate((req) => { + if (!req.actor) throw rejectAuth(req); + assertNotUserSession(req.actor); + }); -export const requireNonAccessTokenGate = (): RequestHandler => { - return (req, _res, next) => { +export const requireNonAccessTokenGate = (): RequestHandler => + gate((req) => { const actor = req.actor; - if (!actor) { - next(rejectAuth(req)); - return; - } + if (!actor) throw rejectAuth(req); // Full-access tokens pass; `requireUserActorGate` still keeps them // off account management. if (actor.accessToken && !actor.accessToken.fullAccess) { - next( - new HttpError( - 403, - 'Access tokens are not allowed to access this resource', - { legacyCode: 'forbidden' }, - ), + throw new HttpError( + 403, + 'Access tokens are not allowed to access this resource', + { legacyCode: 'forbidden' }, ); - return; } - next(); - }; -}; + }); /** Built-in admin usernames that always pass `adminOnly`. */ export const DEFAULT_ADMIN_USERNAMES = ['admin', 'system'] as const; @@ -228,27 +238,19 @@ export const adminOnlyGate = ( ); // Admin follows the name, so a name that grants it must never be claimable. reserveUsernames(allowList); - return (req, _res, next) => { + return gate((req) => { const username = req.actor?.user.username; - if (!username || !allowList.has(username.toLowerCase())) { - next( - new HttpError(403, 'Only admins may request this resource', { - legacyCode: 'forbidden', - }), - ); - return; - } const chainApp = req.actor?.effectiveApp ?? null; - if (chainApp && !(opts.appGated && isAppActor(req.actor))) { - next( - new HttpError(403, 'Only admins may request this resource', { - legacyCode: 'forbidden', - }), - ); - return; + if ( + !username || + !allowList.has(username.toLowerCase()) || + (chainApp && !(opts.appGated && isAppActor(req.actor))) + ) { + throw new HttpError(403, 'Only admins may request this resource', { + legacyCode: 'forbidden', + }); } - next(); - }; + }); }; /** @@ -256,17 +258,8 @@ export const adminOnlyGate = ( * `strict_email_verification_required` is set, so deployments without email * delivery are not locked out. */ -export const requireVerifiedGate = (strictFlag: boolean): RequestHandler => { - return (req, _res, next) => { - try { - assertVerifiedEmail(strictFlag, req.actor?.user); - } catch (err) { - next(err); - return; - } - next(); - }; -}; +export const requireVerifiedGate = (strictFlag: boolean): RequestHandler => + gate((req) => assertVerifiedEmail(strictFlag, req.actor?.user)); /** Just the seat read the 2FA gate needs, so gates stay store-agnostic. */ export interface Team2faLookup { @@ -298,28 +291,15 @@ export const assertTeam2fa = async ( /** {@link assertTeam2fa} as route middleware. */ export const requireTeam2fa = ( teams: Team2faLookup | undefined, -): RequestHandler => { - return (req, _res, next) => { - assertTeam2fa(req.actor?.user, teams).then(() => next(), next); - }; -}; +): RequestHandler => gate((req) => assertTeam2fa(req.actor?.user, teams)); /** * Reject accounts still pending a signup-time verification (email, phone, card, * password change). Default-on for authenticated routes; the flows that clear * these flags opt out with `allowUnconfirmed: true`. */ -export const requireVerifiedAccount = (): RequestHandler => { - return (req, _res, next) => { - try { - assertVerifiedAccount(req.actor?.user); - } catch (err) { - next(err); - return; - } - next(); - }; -}; +export const requireVerifiedAccount = (): RequestHandler => + gate((req) => assertVerifiedAccount(req.actor?.user)); /** * The check behind {@link requireVerifiedAccount}, for auth paths that build @@ -389,17 +369,8 @@ export const assertPhoneVerified = ( }; /** Route-option form of {@link assertPhoneVerified} (`requirePhoneVerified`). */ -export const requirePhoneVerifiedGate = (): RequestHandler => { - return (req, _res, next) => { - try { - assertPhoneVerified(req.actor?.user); - } catch (err) { - next(err); - return; - } - next(); - }; -}; +export const requirePhoneVerifiedGate = (): RequestHandler => + gate((req) => assertPhoneVerified(req.actor?.user)); export interface AnyVerifiedDeps extends CardFallbackDeps { /** A paid plan implies a card on file, so it counts as card-verified. */ @@ -419,21 +390,13 @@ const hasCardEvidence = async ( */ export const requireCardVerifiedGate = ( deps: Pick, -): RequestHandler => { - return (req, _res, next) => { - hasCardEvidence(req.actor, deps).then((verified) => { - if (verified) { - next(); - return; - } - next( - new HttpError(403, CARD_REQUIRED_MESSAGE, { - legacyCode: 'card_verification_required', - }), - ); - }, next); - }; -}; +): RequestHandler => + gate(async (req) => { + if (await hasCardEvidence(req.actor, deps)) return; + throw new HttpError(403, CARD_REQUIRED_MESSAGE, { + legacyCode: 'card_verification_required', + }); + }); const isFactorVerified = ( factor: VerificationFactor, @@ -492,14 +455,7 @@ export const assertAnyVerified = async ( export const requireAnyVerifiedGate = ( factors: readonly VerificationFactor[], deps: AnyVerifiedDeps, -): RequestHandler => { - return (req, _res, next) => { - assertAnyVerified(req.actor, factors, deps).then( - () => next(), - (err) => next(err), - ); - }; -}; +): RequestHandler => gate((req) => assertAnyVerified(req.actor, factors, deps)); export const assertNotSuspended = (user: AccountGateUser | undefined): void => { if (user?.suspended) { @@ -519,16 +475,62 @@ export const allowedAppIdsGate = ( allowedAppUids: readonly string[], ): RequestHandler => { const allowList = new Set(allowedAppUids); - return (req, _res, next) => { + return gate((req) => { const appUid = req.actor?.effectiveApp?.uid; if (appUid && !allowList.has(appUid)) { - next( - new HttpError(403, 'This app may not request this resource', { - legacyCode: 'forbidden', - }), - ); - return; + throw new HttpError(403, 'This app may not request this resource', { + legacyCode: 'forbidden', + }); } - next(); - }; + }); }; + +// -- Metering and reputation gates ----------------------------------- +// +// The decisions live in their own modules because the driver dispatch path +// calls them directly: `/drivers/call` is one shared route, so a per-driver +// requirement can't ride on the route chain. + +/** + * Reject a caller whose account isn't trusted enough for this surface + * (`requireReputation`). Inert unless the running config gives the named tier a + * minimum score. + */ +export const requireReputationGate = ( + config: IConfig, + requirement: ReputationRequirement, +): RequestHandler => + gate((req) => assertActorMeetsReputation(req.actor, requirement, config)); + +/** + * Reject a caller whose plan doesn't include this surface + * (`requireSubscription`): `true` accepts any non-free plan, an array of policy + * ids only those. + */ +export const requireSubscriptionGate = ( + metering: SubscriptionMetering | undefined, + config: IConfig, + requirement: SubscriptionRequirement, + hook?: Omit, +): RequestHandler => + gate((req) => + assertActorHasSubscription( + metering, + req.actor, + requirement, + config, + hook && { ...hook, req }, + ), + ); + +/** + * Reject an authenticated caller with nothing left of their budget + * (`requireCredits`). Anonymous callers pass: signed-URL routes authorize on + * the URL, with no account to charge. Answered from the metering service's + * per-actor cache, so it normally costs a map lookup. + */ +export const requireCreditsGate = ( + metering: CreditMetering | undefined, + config: IConfig, +): RequestHandler => + gate((req) => assertActorHasCredits(metering, req.actor, config)); diff --git a/src/backend/core/http/middleware/puterSite.ts b/src/backend/core/http/middleware/puterSite.ts index 9da8e22e1..ff6a3a4fd 100644 --- a/src/backend/core/http/middleware/puterSite.ts +++ b/src/backend/core/http/middleware/puterSite.ts @@ -191,6 +191,7 @@ export const createPuterSiteMiddleware = ( if (hostingDomains.length === 0) { return (_req, _res, next) => next(); } + const hostingCfg = buildHostingConfig(config); // Longest-first so `foo.bar.puter.site` matches `bar.puter.site` before // falling back to `puter.site`. @@ -253,8 +254,6 @@ export const createPuterSiteMiddleware = ( return; } - const hostingCfg = buildHostingConfig(config); - // The subdomain row's `associated_app_id` column is intentionally // not consulted — it was previously user-writable without an // ownership check (so any value there is either legacy or planted). diff --git a/src/backend/core/http/middleware/rateLimit.js b/src/backend/core/http/middleware/rateLimit.js index 7d1220cd0..a052dcda2 100644 --- a/src/backend/core/http/middleware/rateLimit.js +++ b/src/backend/core/http/middleware/rateLimit.js @@ -594,6 +594,73 @@ function fingerprint(req) { : network; } +// -- Shared charge path ---------------------------------------------- + +const noop = async () => {}; + +/** + * The one path every entry point charges through: resolve the limit (with any + * `bySubscription` override for `req`'s actor), then `rate`, `peek` or + * `acquire` against `backend`. A backend error fails open — `true`, or `null` + * in place of a slot — so a broken Redis/KV doesn't fail every request it + * guards. With `failOpen: false` the error is thrown instead. + */ +async function charge( + req, + key, + spec, + { mode, backend, failOpen = true, label }, +) { + try { + const limit = await resolveSubscriptionLimit(req, spec); + return mode === 'acquire' + ? await backend.acquire(key, limit) + : await backend[mode](key, limit, spec.window); + } catch (err) { + if (!failOpen) throw err; + console.error(`${label} failed, failing open:`, err); + return mode === 'acquire' ? null : true; + } +} + +/** Bucket key for a route-shaped spec: its `key` strategy within its scope. */ +function routeKey(req, spec) { + return resolveKey( + req, + spec.scope ?? req.route?.path ?? 'route', + spec.key ?? 'fingerprint', + ); +} + +/** + * Wrap an acquired slot so `release` runs at most once and neither it nor + * `renew` ever rejects: a failed release is logged, not thrown into the + * caller's `finally` or a response listener. + */ +function guardSlot(slot, label) { + let released = false; + return { + ok: true, + release: async () => { + if (released) return; + released = true; + try { + await slot.release(); + } catch (err) { + console.error(`${label} release failed:`, err); + } + }, + renew: async () => { + if (released) return; + try { + await slot.renew?.(); + } catch (err) { + console.error(`${label} renew failed:`, err); + } + }, + }; +} + // -- Route middleware ------------------------------------------------ /** @@ -602,44 +669,26 @@ function fingerprint(req) { * { rateLimit: { limit: 10, window: 15 * 60_000, key: 'user' } } { rateLimit: { * limit: 100, window: 60_000, backend: 'memory' } } * - * Rejects with 429. Fails open on backend error — a broken Redis/KV shouldn't - * 500 every request. + * Rejects with 429. Fails open on backend error. */ export function rateLimitGate(opts) { - const { - window: windowMs, - key: strategy = 'fingerprint', - scope, - backend, - } = opts; - - const backendPair = resolveBackend(backend); + const backend = resolveBackend(opts.backend); return async (req, _res, next) => { - const key = resolveKey( - req, - scope ?? req.route?.path ?? 'route', - strategy, - ); - try { - // `limit` may be overridden per-actor via `bySubscription`; - // the resolver returns `opts.limit` unchanged when the - // override doesn't apply (no actor, no metering, etc.). - const limit = await resolveSubscriptionLimit(req, opts); - if (!(await backendPair.rate(key, limit, windowMs))) - return next( - new HttpError(429, 'Too many requests.', { - legacyCode: 'too_many_requests', - }), - ); - next(); - } catch (err) { - console.error( - '[rate-limit] backend check failed, failing open:', - err, + const allowed = await charge(req, routeKey(req, opts), opts, { + mode: 'rate', + backend, + label: '[rate-limit] backend check', + }); + if (!allowed) { + next( + new HttpError(429, 'Too many requests.', { + legacyCode: 'too_many_requests', + }), ); - next(); + return; } + next(); }; } @@ -668,23 +717,20 @@ function driverCaller(req) { * Returns true if allowed, false if rate-limited. */ export async function checkDriverRateLimit(req, ifaceName, method, opts = {}) { - const { window: windowMs = 60_000, backend } = opts; - const key = `driver:${ifaceName}:${method}:${driverCaller(req)}`; - const backendPair = resolveBackend(backend); - try { - // Drivers can pin a per-subscription limit via `bySubscription` - // on their decorator config; `resolveSubscriptionLimit` reads - // that through `opts.limit` and falls back to the 600/min - // default when neither the spec nor the override apply. - const limit = await resolveSubscriptionLimit(req, { + return charge( + req, + `driver:${ifaceName}:${method}:${driverCaller(req)}`, + { limit: opts.limit ?? 600, + window: opts.window ?? 60_000, bySubscription: opts.bySubscription, - }); - return await backendPair.rate(key, limit, windowMs); - } catch (err) { - console.error('[rate-limit] driver check failed, failing open:', err); - return true; - } + }, + { + mode: 'rate', + backend: resolveBackend(opts.backend), + label: '[rate-limit] driver check', + }, + ); } // -- Imperative helper ----------------------------------------------- @@ -693,20 +739,30 @@ export async function checkDriverRateLimit(req, ifaceName, method, opts = {}) { * Imperative rate-limit check (no middleware shape). For handlers that need a * second-axis limit after their route-level limit fires — e.g. `/login` clamps * per IP at the route, then tighter still on the requests that carry an - * `auth_id` hint. Returns true if allowed, false if rate-limited. Fails open on - * backend error, matching the rest of this module's policy. + * `auth_id` hint. Returns true if allowed, false if rate-limited. + * + * Fails open on backend error, like the rest of this module; pass `{ failOpen: + * false }` for a limit that must hold even then; the backend error is thrown + * instead. */ -export async function checkRateLimit(key, limit, windowMs, backend) { - const bk = resolveBackend(backend); - try { - return await bk.rate(key, limit, windowMs); - } catch (err) { - console.error( - '[rate-limit] imperative check failed, failing open:', - err, - ); - return true; - } +export async function checkRateLimit( + key, + limit, + windowMs, + backend, + { failOpen = true } = {}, +) { + return charge( + undefined, + key, + { limit, window: windowMs }, + { + mode: 'rate', + backend: resolveBackend(backend), + failOpen, + label: '[rate-limit] imperative check', + }, + ); } /** @@ -727,21 +783,11 @@ export async function consumeRouteRateLimit(req, spec) { } return true; } - const { - window: windowMs, - key: strategy = 'fingerprint', - scope, - backend, - } = spec; - const backendPair = resolveBackend(backend); - const key = resolveKey(req, scope ?? req.route?.path ?? 'route', strategy); - try { - const limit = await resolveSubscriptionLimit(req, spec); - return await backendPair.rate(key, limit, windowMs); - } catch (err) { - console.error('[rate-limit] handler charge failed, failing open:', err); - return true; - } + return charge(req, routeKey(req, spec), spec, { + mode: 'rate', + backend: resolveBackend(spec.backend), + label: '[rate-limit] handler charge', + }); } /** @@ -750,21 +796,11 @@ export async function consumeRouteRateLimit(req, spec) { * consume after it succeeds. Single-window specs only. Fails open. */ export async function peekRouteRateLimit(req, spec) { - const { - window: windowMs, - key: strategy = 'fingerprint', - scope, - backend, - } = spec; - const backendPair = resolveBackend(backend); - const key = resolveKey(req, scope ?? req.route?.path ?? 'route', strategy); - try { - const limit = await resolveSubscriptionLimit(req, spec); - return await backendPair.peek(key, limit, windowMs); - } catch (err) { - console.error('[rate-limit] handler peek failed, failing open:', err); - return true; - } + return charge(req, routeKey(req, spec), spec, { + mode: 'peek', + backend: resolveBackend(spec.backend), + label: '[rate-limit] handler peek', + }); } /** @@ -775,16 +811,16 @@ export async function peekRouteRateLimit(req, spec) { * open on backend error, matching the rest of this module's policy. */ export async function peekRateLimit(key, limit, windowMs, backend) { - const bk = resolveBackend(backend); - try { - return await bk.peek(key, limit, windowMs); - } catch (err) { - console.error( - '[rate-limit] imperative peek failed, failing open:', - err, - ); - return true; - } + return charge( + undefined, + key, + { limit, window: windowMs }, + { + mode: 'peek', + backend: resolveBackend(backend), + label: '[rate-limit] imperative peek', + }, + ); } /** @@ -807,46 +843,19 @@ export async function peekRateLimit(key, limit, windowMs, backend) { * stops counting it. Anything that finishes in seconds can ignore it. */ export async function acquireConcurrent(key, limit, backend) { - const bk = resolveBackend(backend); - try { - const result = await bk.acquire(key, limit); - if (!result.ok) - return { - ok: false, - release: async () => {}, - renew: async () => {}, - }; - let released = false; - return { - ok: true, - release: async () => { - if (released) return; - released = true; - try { - await result.release(); - } catch (err) { - console.error( - '[concurrent] imperative release failed:', - err, - ); - } - }, - renew: async () => { - if (released) return; - try { - await result.renew?.(); - } catch (err) { - console.error('[concurrent] imperative renew failed:', err); - } - }, - }; - } catch (err) { - console.error( - '[concurrent] imperative acquire failed, failing open:', - err, - ); - return { ok: true, release: async () => {}, renew: async () => {} }; - } + const slot = await charge( + undefined, + key, + { limit }, + { + mode: 'acquire', + backend: resolveBackend(backend), + label: '[concurrent] imperative acquire', + }, + ); + if (!slot) return { ok: true, release: noop, renew: noop }; + if (!slot.ok) return { ok: false, release: noop, renew: noop }; + return guardSlot(slot, '[concurrent] imperative'); } /** @@ -859,17 +868,17 @@ export const CONCURRENT_SLOT_TTL_MS = ORPHAN_SAFETY_TTL_MS; // -- Subscription-aware limit resolution ----------------------------- /** - * Per-request limit resolution shared by `rateLimitGate` and `concurrencyGate`. - * The base value is `opts.limit`; if `bySubscription` is set and we have an - * authenticated actor plus a metering service, we look up the actor's - * subscription policy and prefer the matching entry. Failure to resolve (no - * actor, no metering, metering throws) falls through to the base — rate / - * concurrency limiting should never _amplify_ a request failure path. + * Per-request limit resolution shared by every entry point. The base value is + * `opts.limit`; if `bySubscription` is set and we have an authenticated actor + * plus a metering service, we look up the actor's subscription policy and + * prefer the matching entry. Failure to resolve (no actor, no metering, + * metering throws) falls through to the base — rate / concurrency limiting + * should never _amplify_ a request failure path. */ async function resolveSubscriptionLimit(req, opts) { const base = opts.limit; if (!opts.bySubscription || !meteringService) return base; - const actor = req.actor; + const actor = req?.actor; if (!actor?.user?.uuid) return base; try { const sub = await meteringService.getActorSubscription(actor); @@ -895,51 +904,34 @@ async function resolveSubscriptionLimit(req, opts) { * branch). Fails open on backend error. */ export function concurrencyGate(opts) { - const { key: strategy = 'fingerprint', scope, backend } = opts; - const backendPair = resolveBackend(backend); + const backend = resolveBackend(opts.backend); return async (req, res, next) => { - const key = resolveKey( - req, - scope ?? req.route?.path ?? 'route', - strategy, - ); - let result; - try { - const limit = await resolveSubscriptionLimit(req, opts); - result = await backendPair.acquire(key, limit); - } catch (err) { - console.error( - '[concurrent] backend acquire failed, failing open:', - err, - ); - return next(); + const slot = await charge(req, routeKey(req, opts), opts, { + mode: 'acquire', + backend, + label: '[concurrent] backend acquire', + }); + if (!slot) { + next(); + return; } - - if (!result.ok) { - return next( + if (!slot.ok) { + next( new HttpError(429, 'Too many concurrent requests.', { legacyCode: 'too_many_requests', }), ); + return; } - // `finish` (response sent) and `close` (connection closed, - // possibly aborted before finish) can both fire; the once - // guard makes release exactly-once. - let released = false; - const release = () => { - if (released) return; - released = true; - Promise.resolve() - .then(() => result.release()) - .catch((err) => - console.error('[concurrent] release failed:', err), - ); - }; - res.once('finish', release); - res.once('close', release); - if (res.destroyed || res.writableFinished) release(); + // `finish` (response sent) and `close` (connection closed, possibly + // aborted before finish) can both fire; the guard releases once. + const { release } = guardSlot(slot, '[concurrent]'); + const onDone = () => void release(); + res.once('finish', onDone); + res.once('close', onDone); + if (res.destroyed || res.writableFinished) onDone(); next(); }; } @@ -959,31 +951,20 @@ export function concurrencyGate(opts) { */ export async function acquireDriverConcurrent(req, ifaceName, method, opts) { if (!opts || typeof opts.limit !== 'number') { - return { ok: true, release: () => {} }; - } - const { backend } = opts; - const key = `driver:${ifaceName}:${method}:${driverCaller(req)}`; - const backendPair = resolveBackend(backend); - try { - const limit = await resolveSubscriptionLimit(req, opts); - const result = await backendPair.acquire(key, limit); - if (!result.ok) return { ok: false, release: () => {} }; - // Wrap release to swallow errors — a failed release shouldn't - // bubble out of the handler's `finally`. - return { - ok: true, - release: () => - Promise.resolve() - .then(() => result.release()) - .catch((err) => - console.error( - '[concurrent] driver release failed:', - err, - ), - ), - }; - } catch (err) { - console.error('[concurrent] driver acquire failed, failing open:', err); - return { ok: true, release: () => {} }; + return { ok: true, release: noop }; } + const slot = await charge( + req, + `driver:${ifaceName}:${method}:${driverCaller(req)}`, + opts, + { + mode: 'acquire', + backend: resolveBackend(opts.backend), + label: '[concurrent] driver acquire', + }, + ); + if (!slot) return { ok: true, release: noop }; + if (!slot.ok) return { ok: false, release: noop }; + const { release } = guardSlot(slot, '[concurrent] driver'); + return { ok: true, release }; } diff --git a/src/backend/core/http/middleware/rateLimit.test.js b/src/backend/core/http/middleware/rateLimit.test.js index ebd3d1ba3..1e7892698 100644 --- a/src/backend/core/http/middleware/rateLimit.test.js +++ b/src/backend/core/http/middleware/rateLimit.test.js @@ -1328,6 +1328,22 @@ describe('acquireDriverConcurrent', () => { expect(hB.ok).toBe(true); }); + it('releases a slot once even when release is called twice', async () => { + const req = { actor: { user: { uuid: 'double-release' } } }; + const opts = { limit: 2 }; + const h1 = await acquireDriverConcurrent(req, 'iface', 'm', opts); + await acquireDriverConcurrent(req, 'iface', 'm', opts); + await h1.release(); + await h1.release(); + + expect((await acquireDriverConcurrent(req, 'iface', 'm', opts)).ok).toBe( + true, + ); + expect((await acquireDriverConcurrent(req, 'iface', 'm', opts)).ok).toBe( + false, + ); + }); + it('applies bySubscription overrides via the wired metering service', async () => { configureRateLimit({ metering: { @@ -1480,6 +1496,20 @@ describe('checkRateLimit', () => { expect(spy).toHaveBeenCalled(); spy.mockRestore(); }); + + it('throws the backend error instead when asked to fail closed', async () => { + configureRateLimit({ + default: 'redis', + redis: { + multi: () => { + throw new Error('redis exploded'); + }, + }, + }); + await expect( + checkRateLimit('boom', 1, 60_000, undefined, { failOpen: false }), + ).rejects.toThrow('redis exploded'); + }); }); describe('consumeRouteRateLimit', () => { diff --git a/src/backend/core/http/middleware/reputation.ts b/src/backend/core/http/middleware/reputation.ts deleted file mode 100644 index 68c92e54d..000000000 --- a/src/backend/core/http/middleware/reputation.ts +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import type { RequestHandler } from 'express'; -import { - assertActorMeetsReputation, - type ReputationRequirement, -} from '../../reputation.js'; -import type { IConfig } from '../../../types'; -import '../expressAugmentation'; - -/** - * Reject a caller whose account isn't trusted enough for this surface, for - * routes that opt in with `requireReputation`. Off unless a route asks for it, - * and off again unless the running config gives the named tier a minimum - * score. - * - * The decision is `assertActorMeetsReputation`, which the driver dispatch path - * calls directly — `/drivers/call` is one shared route, so a per-driver - * requirement can't ride on the route chain. - */ -export const requireReputationGate = ( - config: IConfig, - requirement: ReputationRequirement, -): RequestHandler => { - return (req, _res, next) => { - assertActorMeetsReputation(req.actor, requirement, config).then( - () => next(), - (err) => next(err), - ); - }; -}; diff --git a/src/backend/core/http/middleware/subscription.ts b/src/backend/core/http/middleware/subscription.ts deleted file mode 100644 index df426b70f..000000000 --- a/src/backend/core/http/middleware/subscription.ts +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2024-present Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ - -import type { RequestHandler } from 'express'; -import { - assertActorHasSubscription, - type SubscriptionGateHook, - type SubscriptionMetering, - type SubscriptionRequirement, -} from '../../../services/metering/enforcement.js'; -import type { IConfig } from '../../../types'; -import '../expressAugmentation'; - -/** - * Reject a caller whose plan doesn't include this surface, for routes that opt - * in with `requireSubscription`. Off unless a route asks for it — no surface is - * subscriber-only by default. - * - * `true` accepts any non-free plan; an array of policy ids accepts only those. - * Either way the decision is `assertActorHasSubscription`, which the driver - * dispatch path calls directly — `/drivers/call` is one shared route, so a - * per-driver requirement can't ride on the route chain. - */ -export const requireSubscriptionGate = ( - metering: SubscriptionMetering | undefined, - config: IConfig, - requirement: SubscriptionRequirement, - hook?: Omit, -): RequestHandler => { - return (req, _res, next) => { - assertActorHasSubscription( - metering, - req.actor, - requirement, - config, - hook && { ...hook, req }, - ).then( - () => next(), - (err) => next(err), - ); - }; -}; diff --git a/src/backend/core/http/routeLifecycle.test.ts b/src/backend/core/http/routeLifecycle.test.ts index 31cb8c94e..4326bcd21 100644 --- a/src/backend/core/http/routeLifecycle.test.ts +++ b/src/backend/core/http/routeLifecycle.test.ts @@ -19,12 +19,9 @@ import { EventEmitter } from 'node:events'; import type { Request, Response } from 'express'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { EventClient } from '../../clients/event/EventClient'; -import type { - EventMap, - RouteLifecycleEvent, -} from '../../clients/event/types'; +import type { EventMap, RouteLifecycleEvent } from '../../clients/event/types'; import type { IConfig } from '../../types'; import { createRouteLifecycleMiddleware, @@ -98,6 +95,78 @@ describe('routeEventKeyBase', () => { }); describe('createRouteLifecycleMiddleware', () => { + it('does no event work for a route nobody listens to', async () => { + const events = makeEvents(); + // A listener elsewhere must not make this route pay. + record(events, 'route.get.other.before' as keyof EventMap); + const emitAndWait = vi.spyOn(events, 'emitAndWait'); + const emit = vi.spyOn(events, 'emit'); + + const mw = createRouteLifecycleMiddleware(events, POST, PATH); + const res = new FakeRes(); + let nextCalled = false; + await mw(makeReq('u-1'), res as unknown as Response, () => { + nextCalled = true; + }); + res.writableFinished = true; + res.emit('finish'); + res.emit('close'); + + expect(nextCalled).toBe(true); + expect(emitAndWait).not.toHaveBeenCalled(); + expect(emit).not.toHaveBeenCalled(); + expect(res.listenerCount('finish')).toBe(0); + expect(res.listenerCount('close')).toBe(0); + }); + + it('skips the awaited before emit when only the terminal phase is observed', async () => { + const events = makeEvents(); + const after = record(events, `${BASE}.after` as keyof EventMap); + const emitAndWait = vi.spyOn(events, 'emitAndWait'); + + const mw = createRouteLifecycleMiddleware(events, POST, PATH); + const res = new FakeRes(); + await mw(makeReq('u-1'), res as unknown as Response, () => {}); + res.writableFinished = true; + res.emit('finish'); + + expect(emitAndWait).not.toHaveBeenCalled(); + expect(after).toHaveLength(1); + expect(after[0]).toMatchObject({ + phase: 'after', + actorUid: 'user:u-1', + }); + }); + + it('skips the response hooks when only before is observed', async () => { + const events = makeEvents(); + const before = record(events, `${BASE}.before` as keyof EventMap); + + const mw = createRouteLifecycleMiddleware(events, POST, PATH); + const res = new FakeRes(); + let nextCalled = false; + await mw(makeReq(), res as unknown as Response, () => { + nextCalled = true; + }); + + expect(nextCalled).toBe(true); + expect(before).toHaveLength(1); + expect(res.listenerCount('finish')).toBe(0); + }); + + it('fires wildcard listeners on every phase', async () => { + const events = makeEvents(); + const seen = record(events, 'route.*' as keyof EventMap); + + const mw = createRouteLifecycleMiddleware(events, POST, PATH); + const res = new FakeRes(); + await mw(makeReq(), res as unknown as Response, () => {}); + res.writableFinished = true; + res.emit('finish'); + + expect(seen.map((e) => e.phase)).toEqual(['before', 'after']); + }); + it('emits before, then after on a clean finish', async () => { const events = makeEvents(); const before = record(events, `${BASE}.before` as keyof EventMap); diff --git a/src/backend/core/http/routeLifecycle.ts b/src/backend/core/http/routeLifecycle.ts index 6a9e0548e..1b4ecdb7c 100644 --- a/src/backend/core/http/routeLifecycle.ts +++ b/src/backend/core/http/routeLifecycle.ts @@ -61,6 +61,8 @@ export const routeEventKeyBase = ( * off the real status, not `reject`. * * Subscribers can listen on `route.*`, `route..*`, or the exact key. + * Each phase is skipped when nothing listens for it, so an unobserved route + * pays for a few map lookups and nothing else. */ export const createRouteLifecycleMiddleware = ( events: EventClient, @@ -70,8 +72,18 @@ export const createRouteLifecycleMiddleware = ( const pathLabel = typeof fullPath === 'string' ? fullPath : String(fullPath); const keyBase = routeEventKeyBase(method, fullPath); + const beforeKey = `${keyBase}.before` as const; return async (req: Request, res, next) => { + const observeBefore = events.hasListeners(beforeKey); + const observeEnd = + events.hasListeners(`${keyBase}.after`) || + events.hasListeners(`${keyBase}.error`); + if (!observeBefore && !observeEnd) { + next(); + return; + } + const actor = req.actor ? actorUid(req.actor) : undefined; const startedAt = Date.now(); const base = { @@ -89,7 +101,9 @@ export const createRouteLifecycleMiddleware = ( allow: true as boolean, rejectReason: undefined as string | undefined, }; - await events.emitAndWait(`${keyBase}.before`, beforeEvent, {}); + if (observeBefore) { + await events.emitAndWait(beforeKey, beforeEvent, {}); + } // Explicit veto: a listener set `allow = false`. Emit `reject` and // answer 403 — unless the listener already wrote its own response, in @@ -137,6 +151,11 @@ export const createRouteLifecycleMiddleware = ( return; } + if (!observeEnd) { + next(); + return; + } + let settled = false; const settle = (aborted: boolean) => { if (settled) return; diff --git a/src/backend/server.ts b/src/backend/server.ts index 92ce5b247..3d5116545 100644 --- a/src/backend/server.ts +++ b/src/backend/server.ts @@ -44,7 +44,10 @@ import { requireAnyVerifiedGate, requireAuthGate, requireCardVerifiedGate, + requireCreditsGate, requirePhoneVerifiedGate, + requireReputationGate, + requireSubscriptionGate, requireTeam2fa, requireVerifiedAccount, requireNonAccessTokenGate, @@ -53,9 +56,6 @@ import { subdomainGate, } from './core/http/middleware/gates'; import { guiOriginGate } from './core/http/middleware/originGate'; -import { requireCreditsGate } from './core/http/middleware/credits'; -import { requireReputationGate } from './core/http/middleware/reputation'; -import { requireSubscriptionGate } from './core/http/middleware/subscription'; import { validateReputationRequirement } from './core/reputation'; import { actorOnPaidPlan, diff --git a/src/backend/util/boundedTtlMap.test.ts b/src/backend/util/boundedTtlMap.test.ts new file mode 100644 index 000000000..722358282 --- /dev/null +++ b/src/backend/util/boundedTtlMap.test.ts @@ -0,0 +1,133 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { BoundedTtlMap } from './boundedTtlMap.ts'; + +describe('BoundedTtlMap', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + afterEach(() => { + vi.useRealTimers(); + }); + + it('stores and returns values, including falsy ones', () => { + const map = new BoundedTtlMap({ + maxEntries: 10, + }); + map.set('a', null).set('b', false); + expect(map.get('a')).toBeNull(); + expect(map.has('a')).toBe(true); + expect(map.get('b')).toBe(false); + expect(map.get('missing')).toBeUndefined(); + expect(map.has('missing')).toBe(false); + }); + + it('expires entries after the TTL', () => { + const map = new BoundedTtlMap({ + maxEntries: 10, + ttlMs: 1000, + }); + map.set('a', 1); + vi.advanceTimersByTime(999); + expect(map.get('a')).toBe(1); + vi.advanceTimersByTime(1); + expect(map.get('a')).toBeUndefined(); + expect(map.size).toBe(0); + }); + + it('keeps entries without a TTL until evicted', () => { + const map = new BoundedTtlMap({ maxEntries: 10 }); + map.set('a', 1); + vi.advanceTimersByTime(365 * 24 * 60 * 60 * 1000); + expect(map.get('a')).toBe(1); + }); + + it('evicts the least recently used entry once full', () => { + const map = new BoundedTtlMap({ maxEntries: 3 }); + map.set('a', 1).set('b', 2).set('c', 3); + // Reading `a` makes `b` the least recently used. + expect(map.get('a')).toBe(1); + map.set('d', 4); + expect(map.size).toBe(3); + expect(map.has('b')).toBe(false); + expect([map.get('a'), map.get('c'), map.get('d')]).toEqual([1, 3, 4]); + }); + + it('overwriting a key does not evict another', () => { + const map = new BoundedTtlMap({ maxEntries: 2 }); + map.set('a', 1).set('b', 2).set('a', 3); + expect(map.size).toBe(2); + expect(map.get('a')).toBe(3); + expect(map.get('b')).toBe(2); + }); + + it('supports delete and clear', () => { + const map = new BoundedTtlMap({ maxEntries: 5 }); + map.set('a', 1).set('b', 2); + expect(map.delete('a')).toBe(true); + expect(map.has('a')).toBe(false); + map.clear(); + expect(map.size).toBe(0); + }); + + describe('shouldEmit', () => { + it('is true once per window per key', () => { + const map = new BoundedTtlMap({ + maxEntries: 10, + ttlMs: 60_000, + }); + expect(map.shouldEmit('k')).toBe(true); + expect(map.shouldEmit('k')).toBe(false); + expect(map.shouldEmit('other')).toBe(true); + vi.advanceTimersByTime(60_000); + expect(map.shouldEmit('k')).toBe(true); + expect(map.shouldEmit('k')).toBe(false); + }); + + it('is always true with a zero TTL', () => { + const map = new BoundedTtlMap({ + maxEntries: 10, + ttlMs: 0, + }); + expect(map.shouldEmit('k')).toBe(true); + expect(map.shouldEmit('k')).toBe(true); + }); + + it('stays bounded under a flood of distinct keys', () => { + const map = new BoundedTtlMap({ + maxEntries: 100, + ttlMs: 60_000, + }); + for (let i = 0; i < 10_000; i++) map.shouldEmit(`k${i}`); + expect(map.size).toBe(100); + }); + + it('emits again after the key is deleted', () => { + const map = new BoundedTtlMap({ + maxEntries: 10, + ttlMs: 60_000, + }); + expect(map.shouldEmit('k')).toBe(true); + map.delete('k'); + expect(map.shouldEmit('k')).toBe(true); + }); + }); +}); diff --git a/src/backend/util/boundedTtlMap.ts b/src/backend/util/boundedTtlMap.ts new file mode 100644 index 000000000..1029ec44c --- /dev/null +++ b/src/backend/util/boundedTtlMap.ts @@ -0,0 +1,102 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +export interface BoundedTtlMapOptions { + /** Past this many entries, the least recently used one is evicted. */ + maxEntries: number; + /** Entry lifetime. Omit for entries that only leave by eviction. */ + ttlMs?: number; +} + +interface Entry { + value: V; + expiresAt: number; +} + +/** + * In-process map with a size cap and an optional per-entry TTL, for caches, + * memos and log throttles. Expired entries read as missing; reads refresh an + * entry's recency, so eviction drops the least recently used one. + */ +export class BoundedTtlMap { + readonly #maxEntries: number; + readonly #ttlMs: number | undefined; + readonly #entries = new Map>(); + + constructor({ maxEntries, ttlMs }: BoundedTtlMapOptions) { + this.#maxEntries = Math.max(1, maxEntries); + this.#ttlMs = ttlMs; + } + + get size(): number { + return this.#entries.size; + } + + get(key: K): V | undefined { + return this.#live(key)?.value; + } + + has(key: K): boolean { + return this.#live(key) !== undefined; + } + + set(key: K, value: V): this { + this.#entries.delete(key); + if (this.#entries.size >= this.#maxEntries) { + const oldest = this.#entries.keys().next(); + if (!oldest.done) this.#entries.delete(oldest.value); + } + this.#entries.set(key, { + value, + expiresAt: + this.#ttlMs === undefined ? Infinity : Date.now() + this.#ttlMs, + }); + return this; + } + + delete(key: K): boolean { + return this.#entries.delete(key); + } + + clear(): void { + this.#entries.clear(); + } + + /** + * True at most once per TTL window for `key`, for "log/post this at most + * once a window" throttles. Calls inside the window return false. + */ + shouldEmit(this: BoundedTtlMap, key: K): boolean { + if (this.has(key)) return false; + this.set(key, true); + return true; + } + + #live(key: K): Entry | undefined { + const entry = this.#entries.get(key); + if (!entry) return undefined; + if (entry.expiresAt <= Date.now()) { + this.#entries.delete(key); + return undefined; + } + this.#entries.delete(key); + this.#entries.set(key, entry); + return entry; + } +} diff --git a/src/backend/util/cardFallback.ts b/src/backend/util/cardFallback.ts index 3626f6771..3da02119c 100644 --- a/src/backend/util/cardFallback.ts +++ b/src/backend/util/cardFallback.ts @@ -30,6 +30,7 @@ import type { EventClient } from '../clients/event/EventClient'; import type { PreludeClient } from '../clients/prelude/PreludeClient'; import type { IConfig } from '../types'; +import { BoundedTtlMap } from './boundedTtlMap'; /** * `/send-confirm-phone` route rate limit: how many verification texts one @@ -133,11 +134,14 @@ export function cardFallbackDepsFrom( * switch still takes effect immediately. */ const CARD_STATUS_TTL_MS = 60_000; -let cardStatusCache: { at: number; enabled: boolean | null } | null = null; +const cardStatusCache = new BoundedTtlMap<'status', boolean | null>({ + maxEntries: 1, + ttlMs: CARD_STATUS_TTL_MS, +}); /** Drops the memoized probe answer. For tests, and for a config reload. */ export function resetCardVerificationStatusCache(): void { - cardStatusCache = null; + cardStatusCache.clear(); } /** @@ -147,10 +151,8 @@ export function resetCardVerificationStatusCache(): void { export async function isCardVerificationEnabled( deps: CardFallbackDeps, ): Promise { - const now = Date.now(); - if (cardStatusCache && now - cardStatusCache.at < CARD_STATUS_TTL_MS) { - return cardStatusCache.enabled; - } + const cached = cardStatusCache.get('status'); + if (cached !== undefined) return cached; let enabled: boolean | null = null; try { enabled = await deps.probeCardVerification(); @@ -159,7 +161,7 @@ export async function isCardVerificationEnabled( // not work is worse than not offering one. console.warn('[card-verification] status probe failed:', e); } - cardStatusCache = { at: now, enabled }; + cardStatusCache.set('status', enabled); return enabled; } diff --git a/src/backend/util/fileSigning.test.ts b/src/backend/util/fileSigning.test.ts index 0b9aa4b71..a35eca03d 100644 --- a/src/backend/util/fileSigning.test.ts +++ b/src/backend/util/fileSigning.test.ts @@ -22,10 +22,24 @@ import { describe, expect, it, vi } from 'vitest'; import type { FSEntry } from '../stores/fs/FSEntry.js'; import { isSignatureValid, + mimeFromName, signFile, verifySignature, } from './fileSigning.js'; +// Pass-through spy, to count how often a key is derived from the raw secret. +const { hmacKeys } = vi.hoisted(() => ({ hmacKeys: [] as unknown[] })); +vi.mock('node:crypto', async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + createHmac: (...args: Parameters) => { + hmacKeys.push(args[1]); + return actual.createHmac(...args); + }, + }; +}); + const CONFIG = { secret: 'a-real-secret-not-the-placeholder', apiBaseUrl: 'https://api.example.test', @@ -290,3 +304,37 @@ describe('fileSigning legacy signature visibility', () => { } }); }); + +describe('signing key derivation', () => { + it('derives the purpose key from a secret once', () => { + const secret = 'a-secret-only-this-test-uses'; + hmacKeys.length = 0; + signFile(makeEntry(), { ...CONFIG, secret }); + signFile(makeEntry(), { ...CONFIG, secret }); + expect(hmacKeys.filter((key) => key === secret)).toHaveLength(1); + }); +}); + +describe('mimeFromName', () => { + it('keeps the types it has always reported', () => { + expect(mimeFromName('photo.JPG')).toBe('image/jpeg'); + expect(mimeFromName('favicon.ico')).toBe('image/x-icon'); + expect(mimeFromName('clip.wav')).toBe('audio/wav'); + expect(mimeFromName('notes.md')).toBe('text/markdown'); + expect(mimeFromName('app.mjs')).toBe('application/javascript'); + }); + + it('covers extensions beyond the old inline table', () => { + expect(mimeFromName('slides.pptx')).toBe( + 'application/vnd.openxmlformats-officedocument.presentationml.presentation', + ); + expect(mimeFromName('song.flac')).toBe('audio/x-flac'); + }); + + it('is null for unknown, missing or leading-dot-only extensions', () => { + expect(mimeFromName('archive.unknownext')).toBeNull(); + expect(mimeFromName('README')).toBeNull(); + expect(mimeFromName('.bashrc')).toBeNull(); + expect(mimeFromName('trailing.')).toBeNull(); + }); +}); diff --git a/src/backend/util/fileSigning.ts b/src/backend/util/fileSigning.ts index 7416c41cb..2ce6e4673 100644 --- a/src/backend/util/fileSigning.ts +++ b/src/backend/util/fileSigning.ts @@ -17,9 +17,12 @@ * along with this program. If not, see . */ -import { createHash, createHmac, timingSafeEqual } from 'node:crypto'; +import { createHash, createHmac } from 'node:crypto'; +import { lookup as lookupMime } from 'mime-types'; import { HttpError } from '../core/http/HttpError.js'; import type { FSEntry } from '../stores/fs/FSEntry.js'; +import { BoundedTtlMap } from './boundedTtlMap.js'; +import { secretsEqual } from './secureCompare.js'; /** * File URL signing. A signed URL carries `uid`, `expires` and `signature`; that @@ -73,6 +76,24 @@ function computeLegacySignature( return sha256(`${uid}/${action}/${secret}/${expires}`); } +// Derived keys by secret; deployments sign with one secret, rarely a few. +const purposeKeys = new BoundedTtlMap({ maxEntries: 8 }); + +/** + * Purpose-labelled key, so no other token signed with the platform secret can + * be replayed as a file signature. + */ +function purposeKey(secret: string): Buffer { + let key = purposeKeys.get(secret); + if (!key) { + key = createHmac('sha256', secret) + .update('puter-fs:signed-url') + .digest(); + purposeKeys.set(secret, key); + } + return key; +} + function computeOwnerBoundSignature( uid: string, action: SignAction, @@ -80,31 +101,21 @@ function computeOwnerBoundSignature( secret: string, expires: number, ): string { - // Purpose-labelled key, so no other token signed with the platform secret - // can be replayed as a file signature. - const key = createHmac('sha256', secret) - .update('puter-fs:signed-url') - .digest(); - return createHmac('sha256', key) + return createHmac('sha256', purposeKey(secret)) .update(`${uid}/${action}/${ownerUserId}/${expires}`) .digest('hex'); } /** - * Constant-time equality for the hex signature strings. A plain `===` - * short-circuits on the first differing character, leaking a byte-by-byte - * timing oracle on the one value an attacker controls and submits repeatedly. - * Length mismatch (or non-hex input) returns false without a timing-variable - * compare. + * Constant-time equality for the hex signature strings; compares the decoded + * bytes, so hex case doesn't matter. */ function signaturesEqual(provided: string, expected: string): boolean { if (provided.length !== expected.length) return false; - const a = Buffer.from(provided, 'hex'); - const b = Buffer.from(expected, 'hex'); - // Malformed hex yields a shorter buffer than the hex length implies; - // timingSafeEqual requires equal lengths, so guard before comparing. - if (a.length !== b.length) return false; - return timingSafeEqual(a, b); + return secretsEqual( + Buffer.from(provided, 'hex'), + Buffer.from(expected, 'hex'), + ); } /** @@ -300,47 +311,16 @@ export function isSignatureValid( } } -// Minimal MIME type inference from file extension. Uses a small inline map -// to avoid pulling in `mime-types`. Callers that need complete coverage -// should import `mime-types` directly. -const MIME_BY_EXT: Record = { - txt: 'text/plain', - html: 'text/html', - htm: 'text/html', - css: 'text/css', - js: 'application/javascript', - mjs: 'application/javascript', - json: 'application/json', - xml: 'application/xml', - pdf: 'application/pdf', - zip: 'application/zip', - gz: 'application/gzip', - tar: 'application/x-tar', - png: 'image/png', - jpg: 'image/jpeg', - jpeg: 'image/jpeg', - gif: 'image/gif', - webp: 'image/webp', - svg: 'image/svg+xml', +// Where mime-types' preferred type differs from what this has always reported. +const MIME_OVERRIDES: Record = { ico: 'image/x-icon', - bmp: 'image/bmp', - tiff: 'image/tiff', - tif: 'image/tiff', - mp3: 'audio/mpeg', wav: 'audio/wav', - ogg: 'audio/ogg', - m4a: 'audio/mp4', - mp4: 'video/mp4', - webm: 'video/webm', - mov: 'video/quicktime', - md: 'text/markdown', - markdown: 'text/markdown', - csv: 'text/csv', }; +/** MIME type for a file name by its extension, or null when unknown. */ export function mimeFromName(name: string): string | null { const dot = name.lastIndexOf('.'); if (dot <= 0) return null; const ext = name.slice(dot + 1).toLowerCase(); - return MIME_BY_EXT[ext] ?? null; + return MIME_OVERRIDES[ext] ?? (lookupMime(ext) || null); } diff --git a/src/backend/util/secureCompare.test.ts b/src/backend/util/secureCompare.test.ts new file mode 100644 index 000000000..0ff24e43c --- /dev/null +++ b/src/backend/util/secureCompare.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { describe, expect, it } from 'vitest'; +import { secretsEqual } from './secureCompare.ts'; + +describe('secretsEqual', () => { + it('is true for equal strings and equal buffers', () => { + expect(secretsEqual('s3cret', 's3cret')).toBe(true); + expect( + secretsEqual(Buffer.from([1, 2, 3]), Buffer.from([1, 2, 3])), + ).toBe(true); + expect(secretsEqual('', '')).toBe(true); + }); + + it('is false for values that differ anywhere', () => { + expect(secretsEqual('s3cret', 's3creT')).toBe(false); + expect(secretsEqual('s3cret', 'X3cret')).toBe(false); + }); + + it('is false for different lengths without throwing', () => { + expect(secretsEqual('short', 'much longer secret')).toBe(false); + expect(secretsEqual('', 'x')).toBe(false); + expect(secretsEqual(Buffer.alloc(16), Buffer.alloc(32))).toBe(false); + }); + + it('compares a string with its utf8 bytes as equal', () => { + expect(secretsEqual('héllo', Buffer.from('héllo', 'utf8'))).toBe(true); + }); +}); diff --git a/src/backend/util/secureCompare.ts b/src/backend/util/secureCompare.ts new file mode 100644 index 000000000..2424ff70a --- /dev/null +++ b/src/backend/util/secureCompare.ts @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +import { createHmac, randomBytes, timingSafeEqual } from 'node:crypto'; + +// Only has to stay unknown to callers for the life of the process. +const COMPARE_KEY = randomBytes(32); + +const digest = (value: string | Buffer): Buffer => + createHmac('sha256', COMPARE_KEY).update(value).digest(); + +/** + * Constant-time equality for secrets, tokens and signatures. Both sides are + * HMACed to a fixed-length digest first, so neither the inputs' lengths nor + * where they first differ shows in the timing, and unequal lengths need no + * early return. + */ +export function secretsEqual(a: string | Buffer, b: string | Buffer): boolean { + return timingSafeEqual(digest(a), digest(b)); +}