/** * Copyright (C) 2024-present Puter Technologies Inc. * * This file is part of Puter. * * Puter is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published * by the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ import { readdirSync, readFileSync } from 'fs'; import { isAbsolute, resolve as resolvePath } from 'path'; import { metrics } from '@opentelemetry/api'; import { createPool, type Pool } from 'mysql2'; import { Span } from '../../util/span.js'; import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient'; import { SQLBatcher } from './SQLBatcher.js'; import { isRetriableError } from './retriableErrors.js'; import { splitMysqlStatements } from './splitMysqlStatements.js'; import { compareMigrationFilenames } from './migrationFilenames.js'; import type { IConfig } from '../../types'; const DEFAULT_SELECT_TIMEOUT_MS = 30_000; const replicaFailoverCounter = metrics .getMeter('puter-backend') .createCounter('db.read.replica_failover', { description: 'Reads that failed on the replica batcher and were retried on the primary', }); export { compareMigrationFilenames }; type PoolConfig = Parameters[0]; enum Configuration { SINGLE, REPLICA, } export class MySQLDatabaseClient extends AbstractDatabaseClient { override readonly engineName = 'mysql'; private primaryPool!: Pool; private replicaPool!: Pool; private db!: SQLBatcher; private dbReplica!: SQLBatcher; private configuration = Configuration.SINGLE; private shutdownStarted = false; private shutdownTimer: ReturnType | null = null; constructor(config: IConfig) { super(config); } // ------------------------------------------------------------------ // Lifecycle // ------------------------------------------------------------------ override async onServerStart(): Promise { const dbConf = this.config.database!; this.primaryPool = this.createPool({ host: dbConf.host ?? '127.0.0.1', port: dbConf.port ?? 3306, user: dbConf.user ?? 'root', password: dbConf.password ?? '', database: dbConf.database ?? 'puter', }); console.log('[mysql] connected to primary'); this.db = this.createPrimaryBatcher(this.primaryPool); if (dbConf.replica) { this.replicaPool = this.createPool(dbConf.replica); this.configuration = Configuration.REPLICA; console.log('[mysql] connected to read-replica'); } else { this.replicaPool = this.primaryPool; this.configuration = Configuration.SINGLE; } this.dbReplica = this.createReplicaBatcher(this.replicaPool); await this.runMigrations(); } override async onServerPrepareShutdown(): Promise { if (this.shutdownStarted) return; this.shutdownStarted = true; // Allow in-flight queries to drain before closing pools const drainMs = 60_000; console.log( `[mysql] draining in-flight queries (${drainMs}ms) before closing pools`, ); this.shutdownTimer = setTimeout(() => { this.shutdownTimer = null; this.closeCurrentPools('drain').catch((e) => console.error('[mysql] error closing pools after drain', e), ); }, drainMs); if (typeof this.shutdownTimer.unref === 'function') { this.shutdownTimer.unref(); } } override async onServerShutdown(): Promise { if (this.shutdownTimer) { clearTimeout(this.shutdownTimer); this.shutdownTimer = null; } await this.closeCurrentPools('shutdown'); } // ------------------------------------------------------------------ // Query interface // ------------------------------------------------------------------ // The db.* spans measure the logical query, including time queued in // the SQLBatcher — the mysql2 auto-instrumentation only sees the // coalesced multi-statement flush, so per-query latency lives here. @Span('db.read', (query: string) => ({ 'db.statement': query })) override async read( query: string, params: unknown[] = [], ): Promise[]> { let result; try { result = await this.dbReplica.execute(query, params); } catch (error) { // Replica-side degradation (batcher load-shed or a transient // connection failure) shouldn't fail reads while the primary is // healthy. Deterministic errors (bad SQL) are rethrown — they // would fail identically on the primary. if ( this.configuration !== Configuration.REPLICA || !MySQLDatabaseClient.isFailoverWorthy(error) ) { throw error; } replicaFailoverCounter.add(1); result = await this.db.execute(query, params); } if (!result) return []; return (result[0] as Record[]) ?? []; } @Span('db.pread', (query: string) => ({ 'db.statement': query })) override async pread( query: string, params: unknown[] = [], ): Promise[]> { const result = await this.db.execute(query, params); if (!result) return []; return (result[0] as Record[]) ?? []; } @Span('db.write', (query: string) => ({ 'db.statement': query })) override async write( query: string, params: unknown[] = [], ): Promise { const result = await this.db.execute(query, params); const header = result[0] as { insertId?: number; affectedRows?: number; }; const affectedRows = header.affectedRows ?? 0; return { insertId: header.insertId ?? 0, affectedRows, anyRowsAffected: affectedRows > 0, }; } @Span('db.batchWrite', (entries: unknown[]) => ({ 'db.batch_size': entries.length, })) override async batchWrite( entries: { statement: string; values: unknown[] }[], ): Promise { if (entries.length === 0) return; // Bypass the SQLBatcher: it coalesces queries from unrelated callers // into a single multi-statement string, which is incompatible with // wrapping a transaction around just *our* statements. Acquire a // dedicated connection so BEGIN/COMMIT/ROLLBACK only scope `entries`. const conn = await this.primaryPool.promise().getConnection(); try { await conn.beginTransaction(); try { for (const { statement, values } of entries) { await conn.execute(statement, values); } await conn.commit(); } catch (err) { await conn.rollback().catch(() => {}); throw err; } } finally { conn.release(); } } @Span('db.tryHardRead', (query: string) => ({ 'db.statement': query })) override async tryHardRead( query: string, params: unknown[] = [], ): Promise[]> { if (this.configuration === Configuration.SINGLE) { return this.read(query, params); } // Run both reads in parallel — prefer replica when it returns rows, // otherwise fall back to primary to handle replication lag. const primaryPromise = this.db.execute(query, params); try { const replicaResult = await this.dbReplica.execute(query, params); if ( Array.isArray(replicaResult?.[0]) && (replicaResult[0] as unknown[]).length > 0 ) { primaryPromise.catch(() => {}); // suppress unhandled rejection return replicaResult[0] as Record[]; } } catch { // fall through to primary } const primaryResult = await primaryPromise; return (primaryResult?.[0] as Record[]) ?? []; } // ------------------------------------------------------------------ // Migrations // ------------------------------------------------------------------ /** * Apply `.sql` files from each configured migration directory in order. * Files within a directory are sorted lexically. Files MUST be * idempotent — there is no per-file applied-state tracking. Failures * abort startup so operators see schema problems loud. */ private async runMigrations(): Promise { const paths = this.config.database?.migrationPaths; if (!paths || paths.length === 0) return; const conn = await this.primaryPool.promise().getConnection(); try { for (const rawPath of paths) { const dir = isAbsolute(rawPath) ? rawPath : resolvePath(process.cwd(), rawPath); let files: string[]; try { files = readdirSync(dir) .filter( (f) => f.endsWith('.sql') && f.startsWith('mysql'), ) .sort(compareMigrationFilenames); } catch (e) { throw new Error( `[mysql] migration path is unreadable: ${dir}`, { cause: e }, ); } if (files.length === 0) { console.log(`[mysql] no migrations in ${dir}`); continue; } console.log( `[mysql] running migrations from ${dir}: ${files.length} file(s)`, ); for (const file of files) { const filePath = resolvePath(dir, file); const contents = readFileSync(filePath, 'utf8'); const statements = splitMysqlStatements(contents); for (let i = 0; i < statements.length; i++) { try { await conn.query(statements[i]); } catch (e) { throw new Error( `[mysql] failed to apply ${file} at statement ${i}`, { cause: e }, ); } } console.log( `[mysql] applied ${file} (${statements.length} statements)`, ); } } } finally { conn.release(); } } // ------------------------------------------------------------------ // Pool management // ------------------------------------------------------------------ private createPool(poolConfig: PoolConfig): Pool { const pool = createPool({ maxPreparedStatements: 900, connectionLimit: 30, enableKeepAlive: true, ...poolConfig, multipleStatements: true, } as PoolConfig); // Server-side kill switch for runaway reads: MySQL applies // max_execution_time to SELECT statements only, so this is // write-safe. Without it, a stalled database turns reads into // indefinite hangs that no client-side timeout ever converts // into a failure. 0 disables. const selectTimeoutMs = Math.floor( Number( this.config.database?.selectTimeoutMs ?? DEFAULT_SELECT_TIMEOUT_MS, ), ); if (selectTimeoutMs > 0) { pool.on('connection', (conn) => { conn.query( `SET SESSION max_execution_time = ${selectTimeoutMs}`, ); }); } return pool; } private createPrimaryBatcher(pool: Pool): SQLBatcher { return new SQLBatcher(pool, { maxTimeInQueue: 30, maxBatchSize: 5, poolLabel: 'primary', acquireTimeoutMs: this.config.database?.acquireTimeoutMs, }); } private createReplicaBatcher(pool: Pool): SQLBatcher { return new SQLBatcher(pool, { maxTimeInQueue: 10, maxBatchSize: 5, poolLabel: 'replica', readOnly: true, acquireTimeoutMs: this.config.database?.acquireTimeoutMs, }); } /** Reinitialize the primary pool (e.g. after a health-check failure). */ reinitPrimary(): void { if (this.shutdownStarted) return; const dbConf = this.config.database!; const previous = this.primaryPool; this.primaryPool = this.createPool({ host: dbConf.host ?? '127.0.0.1', port: dbConf.port ?? 3306, user: dbConf.user ?? 'root', password: dbConf.password ?? '', database: dbConf.database ?? 'puter', }); this.db = this.createPrimaryBatcher(this.primaryPool); if (this.configuration === Configuration.SINGLE) { this.replicaPool = this.primaryPool; this.dbReplica = this.createReplicaBatcher(this.primaryPool); } if (previous && previous !== this.primaryPool) { this.closePool(previous, 'reinit:primary').catch(() => {}); } } /** Reinitialize the replica pool. */ reinitReplica(): void { if (this.shutdownStarted || !this.config.database?.replica) return; const previous = this.replicaPool; this.replicaPool = this.createPool(this.config.database.replica); this.dbReplica = this.createReplicaBatcher(this.replicaPool); if ( previous && previous !== this.replicaPool && previous !== this.primaryPool ) { this.closePool(previous, 'reinit:replica').catch(() => {}); } } // ------------------------------------------------------------------ // Retry helpers (for health checks or resilient reads) // ------------------------------------------------------------------ static isRetriableError(error: unknown): boolean { return isRetriableError(error); } /** Replica failures worth retrying on the primary: batcher load-shed * or transient connection errors — never deterministic SQL errors. */ private static isFailoverWorthy(error: unknown): boolean { const code = (error as { code?: string })?.code; return code === 'dbBatchFailed' || isRetriableError(error); } async readWithRetry( label: string, operation: () => Promise, opts?: { maxAttempts?: number; baseBackoffMs?: number; maxBackoffMs?: number; jitterRatio?: number; }, ): Promise { const maxAttempts = opts?.maxAttempts ?? 3; const baseBackoffMs = opts?.baseBackoffMs ?? 100; const maxBackoffMs = opts?.maxBackoffMs ?? 500; const jitterRatio = opts?.jitterRatio ?? 0.2; let attempt = 1; while (true) { try { return await operation(); } catch (error) { if (this.shutdownStarted) throw error; if ( attempt >= maxAttempts || !MySQLDatabaseClient.isRetriableError(error) ) throw error; const raw = baseBackoffMs * 2 ** (attempt - 1); const capped = Math.min(maxBackoffMs, raw); const window = Math.round(capped * jitterRatio); const jitter = window === 0 ? 0 : Math.floor(Math.random() * (window * 2 + 1)) - window; const delay = Math.max(0, capped + jitter); console.warn( `[${label}] transient mysql error (${(error as { code?: string })?.code ?? 'unknown'}); retry ${attempt + 1}/${maxAttempts} in ${delay}ms`, ); await new Promise((r) => setTimeout(r, delay)); attempt++; } } } // ------------------------------------------------------------------ // Internal pool lifecycle // ------------------------------------------------------------------ private async closePool( pool: Pool, label: string, timeoutMs: number | null = null, ): Promise { if (!pool) return; await new Promise((resolve, reject) => { let settled = false; let timer: ReturnType | null = null; const finish = (err?: unknown) => { if (settled) return; settled = true; if (timer) clearTimeout(timer); if (err) reject(err); else resolve(); }; if (timeoutMs !== null) { timer = setTimeout(() => { console.warn( `[mysql] timed out closing pool (${label}); forcing`, ); this.forceDestroyConnections(pool, `${label}:timeout`); finish(); }, timeoutMs); } try { pool.end((err) => finish(err)); } catch (err) { finish(err); } }); } private forceDestroyConnections(pool: Pool, label: string): void { // mysql2 internal — _allConnections is a CircularBuffer const all = ( pool as unknown as { _allConnections?: { forEach: (fn: (c: { destroy: () => void }) => void) => void; }; } )._allConnections; if (!all || typeof all.forEach !== 'function') return; let count = 0; all.forEach((conn) => { try { conn.destroy(); count++; } catch { // no-op } }); if (count > 0) console.warn( `[mysql] force-closed ${count} connections (${label})`, ); } private async closeCurrentPools(reason: string): Promise { const timeoutMs = reason.startsWith('signal:') ? 45_000 : null; const tasks: Promise[] = []; if (this.primaryPool) { tasks.push( this.closePool( this.primaryPool, `${reason}:primary`, timeoutMs, ), ); } if (this.replicaPool && this.replicaPool !== this.primaryPool) { tasks.push( this.closePool( this.replicaPool, `${reason}:replica`, timeoutMs, ), ); } await Promise.all(tasks); } }