wip: self hosted full setup

src/backend/clients/database/MySQLDatabaseClient.ts

@@ -17,9 +17,12 @@
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
 
+import { readdirSync, readFileSync } from 'fs';
+import { isAbsolute, resolve as resolvePath } from 'path';
 import { createPool, type Pool } from 'mysql2';
 import { AbstractDatabaseClient, type WriteResult } from './DatabaseClient';
 import { SQLBatcher } from './SQLBatcher.js';
+import { splitMysqlStatements } from './splitMysqlStatements.js';
 import type { IConfig } from '../../types';
 
 const RETRIABLE_ERROR_CODES = new Set([
@@ -91,6 +94,8 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
         }
 
         this.dbReplica = new SQLBatcher(this.replicaPool, 10, 5);
+
+        await this.runMigrations();
     }
 
     override async onServerPrepareShutdown(): Promise<void> {
@@ -213,6 +218,72 @@ export class MySQLDatabaseClient extends AbstractDatabaseClient {
         return (primaryResult?.[0] as Record<string, unknown>[]) ?? [];
     }
 
+    // ------------------------------------------------------------------
+    // Migrations
+    // ------------------------------------------------------------------
+
+    /**
+     * Apply `.sql` files from each configured migration directory in order.
+     * Files within a directory are sorted lexically. Files MUST be
+     * idempotent — there is no per-file applied-state tracking. Failures
+     * abort startup so operators see schema problems loud.
+     */
+    private async runMigrations(): Promise<void> {
+        const paths = this.config.database?.migrationPaths;
+        if (!paths || paths.length === 0) return;
+
+        const conn = await this.primaryPool.promise().getConnection();
+        try {
+            for (const rawPath of paths) {
+                const dir = isAbsolute(rawPath)
+                    ? rawPath
+                    : resolvePath(process.cwd(), rawPath);
+
+                let files: string[];
+                try {
+                    files = readdirSync(dir)
+                        .filter((f) => f.endsWith('.sql'))
+                        .sort();
+                } catch (e) {
+                    throw new Error(
+                        `[mysql] migration path is unreadable: ${dir}`,
+                        { cause: e },
+                    );
+                }
+
+                if (files.length === 0) {
+                    console.log(`[mysql] no migrations in ${dir}`);
+                    continue;
+                }
+
+                console.log(
+                    `[mysql] running migrations from ${dir}: ${files.length} file(s)`,
+                );
+
+                for (const file of files) {
+                    const filePath = resolvePath(dir, file);
+                    const contents = readFileSync(filePath, 'utf8');
+                    const statements = splitMysqlStatements(contents);
+                    for (let i = 0; i < statements.length; i++) {
+                        try {
+                            await conn.query(statements[i]);
+                        } catch (e) {
+                            throw new Error(
+                                `[mysql] failed to apply ${file} at statement ${i}`,
+                                { cause: e },
+                            );
+                        }
+                    }
+                    console.log(
+                        `[mysql] applied ${file} (${statements.length} statements)`,
+                    );
+                }
+            }
+        } finally {
+            conn.release();
+        }
+    }
+
     // ------------------------------------------------------------------
     // Pool management
     // ------------------------------------------------------------------

src/backend/clients/database/splitMysqlStatements.test.ts

@@ -0,0 +1,137 @@
+/**
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+import { describe, expect, it } from 'vitest';
+import { splitMysqlStatements } from './splitMysqlStatements.js';
+
+describe('splitMysqlStatements', () => {
+    it('splits simple statements on default delimiter', () => {
+        expect(splitMysqlStatements('SELECT 1; SELECT 2;')).toEqual([
+            'SELECT 1',
+            'SELECT 2',
+        ]);
+    });
+
+    it('returns empty array for whitespace-only input', () => {
+        expect(splitMysqlStatements('  \n\t  ')).toEqual([]);
+    });
+
+    it('keeps a trailing statement without terminating semicolon', () => {
+        expect(splitMysqlStatements('SELECT 1;\nSELECT 2')).toEqual([
+            'SELECT 1',
+            'SELECT 2',
+        ]);
+    });
+
+    it('ignores semicolons inside single-quoted strings', () => {
+        expect(
+            splitMysqlStatements("INSERT INTO t VALUES ('a;b'); SELECT 2;"),
+        ).toEqual(["INSERT INTO t VALUES ('a;b')", 'SELECT 2']);
+    });
+
+    it("handles SQL '' escape inside single-quoted strings", () => {
+        expect(
+            splitMysqlStatements("SELECT 'it''s; ok'; SELECT 2;"),
+        ).toEqual(["SELECT 'it''s; ok'", 'SELECT 2']);
+    });
+
+    it('handles backslash escape inside strings', () => {
+        expect(
+            splitMysqlStatements("SELECT 'a\\'b;c'; SELECT 2;"),
+        ).toEqual(["SELECT 'a\\'b;c'", 'SELECT 2']);
+    });
+
+    it('ignores semicolons inside backtick identifiers', () => {
+        expect(
+            splitMysqlStatements('SELECT `weird;col` FROM t; SELECT 2;'),
+        ).toEqual(['SELECT `weird;col` FROM t', 'SELECT 2']);
+    });
+
+    it('ignores semicolons inside double-quoted strings', () => {
+        expect(splitMysqlStatements('SELECT "a;b"; SELECT 2;')).toEqual([
+            'SELECT "a;b"',
+            'SELECT 2',
+        ]);
+    });
+
+    it('ignores semicolons in line comments', () => {
+        expect(
+            splitMysqlStatements(
+                'SELECT 1; -- a;b\nSELECT 2; # c;d\nSELECT 3;',
+            ),
+        ).toEqual(['SELECT 1', '-- a;b\nSELECT 2', '# c;d\nSELECT 3']);
+    });
+
+    it('ignores semicolons in block comments (multi-line)', () => {
+        expect(
+            splitMysqlStatements('SELECT 1 /* a;\nb;c */; SELECT 2;'),
+        ).toEqual(['SELECT 1 /* a;\nb;c */', 'SELECT 2']);
+    });
+
+    it('honours DELIMITER directive', () => {
+        const sql = `
+SELECT 1;
+DELIMITER //
+CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END//
+DELIMITER ;
+SELECT 3;
+`;
+        expect(splitMysqlStatements(sql)).toEqual([
+            'SELECT 1',
+            'CREATE PROCEDURE p() BEGIN SELECT 1; SELECT 2; END',
+            'SELECT 3',
+        ]);
+    });
+
+    it('handles a stored procedure that uses // delimiter end-to-end', () => {
+        const sql = `DROP PROCEDURE IF EXISTS foo;
+DELIMITER //
+CREATE PROCEDURE foo(IN x INT)
+BEGIN
+  IF x > 0 THEN
+    SET @s := 'hi;';
+    SELECT @s;
+  END IF;
+END//
+DELIMITER ;
+DROP PROCEDURE IF EXISTS foo;
+`;
+        const stmts = splitMysqlStatements(sql);
+        expect(stmts).toHaveLength(3);
+        expect(stmts[0]).toBe('DROP PROCEDURE IF EXISTS foo');
+        expect(stmts[1]).toContain('CREATE PROCEDURE foo');
+        expect(stmts[1]).toContain("SET @s := 'hi;';");
+        expect(stmts[2]).toBe('DROP PROCEDURE IF EXISTS foo');
+    });
+
+    it('strips DELIMITER lines from output even if no statement follows', () => {
+        expect(splitMysqlStatements('DELIMITER //\nDELIMITER ;\n')).toEqual(
+            [],
+        );
+    });
+
+    it('does not treat -- without trailing whitespace as a comment', () => {
+        // `--5` is "minus minus 5" (rare in practice but valid SQL).
+        // MySQL requires whitespace after `--` for it to be a comment.
+        expect(splitMysqlStatements('SELECT 1--5; SELECT 2;')).toEqual([
+            'SELECT 1--5',
+            'SELECT 2',
+        ]);
+    });
+});

src/backend/clients/database/splitMysqlStatements.ts

@@ -0,0 +1,211 @@
+/**
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+/**
+ * DELIMITER-aware splitter for MySQL dump / migration files.
+ *
+ * Splits `sql` into individual statements using the active statement
+ * delimiter (default `;`). Recognises `DELIMITER X` lines, single-quoted
+ * strings, double-quoted strings, backtick-quoted identifiers, line
+ * comments (`-- `, `#`) and block comments (`/* ... *\/`). DELIMITER
+ * directives are stripped from the output (they're a client-side concept,
+ * not server SQL).
+ *
+ * Returns trimmed, non-empty statements without the trailing delimiter.
+ */
+export function splitMysqlStatements(sql: string): string[] {
+    const out: string[] = [];
+    let buf = '';
+    let delim = ';';
+    let i = 0;
+    const n = sql.length;
+
+    // We process the input line-by-line for DELIMITER detection, but track
+    // multi-line state (strings / block comments) across lines.
+    type State =
+        | 'normal'
+        | 'sq' // single-quoted string
+        | 'dq' // double-quoted string
+        | 'bt' // backtick-quoted identifier
+        | 'block'; // /* ... */
+    let state: State = 'normal';
+
+    const pushStatement = () => {
+        const trimmed = buf.trim();
+        if (trimmed.length > 0) out.push(trimmed);
+        buf = '';
+    };
+
+    while (i < n) {
+        // At the start of each line in `normal` state, check for DELIMITER
+        // and full-line comments. We're at line start iff `i === 0` or the
+        // previous char was a newline.
+        const atLineStart = i === 0 || sql[i - 1] === '\n';
+        if (atLineStart && state === 'normal') {
+            // Find end of current line (without consuming).
+            let lineEnd = sql.indexOf('\n', i);
+            if (lineEnd === -1) lineEnd = n;
+            const line = sql.slice(i, lineEnd);
+
+            // DELIMITER directive — only valid when the current statement
+            // buffer is empty (i.e. between statements). MySQL CLI accepts
+            // it almost anywhere, but in practice it's always between
+            // statements; rejecting mid-statement keeps the parser simple
+            // and predictable.
+            const delimMatch = /^\s*DELIMITER\s+(\S+)\s*$/i.exec(line);
+            if (delimMatch && buf.trim() === '') {
+                delim = delimMatch[1];
+                // skip the line including the trailing newline (if any)
+                i = lineEnd + 1;
+                buf = '';
+                continue;
+            }
+        }
+
+        const c = sql[i];
+        const next = i + 1 < n ? sql[i + 1] : '';
+
+        if (state === 'sq') {
+            buf += c;
+            if (c === '\\' && i + 1 < n) {
+                buf += sql[i + 1];
+                i += 2;
+                continue;
+            }
+            if (c === "'") {
+                if (next === "'") {
+                    // SQL-style escaped quote
+                    buf += "'";
+                    i += 2;
+                    continue;
+                }
+                state = 'normal';
+            }
+            i++;
+            continue;
+        }
+
+        if (state === 'dq') {
+            buf += c;
+            if (c === '\\' && i + 1 < n) {
+                buf += sql[i + 1];
+                i += 2;
+                continue;
+            }
+            if (c === '"') {
+                if (next === '"') {
+                    buf += '"';
+                    i += 2;
+                    continue;
+                }
+                state = 'normal';
+            }
+            i++;
+            continue;
+        }
+
+        if (state === 'bt') {
+            buf += c;
+            if (c === '`') {
+                if (next === '`') {
+                    buf += '`';
+                    i += 2;
+                    continue;
+                }
+                state = 'normal';
+            }
+            i++;
+            continue;
+        }
+
+        if (state === 'block') {
+            buf += c;
+            if (c === '*' && next === '/') {
+                buf += '/';
+                i += 2;
+                state = 'normal';
+                continue;
+            }
+            i++;
+            continue;
+        }
+
+        // state === 'normal'
+        // Line comments: `-- ` or `--\n` or `--$` (MySQL requires whitespace
+        // or EOL after `--`); also `#` to EOL.
+        if (
+            (c === '-' &&
+                next === '-' &&
+                (sql[i + 2] === undefined || /\s/.test(sql[i + 2]))) ||
+            c === '#'
+        ) {
+            // consume to end-of-line; keep the comment in the buffer so the
+            // statement text remains faithful (mysql server tolerates it)
+            const lineEnd = sql.indexOf('\n', i);
+            const end = lineEnd === -1 ? n : lineEnd;
+            buf += sql.slice(i, end);
+            i = end;
+            continue;
+        }
+
+        if (c === '/' && next === '*') {
+            buf += '/*';
+            i += 2;
+            state = 'block';
+            continue;
+        }
+
+        if (c === "'") {
+            buf += c;
+            i++;
+            state = 'sq';
+            continue;
+        }
+
+        if (c === '"') {
+            buf += c;
+            i++;
+            state = 'dq';
+            continue;
+        }
+
+        if (c === '`') {
+            buf += c;
+            i++;
+            state = 'bt';
+            continue;
+        }
+
+        // Delimiter match
+        if (sql.startsWith(delim, i)) {
+            // emit current buffer (without the delimiter)
+            pushStatement();
+            i += delim.length;
+            continue;
+        }
+
+        buf += c;
+        i++;
+    }
+
+    // Flush trailing content (no terminating delimiter is allowed for the
+    // last statement, but we still try)
+    pushStatement();
+    return out;
+}

src/backend/types.ts

@@ -265,6 +265,14 @@ export interface IDatabaseConfig {
         password?: string;
         database?: string;
     };
+    /**
+     * Ordered list of directories whose `.sql` files are run sequentially at
+     * server start (mysql engine only). Files within a directory are sorted
+     * lexically; directories are processed in array order. Files MUST be
+     * idempotent — there is no per-file applied-state tracking.
+     * Relative paths resolve from `process.cwd()`.
+     */
+    migrationPaths?: string[];
 }
 
 /**