diff --git a/doc/testing-replica-lag.md b/doc/testing-replica-lag.md new file mode 100644 index 000000000..f4153ee14 --- /dev/null +++ b/doc/testing-replica-lag.md @@ -0,0 +1,107 @@ +# Testing against a lagging read replica + +Some bugs only exist between a primary and a replica that has not caught up: +a row is deleted on the primary, a reader is served the old copy by the +replica, and writes referencing it then fail their foreign keys. + +Local development runs on sqlite, which has neither a replica nor foreign-key +enforcement, so neither half of that can happen. Unit tests inject the driver +error instead. This setup runs the real thing: two MySQL servers in actual +replication, where `STOP REPLICA` freezes the follower on demand. + +## Setup + +Assumes the usual local `puter-mysql` container (mysql:8, port 3306, +`root`/`puter`). Start a follower on 3307: + +```bash +docker run -d --name puter-mysql-replica \ + -e MYSQL_ROOT_PASSWORD=puter -e MYSQL_DATABASE=puter \ + -p 3307:3306 mysql:8 \ + --server-id=2 --log-bin=mysql-bin --relay-log=relay-bin + +until docker exec puter-mysql-replica mysqladmin -uroot -pputer ping >/dev/null 2>&1 +do sleep 2; done +``` + +Give the primary a replication user: + +```bash +docker exec puter-mysql mysql -uroot -pputer -e " +CREATE USER IF NOT EXISTS 'repl'@'%' IDENTIFIED BY 'replpass'; +GRANT REPLICATION SLAVE ON *.* TO 'repl'@'%'; +FLUSH PRIVILEGES;" +``` + +Seed the follower and point it at the primary. `--source-data=2` records the +binlog coordinates the dump was taken at, which is where the follower starts: + +```bash +docker exec puter-mysql mysqldump -uroot -pputer \ + --source-data=2 --single-transaction --databases puter > /tmp/primary.sql + +grep -m1 'CHANGE REPLICATION SOURCE' /tmp/primary.sql +# -- CHANGE REPLICATION SOURCE TO SOURCE_LOG_FILE='binlog.000006', SOURCE_LOG_POS=717382; + +docker exec -i puter-mysql-replica mysql -uroot -pputer < /tmp/primary.sql + +PRIMARY_IP=$(docker inspect puter-mysql \ + --format '{{.NetworkSettings.Networks.bridge.IPAddress}}') + +docker exec puter-mysql-replica mysql -uroot -pputer -e " +STOP REPLICA; RESET REPLICA ALL; +CHANGE REPLICATION SOURCE TO + SOURCE_HOST='${PRIMARY_IP}', SOURCE_PORT=3306, + SOURCE_USER='repl', SOURCE_PASSWORD='replpass', + SOURCE_LOG_FILE='', SOURCE_LOG_POS=, + GET_SOURCE_PUBLIC_KEY=1; +START REPLICA;" +``` + +`GET_SOURCE_PUBLIC_KEY=1` is required because MySQL 8 defaults to +`caching_sha2_password` and this link has no TLS. Confirm both threads are up: + +```bash +docker exec puter-mysql-replica mysql -uroot -pputer -e "SHOW REPLICA STATUS\G" \ + | grep -E 'Replica_IO_Running|Replica_SQL_Running:|Last_Error' +``` + +## Running + +```bash +PUTER_TEST_REPLICA_LAG=1 npx vitest run \ + --config src/backend/vitest.config.ts \ + src/backend/stores/replicaLag.integration.test.ts +``` + +Without the env var the suite skips, so it stays inert in CI and for anyone +without the containers. It builds its own throwaway database +(`puter_replica_lag_verify`), runs the MySQL migrations into it, and drops it +afterwards — your dev data is never touched. + +Container names are overridable via `PUTER_TEST_REPLICA_PRIMARY` and +`PUTER_TEST_REPLICA_FOLLOWER`. + +## Writing a case + +`freeze()` stops replication; everything after it exists only on the primary. +An `afterEach` thaws unconditionally, so a failing assertion cannot strand the +follower and starve later cases of their fixture rows. + +```ts +const app = await makeApp(server, user.id); +await settle(); // let the follower catch up +await server.stores.app.getByUid(app.uid); // warm the cache + +freeze(); // follower is now behind +await server.stores.app.delete(app.id); // primary only + +expect(await server.stores.app.getByUid(app.uid)).toBeNull(); +``` + +## Teardown + +```bash +docker rm -f puter-mysql-replica +docker exec puter-mysql mysql -uroot -pputer -e "DROP USER IF EXISTS 'repl'@'%';" +``` diff --git a/src/backend/stores/replicaLag.integration.test.ts b/src/backend/stores/replicaLag.integration.test.ts new file mode 100644 index 000000000..e767c780a --- /dev/null +++ b/src/backend/stores/replicaLag.integration.test.ts @@ -0,0 +1,266 @@ +/* + * Copyright (C) 2024-present Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Deleted rows must not come back from a replica that hasn't caught up. + * + * Two real MySQL servers in replication, nothing mocked: `STOP REPLICA` freezes + * the follower and MySQL raises the foreign-key errors itself. The co-located + * unit tests inject them, since sqlite has neither a replica nor FK + * enforcement. + * + * Needs both containers, so it is opt-in and skipped in CI — setup and usage in + * doc/testing-replica-lag.md. + */ + +import { execFileSync } from 'node:child_process'; +import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'; +import { runWithContext } from '../core/context.ts'; +import { optionalEnv, skipUnlessEnv } from '../drivers/integrationTestUtil.js'; +import { PuterServer } from '../server.ts'; +import { setupTestServer } from '../testUtil.ts'; + +const ENV_VAR = 'PUTER_TEST_REPLICA_LAG'; +const PRIMARY = optionalEnv('PUTER_TEST_REPLICA_PRIMARY') ?? 'puter-mysql'; +const REPLICA = + optionalEnv('PUTER_TEST_REPLICA_FOLLOWER') ?? 'puter-mysql-replica'; +const DB = 'puter_replica_lag_verify'; + +const mysql = (container: string, statement: string): string => + execFileSync( + 'docker', + [ + 'exec', + container, + 'mysql', + '-uroot', + '-pputer', + '-N', + '-e', + statement, + ], + { encoding: 'utf8', stdio: ['ignore', 'pipe', 'ignore'] }, + ).trim(); + +const onPrimary = (s: string) => mysql(PRIMARY, s); +const onReplica = (s: string) => mysql(REPLICA, s); +const settle = (ms = 1500) => new Promise((r) => setTimeout(r, ms)); + +/** Freeze the follower: everything after this is primary-only. */ +const freeze = () => onReplica('STOP REPLICA;'); +const thaw = async () => { + onReplica('START REPLICA;'); + await settle(); +}; + +const randomName = (prefix: string) => + `${prefix}_${Math.random().toString(36).slice(2, 10)}`; + +const makeUser = (server: PuterServer) => + server.stores.user.create({ + username: randomName('rl'), + uuid: crypto.randomUUID(), + password: null, + email: null, + }); + +const makeApp = (server: PuterServer, ownerUserId: number) => + ( + server.stores.app.create as unknown as ( + fields: Record, + opts: { ownerUserId: number }, + ) => Promise<{ uid: string; id: number; name: string }> + )( + { + name: randomName('rlapp'), + title: 'replica lag', + index_url: 'https://example.test/rl.html', + }, + { ownerUserId }, + ); + +describe.skipIf(skipUnlessEnv(ENV_VAR))( + 'a row deleted while a replica lags (integration)', + () => { + let server: PuterServer; + + beforeAll(async () => { + onReplica('START REPLICA;'); + onPrimary( + `DROP DATABASE IF EXISTS \`${DB}\`; CREATE DATABASE \`${DB}\`;`, + ); + await settle(2000); + + server = await setupTestServer({ + database: { + engine: 'mysql', + host: '127.0.0.1', + port: 3306, + user: 'root', + password: 'puter', + database: DB, + migrationPaths: [ + './src/backend/clients/database/migrations/mysql', + ], + replica: { + host: '127.0.0.1', + port: 3307, + user: 'root', + password: 'puter', + database: DB, + }, + }, + } as never); + + // Let the follower apply everything the migrations just wrote. + await settle(3000); + }, 300_000); + + afterAll(async () => { + try { + onReplica('START REPLICA;'); + } catch { + // best effort + } + await server?.shutdown(); + try { + onPrimary(`DROP DATABASE IF EXISTS \`${DB}\`;`); + } catch { + // best effort + } + }); + + // Unconditional: a failed assertion must not leave the follower frozen. + afterEach(async () => { + await thaw(); + }); + + it('is really stale on the follower — the premise of every case below', async () => { + const user = await makeUser(server); + const app = await makeApp(server, user.id); + await settle(); + + freeze(); + await server.stores.app.delete(app.id); + + expect( + onPrimary( + `SELECT COUNT(*) FROM \`${DB}\`.apps WHERE uid='${app.uid}';`, + ), + ).toBe('0'); + expect( + onReplica( + `SELECT COUNT(*) FROM \`${DB}\`.apps WHERE uid='${app.uid}';`, + ), + ).toBe('1'); + }); + + it('does not let the follower put a deleted app back in cache', async () => { + const user = await makeUser(server); + const app = await makeApp(server, user.id); + await settle(); + + await server.stores.app.getByUid(app.uid); + expect( + await server.clients.redis.get(`apps:uid:${app.uid}`), + ).not.toBeNull(); + + freeze(); + await server.stores.app.delete(app.id); + + expect(await server.stores.app.getByUid(app.uid)).toBeNull(); + expect( + await server.clients.redis.get(`apps:uid:${app.uid}`), + ).toBeNull(); + expect( + await server.clients.redis.get(`apps:uid:${app.uid}:deleted`), + ).not.toBeNull(); + }); + + it('does not let the follower put a deleted account back in cache', async () => { + const user = await makeUser(server); + await settle(); + + await server.stores.user.getByUsername(user.username); + expect( + await server.clients.redis.get( + `users:username:${user.username}`, + ), + ).not.toBeNull(); + + freeze(); + await server.services.userAccount.cascadeDelete(user.id); + + expect( + onReplica( + `SELECT COUNT(*) FROM \`${DB}\`.user WHERE id=${user.id};`, + ), + ).toBe('1'); + expect( + await server.stores.user.getByUsername(user.username), + ).toBeNull(); + expect( + await server.clients.redis.get( + `users:username:${user.username}`, + ), + ).toBeNull(); + }); + + // Raw DELETE leaves the cache warm — the state a lost invalidation leaves. + it('turns a real foreign-key rejection into 404, not an unhandled error', async () => { + const user = await makeUser(server); + const app = await makeApp(server, user.id); + await settle(); + await server.stores.app.getByUid(app.uid); + + freeze(); + await server.clients.db.write('DELETE FROM `apps` WHERE `id` = ?', [ + app.id, + ]); + + const actor = { user } as never; + await expect( + runWithContext({ actor }, () => + server.services.permission.grantUserAppPermission( + actor, + app.uid, + 'test:replica-lag', + {}, + {}, + ), + ), + ).rejects.toMatchObject({ statusCode: 404 }); + }); + + it('turns a real foreign-key rejection on the session insert into 401', async () => { + const user = await makeUser(server); + await settle(); + await server.stores.user.getById(user.id); + + freeze(); + await server.clients.db.write('DELETE FROM `user` WHERE `id` = ?', [ + user.id, + ]); + + await expect( + server.stores.session.create(user.id, { kind: 'web' }), + ).rejects.toMatchObject({ statusCode: 401 }); + }); + }, +);