diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6ac88aee..50b53461 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -73,6 +73,53 @@ jobs: retention-days: 7 if-no-files-found: ignore + # Visual snapshots: pinned container = pixel-stable baselines. Advisory — + # never a required check. + e2e-visual: + runs-on: ubuntu-latest + timeout-minutes: 20 + container: + image: mcr.microsoft.com/playwright:v1.61.1-noble + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Verify container tag matches @playwright/test + run: | + node -e " + const v = require('./frontend/package.json').devDependencies['@playwright/test']; + if (v !== '1.61.1') { + console.error('::error::bump the e2e-visual container tag to v' + v + '-noble (and regenerate baselines)'); + process.exit(1); + } + " + + - name: Install pnpm + uses: pnpm/action-setup@v4 + with: + package_json_file: frontend/package.json + + - name: Install dependencies + working-directory: frontend + run: pnpm install --frozen-lockfile + + - name: Build + working-directory: frontend + run: pnpm run build + + - name: Run visual snapshots + working-directory: frontend + run: E2E_VISUAL=1 npx playwright test -c e2e/playwright.config.ts + + - name: Upload diffs + if: failure() + uses: actions/upload-artifact@v4 + with: + name: e2e-visual-diffs + path: frontend/e2e/test-results + retention-days: 7 + if-no-files-found: ignore + # Tier 2: branch-built image + isolated compose stack + the mock-LLM agent # loop. Not part of the PR gate — nightly and on-demand only. e2e-local: diff --git a/frontend/docs/e2e.md b/frontend/docs/e2e.md index e454c6db..71c39685 100644 --- a/frontend/docs/e2e.md +++ b/frontend/docs/e2e.md @@ -29,6 +29,9 @@ CI=1 pnpm e2e # byte-identical reproduction of a CI run ./e2e/tools/run-local-tier.sh # Tier 2: branch image + isolated docker # stack + mock LLM; runs specs/real/** E2E_TIER=stand E2E_BASE_URL=https://… pnpm e2e # against a live stand + +pnpm e2e:visual # visual snapshots (pinned container) +pnpm e2e:visual:update # regenerate baselines after a UI change ``` Tips for the mock tier: `vite preview` is reused between runs if you keep it @@ -49,6 +52,18 @@ docker daemon, and removes those sandboxes on exit. The deterministic agent transcript lives in `e2e/mock-llm/scenario.mjs`. Iterating: `E2E_SKIP_BUILD=1` reuses the image, `E2E_KEEP_STACK=1` leaves the stack up. +## Visual snapshots + +Baselines live in the repo (`e2e/specs/visual/*-snapshots/`, linux-suffixed) +and are generated ONLY inside the pinned `mcr.microsoft.com/playwright` +container — never run the visual project on the host: macOS pixels produce +parallel baselines that will never match CI. `pnpm e2e:visual` derives the +image tag from the installed `@playwright/test` version, builds `dist` on the +host, and compares inside the container; `pnpm e2e:visual:update` regenerates +baselines (commit them with the UI change that caused the diff). The xterm +canvas is masked — WebGL rendering is driver-dependent. The CI `e2e-visual` +job is advisory and never a required check. + ## Debugging a red CI run 1. Open the failed run (link in the PR comment) and download the `e2e-report` diff --git a/frontend/e2e/playwright.config.ts b/frontend/e2e/playwright.config.ts index 5a4f6641..33cc05a8 100644 --- a/frontend/e2e/playwright.config.ts +++ b/frontend/e2e/playwright.config.ts @@ -7,6 +7,10 @@ export type BackendTier = 'local' | 'mock' | 'stand'; const tier = (process.env.E2E_TIER ?? 'mock') as BackendTier; const isCI = Boolean(process.env.CI); +// Visual snapshots only ever run inside the pinned Playwright Linux container +// (e2e/tools/run-visual.sh) — a darwin run would generate parallel baselines +// that never match CI pixels. +const isVisual = process.env.E2E_VISUAL === '1'; // `vite preview` listens on VITE_PORT + 100 and reuses the dev proxy config. const PREVIEW_PORT = 8100; @@ -38,11 +42,17 @@ export default defineConfig({ projects: tier === 'mock' ? [ - { - name: 'mock-chromium', - testIgnore: '**/specs/real/**', - use: { ...devices['Desktop Chrome'] }, - }, + isVisual + ? { + name: 'visual', + testMatch: '**/specs/visual/**', + use: { ...devices['Desktop Chrome'] }, + } + : { + name: 'mock-chromium', + testIgnore: ['**/specs/real/**', '**/specs/visual/**'], + use: { ...devices['Desktop Chrome'] }, + }, ] : [ { @@ -83,7 +93,9 @@ export default defineConfig({ webServer: tier === 'mock' ? { - command: 'pnpm run build && pnpm exec vite preview', + // The visual container cannot load host-built native vite + // binaries — it serves a pre-built dist with plain Node. + command: isVisual ? 'node e2e/tools/serve-dist.mjs' : 'pnpm run build && pnpm exec vite preview', cwd: fileURLToPath(new URL('..', import.meta.url)), env: { VITE_PORT: '8000', VITE_USE_HTTPS: 'false' }, reuseExistingServer: !isCI, diff --git a/frontend/e2e/specs/visual/routes.spec.ts b/frontend/e2e/specs/visual/routes.spec.ts new file mode 100644 index 00000000..192f615f --- /dev/null +++ b/frontend/e2e/specs/visual/routes.spec.ts @@ -0,0 +1,30 @@ +import { expect, test } from '../../fixtures/test.ts'; +import { ROUTE_MANIFEST } from '../../routes.ts'; + +const THEMES = ['light', 'dark'] as const; + +const slugify = (path: string) => path.replaceAll('/', '-').replace(/^-/, '') || 'root'; + +test.describe('visual', { tag: '@visual' }, () => { + for (const entry of ROUTE_MANIFEST) { + for (const theme of THEMES) { + test.describe(`${entry.path} ${theme}`, () => { + test.use({ cassette: entry.cassette() }); + + test('matches the baseline', async ({ page }) => { + await page.addInitScript((value) => window.localStorage.setItem('theme', String(value)), theme); + await page.goto(entry.path); + await expect(entry.ready(page)).toBeVisible(); + await page.evaluate(() => document.fonts.ready); + + await expect(page).toHaveScreenshot(`${slugify(entry.path)}-${theme}.png`, { + fullPage: true, + // The terminal is a WebGL canvas: SwiftShader pixels differ + // between runners, so its region is masked, not compared. + mask: [page.locator('.xterm')], + }); + }); + }); + } + } +}); diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.png new file mode 100644 index 00000000..a28c93b6 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.png new file mode 100644 index 00000000..70137b7d Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/dashboard-light-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png new file mode 100644 index 00000000..ccbc0902 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png new file mode 100644 index 00000000..76ac600a Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-5-light-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.png new file mode 100644 index 00000000..2018a49e Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.png new file mode 100644 index 00000000..5bf619f9 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/flows-light-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.png new file mode 100644 index 00000000..c6787e6f Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.png new file mode 100644 index 00000000..67aa88c6 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/knowledges-light-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.png new file mode 100644 index 00000000..b6bd0d85 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.png new file mode 100644 index 00000000..a7e8810a Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/resources-light-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.png new file mode 100644 index 00000000..6fd962a1 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.png new file mode 100644 index 00000000..9cd1edef Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-api-tokens-light-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.png new file mode 100644 index 00000000..9ee1ebe0 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.png new file mode 100644 index 00000000..2ee8c294 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-prompts-light-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.png new file mode 100644 index 00000000..336e0601 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.png new file mode 100644 index 00000000..b873bbf9 Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/settings-providers-light-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.png new file mode 100644 index 00000000..28c1779f Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-dark-visual-linux.png differ diff --git a/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.png b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.png new file mode 100644 index 00000000..d93d013c Binary files /dev/null and b/frontend/e2e/specs/visual/routes.spec.ts-snapshots/templates-light-visual-linux.png differ diff --git a/frontend/e2e/tools/run-visual.sh b/frontend/e2e/tools/run-visual.sh new file mode 100755 index 00000000..d0903c09 --- /dev/null +++ b/frontend/e2e/tools/run-visual.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Visual snapshots run ONLY inside the pinned Playwright container so baselines +# are pixel-stable across every machine and CI. The image tag is derived from +# the installed @playwright/test version — the pin cannot drift by construction. +# Usage: run-visual.sh [--update] (E2E_SKIP_BUILD=1 reuses the existing dist) +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +FRONTEND="$REPO_ROOT/frontend" + +PLAYWRIGHT_VERSION="$(node -p "require('$FRONTEND/node_modules/@playwright/test/package.json').version")" +IMAGE="mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" + +if [[ "${E2E_SKIP_BUILD:-}" != "1" ]]; then + (cd "$FRONTEND" && corepack pnpm run build) +fi + +UPDATE_ARGS=() +if [[ "${1:-}" == "--update" ]]; then + UPDATE_ARGS=(--update-snapshots) + shift +fi + +docker run --rm \ + -v "$REPO_ROOT:/work" \ + -w /work/frontend \ + -e E2E_VISUAL=1 \ + -e CI="${CI:-}" \ + --ipc=host \ + "$IMAGE" \ + npx playwright test -c e2e/playwright.config.ts ${UPDATE_ARGS[@]+"${UPDATE_ARGS[@]}"} "$@" diff --git a/frontend/e2e/tools/serve-dist.mjs b/frontend/e2e/tools/serve-dist.mjs new file mode 100644 index 00000000..03e57757 --- /dev/null +++ b/frontend/e2e/tools/serve-dist.mjs @@ -0,0 +1,36 @@ +// Static server for the built dist with an SPA fallback. The visual project +// runs inside the pinned Playwright container where the host-built (darwin) +// native vite binaries cannot load — this replaces `vite preview` with +// dependency-free Node. API calls never reach it: Playwright route mocks +// intercept them in the browser. +import { createReadStream, existsSync, statSync } from 'node:fs'; +import { createServer } from 'node:http'; +import { extname, join, normalize } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const DIST = fileURLToPath(new URL('../../dist', import.meta.url)); +const PORT = Number(process.env.PORT ?? 8100); + +const MIME = { + '.css': 'text/css', + '.html': 'text/html', + '.ico': 'image/x-icon', + '.js': 'text/javascript', + '.json': 'application/json', + '.png': 'image/png', + '.svg': 'image/svg+xml', + '.webp': 'image/webp', + '.woff2': 'font/woff2', +}; + +createServer((request, response) => { + const pathname = normalize(new URL(request.url ?? '/', 'http://localhost').pathname).replace(/^(\.\.[/\\])+/, ''); + let filePath = join(DIST, pathname); + + if (!existsSync(filePath) || statSync(filePath).isDirectory()) { + filePath = join(DIST, 'index.html'); + } + + response.writeHead(200, { 'content-type': MIME[extname(filePath)] ?? 'application/octet-stream' }); + createReadStream(filePath).pipe(response); +}).listen(PORT, () => console.log(`[serve-dist] ${DIST} on :${PORT}`)); diff --git a/frontend/package.json b/frontend/package.json index c905c7d2..a14d4ca5 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -8,6 +8,8 @@ "e2e": "playwright test -c e2e/playwright.config.ts", "e2e:setup": "playwright install chromium", "e2e:ui": "playwright test -c e2e/playwright.config.ts --ui", + "e2e:visual": "./e2e/tools/run-visual.sh", + "e2e:visual:update": "./e2e/tools/run-visual.sh --update", "graphql:generate": "graphql-codegen --config graphql-codegen.ts", "lint": "eslint \"{src,e2e}/**/*.{ts,tsx,js,jsx}\"", "lint:fix": "eslint \"{src,e2e}/**/*.{ts,tsx,js,jsx}\" --fix",