ci: shard backend tests and share workers across server-booting files (#4149)

The suite was CPU-bound on one runner, and about half of that CPU went to
re-importing the backend for every test file.

- Run the suite as 4 shards and merge their blob reports for coverage.
  `test (pr)` stays as the required check and fails if any shard fails.
- Drop the per-PR base leg; pushes to main publish the coverage PRs
  compare against.
- Files that boot a server without module mocks or extensions run in a
  `shared-worker` vitest project (`isolate: false`), so the backend is
  imported once per worker. testSharedWorkerSetup.ts resets the redis
  mock, configContainer and extensionStore between files.

.github/workflows/backend-tests.yaml

@@ -1,17 +1,21 @@
 name: Backend Tests
 
-# Always triggers so it can be a required PR check; the `changes` job below
+# Always triggers on PRs so it can be a required check; the `changes` job below
 # decides whether backend/extension paths actually changed (test-only changes
-# match the same globs, since tests are co-located with the code), and every
-# other job is skipped — not absent — when they didn't, which still satisfies
-# a required check. puter.js SDK changes are covered by puterjs-tests.yaml.
+# match the same globs, since tests are co-located with the code), and `test`
+# still reports success when they didn't. Pushes to main produce the coverage
+# PRs compare against. puter.js SDK changes are covered by puterjs-tests.yaml.
 on:
   pull_request:
     types: [opened, synchronize, reopened]
+  push:
+    branches: [main]
 
 permissions:
   contents: read
   pull-requests: write
+  # Reads main's coverage artifact for the PR comparison.
+  actions: read
 
 jobs:
   changes:
@@ -42,12 +46,9 @@ jobs:
   # The consuming repo runs an equivalent gate, but only once someone bumps the
   # submodule pointer — too late to keep the error off this repo's default
   # branch. Hence a gate here, on this repo's own pull requests.
-  #
-  # Its own job rather than a step in `test`: that one runs a base/PR matrix for
-  # coverage comparison, and the check only needs the PR ref, once.
   typecheck:
     needs: changes
-    if: needs.changes.outputs.backend == 'true'
+    if: github.event_name == 'pull_request' && needs.changes.outputs.backend == 'true'
     runs-on: ubuntu-latest
     steps:
       - name: Checkout repository
@@ -65,85 +66,146 @@ jobs:
       - name: Type check (new errors only)
         run: npm run typecheck
 
-  # `needs.changes` gates the steps below, not this job, on purpose: a
-  # job-level `if` on a matrix job skips the whole job as one unexpanded
-  # check ("test (${{ matrix.artifact }})", literally, since there's no
-  # matrix context to fill it in) instead of the two real per-artifact
-  # checks a required rule pins to ("test (base)" / "test (pr)") — so the
-  # required check never gets reported and the PR blocks forever. Letting
-  # the matrix always expand keeps the check names stable either way.
-  test:
+  # The suite is CPU-bound, so it's split across runners. Each shard writes a
+  # blob report (results plus coverage) that the `coverage` job merges.
+  test-shard:
     needs: changes
-    # Overrides the default matrix naming, which would otherwise bake the
-    # base/head branch name (matrix.ref) into the check name — making it a
-    # different string on every PR and impossible to pin as a required check.
-    name: test (${{ matrix.artifact }})
+    if: needs.changes.outputs.backend == 'true'
+    name: test shard ${{ matrix.shard }}/${{ strategy.job-total }}
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
-        include:
-          - ref: ${{ github.base_ref }}
-            artifact: base
-          - ref: ${{ github.head_ref }}
-            artifact: pr
+        shard: [1, 2, 3, 4]
 
     steps:
-      - name: Checkout ${{ matrix.ref }}
-        if: needs.changes.outputs.backend == 'true'
+      - name: Checkout
         uses: actions/checkout@v7
         with:
-          ref: ${{ matrix.ref }}
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
 
       - name: Set up Node.js
-        if: needs.changes.outputs.backend == 'true'
         uses: actions/setup-node@v7
         with:
           node-version: '24'
           cache: 'npm'
 
       - name: Install dependencies
-        if: needs.changes.outputs.backend == 'true'
         run: npm ci
 
       - name: Run backend tests with coverage
-        if: needs.changes.outputs.backend == 'true'
-        run: npm run test:backend -- --coverage
+        run: >-
+          npm run test:backend --
+          --coverage
+          --shard=${{ matrix.shard }}/${{ strategy.job-total }}
+          --reporter=default
+          --reporter=blob
+        env:
+          CI: 'true'
+
+      - name: Upload blob report
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: backend-blob-${{ matrix.shard }}
+          path: .vitest-reports/
+          include-hidden-files: true
+          retention-days: 1
+
+  # Required check, so its PR name is pinned. A skipped job would satisfy it,
+  # so this always runs and fails on any shard that didn't pass.
+  test:
+    needs: [changes, test-shard]
+    if: ${{ !cancelled() }}
+    name: test (${{ github.event_name == 'push' && 'main' || 'pr' }})
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check shard results
+        env:
+          CHANGES_RESULT: ${{ needs.changes.result }}
+          BACKEND_CHANGED: ${{ needs.changes.outputs.backend }}
+          SHARDS_RESULT: ${{ needs.test-shard.result }}
+        run: |
+          if [ "$CHANGES_RESULT" != "success" ]; then
+            echo "changes job: $CHANGES_RESULT"
+            exit 1
+          fi
+          if [ "$BACKEND_CHANGED" != "true" ]; then
+            echo "No backend changes."
+            exit 0
+          fi
+          if [ "$SHARDS_RESULT" != "success" ]; then
+            echo "test shards: $SHARDS_RESULT"
+            exit 1
+          fi
+
+  coverage:
+    needs: [changes, test-shard]
+    if: ${{ !cancelled() && needs.test-shard.result == 'success' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v7
+        with:
+          node-version: '24'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Download blob reports
+        uses: actions/download-artifact@v8
+        with:
+          pattern: backend-blob-*
+          path: .vitest-reports
+          merge-multiple: true
+
+      - name: Merge shard coverage
+        run: npx vitest run --config src/backend/vitest.config.ts --merge-reports --coverage
         env:
           CI: 'true'
 
       - name: Upload coverage artifact
-        if: needs.changes.outputs.backend == 'true' && !cancelled()
         uses: actions/upload-artifact@v7
         with:
-          name: backend-coverage-${{ matrix.artifact }}
+          name: backend-coverage-${{ github.event_name == 'push' && 'main' || 'pr' }}
           path: src/backend/coverage/
-          retention-days: 14
+          retention-days: ${{ github.event_name == 'push' && 30 || 14 }}
 
-  report-coverage:
-    needs: [changes, test]
-    if: always() && needs.changes.outputs.backend == 'true'
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v7
-
-      - name: Download PR coverage
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-coverage-pr
-          path: src/backend/coverage
-
-      - name: Download base coverage
-        uses: actions/download-artifact@v8
-        with:
-          name: backend-coverage-base
-          path: coverage-base
+      # Latest main coverage from this repo's own runs; a fork can't publish
+      # under this repository id.
+      - name: Download main coverage
+        id: base
+        if: github.event_name == 'pull_request'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+        run: |
+          id=$(gh api "repos/$REPO/actions/artifacts?name=backend-coverage-main&per_page=20" --jq '
+            [.artifacts[]
+              | select(.expired | not)
+              | select(.workflow_run.head_branch == "main")
+              | select(.workflow_run.head_repository_id == .workflow_run.repository_id)
+            ][0].id // empty')
+          if [ -z "$id" ]; then
+            echo "No main coverage artifact; reporting without a comparison."
+            exit 0
+          fi
+          gh api "repos/$REPO/actions/artifacts/$id/zip" > coverage-base.zip
+          unzip -q coverage-base.zip -d coverage-base
+          echo "found=true" >> "$GITHUB_OUTPUT"
 
       - name: Report coverage on PR
+        if: github.event_name == 'pull_request'
         uses: davelosert/vitest-coverage-report-action@v2
         with:
           json-summary-path: src/backend/coverage/coverage-summary.json
           json-final-path: src/backend/coverage/coverage-final.json
-          json-summary-compare-path: coverage-base/coverage-summary.json
+          json-summary-compare-path: ${{ steps.base.outputs.found == 'true' && 'coverage-base/coverage-summary.json' || '' }}

.gitignore

@@ -66,6 +66,7 @@ AGENTS.md
 
 
 coverage/
+.vitest-reports/
 *.log
 undefined
 servers.json

src/backend/testSharedWorkerSetup.ts

@@ -0,0 +1,87 @@
+/*
+ * Copyright (C) 2024-present Puter Technologies Inc.
+ *
+ * This file is part of Puter.
+ *
+ * Puter is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published
+ * by the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+// Setup for the `shared-worker` vitest project, whose files run with
+// `isolate: false` and so share one module graph per worker. Resets the
+// process-wide state that a fresh worker would otherwise have provided.
+import MockRedis from 'ioredis-mock';
+import { afterAll, beforeAll, vi } from 'vitest';
+import { configContainer } from './exports';
+import { extensionStore } from './extensions';
+// Loaded before the snapshot so anything registered at import time is part
+// of the pristine state.
+import './server';
+
+type Container = unknown[] | Record<string, unknown>;
+
+// Copies one level deeper than the container: event listener lists are
+// arrays inside `events`, and registering pushes into them.
+const copyContainer = (value: Container): Container =>
+    Array.isArray(value)
+        ? [...value]
+        : Object.fromEntries(
+              Object.entries(value).map(([k, v]) => [
+                  k,
+                  Array.isArray(v) ? [...v] : v,
+              ]),
+          );
+
+// This file is re-evaluated for every test file, so the pristine snapshot
+// lives on globalThis, taken before the first file in the worker runs.
+const SNAPSHOT_KEY = Symbol.for('puter.test.extensionStoreSnapshot');
+const globals = globalThis as {
+    [SNAPSHOT_KEY]?: Record<string, Container>;
+};
+globals[SNAPSHOT_KEY] ??= Object.fromEntries(
+    Object.entries(extensionStore).map(([k, v]) => [k, copyContainer(v)]),
+);
+
+const restoreExtensionStore = () => {
+    for (const [key, pristine] of Object.entries(globals[SNAPSHOT_KEY]!)) {
+        // The server reads these containers by reference, so refill in place.
+        const live = (extensionStore as Record<string, Container>)[key];
+        const fresh = copyContainer(pristine);
+        if (Array.isArray(live)) {
+            live.splice(0, live.length, ...(fresh as unknown[]));
+        } else {
+            for (const k of Object.keys(live)) delete live[k];
+            Object.assign(live, fresh);
+        }
+    }
+};
+
+beforeAll(async () => {
+    // Every mock client shares the default host/port store, so cached rows
+    // from a previous file's server would otherwise survive.
+    const redis = new MockRedis();
+    await redis.flushall();
+    redis.disconnect();
+
+    for (const key of Object.keys(configContainer)) {
+        delete (configContainer as unknown as Record<string, unknown>)[key];
+    }
+    restoreExtensionStore();
+});
+
+afterAll(() => {
+    vi.unstubAllEnvs();
+    vi.unstubAllGlobals();
+    vi.restoreAllMocks();
+    vi.useRealTimers();
+});

src/backend/vitest.config.ts

@@ -18,6 +18,7 @@
  */
 
 // vite.config.ts - Vite configuration for Puter API tests (TypeScript)
+import { globSync, readFileSync } from 'node:fs';
 import path from 'node:path';
 import { transform } from 'esbuild';
 import { loadEnv } from 'vite';
@@ -44,6 +45,46 @@ const postgresOnlyTests = [
     'src/backend/services/appIcon/AppIconService.test.ts',
 ];
 
+const testInclude = [
+    'src/backend/**/*.test.{js,ts}',
+    'extensions/**/*.test.{js,ts}',
+    // The MCP connector's signed-upload tools call the `/fs` HTTP API
+    // directly, so their tests need a booted backend (`setupPuterTestEnv`).
+    'src/mcp-connector/**/*.test.{js,ts}',
+    // Root-level tools/ scripts are exercised through this suite.
+    'tools/**/*.test.mjs',
+    // The worker runtimes ship as a preamble rather than as their own
+    // package, so their unit tests run with the backend's.
+    'src/worker/**/*.test.{js,ts}',
+];
+
+const testExclude = [
+    ...configDefaults.exclude,
+    ...(isCi && !isPgmockMode ? postgresOnlyTests : []),
+];
+
+// Test files that boot a server pay most of their runtime importing the
+// backend. Sharing a module graph per worker (`isolate: false`) pays that once
+// per worker instead; testSharedWorkerSetup.ts resets process-wide state
+// between files. Stays isolated: module mocks (can't be undone in a shared
+// graph), anything loading extensions (they register into the process-wide
+// extension store and read config only on first import), and pgmock.
+const sharedWorkerTests = isPgmockMode
+    ? []
+    : globSync(testInclude, {
+          cwd: repoRoot,
+          exclude: (p) => path.basename(String(p)) === 'node_modules',
+      }).filter((file) => {
+          if (file.startsWith('extensions/')) return false;
+          if (postgresOnlyTests.includes(file)) return false;
+          const source = readFileSync(path.join(repoRoot, file), 'utf8');
+          return (
+              /\bsetupTestServer\(/.test(source) &&
+              !/\bsetupPuterTestEnv\(/.test(source) &&
+              !/\bvi\.(mock|doMock)\(/.test(source)
+          );
+      });
+
 // Vite 8's oxc transform leaves TC39 stage-3 decorators in place
 // (used by `@Controller`/`@Post`), so they reach Node verbatim and
 // crash with "SyntaxError: Invalid or unexpected token". Pre-transform
@@ -113,25 +154,37 @@ export default defineConfig(({ mode }) => ({
             reportsDirectory: path.join(backendDir, 'coverage'),
         },
         env: loadEnv(mode, '', 'PUTER_'),
-        include: [
-            'src/backend/**/*.test.{js,ts}',
-            'extensions/**/*.test.{js,ts}',
-            // The MCP connector's signed-upload tools call the `/fs` HTTP API
-            // directly, so their tests need a booted backend (`setupPuterTestEnv`).
-            'src/mcp-connector/**/*.test.{js,ts}',
-            // Root-level tools/ scripts are exercised through this suite.
-            'tools/**/*.test.mjs',
-            // The worker runtimes ship as a preamble rather than as their own
-            // package, so their unit tests run with the backend's.
-            'src/worker/**/*.test.{js,ts}',
-        ],
-        exclude: [
-            ...configDefaults.exclude,
-            ...(isCi && !isPgmockMode ? postgresOnlyTests : []),
-        ],
         // Root is the repo root so that the file transformer (which
         // applies `lowerDecoratorsPlugin`) sees both src/backend and
         // extensions/ — vitest skips transform for files outside root.
         root: repoRoot,
+        // `extends: true` concatenates arrays, so include/exclude live only
+        // on the projects.
+        projects: [
+            {
+                extends: true,
+                test: {
+                    name: 'isolated',
+                    include: testInclude,
+                    exclude: [...testExclude, ...sharedWorkerTests],
+                },
+            },
+            ...(sharedWorkerTests.length > 0
+                ? [
+                      {
+                          extends: true as const,
+                          test: {
+                              name: 'shared-worker',
+                              include: sharedWorkerTests,
+                              exclude: testExclude,
+                              isolate: false,
+                              setupFiles: [
+                                  'src/backend/testSharedWorkerSetup.ts',
+                              ],
+                          },
+                      },
+                  ]
+                : []),
+        ],
     },
 }));