Change PR build to generate coverage and comment it (#1474)

This extends the github actions to by default run the coverage
generation targets. The targets will generate the coverage artifacts,
which will then be used by a seprate github action to comment on the PR
the coverage numbers.

This is a sanity check for testing information for new features and
refactors alike. It will allow reviewers to quickly identify if there
are major issues with new tests, or large changes to coverage percentage
that indicate a problem.

.github/workflows/common.yml

@@ -10,6 +10,14 @@ on:
         type: boolean
         description: "Publish this build for release"
         default: false
+      coverage:
+        type: boolean
+        description: "Run tests with code coverage enabled"
+        default: false
+      pr_number:
+        type: string
+        description: "PR number for coverage artifact naming (required when coverage is true)"
+        default: ""
 
 jobs:
   buildAndTest:
@@ -64,13 +72,23 @@ jobs:
         env:
           DEVELOPER_DIR: "/Applications/Xcode-latest.app/Contents/Developer"
 
+      - name: Validate coverage inputs
+        if: ${{ inputs.coverage }}
+        env:
+          PR_NUMBER: ${{ inputs.pr_number }}
+        run: |
+          if [ -z "${PR_NUMBER}" ]; then
+            echo "::error::pr_number input is required when coverage is true"
+            exit 1
+          fi
+
       - name: Create package
         run: |
           mkdir -p outputs
           mv "bin/${BUILD_CONFIGURATION}/container-installer-unsigned.pkg" outputs
           mv "bin/${BUILD_CONFIGURATION}/bundle/container-dSYM.zip" outputs
 
-      - name: Test the container project
+      - name: Set up test environment
         run: |
           APP_ROOT=$(mktemp -d -p "${RUNNER_TEMP}")
           LOG_ROOT="${APP_ROOT}/logs"
@@ -82,10 +100,53 @@ jobs:
           echo no_proxy=${no_proxy}
           echo "APP_ROOT=${APP_ROOT}" >> $GITHUB_ENV
           echo "LOG_ROOT=${LOG_ROOT}" >> $GITHUB_ENV
-          make APP_ROOT="${APP_ROOT}" LOG_ROOT="${LOG_ROOT}" test install-kernel integration
+
+      - name: Test the container project
+        if: ${{ !inputs.coverage }}
+        run: make APP_ROOT="${APP_ROOT}" LOG_ROOT="${LOG_ROOT}" test install-kernel integration
         env:
           DEVELOPER_DIR: "/Applications/Xcode-latest.app/Contents/Developer"
 
+      - name: Test the container project with coverage
+        if: ${{ inputs.coverage }}
+        run: make APP_ROOT="${APP_ROOT}" LOG_ROOT="${LOG_ROOT}" install-kernel coverage
+        env:
+          DEVELOPER_DIR: "/Applications/Xcode-latest.app/Contents/Developer"
+
+      - name: Extract coverage percentages
+        if: ${{ inputs.coverage }}
+        env:
+          PR_NUMBER: ${{ inputs.pr_number }}
+        run: |
+          mkdir -p pr-coverage
+          jq -r '.data[0].totals.lines.percent | . * 100 | round | . / 100' coverage-reports/unit/coverage-summary.json > pr-coverage/unit-line-coverage.txt
+          jq -r '.data[0].totals.lines.percent | . * 100 | round | . / 100' coverage-reports/integration/coverage-summary.json > pr-coverage/integration-line-coverage.txt
+          jq -r '.data[0].totals.lines.percent | . * 100 | round | . / 100' coverage-reports/combined/coverage-summary.json > pr-coverage/combined-line-coverage.txt
+          echo "${PR_NUMBER}" > pr-coverage/pr-number.txt
+          echo "Coverage data:"
+          cat pr-coverage/*.txt
+
+      - name: Upload coverage data
+        if: ${{ inputs.coverage }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: pr-coverage-${{ inputs.pr_number }}
+          path: pr-coverage/
+          retention-days: 1
+          if-no-files-found: warn
+
+      - name: Upload coverage HTML reports
+        if: ${{ inputs.coverage }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: coverage-html-reports
+          path: |
+            coverage-reports/unit/html/
+            coverage-reports/integration/html/
+            coverage-reports/combined/html/
+          retention-days: 14
+          if-no-files-found: ignore
+
       - name: Archive test logs
         if: always()
         run: |
@@ -103,6 +164,7 @@ jobs:
             rm -rf "${APP_ROOT}"
             echo "Removed data directory ${APP_ROOT}"
           fi
+          rm -rf coverage-reports pr-coverage
 
       - name: Upload logs if present
         if: always()

.github/workflows/pr-build.yml

@@ -44,6 +44,8 @@ jobs:
     uses: ./.github/workflows/common.yml
     with:
       release: false
+      coverage: true
+      pr_number: ${{ github.event.pull_request.number }}
     secrets: inherit
     permissions:
       contents: read

.github/workflows/pr-coverage-comment.yml

@@ -0,0 +1,83 @@
+name: PR Coverage Comment
+
+on:
+  workflow_run:
+    workflows: ["container project - PR build"]
+    types:
+      - completed
+
+permissions:
+  contents: read
+
+jobs:
+  comment:
+    name: Post coverage comment
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    permissions:
+      contents: read
+      actions: read
+      pull-requests: write
+
+    steps:
+      - name: Download coverage artifact
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          run-id: ${{ github.event.workflow_run.id }}
+          pattern: pr-coverage-*
+          merge-multiple: false
+        id: download-artifact
+        continue-on-error: true
+
+      - name: Check artifact exists
+        if: steps.download-artifact.outcome == 'success'
+        id: check-artifact
+        run: |
+          if ls pr-coverage-*/pr-number.txt 1>/dev/null 2>&1; then
+            echo "found=true" >> $GITHUB_OUTPUT
+          else
+            echo "No coverage artifact found — coverage job may have failed."
+            echo "found=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Validate and post comment
+        if: steps.check-artifact.outputs.found == 'true'
+        env:
+          GH_TOKEN: ${{ github.token }}
+          REPO: ${{ github.repository }}
+        run: |
+          PR_NUMBER=$(cat pr-coverage-*/pr-number.txt)
+          grep -qxE '[0-9]+' <<< "$PR_NUMBER" || { echo "Invalid PR number: ${PR_NUMBER}"; exit 1; }
+
+          UNIT=$(cat pr-coverage-*/unit-line-coverage.txt)
+          grep -qxE '[0-9]+(\.[0-9]+)?' <<< "$UNIT" || { echo "Invalid unit coverage: ${UNIT}"; exit 1; }
+
+          INTEGRATION=$(cat pr-coverage-*/integration-line-coverage.txt)
+          grep -qxE '[0-9]+(\.[0-9]+)?' <<< "$INTEGRATION" || { echo "Invalid integration coverage: ${INTEGRATION}"; exit 1; }
+
+          COMBINED=$(cat pr-coverage-*/combined-line-coverage.txt)
+          grep -qxE '[0-9]+(\.[0-9]+)?' <<< "$COMBINED" || { echo "Invalid combined coverage: ${COMBINED}"; exit 1; }
+
+          MARKER="<!-- coverage-bot -->"
+          BODY=$(cat <<EOF
+          ${MARKER}
+          ## Code Coverage
+
+          | Tier | Line Coverage |
+          |------|--------------|
+          | Unit | ${UNIT}% |
+          | Integration | ${INTEGRATION}% |
+          | Combined | ${COMBINED}% |
+          EOF
+          )
+
+          EXISTING_COMMENT_ID=$(gh api "repos/${REPO}/issues/${PR_NUMBER}/comments" --paginate --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | head -1)
+
+          if [ -n "${EXISTING_COMMENT_ID}" ]; then
+            gh api "repos/${REPO}/issues/comments/${EXISTING_COMMENT_ID}" -X PATCH -f body="${BODY}"
+            echo "Updated existing coverage comment ${EXISTING_COMMENT_ID}"
+          else
+            gh pr comment "${PR_NUMBER}" --repo "${REPO}" --body "${BODY}"
+            echo "Created new coverage comment"
+          fi

Makefile

@@ -182,7 +182,7 @@ define GENERATE_COV_REPORTS
 		-output-dir=$(COVERAGE_OUTPUT_DIR)/$(2)/html \
 		$(TEST_BINARY)
 	@echo Extracting $(2) coverage percentages...
-	@jq -r '"line coverage: \(.data[0].totals.lines.percent * 100 | round / 100)%\nfunction coverage: \(.data[0].totals.functions.percent * 100 | round / 100)%"' \
+	@jq -r '"line coverage: \(.data[0].totals.lines.percent | . * 100 | round | . / 100)%\nfunction coverage: \(.data[0].totals.functions.percent | . * 100 | round | . / 100)%"' \
 		$(COVERAGE_OUTPUT_DIR)/$(2)/coverage-summary.json > $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt
 	@cat $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt
 endef