diff --git a/.gitignore b/.gitignore index 7f5b4f47..429f000b 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,10 @@ test_results/ *.pkg *.swp +# Coverage artifacts +*.profraw +coverage-reports/ + # API docs for local preview only. _site/ _serve/ diff --git a/Makefile b/Makefile index 96721855..041cf977 100644 --- a/Makefile +++ b/Makefile @@ -24,14 +24,13 @@ SWIFT := "/usr/bin/swift" DEST_DIR ?= /usr/local/ ROOT_DIR := $(shell git rev-parse --show-toplevel) BUILD_BIN_DIR = $(shell $(SWIFT) build -c $(BUILD_CONFIGURATION) --show-bin-path) -COV_DATA_DIR = $(shell $(SWIFT) test --show-coverage-path | xargs dirname) -COV_REPORT_FILE = $(ROOT_DIR)/code-coverage-report STAGING_DIR := bin/$(BUILD_CONFIGURATION)/staging/ PKG_PATH := bin/$(BUILD_CONFIGURATION)/container-installer-unsigned.pkg DSYM_DIR := bin/$(BUILD_CONFIGURATION)/bundle/container-dSYM DSYM_PATH := bin/$(BUILD_CONFIGURATION)/bundle/container-dSYM.zip CODESIGN_OPTS ?= --force --sign - --timestamp=none + # Conditionally use a temporary data directory for integration tests SYSTEM_START_OPTS := ifneq ($(strip $(APP_ROOT)),) @@ -154,29 +153,114 @@ install-kernel: @echo Starting system to install kernel @bin/container --debug system start --timeout 60 --enable-kernel-install $(SYSTEM_START_OPTS) + +# Coverage report generation helpers +# Directory that swift test spits out raw coverage data +COV_DATA_DIR = $(shell $(SWIFT) test --show-coverage-path | xargs dirname) +COV_REPORT_FILE = $(ROOT_DIR)/code-coverage-report +COVERAGE_OUTPUT_DIR := $(ROOT_DIR)/coverage-reports +TEST_BINARY = $(BUILD_BIN_DIR)/containerPackageTests.xctest/Contents/MacOS/containerPackageTests +# Set of files we do not want to get caught in the coverage generation +LLVM_COV_IGNORE := \ + --ignore-filename-regex=".build/" \ + --ignore-filename-regex=".pb.swift" \ + --ignore-filename-regex=".proto" \ + --ignore-filename-regex=".grpc.swift" +# swift test overwrites profraw data on each invocation, so we copy to a safe spot +SAVE_PROFRAW = mkdir -p $(COVERAGE_OUTPUT_DIR)/integration/$(1) && cp $(COV_DATA_DIR)/*.profraw $(COVERAGE_OUTPUT_DIR)/integration/$(1)/ + +# Generate JSON + HTML coverage reports and a coverage-percent.txt from a profdata file. +# $(1) = profdata path, $(2) = tier name (unit/integration/combined) +define GENERATE_COV_REPORTS + @echo Exporting $(2) coverage JSON... + @xcrun llvm-cov export --compilation-dir=`pwd` \ + -instr-profile=$(1) \ + $(LLVM_COV_IGNORE) \ + $(TEST_BINARY) > $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-summary.json + @echo Generating $(2) coverage HTML report... + @xcrun llvm-cov show --compilation-dir=`pwd` --format=html \ + -instr-profile=$(1) \ + $(LLVM_COV_IGNORE) \ + -output-dir=$(COVERAGE_OUTPUT_DIR)/$(2)/html \ + $(TEST_BINARY) + @echo Extracting $(2) coverage percentages... + @jq -r '"line coverage: \(.data[0].totals.lines.percent * 100 | round / 100)%\nfunction coverage: \(.data[0].totals.functions.percent * 100 | round / 100)%"' \ + $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-summary.json > $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt + @cat $(COVERAGE_OUTPUT_DIR)/$(2)/coverage-percent.txt +endef + +INTEGRATION_TEST_SUITES := \ + TestCLIHelp \ + TestCLIStatus \ + TestCLIVersion \ + TestCLINetwork \ + TestCLIRunLifecycle \ + TestCLIRunCapabilities \ + TestCLIExecCommand \ + TestCLICreateCommand \ + TestCLIRunCommand1 \ + TestCLIRunCommand2 \ + TestCLIRunCommand3 \ + TestCLIPruneCommand \ + TestCLIRegistry \ + TestCLIStatsCommand \ + TestCLIImagesCommand \ + TestCLIRunBase \ + TestCLIRunInitImage \ + TestCLIBuildBase \ + TestCLIExportCommand \ + TestCLIVolumes \ + TestCLIKernelSet \ + TestCLIAnonymousVolumes \ + TestCLINotFound \ + TestCLINoParallelCases + .PHONY: coverage -coverage: init-block - @echo Ensuring apiserver stopped before the coverage analysis +# Merge the raw coverage data generated from coverage-unit and coverage-integration into one unified report +coverage: coverage-unit coverage-integration + @echo Merging combined coverage profdata... + @mkdir -p $(COVERAGE_OUTPUT_DIR)/combined + @xcrun llvm-profdata merge -sparse \ + $(COVERAGE_OUTPUT_DIR)/unit/default.profdata \ + $(COVERAGE_OUTPUT_DIR)/integration/default.profdata \ + -o $(COVERAGE_OUTPUT_DIR)/combined/default.profdata + $(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/combined/default.profdata,combined) + +.PHONY: coverage-unit +coverage-unit: + @echo Running unit test coverage... + @rm -f $(COV_DATA_DIR)/*.profraw + @mkdir -p $(COVERAGE_OUTPUT_DIR)/unit +# Run the test suite with profiler (exclude integration tests) + @$(SWIFT) test --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --skip TestCLI +# Move the profiling data to a staging area for later consumption in full coverage aggregation + @echo Merging unit coverage profdata... + @xcrun llvm-profdata merge -sparse $(COV_DATA_DIR)/*.profraw -o $(COVERAGE_OUTPUT_DIR)/unit/default.profdata +# Generate both JSON (for machines) and html (for humans) + $(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/unit/default.profdata,unit) + +.PHONY: coverage-integration +coverage-integration: all + @echo Ensuring apiserver stopped before the coverage integration tests... @bin/container system stop && sleep 3 && scripts/ensure-container-stopped.sh - @bin/container --debug system start $(SYSTEM_START_OPTS) && \ - echo "Starting coverage analysis" && \ + @echo Running integration test coverage... + @rm -f $(COV_DATA_DIR)/*.profraw + @mkdir -p $(COVERAGE_OUTPUT_DIR)/integration + @bin/container --debug system start --timeout 60 $(SYSTEM_START_OPTS) && \ + echo "Starting CLI integration tests with coverage" && \ { \ exit_code=0; \ - $(SWIFT) test --no-parallel --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) || exit_code=1 ; \ - echo Ensuring apiserver stopped after the CLI integration tests ; \ + export CLITEST_LOG_ROOT=$(LOG_ROOT) ; \ + $(foreach suite,$(INTEGRATION_TEST_SUITES), \ + $(SWIFT) test --no-parallel --enable-code-coverage -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter $(suite) || exit_code=1 ; $(call SAVE_PROFRAW,$(suite)) ; \ + ) \ + echo Ensuring apiserver stopped after the coverage integration tests ; \ scripts/ensure-container-stopped.sh ; \ - echo Generating code coverage report... ; \ - xcrun llvm-profdata merge -sparse $(COV_DATA_DIR)/*.profraw -o $(COV_DATA_DIR)/default.profdata ; \ - xcrun llvm-cov show --compilation-dir=`pwd` \ - -instr-profile=$(COV_DATA_DIR)/default.profdata \ - --ignore-filename-regex=".build/" \ - --ignore-filename-regex=".pb.swift" \ - --ignore-filename-regex=".proto" \ - --ignore-filename-regex=".grpc.swift" \ - $(BUILD_BIN_DIR)/containerPackageTests.xctest/Contents/MacOS/containerPackageTests > $(COV_REPORT_FILE) ; \ - echo Code coverage report generated: $(COV_REPORT_FILE) ; \ exit $${exit_code} ; \ } + @echo Merging integration coverage profdata... + @xcrun llvm-profdata merge -sparse $(COVERAGE_OUTPUT_DIR)/integration/*/*.profraw -o $(COVERAGE_OUTPUT_DIR)/integration/default.profdata + $(call GENERATE_COV_REPORTS,$(COVERAGE_OUTPUT_DIR)/integration/default.profdata,integration) .PHONY: integration integration: init-block @@ -193,30 +277,9 @@ integration: init-block { \ exit_code=0; \ CLITEST_LOG_ROOT=$(LOG_ROOT) && export CLITEST_LOG_ROOT ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIHelp || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIStatus || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIVersion || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLINetwork || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunLifecycle || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCapabilities || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIExecCommand || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLICreateCommand || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand1 || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand2 || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand3 || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIPruneCommand || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRegistry || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIStatsCommand || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIImagesCommand || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunBase || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunInitImage || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIBuildBase || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIExportCommand || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIVolumes || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIKernelSet || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIAnonymousVolumes || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLINotFound || exit_code=1 ; \ - $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLINoParallelCases || exit_code=1 ; \ + $(foreach suite,$(INTEGRATION_TEST_SUITES), \ + $(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter $(suite) || exit_code=1 ; \ + ) \ echo Ensuring apiserver stopped after the CLI integration tests ; \ scripts/ensure-container-stopped.sh ; \ exit $${exit_code} ; \ @@ -285,4 +348,5 @@ clean: @rm -rf bin/ libexec/ @rm -rf _site _serve @rm -f $(COV_REPORT_FILE) + @rm -rf $(COVERAGE_OUTPUT_DIR) @$(SWIFT) package clean