Add code coverage to the CTest suite
New AKSL_COVERAGE option instruments the library and its tests with --coverage -O0 and wires the report into the suite itself, so a plain ctest --test-dir build-coverage both runs the tests and produces coverage. Two CTest entries do the work, held in order by a CTest fixture rather than by declaration order so they also hold under ctest -j: coverage_reset (FIXTURES_SETUP) clears the .gcda counters before any test, since gcov counts are cumulative and would otherwise fold in earlier runs; coverage_report (FIXTURES_CLEANUP) aggregates gcov output afterwards. AKSL_COVERAGE_THRESHOLD / AKSL_COVERAGE_BRANCH_THRESHOLD gate the report, the same regression-ratchet idea as the mutation score. The `coverage` target builds, runs and prints in one step. scripts/coverage.py parses gcov's JSON output, aggregates line, branch and function counts across translation units, and lists every uncovered line and never-called function -- the actionable half, as with surviving mutants. Python stdlib plus gcc's own gcov only: no lcov, gcovr or genhtml. It also writes coverage-summary.txt (CTest hides the output of a passing test) and a Cobertura coverage.xml for CI publishers. Instrumentation is per target, so deps/libakerror stays out of the report. The mutation harness now ignores build*/ and gcov artifacts when copying the tree, so a coverage build does not slow it down. Baseline on src/stdlib.c: 52.0% of lines, 23.6% of branches, 8 of 21 functions. The uncovered functions are the untested wrappers the mutation survivors already point at (printf, ato*, stream, realpath, strhash). Verified: cmake -S . -B build-coverage -DAKSL_COVERAGE=ON cmake --build build-coverage --target coverage # 8/8, report printed ctest --test-dir build-coverage -j8 # fixture order holds cmake -S . -B build-coverage -DAKSL_COVERAGE=ON -DAKSL_COVERAGE_THRESHOLD=60 ctest --test-dir build-coverage --output-on-failure # gate fails as expected ctest --test-dir build --output-on-failure # 6/6, no .gcda emitted ctest --test-dir build-asan --output-on-failure # 6/6 scripts/mutation_test.py --target src/stdlib.c --list # 173 mutants, unchanged Totals match gcov itself: 51.98% of 202 lines, 23.60% of 661 branches. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
122
CMakeLists.txt
122
CMakeLists.txt
@@ -20,6 +20,50 @@ if(AKSL_SANITIZE)
|
||||
message(STATUS "AKSL_SANITIZE=ON: building with ASan + UBSan")
|
||||
endif()
|
||||
|
||||
# Coverage build, off by default:
|
||||
# cmake -S . -B build-coverage -DAKSL_COVERAGE=ON
|
||||
# cmake --build build-coverage --target coverage
|
||||
# Instrumentation is applied per target below (the library and the test
|
||||
# binaries) rather than through CMAKE_C_FLAGS, so deps/libakerror is left
|
||||
# uninstrumented -- it has its own suite, and its .gcda would only be noise in
|
||||
# this project's report.
|
||||
option(AKSL_COVERAGE "Build the library and its tests with gcov instrumentation" OFF)
|
||||
# Minimum total line / branch coverage for the `coverage_report` CTest entry
|
||||
# that a -DAKSL_COVERAGE=ON build adds. 0 disables the gate and reports only.
|
||||
set(AKSL_COVERAGE_THRESHOLD 0 CACHE STRING
|
||||
"Fail the coverage_report test below this total line coverage percentage")
|
||||
set(AKSL_COVERAGE_BRANCH_THRESHOLD 0 CACHE STRING
|
||||
"Fail the coverage_report test below this total branch coverage percentage")
|
||||
|
||||
if(AKSL_COVERAGE)
|
||||
if(NOT CMAKE_C_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$")
|
||||
message(FATAL_ERROR
|
||||
"AKSL_COVERAGE=ON needs a gcov-compatible compiler; "
|
||||
"CMAKE_C_COMPILER_ID is ${CMAKE_C_COMPILER_ID}")
|
||||
endif()
|
||||
# -O0 because the optimizer folds and reorders lines until per-line counts
|
||||
# stop matching the source. -fprofile-abs-path makes gcov record absolute
|
||||
# source paths, which is what lets the report be read from any directory.
|
||||
set(AKSL_COVERAGE_COMPILE_FLAGS --coverage -O0 -g)
|
||||
if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
|
||||
list(APPEND AKSL_COVERAGE_COMPILE_FLAGS -fprofile-abs-path)
|
||||
endif()
|
||||
message(STATUS "AKSL_COVERAGE=ON: instrumenting akstdlib and its tests for gcov")
|
||||
endif()
|
||||
|
||||
# Add gcov instrumentation to one target. No-op unless AKSL_COVERAGE is set.
|
||||
function(aksl_target_coverage target)
|
||||
if(NOT AKSL_COVERAGE)
|
||||
return()
|
||||
endif()
|
||||
target_compile_options(${target} PRIVATE ${AKSL_COVERAGE_COMPILE_FLAGS})
|
||||
if(CMAKE_VERSION VERSION_LESS 3.13)
|
||||
set_property(TARGET ${target} APPEND_STRING PROPERTY LINK_FLAGS " --coverage")
|
||||
else()
|
||||
target_link_options(${target} PRIVATE --coverage)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
if(TARGET akerror::akerror)
|
||||
message(STATUS "FOUND akerror::akerror")
|
||||
else()
|
||||
@@ -83,6 +127,8 @@ target_include_directories(akstdlib PUBLIC
|
||||
|
||||
target_link_libraries(akstdlib PUBLIC akerror::akerror)
|
||||
|
||||
aksl_target_coverage(akstdlib)
|
||||
|
||||
set(main_lib_dest "lib/my_library-${MY_LIBRARY_VERSION}")
|
||||
install(TARGETS akstdlib
|
||||
EXPORT akstdlibTargets
|
||||
@@ -147,7 +193,14 @@ foreach(_test IN LISTS AKSL_TESTS AKSL_WILL_FAIL_TESTS AKSL_KNOWN_FAILING_TESTS)
|
||||
add_executable(test_${_test} tests/test_${_test}.c)
|
||||
target_include_directories(test_${_test} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests)
|
||||
target_link_libraries(test_${_test} PRIVATE akstdlib)
|
||||
# The test binaries are instrumented too. The default report filters them out
|
||||
# (only src/ and include/ are shown), but `scripts/coverage.py --include
|
||||
# tests` then answers a different question: whether every test case and
|
||||
# helper branch in tests/ actually runs, which is how a test function that was
|
||||
# written but never wired into AKSL_RUN shows up.
|
||||
aksl_target_coverage(test_${_test})
|
||||
add_test(NAME ${_test} COMMAND test_${_test})
|
||||
list(APPEND AKSL_TEST_TARGETS test_${_test})
|
||||
endforeach()
|
||||
|
||||
if(AKSL_WILL_FAIL_TESTS OR AKSL_KNOWN_FAILING_TESTS)
|
||||
@@ -168,6 +221,74 @@ set_tests_properties(
|
||||
PROPERTIES TIMEOUT 30
|
||||
)
|
||||
|
||||
# Both the coverage report and the mutation harness are Python scripts.
|
||||
find_package(Python3 COMPONENTS Interpreter)
|
||||
|
||||
# Code coverage. A -DAKSL_COVERAGE=ON build wires the report into the suite
|
||||
# itself, so `ctest --test-dir build-coverage` both runs the tests and prints
|
||||
# what they touched:
|
||||
#
|
||||
# coverage_reset deletes the accumulated .gcda counters. gcov counts are
|
||||
# cumulative, so without this every report would fold in
|
||||
# earlier runs and overstate coverage. FIXTURES_SETUP makes
|
||||
# CTest run it before any test that needs the fixture, even
|
||||
# under `ctest -j`.
|
||||
# coverage_report aggregates gcov output and prints the summary plus the
|
||||
# uncovered lines. FIXTURES_CLEANUP makes CTest run it after
|
||||
# the last test in the fixture, which is exactly when the
|
||||
# counters are complete.
|
||||
#
|
||||
# The report is a plain report unless AKSL_COVERAGE_THRESHOLD is set, in which
|
||||
# case coverage_report fails below that percentage. Same ratchet idea as the
|
||||
# mutation threshold: gate on the number you have, raise it as tests land.
|
||||
#
|
||||
# CTest hides the output of a passing test, so coverage_report also writes
|
||||
# <build>/coverage-summary.txt and <build>/coverage.xml (Cobertura). The
|
||||
# `coverage` target below builds, runs and prints in one step.
|
||||
if(AKSL_COVERAGE AND Python3_FOUND)
|
||||
set(AKSL_COVERAGE_ARGS
|
||||
${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/coverage.py
|
||||
--build ${CMAKE_CURRENT_BINARY_DIR}
|
||||
--source-root ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
add_test(NAME coverage_reset COMMAND ${AKSL_COVERAGE_ARGS} --zero)
|
||||
add_test(NAME coverage_report COMMAND ${AKSL_COVERAGE_ARGS}
|
||||
--threshold ${AKSL_COVERAGE_THRESHOLD}
|
||||
--branch-threshold ${AKSL_COVERAGE_BRANCH_THRESHOLD}
|
||||
--output ${CMAKE_CURRENT_BINARY_DIR}/coverage-summary.txt
|
||||
--cobertura ${CMAKE_CURRENT_BINARY_DIR}/coverage.xml)
|
||||
|
||||
set_tests_properties(coverage_reset PROPERTIES FIXTURES_SETUP AKSL_GCDA)
|
||||
set_tests_properties(coverage_report PROPERTIES FIXTURES_CLEANUP AKSL_GCDA)
|
||||
set_tests_properties(
|
||||
${AKSL_TESTS} ${AKSL_WILL_FAIL_TESTS} ${AKSL_KNOWN_FAILING_TESTS}
|
||||
PROPERTIES FIXTURES_REQUIRED AKSL_GCDA
|
||||
)
|
||||
# gcov has to be spawned once per .gcda, so give the report more room than the
|
||||
# 30s the test binaries get.
|
||||
set_tests_properties(coverage_reset coverage_report PROPERTIES TIMEOUT 300)
|
||||
|
||||
# Convenience entry point that also builds the test binaries first. It runs
|
||||
# the suite rather than the script directly, because the two fixture tests
|
||||
# above already reset the counters and produce the report -- and CTest pulls a
|
||||
# required fixture back in even when it is filtered out, so there is no way to
|
||||
# run the tests without them. The second command re-reads the same counters to
|
||||
# print the report that CTest suppressed for the passing coverage_report test
|
||||
# (it only spawns gcov again, it does not re-run anything).
|
||||
add_custom_target(coverage
|
||||
COMMAND ctest --test-dir ${CMAKE_CURRENT_BINARY_DIR} --output-on-failure
|
||||
COMMAND ${AKSL_COVERAGE_ARGS}
|
||||
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||
USES_TERMINAL
|
||||
COMMENT "Running the test suite under gcov and reporting coverage"
|
||||
)
|
||||
add_dependencies(coverage ${AKSL_TEST_TARGETS})
|
||||
elseif(AKSL_COVERAGE)
|
||||
message(WARNING "AKSL_COVERAGE=ON but Python3 was not found: "
|
||||
"instrumenting the build, but the coverage report "
|
||||
"(scripts/coverage.py) will not be wired into CTest")
|
||||
endif()
|
||||
|
||||
# Mutation testing: break the library in small ways and confirm the test suite
|
||||
# notices. This is a meta-check on the tests themselves, and it rebuilds and
|
||||
# re-runs the whole suite once per mutant, so it is a manual target rather than
|
||||
@@ -178,7 +299,6 @@ set_tests_properties(
|
||||
# This target covers both src/stdlib.c and include/akstdlib.h. CI runs the
|
||||
# narrower, faster src/stdlib.c set with a --threshold gate; see
|
||||
# .gitea/workflows/ci.yaml.
|
||||
find_package(Python3 COMPONENTS Interpreter)
|
||||
if(Python3_FOUND)
|
||||
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
|
||||
set(AKSL_MUTATION_TARGET mutation)
|
||||
|
||||
Reference in New Issue
Block a user