Add code coverage to the CTest suite

New AKSL_COVERAGE option instruments the library and its tests with
--coverage -O0 and wires the report into the suite itself, so a plain
ctest --test-dir build-coverage both runs the tests and produces coverage.

Two CTest entries do the work, held in order by a CTest fixture rather
than by declaration order so they also hold under ctest -j:
coverage_reset (FIXTURES_SETUP) clears the .gcda counters before any test,
since gcov counts are cumulative and would otherwise fold in earlier runs;
coverage_report (FIXTURES_CLEANUP) aggregates gcov output afterwards.
AKSL_COVERAGE_THRESHOLD / AKSL_COVERAGE_BRANCH_THRESHOLD gate the report,
the same regression-ratchet idea as the mutation score. The `coverage`
target builds, runs and prints in one step.

scripts/coverage.py parses gcov's JSON output, aggregates line, branch and
function counts across translation units, and lists every uncovered line
and never-called function -- the actionable half, as with surviving
mutants. Python stdlib plus gcc's own gcov only: no lcov, gcovr or
genhtml. It also writes coverage-summary.txt (CTest hides the output of a
passing test) and a Cobertura coverage.xml for CI publishers.

Instrumentation is per target, so deps/libakerror stays out of the report.
The mutation harness now ignores build*/ and gcov artifacts when copying
the tree, so a coverage build does not slow it down.

Baseline on src/stdlib.c: 52.0% of lines, 23.6% of branches, 8 of 21
functions. The uncovered functions are the untested wrappers the mutation
survivors already point at (printf, ato*, stream, realpath, strhash).

Verified:
  cmake -S . -B build-coverage -DAKSL_COVERAGE=ON
  cmake --build build-coverage --target coverage      # 8/8, report printed
  ctest --test-dir build-coverage -j8                 # fixture order holds
  cmake -S . -B build-coverage -DAKSL_COVERAGE=ON -DAKSL_COVERAGE_THRESHOLD=60
  ctest --test-dir build-coverage --output-on-failure  # gate fails as expected
  ctest --test-dir build --output-on-failure           # 6/6, no .gcda emitted
  ctest --test-dir build-asan --output-on-failure      # 6/6
  scripts/mutation_test.py --target src/stdlib.c --list # 173 mutants, unchanged
Totals match gcov itself: 51.98% of 202 lines, 23.60% of 661 branches.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-30 01:48:05 -04:00
parent a87cbfb26d
commit 82c47ed773
5 changed files with 667 additions and 6 deletions

View File

@@ -20,6 +20,50 @@ if(AKSL_SANITIZE)
message(STATUS "AKSL_SANITIZE=ON: building with ASan + UBSan")
endif()
# Coverage build, off by default:
# cmake -S . -B build-coverage -DAKSL_COVERAGE=ON
# cmake --build build-coverage --target coverage
# Instrumentation is applied per target below (the library and the test
# binaries) rather than through CMAKE_C_FLAGS, so deps/libakerror is left
# uninstrumented -- it has its own suite, and its .gcda would only be noise in
# this project's report.
option(AKSL_COVERAGE "Build the library and its tests with gcov instrumentation" OFF)
# Minimum total line / branch coverage for the `coverage_report` CTest entry
# that a -DAKSL_COVERAGE=ON build adds. 0 disables the gate and reports only.
set(AKSL_COVERAGE_THRESHOLD 0 CACHE STRING
"Fail the coverage_report test below this total line coverage percentage")
set(AKSL_COVERAGE_BRANCH_THRESHOLD 0 CACHE STRING
"Fail the coverage_report test below this total branch coverage percentage")
if(AKSL_COVERAGE)
if(NOT CMAKE_C_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$")
message(FATAL_ERROR
"AKSL_COVERAGE=ON needs a gcov-compatible compiler; "
"CMAKE_C_COMPILER_ID is ${CMAKE_C_COMPILER_ID}")
endif()
# -O0 because the optimizer folds and reorders lines until per-line counts
# stop matching the source. -fprofile-abs-path makes gcov record absolute
# source paths, which is what lets the report be read from any directory.
set(AKSL_COVERAGE_COMPILE_FLAGS --coverage -O0 -g)
if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
list(APPEND AKSL_COVERAGE_COMPILE_FLAGS -fprofile-abs-path)
endif()
message(STATUS "AKSL_COVERAGE=ON: instrumenting akstdlib and its tests for gcov")
endif()
# Add gcov instrumentation to one target. No-op unless AKSL_COVERAGE is set.
function(aksl_target_coverage target)
if(NOT AKSL_COVERAGE)
return()
endif()
target_compile_options(${target} PRIVATE ${AKSL_COVERAGE_COMPILE_FLAGS})
if(CMAKE_VERSION VERSION_LESS 3.13)
set_property(TARGET ${target} APPEND_STRING PROPERTY LINK_FLAGS " --coverage")
else()
target_link_options(${target} PRIVATE --coverage)
endif()
endfunction()
if(TARGET akerror::akerror)
message(STATUS "FOUND akerror::akerror")
else()
@@ -83,6 +127,8 @@ target_include_directories(akstdlib PUBLIC
target_link_libraries(akstdlib PUBLIC akerror::akerror)
aksl_target_coverage(akstdlib)
set(main_lib_dest "lib/my_library-${MY_LIBRARY_VERSION}")
install(TARGETS akstdlib
EXPORT akstdlibTargets
@@ -147,7 +193,14 @@ foreach(_test IN LISTS AKSL_TESTS AKSL_WILL_FAIL_TESTS AKSL_KNOWN_FAILING_TESTS)
add_executable(test_${_test} tests/test_${_test}.c)
target_include_directories(test_${_test} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests)
target_link_libraries(test_${_test} PRIVATE akstdlib)
# The test binaries are instrumented too. The default report filters them out
# (only src/ and include/ are shown), but `scripts/coverage.py --include
# tests` then answers a different question: whether every test case and
# helper branch in tests/ actually runs, which is how a test function that was
# written but never wired into AKSL_RUN shows up.
aksl_target_coverage(test_${_test})
add_test(NAME ${_test} COMMAND test_${_test})
list(APPEND AKSL_TEST_TARGETS test_${_test})
endforeach()
if(AKSL_WILL_FAIL_TESTS OR AKSL_KNOWN_FAILING_TESTS)
@@ -168,6 +221,74 @@ set_tests_properties(
PROPERTIES TIMEOUT 30
)
# Both the coverage report and the mutation harness are Python scripts.
find_package(Python3 COMPONENTS Interpreter)
# Code coverage. A -DAKSL_COVERAGE=ON build wires the report into the suite
# itself, so `ctest --test-dir build-coverage` both runs the tests and prints
# what they touched:
#
# coverage_reset deletes the accumulated .gcda counters. gcov counts are
# cumulative, so without this every report would fold in
# earlier runs and overstate coverage. FIXTURES_SETUP makes
# CTest run it before any test that needs the fixture, even
# under `ctest -j`.
# coverage_report aggregates gcov output and prints the summary plus the
# uncovered lines. FIXTURES_CLEANUP makes CTest run it after
# the last test in the fixture, which is exactly when the
# counters are complete.
#
# The report is a plain report unless AKSL_COVERAGE_THRESHOLD is set, in which
# case coverage_report fails below that percentage. Same ratchet idea as the
# mutation threshold: gate on the number you have, raise it as tests land.
#
# CTest hides the output of a passing test, so coverage_report also writes
# <build>/coverage-summary.txt and <build>/coverage.xml (Cobertura). The
# `coverage` target below builds, runs and prints in one step.
if(AKSL_COVERAGE AND Python3_FOUND)
set(AKSL_COVERAGE_ARGS
${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/coverage.py
--build ${CMAKE_CURRENT_BINARY_DIR}
--source-root ${CMAKE_CURRENT_SOURCE_DIR})
add_test(NAME coverage_reset COMMAND ${AKSL_COVERAGE_ARGS} --zero)
add_test(NAME coverage_report COMMAND ${AKSL_COVERAGE_ARGS}
--threshold ${AKSL_COVERAGE_THRESHOLD}
--branch-threshold ${AKSL_COVERAGE_BRANCH_THRESHOLD}
--output ${CMAKE_CURRENT_BINARY_DIR}/coverage-summary.txt
--cobertura ${CMAKE_CURRENT_BINARY_DIR}/coverage.xml)
set_tests_properties(coverage_reset PROPERTIES FIXTURES_SETUP AKSL_GCDA)
set_tests_properties(coverage_report PROPERTIES FIXTURES_CLEANUP AKSL_GCDA)
set_tests_properties(
${AKSL_TESTS} ${AKSL_WILL_FAIL_TESTS} ${AKSL_KNOWN_FAILING_TESTS}
PROPERTIES FIXTURES_REQUIRED AKSL_GCDA
)
# gcov has to be spawned once per .gcda, so give the report more room than the
# 30s the test binaries get.
set_tests_properties(coverage_reset coverage_report PROPERTIES TIMEOUT 300)
# Convenience entry point that also builds the test binaries first. It runs
# the suite rather than the script directly, because the two fixture tests
# above already reset the counters and produce the report -- and CTest pulls a
# required fixture back in even when it is filtered out, so there is no way to
# run the tests without them. The second command re-reads the same counters to
# print the report that CTest suppressed for the passing coverage_report test
# (it only spawns gcov again, it does not re-run anything).
add_custom_target(coverage
COMMAND ctest --test-dir ${CMAKE_CURRENT_BINARY_DIR} --output-on-failure
COMMAND ${AKSL_COVERAGE_ARGS}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
USES_TERMINAL
COMMENT "Running the test suite under gcov and reporting coverage"
)
add_dependencies(coverage ${AKSL_TEST_TARGETS})
elseif(AKSL_COVERAGE)
message(WARNING "AKSL_COVERAGE=ON but Python3 was not found: "
"instrumenting the build, but the coverage report "
"(scripts/coverage.py) will not be wired into CTest")
endif()
# Mutation testing: break the library in small ways and confirm the test suite
# notices. This is a meta-check on the tests themselves, and it rebuilds and
# re-runs the whole suite once per mutant, so it is a manual target rather than
@@ -178,7 +299,6 @@ set_tests_properties(
# This target covers both src/stdlib.c and include/akstdlib.h. CI runs the
# narrower, faster src/stdlib.c set with a --threshold gate; see
# .gitea/workflows/ci.yaml.
find_package(Python3 COMPONENTS Interpreter)
if(Python3_FOUND)
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
set(AKSL_MUTATION_TARGET mutation)