Add gcov code coverage to the test suite

scripts/coverage.py configures an instrumented build tree
(-DAKERR_COVERAGE=ON), runs the CTest suite in it, and reports merged
gcov line/branch/function coverage per library source. Like the mutation
harness it has no third-party dependencies and supports --threshold and
--junit; thresholds gate each file as well as the total so the generated
status-name table cannot mask a regression in src/error.c.

Only the library is instrumented. The public header's macros cannot be
measured this way -- GCC attributes an expanded macro to its call site,
so header logic would report as lines of the test that used it -- which
is what mutation testing against include/akerror.tmpl.h is for.

Coverage flags are applied per target rather than globally, so they do
not leak into the exported/installed target interface.

Current numbers for src/error.c are 94.0% line and 59.5% branch; the CI
gate is set to 90/50 to keep headroom, matching the convention used for
the mutation score threshold.

Tests run: ctest (23/23), cmake --build build --target coverage,
threshold gate verified failing at --threshold 99, cmake --install
checked for flag leakage, out-of-tree --build-dir checked.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-07-30 01:48:07 -04:00
parent 0c0d81249f
commit 9f0034a56e
5 changed files with 517 additions and 8 deletions

View File

@@ -6,8 +6,43 @@ include(CMakePackageConfigHelpers)
include(CTest)
set(AKERR_USE_STDLIB 1 CACHE BOOL "Use the C standard library")
set(AKERR_COVERAGE 0 CACHE BOOL "Instrument the build with gcov coverage counters")
set(akerror_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/akerror")
# Coverage instrumentation. Applied per target (not globally) so it never leaks
# into the exported/installed target interface. Only the library is
# instrumented: the tests are the thing doing the covering, and the public
# header's macros cannot be measured this way at all -- GCC attributes an
# expanded macro to its call site, so header logic shows up as test-file lines.
# Coverage of those macros is what mutation testing (--target
# include/akerror.tmpl.h) is for.
if(AKERR_COVERAGE)
if(NOT CMAKE_C_COMPILER_ID MATCHES "GNU|Clang")
message(FATAL_ERROR
"AKERR_COVERAGE requires GCC or Clang, not ${CMAKE_C_COMPILER_ID}")
endif()
# -O0 keeps line counts attributable; no inlining or code motion.
set(AKERR_COVERAGE_FLAGS --coverage -O0 -g)
if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
# Record absolute source paths so gcov resolves sources built from the
# generated directory (GCC 8+; harmless to check).
include(CheckCCompilerFlag)
check_c_compiler_flag(-fprofile-abs-path AKERR_HAVE_PROFILE_ABS_PATH)
if(AKERR_HAVE_PROFILE_ABS_PATH)
list(APPEND AKERR_COVERAGE_FLAGS -fprofile-abs-path)
endif()
endif()
endif()
# Add coverage compile/link flags to one target, if coverage is enabled.
function(akerr_instrument_for_coverage _target)
if(AKERR_COVERAGE)
target_compile_options(${_target} PRIVATE ${AKERR_COVERAGE_FLAGS})
set_property(TARGET ${_target} APPEND_STRING
PROPERTY LINK_FLAGS " --coverage")
endif()
endfunction()
set(SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generrno.sh)
set(INFILE ${CMAKE_CURRENT_SOURCE_DIR}/include/akerror.tmpl.h)
@@ -44,6 +79,8 @@ target_compile_definitions(akerror
PUBLIC AKERR_USE_STDLIB=${AKERR_USE_STDLIB}
)
akerr_instrument_for_coverage(akerror)
# Each test is one source file in tests/ built into test_<name> and registered
# as CTest <name>. Tests expected to abort (unhandled error / contract
# violation) go in AKERR_WILL_FAIL_TESTS; all others must exit 0.
@@ -95,14 +132,33 @@ set_tests_properties(
PROPERTIES WILL_FAIL TRUE
)
# Mutation testing: break the library in small ways and confirm the test suite
# notices. This is a meta-check on the tests themselves, so it is a manual
# target (it rebuilds and re-runs the whole suite many times), not a CTest test.
# cmake --build build --target mutation
# When embedded in another project, use a namespaced target to avoid collisions
# with mutation targets provided by sibling dependencies.
# Coverage and mutation testing are meta-checks on the test suite itself, and
# both rebuild and re-run the whole suite, so they are manual targets rather
# than CTest tests.
find_package(Python3 COMPONENTS Interpreter)
if(Python3_FOUND)
# Code coverage: which library lines/branches the CTest suite reaches.
# cmake --build build --target coverage
# The script configures and drives its own instrumented build tree (under
# ${CMAKE_BINARY_DIR}/coverage) so this build's binaries and its coverage
# counters can never be stale or half-instrumented. Reports via gcov.
add_custom_target(coverage
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/scripts/coverage.py
--source-root ${CMAKE_CURRENT_SOURCE_DIR}
--build-dir ${CMAKE_CURRENT_BINARY_DIR}/coverage
--cmake ${CMAKE_COMMAND}
--ctest ${CMAKE_CTEST_COMMAND}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
USES_TERMINAL
COMMENT "Running the test suite instrumented for coverage"
)
# Mutation testing: break the library in small ways and confirm the test
# suite notices.
# cmake --build build --target mutation
# When embedded in another project, use a namespaced target to avoid
# collisions with mutation targets provided by sibling dependencies.
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
set(AKERR_MUTATION_TARGET mutation)
else()