andrew/libakerror
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add gcov code coverage to the test suite

Browse Source 
scripts/coverage.py configures an instrumented build tree
(-DAKERR_COVERAGE=ON), runs the CTest suite in it, and reports merged
gcov line/branch/function coverage per library source. Like the mutation
harness it has no third-party dependencies and supports --threshold and
--junit; thresholds gate each file as well as the total so the generated
status-name table cannot mask a regression in src/error.c.

Only the library is instrumented. The public header's macros cannot be
measured this way -- GCC attributes an expanded macro to its call site,
so header logic would report as lines of the test that used it -- which
is what mutation testing against include/akerror.tmpl.h is for.

Coverage flags are applied per target rather than globally, so they do
not leak into the exported/installed target interface.

Current numbers for src/error.c are 94.0% line and 59.5% branch; the CI
gate is set to 90/50 to keep headroom, matching the convention used for
the mutation score threshold.

Tests run: ctest (23/23), cmake --build build --target coverage,
threshold gate verified failing at --threshold 99, cmake --install
checked for flag leakage, out-of-tree --build-dir checked.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Andrew Kesterson
2026-07-30 01:48:07 -04:00
parent 0c0d81249f
commit 9f0034a56e
5 changed files with 517 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
.gitea/workflows/ci.yaml
View File

@@ -37,6 +37,36 @@ jobs:
fail_on_failure: 'true'
- run: echo "🍏 This job's status is ${{ job.status }}."
coverage:
runs-on: ubuntu-latest
steps:
- name: Check out repository code
uses: actions/checkout@v4
- name: dependencies
run: |
sudo apt-get update -y
sudo apt-get install -y cmake gcc moreutils python3
# Run the suite against a gcov-instrumented build and gate on coverage of
# the library sources. Thresholds keep headroom below the current numbers
# (src/error.c: ~94% line, ~60% branch) and apply per file as well as to
# the total, so the generated status-name table cannot mask a regression.
- name: coverage
run: |
python3 scripts/coverage.py --junit coverage-junit.xml --threshold 90 --branch-threshold 50
# Publish even when the threshold gate fails, so gaps are visible.
# Display-only (fail_on_failure: false); the --threshold above is the gate.
# annotate_only avoids the Checks API 404 on Gitea (see note above).
- name: publish coverage results
if: always()
uses: mikepenz/action-junit-report@v4
with:
report_paths: 'coverage-junit.xml'
annotate_only: true
detailed_summary: true
include_passed: true
fail_on_failure: 'false'
- run: echo "🍏 This job's status is ${{ job.status }}."
mutation_test:
runs-on: ubuntu-latest
steps:

19
AGENTS.md
View File

@@ -34,6 +34,20 @@ cmake --build build --target mutation
scripts/mutation_test.py --target src/error.c --threshold 65
```
Code coverage is available through:
```sh
cmake --build build --target coverage
scripts/coverage.py --threshold 90 --branch-threshold 50
```
`scripts/coverage.py` configures its own instrumented build tree (default
`build/coverage`, `-DAKERR_COVERAGE=ON`), runs the CTest suite there, and
reports gcov line/branch coverage per library source. Thresholds gate each
file as well as the total. Coverage measures `src/error.c` and the generated
`src/errno.c` only; the public header's macros expand at their call sites, so
mutation testing is what checks those.
## Coding Style & Naming Conventions
Use C99-compatible C and follow the surrounding style. Functions and types use
@@ -48,8 +62,9 @@ Add tests as `tests/err_<behavior>.c`. Register each new test in the
`AKERR_TESTS` list in `CMakeLists.txt`; tests that intentionally abort must
also be listed in `AKERR_WILL_FAIL_TESTS`. Prefer focused executable tests that
return zero on success and use existing helpers such as `AKERR_CHECK`. Run the
CTest suite before submitting changes, and run mutation testing when changing
core control-flow, reference counting, stack-trace, or handler behavior.
CTest suite before submitting changes, and run mutation testing and coverage
when changing core control-flow, reference counting, stack-trace, or handler
behavior.
## Commit & Pull Request Guidelines

68
CMakeLists.txt
View File

@@ -6,8 +6,43 @@ include(CMakePackageConfigHelpers)
include(CTest)
set(AKERR_USE_STDLIB 1 CACHE BOOL "Use the C standard library")
set(AKERR_COVERAGE 0 CACHE BOOL "Instrument the build with gcov coverage counters")
set(akerror_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/akerror")
# Coverage instrumentation. Applied per target (not globally) so it never leaks
# into the exported/installed target interface. Only the library is
# instrumented: the tests are the thing doing the covering, and the public
# header's macros cannot be measured this way at all -- GCC attributes an
# expanded macro to its call site, so header logic shows up as test-file lines.
# Coverage of those macros is what mutation testing (--target
# include/akerror.tmpl.h) is for.
if(AKERR_COVERAGE)
if(NOT CMAKE_C_COMPILER_ID MATCHES "GNU|Clang")
message(FATAL_ERROR
"AKERR_COVERAGE requires GCC or Clang, not ${CMAKE_C_COMPILER_ID}")
endif()
# -O0 keeps line counts attributable; no inlining or code motion.
set(AKERR_COVERAGE_FLAGS --coverage -O0 -g)
if(CMAKE_C_COMPILER_ID STREQUAL "GNU")
# Record absolute source paths so gcov resolves sources built from the
# generated directory (GCC 8+; harmless to check).
include(CheckCCompilerFlag)
check_c_compiler_flag(-fprofile-abs-path AKERR_HAVE_PROFILE_ABS_PATH)
if(AKERR_HAVE_PROFILE_ABS_PATH)
list(APPEND AKERR_COVERAGE_FLAGS -fprofile-abs-path)
endif()
endif()
endif()
# Add coverage compile/link flags to one target, if coverage is enabled.
function(akerr_instrument_for_coverage _target)
if(AKERR_COVERAGE)
target_compile_options(${_target} PRIVATE ${AKERR_COVERAGE_FLAGS})
set_property(TARGET ${_target} APPEND_STRING
PROPERTY LINK_FLAGS " --coverage")
endif()
endfunction()
set(SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generrno.sh)
set(INFILE ${CMAKE_CURRENT_SOURCE_DIR}/include/akerror.tmpl.h)
@@ -44,6 +79,8 @@ target_compile_definitions(akerror
PUBLIC AKERR_USE_STDLIB=${AKERR_USE_STDLIB}
)
akerr_instrument_for_coverage(akerror)
# Each test is one source file in tests/ built into test_<name> and registered
# as CTest <name>. Tests expected to abort (unhandled error / contract
# violation) go in AKERR_WILL_FAIL_TESTS; all others must exit 0.
@@ -95,14 +132,33 @@ set_tests_properties(
PROPERTIES WILL_FAIL TRUE
)
# Mutation testing: break the library in small ways and confirm the test suite
# notices. This is a meta-check on the tests themselves, so it is a manual
# target (it rebuilds and re-runs the whole suite many times), not a CTest test.
# cmake --build build --target mutation
# When embedded in another project, use a namespaced target to avoid collisions
# with mutation targets provided by sibling dependencies.
# Coverage and mutation testing are meta-checks on the test suite itself, and
# both rebuild and re-run the whole suite, so they are manual targets rather
# than CTest tests.
find_package(Python3 COMPONENTS Interpreter)
if(Python3_FOUND)
# Code coverage: which library lines/branches the CTest suite reaches.
# cmake --build build --target coverage
# The script configures and drives its own instrumented build tree (under
# ${CMAKE_BINARY_DIR}/coverage) so this build's binaries and its coverage
# counters can never be stale or half-instrumented. Reports via gcov.
add_custom_target(coverage
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/scripts/coverage.py
--source-root ${CMAKE_CURRENT_SOURCE_DIR}
--build-dir ${CMAKE_CURRENT_BINARY_DIR}/coverage
--cmake ${CMAKE_COMMAND}
--ctest ${CMAKE_CTEST_COMMAND}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
USES_TERMINAL
COMMENT "Running the test suite instrumented for coverage"
)
# Mutation testing: break the library in small ways and confirm the test
# suite notices.
# cmake --build build --target mutation
# When embedded in another project, use a namespaced target to avoid
# collisions with mutation targets provided by sibling dependencies.
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
set(AKERR_MUTATION_TARGET mutation)
else()

407
scripts/coverage.py Executable file
View File

@@ -0,0 +1,407 @@
#!/usr/bin/env python3
"""
Code coverage harness for libakerror.
Coverage measures which parts of the library the CTest suite actually executes.
It is the complement to mutation testing (scripts/mutation_test.py): coverage
finds code the tests never reach, mutation testing finds code the tests reach
but do not really check.
What is measured is the library's own translation units: src/error.c and the
generated src/errno.c. The macros in the public header are deliberately not
measured -- GCC attributes an expanded macro to its call site, so header logic
would be reported as lines of the test that used it. Use mutation testing
(--target include/akerror.tmpl.h) to check how well those macros are tested.
This harness has no third-party dependencies -- just gcov, which ships with the
compiler, plus the project's normal cmake/ctest toolchain. By default it
configures its own instrumented build directory so the ordinary build tree is
left alone and coverage counters can never be stale.
Usage:
scripts/coverage.py [options]
--source-root DIR repo root (default: parent of this script's dir)
--build-dir DIR instrumented build dir (default: <root>/build/coverage)
--no-configure reuse the build dir as-is (do not cmake/build)
--no-run report existing counters (do not reset and run ctest)
--threshold PCT exit non-zero if line coverage < PCT (default: 0 = off)
--branch-threshold P exit non-zero if branch coverage < P (default: 0 = off)
--junit PATH write a JUnit XML report to this path
--max-uncovered N uncovered lines to list per file (default: 40, 0 = all)
--exclude SUBSTR skip reported paths containing SUBSTR; repeatable
--gcov PROG gcov program (default: $GCOV or "gcov")
"""
import argparse
import collections
import json
import os
import subprocess
import sys
# --------------------------------------------------------------------------- #
# Running the instrumented suite
# --------------------------------------------------------------------------- #
def configure_and_build(cmake, root, build, jobs):
"""Configure an instrumented build dir and build everything in it."""
cmds = [
[cmake, "-S", root, "-B", build, "-DAKERR_COVERAGE=ON"],
[cmake, "--build", build] + (["--parallel", str(jobs)] if jobs else []),
]
for cmd in cmds:
print("+ " + " ".join(cmd))
if subprocess.call(cmd) != 0:
return False
return True
def reset_counters(build):
"""Delete accumulated .gcda files so each run reports one suite run."""
removed = 0
for dirpath, _dirs, files in os.walk(build):
for name in files:
if name.endswith(".gcda"):
os.unlink(os.path.join(dirpath, name))
removed += 1
if removed:
print(f"Reset {removed} coverage data file(s).")
def run_ctest(build, ctest):
"""Run the suite. Returns True if every test passed.
Coverage counters are flushed at exit(), and the tests that are expected to
fail exit via the library's unhandled-error handler (exit(), not abort()),
so WILL_FAIL tests still contribute their counters.
"""
cmd = [ctest, "--output-on-failure"]
print("+ " + " ".join(cmd) + f" (in {build})")
return subprocess.call(cmd, cwd=build) == 0
# --------------------------------------------------------------------------- #
# Collecting gcov data
# --------------------------------------------------------------------------- #
class FileCov:
"""Merged coverage for one source file, across every object that built it.
A file compiled into more than one object -- or a header included by several
translation units -- is reported once, with counts summed. Branches are
merged by (line, index within line), so differing expansions of the same
line contribute the union of their branches.
"""
def __init__(self):
self.lines = collections.Counter() # lineno -> execution count
self.branches = collections.Counter() # (lineno, idx) -> taken count
self.funcs = collections.Counter() # (name, start_line) -> count
def merge(self, entry):
for ln in entry.get("lines", []):
no = ln["line_number"]
self.lines[no] += ln.get("count", 0)
for idx, br in enumerate(ln.get("branches", [])):
if br.get("throw"):
continue
self.branches[(no, idx)] += br.get("count", 0)
for fn in entry.get("functions", []):
key = (fn.get("name", "?"), fn.get("start_line", 0))
self.funcs[key] += fn.get("execution_count", 0)
@staticmethod
def _ratio(counter):
total = len(counter)
hit = sum(1 for v in counter.values() if v > 0)
return hit, total
def line_stats(self):
return self._ratio(self.lines)
def branch_stats(self):
return self._ratio(self.branches)
def func_stats(self):
return self._ratio(self.funcs)
def uncovered_lines(self):
return sorted(no for no, count in self.lines.items() if count == 0)
def find_notes(build):
"""Every .gcno in the build tree: one per instrumented translation unit."""
notes = []
for dirpath, _dirs, files in os.walk(build):
for name in files:
if name.endswith(".gcno"):
notes.append(os.path.join(dirpath, name))
return sorted(notes)
def gcov_json(gcov, note, cwd):
"""Run gcov on one .gcno and return its parsed JSON, or None on failure.
--stdout keeps gcov from littering .gcov files in the build tree. A .gcno
with no matching .gcda still reports, with all counts zero, which is the
correct answer for a translation unit no test executed.
"""
cmd = [gcov, "--branch-probabilities", "--json-format", "--stdout", note]
proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
out, err = proc.communicate()
if proc.returncode != 0:
sys.stderr.write(f"warning: {' '.join(cmd)} failed:\n"
f"{err.decode(errors='replace')}")
return None
try:
return json.loads(out.decode(errors="replace"))
except ValueError as exc:
sys.stderr.write(f"warning: unparseable gcov output for {note}: {exc}\n")
return None
def display_path(path, build, root):
"""Label a source: build-relative for generated files, else repo-relative.
Returns None for anything outside both trees (system headers, toolchain
internals). Generated sources are labelled relative to the build dir so the
report and its thresholds do not change with the build dir's location.
"""
for base in (build, root):
if path.startswith(base + os.sep):
return os.path.relpath(path, base)
return None
def collect(gcov, build, root, excludes):
"""Merge gcov data for every instrumented unit into {display path: FileCov}."""
covs = {}
for note in find_notes(build):
data = gcov_json(gcov, note, build)
if data is None:
continue
# Paths in the report are relative to the directory the unit was
# compiled in, which gcov records in the notes file.
compile_dir = data.get("current_working_directory") or build
for entry in data.get("files", []):
path = entry.get("file", "")
if not os.path.isabs(path):
path = os.path.join(compile_dir, path)
display = display_path(os.path.realpath(path), build, root)
if display is None:
continue # system headers, toolchain internals
if any(x in display for x in excludes):
continue
covs.setdefault(display, FileCov()).merge(entry)
return covs
# --------------------------------------------------------------------------- #
# Reporting
# --------------------------------------------------------------------------- #
def pct(hit, total):
return 100.0 * hit / total if total else 100.0
def fmt_ratio(hit, total):
if not total:
return f"{'-':>9} -"
return f"{hit:>4}/{total:<4} {pct(hit, total):5.1f}%"
def compress(numbers):
"""[1,2,3,7,9,10] -> '1-3, 7, 9-10' for readable uncovered-line lists."""
out, start, prev = [], None, None
for n in numbers:
if start is None:
start = prev = n
elif n == prev + 1:
prev = n
else:
out.append(f"{start}" if start == prev else f"{start}-{prev}")
start = prev = n
if start is not None:
out.append(f"{start}" if start == prev else f"{start}-{prev}")
return ", ".join(out)
def report(covs, max_uncovered):
"""Print the per-file table and uncovered detail; return overall percentages."""
width = max([len(p) for p in covs] + [len("TOTAL")])
print("\n" + "=" * 72)
print("CODE COVERAGE SUMMARY")
print("=" * 72)
print(f" {'FILE':<{width}} {'LINES':^15} {'BRANCHES':^15} FUNCS")
totals = [0, 0, 0, 0, 0, 0] # lines hit/total, branches hit/total, funcs
for path in sorted(covs):
cov = covs[path]
lh, lt = cov.line_stats()
bh, bt = cov.branch_stats()
fh, ft = cov.func_stats()
for i, v in enumerate((lh, lt, bh, bt, fh, ft)):
totals[i] += v
print(f" {path:<{width}} {fmt_ratio(lh, lt)} {fmt_ratio(bh, bt)} "
f"{fh}/{ft}")
lh, lt, bh, bt, fh, ft = totals
print(f" {'-' * width} {'-' * 15} {'-' * 15} -----")
print(f" {'TOTAL':<{width}} {fmt_ratio(lh, lt)} {fmt_ratio(bh, bt)} "
f"{fh}/{ft}")
for path in sorted(covs):
missing = covs[path].uncovered_lines()
if not missing:
continue
shown = missing if not max_uncovered else missing[:max_uncovered]
more = "" if len(shown) == len(missing) else \
f" ... (+{len(missing) - len(shown)} more)"
print(f"\n uncovered in {path} ({len(missing)} line(s)):")
print(f" {compress(shown)}{more}")
return pct(lh, lt), pct(bh, bt)
def _xml_escape(text):
return (str(text).replace("&", "&amp;").replace("<", "&lt;")
.replace(">", "&gt;").replace('"', "&quot;"))
def write_junit(path, covs, line_threshold, branch_threshold):
"""One <testcase> per file per metric; below-threshold is a <failure>."""
cases = []
for f in sorted(covs):
cov = covs[f]
cases.append((f, "lines", cov.line_stats(), line_threshold))
cases.append((f, "branches", cov.branch_stats(), branch_threshold))
for metric, thr, stats in (("lines", line_threshold,
[c.line_stats() for c in covs.values()]),
("branches", branch_threshold,
[c.branch_stats() for c in covs.values()])):
cases.append(("TOTAL", metric,
(sum(h for h, _t in stats), sum(t for _h, t in stats)),
thr))
fails = sum(1 for _f, _m, (h, t), thr in cases
if t and thr > 0 and pct(h, t) < thr)
out = ['<?xml version="1.0" encoding="UTF-8"?>',
f'<testsuites name="coverage" tests="{len(cases)}" '
f'failures="{fails}">',
f' <testsuite name="coverage" tests="{len(cases)}" '
f'failures="{fails}">']
for f, metric, (hit, total), thr in cases:
name = _xml_escape(f"{f} {metric}")
detail = _xml_escape(f"{hit}/{total} ({pct(hit, total):.1f}%)"
if total else "no data")
out.append(f' <testcase name="{name}" '
f'classname="coverage.{_xml_escape(metric)}" time="0">')
if total and thr > 0 and pct(hit, total) < thr:
out.append(f' <failure message="{metric} coverage '
f'{pct(hit, total):.1f}% &lt; threshold {thr:.1f}%">'
f'{detail}</failure>')
else:
out.append(f' <system-out>{detail}</system-out>')
out.append(' </testcase>')
out.append(' </testsuite>')
out.append('</testsuites>')
with open(path, "w") as fh:
fh.write("\n".join(out) + "\n")
def main():
# Line-buffer stdout so progress is visible live under CI / the cmake target.
try:
sys.stdout.reconfigure(line_buffering=True)
except (AttributeError, ValueError):
pass
here = os.path.dirname(os.path.abspath(__file__))
default_root = os.path.dirname(here)
ap = argparse.ArgumentParser(description="Code coverage for libakerror")
ap.add_argument("--source-root", default=default_root)
ap.add_argument("--build-dir", default=None)
ap.add_argument("--configure", dest="configure", action="store_true",
default=True)
ap.add_argument("--no-configure", dest="configure", action="store_false")
ap.add_argument("--run", dest="run", action="store_true", default=True)
ap.add_argument("--no-run", dest="run", action="store_false")
ap.add_argument("--threshold", type=float, default=0.0,
help="fail if line coverage is below this percentage")
ap.add_argument("--branch-threshold", type=float, default=0.0,
help="fail if branch coverage is below this percentage")
ap.add_argument("--junit", default=None,
help="write a JUnit XML report to this path")
ap.add_argument("--max-uncovered", type=int, default=40)
ap.add_argument("--exclude", action="append", default=None,
help="skip reported paths containing this substring")
ap.add_argument("--gcov", default=os.environ.get("GCOV", "gcov"))
ap.add_argument("--cmake", default=os.environ.get("CMAKE", "cmake"))
ap.add_argument("--ctest", default=os.environ.get("CTEST", "ctest"))
ap.add_argument("-j", "--jobs", type=int, default=0)
args = ap.parse_args()
root = os.path.realpath(args.source_root)
build = os.path.realpath(args.build_dir or os.path.join(root, "build",
"coverage"))
# The tests exercise the library; their own source is not what we measure.
excludes = args.exclude if args.exclude is not None else ["tests/"]
if args.configure:
if not configure_and_build(args.cmake, root, build, args.jobs):
sys.stderr.write("\nInstrumented build FAILED; aborting.\n")
return 2
elif not os.path.isdir(build):
sys.stderr.write(f"No build dir at {build} (drop --no-configure).\n")
return 2
tests_ok = True
if args.run:
reset_counters(build)
tests_ok = run_ctest(build, args.ctest)
if not tests_ok:
sys.stderr.write("\nwarning: some tests FAILED; coverage below is "
"still reported, but the run is not green.\n")
covs = collect(args.gcov, build, root, excludes)
if not covs:
sys.stderr.write("No coverage data found. Was the build instrumented "
"(-DAKERR_COVERAGE=ON) and the suite run?\n")
return 2
line_pct, branch_pct = report(covs, args.max_uncovered)
if args.junit:
junit_path = os.path.abspath(args.junit)
write_junit(junit_path, covs, args.threshold, args.branch_threshold)
print(f"\nJUnit report written to: {junit_path}")
rc = 0
if not tests_ok:
print("\nFAIL: the CTest suite did not pass.")
rc = 1
# Thresholds gate every file as well as the total: a small, well-covered
# file (the generated status-name table) must not mask a regression in a
# bigger one. Files with no branches at all are not branch-gated.
checks = [("total", "line", line_pct, args.threshold),
("total", "branch", branch_pct, args.branch_threshold)]
for path in sorted(covs):
lh, lt = covs[path].line_stats()
bh, bt = covs[path].branch_stats()
checks.append((path, "line", pct(lh, lt), args.threshold))
if bt:
checks.append((path, "branch", pct(bh, bt), args.branch_threshold))
for path, metric, value, threshold in checks:
if threshold > 0 and value < threshold:
print(f"\nFAIL: {path} {metric} coverage {value:.1f}% < threshold "
f"{threshold:.1f}%")
rc = 1
return rc
if __name__ == "__main__":
sys.exit(main())

1
test.sh
View File

@@ -2,3 +2,4 @@
cmake --build build
ctest --test-dir build --output-on-failure --output-junit "$(pwd)/ctest-junit.xml"
python3 scripts/mutation_test.py --target src/error.c --junit mutation-junit.xml --threshold 65
python3 scripts/coverage.py --junit coverage-junit.xml --threshold 90 --branch-threshold 50