diff --git a/AGENTS.md b/AGENTS.md index 4146af5..d39c19b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,8 @@ structures in `libakerror` error contexts. Public API declarations live in one-file CTest executables under `tests/test_.c`, with shared test helpers in `tests/aksl_capture.h`. CMake package templates are in `cmake/` and `akstdlib.pc.in`. The vendored dependency is `deps/libakerror`; update it as a -submodule rather than editing generated files under `build/` or `build-asan/`. +submodule rather than editing generated files under `build/`, `build-asan/` or +`build-coverage/`. ## Build, Test, and Development Commands @@ -29,6 +30,15 @@ cmake --build build-asan ctest --test-dir build-asan --output-on-failure ``` +For coverage, configure a third tree; the report is part of that suite (the +`coverage_reset` / `coverage_report` CTest entries) and also lands in +`build-coverage/coverage-summary.txt`: + +```sh +cmake -S . -B build-coverage -DAKSL_COVERAGE=ON +cmake --build build-coverage --target coverage +``` + Run `cmake --build build --target mutation` only when you need the slower mutation harness. `rebuild.sh` installs to `/home/andrew/local` and removes the local build directory, so treat it as a local convenience script. diff --git a/CMakeLists.txt b/CMakeLists.txt index 6c8c120..b873e86 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,6 +20,50 @@ if(AKSL_SANITIZE) message(STATUS "AKSL_SANITIZE=ON: building with ASan + UBSan") endif() +# Coverage build, off by default: +# cmake -S . -B build-coverage -DAKSL_COVERAGE=ON +# cmake --build build-coverage --target coverage +# Instrumentation is applied per target below (the library and the test +# binaries) rather than through CMAKE_C_FLAGS, so deps/libakerror is left +# uninstrumented -- it has its own suite, and its .gcda would only be noise in +# this project's report. +option(AKSL_COVERAGE "Build the library and its tests with gcov instrumentation" OFF) +# Minimum total line / branch coverage for the `coverage_report` CTest entry +# that a -DAKSL_COVERAGE=ON build adds. 0 disables the gate and reports only. +set(AKSL_COVERAGE_THRESHOLD 0 CACHE STRING + "Fail the coverage_report test below this total line coverage percentage") +set(AKSL_COVERAGE_BRANCH_THRESHOLD 0 CACHE STRING + "Fail the coverage_report test below this total branch coverage percentage") + +if(AKSL_COVERAGE) + if(NOT CMAKE_C_COMPILER_ID MATCHES "^(GNU|Clang|AppleClang)$") + message(FATAL_ERROR + "AKSL_COVERAGE=ON needs a gcov-compatible compiler; " + "CMAKE_C_COMPILER_ID is ${CMAKE_C_COMPILER_ID}") + endif() + # -O0 because the optimizer folds and reorders lines until per-line counts + # stop matching the source. -fprofile-abs-path makes gcov record absolute + # source paths, which is what lets the report be read from any directory. + set(AKSL_COVERAGE_COMPILE_FLAGS --coverage -O0 -g) + if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + list(APPEND AKSL_COVERAGE_COMPILE_FLAGS -fprofile-abs-path) + endif() + message(STATUS "AKSL_COVERAGE=ON: instrumenting akstdlib and its tests for gcov") +endif() + +# Add gcov instrumentation to one target. No-op unless AKSL_COVERAGE is set. +function(aksl_target_coverage target) + if(NOT AKSL_COVERAGE) + return() + endif() + target_compile_options(${target} PRIVATE ${AKSL_COVERAGE_COMPILE_FLAGS}) + if(CMAKE_VERSION VERSION_LESS 3.13) + set_property(TARGET ${target} APPEND_STRING PROPERTY LINK_FLAGS " --coverage") + else() + target_link_options(${target} PRIVATE --coverage) + endif() +endfunction() + if(TARGET akerror::akerror) message(STATUS "FOUND akerror::akerror") else() @@ -83,6 +127,8 @@ target_include_directories(akstdlib PUBLIC target_link_libraries(akstdlib PUBLIC akerror::akerror) +aksl_target_coverage(akstdlib) + set(main_lib_dest "lib/my_library-${MY_LIBRARY_VERSION}") install(TARGETS akstdlib EXPORT akstdlibTargets @@ -147,7 +193,14 @@ foreach(_test IN LISTS AKSL_TESTS AKSL_WILL_FAIL_TESTS AKSL_KNOWN_FAILING_TESTS) add_executable(test_${_test} tests/test_${_test}.c) target_include_directories(test_${_test} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/tests) target_link_libraries(test_${_test} PRIVATE akstdlib) + # The test binaries are instrumented too. The default report filters them out + # (only src/ and include/ are shown), but `scripts/coverage.py --include + # tests` then answers a different question: whether every test case and + # helper branch in tests/ actually runs, which is how a test function that was + # written but never wired into AKSL_RUN shows up. + aksl_target_coverage(test_${_test}) add_test(NAME ${_test} COMMAND test_${_test}) + list(APPEND AKSL_TEST_TARGETS test_${_test}) endforeach() if(AKSL_WILL_FAIL_TESTS OR AKSL_KNOWN_FAILING_TESTS) @@ -168,6 +221,74 @@ set_tests_properties( PROPERTIES TIMEOUT 30 ) +# Both the coverage report and the mutation harness are Python scripts. +find_package(Python3 COMPONENTS Interpreter) + +# Code coverage. A -DAKSL_COVERAGE=ON build wires the report into the suite +# itself, so `ctest --test-dir build-coverage` both runs the tests and prints +# what they touched: +# +# coverage_reset deletes the accumulated .gcda counters. gcov counts are +# cumulative, so without this every report would fold in +# earlier runs and overstate coverage. FIXTURES_SETUP makes +# CTest run it before any test that needs the fixture, even +# under `ctest -j`. +# coverage_report aggregates gcov output and prints the summary plus the +# uncovered lines. FIXTURES_CLEANUP makes CTest run it after +# the last test in the fixture, which is exactly when the +# counters are complete. +# +# The report is a plain report unless AKSL_COVERAGE_THRESHOLD is set, in which +# case coverage_report fails below that percentage. Same ratchet idea as the +# mutation threshold: gate on the number you have, raise it as tests land. +# +# CTest hides the output of a passing test, so coverage_report also writes +# /coverage-summary.txt and /coverage.xml (Cobertura). The +# `coverage` target below builds, runs and prints in one step. +if(AKSL_COVERAGE AND Python3_FOUND) + set(AKSL_COVERAGE_ARGS + ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/scripts/coverage.py + --build ${CMAKE_CURRENT_BINARY_DIR} + --source-root ${CMAKE_CURRENT_SOURCE_DIR}) + + add_test(NAME coverage_reset COMMAND ${AKSL_COVERAGE_ARGS} --zero) + add_test(NAME coverage_report COMMAND ${AKSL_COVERAGE_ARGS} + --threshold ${AKSL_COVERAGE_THRESHOLD} + --branch-threshold ${AKSL_COVERAGE_BRANCH_THRESHOLD} + --output ${CMAKE_CURRENT_BINARY_DIR}/coverage-summary.txt + --cobertura ${CMAKE_CURRENT_BINARY_DIR}/coverage.xml) + + set_tests_properties(coverage_reset PROPERTIES FIXTURES_SETUP AKSL_GCDA) + set_tests_properties(coverage_report PROPERTIES FIXTURES_CLEANUP AKSL_GCDA) + set_tests_properties( + ${AKSL_TESTS} ${AKSL_WILL_FAIL_TESTS} ${AKSL_KNOWN_FAILING_TESTS} + PROPERTIES FIXTURES_REQUIRED AKSL_GCDA + ) + # gcov has to be spawned once per .gcda, so give the report more room than the + # 30s the test binaries get. + set_tests_properties(coverage_reset coverage_report PROPERTIES TIMEOUT 300) + + # Convenience entry point that also builds the test binaries first. It runs + # the suite rather than the script directly, because the two fixture tests + # above already reset the counters and produce the report -- and CTest pulls a + # required fixture back in even when it is filtered out, so there is no way to + # run the tests without them. The second command re-reads the same counters to + # print the report that CTest suppressed for the passing coverage_report test + # (it only spawns gcov again, it does not re-run anything). + add_custom_target(coverage + COMMAND ctest --test-dir ${CMAKE_CURRENT_BINARY_DIR} --output-on-failure + COMMAND ${AKSL_COVERAGE_ARGS} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + USES_TERMINAL + COMMENT "Running the test suite under gcov and reporting coverage" + ) + add_dependencies(coverage ${AKSL_TEST_TARGETS}) +elseif(AKSL_COVERAGE) + message(WARNING "AKSL_COVERAGE=ON but Python3 was not found: " + "instrumenting the build, but the coverage report " + "(scripts/coverage.py) will not be wired into CTest") +endif() + # Mutation testing: break the library in small ways and confirm the test suite # notices. This is a meta-check on the tests themselves, and it rebuilds and # re-runs the whole suite once per mutant, so it is a manual target rather than @@ -178,7 +299,6 @@ set_tests_properties( # This target covers both src/stdlib.c and include/akstdlib.h. CI runs the # narrower, faster src/stdlib.c set with a --threshold gate; see # .gitea/workflows/ci.yaml. -find_package(Python3 COMPONENTS Interpreter) if(Python3_FOUND) if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(AKSL_MUTATION_TARGET mutation) diff --git a/README.md b/README.md index 8b1e9b3..62b5c86 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ package the parent provides instead. ## Testing -There are three harnesses. The first two take seconds; the third takes about +There are four harnesses. The first three take seconds; the fourth takes about half an hour. ### 1. The test suite @@ -80,7 +80,66 @@ misbehave under instrumentation — the uninitialised `%s` in `aksl_realpath`, t unbounded `vsprintf` behind `aksl_sprintf`, the missing `va_end` in the `printf` family — so new tests for those should be run this way. -### 3. Mutation testing +### 3. Code coverage + +```sh +cmake -S . -B build-coverage -DAKSL_COVERAGE=ON +cmake --build build-coverage --target coverage +``` + +`-DAKSL_COVERAGE=ON` compiles the library and the tests with `--coverage -O0`, +and wires the report into the suite itself, so a plain +`ctest --test-dir build-coverage` also produces it. Two extra CTest entries +appear, held in place by a CTest fixture rather than by declaration order, so +they work under `ctest -j` too: + +| Test | When | Does | +|---|---|---| +| `coverage_reset` | before every other test | deletes the accumulated `.gcda` counters | +| `coverage_report` | after every other test | aggregates `gcov` output, prints the summary, applies the threshold gate | + +The reset matters: gcov counters are cumulative, so without it each report would +fold in every earlier run and overstate coverage. + +CTest hides the output of a passing test, so `coverage_report` also writes +`build-coverage/coverage-summary.txt` (the same text report) and +`build-coverage/coverage.xml` (Cobertura, for CI publishers). The `coverage` +target above prints the report to the terminal for you; otherwise read the file +or use `ctest --test-dir build-coverage -V -R coverage_report`. + +The report lists per-file line, branch and function coverage, then every +uncovered line and every function the suite never called — that listing is the +actionable part, the same way surviving mutants are for the harness below. + +Drive the script directly for anything narrower: + +```sh +scripts/coverage.py --build build-coverage # report on disk counters +scripts/coverage.py --build build-coverage --summary-only # totals only +scripts/coverage.py --build build-coverage --include tests # coverage of the tests themselves +scripts/coverage.py --build build-coverage --run-tests # reset, run ctest, report +scripts/coverage.py --build build-coverage --threshold 50 --branch-threshold 25 +``` + +It needs nothing but Python 3 and gcc's own `gcov` — no lcov, gcovr or genhtml. + +To gate on coverage, set the threshold at configure time; `coverage_report` then +fails below it, and the same regression-ratchet logic applies as for the mutation +score: + +```sh +cmake -S . -B build-coverage -DAKSL_COVERAGE=ON \ + -DAKSL_COVERAGE_THRESHOLD=50 -DAKSL_COVERAGE_BRANCH_THRESHOLD=20 +``` + +Two caveats. Coverage is measured at `-O0`, because the optimizer reorders lines +until per-line counts stop matching the source — so a coverage build is not the +build to profile. And gcov flushes its counters at normal process exit, which an +`AKSL_WILL_FAIL_TESTS` entry that aborts by design never reaches: such a test +contributes no coverage data at all, so lines only it reaches are reported as +uncovered. + +### 4. Mutation testing The suite tells you the library works. Mutation testing tells you the *suite* works: it breaks the library in small ways, one at a time, and checks that the diff --git a/scripts/coverage.py b/scripts/coverage.py new file mode 100644 index 0000000..191ddba --- /dev/null +++ b/scripts/coverage.py @@ -0,0 +1,469 @@ +#!/usr/bin/env python3 +""" +Code coverage harness for libakstdlib. + +Reports which lines, branches and functions of the library the CTest suite +actually executed. Coverage answers "what did the tests touch"; the mutation +harness in scripts/mutation_test.py answers the stronger question "would the +tests notice if it broke". They are complementary: a line can be covered and +still have no assertion behind it, so an uncovered line is definitely untested +while a covered one is only maybe tested. + +Requires a build configured with -DAKSL_COVERAGE=ON, which compiles the library +and its tests with --coverage. That emits a .gcno next to every object at build +time and a .gcda next to it at run time; this script feeds those to gcov and +aggregates its JSON output. + +There are no third-party dependencies (Python stdlib + gcc's own gcov), so this +works anywhere the project already builds -- no lcov, gcovr or genhtml needed. + +Usage: + scripts/coverage.py --build build-coverage [options] + + --build DIR build tree to read .gcda from (default: build-coverage) + --source-root DIR repo root (default: parent of this script's dir) + --gcov PROG gcov binary to use (default: $GCOV or gcov) + --include PREFIX only report sources under this repo-relative prefix; + repeatable. Default: src, include + --exclude PREFIX drop sources under this repo-relative prefix; + repeatable. Default: tests, deps, build + --zero delete every .gcda in the build tree, then exit. Run + before the suite so counters do not accumulate across + runs. + --run-tests run ctest in the build tree first (implies --zero) + --threshold PCT exit non-zero if total line coverage < PCT (default 0) + --branch-threshold PCT + exit non-zero if total branch coverage < PCT + --summary-only omit the per-file uncovered-line listing + --output PATH also write the text report to PATH (CTest hides the + output of a passing test, so the in-suite report lands + in /coverage-summary.txt this way) + --cobertura PATH also write a Cobertura XML report (for CI publishers) +""" + +import argparse +import glob +import json +import os +import subprocess +import sys +import xml.sax.saxutils as saxutils + +DEFAULT_INCLUDE = ["src", "include"] +DEFAULT_EXCLUDE = ["tests", "deps", "build"] + + +# --------------------------------------------------------------------------- # +# gcov invocation and parsing +# --------------------------------------------------------------------------- # + +class FileCoverage: + """Accumulated coverage for one source file, keyed by line number. + + A single source file can be represented in several .gcda files -- a header + compiled into every test binary, or a .c file linked more than once -- so + counts are summed across all of them rather than overwritten. + """ + + def __init__(self, path): + self.path = path # repo-relative + self.lines = {} # line number -> execution count + self.branches = {} # line number -> list of branch counts + self.functions = {} # function name -> execution count + + def add_line(self, lineno, count): + self.lines[lineno] = self.lines.get(lineno, 0) + count + + def add_branches(self, lineno, counts): + cur = self.branches.setdefault(lineno, [0] * len(counts)) + # Defensive: different translation units should agree on the branch + # count for a line, but never let a mismatch raise. + if len(cur) != len(counts): + if len(counts) > len(cur): + cur.extend([0] * (len(counts) - len(cur))) + counts = counts + [0] * (len(cur) - len(counts)) + for i, c in enumerate(counts): + cur[i] += c + + def add_function(self, name, count): + self.functions[name] = self.functions.get(name, 0) + count + + # -- derived totals ---------------------------------------------------- # + + @property + def lines_total(self): + return len(self.lines) + + @property + def lines_covered(self): + return sum(1 for c in self.lines.values() if c > 0) + + @property + def branches_total(self): + return sum(len(b) for b in self.branches.values()) + + @property + def branches_covered(self): + return sum(1 for b in self.branches.values() for c in b if c > 0) + + @property + def functions_total(self): + return len(self.functions) + + @property + def functions_covered(self): + return sum(1 for c in self.functions.values() if c > 0) + + def uncovered_lines(self): + return sorted(n for n, c in self.lines.items() if c == 0) + + def uncovered_functions(self): + return sorted(n for n, c in self.functions.items() if c == 0) + + +def pct(covered, total): + """Coverage percentage. A file with nothing instrumented counts as 100%, + matching gcov/lcov: there is nothing there to leave untested.""" + return 100.0 * covered / total if total else 100.0 + + +def find_gcda(build): + return sorted(glob.glob(os.path.join(build, "**", "*.gcda"), recursive=True)) + + +def run_gcov(gcov, gcda): + """Run gcov on one .gcda and return the parsed JSON, or None on failure. + + gcov resolves the matching .gcno relative to the .gcda, so it is invoked + from the .gcda's directory with a bare filename. + """ + workdir = os.path.dirname(gcda) or "." + cmd = [gcov, "--json-format", "--stdout", "--branch-probabilities", + os.path.basename(gcda)] + try: + proc = subprocess.run(cmd, cwd=workdir, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + except OSError as exc: + sys.stderr.write(f"warning: cannot run {gcov}: {exc}\n") + return None + if proc.returncode != 0: + sys.stderr.write(f"warning: gcov failed on {gcda}:\n" + f"{proc.stderr.decode(errors='replace')}") + return None + text = proc.stdout.decode(errors="replace").strip() + if not text: + return None + try: + return json.loads(text) + except ValueError as exc: + sys.stderr.write(f"warning: unparseable gcov JSON for {gcda}: {exc}\n") + return None + + +def rel_to_root(path, cwd, root): + """Resolve a path from gcov's JSON to a repo-relative path, or None if it + falls outside the repo (libc headers, the toolchain, an out-of-tree dep).""" + if not os.path.isabs(path): + path = os.path.join(cwd, path) + path = os.path.realpath(path) + try: + rel = os.path.relpath(path, root) + except ValueError: # different drive (Windows) + return None + if rel.startswith(os.pardir): + return None + return rel.replace(os.sep, "/") + + +def selected(rel, includes, excludes): + """True if a repo-relative path passes the include/exclude prefix filters. + Excludes win, so `--include src --exclude src/generated` behaves sanely.""" + def under(prefix): + return rel == prefix or rel.startswith(prefix.rstrip("/") + "/") + if any(under(e) for e in excludes): + return False + if not includes: + return True + return any(under(i) for i in includes) + + +def collect(build, root, gcov, includes, excludes): + """Aggregate coverage for every selected source file in the build tree.""" + files = {} + gcda_files = find_gcda(build) + for gcda in gcda_files: + data = run_gcov(gcov, gcda) + if not data: + continue + cwd = data.get("current_working_directory") or os.path.dirname(gcda) + for entry in data.get("files", []): + rel = rel_to_root(entry.get("file", ""), cwd, root) + if rel is None or not selected(rel, includes, excludes): + continue + fc = files.setdefault(rel, FileCoverage(rel)) + for line in entry.get("lines", []): + lineno = line.get("line_number") + if lineno is None: + continue + fc.add_line(lineno, line.get("count", 0)) + branches = line.get("branches") or [] + if branches: + fc.add_branches(lineno, [b.get("count", 0) for b in branches]) + for fn in entry.get("functions", []): + name = fn.get("demangled_name") or fn.get("name") + if name: + fc.add_function(name, fn.get("execution_count", 0)) + return files, len(gcda_files) + + +# --------------------------------------------------------------------------- # +# Reporting +# --------------------------------------------------------------------------- # + +def line_ranges(numbers): + """Compress [3,4,5,9,11,12] into ['3-5', '9', '11-12'] for readability.""" + out = [] + start = prev = None + for n in numbers: + if start is None: + start = prev = n + elif n == prev + 1: + prev = n + else: + out.append(str(start) if start == prev else f"{start}-{prev}") + start = prev = n + if start is not None: + out.append(str(start) if start == prev else f"{start}-{prev}") + return out + + +def report(files, summary_only): + """Render the text report. Returns (totals dict, report text).""" + names = sorted(files) + width = max([len(n) for n in names] + [len("TOTAL")]) + out = [] + + header = (f" {'file'.ljust(width)} {'lines':>15} {'branches':>15} " + f"{'functions':>15}") + out.append("=" * len(header)) + out.append("CODE COVERAGE SUMMARY") + out.append("=" * len(header)) + out.append(header) + out.append(" " + "-" * (len(header) - 2)) + + def row(name, lc, lt, bc, bt, fc_, ft): + cell = lambda c, t: f"{pct(c, t):5.1f}% {c:4d}/{t:<4d}" + out.append(f" {name.ljust(width)} {cell(lc, lt):>15} " + f"{cell(bc, bt):>15} {cell(fc_, ft):>15}") + + tot = [0, 0, 0, 0, 0, 0] + for name in names: + f = files[name] + vals = (f.lines_covered, f.lines_total, f.branches_covered, + f.branches_total, f.functions_covered, f.functions_total) + row(name, *vals) + tot = [t + v for t, v in zip(tot, vals)] + + out.append(" " + "-" * (len(header) - 2)) + row("TOTAL", *tot) + + if not summary_only: + for name in names: + f = files[name] + gaps = f.uncovered_lines() + missing_fns = f.uncovered_functions() + if not gaps and not missing_fns: + continue + out.append(f"\n{name}: uncovered (each one is a missing test)") + if missing_fns: + out.append(" functions never called:") + for fn in missing_fns: + out.append(f" {fn}") + if gaps: + out.append(f" lines ({len(gaps)}): " + + ", ".join(line_ranges(gaps))) + + text = "\n".join(out) + "\n" + sys.stdout.write(text) + return text, { + "lines_covered": tot[0], "lines_total": tot[1], + "branches_covered": tot[2], "branches_total": tot[3], + "functions_covered": tot[4], "functions_total": tot[5], + } + + +def write_cobertura(path, files, root, totals): + """Emit a minimal Cobertura report. One per source file, which is + what CI coverage publishers expect for C.""" + esc = saxutils.quoteattr + lr = pct(totals["lines_covered"], totals["lines_total"]) / 100.0 + br = pct(totals["branches_covered"], totals["branches_total"]) / 100.0 + out = ['', + f'', + ' ', f' {saxutils.escape(root)}', + ' ', ' ', + ' ', + ' '] + for name in sorted(files): + f = files[name] + flr = pct(f.lines_covered, f.lines_total) / 100.0 + fbr = pct(f.branches_covered, f.branches_total) / 100.0 + cls = os.path.basename(name) + out.append(f' ') + out.append(' ') + out.append(' ') + for lineno in sorted(f.lines): + count = f.lines[lineno] + brs = f.branches.get(lineno) or [] + if brs: + taken = sum(1 for c in brs if c > 0) + cond = (f' branch="true" condition-coverage=' + f'{esc(f"{pct(taken, len(brs)):.0f}% ({taken}/{len(brs)})")}') + else: + cond = ' branch="false"' + out.append(f' ') + out.append(' ') + out.append(' ') + out += [' ', ' ', ' ', + ''] + with open(path, "w") as fh: + fh.write("\n".join(out) + "\n") + + +# --------------------------------------------------------------------------- # +# Driver +# --------------------------------------------------------------------------- # + +def zero_counters(build): + """Delete accumulated .gcda. gcov counters are cumulative across runs, so + without this a report mixes in every earlier suite run (and any test binary + run by hand), which quietly overstates coverage.""" + removed = 0 + for gcda in find_gcda(build): + try: + os.remove(gcda) + removed += 1 + except OSError as exc: + sys.stderr.write(f"warning: cannot remove {gcda}: {exc}\n") + return removed + + +def main(): + try: + sys.stdout.reconfigure(line_buffering=True) + except (AttributeError, ValueError): + pass + here = os.path.dirname(os.path.abspath(__file__)) + default_root = os.path.dirname(here) + + ap = argparse.ArgumentParser(description="Code coverage for libakstdlib") + ap.add_argument("--build", default="build-coverage") + ap.add_argument("--source-root", default=default_root) + ap.add_argument("--gcov", default=os.environ.get("GCOV", "gcov")) + ap.add_argument("--include", action="append", default=None) + ap.add_argument("--exclude", action="append", default=None) + ap.add_argument("--zero", action="store_true") + ap.add_argument("--run-tests", action="store_true") + ap.add_argument("--threshold", type=float, default=0.0) + ap.add_argument("--branch-threshold", type=float, default=0.0) + ap.add_argument("--summary-only", action="store_true") + ap.add_argument("--output", default=None, + help="also write the text report to this path") + ap.add_argument("--cobertura", default=None) + args = ap.parse_args() + + root = os.path.realpath(args.source_root) + build = os.path.abspath(args.build) + includes = args.include if args.include is not None else DEFAULT_INCLUDE + if args.exclude is not None: + excludes = args.exclude + else: + # Excludes win over includes, so a *default* exclude must not silently + # cancel an explicit --include: `--include tests` has to report tests/ + # even though tests/ is excluded by default. + excludes = [e for e in DEFAULT_EXCLUDE + if not any(selected(e, [i], []) for i in (args.include or []))] + + if not os.path.isdir(build): + sys.stderr.write(f"error: no such build directory: {build}\n" + "Configure one with:\n" + " cmake -S . -B build-coverage -DAKSL_COVERAGE=ON\n" + " cmake --build build-coverage\n") + return 2 + + if args.zero and not args.run_tests: + n = zero_counters(build) + print(f"Cleared {n} .gcda counter file(s) in {build}") + return 0 + + if args.run_tests: + n = zero_counters(build) + print(f"Cleared {n} .gcda counter file(s) in {build}") + print("Running ctest ...") + rc = subprocess.call(["ctest", "--test-dir", build, + "--output-on-failure"]) + if rc != 0: + # Report anyway: partial coverage from a red suite is still useful, + # and the ctest output above already shows what failed. + sys.stderr.write(f"warning: ctest exited {rc}; " + "reporting coverage from a failing suite\n") + print() + + files, n_gcda = collect(build, root, args.gcov, includes, excludes) + if not n_gcda: + sys.stderr.write( + f"error: no .gcda files under {build}.\n" + "Either the build was not configured with -DAKSL_COVERAGE=ON, or\n" + "the test suite has not been run yet (try --run-tests).\n") + return 2 + if not files: + sys.stderr.write( + f"error: {n_gcda} .gcda file(s) found, but none of the covered " + "sources matched\nthe include/exclude filters " + f"(include={includes}, exclude={excludes}).\n") + return 2 + + text, totals = report(files, args.summary_only) + + # CTest swallows the output of a passing test, so also drop the report on + # disk: that is what makes the in-suite coverage_report entry useful without + # having to re-run ctest under -V. + if args.output: + path = os.path.abspath(args.output) + try: + with open(path, "w") as fh: + fh.write(text) + print(f"\nReport written to: {path}") + except OSError as exc: + sys.stderr.write(f"warning: cannot write {path}: {exc}\n") + + if args.cobertura: + path = os.path.abspath(args.cobertura) + write_cobertura(path, files, root, totals) + print(f"\nCobertura report written to: {path}") + + line_pct = pct(totals["lines_covered"], totals["lines_total"]) + branch_pct = pct(totals["branches_covered"], totals["branches_total"]) + failed = False + if args.threshold > 0 and line_pct < args.threshold: + print(f"\nFAIL: line coverage {line_pct:.1f}% < " + f"threshold {args.threshold:.1f}%") + failed = True + if args.branch_threshold > 0 and branch_pct < args.branch_threshold: + print(f"\nFAIL: branch coverage {branch_pct:.1f}% < " + f"threshold {args.branch_threshold:.1f}%") + failed = True + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/mutation_test.py b/scripts/mutation_test.py index b0a38a7..890e3a4 100755 --- a/scripts/mutation_test.py +++ b/scripts/mutation_test.py @@ -277,8 +277,11 @@ def write_junit(path, records, targets): def copy_tree(src, dst): - ignore = shutil.ignore_patterns("build", ".git", "*.o", "*.so", "*~", - "#*#", "*.iso", "*.png") + # "build*" covers build/, build-asan/ and build-coverage/: the harness + # configures its own tree inside the copy, so copying those is pure IO -- + # and a coverage tree drags along every .gcno/.gcda as well. + ignore = shutil.ignore_patterns("build*", ".git", "*.o", "*.so", "*~", + "#*#", "*.iso", "*.png", "*.gcno", "*.gcda") shutil.copytree(src, dst, ignore=ignore, symlinks=True)