diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 974f152..7113c07 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -37,6 +37,36 @@ jobs: fail_on_failure: 'true' - run: echo "🍏 This job's status is ${{ job.status }}." + coverage: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y cmake gcc moreutils python3 + # Run the suite against a gcov-instrumented build and gate on coverage of + # the library sources. Thresholds keep headroom below the current numbers + # (src/error.c: ~94% line, ~60% branch) and apply per file as well as to + # the total, so the generated status-name table cannot mask a regression. + - name: coverage + run: | + python3 scripts/coverage.py --junit coverage-junit.xml --threshold 90 --branch-threshold 50 + # Publish even when the threshold gate fails, so gaps are visible. + # Display-only (fail_on_failure: false); the --threshold above is the gate. + # annotate_only avoids the Checks API 404 on Gitea (see note above). + - name: publish coverage results + if: always() + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'coverage-junit.xml' + annotate_only: true + detailed_summary: true + include_passed: true + fail_on_failure: 'false' + - run: echo "🍏 This job's status is ${{ job.status }}." + mutation_test: runs-on: ubuntu-latest steps: diff --git a/AGENTS.md b/AGENTS.md index 8aec74e..9db9382 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -34,6 +34,20 @@ cmake --build build --target mutation scripts/mutation_test.py --target src/error.c --threshold 65 ``` +Code coverage is available through: + +```sh +cmake --build build --target coverage +scripts/coverage.py --threshold 90 --branch-threshold 50 +``` + +`scripts/coverage.py` configures its own instrumented build tree (default +`build/coverage`, `-DAKERR_COVERAGE=ON`), runs the CTest suite there, and +reports gcov line/branch coverage per library source. Thresholds gate each +file as well as the total. Coverage measures `src/error.c` and the generated +`src/errno.c` only; the public header's macros expand at their call sites, so +mutation testing is what checks those. + ## Coding Style & Naming Conventions Use C99-compatible C and follow the surrounding style. Functions and types use @@ -48,8 +62,9 @@ Add tests as `tests/err_.c`. Register each new test in the `AKERR_TESTS` list in `CMakeLists.txt`; tests that intentionally abort must also be listed in `AKERR_WILL_FAIL_TESTS`. Prefer focused executable tests that return zero on success and use existing helpers such as `AKERR_CHECK`. Run the -CTest suite before submitting changes, and run mutation testing when changing -core control-flow, reference counting, stack-trace, or handler behavior. +CTest suite before submitting changes, and run mutation testing and coverage +when changing core control-flow, reference counting, stack-trace, or handler +behavior. ## Commit & Pull Request Guidelines diff --git a/CMakeLists.txt b/CMakeLists.txt index 8447c82..5df4012 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,8 +6,43 @@ include(CMakePackageConfigHelpers) include(CTest) set(AKERR_USE_STDLIB 1 CACHE BOOL "Use the C standard library") +set(AKERR_COVERAGE 0 CACHE BOOL "Instrument the build with gcov coverage counters") set(akerror_install_cmakedir "${CMAKE_INSTALL_LIBDIR}/cmake/akerror") +# Coverage instrumentation. Applied per target (not globally) so it never leaks +# into the exported/installed target interface. Only the library is +# instrumented: the tests are the thing doing the covering, and the public +# header's macros cannot be measured this way at all -- GCC attributes an +# expanded macro to its call site, so header logic shows up as test-file lines. +# Coverage of those macros is what mutation testing (--target +# include/akerror.tmpl.h) is for. +if(AKERR_COVERAGE) + if(NOT CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") + message(FATAL_ERROR + "AKERR_COVERAGE requires GCC or Clang, not ${CMAKE_C_COMPILER_ID}") + endif() + # -O0 keeps line counts attributable; no inlining or code motion. + set(AKERR_COVERAGE_FLAGS --coverage -O0 -g) + if(CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Record absolute source paths so gcov resolves sources built from the + # generated directory (GCC 8+; harmless to check). + include(CheckCCompilerFlag) + check_c_compiler_flag(-fprofile-abs-path AKERR_HAVE_PROFILE_ABS_PATH) + if(AKERR_HAVE_PROFILE_ABS_PATH) + list(APPEND AKERR_COVERAGE_FLAGS -fprofile-abs-path) + endif() + endif() +endif() + +# Add coverage compile/link flags to one target, if coverage is enabled. +function(akerr_instrument_for_coverage _target) + if(AKERR_COVERAGE) + target_compile_options(${_target} PRIVATE ${AKERR_COVERAGE_FLAGS}) + set_property(TARGET ${_target} APPEND_STRING + PROPERTY LINK_FLAGS " --coverage") + endif() +endfunction() + set(SCRIPT ${CMAKE_CURRENT_SOURCE_DIR}/scripts/generrno.sh) set(INFILE ${CMAKE_CURRENT_SOURCE_DIR}/include/akerror.tmpl.h) @@ -44,6 +79,8 @@ target_compile_definitions(akerror PUBLIC AKERR_USE_STDLIB=${AKERR_USE_STDLIB} ) +akerr_instrument_for_coverage(akerror) + # Each test is one source file in tests/ built into test_ and registered # as CTest . Tests expected to abort (unhandled error / contract # violation) go in AKERR_WILL_FAIL_TESTS; all others must exit 0. @@ -95,14 +132,33 @@ set_tests_properties( PROPERTIES WILL_FAIL TRUE ) -# Mutation testing: break the library in small ways and confirm the test suite -# notices. This is a meta-check on the tests themselves, so it is a manual -# target (it rebuilds and re-runs the whole suite many times), not a CTest test. -# cmake --build build --target mutation -# When embedded in another project, use a namespaced target to avoid collisions -# with mutation targets provided by sibling dependencies. +# Coverage and mutation testing are meta-checks on the test suite itself, and +# both rebuild and re-run the whole suite, so they are manual targets rather +# than CTest tests. find_package(Python3 COMPONENTS Interpreter) if(Python3_FOUND) + # Code coverage: which library lines/branches the CTest suite reaches. + # cmake --build build --target coverage + # The script configures and drives its own instrumented build tree (under + # ${CMAKE_BINARY_DIR}/coverage) so this build's binaries and its coverage + # counters can never be stale or half-instrumented. Reports via gcov. + add_custom_target(coverage + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/coverage.py + --source-root ${CMAKE_CURRENT_SOURCE_DIR} + --build-dir ${CMAKE_CURRENT_BINARY_DIR}/coverage + --cmake ${CMAKE_COMMAND} + --ctest ${CMAKE_CTEST_COMMAND} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + USES_TERMINAL + COMMENT "Running the test suite instrumented for coverage" + ) + + # Mutation testing: break the library in small ways and confirm the test + # suite notices. + # cmake --build build --target mutation + # When embedded in another project, use a namespaced target to avoid + # collisions with mutation targets provided by sibling dependencies. if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) set(AKERR_MUTATION_TARGET mutation) else() diff --git a/scripts/coverage.py b/scripts/coverage.py new file mode 100755 index 0000000..30b33ad --- /dev/null +++ b/scripts/coverage.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +""" +Code coverage harness for libakerror. + +Coverage measures which parts of the library the CTest suite actually executes. +It is the complement to mutation testing (scripts/mutation_test.py): coverage +finds code the tests never reach, mutation testing finds code the tests reach +but do not really check. + +What is measured is the library's own translation units: src/error.c and the +generated src/errno.c. The macros in the public header are deliberately not +measured -- GCC attributes an expanded macro to its call site, so header logic +would be reported as lines of the test that used it. Use mutation testing +(--target include/akerror.tmpl.h) to check how well those macros are tested. + +This harness has no third-party dependencies -- just gcov, which ships with the +compiler, plus the project's normal cmake/ctest toolchain. By default it +configures its own instrumented build directory so the ordinary build tree is +left alone and coverage counters can never be stale. + +Usage: + scripts/coverage.py [options] + + --source-root DIR repo root (default: parent of this script's dir) + --build-dir DIR instrumented build dir (default: /build/coverage) + --no-configure reuse the build dir as-is (do not cmake/build) + --no-run report existing counters (do not reset and run ctest) + --threshold PCT exit non-zero if line coverage < PCT (default: 0 = off) + --branch-threshold P exit non-zero if branch coverage < P (default: 0 = off) + --junit PATH write a JUnit XML report to this path + --max-uncovered N uncovered lines to list per file (default: 40, 0 = all) + --exclude SUBSTR skip reported paths containing SUBSTR; repeatable + --gcov PROG gcov program (default: $GCOV or "gcov") +""" + +import argparse +import collections +import json +import os +import subprocess +import sys + + +# --------------------------------------------------------------------------- # +# Running the instrumented suite +# --------------------------------------------------------------------------- # + +def configure_and_build(cmake, root, build, jobs): + """Configure an instrumented build dir and build everything in it.""" + cmds = [ + [cmake, "-S", root, "-B", build, "-DAKERR_COVERAGE=ON"], + [cmake, "--build", build] + (["--parallel", str(jobs)] if jobs else []), + ] + for cmd in cmds: + print("+ " + " ".join(cmd)) + if subprocess.call(cmd) != 0: + return False + return True + + +def reset_counters(build): + """Delete accumulated .gcda files so each run reports one suite run.""" + removed = 0 + for dirpath, _dirs, files in os.walk(build): + for name in files: + if name.endswith(".gcda"): + os.unlink(os.path.join(dirpath, name)) + removed += 1 + if removed: + print(f"Reset {removed} coverage data file(s).") + + +def run_ctest(build, ctest): + """Run the suite. Returns True if every test passed. + + Coverage counters are flushed at exit(), and the tests that are expected to + fail exit via the library's unhandled-error handler (exit(), not abort()), + so WILL_FAIL tests still contribute their counters. + """ + cmd = [ctest, "--output-on-failure"] + print("+ " + " ".join(cmd) + f" (in {build})") + return subprocess.call(cmd, cwd=build) == 0 + + +# --------------------------------------------------------------------------- # +# Collecting gcov data +# --------------------------------------------------------------------------- # + +class FileCov: + """Merged coverage for one source file, across every object that built it. + + A file compiled into more than one object -- or a header included by several + translation units -- is reported once, with counts summed. Branches are + merged by (line, index within line), so differing expansions of the same + line contribute the union of their branches. + """ + + def __init__(self): + self.lines = collections.Counter() # lineno -> execution count + self.branches = collections.Counter() # (lineno, idx) -> taken count + self.funcs = collections.Counter() # (name, start_line) -> count + + def merge(self, entry): + for ln in entry.get("lines", []): + no = ln["line_number"] + self.lines[no] += ln.get("count", 0) + for idx, br in enumerate(ln.get("branches", [])): + if br.get("throw"): + continue + self.branches[(no, idx)] += br.get("count", 0) + for fn in entry.get("functions", []): + key = (fn.get("name", "?"), fn.get("start_line", 0)) + self.funcs[key] += fn.get("execution_count", 0) + + @staticmethod + def _ratio(counter): + total = len(counter) + hit = sum(1 for v in counter.values() if v > 0) + return hit, total + + def line_stats(self): + return self._ratio(self.lines) + + def branch_stats(self): + return self._ratio(self.branches) + + def func_stats(self): + return self._ratio(self.funcs) + + def uncovered_lines(self): + return sorted(no for no, count in self.lines.items() if count == 0) + + +def find_notes(build): + """Every .gcno in the build tree: one per instrumented translation unit.""" + notes = [] + for dirpath, _dirs, files in os.walk(build): + for name in files: + if name.endswith(".gcno"): + notes.append(os.path.join(dirpath, name)) + return sorted(notes) + + +def gcov_json(gcov, note, cwd): + """Run gcov on one .gcno and return its parsed JSON, or None on failure. + + --stdout keeps gcov from littering .gcov files in the build tree. A .gcno + with no matching .gcda still reports, with all counts zero, which is the + correct answer for a translation unit no test executed. + """ + cmd = [gcov, "--branch-probabilities", "--json-format", "--stdout", note] + proc = subprocess.Popen(cmd, cwd=cwd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + out, err = proc.communicate() + if proc.returncode != 0: + sys.stderr.write(f"warning: {' '.join(cmd)} failed:\n" + f"{err.decode(errors='replace')}") + return None + try: + return json.loads(out.decode(errors="replace")) + except ValueError as exc: + sys.stderr.write(f"warning: unparseable gcov output for {note}: {exc}\n") + return None + + +def display_path(path, build, root): + """Label a source: build-relative for generated files, else repo-relative. + + Returns None for anything outside both trees (system headers, toolchain + internals). Generated sources are labelled relative to the build dir so the + report and its thresholds do not change with the build dir's location. + """ + for base in (build, root): + if path.startswith(base + os.sep): + return os.path.relpath(path, base) + return None + + +def collect(gcov, build, root, excludes): + """Merge gcov data for every instrumented unit into {display path: FileCov}.""" + covs = {} + for note in find_notes(build): + data = gcov_json(gcov, note, build) + if data is None: + continue + # Paths in the report are relative to the directory the unit was + # compiled in, which gcov records in the notes file. + compile_dir = data.get("current_working_directory") or build + for entry in data.get("files", []): + path = entry.get("file", "") + if not os.path.isabs(path): + path = os.path.join(compile_dir, path) + display = display_path(os.path.realpath(path), build, root) + if display is None: + continue # system headers, toolchain internals + if any(x in display for x in excludes): + continue + covs.setdefault(display, FileCov()).merge(entry) + return covs + + +# --------------------------------------------------------------------------- # +# Reporting +# --------------------------------------------------------------------------- # + +def pct(hit, total): + return 100.0 * hit / total if total else 100.0 + + +def fmt_ratio(hit, total): + if not total: + return f"{'-':>9} -" + return f"{hit:>4}/{total:<4} {pct(hit, total):5.1f}%" + + +def compress(numbers): + """[1,2,3,7,9,10] -> '1-3, 7, 9-10' for readable uncovered-line lists.""" + out, start, prev = [], None, None + for n in numbers: + if start is None: + start = prev = n + elif n == prev + 1: + prev = n + else: + out.append(f"{start}" if start == prev else f"{start}-{prev}") + start = prev = n + if start is not None: + out.append(f"{start}" if start == prev else f"{start}-{prev}") + return ", ".join(out) + + +def report(covs, max_uncovered): + """Print the per-file table and uncovered detail; return overall percentages.""" + width = max([len(p) for p in covs] + [len("TOTAL")]) + print("\n" + "=" * 72) + print("CODE COVERAGE SUMMARY") + print("=" * 72) + print(f" {'FILE':<{width}} {'LINES':^15} {'BRANCHES':^15} FUNCS") + + totals = [0, 0, 0, 0, 0, 0] # lines hit/total, branches hit/total, funcs + for path in sorted(covs): + cov = covs[path] + lh, lt = cov.line_stats() + bh, bt = cov.branch_stats() + fh, ft = cov.func_stats() + for i, v in enumerate((lh, lt, bh, bt, fh, ft)): + totals[i] += v + print(f" {path:<{width}} {fmt_ratio(lh, lt)} {fmt_ratio(bh, bt)} " + f"{fh}/{ft}") + + lh, lt, bh, bt, fh, ft = totals + print(f" {'-' * width} {'-' * 15} {'-' * 15} -----") + print(f" {'TOTAL':<{width}} {fmt_ratio(lh, lt)} {fmt_ratio(bh, bt)} " + f"{fh}/{ft}") + + for path in sorted(covs): + missing = covs[path].uncovered_lines() + if not missing: + continue + shown = missing if not max_uncovered else missing[:max_uncovered] + more = "" if len(shown) == len(missing) else \ + f" ... (+{len(missing) - len(shown)} more)" + print(f"\n uncovered in {path} ({len(missing)} line(s)):") + print(f" {compress(shown)}{more}") + + return pct(lh, lt), pct(bh, bt) + + +def _xml_escape(text): + return (str(text).replace("&", "&").replace("<", "<") + .replace(">", ">").replace('"', """)) + + +def write_junit(path, covs, line_threshold, branch_threshold): + """One per file per metric; below-threshold is a .""" + cases = [] + for f in sorted(covs): + cov = covs[f] + cases.append((f, "lines", cov.line_stats(), line_threshold)) + cases.append((f, "branches", cov.branch_stats(), branch_threshold)) + for metric, thr, stats in (("lines", line_threshold, + [c.line_stats() for c in covs.values()]), + ("branches", branch_threshold, + [c.branch_stats() for c in covs.values()])): + cases.append(("TOTAL", metric, + (sum(h for h, _t in stats), sum(t for _h, t in stats)), + thr)) + + fails = sum(1 for _f, _m, (h, t), thr in cases + if t and thr > 0 and pct(h, t) < thr) + out = ['', + f'', + f' '] + for f, metric, (hit, total), thr in cases: + name = _xml_escape(f"{f} {metric}") + detail = _xml_escape(f"{hit}/{total} ({pct(hit, total):.1f}%)" + if total else "no data") + out.append(f' ') + if total and thr > 0 and pct(hit, total) < thr: + out.append(f' ' + f'{detail}') + else: + out.append(f' {detail}') + out.append(' ') + out.append(' ') + out.append('') + with open(path, "w") as fh: + fh.write("\n".join(out) + "\n") + + +def main(): + # Line-buffer stdout so progress is visible live under CI / the cmake target. + try: + sys.stdout.reconfigure(line_buffering=True) + except (AttributeError, ValueError): + pass + here = os.path.dirname(os.path.abspath(__file__)) + default_root = os.path.dirname(here) + + ap = argparse.ArgumentParser(description="Code coverage for libakerror") + ap.add_argument("--source-root", default=default_root) + ap.add_argument("--build-dir", default=None) + ap.add_argument("--configure", dest="configure", action="store_true", + default=True) + ap.add_argument("--no-configure", dest="configure", action="store_false") + ap.add_argument("--run", dest="run", action="store_true", default=True) + ap.add_argument("--no-run", dest="run", action="store_false") + ap.add_argument("--threshold", type=float, default=0.0, + help="fail if line coverage is below this percentage") + ap.add_argument("--branch-threshold", type=float, default=0.0, + help="fail if branch coverage is below this percentage") + ap.add_argument("--junit", default=None, + help="write a JUnit XML report to this path") + ap.add_argument("--max-uncovered", type=int, default=40) + ap.add_argument("--exclude", action="append", default=None, + help="skip reported paths containing this substring") + ap.add_argument("--gcov", default=os.environ.get("GCOV", "gcov")) + ap.add_argument("--cmake", default=os.environ.get("CMAKE", "cmake")) + ap.add_argument("--ctest", default=os.environ.get("CTEST", "ctest")) + ap.add_argument("-j", "--jobs", type=int, default=0) + args = ap.parse_args() + + root = os.path.realpath(args.source_root) + build = os.path.realpath(args.build_dir or os.path.join(root, "build", + "coverage")) + # The tests exercise the library; their own source is not what we measure. + excludes = args.exclude if args.exclude is not None else ["tests/"] + + if args.configure: + if not configure_and_build(args.cmake, root, build, args.jobs): + sys.stderr.write("\nInstrumented build FAILED; aborting.\n") + return 2 + elif not os.path.isdir(build): + sys.stderr.write(f"No build dir at {build} (drop --no-configure).\n") + return 2 + + tests_ok = True + if args.run: + reset_counters(build) + tests_ok = run_ctest(build, args.ctest) + if not tests_ok: + sys.stderr.write("\nwarning: some tests FAILED; coverage below is " + "still reported, but the run is not green.\n") + + covs = collect(args.gcov, build, root, excludes) + if not covs: + sys.stderr.write("No coverage data found. Was the build instrumented " + "(-DAKERR_COVERAGE=ON) and the suite run?\n") + return 2 + + line_pct, branch_pct = report(covs, args.max_uncovered) + + if args.junit: + junit_path = os.path.abspath(args.junit) + write_junit(junit_path, covs, args.threshold, args.branch_threshold) + print(f"\nJUnit report written to: {junit_path}") + + rc = 0 + if not tests_ok: + print("\nFAIL: the CTest suite did not pass.") + rc = 1 + + # Thresholds gate every file as well as the total: a small, well-covered + # file (the generated status-name table) must not mask a regression in a + # bigger one. Files with no branches at all are not branch-gated. + checks = [("total", "line", line_pct, args.threshold), + ("total", "branch", branch_pct, args.branch_threshold)] + for path in sorted(covs): + lh, lt = covs[path].line_stats() + bh, bt = covs[path].branch_stats() + checks.append((path, "line", pct(lh, lt), args.threshold)) + if bt: + checks.append((path, "branch", pct(bh, bt), args.branch_threshold)) + for path, metric, value, threshold in checks: + if threshold > 0 and value < threshold: + print(f"\nFAIL: {path} {metric} coverage {value:.1f}% < threshold " + f"{threshold:.1f}%") + rc = 1 + return rc + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/test.sh b/test.sh index 50da858..cdab2e6 100644 --- a/test.sh +++ b/test.sh @@ -2,3 +2,4 @@ cmake --build build ctest --test-dir build --output-on-failure --output-junit "$(pwd)/ctest-junit.xml" python3 scripts/mutation_test.py --target src/error.c --junit mutation-junit.xml --threshold 65 + python3 scripts/coverage.py --junit coverage-junit.xml --threshold 90 --branch-threshold 50