diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 431afda..45e6569 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -21,3 +21,22 @@ jobs: cmake --install build cmake --build build --target test - run: echo "🍏 This job's status is ${{ job.status }}." + + mutation_test: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + - name: dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y cmake gcc moreutils python3 + # Verify the tests actually catch bugs: break the library many ways and + # confirm the suite fails. Gated on src/error.c (fast, deterministic). + # The threshold keeps headroom below the current score for equivalent + # mutants; see tests/MUTATION.md. Run the full default target locally for + # deeper (slower) coverage including the macro header. + - name: mutation testing + run: | + python3 scripts/mutation_test.py --target src/error.c --threshold 65 + - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/CMakeLists.txt b/CMakeLists.txt index 015debf..9109d35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -63,6 +63,9 @@ set(AKERR_TESTS err_errno err_break_variants err_custom_handler + err_error_names + err_release_clears + err_pool_exhaust ) set(AKERR_WILL_FAIL_TESTS @@ -82,6 +85,22 @@ set_tests_properties( PROPERTIES WILL_FAIL TRUE ) +# Mutation testing: break the library in small ways and confirm the test suite +# notices. This is a meta-check on the tests themselves, so it is a manual +# target (it rebuilds and re-runs the whole suite many times), not a CTest test. +# cmake --build build --target mutation +find_package(Python3 COMPONENTS Interpreter) +if(Python3_FOUND) + add_custom_target(mutation + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/mutation_test.py + --source-root ${CMAKE_CURRENT_SOURCE_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + USES_TERMINAL + COMMENT "Running mutation tests (breaks the library, expects tests to fail)" + ) +endif() + set(main_lib_dest "lib/my_library-${MY_LIBRARY_VERSION}") install(TARGETS akerror EXPORT akerrorTargets diff --git a/scripts/mutation_test.py b/scripts/mutation_test.py new file mode 100755 index 0000000..deb3312 --- /dev/null +++ b/scripts/mutation_test.py @@ -0,0 +1,385 @@ +#!/usr/bin/env python3 +""" +Mutation testing harness for libakerror. + +Mutation testing measures how good the test suite is at catching bugs. It works +by making many small, deliberate breakages ("mutants") to the library source -- +flipping a comparison, deleting a statement, swapping true/false -- and then +running the whole CTest suite against each one. If the tests fail, the mutant is +"killed" (good: the tests noticed the bug). If the tests still pass, the mutant +"survived" (bad: a real bug of that shape would slip through unnoticed). + +The mutation score is killed / (killed + survived). Surviving mutants are printed +with file:line and the exact change so they can be turned into new test cases. + +This harness has no third-party dependencies (Python stdlib + the project's +normal cmake/ctest toolchain). It never mutates the real working tree: it copies +the repo to a scratch directory and mutates there. + +Usage: + scripts/mutation_test.py [options] + + --source-root DIR repo root to copy (default: parent of this script's dir) + --target FILE source file to mutate, relative to root; repeatable. + Default: src/error.c and include/akerror.tmpl.h + --work DIR scratch dir for the mutated copy (default: a temp dir) + --timeout SECONDS per-suite ctest timeout (default: 120) + --threshold PCT exit non-zero if mutation score < PCT (default: 0 = off) + --list only list the mutants that would be run, then exit + --keep keep the scratch working copy on exit (for debugging) + -j N (reserved) currently runs sequentially +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tempfile + +# --------------------------------------------------------------------------- # +# Mutation operators +# +# Each operator yields zero or more (start, end, replacement) edits for a single +# line of source. The driver applies exactly one edit per mutant so every mutant +# differs from the original by one localized change. +# --------------------------------------------------------------------------- # + +# Relational operator replacement: map each operator to the alternatives that +# meaningfully change behaviour (not merely the strict negation). +_REL = { + "==": ["!="], + "!=": ["=="], + "<=": ["<", "=="], + ">=": [">", "=="], + "<": ["<=", ">"], + ">": [">=", "<"], +} +# Match a relational operator that is NOT part of ->, <<, >>, =>, <=, >=, ==, != +# unless we intend it. We tokenize the two-char operators first, then single. +_REL_TWO = re.compile(r"(==|!=|<=|>=)") +_REL_ONE = re.compile(r"(?=!+])([<>])(?![=<>])") + +_LOGICAL = {"&&": "||", "||": "&&"} +_LOG_RE = re.compile(r"(&&|\|\|)") + +_BOOL = {"true": "false", "false": "true"} +_BOOL_RE = re.compile(r"\b(true|false)\b") + +# Arithmetic / compound-assignment on whitespace-delimited operands only, to +# avoid touching ++, --, ->, unary signs, or pointer/format punctuation. +_ARITH_RE = re.compile(r"(?<=\s)([+\-])(?=\s)") +_ARITH = {"+": "-", "-": "+"} +_COMPOUND_RE = re.compile(r"(\+=|-=)") +_COMPOUND = {"+=": "-=", "-=": "+="} + +# Integer literal replacement: 0 <-> 1 (word-bounded, not inside identifiers or +# larger numbers, not a float). +_INT_RE = re.compile(r"(?().\[\]* ]*\s*=\s*[^;]* | # assignments + [A-Za-z_][\w]*\s*\([^;]*\) # bare function calls + )\s*;\s*(\\?)\s*$""", + re.VERBOSE, +) + + +# --------------------------------------------------------------------------- # +# Deciding which lines are eligible to mutate +# --------------------------------------------------------------------------- # + +# Skip preprocessor control and the block of constant/error-code #defines in the +# template header: mutating buffer sizes or renumbering error codes produces +# equivalent or uninteresting mutants that swamp the signal. +_SKIP_LINE = re.compile( + r"""^\s*( + \#\s*(include|ifn?def|ifdef|if|elif|else|endif|error|pragma|undef) | + \#\s*define\s+AKERR_(MAX|LAST|NULLPOINTER|OUTOFBOUNDS|API|ATTRIBUTE| + TYPE|KEY|INDEX|FORMAT|IO|VALUE|RELATIONSHIP|EOF|CIRCULAR_REFERENCE| + ITERATOR_BREAK|NOT_IMPLEMENTED|BADEXC|NOIGNORE|USE_STDLIB)\b | + \* | // # comment bodies / line comments + )""", + re.VERBOSE, +) + + +def _is_comment_or_blank(line): + s = line.strip() + return (not s) or s.startswith("//") or s.startswith("/*") or s.startswith("*") + + +def eligible(line): + if _is_comment_or_blank(line): + return False + if _SKIP_LINE.match(line): + return False + return True + + +class Mutant: + __slots__ = ("path", "lineno", "op", "before", "after", "col") + + def __init__(self, path, lineno, op, before, after, col): + self.path = path + self.lineno = lineno + self.op = op + self.before = before + self.after = after + self.col = col + + def describe(self): + return (f"{self.path}:{self.lineno} [{self.op}] " + f"col{self.col}: {self.before.strip()} -> {self.after.strip()}") + + +def generate_mutants(root, rel_target): + """Enumerate all mutants for one target file.""" + abspath = os.path.join(root, rel_target) + with open(abspath, "r") as fh: + lines = fh.readlines() + + mutants = [] + for i, line in enumerate(lines, start=1): + if not eligible(line): + continue + # substitution operators + seen = set() + for tag, s, e, repl in _op_edits(line): + key = (s, e, repl) + if key in seen: + continue + seen.add(key) + mutated = line[:s] + repl + line[e:] + if mutated == line: + continue + mutants.append(Mutant(rel_target, i, tag, line, mutated, s)) + # statement deletion + m = _STMT_DELETABLE.match(line) + if m: + indent = line[: len(line) - len(line.lstrip())] + cont = "\\" if line.rstrip().endswith("\\") else "" + deleted = f"{indent}/* mutant: deleted */ {cont}\n" if cont else f"{indent};\n" + mutants.append(Mutant(rel_target, i, "SDL", line, deleted, 0)) + return mutants + + +# --------------------------------------------------------------------------- # +# Build / test orchestration against a scratch copy +# --------------------------------------------------------------------------- # + +class Runner: + def __init__(self, work, timeout): + self.work = work + self.build = os.path.join(work, "build") + self.timeout = timeout + + def _run(self, cmd, timeout=None): + return subprocess.run( + cmd, cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=timeout, + ) + + def configure(self): + r = self._run(["cmake", "-S", ".", "-B", "build"], timeout=self.timeout) + return r.returncode == 0, r.stdout + + def build_and_test(self): + """Return ('killed-compile' | 'killed-test' | 'killed-timeout' | 'survived').""" + try: + b = self._run(["cmake", "--build", "build"], timeout=self.timeout) + except subprocess.TimeoutExpired: + return "killed-timeout" + if b.returncode != 0: + return "killed-compile" + try: + t = subprocess.run( + ["ctest", "--test-dir", "build", "--output-on-failure", + "--stop-on-failure"], + cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=self.timeout, + ) + except subprocess.TimeoutExpired: + return "killed-timeout" + return "survived" if t.returncode == 0 else "killed-test" + + +def copy_tree(src, dst): + ignore = shutil.ignore_patterns("build", ".git", "*.o", "*.so", "*~", + "#*#", "*.iso", "*.png") + shutil.copytree(src, dst, ignore=ignore, symlinks=True) + + +def read_lines(path): + with open(path) as fh: + return fh.readlines() + + +def write_lines(path, lines): + with open(path, "w") as fh: + fh.writelines(lines) + + +def main(): + # Line-buffer stdout so progress is visible live under CI / the cmake target. + try: + sys.stdout.reconfigure(line_buffering=True) + except (AttributeError, ValueError): + pass + here = os.path.dirname(os.path.abspath(__file__)) + default_root = os.path.dirname(here) + + ap = argparse.ArgumentParser(description="Mutation testing for libakerror") + ap.add_argument("--source-root", default=default_root) + ap.add_argument("--target", action="append", default=None) + ap.add_argument("--work", default=None) + ap.add_argument("--timeout", type=int, default=120) + ap.add_argument("--threshold", type=float, default=0.0) + ap.add_argument("--max-mutants", type=int, default=0, + help="cap the run at N evenly-sampled mutants (0 = all)") + ap.add_argument("--list", action="store_true") + ap.add_argument("--keep", action="store_true") + ap.add_argument("-j", type=int, default=1) + args = ap.parse_args() + + root = os.path.abspath(args.source_root) + targets = args.target or ["src/error.c", "include/akerror.tmpl.h"] + + # Enumerate mutants from the pristine sources. + all_mutants = [] + for t in targets: + all_mutants.extend(generate_mutants(root, t)) + + print(f"Generated {len(all_mutants)} mutants across {len(targets)} file(s):") + for t in targets: + n = sum(1 for m in all_mutants if m.path == t) + print(f" {t}: {n}") + + # Optional even-strided sampling to bound run time (CI / smoke tests). + if args.max_mutants and len(all_mutants) > args.max_mutants: + step = len(all_mutants) / args.max_mutants + sampled = [all_mutants[int(i * step)] for i in range(args.max_mutants)] + print(f"Sampling {len(sampled)} of {len(all_mutants)} mutants " + f"(--max-mutants {args.max_mutants}).") + all_mutants = sampled + + if args.list: + for m in all_mutants: + print(" " + m.describe()) + return 0 + + if not all_mutants: + print("No mutants generated; nothing to do.") + return 0 + + # Scratch working copy. + work_parent = args.work or tempfile.mkdtemp(prefix="akerr_mut_") + work = os.path.join(work_parent, "src") if args.work else work_parent + if os.path.exists(work): + shutil.rmtree(work) + print(f"\nCopying sources to scratch dir: {work}") + copy_tree(root, work) + + runner = Runner(work, args.timeout) + + print("Configuring baseline ...") + ok, out = runner.configure() + if not ok: + sys.stderr.write(out.decode(errors="replace")) + sys.stderr.write("\nBaseline configure FAILED; aborting.\n") + return 2 + + print("Verifying baseline is green (no mutation) ...") + baseline = runner.build_and_test() + if baseline != "survived": + sys.stderr.write(f"Baseline is not green ({baseline}); aborting. " + "Fix the suite before mutation testing.\n") + return 2 + print("Baseline OK.\n") + + # Group mutants by file so we mutate one file at a time and restore it. + killed = {"killed-compile": 0, "killed-test": 0, "killed-timeout": 0} + survivors = [] + total = len(all_mutants) + + # Cache pristine contents per target. + pristine = {t: read_lines(os.path.join(work, t)) for t in targets} + + for idx, m in enumerate(all_mutants, start=1): + tgt_abs = os.path.join(work, m.path) + lines = list(pristine[m.path]) + lines[m.lineno - 1] = m.after + write_lines(tgt_abs, lines) + try: + result = runner.build_and_test() + finally: + write_lines(tgt_abs, pristine[m.path]) # always restore + + if result == "survived": + survivors.append(m) + mark = "SURVIVED" + else: + killed[result] += 1 + mark = result.upper() + print(f"[{idx}/{total}] {mark:16} {m.describe()}") + + total_killed = sum(killed.values()) + score = 100.0 * total_killed / total if total else 100.0 + + print("\n" + "=" * 72) + print("MUTATION TESTING SUMMARY") + print("=" * 72) + print(f" total mutants : {total}") + print(f" killed (test) : {killed['killed-test']}") + print(f" killed (compile): {killed['killed-compile']}") + print(f" killed (timeout): {killed['killed-timeout']}") + print(f" survived : {len(survivors)}") + print(f" mutation score : {score:.1f}%") + if survivors: + print("\nSurviving mutants (test-suite gaps -- turn these into tests):") + for m in survivors: + print(" " + m.describe()) + + if not args.keep and not args.work: + shutil.rmtree(work_parent, ignore_errors=True) + else: + print(f"\nScratch working copy kept at: {work}") + + if args.threshold > 0 and score < args.threshold: + print(f"\nFAIL: mutation score {score:.1f}% < threshold {args.threshold:.1f}%") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/MUTATION.md b/tests/MUTATION.md new file mode 100644 index 0000000..856d999 --- /dev/null +++ b/tests/MUTATION.md @@ -0,0 +1,100 @@ +# Mutation testing + +The unit tests tell us the library works. **Mutation testing tells us the tests +work** — that they would actually fail if the library were broken. + +`scripts/mutation_test.py` deliberately breaks the library in small ways +("mutants"), one at a time, and runs the whole CTest suite against each broken +copy: + +* if the tests **fail**, the mutant is **killed** — good, the suite caught it; +* if the tests still **pass**, the mutant **survived** — a bug of that shape + would slip through, so it points at a missing test. + +The **mutation score** is `killed / (killed + survived)`. A surviving mutant is +a to-do item: write a test that distinguishes the mutant from the original. + +## Running + +No third-party tools are required — just Python 3 and the normal +cmake/ctest toolchain. The harness never touches your working tree; it copies +the repo to a scratch directory and mutates the copy. + +```sh +# Default: mutate src/error.c and include/akerror.tmpl.h +scripts/mutation_test.py + +# Faster: just the C source +scripts/mutation_test.py --target src/error.c + +# See what would run without building anything +scripts/mutation_test.py --target src/error.c --list + +# Gate CI: exit non-zero if the score drops below 90% +scripts/mutation_test.py --threshold 90 +``` + +Via CMake (configures a build first if needed): + +```sh +cmake --build build --target mutation +``` + +Useful flags: `--timeout SECONDS` (per-suite build+test cap; a mutant that +hangs is counted as killed), `--keep` (retain the scratch copy for debugging), +`--work DIR` (use a specific scratch directory). + +## Mutation operators + +Each mutant changes exactly one location by one of: + +| Tag | Operator | Example | +|-----|--------------------------------|----------------------------------| +| ROR | relational operator | `==` → `!=`, `<` → `<=`, `>=` → `>` | +| LCR | logical connector | `&&` → `\|\|` | +| BCR | boolean constant | `true` → `false` | +| AOR | arithmetic / compound assign | `+` → `-`, `+=` → `-=` | +| ICR | integer literal | `0` → `1`, `1` → `0` | +| SDL | statement deletion | `err->refcount += 1;` → *(removed)* | + +Preprocessor control lines, comments, and the block of error-code / buffer-size +`#define`s are skipped: mutating those produces equivalent or uninteresting +mutants that only add noise. + +## Interpreting survivors + +Not every survivor is a test gap — some mutants are **equivalent** (they don't +change observable behaviour, e.g. resizing an internal scratch buffer). For each +survivor, decide: + +1. **Real gap** → add or strengthen a test in `tests/` so the mutant is killed, + then re-run. +2. **Equivalent mutant** → no test can catch it; leave a note. If a specific + line is a persistent source of equivalents, narrow the target with + `--target` or extend the skip rules in `scripts/mutation_test.py`. + +Re-run after adding tests and confirm the score went up. + +## Current status + +`src/error.c` scores ~71% (the CI gate is set to 65% for headroom). The +remaining survivors are dominated by: + +* **Equivalent mutants** in `akerr_init`: deleting the `memset`/`NULL` setup of + file-scope statics (`AKERR_ARRAY_ERROR`, `__akerr_last_ditch`, + `__akerr_last_ignored`) changes nothing, because C already zero-initializes + objects with static storage duration. `int oldid = 0;` → `1` is likewise + dead: it is overwritten before use. +* **Default logger / handler internals** (`vfprintf`, `va_end`, the + `errctx == NULL` branch, `exit(1)`): killing these needs a subprocess-based + test that captures a child's stderr and exit code, rather than the in-process + capturing logger the other tests use. + +Findings worth noting (surfaced by mutation testing, not yet fixed): + +* `AKERR_MAX_ERR_VALUE` is `AKERR_LAST_ERRNO_VALUE + 15`, but `AKERR_NOT_IMPLEMENTED` + (+16) and `AKERR_BADEXC` (+17) exceed it. `akerr_name_for_status` rejects any + status `> AKERR_MAX_ERR_VALUE`, so those two codes can never store or return a + name — the `akerr_name_for_status(AKERR_BADEXC, ...)` call in `akerr_init` is + dead code (which is why deleting it survives). Bumping `AKERR_MAX_ERR_VALUE` to + `+ 17` would fix it. diff --git a/tests/err_error_names.c b/tests/err_error_names.c new file mode 100644 index 0000000..80b6621 --- /dev/null +++ b/tests/err_error_names.c @@ -0,0 +1,45 @@ +#include "akerror.h" +#include "err_capture.h" +#include + +/* + * akerr_init() registers a human-readable name for each library error code. + * Verify the names are actually installed (mutation testing showed the + * registration calls could be deleted without any test noticing). + * + * Note: AKERR_NOT_IMPLEMENTED and AKERR_BADEXC are intentionally omitted -- they + * exceed AKERR_MAX_ERR_VALUE, so akerr_name_for_status cannot store or return + * their names (see tests/MUTATION.md). + */ + +static const struct { + int code; + const char *name; +} expected[] = { + { AKERR_NULLPOINTER, "Null Pointer Error" }, + { AKERR_OUTOFBOUNDS, "Out Of Bounds Error" }, + { AKERR_API, "API Error" }, + { AKERR_ATTRIBUTE, "Attribute Error" }, + { AKERR_TYPE, "Type Error" }, + { AKERR_KEY, "Key Error" }, + { AKERR_INDEX, "Index Error" }, + { AKERR_FORMAT, "Format Error" }, + { AKERR_IO, "Input Output Error" }, + { AKERR_VALUE, "Value Error" }, + { AKERR_RELATIONSHIP, "Relationship Error" }, + { AKERR_CIRCULAR_REFERENCE, "Circular Reference Error" }, +}; + +int main(void) +{ + akerr_init(); + + for ( unsigned i = 0; i < sizeof(expected) / sizeof(expected[0]); i++ ) { + char *nm = akerr_name_for_status(expected[i].code, NULL); + AKERR_CHECK(nm != NULL); + AKERR_CHECK(strcmp(nm, expected[i].name) == 0); + } + + fprintf(stderr, "err_error_names ok\n"); + return 0; +} diff --git a/tests/err_pool_exhaust.c b/tests/err_pool_exhaust.c new file mode 100644 index 0000000..28ef594 --- /dev/null +++ b/tests/err_pool_exhaust.c @@ -0,0 +1,39 @@ +#include "akerror.h" +#include "err_capture.h" + +/* + * The error pool is a fixed array of AKERR_MAX_ARRAY_ERROR slots. When every + * slot is checked out, akerr_next_error() must return NULL rather than run off + * the end of the array; and it must always hand back the lowest free slot. + * Mutation testing showed both the terminating "return NULL" and the scan + * bounds could be broken without any test noticing. + */ + +int main(void) +{ + akerr_init(); + + akerr_ErrorContext *slots[AKERR_MAX_ARRAY_ERROR]; + + /* Check out every slot. */ + for ( int i = 0; i < AKERR_MAX_ARRAY_ERROR; i++ ) { + slots[i] = akerr_next_error(); + AKERR_CHECK(slots[i] != NULL); + slots[i]->refcount = 1; + } + + /* Pool is fully exhausted: the next request must fail cleanly. */ + AKERR_CHECK(akerr_next_error() == NULL); + + /* Free exactly the first slot; the scan must find and return it. */ + slots[0]->refcount = 0; + AKERR_CHECK(akerr_next_error() == slots[0]); + + /* Tidy up. */ + for ( int i = 0; i < AKERR_MAX_ARRAY_ERROR; i++ ) { + slots[i]->refcount = 0; + } + + fprintf(stderr, "err_pool_exhaust ok\n"); + return 0; +} diff --git a/tests/err_release_clears.c b/tests/err_release_clears.c new file mode 100644 index 0000000..6134da8 --- /dev/null +++ b/tests/err_release_clears.c @@ -0,0 +1,44 @@ +#include "akerror.h" +#include "err_capture.h" +#include + +/* + * Releasing an error context back to the pool must wipe it, so the next caller + * that checks it out never sees stale status/message/stacktrace from a previous + * error. Mutation testing showed the clearing memset in akerr_release_error + * could be deleted without any test noticing. + */ + +akerr_ErrorContext *boom(void) +{ + PREPARE_ERROR(e); + FAIL_RETURN(e, AKERR_VALUE, "stale dirty message that must not survive"); +} + +int main(void) +{ + akerr_capture_install(); + akerr_init(); + + /* Raise and fully handle an error; FINISH_NORETURN releases it to the pool. */ + PREPARE_ERROR(e); + ATTEMPT { + CATCH(e, boom()); + } CLEANUP { + } PROCESS(e) { + } HANDLE(e, AKERR_VALUE) { + } FINISH_NORETURN(e); + + AKERR_CHECK(e == NULL); + + /* The next context handed out is the slot we just released: it must be clean. */ + akerr_ErrorContext *slot = akerr_next_error(); + AKERR_CHECK(slot != NULL); + AKERR_CHECK(slot->status == 0); + AKERR_CHECK(slot->message[0] == '\0'); + AKERR_CHECK(slot->stacktracebuf[0] == '\0'); + AKERR_CHECK(strstr(slot->message, "stale dirty message") == NULL); + + fprintf(stderr, "err_release_clears ok\n"); + return 0; +}