Add mutation testing to validate the test suite
Introduce a self-contained mutation testing harness that verifies the unit tests actually catch bugs: it makes small deliberate breakages to the library (flip comparisons, delete statements, swap true/false, etc.), rebuilds, and runs the whole CTest suite against each mutant. Tests that still pass reveal a gap; tests that fail "kill" the mutant. - scripts/mutation_test.py: the engine (stdlib only, no LLVM/clang deps). Operators ROR/LCR/BCR/AOR/ICR/SDL over src/error.c and the macro header. Mutates a scratch copy, never the working tree. Supports --target, --list, --max-mutants sampling, --threshold gating, --timeout. - CMakeLists.txt: 'mutation' custom target (cmake --build build --target mutation). - .gitea/workflows/ci.yaml: gated mutation job on src/error.c (threshold 65%). - tests/MUTATION.md: how to run, interpret survivors, and known equivalents. Close the real gaps the harness found in src/error.c (score 53% -> 71%): - err_error_names: the AKERR_* codes have their names registered by akerr_init - err_release_clears: releasing a context wipes it before reuse - err_pool_exhaust: akerr_next_error returns NULL when the pool is full and always hands back the lowest free slot Also surfaced (documented, not fixed): AKERR_MAX_ERR_VALUE (+15) is below AKERR_NOT_IMPLEMENTED (+16) and AKERR_BADEXC (+17), so those codes can never have a name registered. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -21,3 +21,22 @@ jobs:
|
|||||||
cmake --install build
|
cmake --install build
|
||||||
cmake --build build --target test
|
cmake --build build --target test
|
||||||
- run: echo "🍏 This job's status is ${{ job.status }}."
|
- run: echo "🍏 This job's status is ${{ job.status }}."
|
||||||
|
|
||||||
|
mutation_test:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- name: Check out repository code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
- name: dependencies
|
||||||
|
run: |
|
||||||
|
sudo apt-get update -y
|
||||||
|
sudo apt-get install -y cmake gcc moreutils python3
|
||||||
|
# Verify the tests actually catch bugs: break the library many ways and
|
||||||
|
# confirm the suite fails. Gated on src/error.c (fast, deterministic).
|
||||||
|
# The threshold keeps headroom below the current score for equivalent
|
||||||
|
# mutants; see tests/MUTATION.md. Run the full default target locally for
|
||||||
|
# deeper (slower) coverage including the macro header.
|
||||||
|
- name: mutation testing
|
||||||
|
run: |
|
||||||
|
python3 scripts/mutation_test.py --target src/error.c --threshold 65
|
||||||
|
- run: echo "🍏 This job's status is ${{ job.status }}."
|
||||||
|
|||||||
@@ -63,6 +63,9 @@ set(AKERR_TESTS
|
|||||||
err_errno
|
err_errno
|
||||||
err_break_variants
|
err_break_variants
|
||||||
err_custom_handler
|
err_custom_handler
|
||||||
|
err_error_names
|
||||||
|
err_release_clears
|
||||||
|
err_pool_exhaust
|
||||||
)
|
)
|
||||||
|
|
||||||
set(AKERR_WILL_FAIL_TESTS
|
set(AKERR_WILL_FAIL_TESTS
|
||||||
@@ -82,6 +85,22 @@ set_tests_properties(
|
|||||||
PROPERTIES WILL_FAIL TRUE
|
PROPERTIES WILL_FAIL TRUE
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Mutation testing: break the library in small ways and confirm the test suite
|
||||||
|
# notices. This is a meta-check on the tests themselves, so it is a manual
|
||||||
|
# target (it rebuilds and re-runs the whole suite many times), not a CTest test.
|
||||||
|
# cmake --build build --target mutation
|
||||||
|
find_package(Python3 COMPONENTS Interpreter)
|
||||||
|
if(Python3_FOUND)
|
||||||
|
add_custom_target(mutation
|
||||||
|
COMMAND ${Python3_EXECUTABLE}
|
||||||
|
${CMAKE_CURRENT_SOURCE_DIR}/scripts/mutation_test.py
|
||||||
|
--source-root ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
||||||
|
USES_TERMINAL
|
||||||
|
COMMENT "Running mutation tests (breaks the library, expects tests to fail)"
|
||||||
|
)
|
||||||
|
endif()
|
||||||
|
|
||||||
set(main_lib_dest "lib/my_library-${MY_LIBRARY_VERSION}")
|
set(main_lib_dest "lib/my_library-${MY_LIBRARY_VERSION}")
|
||||||
install(TARGETS akerror
|
install(TARGETS akerror
|
||||||
EXPORT akerrorTargets
|
EXPORT akerrorTargets
|
||||||
|
|||||||
385
scripts/mutation_test.py
Executable file
385
scripts/mutation_test.py
Executable file
@@ -0,0 +1,385 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Mutation testing harness for libakerror.
|
||||||
|
|
||||||
|
Mutation testing measures how good the test suite is at catching bugs. It works
|
||||||
|
by making many small, deliberate breakages ("mutants") to the library source --
|
||||||
|
flipping a comparison, deleting a statement, swapping true/false -- and then
|
||||||
|
running the whole CTest suite against each one. If the tests fail, the mutant is
|
||||||
|
"killed" (good: the tests noticed the bug). If the tests still pass, the mutant
|
||||||
|
"survived" (bad: a real bug of that shape would slip through unnoticed).
|
||||||
|
|
||||||
|
The mutation score is killed / (killed + survived). Surviving mutants are printed
|
||||||
|
with file:line and the exact change so they can be turned into new test cases.
|
||||||
|
|
||||||
|
This harness has no third-party dependencies (Python stdlib + the project's
|
||||||
|
normal cmake/ctest toolchain). It never mutates the real working tree: it copies
|
||||||
|
the repo to a scratch directory and mutates there.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
scripts/mutation_test.py [options]
|
||||||
|
|
||||||
|
--source-root DIR repo root to copy (default: parent of this script's dir)
|
||||||
|
--target FILE source file to mutate, relative to root; repeatable.
|
||||||
|
Default: src/error.c and include/akerror.tmpl.h
|
||||||
|
--work DIR scratch dir for the mutated copy (default: a temp dir)
|
||||||
|
--timeout SECONDS per-suite ctest timeout (default: 120)
|
||||||
|
--threshold PCT exit non-zero if mutation score < PCT (default: 0 = off)
|
||||||
|
--list only list the mutants that would be run, then exit
|
||||||
|
--keep keep the scratch working copy on exit (for debugging)
|
||||||
|
-j N (reserved) currently runs sequentially
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import sys
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Mutation operators
|
||||||
|
#
|
||||||
|
# Each operator yields zero or more (start, end, replacement) edits for a single
|
||||||
|
# line of source. The driver applies exactly one edit per mutant so every mutant
|
||||||
|
# differs from the original by one localized change.
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
# Relational operator replacement: map each operator to the alternatives that
|
||||||
|
# meaningfully change behaviour (not merely the strict negation).
|
||||||
|
_REL = {
|
||||||
|
"==": ["!="],
|
||||||
|
"!=": ["=="],
|
||||||
|
"<=": ["<", "=="],
|
||||||
|
">=": [">", "=="],
|
||||||
|
"<": ["<=", ">"],
|
||||||
|
">": [">=", "<"],
|
||||||
|
}
|
||||||
|
# Match a relational operator that is NOT part of ->, <<, >>, =>, <=, >=, ==, !=
|
||||||
|
# unless we intend it. We tokenize the two-char operators first, then single.
|
||||||
|
_REL_TWO = re.compile(r"(==|!=|<=|>=)")
|
||||||
|
_REL_ONE = re.compile(r"(?<![-<>=!+])([<>])(?![=<>])")
|
||||||
|
|
||||||
|
_LOGICAL = {"&&": "||", "||": "&&"}
|
||||||
|
_LOG_RE = re.compile(r"(&&|\|\|)")
|
||||||
|
|
||||||
|
_BOOL = {"true": "false", "false": "true"}
|
||||||
|
_BOOL_RE = re.compile(r"\b(true|false)\b")
|
||||||
|
|
||||||
|
# Arithmetic / compound-assignment on whitespace-delimited operands only, to
|
||||||
|
# avoid touching ++, --, ->, unary signs, or pointer/format punctuation.
|
||||||
|
_ARITH_RE = re.compile(r"(?<=\s)([+\-])(?=\s)")
|
||||||
|
_ARITH = {"+": "-", "-": "+"}
|
||||||
|
_COMPOUND_RE = re.compile(r"(\+=|-=)")
|
||||||
|
_COMPOUND = {"+=": "-=", "-=": "+="}
|
||||||
|
|
||||||
|
# Integer literal replacement: 0 <-> 1 (word-bounded, not inside identifiers or
|
||||||
|
# larger numbers, not a float).
|
||||||
|
_INT_RE = re.compile(r"(?<![\w.])([01])(?![\w.])")
|
||||||
|
_INT = {"0": "1", "1": "0"}
|
||||||
|
|
||||||
|
|
||||||
|
def _op_edits(line):
|
||||||
|
"""Yield (tag, start, end, replacement) for every candidate mutation."""
|
||||||
|
# Relational (two-char first so we don't split them with the one-char pass)
|
||||||
|
for m in _REL_TWO.finditer(line):
|
||||||
|
for alt in _REL[m.group(1)]:
|
||||||
|
yield ("ROR", m.start(1), m.end(1), alt)
|
||||||
|
for m in _REL_ONE.finditer(line):
|
||||||
|
for alt in _REL[m.group(1)]:
|
||||||
|
yield ("ROR", m.start(1), m.end(1), alt)
|
||||||
|
for m in _LOG_RE.finditer(line):
|
||||||
|
yield ("LCR", m.start(1), m.end(1), _LOGICAL[m.group(1)])
|
||||||
|
for m in _BOOL_RE.finditer(line):
|
||||||
|
yield ("BCR", m.start(1), m.end(1), _BOOL[m.group(1)])
|
||||||
|
for m in _COMPOUND_RE.finditer(line):
|
||||||
|
yield ("AOR", m.start(1), m.end(1), _COMPOUND[m.group(1)])
|
||||||
|
for m in _ARITH_RE.finditer(line):
|
||||||
|
yield ("AOR", m.start(1), m.end(1), _ARITH[m.group(1)])
|
||||||
|
for m in _INT_RE.finditer(line):
|
||||||
|
yield ("ICR", m.start(1), m.end(1), _INT[m.group(1)])
|
||||||
|
|
||||||
|
|
||||||
|
# Statement-deletion: neutralize a whole statement. We only delete statements
|
||||||
|
# that are safe to drop without guaranteeing a compile error, so a surviving
|
||||||
|
# deletion is a genuine test gap rather than compiler noise.
|
||||||
|
_STMT_DELETABLE = re.compile(
|
||||||
|
r"""^\s*(
|
||||||
|
break |
|
||||||
|
return\b[^;]* |
|
||||||
|
[A-Za-z_][-\w>().\[\]* ]*\s*=\s*[^;]* | # assignments
|
||||||
|
[A-Za-z_][\w]*\s*\([^;]*\) # bare function calls
|
||||||
|
)\s*;\s*(\\?)\s*$""",
|
||||||
|
re.VERBOSE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Deciding which lines are eligible to mutate
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
# Skip preprocessor control and the block of constant/error-code #defines in the
|
||||||
|
# template header: mutating buffer sizes or renumbering error codes produces
|
||||||
|
# equivalent or uninteresting mutants that swamp the signal.
|
||||||
|
_SKIP_LINE = re.compile(
|
||||||
|
r"""^\s*(
|
||||||
|
\#\s*(include|ifn?def|ifdef|if|elif|else|endif|error|pragma|undef) |
|
||||||
|
\#\s*define\s+AKERR_(MAX|LAST|NULLPOINTER|OUTOFBOUNDS|API|ATTRIBUTE|
|
||||||
|
TYPE|KEY|INDEX|FORMAT|IO|VALUE|RELATIONSHIP|EOF|CIRCULAR_REFERENCE|
|
||||||
|
ITERATOR_BREAK|NOT_IMPLEMENTED|BADEXC|NOIGNORE|USE_STDLIB)\b |
|
||||||
|
\* | // # comment bodies / line comments
|
||||||
|
)""",
|
||||||
|
re.VERBOSE,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _is_comment_or_blank(line):
|
||||||
|
s = line.strip()
|
||||||
|
return (not s) or s.startswith("//") or s.startswith("/*") or s.startswith("*")
|
||||||
|
|
||||||
|
|
||||||
|
def eligible(line):
|
||||||
|
if _is_comment_or_blank(line):
|
||||||
|
return False
|
||||||
|
if _SKIP_LINE.match(line):
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
class Mutant:
|
||||||
|
__slots__ = ("path", "lineno", "op", "before", "after", "col")
|
||||||
|
|
||||||
|
def __init__(self, path, lineno, op, before, after, col):
|
||||||
|
self.path = path
|
||||||
|
self.lineno = lineno
|
||||||
|
self.op = op
|
||||||
|
self.before = before
|
||||||
|
self.after = after
|
||||||
|
self.col = col
|
||||||
|
|
||||||
|
def describe(self):
|
||||||
|
return (f"{self.path}:{self.lineno} [{self.op}] "
|
||||||
|
f"col{self.col}: {self.before.strip()} -> {self.after.strip()}")
|
||||||
|
|
||||||
|
|
||||||
|
def generate_mutants(root, rel_target):
|
||||||
|
"""Enumerate all mutants for one target file."""
|
||||||
|
abspath = os.path.join(root, rel_target)
|
||||||
|
with open(abspath, "r") as fh:
|
||||||
|
lines = fh.readlines()
|
||||||
|
|
||||||
|
mutants = []
|
||||||
|
for i, line in enumerate(lines, start=1):
|
||||||
|
if not eligible(line):
|
||||||
|
continue
|
||||||
|
# substitution operators
|
||||||
|
seen = set()
|
||||||
|
for tag, s, e, repl in _op_edits(line):
|
||||||
|
key = (s, e, repl)
|
||||||
|
if key in seen:
|
||||||
|
continue
|
||||||
|
seen.add(key)
|
||||||
|
mutated = line[:s] + repl + line[e:]
|
||||||
|
if mutated == line:
|
||||||
|
continue
|
||||||
|
mutants.append(Mutant(rel_target, i, tag, line, mutated, s))
|
||||||
|
# statement deletion
|
||||||
|
m = _STMT_DELETABLE.match(line)
|
||||||
|
if m:
|
||||||
|
indent = line[: len(line) - len(line.lstrip())]
|
||||||
|
cont = "\\" if line.rstrip().endswith("\\") else ""
|
||||||
|
deleted = f"{indent}/* mutant: deleted */ {cont}\n" if cont else f"{indent};\n"
|
||||||
|
mutants.append(Mutant(rel_target, i, "SDL", line, deleted, 0))
|
||||||
|
return mutants
|
||||||
|
|
||||||
|
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
# Build / test orchestration against a scratch copy
|
||||||
|
# --------------------------------------------------------------------------- #
|
||||||
|
|
||||||
|
class Runner:
|
||||||
|
def __init__(self, work, timeout):
|
||||||
|
self.work = work
|
||||||
|
self.build = os.path.join(work, "build")
|
||||||
|
self.timeout = timeout
|
||||||
|
|
||||||
|
def _run(self, cmd, timeout=None):
|
||||||
|
return subprocess.run(
|
||||||
|
cmd, cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||||
|
timeout=timeout,
|
||||||
|
)
|
||||||
|
|
||||||
|
def configure(self):
|
||||||
|
r = self._run(["cmake", "-S", ".", "-B", "build"], timeout=self.timeout)
|
||||||
|
return r.returncode == 0, r.stdout
|
||||||
|
|
||||||
|
def build_and_test(self):
|
||||||
|
"""Return ('killed-compile' | 'killed-test' | 'killed-timeout' | 'survived')."""
|
||||||
|
try:
|
||||||
|
b = self._run(["cmake", "--build", "build"], timeout=self.timeout)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return "killed-timeout"
|
||||||
|
if b.returncode != 0:
|
||||||
|
return "killed-compile"
|
||||||
|
try:
|
||||||
|
t = subprocess.run(
|
||||||
|
["ctest", "--test-dir", "build", "--output-on-failure",
|
||||||
|
"--stop-on-failure"],
|
||||||
|
cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
|
||||||
|
timeout=self.timeout,
|
||||||
|
)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
return "killed-timeout"
|
||||||
|
return "survived" if t.returncode == 0 else "killed-test"
|
||||||
|
|
||||||
|
|
||||||
|
def copy_tree(src, dst):
|
||||||
|
ignore = shutil.ignore_patterns("build", ".git", "*.o", "*.so", "*~",
|
||||||
|
"#*#", "*.iso", "*.png")
|
||||||
|
shutil.copytree(src, dst, ignore=ignore, symlinks=True)
|
||||||
|
|
||||||
|
|
||||||
|
def read_lines(path):
|
||||||
|
with open(path) as fh:
|
||||||
|
return fh.readlines()
|
||||||
|
|
||||||
|
|
||||||
|
def write_lines(path, lines):
|
||||||
|
with open(path, "w") as fh:
|
||||||
|
fh.writelines(lines)
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# Line-buffer stdout so progress is visible live under CI / the cmake target.
|
||||||
|
try:
|
||||||
|
sys.stdout.reconfigure(line_buffering=True)
|
||||||
|
except (AttributeError, ValueError):
|
||||||
|
pass
|
||||||
|
here = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
default_root = os.path.dirname(here)
|
||||||
|
|
||||||
|
ap = argparse.ArgumentParser(description="Mutation testing for libakerror")
|
||||||
|
ap.add_argument("--source-root", default=default_root)
|
||||||
|
ap.add_argument("--target", action="append", default=None)
|
||||||
|
ap.add_argument("--work", default=None)
|
||||||
|
ap.add_argument("--timeout", type=int, default=120)
|
||||||
|
ap.add_argument("--threshold", type=float, default=0.0)
|
||||||
|
ap.add_argument("--max-mutants", type=int, default=0,
|
||||||
|
help="cap the run at N evenly-sampled mutants (0 = all)")
|
||||||
|
ap.add_argument("--list", action="store_true")
|
||||||
|
ap.add_argument("--keep", action="store_true")
|
||||||
|
ap.add_argument("-j", type=int, default=1)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
root = os.path.abspath(args.source_root)
|
||||||
|
targets = args.target or ["src/error.c", "include/akerror.tmpl.h"]
|
||||||
|
|
||||||
|
# Enumerate mutants from the pristine sources.
|
||||||
|
all_mutants = []
|
||||||
|
for t in targets:
|
||||||
|
all_mutants.extend(generate_mutants(root, t))
|
||||||
|
|
||||||
|
print(f"Generated {len(all_mutants)} mutants across {len(targets)} file(s):")
|
||||||
|
for t in targets:
|
||||||
|
n = sum(1 for m in all_mutants if m.path == t)
|
||||||
|
print(f" {t}: {n}")
|
||||||
|
|
||||||
|
# Optional even-strided sampling to bound run time (CI / smoke tests).
|
||||||
|
if args.max_mutants and len(all_mutants) > args.max_mutants:
|
||||||
|
step = len(all_mutants) / args.max_mutants
|
||||||
|
sampled = [all_mutants[int(i * step)] for i in range(args.max_mutants)]
|
||||||
|
print(f"Sampling {len(sampled)} of {len(all_mutants)} mutants "
|
||||||
|
f"(--max-mutants {args.max_mutants}).")
|
||||||
|
all_mutants = sampled
|
||||||
|
|
||||||
|
if args.list:
|
||||||
|
for m in all_mutants:
|
||||||
|
print(" " + m.describe())
|
||||||
|
return 0
|
||||||
|
|
||||||
|
if not all_mutants:
|
||||||
|
print("No mutants generated; nothing to do.")
|
||||||
|
return 0
|
||||||
|
|
||||||
|
# Scratch working copy.
|
||||||
|
work_parent = args.work or tempfile.mkdtemp(prefix="akerr_mut_")
|
||||||
|
work = os.path.join(work_parent, "src") if args.work else work_parent
|
||||||
|
if os.path.exists(work):
|
||||||
|
shutil.rmtree(work)
|
||||||
|
print(f"\nCopying sources to scratch dir: {work}")
|
||||||
|
copy_tree(root, work)
|
||||||
|
|
||||||
|
runner = Runner(work, args.timeout)
|
||||||
|
|
||||||
|
print("Configuring baseline ...")
|
||||||
|
ok, out = runner.configure()
|
||||||
|
if not ok:
|
||||||
|
sys.stderr.write(out.decode(errors="replace"))
|
||||||
|
sys.stderr.write("\nBaseline configure FAILED; aborting.\n")
|
||||||
|
return 2
|
||||||
|
|
||||||
|
print("Verifying baseline is green (no mutation) ...")
|
||||||
|
baseline = runner.build_and_test()
|
||||||
|
if baseline != "survived":
|
||||||
|
sys.stderr.write(f"Baseline is not green ({baseline}); aborting. "
|
||||||
|
"Fix the suite before mutation testing.\n")
|
||||||
|
return 2
|
||||||
|
print("Baseline OK.\n")
|
||||||
|
|
||||||
|
# Group mutants by file so we mutate one file at a time and restore it.
|
||||||
|
killed = {"killed-compile": 0, "killed-test": 0, "killed-timeout": 0}
|
||||||
|
survivors = []
|
||||||
|
total = len(all_mutants)
|
||||||
|
|
||||||
|
# Cache pristine contents per target.
|
||||||
|
pristine = {t: read_lines(os.path.join(work, t)) for t in targets}
|
||||||
|
|
||||||
|
for idx, m in enumerate(all_mutants, start=1):
|
||||||
|
tgt_abs = os.path.join(work, m.path)
|
||||||
|
lines = list(pristine[m.path])
|
||||||
|
lines[m.lineno - 1] = m.after
|
||||||
|
write_lines(tgt_abs, lines)
|
||||||
|
try:
|
||||||
|
result = runner.build_and_test()
|
||||||
|
finally:
|
||||||
|
write_lines(tgt_abs, pristine[m.path]) # always restore
|
||||||
|
|
||||||
|
if result == "survived":
|
||||||
|
survivors.append(m)
|
||||||
|
mark = "SURVIVED"
|
||||||
|
else:
|
||||||
|
killed[result] += 1
|
||||||
|
mark = result.upper()
|
||||||
|
print(f"[{idx}/{total}] {mark:16} {m.describe()}")
|
||||||
|
|
||||||
|
total_killed = sum(killed.values())
|
||||||
|
score = 100.0 * total_killed / total if total else 100.0
|
||||||
|
|
||||||
|
print("\n" + "=" * 72)
|
||||||
|
print("MUTATION TESTING SUMMARY")
|
||||||
|
print("=" * 72)
|
||||||
|
print(f" total mutants : {total}")
|
||||||
|
print(f" killed (test) : {killed['killed-test']}")
|
||||||
|
print(f" killed (compile): {killed['killed-compile']}")
|
||||||
|
print(f" killed (timeout): {killed['killed-timeout']}")
|
||||||
|
print(f" survived : {len(survivors)}")
|
||||||
|
print(f" mutation score : {score:.1f}%")
|
||||||
|
if survivors:
|
||||||
|
print("\nSurviving mutants (test-suite gaps -- turn these into tests):")
|
||||||
|
for m in survivors:
|
||||||
|
print(" " + m.describe())
|
||||||
|
|
||||||
|
if not args.keep and not args.work:
|
||||||
|
shutil.rmtree(work_parent, ignore_errors=True)
|
||||||
|
else:
|
||||||
|
print(f"\nScratch working copy kept at: {work}")
|
||||||
|
|
||||||
|
if args.threshold > 0 and score < args.threshold:
|
||||||
|
print(f"\nFAIL: mutation score {score:.1f}% < threshold {args.threshold:.1f}%")
|
||||||
|
return 1
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
100
tests/MUTATION.md
Normal file
100
tests/MUTATION.md
Normal file
@@ -0,0 +1,100 @@
|
|||||||
|
# Mutation testing
|
||||||
|
|
||||||
|
The unit tests tell us the library works. **Mutation testing tells us the tests
|
||||||
|
work** — that they would actually fail if the library were broken.
|
||||||
|
|
||||||
|
`scripts/mutation_test.py` deliberately breaks the library in small ways
|
||||||
|
("mutants"), one at a time, and runs the whole CTest suite against each broken
|
||||||
|
copy:
|
||||||
|
|
||||||
|
* if the tests **fail**, the mutant is **killed** — good, the suite caught it;
|
||||||
|
* if the tests still **pass**, the mutant **survived** — a bug of that shape
|
||||||
|
would slip through, so it points at a missing test.
|
||||||
|
|
||||||
|
The **mutation score** is `killed / (killed + survived)`. A surviving mutant is
|
||||||
|
a to-do item: write a test that distinguishes the mutant from the original.
|
||||||
|
|
||||||
|
## Running
|
||||||
|
|
||||||
|
No third-party tools are required — just Python 3 and the normal
|
||||||
|
cmake/ctest toolchain. The harness never touches your working tree; it copies
|
||||||
|
the repo to a scratch directory and mutates the copy.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# Default: mutate src/error.c and include/akerror.tmpl.h
|
||||||
|
scripts/mutation_test.py
|
||||||
|
|
||||||
|
# Faster: just the C source
|
||||||
|
scripts/mutation_test.py --target src/error.c
|
||||||
|
|
||||||
|
# See what would run without building anything
|
||||||
|
scripts/mutation_test.py --target src/error.c --list
|
||||||
|
|
||||||
|
# Gate CI: exit non-zero if the score drops below 90%
|
||||||
|
scripts/mutation_test.py --threshold 90
|
||||||
|
```
|
||||||
|
|
||||||
|
Via CMake (configures a build first if needed):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cmake --build build --target mutation
|
||||||
|
```
|
||||||
|
|
||||||
|
Useful flags: `--timeout SECONDS` (per-suite build+test cap; a mutant that
|
||||||
|
hangs is counted as killed), `--keep` (retain the scratch copy for debugging),
|
||||||
|
`--work DIR` (use a specific scratch directory).
|
||||||
|
|
||||||
|
## Mutation operators
|
||||||
|
|
||||||
|
Each mutant changes exactly one location by one of:
|
||||||
|
|
||||||
|
| Tag | Operator | Example |
|
||||||
|
|-----|--------------------------------|----------------------------------|
|
||||||
|
| ROR | relational operator | `==` → `!=`, `<` → `<=`, `>=` → `>` |
|
||||||
|
| LCR | logical connector | `&&` → `\|\|` |
|
||||||
|
| BCR | boolean constant | `true` → `false` |
|
||||||
|
| AOR | arithmetic / compound assign | `+` → `-`, `+=` → `-=` |
|
||||||
|
| ICR | integer literal | `0` → `1`, `1` → `0` |
|
||||||
|
| SDL | statement deletion | `err->refcount += 1;` → *(removed)* |
|
||||||
|
|
||||||
|
Preprocessor control lines, comments, and the block of error-code / buffer-size
|
||||||
|
`#define`s are skipped: mutating those produces equivalent or uninteresting
|
||||||
|
mutants that only add noise.
|
||||||
|
|
||||||
|
## Interpreting survivors
|
||||||
|
|
||||||
|
Not every survivor is a test gap — some mutants are **equivalent** (they don't
|
||||||
|
change observable behaviour, e.g. resizing an internal scratch buffer). For each
|
||||||
|
survivor, decide:
|
||||||
|
|
||||||
|
1. **Real gap** → add or strengthen a test in `tests/` so the mutant is killed,
|
||||||
|
then re-run.
|
||||||
|
2. **Equivalent mutant** → no test can catch it; leave a note. If a specific
|
||||||
|
line is a persistent source of equivalents, narrow the target with
|
||||||
|
`--target` or extend the skip rules in `scripts/mutation_test.py`.
|
||||||
|
|
||||||
|
Re-run after adding tests and confirm the score went up.
|
||||||
|
|
||||||
|
## Current status
|
||||||
|
|
||||||
|
`src/error.c` scores ~71% (the CI gate is set to 65% for headroom). The
|
||||||
|
remaining survivors are dominated by:
|
||||||
|
|
||||||
|
* **Equivalent mutants** in `akerr_init`: deleting the `memset`/`NULL` setup of
|
||||||
|
file-scope statics (`AKERR_ARRAY_ERROR`, `__akerr_last_ditch`,
|
||||||
|
`__akerr_last_ignored`) changes nothing, because C already zero-initializes
|
||||||
|
objects with static storage duration. `int oldid = 0;` → `1` is likewise
|
||||||
|
dead: it is overwritten before use.
|
||||||
|
* **Default logger / handler internals** (`vfprintf`, `va_end`, the
|
||||||
|
`errctx == NULL` branch, `exit(1)`): killing these needs a subprocess-based
|
||||||
|
test that captures a child's stderr and exit code, rather than the in-process
|
||||||
|
capturing logger the other tests use.
|
||||||
|
|
||||||
|
Findings worth noting (surfaced by mutation testing, not yet fixed):
|
||||||
|
|
||||||
|
* `AKERR_MAX_ERR_VALUE` is `AKERR_LAST_ERRNO_VALUE + 15`, but `AKERR_NOT_IMPLEMENTED`
|
||||||
|
(+16) and `AKERR_BADEXC` (+17) exceed it. `akerr_name_for_status` rejects any
|
||||||
|
status `> AKERR_MAX_ERR_VALUE`, so those two codes can never store or return a
|
||||||
|
name — the `akerr_name_for_status(AKERR_BADEXC, ...)` call in `akerr_init` is
|
||||||
|
dead code (which is why deleting it survives). Bumping `AKERR_MAX_ERR_VALUE` to
|
||||||
|
`+ 17` would fix it.
|
||||||
45
tests/err_error_names.c
Normal file
45
tests/err_error_names.c
Normal file
@@ -0,0 +1,45 @@
|
|||||||
|
#include "akerror.h"
|
||||||
|
#include "err_capture.h"
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* akerr_init() registers a human-readable name for each library error code.
|
||||||
|
* Verify the names are actually installed (mutation testing showed the
|
||||||
|
* registration calls could be deleted without any test noticing).
|
||||||
|
*
|
||||||
|
* Note: AKERR_NOT_IMPLEMENTED and AKERR_BADEXC are intentionally omitted -- they
|
||||||
|
* exceed AKERR_MAX_ERR_VALUE, so akerr_name_for_status cannot store or return
|
||||||
|
* their names (see tests/MUTATION.md).
|
||||||
|
*/
|
||||||
|
|
||||||
|
static const struct {
|
||||||
|
int code;
|
||||||
|
const char *name;
|
||||||
|
} expected[] = {
|
||||||
|
{ AKERR_NULLPOINTER, "Null Pointer Error" },
|
||||||
|
{ AKERR_OUTOFBOUNDS, "Out Of Bounds Error" },
|
||||||
|
{ AKERR_API, "API Error" },
|
||||||
|
{ AKERR_ATTRIBUTE, "Attribute Error" },
|
||||||
|
{ AKERR_TYPE, "Type Error" },
|
||||||
|
{ AKERR_KEY, "Key Error" },
|
||||||
|
{ AKERR_INDEX, "Index Error" },
|
||||||
|
{ AKERR_FORMAT, "Format Error" },
|
||||||
|
{ AKERR_IO, "Input Output Error" },
|
||||||
|
{ AKERR_VALUE, "Value Error" },
|
||||||
|
{ AKERR_RELATIONSHIP, "Relationship Error" },
|
||||||
|
{ AKERR_CIRCULAR_REFERENCE, "Circular Reference Error" },
|
||||||
|
};
|
||||||
|
|
||||||
|
int main(void)
|
||||||
|
{
|
||||||
|
akerr_init();
|
||||||
|
|
||||||
|
for ( unsigned i = 0; i < sizeof(expected) / sizeof(expected[0]); i++ ) {
|
||||||
|
char *nm = akerr_name_for_status(expected[i].code, NULL);
|
||||||
|
AKERR_CHECK(nm != NULL);
|
||||||
|
AKERR_CHECK(strcmp(nm, expected[i].name) == 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "err_error_names ok\n");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
39
tests/err_pool_exhaust.c
Normal file
39
tests/err_pool_exhaust.c
Normal file
@@ -0,0 +1,39 @@
|
|||||||
|
#include "akerror.h"
|
||||||
|
#include "err_capture.h"
|
||||||
|
|
||||||
|
/*
|
||||||
|
* The error pool is a fixed array of AKERR_MAX_ARRAY_ERROR slots. When every
|
||||||
|
* slot is checked out, akerr_next_error() must return NULL rather than run off
|
||||||
|
* the end of the array; and it must always hand back the lowest free slot.
|
||||||
|
* Mutation testing showed both the terminating "return NULL" and the scan
|
||||||
|
* bounds could be broken without any test noticing.
|
||||||
|
*/
|
||||||
|
|
||||||
|
int main(void)
|
||||||
|
{
|
||||||
|
akerr_init();
|
||||||
|
|
||||||
|
akerr_ErrorContext *slots[AKERR_MAX_ARRAY_ERROR];
|
||||||
|
|
||||||
|
/* Check out every slot. */
|
||||||
|
for ( int i = 0; i < AKERR_MAX_ARRAY_ERROR; i++ ) {
|
||||||
|
slots[i] = akerr_next_error();
|
||||||
|
AKERR_CHECK(slots[i] != NULL);
|
||||||
|
slots[i]->refcount = 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* Pool is fully exhausted: the next request must fail cleanly. */
|
||||||
|
AKERR_CHECK(akerr_next_error() == NULL);
|
||||||
|
|
||||||
|
/* Free exactly the first slot; the scan must find and return it. */
|
||||||
|
slots[0]->refcount = 0;
|
||||||
|
AKERR_CHECK(akerr_next_error() == slots[0]);
|
||||||
|
|
||||||
|
/* Tidy up. */
|
||||||
|
for ( int i = 0; i < AKERR_MAX_ARRAY_ERROR; i++ ) {
|
||||||
|
slots[i]->refcount = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
fprintf(stderr, "err_pool_exhaust ok\n");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
44
tests/err_release_clears.c
Normal file
44
tests/err_release_clears.c
Normal file
@@ -0,0 +1,44 @@
|
|||||||
|
#include "akerror.h"
|
||||||
|
#include "err_capture.h"
|
||||||
|
#include <string.h>
|
||||||
|
|
||||||
|
/*
|
||||||
|
* Releasing an error context back to the pool must wipe it, so the next caller
|
||||||
|
* that checks it out never sees stale status/message/stacktrace from a previous
|
||||||
|
* error. Mutation testing showed the clearing memset in akerr_release_error
|
||||||
|
* could be deleted without any test noticing.
|
||||||
|
*/
|
||||||
|
|
||||||
|
akerr_ErrorContext *boom(void)
|
||||||
|
{
|
||||||
|
PREPARE_ERROR(e);
|
||||||
|
FAIL_RETURN(e, AKERR_VALUE, "stale dirty message that must not survive");
|
||||||
|
}
|
||||||
|
|
||||||
|
int main(void)
|
||||||
|
{
|
||||||
|
akerr_capture_install();
|
||||||
|
akerr_init();
|
||||||
|
|
||||||
|
/* Raise and fully handle an error; FINISH_NORETURN releases it to the pool. */
|
||||||
|
PREPARE_ERROR(e);
|
||||||
|
ATTEMPT {
|
||||||
|
CATCH(e, boom());
|
||||||
|
} CLEANUP {
|
||||||
|
} PROCESS(e) {
|
||||||
|
} HANDLE(e, AKERR_VALUE) {
|
||||||
|
} FINISH_NORETURN(e);
|
||||||
|
|
||||||
|
AKERR_CHECK(e == NULL);
|
||||||
|
|
||||||
|
/* The next context handed out is the slot we just released: it must be clean. */
|
||||||
|
akerr_ErrorContext *slot = akerr_next_error();
|
||||||
|
AKERR_CHECK(slot != NULL);
|
||||||
|
AKERR_CHECK(slot->status == 0);
|
||||||
|
AKERR_CHECK(slot->message[0] == '\0');
|
||||||
|
AKERR_CHECK(slot->stacktracebuf[0] == '\0');
|
||||||
|
AKERR_CHECK(strstr(slot->message, "stale dirty message") == NULL);
|
||||||
|
|
||||||
|
fprintf(stderr, "err_release_clears ok\n");
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user