From dc1bd0a7983dfaeb03affed519ed4448b3af8172 Mon Sep 17 00:00:00 2001 From: Andrew Kesterson Date: Wed, 29 Jul 2026 18:42:09 -0400 Subject: [PATCH] Add mutation testing harness Port the scratch-copy mutation runner used by libakerror and libakstdlib, covering libakgl source files with sampling, thresholds, JUnit reports, and test timeouts. Add a CMake target and documentation, preserve the intentionally failing character test exclusion, and migrate the sprite test to the renderer backend so mutation baselines are green. Co-authored-by: Codex (GPT-5) --- AGENTS.md | 2 + CMakeLists.txt | 22 +- README.md | 13 ++ scripts/mutation_test.py | 466 +++++++++++++++++++++++++++++++++++++++ tests/sprite.c | 12 +- 5 files changed, 509 insertions(+), 6 deletions(-) create mode 100755 scripts/mutation_test.py diff --git a/AGENTS.md b/AGENTS.md index 0454fa9..91c8462 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -26,6 +26,8 @@ ctest --test-dir build --output-on-failure Run one test while iterating, for example `ctest --test-dir build -R sprite --output-on-failure`. The `rebuild.sh` script also installs into a developer-specific `/home/andrew/local` prefix and removes existing outputs; prefer the portable commands above unless that exact workflow is intended. +Run mutation testing with `cmake --build build --target mutation`. For a quick smoke run, use `scripts/mutation_test.py --target src/tilemap.c --max-mutants 10`; the harness mutates only a scratch copy and excludes the intentionally failing character test. + ## Coding Style & Naming Conventions Follow the surrounding C style: braces on the next line for function bodies, spaces inside control-flow parentheses, and short, focused functions. Preserve the indentation of the file being edited; older files contain both spaces and tabs. Public symbols use the `akgl_` prefix, public types use `akgl_TypeName`, and constants/macros use `AKGL_UPPER_SNAKE_CASE`. Name matching header/source pairs by feature, such as `include/akgl/sprite.h` and `src/sprite.c`. No repository-wide formatter or linter is configured, so avoid unrelated formatting churn. diff --git a/CMakeLists.txt b/CMakeLists.txt index 1de6d1b..f86c2ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -132,7 +132,7 @@ add_test(NAME semver_unit COMMAND test_semver_unit) set_tests_properties( actor bitmasks character registry sprite staticstring tilemap util semver_unit - PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests" + PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests" TIMEOUT 30 ) # Specify include directories for the library's headers (if applicable) @@ -163,6 +163,26 @@ target_link_libraries(test_util PRIVATE akstdlib::akstdlib akerror::akerror akgl target_link_libraries(charviewer PRIVATE akstdlib::akstdlib akerror::akerror akgl SDL3::SDL3 SDL3_ttf::SDL3_ttf SDL3_image::SDL3_image SDL3_mixer::SDL3_mixer jansson::jansson -lm) +# Mutation testing copies the repository to scratch space, applies one small +# source change at a time, and verifies that the passing tests detect it. The +# intentionally failing character test is excluded by the harness. +find_package(Python3 COMPONENTS Interpreter) +if(Python3_FOUND) + if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(AKGL_MUTATION_TARGET mutation) + else() + set(AKGL_MUTATION_TARGET akgl_mutation) + endif() + add_custom_target(${AKGL_MUTATION_TARGET} + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/mutation_test.py + --source-root ${CMAKE_CURRENT_SOURCE_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + USES_TERMINAL + COMMENT "Running mutation tests (breaks a scratch copy, expects tests to fail)" + ) +endif() + set(main_lib_dest "lib/akgl-${MY_LIBRARY_VERSION}") install(FILES ${CMAKE_CURRENT_BINARY_DIR}/akgl.pc DESTINATION "lib/pkgconfig/") install(TARGETS akgl DESTINATION "lib/") diff --git a/README.md b/README.md index 5183c50..ed529de 100644 --- a/README.md +++ b/README.md @@ -269,3 +269,16 @@ PASS(e, aksl_atoi(width->data, &screenwidth)); PASS(e, akgl_heap_release_string(width)); ``` + +## Mutation testing + +The mutation harness makes one deliberate source-code change at a time in a scratch copy, then rebuilds and runs the passing CTest suite to measure whether tests detect the change. The known-failing `character` test is excluded by default. + +```sh +cmake --build build --target mutation +scripts/mutation_test.py --target src/tilemap.c --list +scripts/mutation_test.py --target src/tilemap.c --max-mutants 10 +scripts/mutation_test.py --threshold 40 --junit mutation-junit.xml +``` + +The default run covers all libakgl-owned files under `src/`. Use repeated `--target` options to narrow the scope. A surviving mutant identifies behavior that the current tests do not verify; the script prints its file, line, operator, and exact edit. The real working tree is never mutated. diff --git a/scripts/mutation_test.py b/scripts/mutation_test.py new file mode 100755 index 0000000..ad82b38 --- /dev/null +++ b/scripts/mutation_test.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 +""" +Mutation testing harness for libakgl. + +Mutation testing measures how good the test suite is at catching bugs. It works +by making many small, deliberate breakages ("mutants") to the library source -- +flipping a comparison, deleting a statement, swapping true/false -- and then +running the whole CTest suite against each one. If the tests fail, the mutant is +"killed" (good: the tests noticed the bug). If the tests still pass, the mutant +"survived" (bad: a real bug of that shape would slip through unnoticed). + +The mutation score is killed / (killed + survived). Surviving mutants are printed +with file:line and the exact change so they can be turned into new test cases. + +This harness has no third-party dependencies (Python stdlib + the project's +normal cmake/ctest toolchain). It never mutates the real working tree: it copies +the repo to a scratch directory and mutates there. + +Usage: + scripts/mutation_test.py [options] + + --source-root DIR repo root to copy (default: parent of this script's dir) + --target FILE source file to mutate, relative to root; repeatable. + Default: all libakgl-owned C files under src/ + --work DIR scratch dir for the mutated copy (default: a temp dir) + --timeout SECONDS per-suite ctest timeout (default: 120) + --exclude-test REGEX CTest regex to exclude (default: ^character$) + --threshold PCT exit non-zero if mutation score < PCT (default: 0 = off) + --list only list the mutants that would be run, then exit + --keep keep the scratch working copy on exit (for debugging) + -j N (reserved) currently runs sequentially +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tempfile + +# --------------------------------------------------------------------------- # +# Mutation operators +# +# Each operator yields zero or more (start, end, replacement) edits for a single +# line of source. The driver applies exactly one edit per mutant so every mutant +# differs from the original by one localized change. +# --------------------------------------------------------------------------- # + +# Relational operator replacement: map each operator to the alternatives that +# meaningfully change behaviour (not merely the strict negation). +_REL = { + "==": ["!="], + "!=": ["=="], + "<=": ["<", "=="], + ">=": [">", "=="], + "<": ["<=", ">"], + ">": [">=", "<"], +} +# Match a relational operator that is NOT part of ->, <<, >>, =>, <=, >=, ==, != +# unless we intend it. We tokenize the two-char operators first, then single. +_REL_TWO = re.compile(r"(==|!=|<=|>=)") +_REL_ONE = re.compile(r"(?=!+])([<>])(?![=<>])") + +_LOGICAL = {"&&": "||", "||": "&&"} +_LOG_RE = re.compile(r"(&&|\|\|)") + +_BOOL = {"true": "false", "false": "true"} +_BOOL_RE = re.compile(r"\b(true|false)\b") + +# Arithmetic / compound-assignment on whitespace-delimited operands only, to +# avoid touching ++, --, ->, unary signs, or pointer/format punctuation. +_ARITH_RE = re.compile(r"(?<=\s)([+\-])(?=\s)") +_ARITH = {"+": "-", "-": "+"} +_COMPOUND_RE = re.compile(r"(\+=|-=)") +_COMPOUND = {"+=": "-=", "-=": "+="} + +# Integer literal replacement: 0 <-> 1 (word-bounded, not inside identifiers or +# larger numbers, not a float). +_INT_RE = re.compile(r"(?().\[\]* ]*\s*=\s*[^;]* | # assignments + [A-Za-z_][\w]*\s*\([^;]*\) # bare function calls + )\s*;\s*(\\?)\s*$""", + re.VERBOSE, +) + + +# --------------------------------------------------------------------------- # +# Deciding which lines are eligible to mutate +# --------------------------------------------------------------------------- # + +# Skip preprocessor control and the block of constant/error-code #defines in the +# template header: mutating buffer sizes or renumbering error codes produces +# equivalent or uninteresting mutants that swamp the signal. +_SKIP_LINE = re.compile( + r"""^\s*( + \#\s*(include|ifn?def|ifdef|if|elif|else|endif|error|pragma|undef) | + \#\s*define\s+AKERR_(MAX|LAST|NULLPOINTER|OUTOFBOUNDS|API|ATTRIBUTE| + TYPE|KEY|INDEX|FORMAT|IO|VALUE|RELATIONSHIP|EOF|CIRCULAR_REFERENCE| + ITERATOR_BREAK|NOT_IMPLEMENTED|BADEXC|NOIGNORE|USE_STDLIB)\b | + \* | // # comment bodies / line comments + )""", + re.VERBOSE, +) + + +def _is_comment_or_blank(line): + s = line.strip() + return (not s) or s.startswith("//") or s.startswith("/*") or s.startswith("*") + + +def eligible(line): + if _is_comment_or_blank(line): + return False + if _SKIP_LINE.match(line): + return False + return True + + +class Mutant: + __slots__ = ("path", "lineno", "op", "before", "after", "col") + + def __init__(self, path, lineno, op, before, after, col): + self.path = path + self.lineno = lineno + self.op = op + self.before = before + self.after = after + self.col = col + + def describe(self): + return (f"{self.path}:{self.lineno} [{self.op}] " + f"col{self.col}: {self.before.strip()} -> {self.after.strip()}") + + +def generate_mutants(root, rel_target): + """Enumerate all mutants for one target file.""" + abspath = os.path.join(root, rel_target) + with open(abspath, "r") as fh: + lines = fh.readlines() + + mutants = [] + for i, line in enumerate(lines, start=1): + if not eligible(line): + continue + # substitution operators + seen = set() + for tag, s, e, repl in _op_edits(line): + key = (s, e, repl) + if key in seen: + continue + seen.add(key) + mutated = line[:s] + repl + line[e:] + if mutated == line: + continue + mutants.append(Mutant(rel_target, i, tag, line, mutated, s)) + # statement deletion + m = _STMT_DELETABLE.match(line) + if m: + indent = line[: len(line) - len(line.lstrip())] + cont = "\\" if line.rstrip().endswith("\\") else "" + deleted = f"{indent}/* mutant: deleted */ {cont}\n" if cont else f"{indent};\n" + mutants.append(Mutant(rel_target, i, "SDL", line, deleted, 0)) + return mutants + + +# --------------------------------------------------------------------------- # +# Build / test orchestration against a scratch copy +# --------------------------------------------------------------------------- # + +class Runner: + def __init__(self, work, timeout, exclude_test): + self.work = work + self.build = os.path.join(work, "build") + self.timeout = timeout + self.exclude_test = exclude_test + self.env = os.environ.copy() + library_dirs = [ + self.build, + os.path.join(self.build, "deps", "SDL"), + os.path.join(self.build, "deps", "SDL_image"), + os.path.join(self.build, "deps", "SDL_mixer"), + os.path.join(self.build, "deps", "SDL_ttf"), + ] + current_library_path = self.env.get("LD_LIBRARY_PATH", "") + if current_library_path: + library_dirs.append(current_library_path) + self.env["LD_LIBRARY_PATH"] = os.pathsep.join(library_dirs) + + def _run(self, cmd, timeout=None): + return subprocess.run( + cmd, cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=timeout, env=self.env, + ) + + def configure(self): + r = self._run(["cmake", "-S", ".", "-B", "build"], timeout=self.timeout) + return r.returncode == 0, r.stdout + + def ctest_command(self): + command = ["ctest", "--test-dir", "build", "--output-on-failure", + "--stop-on-failure"] + if self.exclude_test: + command.extend(["-E", self.exclude_test]) + return command + + def build_and_test(self): + """Return ('killed-compile' | 'killed-test' | 'killed-timeout' | 'survived').""" + try: + b = self._run(["cmake", "--build", "build"], timeout=self.timeout) + except subprocess.TimeoutExpired: + return "killed-timeout" + if b.returncode != 0: + return "killed-compile" + try: + t = subprocess.run( + self.ctest_command(), + env=self.env, + cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=self.timeout, + ) + except subprocess.TimeoutExpired: + return "killed-timeout" + return "survived" if t.returncode == 0 else "killed-test" + + +def _xml_escape(s): + return (s.replace("&", "&").replace("<", "<").replace(">", ">") + .replace('"', """)) + + +def write_junit(path, records, targets): + """Write a JUnit XML report. One per mutant; a surviving mutant + is a (test-suite gap), a killed mutant is a passing case.""" + by_file = {t: [] for t in targets} + for m, result in records: + by_file.setdefault(m.path, []).append((m, result)) + + total = len(records) + total_fail = sum(1 for _, r in records if r == "survived") + out = ['', + f''] + for f, items in by_file.items(): + if not items: + continue + fails = sum(1 for _, r in items if r == "survived") + out.append(f' ') + for m, result in items: + name = _xml_escape(m.describe()) + cls = "mutation." + _xml_escape(m.path) + if result == "survived": + detail = _xml_escape(f"{m.before.strip()} -> {m.after.strip()}") + out.append(f' ') + out.append(f' {detail}') + out.append(' ') + else: + out.append(f' {_xml_escape(result)}' + '') + out.append(' ') + out.append('') + with open(path, "w") as fh: + fh.write("\n".join(out) + "\n") + + +def copy_tree(src, dst): + ignore = shutil.ignore_patterns("build", ".git", "*.o", "*.so", "*~", + "#*#", "*.iso") + shutil.copytree(src, dst, ignore=ignore, symlinks=True) + + +def read_lines(path): + with open(path) as fh: + return fh.readlines() + + +def write_lines(path, lines): + with open(path, "w") as fh: + fh.writelines(lines) + + +def main(): + # Line-buffer stdout so progress is visible live under CI / the cmake target. + try: + sys.stdout.reconfigure(line_buffering=True) + except (AttributeError, ValueError): + pass + here = os.path.dirname(os.path.abspath(__file__)) + default_root = os.path.dirname(here) + + ap = argparse.ArgumentParser(description="Mutation testing for libakgl") + ap.add_argument("--source-root", default=default_root) + ap.add_argument("--target", action="append", default=None) + ap.add_argument("--work", default=None) + ap.add_argument("--timeout", type=int, default=120) + ap.add_argument("--exclude-test", default="^character$", + help="CTest regex to exclude (default: ^character$)") + ap.add_argument("--threshold", type=float, default=0.0) + ap.add_argument("--junit", default=None, + help="write a JUnit XML report to this path") + ap.add_argument("--max-mutants", type=int, default=0, + help="cap the run at N evenly-sampled mutants (0 = all)") + ap.add_argument("--list", action="store_true") + ap.add_argument("--keep", action="store_true") + ap.add_argument("-j", type=int, default=1) + args = ap.parse_args() + + root = os.path.abspath(args.source_root) + targets = args.target or [ + "src/actor.c", "src/actor_state_string_names.c", "src/assets.c", + "src/character.c", "src/controller.c", "src/draw.c", + "src/game.c", "src/heap.c", "src/json_helpers.c", + "src/physics.c", "src/registry.c", "src/renderer.c", + "src/sprite.c", "src/staticstring.c", "src/text.c", + "src/tilemap.c", "src/util.c", + ] + + # Enumerate mutants from the pristine sources. + all_mutants = [] + for t in targets: + all_mutants.extend(generate_mutants(root, t)) + + print(f"Generated {len(all_mutants)} mutants across {len(targets)} file(s):") + for t in targets: + n = sum(1 for m in all_mutants if m.path == t) + print(f" {t}: {n}") + + # Optional even-strided sampling to bound run time (CI / smoke tests). + if args.max_mutants and len(all_mutants) > args.max_mutants: + step = len(all_mutants) / args.max_mutants + sampled = [all_mutants[int(i * step)] for i in range(args.max_mutants)] + print(f"Sampling {len(sampled)} of {len(all_mutants)} mutants " + f"(--max-mutants {args.max_mutants}).") + all_mutants = sampled + + if args.list: + for m in all_mutants: + print(" " + m.describe()) + return 0 + + if not all_mutants: + print("No mutants generated; nothing to do.") + return 0 + + # Scratch working copy. + work_parent = args.work or tempfile.mkdtemp(prefix="akgl_mut_") + work = os.path.join(work_parent, "src") if args.work else work_parent + if os.path.exists(work): + shutil.rmtree(work) + print(f"\nCopying sources to scratch dir: {work}") + copy_tree(root, work) + + runner = Runner(work, args.timeout, args.exclude_test) + + print("Configuring baseline ...") + ok, out = runner.configure() + if not ok: + sys.stderr.write(out.decode(errors="replace")) + sys.stderr.write("\nBaseline configure FAILED; aborting.\n") + return 2 + + print("Verifying baseline is green (no mutation) ...") + baseline = runner.build_and_test() + if baseline != "survived": + sys.stderr.write(f"Baseline is not green ({baseline}); aborting. " + "Fix the suite before mutation testing.\n") + return 2 + print("Baseline OK.\n") + + # Group mutants by file so we mutate one file at a time and restore it. + killed = {"killed-compile": 0, "killed-test": 0, "killed-timeout": 0} + survivors = [] + records = [] + total = len(all_mutants) + + # Cache pristine contents per target. + pristine = {t: read_lines(os.path.join(work, t)) for t in targets} + + for idx, m in enumerate(all_mutants, start=1): + tgt_abs = os.path.join(work, m.path) + lines = list(pristine[m.path]) + lines[m.lineno - 1] = m.after + write_lines(tgt_abs, lines) + try: + result = runner.build_and_test() + finally: + write_lines(tgt_abs, pristine[m.path]) # always restore + + records.append((m, result)) + if result == "survived": + survivors.append(m) + mark = "SURVIVED" + else: + killed[result] += 1 + mark = result.upper() + print(f"[{idx}/{total}] {mark:16} {m.describe()}") + + total_killed = sum(killed.values()) + score = 100.0 * total_killed / total if total else 100.0 + + print("\n" + "=" * 72) + print("MUTATION TESTING SUMMARY") + print("=" * 72) + print(f" total mutants : {total}") + print(f" killed (test) : {killed['killed-test']}") + print(f" killed (compile): {killed['killed-compile']}") + print(f" killed (timeout): {killed['killed-timeout']}") + print(f" survived : {len(survivors)}") + print(f" mutation score : {score:.1f}%") + if survivors: + print("\nSurviving mutants (test-suite gaps -- turn these into tests):") + for m in survivors: + print(" " + m.describe()) + + if args.junit: + junit_path = os.path.abspath(args.junit) + write_junit(junit_path, records, targets) + print(f"\nJUnit report written to: {junit_path}") + + if not args.keep and not args.work: + shutil.rmtree(work_parent, ignore_errors=True) + else: + print(f"\nScratch working copy kept at: {work}") + + if args.threshold > 0 and score < args.threshold: + print(f"\nFAIL: mutation score {score:.1f}% < threshold {args.threshold:.1f}%") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/sprite.c b/tests/sprite.c index a1f52b6..fcdd575 100644 --- a/tests/sprite.c +++ b/tests/sprite.c @@ -10,9 +10,9 @@ #include #include #include +#include +#include -SDL_Window *window; -SDL_Renderer *renderer; akerr_ErrorContext *test_akgl_spritesheet_initialize(void) { @@ -37,7 +37,7 @@ akerr_ErrorContext *test_akgl_spritesheet_initialize(void) "Loaded texture was not the correct size"); snprintf((char *)&tmpstr->data, AKGL_MAX_STRING_LENGTH, "%s%s", SDL_GetBasePath(), "assets/spritesheet.png"); - image = IMG_LoadTexture(renderer, (char *)&tmpstr->data); + image = IMG_LoadTexture(renderer->sdl_renderer, (char *)&tmpstr->data); FAIL_ZERO_BREAK(errctx, image, AKGL_ERR_SDL, "Failed to load comparison image"); CATCH( @@ -138,7 +138,7 @@ akerr_ErrorContext *test_akgl_sprite_load_json(void) // Is it using the right spritesheet? snprintf((char *)&tmpstr->data, AKGL_MAX_STRING_LENGTH, "%s%s", SDL_GetBasePath(), "assets/spritesheet.png"); - image = IMG_LoadTexture(renderer, (char *)&tmpstr->data); + image = IMG_LoadTexture(renderer->sdl_renderer, (char *)&tmpstr->data); FAIL_ZERO_BREAK(errctx, image, AKGL_ERR_SDL, "Failed to load comparison image"); CATCH( @@ -187,6 +187,7 @@ int main(void) PREPARE_ERROR(errctx); ATTEMPT { + renderer = &_akgl_renderer; SDL_SetAppMetadata("SDL3-GameTest", "0.1", "net.aklabs.sdl3-gametest"); @@ -194,9 +195,10 @@ int main(void) FAIL_BREAK(errctx, AKGL_ERR_SDL, "Couldn't initialize SDL: %s", SDL_GetError()); } - if (!SDL_CreateWindowAndRenderer("net/aklabs/libakgl/test_sprite", 640, 480, 0, &window, &renderer)) { + if (!SDL_CreateWindowAndRenderer("net/aklabs/libakgl/test_sprite", 640, 480, 0, &window, &renderer->sdl_renderer)) { FAIL_BREAK(errctx, AKGL_ERR_SDL, "Couldn't create window/renderer: %s", SDL_GetError()); } + renderer->draw_texture = &akgl_render_2d_draw_texture; CATCH(errctx, akgl_heap_init()); CATCH(errctx, akgl_registry_init_sprite());