andrew/libakgl
1
0
Fork 0
Code Issues Pull Requests 1 Actions 1 Packages Projects Releases Wiki Activity

Add mutation testing harness

Browse Source 
Port the scratch-copy mutation runner used by libakerror and libakstdlib, covering libakgl source files with sampling, thresholds, JUnit reports, and test timeouts. Add a CMake target and documentation, preserve the intentionally failing character test exclusion, and migrate the sprite test to the renderer backend so mutation baselines are green.

Co-authored-by: Codex (GPT-5) <noreply@openai.com>
This commit is contained in:
Andrew Kesterson
2026-07-29 18:42:09 -04:00
parent 3854b33750
commit dc1bd0a798
5 changed files with 509 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
AGENTS.md
View File

@@ -26,6 +26,8 @@ ctest --test-dir build --output-on-failure
Run one test while iterating, for example `ctest --test-dir build -R sprite --output-on-failure`. The `rebuild.sh` script also installs into a developer-specific `/home/andrew/local` prefix and removes existing outputs; prefer the portable commands above unless that exact workflow is intended.
Run mutation testing with `cmake --build build --target mutation`. For a quick smoke run, use `scripts/mutation_test.py --target src/tilemap.c --max-mutants 10`; the harness mutates only a scratch copy and excludes the intentionally failing character test.
## Coding Style & Naming Conventions
Follow the surrounding C style: braces on the next line for function bodies, spaces inside control-flow parentheses, and short, focused functions. Preserve the indentation of the file being edited; older files contain both spaces and tabs. Public symbols use the `akgl_` prefix, public types use `akgl_TypeName`, and constants/macros use `AKGL_UPPER_SNAKE_CASE`. Name matching header/source pairs by feature, such as `include/akgl/sprite.h` and `src/sprite.c`. No repository-wide formatter or linter is configured, so avoid unrelated formatting churn.

22
CMakeLists.txt
View File

@@ -132,7 +132,7 @@ add_test(NAME semver_unit COMMAND test_semver_unit)
set_tests_properties(
actor bitmasks character registry sprite staticstring tilemap util semver_unit
PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests"
PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests" TIMEOUT 30
)
# Specify include directories for the library's headers (if applicable)
@@ -163,6 +163,26 @@ target_link_libraries(test_util PRIVATE akstdlib::akstdlib akerror::akerror akgl
target_link_libraries(charviewer PRIVATE akstdlib::akstdlib akerror::akerror akgl SDL3::SDL3 SDL3_ttf::SDL3_ttf SDL3_image::SDL3_image SDL3_mixer::SDL3_mixer jansson::jansson -lm)
# Mutation testing copies the repository to scratch space, applies one small
# source change at a time, and verifies that the passing tests detect it. The
# intentionally failing character test is excluded by the harness.
find_package(Python3 COMPONENTS Interpreter)
if(Python3_FOUND)
if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR)
set(AKGL_MUTATION_TARGET mutation)
else()
set(AKGL_MUTATION_TARGET akgl_mutation)
endif()
add_custom_target(${AKGL_MUTATION_TARGET}
COMMAND ${Python3_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/scripts/mutation_test.py
--source-root ${CMAKE_CURRENT_SOURCE_DIR}
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
USES_TERMINAL
COMMENT "Running mutation tests (breaks a scratch copy, expects tests to fail)"
)
endif()
set(main_lib_dest "lib/akgl-${MY_LIBRARY_VERSION}")
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/akgl.pc DESTINATION "lib/pkgconfig/")
install(TARGETS akgl DESTINATION "lib/")

13
README.md
View File

@@ -269,3 +269,16 @@ PASS(e, aksl_atoi(width->data, &screenwidth));
PASS(e, akgl_heap_release_string(width));
```
## Mutation testing
The mutation harness makes one deliberate source-code change at a time in a scratch copy, then rebuilds and runs the passing CTest suite to measure whether tests detect the change. The known-failing `character` test is excluded by default.
```sh
cmake --build build --target mutation
scripts/mutation_test.py --target src/tilemap.c --list
scripts/mutation_test.py --target src/tilemap.c --max-mutants 10
scripts/mutation_test.py --threshold 40 --junit mutation-junit.xml
```
The default run covers all libakgl-owned files under `src/`. Use repeated `--target` options to narrow the scope. A surviving mutant identifies behavior that the current tests do not verify; the script prints its file, line, operator, and exact edit. The real working tree is never mutated.

466
scripts/mutation_test.py Executable file
View File

@@ -0,0 +1,466 @@
#!/usr/bin/env python3
"""
Mutation testing harness for libakgl.
Mutation testing measures how good the test suite is at catching bugs. It works
by making many small, deliberate breakages ("mutants") to the library source --
flipping a comparison, deleting a statement, swapping true/false -- and then
running the whole CTest suite against each one. If the tests fail, the mutant is
"killed" (good: the tests noticed the bug). If the tests still pass, the mutant
"survived" (bad: a real bug of that shape would slip through unnoticed).
The mutation score is killed / (killed + survived). Surviving mutants are printed
with file:line and the exact change so they can be turned into new test cases.
This harness has no third-party dependencies (Python stdlib + the project's
normal cmake/ctest toolchain). It never mutates the real working tree: it copies
the repo to a scratch directory and mutates there.
Usage:
scripts/mutation_test.py [options]
--source-root DIR repo root to copy (default: parent of this script's dir)
--target FILE source file to mutate, relative to root; repeatable.
Default: all libakgl-owned C files under src/
--work DIR scratch dir for the mutated copy (default: a temp dir)
--timeout SECONDS per-suite ctest timeout (default: 120)
--exclude-test REGEX CTest regex to exclude (default: ^character$)
--threshold PCT exit non-zero if mutation score < PCT (default: 0 = off)
--list only list the mutants that would be run, then exit
--keep keep the scratch working copy on exit (for debugging)
-j N (reserved) currently runs sequentially
"""
import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
# --------------------------------------------------------------------------- #
# Mutation operators
#
# Each operator yields zero or more (start, end, replacement) edits for a single
# line of source. The driver applies exactly one edit per mutant so every mutant
# differs from the original by one localized change.
# --------------------------------------------------------------------------- #
# Relational operator replacement: map each operator to the alternatives that
# meaningfully change behaviour (not merely the strict negation).
_REL = {
"==": ["!="],
"!=": ["=="],
"<=": ["<", "=="],
">=": [">", "=="],
"<": ["<=", ">"],
">": [">=", "<"],
}
# Match a relational operator that is NOT part of ->, <<, >>, =>, <=, >=, ==, !=
# unless we intend it. We tokenize the two-char operators first, then single.
_REL_TWO = re.compile(r"(==|!=|<=|>=)")
_REL_ONE = re.compile(r"(?<![-<>=!+])([<>])(?![=<>])")
_LOGICAL = {"&&": "||", "||": "&&"}
_LOG_RE = re.compile(r"(&&|\|\|)")
_BOOL = {"true": "false", "false": "true"}
_BOOL_RE = re.compile(r"\b(true|false)\b")
# Arithmetic / compound-assignment on whitespace-delimited operands only, to
# avoid touching ++, --, ->, unary signs, or pointer/format punctuation.
_ARITH_RE = re.compile(r"(?<=\s)([+\-])(?=\s)")
_ARITH = {"+": "-", "-": "+"}
_COMPOUND_RE = re.compile(r"(\+=|-=)")
_COMPOUND = {"+=": "-=", "-=": "+="}
# Integer literal replacement: 0 <-> 1 (word-bounded, not inside identifiers or
# larger numbers, not a float).
_INT_RE = re.compile(r"(?<![\w.])([01])(?![\w.])")
_INT = {"0": "1", "1": "0"}
def _op_edits(line):
"""Yield (tag, start, end, replacement) for every candidate mutation."""
# Relational (two-char first so we don't split them with the one-char pass)
for m in _REL_TWO.finditer(line):
for alt in _REL[m.group(1)]:
yield ("ROR", m.start(1), m.end(1), alt)
for m in _REL_ONE.finditer(line):
for alt in _REL[m.group(1)]:
yield ("ROR", m.start(1), m.end(1), alt)
for m in _LOG_RE.finditer(line):
yield ("LCR", m.start(1), m.end(1), _LOGICAL[m.group(1)])
for m in _BOOL_RE.finditer(line):
yield ("BCR", m.start(1), m.end(1), _BOOL[m.group(1)])
for m in _COMPOUND_RE.finditer(line):
yield ("AOR", m.start(1), m.end(1), _COMPOUND[m.group(1)])
for m in _ARITH_RE.finditer(line):
yield ("AOR", m.start(1), m.end(1), _ARITH[m.group(1)])
for m in _INT_RE.finditer(line):
yield ("ICR", m.start(1), m.end(1), _INT[m.group(1)])
# Statement-deletion: neutralize a whole statement. We only delete statements
# that are safe to drop without guaranteeing a compile error, so a surviving
# deletion is a genuine test gap rather than compiler noise.
_STMT_DELETABLE = re.compile(
r"""^\s*(
break |
return\b[^;]* |
[A-Za-z_][-\w>().\[\]* ]*\s*=\s*[^;]* | # assignments
[A-Za-z_][\w]*\s*\([^;]*\) # bare function calls
)\s*;\s*(\\?)\s*$""",
re.VERBOSE,
)
# --------------------------------------------------------------------------- #
# Deciding which lines are eligible to mutate
# --------------------------------------------------------------------------- #
# Skip preprocessor control and the block of constant/error-code #defines in the
# template header: mutating buffer sizes or renumbering error codes produces
# equivalent or uninteresting mutants that swamp the signal.
_SKIP_LINE = re.compile(
r"""^\s*(
\#\s*(include|ifn?def|ifdef|if|elif|else|endif|error|pragma|undef) |
\#\s*define\s+AKERR_(MAX|LAST|NULLPOINTER|OUTOFBOUNDS|API|ATTRIBUTE|
TYPE|KEY|INDEX|FORMAT|IO|VALUE|RELATIONSHIP|EOF|CIRCULAR_REFERENCE|
ITERATOR_BREAK|NOT_IMPLEMENTED|BADEXC|NOIGNORE|USE_STDLIB)\b |
\* | // # comment bodies / line comments
)""",
re.VERBOSE,
)
def _is_comment_or_blank(line):
s = line.strip()
return (not s) or s.startswith("//") or s.startswith("/*") or s.startswith("*")
def eligible(line):
if _is_comment_or_blank(line):
return False
if _SKIP_LINE.match(line):
return False
return True
class Mutant:
__slots__ = ("path", "lineno", "op", "before", "after", "col")
def __init__(self, path, lineno, op, before, after, col):
self.path = path
self.lineno = lineno
self.op = op
self.before = before
self.after = after
self.col = col
def describe(self):
return (f"{self.path}:{self.lineno} [{self.op}] "
f"col{self.col}: {self.before.strip()} -> {self.after.strip()}")
def generate_mutants(root, rel_target):
"""Enumerate all mutants for one target file."""
abspath = os.path.join(root, rel_target)
with open(abspath, "r") as fh:
lines = fh.readlines()
mutants = []
for i, line in enumerate(lines, start=1):
if not eligible(line):
continue
# substitution operators
seen = set()
for tag, s, e, repl in _op_edits(line):
key = (s, e, repl)
if key in seen:
continue
seen.add(key)
mutated = line[:s] + repl + line[e:]
if mutated == line:
continue
mutants.append(Mutant(rel_target, i, tag, line, mutated, s))
# statement deletion
m = _STMT_DELETABLE.match(line)
if m:
indent = line[: len(line) - len(line.lstrip())]
cont = "\\" if line.rstrip().endswith("\\") else ""
deleted = f"{indent}/* mutant: deleted */ {cont}\n" if cont else f"{indent};\n"
mutants.append(Mutant(rel_target, i, "SDL", line, deleted, 0))
return mutants
# --------------------------------------------------------------------------- #
# Build / test orchestration against a scratch copy
# --------------------------------------------------------------------------- #
class Runner:
def __init__(self, work, timeout, exclude_test):
self.work = work
self.build = os.path.join(work, "build")
self.timeout = timeout
self.exclude_test = exclude_test
self.env = os.environ.copy()
library_dirs = [
self.build,
os.path.join(self.build, "deps", "SDL"),
os.path.join(self.build, "deps", "SDL_image"),
os.path.join(self.build, "deps", "SDL_mixer"),
os.path.join(self.build, "deps", "SDL_ttf"),
]
current_library_path = self.env.get("LD_LIBRARY_PATH", "")
if current_library_path:
library_dirs.append(current_library_path)
self.env["LD_LIBRARY_PATH"] = os.pathsep.join(library_dirs)
def _run(self, cmd, timeout=None):
return subprocess.run(
cmd, cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
timeout=timeout, env=self.env,
)
def configure(self):
r = self._run(["cmake", "-S", ".", "-B", "build"], timeout=self.timeout)
return r.returncode == 0, r.stdout
def ctest_command(self):
command = ["ctest", "--test-dir", "build", "--output-on-failure",
"--stop-on-failure"]
if self.exclude_test:
command.extend(["-E", self.exclude_test])
return command
def build_and_test(self):
"""Return ('killed-compile' | 'killed-test' | 'killed-timeout' | 'survived')."""
try:
b = self._run(["cmake", "--build", "build"], timeout=self.timeout)
except subprocess.TimeoutExpired:
return "killed-timeout"
if b.returncode != 0:
return "killed-compile"
try:
t = subprocess.run(
self.ctest_command(),
env=self.env,
cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
timeout=self.timeout,
)
except subprocess.TimeoutExpired:
return "killed-timeout"
return "survived" if t.returncode == 0 else "killed-test"
def _xml_escape(s):
return (s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
.replace('"', "&quot;"))
def write_junit(path, records, targets):
"""Write a JUnit XML report. One <testcase> per mutant; a surviving mutant
is a <failure> (test-suite gap), a killed mutant is a passing case."""
by_file = {t: [] for t in targets}
for m, result in records:
by_file.setdefault(m.path, []).append((m, result))
total = len(records)
total_fail = sum(1 for _, r in records if r == "survived")
out = ['<?xml version="1.0" encoding="UTF-8"?>',
f'<testsuites name="mutation" tests="{total}" failures="{total_fail}">']
for f, items in by_file.items():
if not items:
continue
fails = sum(1 for _, r in items if r == "survived")
out.append(f' <testsuite name="mutation:{_xml_escape(f)}" '
f'tests="{len(items)}" failures="{fails}">')
for m, result in items:
name = _xml_escape(m.describe())
cls = "mutation." + _xml_escape(m.path)
if result == "survived":
detail = _xml_escape(f"{m.before.strip()} -> {m.after.strip()}")
out.append(f' <testcase name="{name}" classname="{cls}" time="0">')
out.append(f' <failure message="survived mutant '
f'({_xml_escape(m.op)} at {_xml_escape(m.path)}:'
f'{m.lineno})">{detail}</failure>')
out.append(' </testcase>')
else:
out.append(f' <testcase name="{name}" classname="{cls}" '
f'time="0"><system-out>{_xml_escape(result)}'
'</system-out></testcase>')
out.append(' </testsuite>')
out.append('</testsuites>')
with open(path, "w") as fh:
fh.write("\n".join(out) + "\n")
def copy_tree(src, dst):
ignore = shutil.ignore_patterns("build", ".git", "*.o", "*.so", "*~",
"#*#", "*.iso")
shutil.copytree(src, dst, ignore=ignore, symlinks=True)
def read_lines(path):
with open(path) as fh:
return fh.readlines()
def write_lines(path, lines):
with open(path, "w") as fh:
fh.writelines(lines)
def main():
# Line-buffer stdout so progress is visible live under CI / the cmake target.
try:
sys.stdout.reconfigure(line_buffering=True)
except (AttributeError, ValueError):
pass
here = os.path.dirname(os.path.abspath(__file__))
default_root = os.path.dirname(here)
ap = argparse.ArgumentParser(description="Mutation testing for libakgl")
ap.add_argument("--source-root", default=default_root)
ap.add_argument("--target", action="append", default=None)
ap.add_argument("--work", default=None)
ap.add_argument("--timeout", type=int, default=120)
ap.add_argument("--exclude-test", default="^character$",
help="CTest regex to exclude (default: ^character$)")
ap.add_argument("--threshold", type=float, default=0.0)
ap.add_argument("--junit", default=None,
help="write a JUnit XML report to this path")
ap.add_argument("--max-mutants", type=int, default=0,
help="cap the run at N evenly-sampled mutants (0 = all)")
ap.add_argument("--list", action="store_true")
ap.add_argument("--keep", action="store_true")
ap.add_argument("-j", type=int, default=1)
args = ap.parse_args()
root = os.path.abspath(args.source_root)
targets = args.target or [
"src/actor.c", "src/actor_state_string_names.c", "src/assets.c",
"src/character.c", "src/controller.c", "src/draw.c",
"src/game.c", "src/heap.c", "src/json_helpers.c",
"src/physics.c", "src/registry.c", "src/renderer.c",
"src/sprite.c", "src/staticstring.c", "src/text.c",
"src/tilemap.c", "src/util.c",
]
# Enumerate mutants from the pristine sources.
all_mutants = []
for t in targets:
all_mutants.extend(generate_mutants(root, t))
print(f"Generated {len(all_mutants)} mutants across {len(targets)} file(s):")
for t in targets:
n = sum(1 for m in all_mutants if m.path == t)
print(f" {t}: {n}")
# Optional even-strided sampling to bound run time (CI / smoke tests).
if args.max_mutants and len(all_mutants) > args.max_mutants:
step = len(all_mutants) / args.max_mutants
sampled = [all_mutants[int(i * step)] for i in range(args.max_mutants)]
print(f"Sampling {len(sampled)} of {len(all_mutants)} mutants "
f"(--max-mutants {args.max_mutants}).")
all_mutants = sampled
if args.list:
for m in all_mutants:
print(" " + m.describe())
return 0
if not all_mutants:
print("No mutants generated; nothing to do.")
return 0
# Scratch working copy.
work_parent = args.work or tempfile.mkdtemp(prefix="akgl_mut_")
work = os.path.join(work_parent, "src") if args.work else work_parent
if os.path.exists(work):
shutil.rmtree(work)
print(f"\nCopying sources to scratch dir: {work}")
copy_tree(root, work)
runner = Runner(work, args.timeout, args.exclude_test)
print("Configuring baseline ...")
ok, out = runner.configure()
if not ok:
sys.stderr.write(out.decode(errors="replace"))
sys.stderr.write("\nBaseline configure FAILED; aborting.\n")
return 2
print("Verifying baseline is green (no mutation) ...")
baseline = runner.build_and_test()
if baseline != "survived":
sys.stderr.write(f"Baseline is not green ({baseline}); aborting. "
"Fix the suite before mutation testing.\n")
return 2
print("Baseline OK.\n")
# Group mutants by file so we mutate one file at a time and restore it.
killed = {"killed-compile": 0, "killed-test": 0, "killed-timeout": 0}
survivors = []
records = []
total = len(all_mutants)
# Cache pristine contents per target.
pristine = {t: read_lines(os.path.join(work, t)) for t in targets}
for idx, m in enumerate(all_mutants, start=1):
tgt_abs = os.path.join(work, m.path)
lines = list(pristine[m.path])
lines[m.lineno - 1] = m.after
write_lines(tgt_abs, lines)
try:
result = runner.build_and_test()
finally:
write_lines(tgt_abs, pristine[m.path]) # always restore
records.append((m, result))
if result == "survived":
survivors.append(m)
mark = "SURVIVED"
else:
killed[result] += 1
mark = result.upper()
print(f"[{idx}/{total}] {mark:16} {m.describe()}")
total_killed = sum(killed.values())
score = 100.0 * total_killed / total if total else 100.0
print("\n" + "=" * 72)
print("MUTATION TESTING SUMMARY")
print("=" * 72)
print(f" total mutants : {total}")
print(f" killed (test) : {killed['killed-test']}")
print(f" killed (compile): {killed['killed-compile']}")
print(f" killed (timeout): {killed['killed-timeout']}")
print(f" survived : {len(survivors)}")
print(f" mutation score : {score:.1f}%")
if survivors:
print("\nSurviving mutants (test-suite gaps -- turn these into tests):")
for m in survivors:
print(" " + m.describe())
if args.junit:
junit_path = os.path.abspath(args.junit)
write_junit(junit_path, records, targets)
print(f"\nJUnit report written to: {junit_path}")
if not args.keep and not args.work:
shutil.rmtree(work_parent, ignore_errors=True)
else:
print(f"\nScratch working copy kept at: {work}")
if args.threshold > 0 and score < args.threshold:
print(f"\nFAIL: mutation score {score:.1f}% < threshold {args.threshold:.1f}%")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())

12
tests/sprite.c
View File

@@ -10,9 +10,9 @@
#include <akgl/heap.h>
#include <akgl/util.h>
#include <akgl/error.h>
#include <akgl/game.h>
#include <akgl/renderer.h>
SDL_Window *window;
SDL_Renderer *renderer;
akerr_ErrorContext *test_akgl_spritesheet_initialize(void)
{
@@ -37,7 +37,7 @@ akerr_ErrorContext *test_akgl_spritesheet_initialize(void)
"Loaded texture was not the correct size");
snprintf((char *)&tmpstr->data, AKGL_MAX_STRING_LENGTH, "%s%s", SDL_GetBasePath(), "assets/spritesheet.png");
image = IMG_LoadTexture(renderer, (char *)&tmpstr->data);
image = IMG_LoadTexture(renderer->sdl_renderer, (char *)&tmpstr->data);
FAIL_ZERO_BREAK(errctx, image, AKGL_ERR_SDL, "Failed to load comparison image");
CATCH(
@@ -138,7 +138,7 @@ akerr_ErrorContext *test_akgl_sprite_load_json(void)
// Is it using the right spritesheet?
snprintf((char *)&tmpstr->data, AKGL_MAX_STRING_LENGTH, "%s%s", SDL_GetBasePath(), "assets/spritesheet.png");
image = IMG_LoadTexture(renderer, (char *)&tmpstr->data);
image = IMG_LoadTexture(renderer->sdl_renderer, (char *)&tmpstr->data);
FAIL_ZERO_BREAK(errctx, image, AKGL_ERR_SDL, "Failed to load comparison image");
CATCH(
@@ -187,6 +187,7 @@ int main(void)
PREPARE_ERROR(errctx);
ATTEMPT {
renderer = &_akgl_renderer;
SDL_SetAppMetadata("SDL3-GameTest", "0.1", "net.aklabs.sdl3-gametest");
@@ -194,9 +195,10 @@ int main(void)
FAIL_BREAK(errctx, AKGL_ERR_SDL, "Couldn't initialize SDL: %s", SDL_GetError());
}
if (!SDL_CreateWindowAndRenderer("net/aklabs/libakgl/test_sprite", 640, 480, 0, &window, &renderer)) {
if (!SDL_CreateWindowAndRenderer("net/aklabs/libakgl/test_sprite", 640, 480, 0, &window, &renderer->sdl_renderer)) {
FAIL_BREAK(errctx, AKGL_ERR_SDL, "Couldn't create window/renderer: %s", SDL_GetError());
}
renderer->draw_texture = &akgl_render_2d_draw_texture;
CATCH(errctx, akgl_heap_init());
CATCH(errctx, akgl_registry_init_sprite());