andrew/akbasic
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
c8b917d2053a5060323800ae4b1145383f6c945d
akbasic/scripts/mutation_test.py
Andrew Kesterson caebc1a174 Stop the mutation harness dropping the documentation's figures 
`copy_tree()` excluded `*.png` from the scratch tree, which was right when the
only PNGs anywhere were libpng's and SDL_image's test corpora -- 39 MB that
nothing in this build reads. `docs/images/` changed that: `docs_examples`
asserts that every `screenshot=` block has its figure, so a copy with no images
fails thirteen blocks, the *baseline* comes back red, and the run aborts with
"Fix the suite before mutation testing" before mutating anything.

Mutation testing has therefore been unrunnable since the figures landed, and it
fails in the one way this repository's own notes warn about: looking like the
thing it measures is broken.

Keep the project's PNGs, still drop the dependencies'. The pattern list cannot
express that -- `ignore_patterns` matches basenames -- so it becomes a function
that adds the `*.png` rule only under `deps/`.

Verified: `--target src/symtab.c --max-mutants 2` now reports "Baseline OK." and
scores both mutants, where every invocation before this aborted.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-01 23:36:02 -04:00

469 lines
18 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
"""
Mutation testing harness for akbasic.
Mutation testing measures how good the test suite is at catching bugs. It works
by making many small, deliberate breakages ("mutants") to the library source --
flipping a comparison, deleting a statement, swapping true/false -- and then
running the whole CTest suite against each one. If the tests fail, the mutant is
"killed" (good: the tests noticed the bug). If the tests still pass, the mutant
"survived" (bad: a real bug of that shape would slip through unnoticed).
The mutation score is killed / (killed + survived). Surviving mutants are printed
with file:line and the exact change so they can be turned into new test cases.
This harness has no third-party dependencies (Python stdlib + the project's
normal cmake/ctest toolchain). It never mutates the real working tree: it copies
the repo to a scratch directory and mutates there.
Usage:
scripts/mutation_test.py [options]
--source-root DIR repo root to copy (default: parent of this script's dir)
--target FILE source file to mutate, relative to root; repeatable.
Default: every akbasic-owned C file under src/
--work DIR scratch dir for the mutated copy (default: a temp dir)
--timeout SECONDS per-suite ctest timeout (default: 120)
--threshold PCT exit non-zero if mutation score < PCT (default: 0 = off)
--list only list the mutants that would be run, then exit
--keep keep the scratch working copy on exit (for debugging)
-j N (reserved) currently runs sequentially
"""
import argparse
import os
import re
import shutil
import subprocess
import sys
import tempfile
# --------------------------------------------------------------------------- #
# Mutation operators
#
# Each operator yields zero or more (start, end, replacement) edits for a single
# line of source. The driver applies exactly one edit per mutant so every mutant
# differs from the original by one localized change.
# --------------------------------------------------------------------------- #
# Relational operator replacement: map each operator to the alternatives that
# meaningfully change behaviour (not merely the strict negation).
_REL = {
"==": ["!="],
"!=": ["=="],
"<=": ["<", "=="],
">=": [">", "=="],
"<": ["<=", ">"],
">": [">=", "<"],
}
# Match a relational operator that is NOT part of ->, <<, >>, =>, <=, >=, ==, !=
# unless we intend it. We tokenize the two-char operators first, then single.
_REL_TWO = re.compile(r"(==|!=|<=|>=)")
_REL_ONE = re.compile(r"(?<![-<>=!+])([<>])(?![=<>])")
_LOGICAL = {"&&": "||", "||": "&&"}
_LOG_RE = re.compile(r"(&&|\|\|)")
_BOOL = {"true": "false", "false": "true"}
_BOOL_RE = re.compile(r"\b(true|false)\b")
# Arithmetic / compound-assignment on whitespace-delimited operands only, to
# avoid touching ++, --, ->, unary signs, or pointer/format punctuation.
_ARITH_RE = re.compile(r"(?<=\s)([+\-])(?=\s)")
_ARITH = {"+": "-", "-": "+"}
_COMPOUND_RE = re.compile(r"(\+=|-=)")
_COMPOUND = {"+=": "-=", "-=": "+="}
# Integer literal replacement: 0 <-> 1 (word-bounded, not inside identifiers or
# larger numbers, not a float).
_INT_RE = re.compile(r"(?<![\w.])([01])(?![\w.])")
_INT = {"0": "1", "1": "0"}
def _op_edits(line):
"""Yield (tag, start, end, replacement) for every candidate mutation."""
# Relational (two-char first so we don't split them with the one-char pass)
for m in _REL_TWO.finditer(line):
for alt in _REL[m.group(1)]:
yield ("ROR", m.start(1), m.end(1), alt)
for m in _REL_ONE.finditer(line):
for alt in _REL[m.group(1)]:
yield ("ROR", m.start(1), m.end(1), alt)
for m in _LOG_RE.finditer(line):
yield ("LCR", m.start(1), m.end(1), _LOGICAL[m.group(1)])
for m in _BOOL_RE.finditer(line):
yield ("BCR", m.start(1), m.end(1), _BOOL[m.group(1)])
for m in _COMPOUND_RE.finditer(line):
yield ("AOR", m.start(1), m.end(1), _COMPOUND[m.group(1)])
for m in _ARITH_RE.finditer(line):
yield ("AOR", m.start(1), m.end(1), _ARITH[m.group(1)])
for m in _INT_RE.finditer(line):
yield ("ICR", m.start(1), m.end(1), _INT[m.group(1)])
# Statement-deletion: neutralize a whole statement. We only delete statements
# that are safe to drop without guaranteeing a compile error, so a surviving
# deletion is a genuine test gap rather than compiler noise.
_STMT_DELETABLE = re.compile(
r"""^\s*(
break |
return\b[^;]* |
[A-Za-z_][-\w>().\[\]* ]*\s*=\s*[^;]* | # assignments
[A-Za-z_][\w]*\s*\([^;]*\) # bare function calls
)\s*;\s*(\\?)\s*$""",
re.VERBOSE,
)
# --------------------------------------------------------------------------- #
# Deciding which lines are eligible to mutate
# --------------------------------------------------------------------------- #
# Skip preprocessor control and the block of constant/error-code #defines in the
# template header: mutating buffer sizes or renumbering error codes produces
# equivalent or uninteresting mutants that swamp the signal.
_SKIP_LINE = re.compile(
r"""^\s*(
\#\s*(include|ifn?def|ifdef|if|elif|else|endif|error|pragma|undef) |
\#\s*define\s+AKERR_(MAX|LAST|NULLPOINTER|OUTOFBOUNDS|API|ATTRIBUTE|
TYPE|KEY|INDEX|FORMAT|IO|VALUE|RELATIONSHIP|EOF|CIRCULAR_REFERENCE|
ITERATOR_BREAK|NOT_IMPLEMENTED|BADEXC|NOIGNORE|USE_STDLIB)\b |
\* | // # comment bodies / line comments
)""",
re.VERBOSE,
)
def _is_comment_or_blank(line):
s = line.strip()
return (not s) or s.startswith("//") or s.startswith("/*") or s.startswith("*")
def eligible(line):
if _is_comment_or_blank(line):
return False
if _SKIP_LINE.match(line):
return False
return True
class Mutant:
__slots__ = ("path", "lineno", "op", "before", "after", "col")
def __init__(self, path, lineno, op, before, after, col):
self.path = path
self.lineno = lineno
self.op = op
self.before = before
self.after = after
self.col = col
def describe(self):
return (f"{self.path}:{self.lineno} [{self.op}] "
f"col{self.col}: {self.before.strip()} -> {self.after.strip()}")
def generate_mutants(root, rel_target):
"""Enumerate all mutants for one target file."""
abspath = os.path.join(root, rel_target)
with open(abspath, "r") as fh:
lines = fh.readlines()
mutants = []
for i, line in enumerate(lines, start=1):
if not eligible(line):
continue
# substitution operators
seen = set()
for tag, s, e, repl in _op_edits(line):
key = (s, e, repl)
if key in seen:
continue
seen.add(key)
mutated = line[:s] + repl + line[e:]
if mutated == line:
continue
mutants.append(Mutant(rel_target, i, tag, line, mutated, s))
# statement deletion
m = _STMT_DELETABLE.match(line)
if m:
indent = line[: len(line) - len(line.lstrip())]
cont = "\\" if line.rstrip().endswith("\\") else ""
deleted = f"{indent}/* mutant: deleted */ {cont}\n" if cont else f"{indent};\n"
mutants.append(Mutant(rel_target, i, "SDL", line, deleted, 0))
return mutants
# --------------------------------------------------------------------------- #
# Build / test orchestration against a scratch copy
# --------------------------------------------------------------------------- #
class Runner:
def __init__(self, work, timeout):
self.work = work
self.build = os.path.join(work, "build")
self.timeout = timeout
def _run(self, cmd, timeout=None):
return subprocess.run(
cmd, cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
timeout=timeout,
)
def configure(self):
r = self._run(["cmake", "-S", ".", "-B", "build"], timeout=self.timeout)
return r.returncode == 0, r.stdout
def build_and_test(self):
"""Return ('killed-compile' | 'killed-test' | 'killed-timeout' | 'survived')."""
try:
b = self._run(["cmake", "--build", "build"], timeout=self.timeout)
except subprocess.TimeoutExpired:
return "killed-timeout"
if b.returncode != 0:
return "killed-compile"
try:
t = subprocess.run(
["ctest", "--test-dir", "build", "--output-on-failure",
"--stop-on-failure"],
cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
timeout=self.timeout,
)
except subprocess.TimeoutExpired:
return "killed-timeout"
return "survived" if t.returncode == 0 else "killed-test"
def _xml_escape(s):
return (s.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;")
.replace('"', "&quot;"))
def write_junit(path, records, targets):
"""Write a JUnit XML report. One <testcase> per mutant; a surviving mutant
is a <failure> (test-suite gap), a killed mutant is a passing case."""
by_file = {t: [] for t in targets}
for m, result in records:
by_file.setdefault(m.path, []).append((m, result))
total = len(records)
total_fail = sum(1 for _, r in records if r == "survived")
out = ['<?xml version="1.0" encoding="UTF-8"?>',
f'<testsuites name="mutation" tests="{total}" failures="{total_fail}">']
for f, items in by_file.items():
if not items:
continue
fails = sum(1 for _, r in items if r == "survived")
out.append(f' <testsuite name="mutation:{_xml_escape(f)}" '
f'tests="{len(items)}" failures="{fails}">')
for m, result in items:
name = _xml_escape(m.describe())
cls = "mutation." + _xml_escape(m.path)
if result == "survived":
detail = _xml_escape(f"{m.before.strip()} -> {m.after.strip()}")
out.append(f' <testcase name="{name}" classname="{cls}" time="0">')
out.append(f' <failure message="survived mutant '
f'({_xml_escape(m.op)} at {_xml_escape(m.path)}:'
f'{m.lineno})">{detail}</failure>')
out.append(' </testcase>')
else:
out.append(f' <testcase name="{name}" classname="{cls}" '
f'time="0"><system-out>{_xml_escape(result)}'
'</system-out></testcase>')
out.append(' </testsuite>')
out.append('</testsuites>')
with open(path, "w") as fh:
fh.write("\n".join(out) + "\n")
def copy_tree(src, dst):
# "build*" covers build/, build-asan/ and build-coverage/: the harness
# configures its own tree inside the copy, so copying those is pure IO --
# and a coverage tree drags along every .gcno/.gcda as well.
#
# deps/ is copied wholesale and has to be: the build pulls libakerror and
# libakstdlib in with add_subdirectory. The golden corpus used to be the
# other reason -- it was driven in place out of deps/basicinterpret -- and
# now lives in tests/reference/, which comes along with the rest of the tree.
# Only vendored Windows DLLs are dead weight, hence "*.dll".
#
# **PNGs are ours or theirs, and the difference matters.** This used to drop
# every one of them, which was right when the only PNGs in the tree were
# SDL_image's and libpng's test corpora -- 39 MB that nothing here reads.
# docs/images/ changed that: docs_examples asserts every `screenshot=` block
# has its figure, so dropping them made the *baseline* fail and no mutation
# run could start at all. Keep the project's thirteen; still drop the
# dependencies' thousands.
patterns = shutil.ignore_patterns("build*", ".git", "*.o", "*.so", "*~",
"#*#", "*.iso", "*.gcno", "*.gcda",
"*.dll")
depsdir = os.path.join(os.path.abspath(src), "deps")
def ignore(directory, names):
skip = set(patterns(directory, names))
if os.path.abspath(directory).startswith(depsdir):
skip.update(n for n in names if n.endswith(".png"))
return skip
shutil.copytree(src, dst, ignore=ignore, symlinks=True)
def read_lines(path):
with open(path) as fh:
return fh.readlines()
def write_lines(path, lines):
with open(path, "w") as fh:
fh.writelines(lines)
def main():
# Line-buffer stdout so progress is visible live under CI / the cmake target.
try:
sys.stdout.reconfigure(line_buffering=True)
except (AttributeError, ValueError):
pass
here = os.path.dirname(os.path.abspath(__file__))
default_root = os.path.dirname(here)
ap = argparse.ArgumentParser(description="Mutation testing for akbasic")
ap.add_argument("--source-root", default=default_root)
ap.add_argument("--target", action="append", default=None)
ap.add_argument("--work", default=None)
ap.add_argument("--timeout", type=int, default=120)
ap.add_argument("--threshold", type=float, default=0.0)
ap.add_argument("--junit", default=None,
help="write a JUnit XML report to this path")
ap.add_argument("--max-mutants", type=int, default=0,
help="cap the run at N evenly-sampled mutants (0 = all)")
ap.add_argument("--list", action="store_true")
ap.add_argument("--keep", action="store_true")
ap.add_argument("-j", type=int, default=1)
args = ap.parse_args()
root = os.path.abspath(args.source_root)
targets = args.target or [
"src/convert.c", "src/environment.c", "src/error.c", "src/grammar.c",
"src/parser.c", "src/parser_commands.c", "src/runtime.c",
"src/runtime_commands.c", "src/runtime_functions.c", "src/scanner.c",
"src/sink_stdio.c", "src/symtab.c", "src/value.c", "src/variable.c",
"src/verbs.c",
]
# Enumerate mutants from the pristine sources.
all_mutants = []
for t in targets:
all_mutants.extend(generate_mutants(root, t))
print(f"Generated {len(all_mutants)} mutants across {len(targets)} file(s):")
for t in targets:
n = sum(1 for m in all_mutants if m.path == t)
print(f" {t}: {n}")
# Optional even-strided sampling to bound run time (CI / smoke tests).
if args.max_mutants and len(all_mutants) > args.max_mutants:
step = len(all_mutants) / args.max_mutants
sampled = [all_mutants[int(i * step)] for i in range(args.max_mutants)]
print(f"Sampling {len(sampled)} of {len(all_mutants)} mutants "
f"(--max-mutants {args.max_mutants}).")
all_mutants = sampled
if args.list:
for m in all_mutants:
print(" " + m.describe())
return 0
if not all_mutants:
print("No mutants generated; nothing to do.")
return 0
# Scratch working copy.
work_parent = args.work or tempfile.mkdtemp(prefix="akbasic_mut_")
work = os.path.join(work_parent, "src") if args.work else work_parent
if os.path.exists(work):
shutil.rmtree(work)
print(f"\nCopying sources to scratch dir: {work}")
copy_tree(root, work)
runner = Runner(work, args.timeout)
print("Configuring baseline ...")
ok, out = runner.configure()
if not ok:
sys.stderr.write(out.decode(errors="replace"))
sys.stderr.write("\nBaseline configure FAILED; aborting.\n")
return 2
print("Verifying baseline is green (no mutation) ...")
baseline = runner.build_and_test()
if baseline != "survived":
sys.stderr.write(f"Baseline is not green ({baseline}); aborting. "
"Fix the suite before mutation testing.\n")
return 2
print("Baseline OK.\n")
# Group mutants by file so we mutate one file at a time and restore it.
killed = {"killed-compile": 0, "killed-test": 0, "killed-timeout": 0}
survivors = []
records = []
total = len(all_mutants)
# Cache pristine contents per target.
pristine = {t: read_lines(os.path.join(work, t)) for t in targets}
for idx, m in enumerate(all_mutants, start=1):
tgt_abs = os.path.join(work, m.path)
lines = list(pristine[m.path])
lines[m.lineno - 1] = m.after
write_lines(tgt_abs, lines)
try:
result = runner.build_and_test()
finally:
write_lines(tgt_abs, pristine[m.path]) # always restore
records.append((m, result))
if result == "survived":
survivors.append(m)
mark = "SURVIVED"
else:
killed[result] += 1
mark = result.upper()
print(f"[{idx}/{total}] {mark:16} {m.describe()}")
total_killed = sum(killed.values())
score = 100.0 * total_killed / total if total else 100.0
print("\n" + "=" * 72)
print("MUTATION TESTING SUMMARY")
print("=" * 72)
print(f" total mutants : {total}")
print(f" killed (test) : {killed['killed-test']}")
print(f" killed (compile): {killed['killed-compile']}")
print(f" killed (timeout): {killed['killed-timeout']}")
print(f" survived : {len(survivors)}")
print(f" mutation score : {score:.1f}%")
if survivors:
print("\nSurviving mutants (test-suite gaps -- turn these into tests):")
for m in survivors:
print(" " + m.describe())
if args.junit:
junit_path = os.path.abspath(args.junit)
write_junit(junit_path, records, targets)
print(f"\nJUnit report written to: {junit_path}")
if not args.keep and not args.work:
shutil.rmtree(work_parent, ignore_errors=True)
else:
print(f"\nScratch working copy kept at: {work}")
if args.threshold > 0 and score < args.threshold:
print(f"\nFAIL: mutation score {score:.1f}% < threshold {args.threshold:.1f}%")
return 1
return 0
if __name__ == "__main__":
sys.exit(main())
Reference in New Issue View Git Blame Copy Permalink