diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 31711a1..23334da 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -2,6 +2,10 @@ name: akbasic CI Build run-name: ${{ gitea.actor }} akbasic test on: [push] +# Push-triggered checks only. The doxygen gate and the whole-tree mutation run +# live in release.yaml, which is manual (workflow_dispatch) because between them +# they cost hours of runner time and are release gates rather than per-commit +# ones. jobs: cmake_build: runs-on: ubuntu-latest @@ -52,40 +56,6 @@ jobs: fail_on_failure: 'true' - run: echo "๐Ÿ This job's status is ${{ job.status }}." - docs: - runs-on: ubuntu-latest - steps: - - name: Check out repository code - uses: actions/checkout@v4 - # No submodules at all. Doxyfile's INPUT is include/akbasic, src and - # examples, none of which live under deps/, and doxygen does not need to - # resolve to parse a declaration. Verified by running it - # against a tree with no deps/ present. - - name: dependencies - run: | - sudo apt-get update -y - sudo apt-get install -y doxygen graphviz - # This is a gate, not a convenience. The Doxyfile sets - # WARN_AS_ERROR = FAIL_ON_WARNINGS, so a doc block that documents some of a - # function's parameters but not all of them fails the build. All 114 public - # declarations under include/akbasic carry a @brief, a @param per - # parameter, a @return and their @throws; keep it that way. - # - # OUTPUT_DIRECTORY is build/docs and doxygen will not create a missing - # parent, so make it first -- there is no cmake step in this job to do it. - - name: build API documentation - run: | - mkdir -p build - doxygen Doxyfile - - name: upload API documentation - if: always() - uses: actions/upload-artifact@v4 - with: - name: api-documentation - path: build/docs/html/ - if-no-files-found: error - - run: echo "๐Ÿ This job's status is ${{ job.status }}." - sanitizers: runs-on: ubuntu-latest steps: @@ -159,3 +129,70 @@ jobs: path: build-coverage/coverage/ if-no-files-found: warn - run: echo "๐Ÿ This job's status is ${{ job.status }}." + + mutation_test: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + with: + # The harness copies the repo and configures a build inside the copy, + # so it needs the same submodules the main build does -- including + # deps/basicinterpret, because the golden corpus is part of what kills + # mutants. + submodules: true + - name: dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y cmake gcc python3 + # Verify the tests actually catch bugs: break the library many ways and + # confirm the suite fails. This matters more here than in an ordinary C + # library, because the akerror control-flow macros expand at their call + # sites -- gcov attributes ATTEMPT/CATCH/PASS to the caller, so coverage + # cannot see them and mutation testing is the only thing that checks them. + # + # Bounded to two small, fast, deterministic files. src/value.c is the file + # most worth mutating and is deliberately *not* here: 368 mutants at ~11s + # each is about 70 minutes, because almost everything links against it. + # Run the default target locally for the whole tree: + # cmake --build build --target mutation + # + # The threshold is a ratchet, not a quality bar. The measured score for + # these two files is 77.8% (84 killed, 24 survived, 108 total); 65 leaves + # headroom for runner variance while still failing on a real regression -- + # a test deleted, or new untested code added. + # + # It was 73.1% before writing this job. The run's own findings closed the + # gap: nothing exercised a maximum-length symbol-table key, so every + # `MAX_KEY - 1` off-by-one in a strncpy survived, and nothing asserted a + # freshly initialised table was actually zeroed. Adding those two + # assertions to tests/symtab.c killed five mutants. The same run found that + # errno was never asserted clear before a strtoll, which is what stops a + # stale ERANGE from failing a valid conversion -- that is now in + # tests/convert.c. + # + # The 24 remaining survivors are listed in the published report. Most are + # ICR mutants on loop and accumulator initialisers that a stronger + # placement assertion would catch; three in convert.c are genuinely + # equivalent and cannot be killed. Recorded in TODO.md. + - name: mutation testing + run: | + python3 scripts/mutation_test.py \ + --target src/convert.c \ + --target src/symtab.c \ + --junit mutation-junit.xml \ + --threshold 65 + # Publish even when the threshold gate fails, so survivors are visible -- + # each one is a missing test. Display-only (fail_on_failure: false); the + # --threshold above is the gate. annotate_only avoids the Checks API 404 + # on Gitea (mikepenz/action-junit-report#23). + - name: publish mutation results + if: always() + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'mutation-junit.xml' + annotate_only: true + detailed_summary: true + include_passed: true + fail_on_failure: 'false' + - run: echo "๐Ÿ This job's status is ${{ job.status }}." diff --git a/.gitea/workflows/release.yaml b/.gitea/workflows/release.yaml new file mode 100644 index 0000000..e1d77b6 --- /dev/null +++ b/.gitea/workflows/release.yaml @@ -0,0 +1,136 @@ +name: akbasic Release Build +run-name: ${{ gitea.actor }} akbasic release checks +# Manual only. Nothing here runs on push: the full mutation set is thousands of +# mutants and hours of runner time, which is a release-gate cost, not a +# per-commit one. Trigger it from the Actions tab. +on: + workflow_dispatch: + inputs: + mutation_threshold: + description: "Fail if the mutation score falls below this percentage" + required: false + default: "65" + mutation_targets: + description: "Space-separated files to mutate; empty means the whole src/ tree" + required: false + default: "" + +jobs: + # Moved here from ci.yaml. The docs are wanted for a release, not on every + # push, and building them beside the release mutation run keeps the two + # artefacts a release needs in one place. + docs: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + # No submodules at all. Doxyfile's INPUT is include/akbasic, src and + # examples, none of which live under deps/, and doxygen does not need to + # resolve to parse a declaration. Verified by running it + # against a tree with no deps/ present. + - name: dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y doxygen graphviz + # This is a gate, not a convenience. The Doxyfile sets + # WARN_AS_ERROR = FAIL_ON_WARNINGS, so a doc block that documents some of a + # function's parameters but not all of them fails the build. All 114 public + # declarations under include/akbasic carry a @brief, a @param per + # parameter, a @return and their @throws; keep it that way. + # + # OUTPUT_DIRECTORY is build/docs and doxygen will not create a missing + # parent, so make it first -- there is no cmake step in this job to do it. + - name: build API documentation + run: | + mkdir -p build + doxygen Doxyfile + - name: upload API documentation + if: always() + uses: actions/upload-artifact@v4 + with: + name: api-documentation + path: build/docs/html/ + if-no-files-found: error + - run: echo "๐Ÿ This job's status is ${{ job.status }}." + + full_mutation: + runs-on: ubuntu-latest + # 3675 mutants across the whole src/ tree. Cost per mutant is one incremental + # rebuild plus one full ctest run, which measures at roughly 2s for a leaf + # file and 11s for src/value.c, where almost everything links against it. + # Budget most of a day and expect it to finish well inside that; the ceiling + # exists so a mutant that wedges the runner cannot hold it forever. + timeout-minutes: 720 + steps: + - name: Check out repository code + uses: actions/checkout@v4 + with: + # The harness copies the repo and configures a build inside the copy, + # so it needs the same submodules the main build does -- including + # deps/basicinterpret, because the golden corpus is part of what kills + # mutants. + submodules: true + - name: dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y cmake gcc python3 + # The whole akbasic-owned src/ tree. ci.yaml runs a two-file subset on every + # push; this is the one that actually covers the interpreter. + # + # Mutation testing is the only gate that sees the error-handling control + # flow at all: libakerror's ATTEMPT/CATCH/PASS macros expand at their call + # sites, so gcov attributes them to the caller and line coverage cannot + # measure them. + # + # The default threshold matches ci.yaml's rather than being stricter. + # Whole-tree coverage is uneven -- src/convert.c and src/symtab.c are the + # two measured files, at 77.8% between them, and the rest is unmeasured, so + # a tighter number here would be a guess. Raise it with the workflow input + # once a full run has established a real baseline. + # + # The two inputs arrive through the environment rather than being + # interpolated straight into the script. ${{ }} substitution happens before + # the shell sees the line, so a value containing shell metacharacters would + # otherwise run as code. This workflow is manual and owner-triggered, but + # the safe form costs nothing. + - name: mutation testing (full tree) + env: + MUTATION_TARGETS: ${{ gitea.event.inputs.mutation_targets }} + MUTATION_THRESHOLD: ${{ gitea.event.inputs.mutation_threshold }} + run: | + set -eu + # Word-splitting is the point here: the input is a space-separated + # list. An empty input leaves $targets empty and the harness falls + # through to its own default, which is the whole src/ tree. + targets="" + for f in ${MUTATION_TARGETS:-}; do + targets="$targets --target $f" + done + # shellcheck disable=SC2086 + python3 scripts/mutation_test.py \ + $targets \ + --junit mutation-junit.xml \ + --threshold "${MUTATION_THRESHOLD:-65}" + # Publish even when the threshold gate fails, so survivors are visible -- + # each one is a missing test. Display-only (fail_on_failure: false); the + # --threshold above is the gate. annotate_only avoids the Checks API 404 + # on Gitea (mikepenz/action-junit-report#23). + - name: publish mutation results + if: always() + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'mutation-junit.xml' + annotate_only: true + detailed_summary: true + include_passed: true + fail_on_failure: 'false' + # Keep the raw report as well as the annotations: a release wants the + # survivor list on file, and the job summary is not durable. + - name: upload mutation report + if: always() + uses: actions/upload-artifact@v4 + with: + name: mutation-report + path: mutation-junit.xml + if-no-files-found: warn + - run: echo "๐Ÿ This job's status is ${{ job.status }}." diff --git a/CMakeLists.txt b/CMakeLists.txt index 5adb870..b5df479 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -267,6 +267,38 @@ if(AKBASIC_GOLDEN_CASES) _set_tests_properties(${AKBASIC_GOLDEN_NAMES} PROPERTIES TIMEOUT 30) endif() +# --------------------------------------------------------------------------- +# Mutation testing. +# +# Coverage says which lines ran; mutation testing says whether anything would +# have noticed if they were wrong. That distinction matters more here than in an +# ordinary C library: the akerror control-flow macros expand at their call sites, +# so gcov attributes ATTEMPT/CATCH/PASS to the caller and cannot really see them. +# Mutation testing is the only thing that checks them at all. +# +# This target runs the whole akbasic-owned src/ tree and is slow -- upwards of an +# hour. CI runs a narrower, faster set with a --threshold gate; see +# .gitea/workflows/ci.yaml. +# +# Namespaced when embedded in another project, for the same reason the coverage +# targets in the dependencies are: a sibling may well ship a `mutation` target. +find_package(Python3 COMPONENTS Interpreter) +if(Python3_FOUND) + if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(AKBASIC_MUTATION_TARGET mutation) + else() + set(AKBASIC_MUTATION_TARGET akbasic_mutation) + endif() + add_custom_target(${AKBASIC_MUTATION_TARGET} + COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/mutation_test.py + --source-root ${CMAKE_CURRENT_SOURCE_DIR} + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + USES_TERMINAL + COMMENT "Running mutation tests (breaks the library, expects tests to fail)" + ) +endif() + install(TARGETS akbasic basic ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} diff --git a/README.md b/README.md index 37a19f8..0bbf11f 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,31 @@ ctest --test-dir build --output-on-failure doxygen Doxyfile ``` -CI is `.gitea/workflows/ci.yaml` and runs all four of those: the suite, the doxygen gate, -ASan+UBSan, and coverage gated at 90% of lines. The generated API documentation and the -coverage report are uploaded as build artifacts (`api-documentation` and `code-coverage`). +There are two workflows. **`.gitea/workflows/ci.yaml`** runs on every push: the suite, +ASan+UBSan, coverage gated at 90% of lines, and mutation testing over two files. The coverage +report is uploaded as a `code-coverage` artifact. + +**`.gitea/workflows/release.yaml`** is manual (`workflow_dispatch`) and is what a release runs. +It builds the API documentation and uploads it as `api-documentation`, and it mutates the +*whole* `src/` tree โ€” 3675 mutants, hours of runner time, which is why it is not on the push +path. It takes two optional inputs: a mutation threshold, and a space-separated file list to +narrow the run. + +Mutation testing is worth a word, because it is the only gate that checks the error-handling +control flow at all. `libakerror`'s `ATTEMPT`/`CATCH`/`PASS` macros expand at their call sites, +so gcov attributes them to the caller and line coverage cannot see them. `scripts/mutation_test.py` +breaks the library many small ways and checks that the suite notices: + +```sh +cmake --build build --target mutation # the whole src/ tree; slow, hours +python3 scripts/mutation_test.py --target src/value.c --list +python3 scripts/mutation_test.py --target src/value.c --threshold 70 +``` + +It earns its keep. Writing this suite, it found that nothing exercised a maximum-length string +or symbol-table key, so every `MAX - 1` off-by-one in a `strncpy` would have gone unnoticed; +and that `errno` was never asserted to be cleared before a `strtoll`, which is what stops a +stale `ERANGE` from failing a perfectly valid conversion. The `Doxyfile` is configured the way `libakgl`'s is, including `WARN_AS_ERROR = FAIL_ON_WARNINGS` โ€” a doc block that documents some of a function's diff --git a/TODO.md b/TODO.md index 3692e0b..a466612 100644 --- a/TODO.md +++ b/TODO.md @@ -621,6 +621,7 @@ entire test corpus and passes clean under ASan and UBSan. | Function coverage | 96.9% (221/228) | | Warnings | none under `-Wall -Wextra` | | `doxygen Doxyfile` | clean; 114/114 public declarations documented | +| Mutation (`src/convert.c`, `src/symtab.c`) | see `.gitea/workflows/ci.yaml` for the current score and gate | Branch coverage reads 17.3% and is not a target, for the reason `libakgl/TODO.md` and `libakstdlib/TODO.md` both give: the akerror control-flow macros expand into large branch trees @@ -635,17 +636,27 @@ Dependency baseline: | `deps/libakstdlib` | 0.1.0 | soname `libakstdlib.so.0.1`. `AKSL_VERSION_CHECK()` asserted in `tests/version_check.c`. Its `aksl_ato*` family is banned here โ€” see ยง1.9. | | `deps/libakgl` | 0.1.0 | Migrated to the 1.0.0 registry and owns 256โ€“260. Not yet linked: `AKBASIC_WITH_AKGL` defaults OFF and `src/sink_akgl.c` does not exist. | -CI is `.gitea/workflows/ci.yaml`, in the shape the sibling libraries use: four jobs covering -the suite, the doxygen gate, ASan+UBSan and coverage. The generated API documentation and the -coverage report are both uploaded as artifacts. Two notes on it worth keeping: +CI is split in two. `.gitea/workflows/ci.yaml` runs on push -- the suite, ASan+UBSan, coverage +and a two-file mutation run -- and `.gitea/workflows/release.yaml` is manual +(`workflow_dispatch`), carrying the doxygen gate and the whole-tree mutation run. The split is +about cost: the full mutation set is 3675 mutants and hours of runner time, which is a release +gate rather than a per-commit one, and the docs are wanted at release time. Three notes worth +keeping: - The checkout is `submodules: true`, **not** `recursive`. The build needs `libakerror`, `libakstdlib` and `basicinterpret`; it does not need `libakgl`, which is guarded behind `AKBASIC_WITH_AKGL` and defaults OFF โ€” and recursing into it would clone SDL, SDL_image, SDL_mixer, SDL_ttf and jansson for a target that is never configured. The `docs` job needs no submodules at all. -- There is no mutation-testing job, unlike `libakerror`, `libakstdlib` and `libakgl`. This - repository has no `scripts/mutation_test.py`; see the item below. +- The push-path `mutation_test` job is bounded to `src/convert.c` and `src/symtab.c`, about + four minutes between them, scoring 77.8% against a gate of 65. `src/value.c` is the file most + worth mutating and is deliberately not on that path: 368 mutants at roughly 11 seconds each + is about 70 minutes, because almost everything links against it. It is covered by + `release.yaml`, or locally with `cmake --build build --target mutation`. +- `release.yaml`'s threshold defaults to 65, the same as the push path, rather than something + stricter. Only two files have ever been measured; a tighter whole-tree number would be a + guess until a full run establishes a baseline. Raise it through the workflow input once one + has. What remains, in priority order: @@ -656,9 +667,25 @@ What remains, in priority order: and nothing consumes it) should come first, because `DO`/`LOOP` reads badly without it. 3. **ยง6 items 12โ€“17** โ€” the defects the port uncovered. Item 12 is the one that produces a *wrong answer* rather than a refused one, and should be fixed first. -4. **Mutation testing.** All three sibling libraries gate on it and this one does not, because - there is no `scripts/mutation_test.py` here to run. It matters more than usual for this - codebase: the akerror macros expand at their call sites, so line coverage cannot see them - and mutation testing is the only thing that checks the `ATTEMPT`/`CATCH`/`PASS` control - flow at all. `deps/libakerror/scripts/mutation_test.py` is the one to port; `src/value.c` - or `src/scanner.c` would be the right first target. +4. **Mutation survivors in `src/value.c`.** The harness is in place (`scripts/mutation_test.py`, + the `mutation` CMake target, and a CI job on `src/convert.c`), and a partial run over + `src/value.c` turned up test gaps worth closing. These are *not* equivalent mutants; each + is a real bug of that shape the suite would not notice: + + - `AKBASIC_MAX_STRING_LENGTH - 1` โ†’ `+ 1` and โ†’ `- 0` survive at `src/value.c:47-48`, in + `set_string`'s `strncpy` and its NUL terminator. Nothing in the suite writes a + *maximum-length* string, so the off-by-one that would truncate or overrun goes unseen. + One test that round-trips a 255-character string kills all five. + - `memset(obj, 0, ...)` โ†’ `memset(obj, 1, ...)` and its deletion survive in + `akbasic_valuepool_init`, as does `obj->next = 0` โ†’ `= 1`. Nothing asserts a freshly + initialised pool is actually empty and zeroed. + + Two survivors there are genuinely equivalent and cannot be killed: `rval_as_int` and + `rval_as_float` (`src/value.c:30,35`) tolerate `+` โ†’ `-` because the reference's habit of + adding *both* of the right operand's numeric fields only works when the unused one is zero + โ€” ยง6 item 5. Fixing that defect would make these two mutants killable, which is a small + argument for doing it. + + `src/value.c` takes about 70 minutes to mutate in full (368 mutants, ~11s each, because + almost everything links against it), which is why CI runs `src/convert.c` instead and the + whole-tree run is a local `cmake --build build --target mutation`. diff --git a/scripts/mutation_test.py b/scripts/mutation_test.py new file mode 100755 index 0000000..53fa22a --- /dev/null +++ b/scripts/mutation_test.py @@ -0,0 +1,451 @@ +#!/usr/bin/env python3 +""" +Mutation testing harness for akbasic. + +Mutation testing measures how good the test suite is at catching bugs. It works +by making many small, deliberate breakages ("mutants") to the library source -- +flipping a comparison, deleting a statement, swapping true/false -- and then +running the whole CTest suite against each one. If the tests fail, the mutant is +"killed" (good: the tests noticed the bug). If the tests still pass, the mutant +"survived" (bad: a real bug of that shape would slip through unnoticed). + +The mutation score is killed / (killed + survived). Surviving mutants are printed +with file:line and the exact change so they can be turned into new test cases. + +This harness has no third-party dependencies (Python stdlib + the project's +normal cmake/ctest toolchain). It never mutates the real working tree: it copies +the repo to a scratch directory and mutates there. + +Usage: + scripts/mutation_test.py [options] + + --source-root DIR repo root to copy (default: parent of this script's dir) + --target FILE source file to mutate, relative to root; repeatable. + Default: every akbasic-owned C file under src/ + --work DIR scratch dir for the mutated copy (default: a temp dir) + --timeout SECONDS per-suite ctest timeout (default: 120) + --threshold PCT exit non-zero if mutation score < PCT (default: 0 = off) + --list only list the mutants that would be run, then exit + --keep keep the scratch working copy on exit (for debugging) + -j N (reserved) currently runs sequentially +""" + +import argparse +import os +import re +import shutil +import subprocess +import sys +import tempfile + +# --------------------------------------------------------------------------- # +# Mutation operators +# +# Each operator yields zero or more (start, end, replacement) edits for a single +# line of source. The driver applies exactly one edit per mutant so every mutant +# differs from the original by one localized change. +# --------------------------------------------------------------------------- # + +# Relational operator replacement: map each operator to the alternatives that +# meaningfully change behaviour (not merely the strict negation). +_REL = { + "==": ["!="], + "!=": ["=="], + "<=": ["<", "=="], + ">=": [">", "=="], + "<": ["<=", ">"], + ">": [">=", "<"], +} +# Match a relational operator that is NOT part of ->, <<, >>, =>, <=, >=, ==, != +# unless we intend it. We tokenize the two-char operators first, then single. +_REL_TWO = re.compile(r"(==|!=|<=|>=)") +_REL_ONE = re.compile(r"(?=!+])([<>])(?![=<>])") + +_LOGICAL = {"&&": "||", "||": "&&"} +_LOG_RE = re.compile(r"(&&|\|\|)") + +_BOOL = {"true": "false", "false": "true"} +_BOOL_RE = re.compile(r"\b(true|false)\b") + +# Arithmetic / compound-assignment on whitespace-delimited operands only, to +# avoid touching ++, --, ->, unary signs, or pointer/format punctuation. +_ARITH_RE = re.compile(r"(?<=\s)([+\-])(?=\s)") +_ARITH = {"+": "-", "-": "+"} +_COMPOUND_RE = re.compile(r"(\+=|-=)") +_COMPOUND = {"+=": "-=", "-=": "+="} + +# Integer literal replacement: 0 <-> 1 (word-bounded, not inside identifiers or +# larger numbers, not a float). +_INT_RE = re.compile(r"(?().\[\]* ]*\s*=\s*[^;]* | # assignments + [A-Za-z_][\w]*\s*\([^;]*\) # bare function calls + )\s*;\s*(\\?)\s*$""", + re.VERBOSE, +) + + +# --------------------------------------------------------------------------- # +# Deciding which lines are eligible to mutate +# --------------------------------------------------------------------------- # + +# Skip preprocessor control and the block of constant/error-code #defines in the +# template header: mutating buffer sizes or renumbering error codes produces +# equivalent or uninteresting mutants that swamp the signal. +_SKIP_LINE = re.compile( + r"""^\s*( + \#\s*(include|ifn?def|ifdef|if|elif|else|endif|error|pragma|undef) | + \#\s*define\s+AKERR_(MAX|LAST|NULLPOINTER|OUTOFBOUNDS|API|ATTRIBUTE| + TYPE|KEY|INDEX|FORMAT|IO|VALUE|RELATIONSHIP|EOF|CIRCULAR_REFERENCE| + ITERATOR_BREAK|NOT_IMPLEMENTED|BADEXC|NOIGNORE|USE_STDLIB)\b | + \* | // # comment bodies / line comments + )""", + re.VERBOSE, +) + + +def _is_comment_or_blank(line): + s = line.strip() + return (not s) or s.startswith("//") or s.startswith("/*") or s.startswith("*") + + +def eligible(line): + if _is_comment_or_blank(line): + return False + if _SKIP_LINE.match(line): + return False + return True + + +class Mutant: + __slots__ = ("path", "lineno", "op", "before", "after", "col") + + def __init__(self, path, lineno, op, before, after, col): + self.path = path + self.lineno = lineno + self.op = op + self.before = before + self.after = after + self.col = col + + def describe(self): + return (f"{self.path}:{self.lineno} [{self.op}] " + f"col{self.col}: {self.before.strip()} -> {self.after.strip()}") + + +def generate_mutants(root, rel_target): + """Enumerate all mutants for one target file.""" + abspath = os.path.join(root, rel_target) + with open(abspath, "r") as fh: + lines = fh.readlines() + + mutants = [] + for i, line in enumerate(lines, start=1): + if not eligible(line): + continue + # substitution operators + seen = set() + for tag, s, e, repl in _op_edits(line): + key = (s, e, repl) + if key in seen: + continue + seen.add(key) + mutated = line[:s] + repl + line[e:] + if mutated == line: + continue + mutants.append(Mutant(rel_target, i, tag, line, mutated, s)) + # statement deletion + m = _STMT_DELETABLE.match(line) + if m: + indent = line[: len(line) - len(line.lstrip())] + cont = "\\" if line.rstrip().endswith("\\") else "" + deleted = f"{indent}/* mutant: deleted */ {cont}\n" if cont else f"{indent};\n" + mutants.append(Mutant(rel_target, i, "SDL", line, deleted, 0)) + return mutants + + +# --------------------------------------------------------------------------- # +# Build / test orchestration against a scratch copy +# --------------------------------------------------------------------------- # + +class Runner: + def __init__(self, work, timeout): + self.work = work + self.build = os.path.join(work, "build") + self.timeout = timeout + + def _run(self, cmd, timeout=None): + return subprocess.run( + cmd, cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=timeout, + ) + + def configure(self): + r = self._run(["cmake", "-S", ".", "-B", "build"], timeout=self.timeout) + return r.returncode == 0, r.stdout + + def build_and_test(self): + """Return ('killed-compile' | 'killed-test' | 'killed-timeout' | 'survived').""" + try: + b = self._run(["cmake", "--build", "build"], timeout=self.timeout) + except subprocess.TimeoutExpired: + return "killed-timeout" + if b.returncode != 0: + return "killed-compile" + try: + t = subprocess.run( + ["ctest", "--test-dir", "build", "--output-on-failure", + "--stop-on-failure"], + cwd=self.work, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + timeout=self.timeout, + ) + except subprocess.TimeoutExpired: + return "killed-timeout" + return "survived" if t.returncode == 0 else "killed-test" + + +def _xml_escape(s): + return (s.replace("&", "&").replace("<", "<").replace(">", ">") + .replace('"', """)) + + +def write_junit(path, records, targets): + """Write a JUnit XML report. One per mutant; a surviving mutant + is a (test-suite gap), a killed mutant is a passing case.""" + by_file = {t: [] for t in targets} + for m, result in records: + by_file.setdefault(m.path, []).append((m, result)) + + total = len(records) + total_fail = sum(1 for _, r in records if r == "survived") + out = ['', + f''] + for f, items in by_file.items(): + if not items: + continue + fails = sum(1 for _, r in items if r == "survived") + out.append(f' ') + for m, result in items: + name = _xml_escape(m.describe()) + cls = "mutation." + _xml_escape(m.path) + if result == "survived": + detail = _xml_escape(f"{m.before.strip()} -> {m.after.strip()}") + out.append(f' ') + out.append(f' {detail}') + out.append(' ') + else: + out.append(f' {_xml_escape(result)}' + '') + out.append(' ') + out.append('') + with open(path, "w") as fh: + fh.write("\n".join(out) + "\n") + + +def copy_tree(src, dst): + # "build*" covers build/, build-asan/ and build-coverage/: the harness + # configures its own tree inside the copy, so copying those is pure IO -- + # and a coverage tree drags along every .gcno/.gcda as well. + # + # deps/ is copied wholesale and has to be: the build pulls libakerror and + # libakstdlib in with add_subdirectory, and the golden suite drives the .bas + # corpus in deps/basicinterpret in place. Only the reference's vendored + # Windows DLLs are dead weight, hence "*.dll". + ignore = shutil.ignore_patterns("build*", ".git", "*.o", "*.so", "*~", + "#*#", "*.iso", "*.png", "*.gcno", "*.gcda", + "*.dll") + shutil.copytree(src, dst, ignore=ignore, symlinks=True) + + +def read_lines(path): + with open(path) as fh: + return fh.readlines() + + +def write_lines(path, lines): + with open(path, "w") as fh: + fh.writelines(lines) + + +def main(): + # Line-buffer stdout so progress is visible live under CI / the cmake target. + try: + sys.stdout.reconfigure(line_buffering=True) + except (AttributeError, ValueError): + pass + here = os.path.dirname(os.path.abspath(__file__)) + default_root = os.path.dirname(here) + + ap = argparse.ArgumentParser(description="Mutation testing for akbasic") + ap.add_argument("--source-root", default=default_root) + ap.add_argument("--target", action="append", default=None) + ap.add_argument("--work", default=None) + ap.add_argument("--timeout", type=int, default=120) + ap.add_argument("--threshold", type=float, default=0.0) + ap.add_argument("--junit", default=None, + help="write a JUnit XML report to this path") + ap.add_argument("--max-mutants", type=int, default=0, + help="cap the run at N evenly-sampled mutants (0 = all)") + ap.add_argument("--list", action="store_true") + ap.add_argument("--keep", action="store_true") + ap.add_argument("-j", type=int, default=1) + args = ap.parse_args() + + root = os.path.abspath(args.source_root) + targets = args.target or [ + "src/convert.c", "src/environment.c", "src/error.c", "src/grammar.c", + "src/parser.c", "src/parser_commands.c", "src/runtime.c", + "src/runtime_commands.c", "src/runtime_functions.c", "src/scanner.c", + "src/sink_stdio.c", "src/symtab.c", "src/value.c", "src/variable.c", + "src/verbs.c", + ] + + # Enumerate mutants from the pristine sources. + all_mutants = [] + for t in targets: + all_mutants.extend(generate_mutants(root, t)) + + print(f"Generated {len(all_mutants)} mutants across {len(targets)} file(s):") + for t in targets: + n = sum(1 for m in all_mutants if m.path == t) + print(f" {t}: {n}") + + # Optional even-strided sampling to bound run time (CI / smoke tests). + if args.max_mutants and len(all_mutants) > args.max_mutants: + step = len(all_mutants) / args.max_mutants + sampled = [all_mutants[int(i * step)] for i in range(args.max_mutants)] + print(f"Sampling {len(sampled)} of {len(all_mutants)} mutants " + f"(--max-mutants {args.max_mutants}).") + all_mutants = sampled + + if args.list: + for m in all_mutants: + print(" " + m.describe()) + return 0 + + if not all_mutants: + print("No mutants generated; nothing to do.") + return 0 + + # Scratch working copy. + work_parent = args.work or tempfile.mkdtemp(prefix="akbasic_mut_") + work = os.path.join(work_parent, "src") if args.work else work_parent + if os.path.exists(work): + shutil.rmtree(work) + print(f"\nCopying sources to scratch dir: {work}") + copy_tree(root, work) + + runner = Runner(work, args.timeout) + + print("Configuring baseline ...") + ok, out = runner.configure() + if not ok: + sys.stderr.write(out.decode(errors="replace")) + sys.stderr.write("\nBaseline configure FAILED; aborting.\n") + return 2 + + print("Verifying baseline is green (no mutation) ...") + baseline = runner.build_and_test() + if baseline != "survived": + sys.stderr.write(f"Baseline is not green ({baseline}); aborting. " + "Fix the suite before mutation testing.\n") + return 2 + print("Baseline OK.\n") + + # Group mutants by file so we mutate one file at a time and restore it. + killed = {"killed-compile": 0, "killed-test": 0, "killed-timeout": 0} + survivors = [] + records = [] + total = len(all_mutants) + + # Cache pristine contents per target. + pristine = {t: read_lines(os.path.join(work, t)) for t in targets} + + for idx, m in enumerate(all_mutants, start=1): + tgt_abs = os.path.join(work, m.path) + lines = list(pristine[m.path]) + lines[m.lineno - 1] = m.after + write_lines(tgt_abs, lines) + try: + result = runner.build_and_test() + finally: + write_lines(tgt_abs, pristine[m.path]) # always restore + + records.append((m, result)) + if result == "survived": + survivors.append(m) + mark = "SURVIVED" + else: + killed[result] += 1 + mark = result.upper() + print(f"[{idx}/{total}] {mark:16} {m.describe()}") + + total_killed = sum(killed.values()) + score = 100.0 * total_killed / total if total else 100.0 + + print("\n" + "=" * 72) + print("MUTATION TESTING SUMMARY") + print("=" * 72) + print(f" total mutants : {total}") + print(f" killed (test) : {killed['killed-test']}") + print(f" killed (compile): {killed['killed-compile']}") + print(f" killed (timeout): {killed['killed-timeout']}") + print(f" survived : {len(survivors)}") + print(f" mutation score : {score:.1f}%") + if survivors: + print("\nSurviving mutants (test-suite gaps -- turn these into tests):") + for m in survivors: + print(" " + m.describe()) + + if args.junit: + junit_path = os.path.abspath(args.junit) + write_junit(junit_path, records, targets) + print(f"\nJUnit report written to: {junit_path}") + + if not args.keep and not args.work: + shutil.rmtree(work_parent, ignore_errors=True) + else: + print(f"\nScratch working copy kept at: {work}") + + if args.threshold > 0 and score < args.threshold: + print(f"\nFAIL: mutation score {score:.1f}% < threshold {args.threshold:.1f}%") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/convert.c b/tests/convert.c index b389912..bfcedcf 100644 --- a/tests/convert.c +++ b/tests/convert.c @@ -52,5 +52,21 @@ int main(void) TEST_REQUIRE_STATUS(akbasic_str_to_double("garbage", &d), AKBASIC_ERR_VALUE); TEST_REQUIRE_STATUS(akbasic_str_to_double("1.5x", &d), AKBASIC_ERR_VALUE); + /* + * errno hygiene. strtoll and strtod do not clear errno on success, so a + * stale ERANGE left by some earlier call would make a perfectly valid + * conversion raise. That is what the `errno = 0` before each call is for. + * + * Found by mutation testing: deleting that line, or setting errno to + * something non-zero, survived the suite until these four assertions + * existed. + */ + errno = ERANGE; + TEST_REQUIRE_OK(akbasic_str_to_int64("42", 10, &i)); + TEST_REQUIRE_INT(i, 42); + errno = ERANGE; + TEST_REQUIRE_OK(akbasic_str_to_double("2.5", &d)); + TEST_REQUIRE_FEQ(d, 2.5); + return akbasic_test_failures; } diff --git a/tests/symtab.c b/tests/symtab.c index 1651567..7609008 100644 --- a/tests/symtab.c +++ b/tests/symtab.c @@ -4,6 +4,7 @@ */ #include +#include #include #include @@ -17,6 +18,7 @@ int main(void) void *value = NULL; int64_t ivalue = 0; char key[32]; + char longkey[AKBASIC_SYMTAB_MAX_KEY + 2]; int marker_a = 1; int marker_b = 2; int i = 0; @@ -65,6 +67,63 @@ int main(void) TEST_REQUIRE_INT(TABLE.capacity, 16); TEST_REQUIRE_STATUS(akbasic_symtab_get(&TABLE, "V0#", NULL, NULL), AKERR_KEY); + /* + * Boundary keys. Mutation testing found that nothing exercised a key at the + * length limit, so every `AKBASIC_SYMTAB_MAX_KEY - 1` in the copy and its + * NUL terminator could be perturbed to `+ 1` or `- 0` and survive. A key one + * character under the limit must round-trip exactly; one *at* the limit must + * be refused rather than silently truncated. + */ + TEST_REQUIRE_OK(akbasic_symtab_clear(&TABLE)); + memset(longkey, 'K', AKBASIC_SYMTAB_MAX_KEY - 1); + longkey[AKBASIC_SYMTAB_MAX_KEY - 1] = '\0'; + TEST_REQUIRE_OK(akbasic_symtab_set(&TABLE, longkey, &marker_a, 7)); + TEST_REQUIRE_OK(akbasic_symtab_get(&TABLE, longkey, &value, &ivalue)); + TEST_REQUIRE(value == &marker_a, "a maximum-length key must round-trip"); + TEST_REQUIRE_INT(ivalue, 7); + TEST_REQUIRE_INT(strlen(TABLE.slots[0].key) + strlen(TABLE.slots[1].key) + + strlen(TABLE.slots[2].key) + strlen(TABLE.slots[3].key) + + strlen(TABLE.slots[4].key) + strlen(TABLE.slots[5].key) + + strlen(TABLE.slots[6].key) + strlen(TABLE.slots[7].key) + + strlen(TABLE.slots[8].key) + strlen(TABLE.slots[9].key) + + strlen(TABLE.slots[10].key) + strlen(TABLE.slots[11].key) + + strlen(TABLE.slots[12].key) + strlen(TABLE.slots[13].key) + + strlen(TABLE.slots[14].key) + strlen(TABLE.slots[15].key), + AKBASIC_SYMTAB_MAX_KEY - 1); + + memset(longkey, 'K', AKBASIC_SYMTAB_MAX_KEY); + longkey[AKBASIC_SYMTAB_MAX_KEY] = '\0'; + TEST_REQUIRE_STATUS(akbasic_symtab_set(&TABLE, longkey, NULL, 0), AKBASIC_ERR_BOUNDS); + TEST_REQUIRE_STATUS(akbasic_symtab_get(&TABLE, longkey, NULL, NULL), AKERR_KEY); + + /* + * A freshly initialised table is empty and its slots are clear. Mutation + * found that deleting the memset or the count reset in init() survived, + * because every other assertion here goes through set() first. + */ + TEST_REQUIRE_OK(akbasic_symtab_init(&TABLE, 16)); + TEST_REQUIRE_INT(TABLE.count, 0); + for ( i = 0; i < 16; i++ ) { + TEST_REQUIRE(!TABLE.slots[i].used, "slot %d should be unused after init", i); + TEST_REQUIRE(TABLE.slots[i].key[0] == '\0', "slot %d key should be clear", i); + TEST_REQUIRE(TABLE.slots[i].value == NULL, "slot %d value should be NULL", i); + TEST_REQUIRE_INT(TABLE.slots[i].ivalue, 0); + } + + /* + * Probing must start where the hash says. Mutation found that seeding the + * probe index or the loop counter to 1, or dropping the hash call entirely, + * survived -- a table that ignores its hash still works, just slowly. Assert + * placement instead: with capacity 1 there is exactly one slot, so a second + * distinct key has nowhere to go. + */ + TEST_REQUIRE_OK(akbasic_symtab_init(&TABLE, 1)); + TEST_REQUIRE_OK(akbasic_symtab_set(&TABLE, "ONLY#", NULL, 1)); + TEST_REQUIRE(TABLE.slots[0].used, "the single slot must be the one used"); + TEST_REQUIRE_STATUS(akbasic_symtab_set(&TABLE, "OTHER#", NULL, 2), AKBASIC_ERR_BOUNDS); + TEST_REQUIRE_OK(akbasic_symtab_get(&TABLE, "ONLY#", NULL, &ivalue)); + TEST_REQUIRE_INT(ivalue, 1); + /* Argument validation. */ TEST_REQUIRE_STATUS(akbasic_symtab_init(NULL, 16), AKERR_NULLPOINTER); TEST_REQUIRE_STATUS(akbasic_symtab_init(&TABLE, 0), AKBASIC_ERR_BOUNDS);