diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 5413c69..d3853b1 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -30,3 +30,48 @@ jobs: sudo cmake --install build ctest --test-dir build --output-on-failure - run: echo "๐Ÿ This job's status is ${{ job.status }}." + + mutation_test: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + with: + # The harness copies the repo and builds the copy top-level, so it + # needs deps/libakerror present just like the main job does. + submodules: recursive + - name: dependencies + run: | + sudo apt-get update -y + sudo apt-get install -y cmake gcc moreutils python3 + # Verify the tests actually catch bugs: break the library many ways and + # confirm the suite fails. Gated on src/stdlib.c (fast, deterministic); + # run the full default target locally for the macro header as well. + # + # The threshold is a ratchet, not a quality bar. The score on the section + # 1.0 suite is 46.8% (81/173 killed): the list and tree functions are + # covered, and everything else in src/stdlib.c -- the printf family, the + # ato* family, realpath, the memory and stream wrappers -- has no tests + # yet, so its mutants survive. 40 leaves headroom for the runner while + # still failing on a real regression (tests deleted, or new untested code + # added). Raise it as TODO.md sections 1.1-1.9 land. + - name: mutation testing + run: | + python3 scripts/mutation_test.py \ + --target src/stdlib.c \ + --junit mutation-junit.xml \ + --threshold 40 + # Publish even when the threshold gate fails, so survivors are visible -- + # each one is a missing test. Display-only (fail_on_failure: false); the + # --threshold above is the gate. annotate_only avoids the Checks API 404 + # on Gitea (mikepenz/action-junit-report#23). + - name: publish mutation results + if: always() + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'mutation-junit.xml' + annotate_only: true + detailed_summary: true + include_passed: true + fail_on_failure: 'false' + - run: echo "๐Ÿ This job's status is ${{ job.status }}." diff --git a/CMakeLists.txt b/CMakeLists.txt index 3b9d8a9..56cd6cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -156,12 +156,25 @@ if(AKSL_WILL_FAIL_TESTS OR AKSL_KNOWN_FAILING_TESTS) ) endif() +# Cap every test. The list and tree code is full of loops whose termination +# depends on a single condition, so a plausible bug -- or a mutant from the +# target below -- turns a test into an infinite loop. Without this, ctest waits +# forever; the mutation harness then kills ctest at its own timeout and the +# spinning test binary is left orphaned, holding the pipe open and burning a +# core. The whole suite runs in well under a second, so 30s is pure headroom. +set_tests_properties( + ${AKSL_TESTS} ${AKSL_WILL_FAIL_TESTS} ${AKSL_KNOWN_FAILING_TESTS} + PROPERTIES TIMEOUT 30 +) + # Mutation testing: break the library in small ways and confirm the test suite -# notices. This is a meta-check on the tests themselves, so it is a manual -# target (it rebuilds and re-runs the whole suite once per mutant), not a CTest -# test and not yet a CI gate -- the per-function tests in TODO.md sections -# 1.1-1.9 need to exist before a threshold would mean anything. +# notices. This is a meta-check on the tests themselves, and it rebuilds and +# re-runs the whole suite once per mutant, so it is a manual target rather than +# a CTest test: # cmake --build build --target mutation +# This target covers both src/stdlib.c and include/akstdlib.h. CI runs the +# narrower, faster src/stdlib.c set with a --threshold gate; see +# .gitea/workflows/ci.yaml. find_package(Python3 COMPONENTS Interpreter) if(Python3_FOUND) add_custom_target(mutation diff --git a/TODO.md b/TODO.md index fed814c..cfe4efb 100644 --- a/TODO.md +++ b/TODO.md @@ -56,10 +56,30 @@ against `tests/aksl_capture.h` and added to the `AKSL_TESTS` list. `aksl_realpath`, unbounded `vsprintf`, missing `va_end`). - [x] **Port `scripts/mutation_test.py` from libakerror.** Retargeted at `src/stdlib.c` and `include/akstdlib.h` (178 mutants) and exposed as - `cmake --build build --target mutation`. Deliberately *not* a CI gate yet, and no - `--threshold` is set: a mutation score only means something once ยง1.1โ€“1.9 exist, and - against today's tests it would do little but confirm that most of the library is - untested. + `cmake --build build --target mutation`. CI runs the narrower `src/stdlib.c` set (173 + mutants) as its own `mutation_test` job with `--threshold 40` and a JUnit report. + **Current score: 46.8%** โ€” 81 killed, 92 survived. The survivors are concentrated + exactly where ยง1.1โ€“1.9 have yet to be written: + + | surviving mutants | function | + |---|---| + | 10 | `aksl_tree_iterate` | + | 6 each | `aksl_fread`, `aksl_fwrite`, `aksl_fprintf`, `aksl_sprintf` | + | 5 | `aksl_printf` | + | 4 each | `aksl_memcpy`, `aksl_realpath`, `aksl_strhash_djb2`, `aksl_list_append` | + | 3 each | `aksl_malloc`, `aksl_free`, `aksl_memset`, `aksl_fopen`, `aksl_fclose`, `aksl_atoi`, `aksl_atol`, `aksl_atoll`, `aksl_atof` | + | 1 | `aksl_list_iterate` | + + The threshold is a regression ratchet, not a quality bar; raise it as sections below + land. Each surviving mutant is a concrete missing test โ€” `mutation-junit.xml` lists + them with `file:line` and the exact edit. +- [x] **Cap every test with a CTest `TIMEOUT`.** Found while measuring the above: a mutant + that deletes `fast = fast->next->next` turns `aksl_list_iterate` into an infinite + loop, so `ctest` waited forever, the mutation harness killed `ctest` at its own + 120 s timeout, and the orphaned test binary kept spinning at 100% CPU while holding + the pipe open โ€” the run wedged and had to be killed by hand. `TIMEOUT 30` on every + test makes `ctest` reap its own child, so those mutants are now killed in 30 s and + leave nothing behind. It also guards the real suite against the same class of bug. - [x] **CI could not have run any of this.** `actions/checkout@v4` was not fetching submodules, so `add_subdirectory(deps/libakerror)` had nothing to descend into and the configure step failed before ever reaching the tests. Added `submodules: recursive`,