From 43516c7e736d64cdfcf2861e3c7e39e7e4e156af Mon Sep 17 00:00:00 2001 From: Andrew Kesterson Date: Mon, 27 Jul 2026 20:51:28 -0400 Subject: [PATCH] Emit JUnit XML from tests + mutation, consume it in CI Produce machine-readable results and surface them in the Gitea pipeline: - ctest: run with --output-junit to write ctest-junit.xml. The path must be absolute ("$(pwd)/...") because --output-junit otherwise resolves relative to the --test-dir build directory. - mutation_test.py: new --junit FILE option writes a JUnit report where each mutant is a test case and a surviving mutant is a (so gaps show up as failing tests). - .gitea/workflows/ci.yaml: both jobs generate their XML and feed it to mikepenz/action-junit-report with `if: always()`, so results publish even when a gate fails. Mutation publishing is display-only (fail_on_failure: false); the --threshold flag remains the gate. - .gitignore: ignore the generated *-junit.xml artifacts. Co-Authored-By: Claude Opus 4.8 (1M context) --- .gitea/workflows/ci.yaml | 25 +++++++++++++++++--- .gitignore | 1 + scripts/mutation_test.py | 51 ++++++++++++++++++++++++++++++++++++++++ tests/MUTATION.md | 15 +++++++++++- 4 files changed, 88 insertions(+), 4 deletions(-) diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 45e6569..3d9fd18 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -13,13 +13,23 @@ jobs: run: | sudo apt-get update -y sudo apt-get install -y cmake gcc moreutils - - name: build and test + - name: build and install run: | mkdir installdir cmake -S . -B build -DCMAKE_INSTALL_PREFIX=installdir cmake --build build cmake --install build - cmake --build build --target test + # --output-junit resolves relative to the test dir, so give an absolute + # path to land the report in the workspace root for the reporter below. + - name: test (JUnit) + run: ctest --test-dir build --output-on-failure --output-junit "$(pwd)/ctest-junit.xml" + - name: publish test results + if: always() + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'ctest-junit.xml' + check_name: 'ctest results' + fail_on_failure: 'true' - run: echo "🍏 This job's status is ${{ job.status }}." mutation_test: @@ -38,5 +48,14 @@ jobs: # deeper (slower) coverage including the macro header. - name: mutation testing run: | - python3 scripts/mutation_test.py --target src/error.c --threshold 65 + python3 scripts/mutation_test.py --target src/error.c --junit mutation-junit.xml --threshold 65 + # Publish even when the threshold gate fails, so survivors are visible. + # Display-only (fail_on_failure: false); the --threshold above is the gate. + - name: publish mutation results + if: always() + uses: mikepenz/action-junit-report@v4 + with: + report_paths: 'mutation-junit.xml' + check_name: 'mutation results' + fail_on_failure: 'false' - run: echo "🍏 This job's status is ${{ job.status }}." diff --git a/.gitignore b/.gitignore index 94aab09..54ffe13 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ include/akerror.h build/ +*-junit.xml diff --git a/scripts/mutation_test.py b/scripts/mutation_test.py index deb3312..1b2828e 100755 --- a/scripts/mutation_test.py +++ b/scripts/mutation_test.py @@ -234,6 +234,48 @@ class Runner: return "survived" if t.returncode == 0 else "killed-test" +def _xml_escape(s): + return (s.replace("&", "&").replace("<", "<").replace(">", ">") + .replace('"', """)) + + +def write_junit(path, records, targets): + """Write a JUnit XML report. One per mutant; a surviving mutant + is a (test-suite gap), a killed mutant is a passing case.""" + by_file = {t: [] for t in targets} + for m, result in records: + by_file.setdefault(m.path, []).append((m, result)) + + total = len(records) + total_fail = sum(1 for _, r in records if r == "survived") + out = ['', + f''] + for f, items in by_file.items(): + if not items: + continue + fails = sum(1 for _, r in items if r == "survived") + out.append(f' ') + for m, result in items: + name = _xml_escape(m.describe()) + cls = "mutation." + _xml_escape(m.path) + if result == "survived": + detail = _xml_escape(f"{m.before.strip()} -> {m.after.strip()}") + out.append(f' ') + out.append(f' {detail}') + out.append(' ') + else: + out.append(f' {_xml_escape(result)}' + '') + out.append(' ') + out.append('') + with open(path, "w") as fh: + fh.write("\n".join(out) + "\n") + + def copy_tree(src, dst): ignore = shutil.ignore_patterns("build", ".git", "*.o", "*.so", "*~", "#*#", "*.iso", "*.png") @@ -265,6 +307,8 @@ def main(): ap.add_argument("--work", default=None) ap.add_argument("--timeout", type=int, default=120) ap.add_argument("--threshold", type=float, default=0.0) + ap.add_argument("--junit", default=None, + help="write a JUnit XML report to this path") ap.add_argument("--max-mutants", type=int, default=0, help="cap the run at N evenly-sampled mutants (0 = all)") ap.add_argument("--list", action="store_true") @@ -330,6 +374,7 @@ def main(): # Group mutants by file so we mutate one file at a time and restore it. killed = {"killed-compile": 0, "killed-test": 0, "killed-timeout": 0} survivors = [] + records = [] total = len(all_mutants) # Cache pristine contents per target. @@ -345,6 +390,7 @@ def main(): finally: write_lines(tgt_abs, pristine[m.path]) # always restore + records.append((m, result)) if result == "survived": survivors.append(m) mark = "SURVIVED" @@ -370,6 +416,11 @@ def main(): for m in survivors: print(" " + m.describe()) + if args.junit: + junit_path = os.path.abspath(args.junit) + write_junit(junit_path, records, targets) + print(f"\nJUnit report written to: {junit_path}") + if not args.keep and not args.work: shutil.rmtree(work_parent, ignore_errors=True) else: diff --git a/tests/MUTATION.md b/tests/MUTATION.md index 714dd1a..4192cd1 100644 --- a/tests/MUTATION.md +++ b/tests/MUTATION.md @@ -42,7 +42,20 @@ cmake --build build --target mutation Useful flags: `--timeout SECONDS` (per-suite build+test cap; a mutant that hangs is counted as killed), `--keep` (retain the scratch copy for debugging), -`--work DIR` (use a specific scratch directory). +`--work DIR` (use a specific scratch directory), `--junit FILE` (write a JUnit +XML report — surviving mutants appear as failing test cases). + +## CI reporting + +Both the unit tests and the mutation run emit JUnit XML that CI consumes: + +* `ctest --test-dir build --output-junit "$(pwd)/ctest-junit.xml"` — note the + absolute path; `--output-junit` otherwise resolves relative to the test dir. +* `scripts/mutation_test.py --junit mutation-junit.xml` + +`.gitea/workflows/ci.yaml` runs both and feeds the XML to +`mikepenz/action-junit-report` (with `if: always()`, so results publish even +when a gate fails). The generated `*-junit.xml` files are git-ignored. ## Mutation operators