name: akbasic CI Build run-name: ${{ gitea.actor }} akbasic test on: [push] # Push-triggered checks only. The doxygen gate and the whole-tree mutation run # live in release.yaml, which is manual (workflow_dispatch) because between them # they cost hours of runner time and are release gates rather than per-commit # ones. jobs: cmake_build: runs-on: ubuntu-latest steps: - run: echo "Triggered by ${{ gitea.event_name }} from ${{ gitea.repository }}@${{ gitea.ref }}. Building on ${{ runner.os }}." - name: Check out repository code uses: actions/checkout@v4 with: # Not recursive, deliberately. The top-level build needs # deps/libakerror and deps/libakstdlib (via add_subdirectory) and # deps/basicinterpret (the golden corpus is driven in place from it). # It does *not* need deps/libakgl, which is guarded behind # AKBASIC_WITH_AKGL and defaults OFF -- and recursing into it would # clone SDL, SDL_image, SDL_mixer, SDL_ttf and jansson for a target # that is never configured. libakstdlib's own nested deps/libakerror is # skipped for the same reason: our CMakeLists declares akerror::akerror # first and libakstdlib guards on if(NOT TARGET ...). submodules: true - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc - name: build run: | cmake -S . -B build cmake --build build --parallel # The suite is 61 cases: 41 golden files byte-compared against the Go # reference's own corpus, 17 unit tests, 2 embedding examples, and 1 # known-failing test that asserts the *correct* contract for defects # carried over from the reference (TODO.md section 6). A green run # therefore does not mean defect-free -- see AKBASIC_KNOWN_FAILING_TESTS. # # --output-junit resolves relative to the test dir, so give an absolute # path to land the report in the workspace root for the reporter below. - name: test (JUnit) run: ctest --test-dir build --output-on-failure --output-junit "$(pwd)/ctest-junit.xml" # annotate_only: true skips creating a check run via the Checks API, which # Gitea does not support and 404s on (mikepenz/action-junit-report#23). # Results surface via the job summary instead. - name: publish test results if: always() uses: mikepenz/action-junit-report@v4 with: report_paths: 'ctest-junit.xml' annotate_only: true detailed_summary: true include_passed: true fail_on_failure: 'true' - run: echo "🍏 This job's status is ${{ job.status }}." sanitizers: runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 with: submodules: true - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc # The whole suite under ASan and UBSan, golden files included. This is the # gate libakstdlib's TODO.md section 1 calls its highest-value missing item # -- worth having here from the start, because this library is all fixed # pools and manual buffer arithmetic, which is exactly what it catches. - name: build and test under ASan + UBSan run: | cmake -S . -B build-asan \ -DAKBASIC_SANITIZE=ON \ -DCMAKE_BUILD_TYPE=Debug cmake --build build-asan --parallel ctest --test-dir build-asan --output-on-failure - run: echo "🍏 This job's status is ${{ job.status }}." coverage: runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 with: submodules: true - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc gcovr # The gate is a ratchet, not a target: src/ sits at 92.3% of lines and # 96.9% of functions, so 90 fails on a real regression (a test deleted, or # new untested code added) without tripping over rounding. # # There is deliberately no branch gate. Branch coverage reads about 17% # and is not a meaningful number here, for the reason libakgl's and # libakstdlib's TODO.md both record: the akerror control-flow macros expand # into large branch trees at every call site, most of them unreachable in # normal operation. Gating on it would mean writing tests for libakerror's # macros, which is libakerror's mutation suite's job. # # --root . with a src/ filter keeps the dependency trees out of the report. # Note gcovr searches for .gcda under --root, so a stale instrumented build # left in the source directory would be folded in -- that is libakgl defect # #13, and the reason build*/ is gitignored rather than kept around. - name: coverage run: | cmake -S . -B build-coverage \ -DAKBASIC_COVERAGE=ON \ -DCMAKE_BUILD_TYPE=Debug cmake --build build-coverage --parallel ctest --test-dir build-coverage --output-on-failure mkdir -p build-coverage/coverage gcovr --root . --filter 'src/.*' \ --print-summary \ --html-details build-coverage/coverage/index.html \ --xml build-coverage/coverage/coverage.xml \ --fail-under-line 90 # Publish even when the threshold gate fails, so the uncovered lines are # visible -- each one is a missing test. - name: upload coverage reports if: always() uses: actions/upload-artifact@v4 with: name: code-coverage path: build-coverage/coverage/ if-no-files-found: warn - run: echo "🍏 This job's status is ${{ job.status }}." mutation_test: runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 with: # The harness copies the repo and configures a build inside the copy, # so it needs the same submodules the main build does -- including # deps/basicinterpret, because the golden corpus is part of what kills # mutants. submodules: true - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc python3 # Verify the tests actually catch bugs: break the library many ways and # confirm the suite fails. This matters more here than in an ordinary C # library, because the akerror control-flow macros expand at their call # sites -- gcov attributes ATTEMPT/CATCH/PASS to the caller, so coverage # cannot see them and mutation testing is the only thing that checks them. # # Bounded to two small, fast, deterministic files. src/value.c is the file # most worth mutating and is deliberately *not* here: 368 mutants at ~11s # each is about 70 minutes, because almost everything links against it. # Run the default target locally for the whole tree: # cmake --build build --target mutation # # The threshold is a ratchet, not a quality bar. The measured score for # these two files is 77.8% (84 killed, 24 survived, 108 total); 65 leaves # headroom for runner variance while still failing on a real regression -- # a test deleted, or new untested code added. # # It was 73.1% before writing this job. The run's own findings closed the # gap: nothing exercised a maximum-length symbol-table key, so every # `MAX_KEY - 1` off-by-one in a strncpy survived, and nothing asserted a # freshly initialised table was actually zeroed. Adding those two # assertions to tests/symtab.c killed five mutants. The same run found that # errno was never asserted clear before a strtoll, which is what stops a # stale ERANGE from failing a valid conversion -- that is now in # tests/convert.c. # # The 24 remaining survivors are listed in the published report. Most are # ICR mutants on loop and accumulator initialisers that a stronger # placement assertion would catch; three in convert.c are genuinely # equivalent and cannot be killed. Recorded in TODO.md. - name: mutation testing run: | python3 scripts/mutation_test.py \ --target src/convert.c \ --target src/symtab.c \ --junit mutation-junit.xml \ --threshold 65 # Publish even when the threshold gate fails, so survivors are visible -- # each one is a missing test. Display-only (fail_on_failure: false); the # --threshold above is the gate. annotate_only avoids the Checks API 404 # on Gitea (mikepenz/action-junit-report#23). - name: publish mutation results if: always() uses: mikepenz/action-junit-report@v4 with: report_paths: 'mutation-junit.xml' annotate_only: true detailed_summary: true include_passed: true fail_on_failure: 'false' - run: echo "🍏 This job's status is ${{ job.status }}."