name: akbasic CI Build run-name: ${{ gitea.actor }} akbasic test on: [push] # Push-triggered checks only. The doxygen gate and the whole-tree mutation run # live in release.yaml, which is manual (workflow_dispatch) because between them # they cost hours of runner time and are release gates rather than per-commit # ones. jobs: cmake_build: runs-on: ubuntu-latest steps: - run: echo "Triggered by ${{ gitea.event_name }} from ${{ gitea.repository }}@${{ gitea.ref }}. Building on ${{ runner.os }}." - name: Check out repository code uses: actions/checkout@v4 with: # Not recursive, deliberately. The top-level build needs # deps/libakerror and deps/libakstdlib (via add_subdirectory) and # deps/basicinterpret (the golden corpus is driven in place from it). # It does *not* need deps/libakgl, which is guarded behind # AKBASIC_WITH_AKGL and defaults OFF -- and recursing into it would # clone SDL, SDL_image, SDL_mixer, SDL_ttf and jansson for a target # that is never configured. libakstdlib's own nested deps/libakerror is # skipped for the same reason: our CMakeLists declares akerror::akerror # first and libakstdlib guards on if(NOT TARGET ...). submodules: true - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc - name: build run: | cmake -S . -B build cmake --build build --parallel # The suite is 70 cases: 41 golden files byte-compared against the Go # reference's own corpus, 5 local golden cases for verbs the reference # never implemented, 21 unit tests, 2 embedding examples, and 1 # known-failing test that asserts the *correct* contract for defects # carried over from the reference (TODO.md section 6). A green run # therefore does not mean defect-free -- see AKBASIC_KNOWN_FAILING_TESTS. # # AKBASIC_WITH_AKGL is off here, which is the point rather than an # omission: this job is what proves the interpreter builds and passes on a # machine with no SDL. The akgl_build job below covers the other half. # # --output-junit resolves relative to the test dir, so give an absolute # path to land the report in the workspace root for the reporter below. - name: test (JUnit) run: ctest --test-dir build --output-on-failure --output-junit "$(pwd)/ctest-junit.xml" # annotate_only: true skips creating a check run via the Checks API, which # Gitea does not support and 404s on (mikepenz/action-junit-report#23). # Results surface via the job summary instead. - name: publish test results if: always() uses: mikepenz/action-junit-report@v4 with: report_paths: 'ctest-junit.xml' annotate_only: true detailed_summary: true include_passed: true fail_on_failure: 'true' - run: echo "🍏 This job's status is ${{ job.status }}." sanitizers: runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 with: submodules: true - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc # The whole suite under ASan and UBSan, golden files included. This is the # gate libakstdlib's TODO.md section 1 calls its highest-value missing item # -- worth having here from the start, because this library is all fixed # pools and manual buffer arithmetic, which is exactly what it catches. - name: build and test under ASan + UBSan run: | cmake -S . -B build-asan \ -DAKBASIC_SANITIZE=ON \ -DCMAKE_BUILD_TYPE=Debug cmake --build build-asan --parallel ctest --test-dir build-asan --output-on-failure - run: echo "🍏 This job's status is ${{ job.status }}." coverage: runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 with: submodules: true - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc gcovr # The gate is a ratchet, not a target: src/ sits at 93.5% of lines and # 97.8% of functions, so 90 fails on a real regression (a test deleted, or # new untested code added) without tripping over rounding. # # The akbasic_akgl target is not in this figure. It is not built in a # default configuration, and instrumenting it would drag SDL into the # coverage job for four thin adaptors. # # There is deliberately no branch gate. Branch coverage reads about 18% # and is not a meaningful number here, for the reason libakgl's and # libakstdlib's TODO.md both record: the akerror control-flow macros expand # into large branch trees at every call site, most of them unreachable in # normal operation. Gating on it would mean writing tests for libakerror's # macros, which is libakerror's mutation suite's job. # # --root . with a src/ filter keeps the dependency trees out of the report. # Note gcovr searches for .gcda under --root, so a stale instrumented build # left in the source directory would be folded in -- that is libakgl defect # #13, and the reason build*/ is gitignored rather than kept around. - name: coverage run: | cmake -S . -B build-coverage \ -DAKBASIC_COVERAGE=ON \ -DCMAKE_BUILD_TYPE=Debug cmake --build build-coverage --parallel ctest --test-dir build-coverage --output-on-failure mkdir -p build-coverage/coverage gcovr --root . --filter 'src/.*' \ --print-summary \ --html-details build-coverage/coverage/index.html \ --xml build-coverage/coverage/coverage.xml \ --fail-under-line 90 # Publish even when the threshold gate fails, so the uncovered lines are # visible -- each one is a missing test. - name: upload coverage reports if: always() uses: actions/upload-artifact@v4 with: name: code-coverage path: build-coverage/coverage/ if-no-files-found: warn - run: echo "🍏 This job's status is ${{ job.status }}." akgl_build: runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 with: # submodules: true, *not* recursive, and then the libakgl dependencies # by hand in the next step. Recursive would work and costs an extra # 461 MB: it descends into SDL_image/external, SDL_mixer/external and # SDL_ttf/external, which are aom, dav1d, libjxl, libtiff, mpg123, # opus, flac, freetype, harfbuzz and a dozen more. None of them is # used -- every one configures as "Could NOT find" or falls back to # the system copy -- so cloning them is pure checkout time. submodules: true # libakgl builds its vendored SDL only when it is the top-level project; # embedded it takes a find_package path instead, so our CMakeLists declares # those targets first and needs the submodules present. Six of them, none # recursive. Filed upstream as libakgl API-gap item 5. - name: libakgl dependencies run: | git -C deps/libakgl submodule update --init \ deps/SDL deps/SDL_image deps/SDL_mixer deps/SDL_ttf \ deps/jansson deps/semver # libfreetype-dev and libharfbuzz-dev are load-bearing, not incidental. # SDL_ttf prefers the system copies -- it reports "Using system freetype # library" and links libfreetype.so.6 -- and without them it would reach # for deps/SDL_ttf/external/freetype, which the checkout above # deliberately does not clone. Installing two dev packages is much cheaper # than cloning freetype and harfbuzz. - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc g++ pkg-config \ libfreetype-dev libharfbuzz-dev # The akgl-backed half: the text sink and the graphics, audio and input # backends, plus the akgl_backends suite that drives them against a real # software renderer and reads the pixels back. # # It is a separate job rather than a flag on cmake_build because # AKBASIC_WITH_AKGL is off by default and that default is the point: the # interpreter and its whole 70-case suite build and pass on a machine with # no SDL. Keeping the two apart is what proves that claim on every push # rather than asserting it. # # Cost, measured rather than guessed: about 25 s to clone the six # submodules, 25 s to configure and 330 object files to compile -- roughly # a minute of CPU. Cheaper than it looks, because SDL3 compiles out almost # every backend it does not need here. - name: build with libakgl run: | cmake -S . -B build-akgl -DAKBASIC_WITH_AKGL=ON cmake --build build-akgl --parallel # Dummy video and audio drivers: the suite creates a 128x128 software # renderer and reads it back with SDL_RenderReadPixels, so it needs no # display, no sound card and no offscreen harness. Same approach # deps/libakgl/tests/draw.c takes. # # 71 cases: the 70 the default build runs, plus akgl_backends. The other # 70 are run here too on purpose -- they are the ones that must keep # passing when SDL *is* present, and a linker that picked up the wrong # akstdlib or akerror would show up here first. - name: test with libakgl env: SDL_VIDEODRIVER: dummy SDL_AUDIODRIVER: dummy run: ctest --test-dir build-akgl --output-on-failure --output-junit "$(pwd)/ctest-akgl-junit.xml" - name: publish test results if: always() uses: mikepenz/action-junit-report@v4 with: report_paths: 'ctest-akgl-junit.xml' annotate_only: true detailed_summary: true include_passed: true fail_on_failure: 'true' check_name: 'akgl test results' - run: echo "🍏 This job's status is ${{ job.status }}." mutation_test: runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 with: # The harness copies the repo and configures a build inside the copy, # so it needs the same submodules the main build does -- including # deps/basicinterpret, because the golden corpus is part of what kills # mutants. submodules: true - name: dependencies run: | sudo apt-get update -y sudo apt-get install -y cmake gcc python3 # Verify the tests actually catch bugs: break the library many ways and # confirm the suite fails. This matters more here than in an ordinary C # library, because the akerror control-flow macros expand at their call # sites -- gcov attributes ATTEMPT/CATCH/PASS to the caller, so coverage # cannot see them and mutation testing is the only thing that checks them. # # Bounded to two small, fast, deterministic files. src/value.c is the file # most worth mutating and is deliberately *not* here: 368 mutants at ~11s # each is about 70 minutes, because almost everything links against it. # Run the default target locally for the whole tree: # cmake --build build --target mutation # # The threshold is a ratchet, not a quality bar. The measured score for # these two files is 77.8% (84 killed, 24 survived, 108 total); 65 leaves # headroom for runner variance while still failing on a real regression -- # a test deleted, or new untested code added. # # It was 73.1% before writing this job. The run's own findings closed the # gap: nothing exercised a maximum-length symbol-table key, so every # `MAX_KEY - 1` off-by-one in a strncpy survived, and nothing asserted a # freshly initialised table was actually zeroed. Adding those two # assertions to tests/symtab.c killed five mutants. The same run found that # errno was never asserted clear before a strtoll, which is what stops a # stale ERANGE from failing a valid conversion -- that is now in # tests/convert.c. # # The 24 remaining survivors are listed in the published report. Most are # ICR mutants on loop and accumulator initialisers that a stronger # placement assertion would catch; three in convert.c are genuinely # equivalent and cannot be killed. Recorded in TODO.md. - name: mutation testing run: | python3 scripts/mutation_test.py \ --target src/convert.c \ --target src/symtab.c \ --junit mutation-junit.xml \ --threshold 65 # Publish even when the threshold gate fails, so survivors are visible -- # each one is a missing test. Display-only (fail_on_failure: false); the # --threshold above is the gate. annotate_only avoids the Checks API 404 # on Gitea (mikepenz/action-junit-report#23). - name: publish mutation results if: always() uses: mikepenz/action-junit-report@v4 with: report_paths: 'mutation-junit.xml' annotate_only: true detailed_summary: true include_passed: true fail_on_failure: 'false' - run: echo "🍏 This job's status is ${{ job.status }}."