Compare commits
2 Commits
ebec1f1621
...
52
| Author | SHA1 | Date | |
|---|---|---|---|
|
1532cb2d1d
|
|||
|
e20fd445a4
|
48
.gitea/workflows/mutation.yaml
Normal file
48
.gitea/workflows/mutation.yaml
Normal file
@@ -0,0 +1,48 @@
|
|||||||
|
name: akbasic Mutation Testing
|
||||||
|
run-name: ${{ gitea.actor }} akbasic mutation test
|
||||||
|
|
||||||
|
# Mutation testing is intentionally isolated from push and merge checks. The
|
||||||
|
# complete src/ tree takes hours and must not consume the shared CI capacity
|
||||||
|
# during normal development.
|
||||||
|
on:
|
||||||
|
# Gitea schedules use UTC. 07:00 UTC is 02:00 EST (the requested fixed
|
||||||
|
# Eastern Standard Time slot).
|
||||||
|
schedule:
|
||||||
|
- cron: '0 7 * * *'
|
||||||
|
workflow_dispatch:
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
full_mutation:
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
timeout-minutes: 720
|
||||||
|
steps:
|
||||||
|
- name: Check out repository code
|
||||||
|
uses: actions/checkout@v4
|
||||||
|
with:
|
||||||
|
submodules: true
|
||||||
|
- name: dependencies
|
||||||
|
run: |
|
||||||
|
sudo apt-get update -y
|
||||||
|
sudo apt-get install -y cmake gcc python3 moreutils
|
||||||
|
- name: mutation testing (full tree)
|
||||||
|
run: |
|
||||||
|
python3 scripts/mutation_test.py \
|
||||||
|
--junit mutation-junit.xml \
|
||||||
|
--threshold 65
|
||||||
|
- name: publish mutation results
|
||||||
|
if: always()
|
||||||
|
uses: mikepenz/action-junit-report@v4
|
||||||
|
with:
|
||||||
|
report_paths: 'mutation-junit.xml'
|
||||||
|
annotate_only: true
|
||||||
|
detailed_summary: true
|
||||||
|
include_passed: true
|
||||||
|
fail_on_failure: 'false'
|
||||||
|
- name: upload mutation report
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: mutation-report
|
||||||
|
path: mutation-junit.xml
|
||||||
|
if-no-files-found: warn
|
||||||
|
- run: echo "🍏 This job's status is ${{ job.status }}."
|
||||||
@@ -1,24 +1,13 @@
|
|||||||
name: akbasic Release Build
|
name: akbasic Release Build
|
||||||
run-name: ${{ gitea.actor }} akbasic release checks
|
run-name: ${{ gitea.actor }} akbasic release checks
|
||||||
# Manual only. Nothing here runs on push: the full mutation set is thousands of
|
# Manual only. Nothing here runs on push: documentation is a release gate, not
|
||||||
# mutants and hours of runner time, which is a release-gate cost, not a
|
# a per-commit check. The expensive mutation suite has its own daily schedule.
|
||||||
# per-commit one. Trigger it from the Actions tab.
|
|
||||||
on:
|
on:
|
||||||
workflow_dispatch:
|
workflow_dispatch:
|
||||||
inputs:
|
|
||||||
mutation_threshold:
|
|
||||||
description: "Fail if the mutation score falls below this percentage"
|
|
||||||
required: false
|
|
||||||
default: "65"
|
|
||||||
mutation_targets:
|
|
||||||
description: "Space-separated files to mutate; empty means the whole src/ tree"
|
|
||||||
required: false
|
|
||||||
default: ""
|
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
# Moved here from ci.yaml. The docs are wanted for a release, not on every
|
# Moved here from ci.yaml. The docs are wanted for a release, not on every
|
||||||
# push, and building them beside the release mutation run keeps the two
|
# push.
|
||||||
# artefacts a release needs in one place.
|
|
||||||
docs:
|
docs:
|
||||||
runs-on: ubuntu-latest
|
runs-on: ubuntu-latest
|
||||||
steps:
|
steps:
|
||||||
@@ -52,89 +41,3 @@ jobs:
|
|||||||
path: build/docs/html/
|
path: build/docs/html/
|
||||||
if-no-files-found: error
|
if-no-files-found: error
|
||||||
- run: echo "🍏 This job's status is ${{ job.status }}."
|
- run: echo "🍏 This job's status is ${{ job.status }}."
|
||||||
|
|
||||||
full_mutation:
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
# 3675 mutants across the whole src/ tree. Cost per mutant is one incremental
|
|
||||||
# rebuild plus one full ctest run, which measures at roughly 2s for a leaf
|
|
||||||
# file and 11s for src/value.c, where almost everything links against it.
|
|
||||||
# Budget most of a day and expect it to finish well inside that; the ceiling
|
|
||||||
# exists so a mutant that wedges the runner cannot hold it forever.
|
|
||||||
timeout-minutes: 720
|
|
||||||
steps:
|
|
||||||
- name: Check out repository code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
# The harness copies the repo and configures a build inside the copy,
|
|
||||||
# so it needs the same submodules the main build does. The golden
|
|
||||||
# corpus is part of what kills mutants and it is checked in now, so
|
|
||||||
# that is libakerror and libakstdlib and nothing else.
|
|
||||||
submodules: true
|
|
||||||
# moreutils for errno(1), which deps/libakerror's scripts/generrno.sh needs
|
|
||||||
# to generate its errno table. Without it the generated table is empty and
|
|
||||||
# every AKERR_* code collapses onto a low integer -- see the long note on
|
|
||||||
# ci.yaml's cmake_build job for what that breaks and how it presents.
|
|
||||||
- name: dependencies
|
|
||||||
run: |
|
|
||||||
sudo apt-get update -y
|
|
||||||
sudo apt-get install -y cmake gcc python3 moreutils
|
|
||||||
# The whole akbasic-owned src/ tree. ci.yaml runs a two-file subset on every
|
|
||||||
# push; this is the one that actually covers the interpreter.
|
|
||||||
#
|
|
||||||
# Mutation testing is the only gate that sees the error-handling control
|
|
||||||
# flow at all: libakerror's ATTEMPT/CATCH/PASS macros expand at their call
|
|
||||||
# sites, so gcov attributes them to the caller and line coverage cannot
|
|
||||||
# measure them.
|
|
||||||
#
|
|
||||||
# The default threshold matches ci.yaml's rather than being stricter.
|
|
||||||
# Whole-tree coverage is uneven -- src/symtab.c is the only file measured
|
|
||||||
# on the push path, at 74.1%, and the rest is unmeasured, so a tighter
|
|
||||||
# number here would be a guess. Raise it with the workflow input once a
|
|
||||||
# full run has established a real baseline.
|
|
||||||
#
|
|
||||||
# The two inputs arrive through the environment rather than being
|
|
||||||
# interpolated straight into the script. ${{ }} substitution happens before
|
|
||||||
# the shell sees the line, so a value containing shell metacharacters would
|
|
||||||
# otherwise run as code. This workflow is manual and owner-triggered, but
|
|
||||||
# the safe form costs nothing.
|
|
||||||
- name: mutation testing (full tree)
|
|
||||||
env:
|
|
||||||
MUTATION_TARGETS: ${{ gitea.event.inputs.mutation_targets }}
|
|
||||||
MUTATION_THRESHOLD: ${{ gitea.event.inputs.mutation_threshold }}
|
|
||||||
run: |
|
|
||||||
set -eu
|
|
||||||
# Word-splitting is the point here: the input is a space-separated
|
|
||||||
# list. An empty input leaves $targets empty and the harness falls
|
|
||||||
# through to its own default, which is the whole src/ tree.
|
|
||||||
targets=""
|
|
||||||
for f in ${MUTATION_TARGETS:-}; do
|
|
||||||
targets="$targets --target $f"
|
|
||||||
done
|
|
||||||
# shellcheck disable=SC2086
|
|
||||||
python3 scripts/mutation_test.py \
|
|
||||||
$targets \
|
|
||||||
--junit mutation-junit.xml \
|
|
||||||
--threshold "${MUTATION_THRESHOLD:-65}"
|
|
||||||
# Publish even when the threshold gate fails, so survivors are visible --
|
|
||||||
# each one is a missing test. Display-only (fail_on_failure: false); the
|
|
||||||
# --threshold above is the gate. annotate_only avoids the Checks API 404
|
|
||||||
# on Gitea (mikepenz/action-junit-report#23).
|
|
||||||
- name: publish mutation results
|
|
||||||
if: always()
|
|
||||||
uses: mikepenz/action-junit-report@v4
|
|
||||||
with:
|
|
||||||
report_paths: 'mutation-junit.xml'
|
|
||||||
annotate_only: true
|
|
||||||
detailed_summary: true
|
|
||||||
include_passed: true
|
|
||||||
fail_on_failure: 'false'
|
|
||||||
# Keep the raw report as well as the annotations: a release wants the
|
|
||||||
# survivor list on file, and the job summary is not durable.
|
|
||||||
- name: upload mutation report
|
|
||||||
if: always()
|
|
||||||
uses: actions/upload-artifact@v4
|
|
||||||
with:
|
|
||||||
name: mutation-report
|
|
||||||
path: mutation-junit.xml
|
|
||||||
if-no-files-found: warn
|
|
||||||
- run: echo "🍏 This job's status is ${{ job.status }}."
|
|
||||||
|
|||||||
@@ -118,6 +118,12 @@ typedef struct akbasic_Runtime
|
|||||||
{
|
{
|
||||||
akbasic_SourceLine source[AKBASIC_MAX_SOURCE_LINES];
|
akbasic_SourceLine source[AKBASIC_MAX_SOURCE_LINES];
|
||||||
|
|
||||||
|
/* Scratch owned by this runtime for RENUMBER and its target prescan. */
|
||||||
|
int16_t renumber_map[AKBASIC_MAX_SOURCE_LINES];
|
||||||
|
uint8_t renumber_visited[AKBASIC_MAX_SOURCE_LINES];
|
||||||
|
akbasic_SourceLine renumber_line;
|
||||||
|
char renumber_discard[AKBASIC_MAX_LINE_LENGTH * 2];
|
||||||
|
|
||||||
/* Pools. Nothing here is malloc'd; everything is drawn from and returned. */
|
/* Pools. Nothing here is malloc'd; everything is drawn from and returned. */
|
||||||
akbasic_Environment environments[AKBASIC_MAX_ENVIRONMENTS];
|
akbasic_Environment environments[AKBASIC_MAX_ENVIRONMENTS];
|
||||||
akbasic_Variable variables[AKBASIC_MAX_VARIABLES];
|
akbasic_Variable variables[AKBASIC_MAX_VARIABLES];
|
||||||
|
|||||||
@@ -72,7 +72,7 @@ struct akbasic_TargetWalk
|
|||||||
* rewritten: `GOTO 9999` in a program with no line 9999 is already broken, and
|
* rewritten: `GOTO 9999` in a program with no line 9999 is already broken, and
|
||||||
* inventing a destination for it would hide that.
|
* inventing a destination for it would hide that.
|
||||||
*/
|
*/
|
||||||
static int64_t mapped(const int64_t *map, int64_t line)
|
static int64_t mapped(const int16_t *map, int64_t line)
|
||||||
{
|
{
|
||||||
if ( line < 0 || line >= AKBASIC_MAX_SOURCE_LINES ) {
|
if ( line < 0 || line >= AKBASIC_MAX_SOURCE_LINES ) {
|
||||||
return line;
|
return line;
|
||||||
@@ -306,7 +306,7 @@ static akerr_ErrorContext *rewrite_line(akbasic_TargetWalk *walk, const char *co
|
|||||||
static akerr_ErrorContext *visit_renumber(akbasic_TargetWalk *walk, int64_t target, char *dest, size_t len)
|
static akerr_ErrorContext *visit_renumber(akbasic_TargetWalk *walk, int64_t target, char *dest, size_t len)
|
||||||
{
|
{
|
||||||
PREPARE_ERROR(errctx);
|
PREPARE_ERROR(errctx);
|
||||||
const int64_t *map = (const int64_t *)walk->self;
|
const int16_t *map = (const int16_t *)walk->self;
|
||||||
int written = 0;
|
int written = 0;
|
||||||
|
|
||||||
PASS(errctx, aksl_snprintf(&written, dest, len, "%" PRId64, mapped(map, target)));
|
PASS(errctx, aksl_snprintf(&written, dest, len, "%" PRId64, mapped(map, target)));
|
||||||
@@ -316,8 +316,7 @@ static akerr_ErrorContext *visit_renumber(akbasic_TargetWalk *walk, int64_t targ
|
|||||||
akerr_ErrorContext *akbasic_renumber(akbasic_Runtime *obj, int64_t newstart, int64_t increment, int64_t oldstart)
|
akerr_ErrorContext *akbasic_renumber(akbasic_Runtime *obj, int64_t newstart, int64_t increment, int64_t oldstart)
|
||||||
{
|
{
|
||||||
PREPARE_ERROR(errctx);
|
PREPARE_ERROR(errctx);
|
||||||
static int64_t map[AKBASIC_MAX_SOURCE_LINES];
|
int16_t *map = obj == NULL ? NULL : obj->renumber_map;
|
||||||
static akbasic_SourceLine rewritten[AKBASIC_MAX_SOURCE_LINES];
|
|
||||||
akbasic_TargetWalk walk = { map, visit_renumber };
|
akbasic_TargetWalk walk = { map, visit_renumber };
|
||||||
int64_t next = newstart;
|
int64_t next = newstart;
|
||||||
int64_t i = 0;
|
int64_t i = 0;
|
||||||
@@ -356,10 +355,8 @@ akerr_ErrorContext *akbasic_renumber(akbasic_Runtime *obj, int64_t newstart, int
|
|||||||
next += increment;
|
next += increment;
|
||||||
}
|
}
|
||||||
|
|
||||||
PASS(errctx, aksl_memset(rewritten, 0, sizeof(rewritten)));
|
PASS(errctx, aksl_memset(obj->renumber_visited, 0, sizeof(obj->renumber_visited)));
|
||||||
for ( i = 0; i < AKBASIC_MAX_SOURCE_LINES; i++ ) {
|
for ( i = 0; i < AKBASIC_MAX_SOURCE_LINES; i++ ) {
|
||||||
int64_t target = 0;
|
|
||||||
|
|
||||||
if ( obj->source[i].code[0] == '\0' ) {
|
if ( obj->source[i].code[0] == '\0' ) {
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
@@ -367,20 +364,62 @@ akerr_ErrorContext *akbasic_renumber(akbasic_Runtime *obj, int64_t newstart, int
|
|||||||
* Every line is rewritten, not just the moved ones: a line before
|
* Every line is rewritten, not just the moved ones: a line before
|
||||||
* `oldstart` can branch into the region that moved.
|
* `oldstart` can branch into the region that moved.
|
||||||
*/
|
*/
|
||||||
target = mapped(map, i);
|
|
||||||
PASS(errctx, rewrite_line(&walk, obj->source[i].code,
|
PASS(errctx, rewrite_line(&walk, obj->source[i].code,
|
||||||
rewritten[target].code, sizeof(rewritten[target].code)));
|
obj->renumber_line.code, sizeof(obj->renumber_line.code)));
|
||||||
rewritten[target].lineno = target;
|
obj->renumber_line.lineno = i;
|
||||||
/*
|
/*
|
||||||
* Every line comes out numbered, whether or not it went in that way.
|
* Every line comes out numbered, whether or not it went in that way.
|
||||||
* Asking for numbers is what RENUMBER is, and a program that has been
|
* Asking for numbers is what RENUMBER is, and a program that has been
|
||||||
* through it can be branched into by number -- which is the whole point
|
* through it can be branched into by number -- which is the whole point
|
||||||
* of running it over source that arrived without any.
|
* of running it over source that arrived without any.
|
||||||
*/
|
*/
|
||||||
rewritten[target].numbered = true;
|
obj->renumber_line.numbered = true;
|
||||||
|
PASS(errctx, aksl_memcpy(&obj->source[i], &obj->renumber_line,
|
||||||
|
sizeof(obj->source[i])));
|
||||||
}
|
}
|
||||||
|
|
||||||
PASS(errctx, aksl_memcpy(obj->source, rewritten, sizeof(obj->source)));
|
/* Move the already-rewritten lines in place. The map is a partial
|
||||||
|
* permutation: a chain ends at an empty slot, while a cycle closes back
|
||||||
|
* on its starting line. A single displaced line is sufficient for both. */
|
||||||
|
for ( i = 0; i < AKBASIC_MAX_SOURCE_LINES; i++ ) {
|
||||||
|
int64_t current = i;
|
||||||
|
akbasic_SourceLine displaced;
|
||||||
|
akbasic_SourceLine next_line;
|
||||||
|
|
||||||
|
if ( map[i] < 0 || obj->renumber_visited[i] ) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
PASS(errctx, aksl_memcpy(&displaced, &obj->source[i], sizeof(displaced)));
|
||||||
|
for ( ;; ) {
|
||||||
|
int64_t destination = map[current];
|
||||||
|
|
||||||
|
obj->renumber_visited[current] = 1;
|
||||||
|
if ( destination == i ) {
|
||||||
|
displaced.lineno = destination;
|
||||||
|
PASS(errctx, aksl_memcpy(&obj->source[destination], &displaced,
|
||||||
|
sizeof(displaced)));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if ( map[destination] < 0 ) {
|
||||||
|
displaced.lineno = destination;
|
||||||
|
PASS(errctx, aksl_memcpy(&obj->source[destination], &displaced,
|
||||||
|
sizeof(displaced)));
|
||||||
|
PASS(errctx, aksl_memset(&obj->source[current], 0,
|
||||||
|
sizeof(obj->source[current])));
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
PASS(errctx, aksl_memcpy(&next_line, &obj->source[destination],
|
||||||
|
sizeof(displaced)));
|
||||||
|
displaced.lineno = destination;
|
||||||
|
PASS(errctx, aksl_memcpy(&obj->source[destination], &displaced,
|
||||||
|
sizeof(obj->renumber_line)));
|
||||||
|
PASS(errctx, aksl_memset(&obj->source[current], 0,
|
||||||
|
sizeof(obj->source[current])));
|
||||||
|
PASS(errctx, aksl_memcpy(&displaced, &next_line,
|
||||||
|
sizeof(displaced)));
|
||||||
|
current = destination;
|
||||||
|
}
|
||||||
|
}
|
||||||
SUCCEED_RETURN(errctx);
|
SUCCEED_RETURN(errctx);
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -435,7 +474,6 @@ akerr_ErrorContext *akbasic_runtime_check_targets(akbasic_Runtime *obj)
|
|||||||
* step(). Nothing is read back out of it -- the walk needs somewhere to put
|
* step(). Nothing is read back out of it -- the walk needs somewhere to put
|
||||||
* the text it would have written, and this is it.
|
* the text it would have written, and this is it.
|
||||||
*/
|
*/
|
||||||
static char discard[AKBASIC_MAX_LINE_LENGTH * 2];
|
|
||||||
CheckState state = { NULL, 0 };
|
CheckState state = { NULL, 0 };
|
||||||
akbasic_TargetWalk walk = { &state, visit_check };
|
akbasic_TargetWalk walk = { &state, visit_check };
|
||||||
int64_t entry = 0;
|
int64_t entry = 0;
|
||||||
@@ -463,7 +501,8 @@ akerr_ErrorContext *akbasic_runtime_check_targets(akbasic_Runtime *obj)
|
|||||||
* the whole point of setting it.
|
* the whole point of setting it.
|
||||||
*/
|
*/
|
||||||
obj->environment->lineno = i;
|
obj->environment->lineno = i;
|
||||||
PASS(errctx, rewrite_line(&walk, obj->source[i].code, discard, sizeof(discard)));
|
PASS(errctx, rewrite_line(&walk, obj->source[i].code,
|
||||||
|
obj->renumber_discard, sizeof(obj->renumber_discard)));
|
||||||
}
|
}
|
||||||
/* Nothing was refused, so leave the cursor as the caller had it. */
|
/* Nothing was refused, so leave the cursor as the caller had it. */
|
||||||
obj->environment->lineno = entry;
|
obj->environment->lineno = entry;
|
||||||
|
|||||||
Reference in New Issue
Block a user