diff --git a/AGENTS.md b/AGENTS.md index 81c42ca..8e89e5d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -64,6 +64,17 @@ Run one test while iterating, for example `ctest --test-dir build -R sprite --ou Run only the performance suites with `ctest --test-dir build -L perf --output-on-failure`, or leave them out of an ordinary run with `-LE perf`. They take about 30 seconds together, print a table of nanoseconds per operation, and fail only when a measurement exceeds a budget set at roughly ten times the recorded baseline. `AKGL_BENCH_SCALE` scales every iteration count — `AKGL_BENCH_SCALE=0.1 ctest --test-dir build -L perf` for a quick look — and below 1.0 the budgets are reported but not enforced. The recorded baseline and what it means are in `PERFORMANCE.md`. +Check for memory defects with `cmake --build build --target memcheck`, or +`scripts/memcheck.sh` directly — it takes ctest's selection flags, so +`scripts/memcheck.sh -R tilemap` and `scripts/memcheck.sh -LE perf` both work. +It runs the suites that already exist under valgrind (`ctest -T memcheck`) with +the headless drivers forced, and exits non-zero when valgrind finds a definite +leak, an invalid access, or a read of uninitialised memory — which `ctest -T +memcheck` on its own will not do. The whole run takes about thirty seconds +because the perf suites scale themselves down under valgrind. Suppressions for +third-party findings live in `scripts/valgrind.supp`; add one only when the +finding genuinely cannot be fixed from this repository. + Run mutation testing with `cmake --build build --target mutation`. For a quick smoke run, use `scripts/mutation_test.py --target src/tilemap.c --max-mutants 10`; the harness mutates only a scratch copy and excludes the intentionally failing character test. Generate HTML and Cobertura coverage reports with `cmake -S . -B build-coverage -DAKGL_COVERAGE=ON -DCMAKE_BUILD_TYPE=Debug`, then build and run CTest. Reports are written to `build-coverage/coverage/`; this mode requires `gcovr` and GCC or Clang. @@ -333,6 +344,13 @@ they are enforced only in an optimized build at full scale. When a change moves a number for a good reason, re-record the baseline in `PERFORMANCE.md` in the same commit and say why it moved. +These two suites are also the memory-check vehicle. `tests/benchutil.h` detects +valgrind from `LD_PRELOAD` and drops the scale to +`AKGL_BENCH_VALGRIND_SCALE`, so the same binaries walk every path once instead +of a hundred thousand times, and the timings a checked run prints are labelled +as meaningless. **Do not add memory-check suites**: a new path worth checking +belongs in a benchmark, where it gets both. + ## Commit & Pull Request Guidelines Recent commits use concise, imperative summaries such as `Fix a scale bug...` and often explain related changes in one sentence. Keep each commit scoped and describe user-visible behavior. Pull requests should summarize the change, identify affected modules, list the CMake/CTest commands run, and link relevant issues. Include screenshots only for rendering, map, or `charviewer` changes. diff --git a/CMakeLists.txt b/CMakeLists.txt index a18a4b8..05075d0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,6 +4,37 @@ cmake_minimum_required(VERSION 3.10) # Version field in akgl.pc. Bump it here and nowhere else. project(akgl VERSION 0.3.0 LANGUAGES C) +# Memory checking reuses the suites that already exist -- `ctest -T memcheck` +# runs every registered test under valgrind -- rather than adding programs of its +# own. The perf suites carry most of the weight there: they are the only things +# in the tree that load assets, draw a scene, and run a frame loop in one +# process, and tests/benchutil.h drops their iteration counts by three orders of +# magnitude when it finds itself under valgrind, which turns a benchmark into +# exactly the broad, once-through path coverage a leak check wants. +# +# These have to be set before include(CTest): that is what writes them into +# DartConfiguration.tcl, and a memcheck run reads them from there. +find_program(MEMORYCHECK_COMMAND valgrind) +# Set with FORCE, but only when empty. include(CTest) declares both of these as +# empty cache entries, so a build tree configured before this block existed has +# them already and a plain set(... CACHE ...) would be ignored -- the values +# would silently never reach DartConfiguration.tcl. Guarding on emptiness still +# leaves a deliberate -DMEMORYCHECK_SUPPRESSIONS_FILE=... alone. +if(NOT MEMORYCHECK_SUPPRESSIONS_FILE) + set(MEMORYCHECK_SUPPRESSIONS_FILE + "${CMAKE_CURRENT_SOURCE_DIR}/scripts/valgrind.supp" + CACHE FILEPATH "Suppressions for third-party findings the memcheck run cannot fix" FORCE) +endif() +# Definite losses only. "Still reachable" is every global SDL and FreeType keeps +# for the process lifetime and says nothing about libakgl; "possibly lost" is +# dominated by interior pointers into pools and thread stacks. Neither is worth +# the false positives. +if(NOT MEMORYCHECK_COMMAND_OPTIONS) + set(MEMORYCHECK_COMMAND_OPTIONS + "--leak-check=full --show-leak-kinds=definite --errors-for-leak-kinds=definite --track-origins=yes --num-callers=25" + CACHE STRING "Options passed to the memory checker" FORCE) +endif() + include(CTest) option(AKGL_COVERAGE "Instrument libakgl and generate coverage reports with CTest" OFF) @@ -229,9 +260,13 @@ endforeach() add_test(NAME semver_unit COMMAND akgl_test_semver_unit) +# TIMEOUT is generous because CTest applies the same property to `ctest -T +# memcheck`, where every suite runs under valgrind at something like twenty +# times its normal cost. The unit suites finish in well under a second each +# without it; the ceiling is there for the checked run, not the ordinary one. set_tests_properties( ${AKGL_TEST_SUITES} semver_unit - PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests" TIMEOUT 30 + PROPERTIES WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/tests" TIMEOUT 300 ) set_tests_properties( @@ -382,6 +417,26 @@ if(Python3_FOUND) ) endif() +# Memory checking runs the whole registered suite under valgrind. The wrapper +# script exists because `ctest -T memcheck` records defects and still exits 0, +# which cannot gate anything; it also forces the headless drivers, so the vendor +# GPU stack is never loaded and never has to be suppressed. +if(MEMORYCHECK_COMMAND) + if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + set(AKGL_MEMCHECK_TARGET memcheck) + else() + set(AKGL_MEMCHECK_TARGET akgl_memcheck) + endif() + add_custom_target(${AKGL_MEMCHECK_TARGET} + COMMAND ${CMAKE_COMMAND} -E env + AKGL_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR} + ${CMAKE_CURRENT_SOURCE_DIR}/scripts/memcheck.sh + WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} + USES_TERMINAL + COMMENT "Running every test suite under valgrind (slow; the perf suites scale themselves down)" + ) +endif() + install(FILES ${CMAKE_CURRENT_BINARY_DIR}/akgl.pc DESTINATION "lib/pkgconfig/") install(FILES ${CMAKE_CURRENT_BINARY_DIR}/include/akgl/version.h DESTINATION "include/akgl/") install(TARGETS akgl DESTINATION "lib/") diff --git a/PERFORMANCE.md b/PERFORMANCE.md index 68e8e76..ffe2748 100644 --- a/PERFORMANCE.md +++ b/PERFORMANCE.md @@ -331,8 +331,14 @@ nothing else in this tree did. dummy driver does not do the work a real device would. - **No controller input.** The event path is driven by SDL and has no gamepad to drive it under the dummy driver. -- **No memory profiling beyond the static footprint.** The leaks above were found - by counting pool slots, not by valgrind or ASan. A run under either would - likely find more. +- **No heap profiling beyond the static footprint.** Nothing here measures peak + or steady-state heap usage. Correctness of the heap *is* covered now, by + `cmake --build build --target memcheck`, which runs these same binaries under + valgrind — `tests/benchutil.h` cuts the iteration counts by three orders of + magnitude when it finds itself there, so a benchmark becomes a path-coverage + program. That run found six more defects, including a leak of the parsed JSON + document in every asset loader; they are in `TODO.md` under **Memory + checking**. What is still missing is a profile: how much a running game + actually holds at once, and how that grows over an hour. - **Single-threaded throughout.** The library is not thread-safe by design (`akgl_game_state_lock` guards one field), and nothing here tests contention. diff --git a/README.md b/README.md index 6b3a86b..e5cf6d3 100644 --- a/README.md +++ b/README.md @@ -388,3 +388,17 @@ AKGL_BENCH_SCALE=0.1 ctest --test-dir build -L perf # a tenth of the iterations Each measurement is the best of five runs and is held to a budget set at roughly ten times the recorded baseline, so a suite that turns red means an algorithmic regression rather than a busy machine. Budgets are enforced only in an optimized build at full scale; below `AKGL_BENCH_SCALE=1.0`, and in a coverage build, they are reported without failing. `PERFORMANCE.md` carries the recorded baseline, the frame budget it adds up to, and what the numbers say — including the raw-SDL control rows that separate what libakgl costs from what the rasterizer costs. The six defects the stress tests turned up, and the targets the numbers are measured against, are in `TODO.md` under **Performance**. + +## Memory checking + +`memcheck` runs the suites that already exist under valgrind rather than adding suites of its own. The perf binaries carry most of it: they are the only programs in the tree that load assets, draw a scene, and run a frame loop in one process, and `tests/benchutil.h` cuts their iteration counts by three orders of magnitude when it detects valgrind — a benchmark walks one path a hundred thousand times, a leak check wants every path walked once. + +```sh +cmake --build build --target memcheck # everything, about 30 seconds +scripts/memcheck.sh -R tilemap # one suite; ctest selection flags pass through +scripts/memcheck.sh -LE perf # skip the benchmarks +``` + +The run forces `SDL_VIDEO_DRIVER=dummy`, `SDL_RENDER_DRIVER=software` and `SDL_AUDIO_DRIVER=dummy` so the vendor GPU stack is never loaded — that removes thousands of unfixable findings inside the driver without suppressing anything. Definite leaks, invalid accesses and uninitialised reads are counted and set the exit status; "still reachable" is not, because that is what SDL and FreeType keep for the process lifetime. Suppressions for genuine third-party findings are in `scripts/valgrind.supp`. + +What it found the first time it ran is in `TODO.md` under **Memory checking**. diff --git a/TODO.md b/TODO.md index cc55c7c..39c8931 100644 --- a/TODO.md +++ b/TODO.md @@ -1101,6 +1101,149 @@ Notes on the ones worth arguing about: layers, several tilesets — would make target 13 measurable and would probably find something. +## Memory checking + +`cmake --build build --target memcheck` runs **the suites that already exist** +under valgrind — `ctest -T memcheck` with the headless drivers forced, wrapped by +`scripts/memcheck.sh` so that a finding is an exit status rather than a line in a +log nobody reads. There are no memory-check test programs, and there should never +be any: `tests/benchutil.h` notices that it is running under valgrind and divides +every benchmark's iteration count by two thousand, which turns the perf suites +into the broadest path coverage in the tree at a cost valgrind can survive. The +whole run is about thirty seconds. + +The two halves fit together on purpose. A benchmark is a program that walks one +path a hundred thousand times; a leak check wants every path walked once. Same +binaries, same registration, one flag apart. + +Third-party findings are suppressed in `scripts/valgrind.supp`, and the run +forces `SDL_VIDEO_DRIVER=dummy` / `SDL_RENDER_DRIVER=software` / +`SDL_AUDIO_DRIVER=dummy` so the vendor GPU stack is never loaded — that removes +thousands of unfixable findings inside `amdgpu_dri.so` without suppressing +anything at all. Only *definite* losses and invalid accesses are counted; "still +reachable" is what SDL and FreeType keep for the process lifetime and says +nothing about this library. + +### Defects the memory checker found + +Ordered by blast radius. Numbering continues the lists above. Every size below +is measured, not estimated. + +34. **Every JSON loader leaks its parsed document.** There are four + `json_load_file` calls in `src/` and not one `json_decref` anywhere in the + library, so the whole parsed tree — objects, hashtables, strings — is + abandoned on both the success and failure paths: + + | Loader | Site | Leaked per call | + |---|---|---:| + | `akgl_sprite_load_json` | `src/sprite.c:140` | ~1,500 bytes | + | `akgl_character_load_json` | `src/character.c:232` | ~2,150 bytes | + | `akgl_tilemap_load` | `src/tilemap.c:693` | ~9,000 bytes (2x2 fixture map) | + | `akgl_registry_load_properties` | `src/registry.c:134` | not exercised by any test | + + Blast radius: every asset load, forever. The map figure is for the 2x2 + fixture; a real map's JSON is the size of its layer data, so a 128x128 map + leaks on the order of a megabyte per load. A game that reloads a level on + death leaks a level's worth of JSON each time, and this is the one item in + this list that grows without bound. + + Fix: `json_decref(json)` in the `CLEANUP` block of each, which is where the + other resources are already released. It is four lines. What makes it worth + its own commit rather than a footnote is that each one needs a test proving + the document is released, and `akgl_registry_load_properties` has no test at + all yet. + +35. **`akgl_get_property` reads up to 4 KiB past the end of the property + value.** `src/registry.c:181` copies a fixed `AKGL_MAX_STRING_LENGTH` bytes + out of whatever `SDL_GetStringProperty` returns, and what it returns is an + `SDL_strdup` of the value — four bytes for `"0.0"`. Valgrind reports an + invalid read on **every call**, twelve of them in `tests/physics.c` alone, + because `akgl_physics_init_arcade` reads six properties and + `akgl_render_init2d` reads two more. + + This has been in `registry.h` as a `@note` about the copy being "a fixed + #AKGL_MAX_STRING_LENGTH bytes rather than the length" — filed as waste. It + is not waste, it is an out-of-bounds read: today it walks into the rest of + SDL's heap and returns garbage past the terminator, and on a value that + lands at the end of a page it is a segfault in a getter. + + Fix: bound the copy by `strlen` of the source, still capped at + `AKGL_MAX_STRING_LENGTH`. Touches `akgl_get_property` only, and the header + note becomes a description of correct behaviour instead of a confession. + Worth a test that stores a short property, reads it back, and asserts the + bytes after the terminator in the destination are untouched. + +36. **The savegame name tables read past the end of every registry key, and + write what they find into the file.** `akgl_game_save_actorname_iterator` + (`src/game.c:219`) writes `AKGL_ACTOR_MAX_NAME_LENGTH` — 128 — bytes + starting at the key SDL handed it, and SDL allocated that key to fit the + name. Valgrind catches it on a 40-byte allocation. The three sibling + iterators do the same thing at `src/game.c:248`, `src/game.c:280` and + `src/game.c:308`, with 128, 512 and 128 byte fixed widths; only the actor + one is reached by the current tests, because the other registries are empty + in the save roundtrip. + + Two consequences, and the second is the interesting one. The read can fault + if the key sits at the end of a page. And whatever it reads goes into the + save file, so a savegame contains up to 500 bytes of this process's heap per + registered object — anything that happened to be next to the key. That is a + file a player might send someone. + + Fix: copy the name into a zeroed fixed-width buffer and write that. It + pairs naturally with **Defects -> Known and still open** item 7, which is + the same tables disagreeing about their widths between writer and reader. + +37. **`akgl_controller_list_keyboards` leaks the array SDL gives it.** + `src/controller.c:188` calls `SDL_GetKeyboards`, which allocates, and never + calls `SDL_free` on the result. Four bytes per call in the test environment + — one keyboard id — but it is per call, and the function is shaped like + something a game calls when a device is hotplugged. + + Fix: one `SDL_free`, in a `CLEANUP` block so the early-return path is + covered too. The same question should be asked of every SDL enumeration in + `src/controller.c`; `SDL_GetGamepads` has the same contract and the dummy + driver reports no gamepads, so no test reaches it. + +38. **A font, once loaded, is never freed and cannot be.** + `akgl_text_loadfont` (`src/text.c:20`) opens a `TTF_Font`, puts the pointer + in `AKGL_REGISTRY_FONT`, and that is the last anyone can do about it: the + header exposes no way to close a font, and `SDL_Quit` destroying the + property registry drops the last reference. 10,523 bytes per font — 736 of + them SDL_ttf's, the rest FreeType's. + + This is bounded by how many fonts a game loads, so it is not the runaway + that item 34 is. It is still a gap in the API rather than only a leak: a + game that switches fonts between scenes, or a tool like `charviewer` that + reloads one while the user picks a size, has no way to give the old one + back. Fix: `akgl_text_unloadfont(char *name)` that clears the registry entry + and calls `TTF_CloseFont`, and a matching sweep at shutdown. + +39. **Vendored `deps/semver`'s own unit test leaks 188 bytes** across 16 blocks, + from the `calloc`s in `test_strcut_first` and `test_strcut_second` + (`deps/semver/semver_unit.c:8` and `:21`). Not libakgl's code and not + libakgl's to fix; recorded so that nobody re-diagnoses it, and because + `semver_unit` is registered as one of our CTest tests and so shows up in our + memcheck run. If it becomes noise, the answer is a suppression naming those + two functions, not a local edit to a vendored file. + +### Not defects, and why + +- **`tests/util.c` fixtures were reading uninitialised stack floats.** Three + null-pointer tests declared `SDL_FRect` and `point` fixtures without + initializing them and then made one real call with them at the end, which is + sixteen "conditional jump depends on uninitialised value" findings for a test + that is not about coordinates. The fixtures are zeroed now. The library was + never at fault; a memory checker that reports noise gets ignored, which is the + only reason this was worth touching. +- **"Still reachable" at exit is not counted.** SDL's global state, the hint + table, the property registry and FreeType's library instance are one per + process and are reclaimed by `SDL_Quit` / `TTF_Quit`. Counting them would + bury the six findings above under a hundred that mean nothing. +- **The GPU driver's findings are avoided rather than suppressed.** Running the + suite against the real driver produces thousands of findings inside + `amdgpu_dri.so`; running it headless produces none, and headless is what the + suites are written for anyway. + ## Build notes The vendored SDL satellite libraries are built into per-project subdirectories diff --git a/scripts/memcheck.sh b/scripts/memcheck.sh new file mode 100755 index 0000000..df8c9d9 --- /dev/null +++ b/scripts/memcheck.sh @@ -0,0 +1,108 @@ +#!/bin/bash +# +# Run the existing CTest suites under valgrind and report what leaked. +# +# scripts/memcheck.sh check every suite in ./build +# scripts/memcheck.sh -R tilemap check only the suites matching a regex +# scripts/memcheck.sh -LE perf skip the perf suites +# +# Anything after the build directory is handed straight to ctest, so every +# selection flag ctest understands works here. +# +# Environment: +# AKGL_BUILD_DIR build tree to check (default: build) +# AKGL_MEMCHECK_LOG directory for the per-suite valgrind logs +# (default: $AKGL_BUILD_DIR/Testing/Temporary) +# +# Exit status: 0 when valgrind found nothing, 1 when it found something, 2 on a +# usage or environment error. This is the part `ctest -T memcheck` will not do +# on its own -- it records defects and still exits 0, which is no use as a gate. + +set -eu + +AKGL_BUILD_DIR="${AKGL_BUILD_DIR:-build}" + +function memcheck_die() +{ + echo "memcheck: $*" >&2 + exit 2 +} + +# The suite runs headless on purpose. Loading the real GPU stack costs thousands +# of unfixable findings inside the vendor driver, and none of them are libakgl's. +# A suite that sets these hints itself is unaffected; the ones that do not -- +# tests/tilemap.c and tests/sprite.c -- pick them up from here. +export SDL_VIDEO_DRIVER="${SDL_VIDEO_DRIVER:-dummy}" +export SDL_RENDER_DRIVER="${SDL_RENDER_DRIVER:-software}" +export SDL_AUDIO_DRIVER="${SDL_AUDIO_DRIVER:-dummy}" + +command -v valgrind >/dev/null 2>&1 || memcheck_die "valgrind not found" +command -v ctest >/dev/null 2>&1 || memcheck_die "ctest not found" +[ -f "$AKGL_BUILD_DIR/CMakeCache.txt" ] || memcheck_die "$AKGL_BUILD_DIR is not a configured build tree" + +AKGL_MEMCHECK_LOG="${AKGL_MEMCHECK_LOG:-$AKGL_BUILD_DIR/Testing/Temporary}" + +# Old logs would be counted as this run's findings. CTest numbers them by test +# id, so a run narrowed with -R leaves the others behind. +rm -f "$AKGL_MEMCHECK_LOG"/MemoryChecker.*.log + +# The run itself is allowed to fail: a suite that valgrind makes slow enough to +# trip an assertion still produced a log worth reading, and the log is what +# decides the exit status below. +set +e +ctest --test-dir "$AKGL_BUILD_DIR" -T memcheck --output-on-failure "$@" +AKGL_CTEST_STATUS=$? +set -e + +# What counts as a finding. "Definitely lost" is a leak nobody can argue with; +# the invalid-access lines are memcheck telling us the library read or wrote +# outside an allocation, which is worse than a leak and easier to fix. +# +# "are definitely lost", not "definitely lost": the LEAK SUMMARY block ends every +# clean log with "definitely lost: 0 bytes in 0 blocks", and matching that +# reported a finding in all twenty-three suites, including the ones that never +# call malloc. +AKGL_MEMCHECK_PATTERN='are definitely lost|Invalid read|Invalid write|Invalid free|Mismatched free|Source and destination overlap|depends on uninitialised|Use of uninitialised' + +total=0 +echo +echo "== valgrind findings by suite ==" +for log in "$AKGL_MEMCHECK_LOG"/MemoryChecker.*.log; do + [ -f "$log" ] || continue + # CTest writes the test name into the log's first line as the command it ran. + suite=$(grep -m1 -oE 'akgl_test_[a-z_]+' "$log" || basename "$log") + count=$(grep -cE "$AKGL_MEMCHECK_PATTERN" "$log" || true) + total=$((total + count)) + if [ "$count" -gt 0 ]; then + printf '%-24s %3d finding(s) %s\n' "$suite" "$count" "$log" + fi +done + +if [ "$total" -eq 0 ]; then + echo "none" + echo + # A clean valgrind run and a failing test are different things: a suite can + # assert its way to a non-zero exit without leaking a byte, and saying + # "clean" and returning 0 for that would hide it. + if [ "$AKGL_CTEST_STATUS" -ne 0 ]; then + echo "memcheck: valgrind found nothing, but ctest exited $AKGL_CTEST_STATUS -- a suite failed on its own terms" >&2 + exit 1 + fi + echo "memcheck: clean ($AKGL_MEMCHECK_LOG)" + exit 0 +fi + +echo +echo "== library frames in those stacks, most frequent first ==" +# Keyed on the function name rather than the file, because src/physics.c and +# tests/physics.c are both "physics.c" in a valgrind frame and only one of them +# is the library. Anything a caller can reach is an akgl_ symbol by convention, +# which makes the prefix a usable filter; a static helper appears under whatever +# akgl_ function called it, one frame further down. +grep -hoE 'by 0x[0-9A-Fa-f]+: akgl_[A-Za-z0-9_]+ \([a-z_]+\.c:[0-9]+\)' \ + "$AKGL_MEMCHECK_LOG"/MemoryChecker.*.log 2>/dev/null | + sed -E 's/^by 0x[0-9A-Fa-f]+: //' | sort | uniq -c | sort -rn | head -20 + +echo +echo "memcheck: $total finding(s); full stacks in $AKGL_MEMCHECK_LOG" >&2 +exit 1 diff --git a/scripts/valgrind.supp b/scripts/valgrind.supp new file mode 100644 index 0000000..f31a7a2 --- /dev/null +++ b/scripts/valgrind.supp @@ -0,0 +1,56 @@ +# Valgrind suppressions for the libakgl memory-check run. +# +# Used by `cmake --build build --target memcheck` and by any hand-run +# `ctest -T memcheck`; see AGENTS.md -> "Memory checking". +# +# Every entry here is a deliberate decision that a finding belongs to somebody +# else's code and cannot be fixed from this repository. Nothing that libakgl +# allocates is suppressed, and nothing is suppressed merely because it is noisy. +# If a suppression starts hiding a real finding, delete it -- a false negative in +# a leak check is worth more than a quiet log. +# +# The memcheck target runs the suite headless (SDL_VIDEO_DRIVER=dummy, +# SDL_RENDER_DRIVER=software, SDL_AUDIO_DRIVER=dummy) precisely so that the GPU +# stack is never loaded. That removes thousands of unfixable findings inside +# amdgpu_dri.so / Mesa / libGLX without suppressing anything, which is why there +# are no driver entries below. Run the suite against a real driver and you will +# need them; do not add them here on that account. + +# SDL allocates its global state -- the hint table, the property registry, the +# log category array, the clipboard -- on first use and reclaims it in SDL_Quit. +# A test that fails before SDL_Quit, and every test that deliberately leaves SDL +# up so a later assertion can look at it, leaves those blocks behind. They are +# one-per-process, not per-call, so they cannot accumulate into a leak. +{ + sdl3-global-state-still-reachable + Memcheck:Leak + match-leak-kinds: reachable + ... + fun:SDL_InitSubSystem_REAL +} + +# dlopen keeps the link map and the loaded objects' TLS blocks for the process +# lifetime; a dlclose does not return them. SDL loads its video, audio, and +# render backends this way, and SDL_image and SDL_mixer load their codecs the +# same way. +{ + dl-open-keeps-its-link-map + Memcheck:Leak + match-leak-kinds: reachable,possible + ... + fun:_dl_open +} + +# FreeType, reached through SDL_ttf, caches per-face and per-size structures +# inside the library instance and frees them in FT_Done_FreeType, which SDL_ttf +# calls from TTF_Quit. A suite that opens a font and exits without TTF_Quit +# reports them; a font libakgl itself failed to close is a different finding and +# is NOT covered here -- it appears under akgl_text_loadfont, not under +# FT_Init_FreeType. +{ + freetype-library-instance + Memcheck:Leak + match-leak-kinds: reachable + ... + fun:FT_Init_FreeType +} diff --git a/tests/benchutil.h b/tests/benchutil.h index defbc04..f8a0446 100644 --- a/tests/benchutil.h +++ b/tests/benchutil.h @@ -33,6 +33,16 @@ * Set `AKGL_BENCH_SCALE` in the environment to change how long the suite runs: * 0.1 for a tenth of the iterations, 10 for ten times as many. Below 1.0 the * budgets are not enforced, because a short run is a noisy one. + * + * **Under valgrind these suites become the memory-check suites**, and the scale + * drops itself to #AKGL_BENCH_VALGRIND_SCALE without being asked. That is the + * whole reason `ctest -T memcheck` does not need benchmark programs of its own: + * a leak check wants every path walked *once*, and a benchmark is a program that + * walks one path a hundred thousand times. Divide the iteration counts by two + * thousand and the same binary is exactly the right shape for memcheck -- + * broader coverage of asset loading, drawing, and the frame loop than any unit + * suite, at a runtime valgrind can survive. Timings from such a run are + * meaningless and the report says so. */ #ifndef _AKGL_BENCHUTIL_H_ @@ -54,6 +64,8 @@ #define AKGL_BENCH_REPETITIONS 5 /** @brief Environment variable scaling every iteration count. */ #define AKGL_BENCH_SCALE_ENV "AKGL_BENCH_SCALE" +/** @brief Scale forced on when running under valgrind. Enough to walk every path, few enough to finish. */ +#define AKGL_BENCH_VALGRIND_SCALE 0.0005 /** @brief One row of the report: what was measured, how fast, and what it was allowed to cost. */ typedef struct akgl_Benchmark { @@ -75,10 +87,38 @@ static akgl_Benchmark *bench_running = NULL; /** @brief `SDL_GetTicksNS()` at the last bench_start(). */ static uint64_t bench_started_ns = 0; +/** + * @brief Report whether this process is running under valgrind. + * + * Detected from `LD_PRELOAD`, which valgrind fills with its own + * `vgpreload_*.so` before handing the process over. That is a deliberate choice + * over `RUNNING_ON_VALGRIND` from `valgrind/valgrind.h`: the macro is exact, but + * it makes the test suite fail to compile anywhere the valgrind headers are not + * installed, and a benchmark is not worth a build dependency. The cost of the + * check being wrong is a slow run or a fast one, never a wrong answer. + */ +static bool bench_under_valgrind(void) +{ + static int detected = -1; + const char *preload = NULL; + + if ( detected >= 0 ) { + return ( detected == 1 ); + } + detected = 0; + preload = SDL_getenv("LD_PRELOAD"); + if ( preload != NULL && strstr(preload, "vgpreload") != NULL ) { + detected = 1; + } + return ( detected == 1 ); +} + /** * @brief Multiplier applied to every iteration count, from `AKGL_BENCH_SCALE`. * * Read once and cached. Anything unparseable, negative, or absent gives 1.0. + * Under valgrind the result is capped at #AKGL_BENCH_VALGRIND_SCALE -- the + * smaller of the two wins, so asking for an even shorter run still works. */ static double bench_scale(void) { @@ -96,6 +136,9 @@ static double bench_scale(void) scale = 1.0; } } + if ( bench_under_valgrind() == true && scale > AKGL_BENCH_VALGRIND_SCALE ) { + scale = AKGL_BENCH_VALGRIND_SCALE; + } return scale; } @@ -117,11 +160,15 @@ static int bench_iterations(int count) /** * @brief Report whether budgets are being enforced in this run. * - * An unoptimized build measures the instrumentation rather than the library, - * and a scaled-down run is too short to trust, so both report without failing. + * An unoptimized build measures the instrumentation rather than the library, a + * scaled-down run is too short to trust, and a run under valgrind is measuring + * valgrind. All three report without failing. */ static bool bench_budgets_enforced(void) { + if ( bench_under_valgrind() == true ) { + return false; + } #ifdef __OPTIMIZE__ return ( bench_scale() >= 1.0 ); #else @@ -229,7 +276,11 @@ static int bench_report(void) char *verdict = NULL; printf("\n"); - printf("scale %.2fx, best of %d runs, budgets %s\n", + if ( bench_under_valgrind() == true ) { + printf("running under valgrind: this is a memory check, not a measurement.\n"); + printf("The timings below are valgrind's and mean nothing about libakgl.\n"); + } + printf("scale %.4gx, best of %d runs, budgets %s\n", bench_scale(), AKGL_BENCH_REPETITIONS, enforced ? "enforced" : "reported only"); diff --git a/tests/util.c b/tests/util.c index ecf74e8..d1d33a4 100644 --- a/tests/util.c +++ b/tests/util.c @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -28,9 +29,14 @@ static int live_error_contexts(void) akerr_ErrorContext *test_akgl_rectangle_points_nullpointers(void) { RectanglePoints points; - SDL_FRect testrect; + // Zeroed for the same reason as the fixtures in + // test_akgl_collide_point_rectangle_nullpointers: the last case here is a + // real call, and feeding it stack garbage is noise under `memcheck`. + SDL_FRect testrect = {.x = 0, .y = 0, .w = 0, .h = 0}; PREPARE_ERROR(errctx); + memset((void *)&points, 0x00, sizeof(RectanglePoints)); + ATTEMPT { CATCH(errctx, akgl_rectangle_points(NULL, NULL)); FAIL_BREAK(errctx, AKGL_ERR_BEHAVIOR, "akgl_rectangle_points fails to FAIL with all NULL pointers"); @@ -102,12 +108,18 @@ akerr_ErrorContext *test_akgl_rectangle_points_math(void) akerr_ErrorContext *test_akgl_collide_point_rectangle_nullpointers(void) { - point testpoint; + // Zeroed rather than left as whatever the stack held. The last case in this + // function is a real call with real arguments, and reading uninitialised + // floats out of it is sixteen findings under `memcheck` for a test that is + // not about coordinates at all. + point testpoint = { .x = 0, .y = 0 }; RectanglePoints testrectpoints; - bool testcollide; + bool testcollide = false; PREPARE_ERROR(errctx); + memset(&testrectpoints, 0x00, sizeof(RectanglePoints)); + ATTEMPT { CATCH(errctx, akgl_collide_point_rectangle(&testpoint, &testrectpoints, NULL)); FAIL_BREAK(errctx, AKGL_ERR_BEHAVIOR, "akgl_collide_point_rectangle(*, *, NULL) failed"); @@ -182,9 +194,9 @@ akerr_ErrorContext *test_akgl_collide_point_rectangle_logic(void) akerr_ErrorContext *test_akgl_collide_rectangles_nullpointers(void) { - SDL_FRect testrect1; - SDL_FRect testrect2; - bool testcollide; + SDL_FRect testrect1 = {.x = 0, .y = 0, .w = 0, .h = 0}; + SDL_FRect testrect2 = {.x = 0, .y = 0, .w = 0, .h = 0}; + bool testcollide = false; PREPARE_ERROR(errctx);