/** * @file benchutil.h * @brief The timing harness behind the perf suites: run it, record it, report it, hold it to a budget. * * A benchmark here is a timed region: bench_start() takes a timestamp, * bench_stop() takes another and divides by the number of units of work that * happened in between. The unit is whatever the measurement is *about* -- one * call, one actor, one tile, one frame -- and it is printed alongside the * number so nobody has to guess what "42 ns" was 42 nanoseconds of. * * Two things make the numbers usable rather than merely present: * * 1. **Best of #AKGL_BENCH_REPETITIONS.** Calling bench_start() again with a * name already in the table folds the new run into the old entry and keeps * the *lowest* ns/op seen. A benchmark loop therefore repeats itself and the * harness reports the run that was interrupted least. The mean is the wrong * statistic here: every source of noise on a shared machine makes a run * slower and none makes it faster. * 2. **The timed region contains no error checking.** `PASS` and `CATCH` both * call `akerr_valid_error_address`, which walks `AKERR_ARRAY_ERROR` looking * for a match -- more work than several of the calls being measured. Use * #BENCH_LOOP, which stashes the context and stops at the first failure, and * hand it to `PASS` after the clock has stopped. * * A budget is a per-op ceiling in nanoseconds, and a benchmark that exceeds it * fails the suite. Budgets are set at roughly ten times the measured baseline: * loose enough that a busy machine does not turn CI red, tight enough that an * algorithmic regression -- a linear scan that becomes quadratic, a per-frame * allocation that becomes a per-actor one -- cannot hide. They are enforced * only in an optimized build at full scale; a coverage build measures the * instrumentation, not the library, and reports without judging. * * Set `AKGL_BENCH_SCALE` in the environment to change how long the suite runs: * 0.1 for a tenth of the iterations, 10 for ten times as many. Below 1.0 the * budgets are not enforced, because a short run is a noisy one. * * **Under valgrind these suites become the memory-check suites**, and the scale * drops itself to #AKGL_BENCH_VALGRIND_SCALE without being asked. That is the * whole reason `ctest -T memcheck` does not need benchmark programs of its own: * a leak check wants every path walked *once*, and a benchmark is a program that * walks one path a hundred thousand times. Divide the iteration counts by two * thousand and the same binary is exactly the right shape for memcheck -- * broader coverage of asset loading, drawing, and the frame loop than any unit * suite, at a runtime valgrind can survive. Timings from such a run are * meaningless and the report says so. */ #ifndef _AKGL_BENCHUTIL_H_ #define _AKGL_BENCHUTIL_H_ #include #include #include #include #include /** @brief How many benchmarks one suite can record. */ #define AKGL_BENCH_MAX_RESULTS 48 /** @brief Longest benchmark name kept, including the terminator. */ #define AKGL_BENCH_MAX_NAME 56 /** @brief Longest unit name kept, including the terminator. */ #define AKGL_BENCH_MAX_UNIT 12 /** @brief How many times a benchmark is repeated before the best run is reported. */ #define AKGL_BENCH_REPETITIONS 5 /** @brief Environment variable scaling every iteration count. */ #define AKGL_BENCH_SCALE_ENV "AKGL_BENCH_SCALE" /** @brief Scale forced on when running under valgrind. Enough to walk every path, few enough to finish. */ #define AKGL_BENCH_VALGRIND_SCALE 0.0005 /** @brief One row of the report: what was measured, how fast, and what it was allowed to cost. */ typedef struct akgl_Benchmark { char name[AKGL_BENCH_MAX_NAME]; /**< What was measured. Also the key runs are merged under. */ char unit[AKGL_BENCH_MAX_UNIT]; /**< What one op is: "call", "actor", "tile", "frame". */ double budget_ns; /**< Per-op ceiling in nanoseconds. 0 reports without judging. */ double best_ns; /**< Lowest ns/op seen across every run under this name. */ uint64_t ops; /**< Units of work in the run that produced best_ns. */ uint64_t elapsed_ns; /**< Wall time of that run. */ int runs; /**< How many runs were folded in. */ } akgl_Benchmark; /** @brief Every benchmark this suite has recorded, in the order it first ran. */ static akgl_Benchmark bench_results[AKGL_BENCH_MAX_RESULTS]; /** @brief How many entries of #bench_results are in use. */ static int bench_result_count = 0; /** @brief The entry bench_stop() will write to, claimed by bench_start(). */ static akgl_Benchmark *bench_running = NULL; /** @brief `SDL_GetTicksNS()` at the last bench_start(). */ static uint64_t bench_started_ns = 0; /** * @brief Report whether this process is running under valgrind. * * Detected from `LD_PRELOAD`, which valgrind fills with its own * `vgpreload_*.so` before handing the process over. That is a deliberate choice * over `RUNNING_ON_VALGRIND` from `valgrind/valgrind.h`: the macro is exact, but * it makes the test suite fail to compile anywhere the valgrind headers are not * installed, and a benchmark is not worth a build dependency. The cost of the * check being wrong is a slow run or a fast one, never a wrong answer. */ static bool bench_under_valgrind(void) { static int detected = -1; const char *preload = NULL; if ( detected >= 0 ) { return ( detected == 1 ); } detected = 0; preload = SDL_getenv("LD_PRELOAD"); if ( preload != NULL && strstr(preload, "vgpreload") != NULL ) { detected = 1; } return ( detected == 1 ); } /** * @brief Multiplier applied to every iteration count, from `AKGL_BENCH_SCALE`. * * Read once and cached. Anything unparseable, negative, or absent gives 1.0. * Under valgrind the result is capped at #AKGL_BENCH_VALGRIND_SCALE -- the * smaller of the two wins, so asking for an even shorter run still works. */ static double bench_scale(void) { static double scale = -1.0; const char *env = NULL; if ( scale >= 0.0 ) { return scale; } scale = 1.0; env = SDL_getenv(AKGL_BENCH_SCALE_ENV); if ( env != NULL ) { scale = SDL_atof(env); if ( scale <= 0.0 ) { scale = 1.0; } } if ( bench_under_valgrind() == true && scale > AKGL_BENCH_VALGRIND_SCALE ) { scale = AKGL_BENCH_VALGRIND_SCALE; } return scale; } /** * @brief Scale a nominal iteration count, never down to zero. * @param count The count the benchmark was written for. * @return @p count times the scale factor, at least 1. */ static int bench_iterations(int count) { int scaled = (int)((double)count * bench_scale()); if ( scaled < 1 ) { scaled = 1; } return scaled; } /** * @brief Report whether budgets are being enforced in this run. * * An unoptimized build measures the instrumentation rather than the library, a * scaled-down run is too short to trust, and a run under valgrind is measuring * valgrind. All three report without failing. */ static bool bench_budgets_enforced(void) { if ( bench_under_valgrind() == true ) { return false; } #ifdef __OPTIMIZE__ return ( bench_scale() >= 1.0 ); #else return false; #endif } /** * @brief Start timing, claiming or reusing the table entry named @p name. * * @param name What is being measured. Truncated at #AKGL_BENCH_MAX_NAME. * Reusing a name folds this run into that entry. * @param unit What one op is. Truncated at #AKGL_BENCH_MAX_UNIT. * @param budget_ns Per-op ceiling in nanoseconds; 0 to report only. The value * from the first run under a name wins. * * @note The table is fixed. Past #AKGL_BENCH_MAX_RESULTS entries the run is * dropped with a message rather than overwriting somebody else's row. */ static void bench_start(char *name, char *unit, double budget_ns) { int i = 0; bench_running = NULL; for ( i = 0; i < bench_result_count; i++ ) { if ( strncmp(bench_results[i].name, name, AKGL_BENCH_MAX_NAME - 1) == 0 ) { bench_running = &bench_results[i]; break; } } if ( bench_running == NULL ) { if ( bench_result_count >= AKGL_BENCH_MAX_RESULTS ) { SDL_Log("benchutil: no room for benchmark '%s', raise AKGL_BENCH_MAX_RESULTS", name); return; } bench_running = &bench_results[bench_result_count]; bench_result_count += 1; memset(bench_running, 0x00, sizeof(akgl_Benchmark)); strncpy(bench_running->name, name, AKGL_BENCH_MAX_NAME - 1); strncpy(bench_running->unit, unit, AKGL_BENCH_MAX_UNIT - 1); bench_running->budget_ns = budget_ns; bench_running->best_ns = -1.0; } bench_started_ns = SDL_GetTicksNS(); } /** * @brief Stop timing and keep the run if it beat every previous one. * @param ops Units of work done since bench_start(). Zero is ignored -- a * benchmark that did nothing has no rate to report. */ static void bench_stop(uint64_t ops) { uint64_t elapsed = SDL_GetTicksNS() - bench_started_ns; double per_op = 0.0; if ( bench_running == NULL || ops == 0 ) { return; } per_op = (double)elapsed / (double)ops; bench_running->runs += 1; if ( bench_running->best_ns < 0.0 || per_op < bench_running->best_ns ) { bench_running->best_ns = per_op; bench_running->ops = ops; bench_running->elapsed_ns = elapsed; } bench_running = NULL; } /** * @brief Run @p stmt @p count times, stopping at the first error. * * The timed loop deliberately holds no `PASS` or `CATCH`: their validity check * walks the whole error array and costs more than some of the calls being * measured. Hand @p errvar to `PASS` once the clock has stopped. * * @param errvar Receives the first non-`NULL` context returned, or `NULL`. * @param i Loop variable, declared by the caller. * @param count How many times to run @p stmt. * @param stmt The call under test. Must evaluate to an `akerr_ErrorContext *`. */ #define BENCH_LOOP(errvar, i, count, stmt) \ for ( i = 0; i < (count); i++ ) { \ errvar = (stmt); \ if ( errvar != NULL ) { \ break; \ } \ } /** * @brief Print the recorded benchmarks as a table and count the ones over budget. * * The table goes to stdout so `ctest --output-on-failure` and a plain run of the * executable both show it. Rates are printed as ops per second, which is the * number a frame budget is actually built from. * * @return How many benchmarks exceeded their budget. Always 0 when * bench_budgets_enforced() is false. */ static int bench_report(void) { int i = 0; int over = 0; bool enforced = bench_budgets_enforced(); char *verdict = NULL; printf("\n"); if ( bench_under_valgrind() == true ) { printf("running under valgrind: this is a memory check, not a measurement.\n"); printf("The timings below are valgrind's and mean nothing about libakgl.\n"); } printf("scale %.4gx, best of %d runs, budgets %s\n", bench_scale(), AKGL_BENCH_REPETITIONS, enforced ? "enforced" : "reported only"); printf("%-54s %-6s %9s %12s %16s %10s %s\n", "benchmark", "unit", "ops", "ns/op", "ops/sec", "budget", "verdict"); printf("%-54s %-6s %9s %12s %16s %10s %s\n", "------------------------------------------------------", "------", "---------", "------------", "----------------", "----------", "-------"); for ( i = 0; i < bench_result_count; i++ ) { verdict = "-"; if ( bench_results[i].budget_ns > 0.0 ) { if ( bench_results[i].best_ns > bench_results[i].budget_ns ) { verdict = enforced ? "OVER" : "over"; if ( enforced ) { over += 1; } } else { verdict = "ok"; } } printf("%-54s %-6s %9llu %12.1f %16.0f %10.0f %s\n", bench_results[i].name, bench_results[i].unit, (unsigned long long)bench_results[i].ops, bench_results[i].best_ns, ( bench_results[i].best_ns > 0.0 ) ? (1000000000.0 / bench_results[i].best_ns) : 0.0, bench_results[i].budget_ns, verdict); } printf("\n"); fflush(stdout); return over; } /** * @brief Fail the enclosing ATTEMPT block if any benchmark blew its budget. * * The report is printed either way -- a run that fails still has to say what the * numbers were. */ #define BENCH_REPORT_BREAK(e) \ { \ int __bench_over = bench_report(); \ if ( __bench_over > 0 ) { \ FAIL_BREAK( \ e, \ AKGL_ERR_BEHAVIOR, \ "%d benchmark(s) exceeded their budget", \ __bench_over); \ } \ } #endif // _AKGL_BENCHUTIL_H_