/** * @file benchutil.h * @brief The timing harness behind the perf suites: run it, record it, report it, hold it to a budget. * * A benchmark here is a timed region: bench_start() takes a timestamp, * bench_stop() takes another and divides by the number of units of work that * happened in between. The unit is whatever the measurement is *about* -- one * call, one actor, one tile, one frame -- and it is printed alongside the * number so nobody has to guess what "42 ns" was 42 nanoseconds of. * * Two things make the numbers usable rather than merely present: * * 1. **Best of #AKGL_BENCH_REPETITIONS.** Calling bench_start() again with a * name already in the table folds the new run into the old entry and keeps * the *lowest* ns/op seen. A benchmark loop therefore repeats itself and the * harness reports the run that was interrupted least. The mean is the wrong * statistic here: every source of noise on a shared machine makes a run * slower and none makes it faster. * 2. **The timed region contains no error checking.** `PASS` and `CATCH` both * call `akerr_valid_error_address`, which walks `AKERR_ARRAY_ERROR` looking * for a match -- more work than several of the calls being measured. Use * #BENCH_LOOP, which stashes the context and stops at the first failure, and * hand it to `PASS` after the clock has stopped. * * A budget is a per-op ceiling in nanoseconds, and a benchmark that exceeds it * fails the suite. Budgets are set at roughly ten times the measured baseline: * loose enough that a busy machine does not turn CI red, tight enough that an * algorithmic regression -- a linear scan that becomes quadratic, a per-frame * allocation that becomes a per-actor one -- cannot hide. They are enforced * only in an optimized build at full scale; a coverage build measures the * instrumentation, not the library, and reports without judging. * * Set `AKGL_BENCH_SCALE` in the environment to change how long the suite runs: * 0.1 for a tenth of the iterations, 10 for ten times as many. Below 1.0 the * budgets are not enforced, because a short run is a noisy one. */ #ifndef _AKGL_BENCHUTIL_H_ #define _AKGL_BENCHUTIL_H_ #include #include #include #include #include /** @brief How many benchmarks one suite can record. */ #define AKGL_BENCH_MAX_RESULTS 48 /** @brief Longest benchmark name kept, including the terminator. */ #define AKGL_BENCH_MAX_NAME 56 /** @brief Longest unit name kept, including the terminator. */ #define AKGL_BENCH_MAX_UNIT 12 /** @brief How many times a benchmark is repeated before the best run is reported. */ #define AKGL_BENCH_REPETITIONS 5 /** @brief Environment variable scaling every iteration count. */ #define AKGL_BENCH_SCALE_ENV "AKGL_BENCH_SCALE" /** @brief One row of the report: what was measured, how fast, and what it was allowed to cost. */ typedef struct akgl_Benchmark { char name[AKGL_BENCH_MAX_NAME]; /**< What was measured. Also the key runs are merged under. */ char unit[AKGL_BENCH_MAX_UNIT]; /**< What one op is: "call", "actor", "tile", "frame". */ double budget_ns; /**< Per-op ceiling in nanoseconds. 0 reports without judging. */ double best_ns; /**< Lowest ns/op seen across every run under this name. */ uint64_t ops; /**< Units of work in the run that produced best_ns. */ uint64_t elapsed_ns; /**< Wall time of that run. */ int runs; /**< How many runs were folded in. */ } akgl_Benchmark; /** @brief Every benchmark this suite has recorded, in the order it first ran. */ static akgl_Benchmark bench_results[AKGL_BENCH_MAX_RESULTS]; /** @brief How many entries of #bench_results are in use. */ static int bench_result_count = 0; /** @brief The entry bench_stop() will write to, claimed by bench_start(). */ static akgl_Benchmark *bench_running = NULL; /** @brief `SDL_GetTicksNS()` at the last bench_start(). */ static uint64_t bench_started_ns = 0; /** * @brief Multiplier applied to every iteration count, from `AKGL_BENCH_SCALE`. * * Read once and cached. Anything unparseable, negative, or absent gives 1.0. */ static double bench_scale(void) { static double scale = -1.0; const char *env = NULL; if ( scale >= 0.0 ) { return scale; } scale = 1.0; env = SDL_getenv(AKGL_BENCH_SCALE_ENV); if ( env != NULL ) { scale = SDL_atof(env); if ( scale <= 0.0 ) { scale = 1.0; } } return scale; } /** * @brief Scale a nominal iteration count, never down to zero. * @param count The count the benchmark was written for. * @return @p count times the scale factor, at least 1. */ static int bench_iterations(int count) { int scaled = (int)((double)count * bench_scale()); if ( scaled < 1 ) { scaled = 1; } return scaled; } /** * @brief Report whether budgets are being enforced in this run. * * An unoptimized build measures the instrumentation rather than the library, * and a scaled-down run is too short to trust, so both report without failing. */ static bool bench_budgets_enforced(void) { #ifdef __OPTIMIZE__ return ( bench_scale() >= 1.0 ); #else return false; #endif } /** * @brief Start timing, claiming or reusing the table entry named @p name. * * @param name What is being measured. Truncated at #AKGL_BENCH_MAX_NAME. * Reusing a name folds this run into that entry. * @param unit What one op is. Truncated at #AKGL_BENCH_MAX_UNIT. * @param budget_ns Per-op ceiling in nanoseconds; 0 to report only. The value * from the first run under a name wins. * * @note The table is fixed. Past #AKGL_BENCH_MAX_RESULTS entries the run is * dropped with a message rather than overwriting somebody else's row. */ static void bench_start(char *name, char *unit, double budget_ns) { int i = 0; bench_running = NULL; for ( i = 0; i < bench_result_count; i++ ) { if ( strncmp(bench_results[i].name, name, AKGL_BENCH_MAX_NAME - 1) == 0 ) { bench_running = &bench_results[i]; break; } } if ( bench_running == NULL ) { if ( bench_result_count >= AKGL_BENCH_MAX_RESULTS ) { SDL_Log("benchutil: no room for benchmark '%s', raise AKGL_BENCH_MAX_RESULTS", name); return; } bench_running = &bench_results[bench_result_count]; bench_result_count += 1; memset(bench_running, 0x00, sizeof(akgl_Benchmark)); strncpy(bench_running->name, name, AKGL_BENCH_MAX_NAME - 1); strncpy(bench_running->unit, unit, AKGL_BENCH_MAX_UNIT - 1); bench_running->budget_ns = budget_ns; bench_running->best_ns = -1.0; } bench_started_ns = SDL_GetTicksNS(); } /** * @brief Stop timing and keep the run if it beat every previous one. * @param ops Units of work done since bench_start(). Zero is ignored -- a * benchmark that did nothing has no rate to report. */ static void bench_stop(uint64_t ops) { uint64_t elapsed = SDL_GetTicksNS() - bench_started_ns; double per_op = 0.0; if ( bench_running == NULL || ops == 0 ) { return; } per_op = (double)elapsed / (double)ops; bench_running->runs += 1; if ( bench_running->best_ns < 0.0 || per_op < bench_running->best_ns ) { bench_running->best_ns = per_op; bench_running->ops = ops; bench_running->elapsed_ns = elapsed; } bench_running = NULL; } /** * @brief Run @p stmt @p count times, stopping at the first error. * * The timed loop deliberately holds no `PASS` or `CATCH`: their validity check * walks the whole error array and costs more than some of the calls being * measured. Hand @p errvar to `PASS` once the clock has stopped. * * @param errvar Receives the first non-`NULL` context returned, or `NULL`. * @param i Loop variable, declared by the caller. * @param count How many times to run @p stmt. * @param stmt The call under test. Must evaluate to an `akerr_ErrorContext *`. */ #define BENCH_LOOP(errvar, i, count, stmt) \ for ( i = 0; i < (count); i++ ) { \ errvar = (stmt); \ if ( errvar != NULL ) { \ break; \ } \ } /** * @brief Print the recorded benchmarks as a table and count the ones over budget. * * The table goes to stdout so `ctest --output-on-failure` and a plain run of the * executable both show it. Rates are printed as ops per second, which is the * number a frame budget is actually built from. * * @return How many benchmarks exceeded their budget. Always 0 when * bench_budgets_enforced() is false. */ static int bench_report(void) { int i = 0; int over = 0; bool enforced = bench_budgets_enforced(); char *verdict = NULL; printf("\n"); printf("scale %.2fx, best of %d runs, budgets %s\n", bench_scale(), AKGL_BENCH_REPETITIONS, enforced ? "enforced" : "reported only"); printf("%-54s %-6s %9s %12s %16s %10s %s\n", "benchmark", "unit", "ops", "ns/op", "ops/sec", "budget", "verdict"); printf("%-54s %-6s %9s %12s %16s %10s %s\n", "------------------------------------------------------", "------", "---------", "------------", "----------------", "----------", "-------"); for ( i = 0; i < bench_result_count; i++ ) { verdict = "-"; if ( bench_results[i].budget_ns > 0.0 ) { if ( bench_results[i].best_ns > bench_results[i].budget_ns ) { verdict = enforced ? "OVER" : "over"; if ( enforced ) { over += 1; } } else { verdict = "ok"; } } printf("%-54s %-6s %9llu %12.1f %16.0f %10.0f %s\n", bench_results[i].name, bench_results[i].unit, (unsigned long long)bench_results[i].ops, bench_results[i].best_ns, ( bench_results[i].best_ns > 0.0 ) ? (1000000000.0 / bench_results[i].best_ns) : 0.0, bench_results[i].budget_ns, verdict); } printf("\n"); fflush(stdout); return over; } /** * @brief Fail the enclosing ATTEMPT block if any benchmark blew its budget. * * The report is printed either way -- a run that fails still has to say what the * numbers were. */ #define BENCH_REPORT_BREAK(e) \ { \ int __bench_over = bench_report(); \ if ( __bench_over > 0 ) { \ FAIL_BREAK( \ e, \ AKGL_ERR_BEHAVIOR, \ "%d benchmark(s) exceeded their budget", \ __bench_over); \ } \ } #endif // _AKGL_BENCHUTIL_H_