libakgl/tests/benchutil.h

/**
 * @file benchutil.h
 * @brief The timing harness behind the perf suites: run it, record it, report it, hold it to a budget.
 *
 * A benchmark here is a timed region: bench_start() takes a timestamp,
 * bench_stop() takes another and divides by the number of units of work that
 * happened in between. The unit is whatever the measurement is *about* -- one
 * call, one actor, one tile, one frame -- and it is printed alongside the
 * number so nobody has to guess what "42 ns" was 42 nanoseconds of.
 *
 * Two things make the numbers usable rather than merely present:
 *
 * 1. **Best of #AKGL_BENCH_REPETITIONS.** Calling bench_start() again with a
 *    name already in the table folds the new run into the old entry and keeps
 *    the *lowest* ns/op seen. A benchmark loop therefore repeats itself and the
 *    harness reports the run that was interrupted least. The mean is the wrong
 *    statistic here: every source of noise on a shared machine makes a run
 *    slower and none makes it faster.
 * 2. **The timed region contains no error checking.** `PASS` and `CATCH` both
 *    call `akerr_valid_error_address`, which walks `AKERR_ARRAY_ERROR` looking
 *    for a match -- more work than several of the calls being measured. Use
 *    #BENCH_LOOP, which stashes the context and stops at the first failure, and
 *    hand it to `PASS` after the clock has stopped.
 *
 * A budget is a per-op ceiling in nanoseconds, and a benchmark that exceeds it
 * fails the suite. Budgets are set at roughly ten times the measured baseline:
 * loose enough that a busy machine does not turn CI red, tight enough that an
 * algorithmic regression -- a linear scan that becomes quadratic, a per-frame
 * allocation that becomes a per-actor one -- cannot hide. They are enforced
 * only in an optimized build at full scale; a coverage build measures the
 * instrumentation, not the library, and reports without judging.
 *
 * Set `AKGL_BENCH_SCALE` in the environment to change how long the suite runs:
 * 0.1 for a tenth of the iterations, 10 for ten times as many. Below 1.0 the
 * budgets are not enforced, because a short run is a noisy one.
 *
 * **Under valgrind these suites become the memory-check suites**, and the scale
 * drops itself to #AKGL_BENCH_VALGRIND_SCALE without being asked. That is the
 * whole reason `ctest -T memcheck` does not need benchmark programs of its own:
 * a leak check wants every path walked *once*, and a benchmark is a program that
 * walks one path a hundred thousand times. Divide the iteration counts by two
 * thousand and the same binary is exactly the right shape for memcheck --
 * broader coverage of asset loading, drawing, and the frame loop than any unit
 * suite, at a runtime valgrind can survive. Timings from such a run are
 * meaningless and the report says so.
 */

#ifndef _AKGL_BENCHUTIL_H_
#define _AKGL_BENCHUTIL_H_

#include <stdio.h>
#include <string.h>
#include <SDL3/SDL.h>
#include <akerror.h>
#include <akgl/error.h>

/** @brief How many benchmarks one suite can record. */
#define AKGL_BENCH_MAX_RESULTS          48
/** @brief Longest benchmark name kept, including the terminator. */
#define AKGL_BENCH_MAX_NAME             56
/** @brief Longest unit name kept, including the terminator. */
#define AKGL_BENCH_MAX_UNIT             12
/** @brief How many times a benchmark is repeated before the best run is reported. */
#define AKGL_BENCH_REPETITIONS          5
/** @brief Environment variable scaling every iteration count. */
#define AKGL_BENCH_SCALE_ENV            "AKGL_BENCH_SCALE"
/** @brief Scale forced on when running under valgrind. Enough to walk every path, few enough to finish. */
#define AKGL_BENCH_VALGRIND_SCALE       0.0005

/** @brief One row of the report: what was measured, how fast, and what it was allowed to cost. */
typedef struct akgl_Benchmark {
    char name[AKGL_BENCH_MAX_NAME];     /**< What was measured. Also the key runs are merged under. */
    char unit[AKGL_BENCH_MAX_UNIT];     /**< What one op is: "call", "actor", "tile", "frame". */
    double budget_ns;                   /**< Per-op ceiling in nanoseconds. 0 reports without judging. */
    double best_ns;                     /**< Lowest ns/op seen across every run under this name. */
    uint64_t ops;                       /**< Units of work in the run that produced best_ns. */
    uint64_t elapsed_ns;                /**< Wall time of that run. */
    int runs;                           /**< How many runs were folded in. */
} akgl_Benchmark;

/** @brief Every benchmark this suite has recorded, in the order it first ran. */
static akgl_Benchmark bench_results[AKGL_BENCH_MAX_RESULTS];
/** @brief How many entries of #bench_results are in use. */
static int bench_result_count = 0;
/** @brief The entry bench_stop() will write to, claimed by bench_start(). */
static akgl_Benchmark *bench_running = NULL;
/** @brief `SDL_GetTicksNS()` at the last bench_start(). */
static uint64_t bench_started_ns = 0;

/**
 * @brief Report whether this process is running under valgrind.
 *
 * Detected from `LD_PRELOAD`, which valgrind fills with its own
 * `vgpreload_*.so` before handing the process over. That is a deliberate choice
 * over `RUNNING_ON_VALGRIND` from `valgrind/valgrind.h`: the macro is exact, but
 * it makes the test suite fail to compile anywhere the valgrind headers are not
 * installed, and a benchmark is not worth a build dependency. The cost of the
 * check being wrong is a slow run or a fast one, never a wrong answer.
 */
static bool bench_under_valgrind(void)
{
    static int detected = -1;
    const char *preload = NULL;

    if ( detected >= 0 ) {
	return ( detected == 1 );
    }
    detected = 0;
    preload = SDL_getenv("LD_PRELOAD");
    if ( preload != NULL && strstr(preload, "vgpreload") != NULL ) {
	detected = 1;
    }
    return ( detected == 1 );
}

/**
 * @brief Multiplier applied to every iteration count, from `AKGL_BENCH_SCALE`.
 *
 * Read once and cached. Anything unparseable, negative, or absent gives 1.0.
 * Under valgrind the result is capped at #AKGL_BENCH_VALGRIND_SCALE -- the
 * smaller of the two wins, so asking for an even shorter run still works.
 */
static double bench_scale(void)
{
    static double scale = -1.0;
    const char *env = NULL;

    if ( scale >= 0.0 ) {
	return scale;
    }
    scale = 1.0;
    env = SDL_getenv(AKGL_BENCH_SCALE_ENV);
    if ( env != NULL ) {
	scale = SDL_atof(env);
	if ( scale <= 0.0 ) {
	    scale = 1.0;
	}
    }
    if ( bench_under_valgrind() == true && scale > AKGL_BENCH_VALGRIND_SCALE ) {
	scale = AKGL_BENCH_VALGRIND_SCALE;
    }
    return scale;
}

/**
 * @brief Scale a nominal iteration count, never down to zero.
 * @param count The count the benchmark was written for.
 * @return @p count times the scale factor, at least 1.
 */
static int bench_iterations(int count)
{
    int scaled = (int)((double)count * bench_scale());

    if ( scaled < 1 ) {
	scaled = 1;
    }
    return scaled;
}

/**
 * @brief Report whether budgets are being enforced in this run.
 *
 * An unoptimized build measures the instrumentation rather than the library, a
 * scaled-down run is too short to trust, and a run under valgrind is measuring
 * valgrind. All three report without failing.
 */
static bool bench_budgets_enforced(void)
{
    if ( bench_under_valgrind() == true ) {
	return false;
    }
#ifdef __OPTIMIZE__
    return ( bench_scale() >= 1.0 );
#else
    return false;
#endif
}

/**
 * @brief Start timing, claiming or reusing the table entry named @p name.
 *
 * @param name      What is being measured. Truncated at #AKGL_BENCH_MAX_NAME.
 *                  Reusing a name folds this run into that entry.
 * @param unit      What one op is. Truncated at #AKGL_BENCH_MAX_UNIT.
 * @param budget_ns Per-op ceiling in nanoseconds; 0 to report only. The value
 *                  from the first run under a name wins.
 *
 * @note The table is fixed. Past #AKGL_BENCH_MAX_RESULTS entries the run is
 *       dropped with a message rather than overwriting somebody else's row.
 */
static void bench_start(char *name, char *unit, double budget_ns)
{
    int i = 0;

    bench_running = NULL;
    for ( i = 0; i < bench_result_count; i++ ) {
	if ( strncmp(bench_results[i].name, name, AKGL_BENCH_MAX_NAME - 1) == 0 ) {
	    bench_running = &bench_results[i];
	    break;
	}
    }
    if ( bench_running == NULL ) {
	if ( bench_result_count >= AKGL_BENCH_MAX_RESULTS ) {
	    SDL_Log("benchutil: no room for benchmark '%s', raise AKGL_BENCH_MAX_RESULTS", name);
	    return;
	}
	bench_running = &bench_results[bench_result_count];
	bench_result_count += 1;
	memset(bench_running, 0x00, sizeof(akgl_Benchmark));
	strncpy(bench_running->name, name, AKGL_BENCH_MAX_NAME - 1);
	strncpy(bench_running->unit, unit, AKGL_BENCH_MAX_UNIT - 1);
	bench_running->budget_ns = budget_ns;
	bench_running->best_ns = -1.0;
    }
    bench_started_ns = SDL_GetTicksNS();
}

/**
 * @brief Stop timing and keep the run if it beat every previous one.
 * @param ops Units of work done since bench_start(). Zero is ignored -- a
 *            benchmark that did nothing has no rate to report.
 */
static void bench_stop(uint64_t ops)
{
    uint64_t elapsed = SDL_GetTicksNS() - bench_started_ns;
    double per_op = 0.0;

    if ( bench_running == NULL || ops == 0 ) {
	return;
    }
    per_op = (double)elapsed / (double)ops;
    bench_running->runs += 1;
    if ( bench_running->best_ns < 0.0 || per_op < bench_running->best_ns ) {
	bench_running->best_ns = per_op;
	bench_running->ops = ops;
	bench_running->elapsed_ns = elapsed;
    }
    bench_running = NULL;
}

/**
 * @brief Run @p stmt @p count times, stopping at the first error.
 *
 * The timed loop deliberately holds no `PASS` or `CATCH`: their validity check
 * walks the whole error array and costs more than some of the calls being
 * measured. Hand @p errvar to `PASS` once the clock has stopped.
 *
 * @param errvar Receives the first non-`NULL` context returned, or `NULL`.
 * @param i      Loop variable, declared by the caller.
 * @param count  How many times to run @p stmt.
 * @param stmt   The call under test. Must evaluate to an `akerr_ErrorContext *`.
 */
#define BENCH_LOOP(errvar, i, count, stmt)      \
    for ( i = 0; i < (count); i++ ) {           \
	errvar = (stmt);                        \
	if ( errvar != NULL ) {                 \
	    break;                              \
	}                                       \
    }

/**
 * @brief Print the recorded benchmarks as a table and count the ones over budget.
 *
 * The table goes to stdout so `ctest --output-on-failure` and a plain run of the
 * executable both show it. Rates are printed as ops per second, which is the
 * number a frame budget is actually built from.
 *
 * @return How many benchmarks exceeded their budget. Always 0 when
 *         bench_budgets_enforced() is false.
 */
static int bench_report(void)
{
    int i = 0;
    int over = 0;
    bool enforced = bench_budgets_enforced();
    char *verdict = NULL;

    printf("\n");
    if ( bench_under_valgrind() == true ) {
	printf("running under valgrind: this is a memory check, not a measurement.\n");
	printf("The timings below are valgrind's and mean nothing about libakgl.\n");
    }
    printf("scale %.4gx, best of %d runs, budgets %s\n",
	   bench_scale(),
	   AKGL_BENCH_REPETITIONS,
	   enforced ? "enforced" : "reported only");
    printf("%-54s %-6s %9s %12s %16s %10s %s\n",
	   "benchmark", "unit", "ops", "ns/op", "ops/sec", "budget", "verdict");
    printf("%-54s %-6s %9s %12s %16s %10s %s\n",
	   "------------------------------------------------------",
	   "------", "---------", "------------", "----------------", "----------", "-------");
    for ( i = 0; i < bench_result_count; i++ ) {
	verdict = "-";
	if ( bench_results[i].budget_ns > 0.0 ) {
	    if ( bench_results[i].best_ns > bench_results[i].budget_ns ) {
		verdict = enforced ? "OVER" : "over";
		if ( enforced ) {
		    over += 1;
		}
	    } else {
		verdict = "ok";
	    }
	}
	printf("%-54s %-6s %9llu %12.1f %16.0f %10.0f %s\n",
	       bench_results[i].name,
	       bench_results[i].unit,
	       (unsigned long long)bench_results[i].ops,
	       bench_results[i].best_ns,
	       ( bench_results[i].best_ns > 0.0 ) ? (1000000000.0 / bench_results[i].best_ns) : 0.0,
	       bench_results[i].budget_ns,
	       verdict);
    }
    printf("\n");
    fflush(stdout);
    return over;
}

/**
 * @brief Fail the enclosing ATTEMPT block if any benchmark blew its budget.
 *
 * The report is printed either way -- a run that fails still has to say what the
 * numbers were.
 */
#define BENCH_REPORT_BREAK(e)					\
    {								\
	int __bench_over = bench_report();			\
	if ( __bench_over > 0 ) {				\
	    FAIL_BREAK(						\
		e,						\
		AKGL_ERR_BEHAVIOR,				\
		"%d benchmark(s) exceeded their budget",	\
		__bench_over);					\
	}							\
    }

#endif // _AKGL_BENCHUTIL_H_