akbasic/tests/docs_examples.sh

#!/bin/bash
#
# Run every example in the documentation and check what it produces.
#
# The guide in docs/ and the walkthrough in README.md are full of programs,
# transcripts, C snippets and shell commands. Every one of them was checked by
# hand once, when it was written, and that is not a standard that survives
# contact with a changing interpreter. Three of them were already wrong when this
# harness was written -- two transcripts carrying a leading space PRINT does not
# emit, and a struct in README.md that had grown two members hours earlier.
#
# Documentation goes stale because the *code* moved, not because somebody edited
# a chapter, so this runs on every ctest rather than on a docs path filter.
#
# **Exit status is the number of failed examples**, the house convention.
#
# The contract with the documentation is a set of fence info strings. They are
# ordinary markdown, they render as syntax highlighting on the forge, and they
# sit next to the thing they describe. MAINTENANCE.md is the reference; the short
# version:
#
#   ```basic              a whole program: run it, and it must not error
#   ```basic repl         input lines fed to a fresh interpreter on stdin
#   ```basic norun        a fragment, shown but not run
#   ```basic requires=akgl   only run in the -DAKBASIC_WITH_AKGL=ON build
#   ```basic requires=noakgl only run in the build with no devices attached
#   ```output             the exact stdout of the block above, byte for byte
#   ```c                  a translation unit: compile it with -fsyntax-only
#   ```c wrap=NAME        the same, wrapped in tests/docs_preludes/NAME.pre/.post
#   ```c excerpt=PATH     must appear verbatim in PATH; not compiled
#   ```c norun            shown but not compiled
#   ```sh                 run in a sandbox; must exit 0
#   ```sh setup=NAME      the same, after tests/docs_setups/NAME.sh
#   ```sh norun           destructive, networked, or re-enters this suite
#   ```cmake              never executed; hand-maintained, by decision
#   ```text               a diagram or a captured dump; there is nothing to execute
#
# A block with **no** info string is a hard error. That is deliberate: the
# failure mode this whole harness exists to avoid is passing because it quietly
# ran nothing, so an unannotated block is a missing decision rather than a
# default.

set -u

ROOT=""
BASIC=""
CFLAGS_FILE=""
WITH_AKGL=0

usage()
{
    cat >&2 <<'EOF'
usage: docs_examples.sh --root DIR --basic PATH [--cflags-file FILE] [--akgl] [FILE...]

  --root DIR          repository root; documentation paths are relative to it
  --basic PATH        the built interpreter
  --cflags-file FILE  one compiler flag per line, for the ```c blocks
  --akgl              this is an AKBASIC_WITH_AKGL build; run requires=akgl blocks
  FILE...             which documents to check (default: README.md docs/*.md)
EOF
    exit 2
}

while [ $# -gt 0 ]; do
    case "$1" in
    --root)        ROOT="$2";        shift 2 ;;
    --basic)       BASIC="$2";       shift 2 ;;
    --cflags-file) CFLAGS_FILE="$2"; shift 2 ;;
    --akgl)        WITH_AKGL=1;      shift ;;
    --help|-h)     usage ;;
    --*)           echo "unknown option $1" >&2; usage ;;
    *)             break ;;
    esac
done

[ -n "${ROOT}" ]  || usage
[ -n "${BASIC}" ] || usage
[ -x "${BASIC}" ] || { echo "FAIL: no interpreter at ${BASIC}" >&2; exit 2; }

# Both of these are used from inside a sandbox directory this script cd's into,
# so a relative one silently resolves against the wrong place: `--basic
# ./build/basic` fails every example with "exited 127", and `--root .` cannot
# find tests/docs_setups. CTest passes absolute paths and never saw it; a person
# running one document by hand hits it immediately. Resolve them here instead of
# documenting the trap.
case "${BASIC}" in
/*) ;;
*)  BASIC="${PWD}/${BASIC}" ;;
esac

cd "${ROOT}" || exit 2
ROOT="${PWD}"

DOCS=("$@")
if [ ${#DOCS[@]} -eq 0 ]; then
    # MAINTENANCE.md is here for the tagging rule rather than for its examples:
    # every one of its blocks is `norun`, and the point is that the file which
    # documents the convention is held to it.
    DOCS=(README.md MAINTENANCE.md)
    for _doc in docs/*.md; do
	DOCS+=("${_doc}")
    done
fi

# Refuse a document that is not there rather than checking nothing and passing.
# An empty argument is the specific case that got here: a CMake generator
# expression evaluating to nothing still contributes an empty argument, which
# looked like a filename and silently replaced the whole default list.
for _doc in "${DOCS[@]}"; do
    if [ ! -r "${_doc}" ]; then
	echo "FAIL: no document at \"${_doc}\"" >&2
	exit 2
    fi
done

WORK="$(mktemp -d)"
trap 'rm -rf "${WORK}"' EXIT

# The compiler flags the ```c blocks need. CMake writes them, because the
# include path is transitive through akerror, akstdlib and (in the AKGL build)
# akgl, and hardcoding it here would go stale exactly the way the docs do.
CFLAGS=()
if [ -n "${CFLAGS_FILE}" ] && [ -r "${CFLAGS_FILE}" ]; then
    while IFS= read -r _flag; do
	[ -n "${_flag}" ] && CFLAGS+=("${_flag}")
    done < "${CFLAGS_FILE}"
fi
CC="${CC:-cc}"

FAILURES=0
declare -A RAN=([basic]=0 [repl]=0 [output]=0 [c]=0 [excerpt]=0 [sh]=0)
declare -A SKIPPED=([norun]=0 [akgl]=0 [cmake]=0 [text]=0 [nocc]=0)

# Every failure names the file and line of the block, so the message points at
# the thing to edit rather than at this script.
fail()
{
    local where="$1"; shift
    echo "FAIL ${where}: $*" >&2
    FAILURES=$((FAILURES + 1))
}

# Show a byte-exact comparison. `cat -A` because the errors this catches are
# leading spaces and missing newlines, which a plain diff renders invisibly.
show_diff()
{
    local want="$1" got="$2"
    echo "--- expected ---" >&2
    cat -A "${want}" >&2
    echo "--- actual ---" >&2
    cat -A "${got}" >&2
}

# ---------------------------------------------------------------- extraction
#
# One pass per document, writing each block's body to ${WORK}/blocks/NNNN and a
# line to ${WORK}/index. Keeping the bodies in files rather than shell variables
# means a block containing a NUL, a backslash or an unbalanced quote is carried
# through untouched -- and BASIC string literals contain plenty of all three.
extract()
{
    local doc="$1" out="$2"

    awk -v OUT="${out}" -v INDEX="${out}/index" '
	function pad(n) { return sprintf("%04d", n) }
	/^```/ {
	    if ( !inblock ) {
		inblock = 1
		tag = substr($0, 4)
		sub(/[ \t]+$/, "", tag)
		start = NR
		count++
		body = OUT "/" pad(count)
		printf "" > body
		next
	    }
	    if ( $0 == "```" ) {
		close(body)
		printf "%s\t%d\t%s\n", pad(count), start, tag >> INDEX
		inblock = 0
		next
	    }
	}
	inblock { print >> body }
    ' "${doc}"
}

# ------------------------------------------------------------------- helpers

# The value of attribute $1 in an info string $2, or empty. `wrap=embed` and
# `requires=akgl` are both read this way.
attr()
{
    local name="$1" info="$2" word
    for word in ${info}; do
	case "${word}" in
	"${name}"=*) echo "${word#*=}"; return ;;
	esac
    done
}

# True when a bare word appears in an info string. Used for `norun` and `repl`.
has_word()
{
    local want="$1" info="$2" word
    for word in ${info}; do
	[ "${word}" = "${want}" ] && return 0
    done
    return 1
}

# Strip C comments and collapse whitespace, so an excerpt can be compared to the
# header it came from without either side having to carry the other's doc
# comments or indentation.
normalise_c()
{
    awk '
	{
	    line = $0
	    out = ""
	    while ( length(line) > 0 ) {
		if ( incomment ) {
		    i = index(line, "*/")
		    if ( i == 0 ) { line = ""; break }
		    incomment = 0
		    line = substr(line, i + 2)
		    continue
		}
		i = index(line, "/*")
		j = index(line, "//")
		if ( j > 0 && (i == 0 || j < i) ) {
		    out = out substr(line, 1, j - 1)
		    line = ""
		    break
		}
		if ( i == 0 ) { out = out line; line = ""; break }
		out = out substr(line, 1, i - 1)
		incomment = 1
		line = substr(line, i + 2)
	    }
	    printf "%s ", out
	}
	END { printf "\n" }
    ' "$1" | tr -s ' \t' ' ' | sed -e 's/^ //' -e 's/ $//'
}

# --------------------------------------------------------------- block kinds

# A BASIC program in a file, or a transcript on stdin. Both go through the same
# comparison, because the only difference is how the interpreter is invoked.
run_basic()
{
    local where="$1" body="$2" info="$3" expected="$4"
    local sandbox="${WORK}/run" got="${WORK}/got" status=0 setup

    rm -rf "${sandbox}"
    mkdir -p "${sandbox}"

    # An example that reads a file needs that file. Putting the fixture in a
    # setup script rather than in the chapter keeps the example the shape a
    # reader wants to see -- `SPRSAV "ship.png", 1` and nothing else.
    setup="$(attr setup "${info}")"
    if [ -n "${setup}" ]; then
	if [ ! -r "tests/docs_setups/${setup}.sh" ]; then
	    fail "${where}" "setup=${setup} names no tests/docs_setups/${setup}.sh"
	    return
	fi
	if ! ( cd "${sandbox}" && bash "${ROOT}/tests/docs_setups/${setup}.sh" ) >/dev/null 2>&1; then
	    fail "${where}" "setup=${setup} failed"
	    return
	fi
    fi

    if has_word repl "${info}"; then
	( cd "${sandbox}" && "${BASIC}" < "${body}" ) > "${got}.raw" 2> "${WORK}/stderr"
	status=$?
	# Drop the startup banner. A piped interpreter prints READY once, before
	# it has read anything, so carrying it in every transcript would be one
	# line of noise per example and would put it in the wrong place besides:
	# at a real prompt READY comes back *after* each line. Chapter 2 shows
	# that interactively, which is where it belongs.
	sed -e '1{/^READY$/d}' "${got}.raw" > "${got}"
	RAN[repl]=$((RAN[repl] + 1))
    else
	cp "${body}" "${sandbox}/example.bas"
	( cd "${sandbox}" && "${BASIC}" example.bas ) > "${got}" 2> "${WORK}/stderr"
	status=$?
	RAN[basic]=$((RAN[basic] + 1))
    fi

    if [ "${status}" -ne 0 ]; then
	fail "${where}" "the interpreter exited ${status}"
	sed -n '1,20p' "${got}" "${WORK}/stderr" >&2
	return
    fi

    # stdout only, the way tests/golden.cmake compares its cases. libakgl logs
    # registry activity to stderr -- three lines every time a sprite is created
    # -- and folding that into the comparison would put library chatter into the
    # documentation of a BASIC verb.
    if [ -n "${expected}" ]; then
	RAN[output]=$((RAN[output] + 1))
	if ! cmp -s "${expected}" "${got}"; then
	    fail "${where}" "output does not match the block below it"
	    show_diff "${expected}" "${got}"
	fi
	return
    fi

    # No expectation given, so the only thing to assert is that it ran cleanly.
    # A BASIC-level error prints "? LINE : CLASS" and still exits 0 -- that is
    # correct for the interpreter and useless as a test result, so look for it.
    if grep -qE '^\? [0-9]+ :' "${got}"; then
	fail "${where}" "the program raised a BASIC error"
	grep -E '^\? [0-9]+ :' "${got}" >&2
    fi
}

# A C snippet. Compiled, never linked and never run: what these examples are for
# is showing the API, and the API is exactly what -fsyntax-only checks.
run_c()
{
    local where="$1" body="$2" info="$3" doc="$4" line="$5"
    local wrap unit="${WORK}/unit.c" log="${WORK}/cc.log"

    if [ ${#CFLAGS[@]} -eq 0 ]; then
	SKIPPED[nocc]=$((SKIPPED[nocc] + 1))
	return
    fi

    wrap="$(attr wrap "${info}")"
    : > "${unit}"
    if [ -n "${wrap}" ]; then
	if [ ! -r "tests/docs_preludes/${wrap}.pre" ]; then
	    fail "${where}" "wrap=${wrap} names no tests/docs_preludes/${wrap}.pre"
	    return
	fi
	cat "tests/docs_preludes/${wrap}.pre" >> "${unit}"
    fi
    # Hand the compiler the document's own coordinates, so a diagnostic points
    # at the line of markdown to fix rather than at a scratch file.
    printf '#line %d "%s"\n' "$((line + 1))" "${doc}" >> "${unit}"
    cat "${body}" >> "${unit}"
    if [ -n "${wrap}" ] && [ -r "tests/docs_preludes/${wrap}.post" ]; then
	printf '#line 1 "tests/docs_preludes/%s.post"\n' "${wrap}" >> "${unit}"
	cat "tests/docs_preludes/${wrap}.post" >> "${unit}"
    fi

    # gnu99 rather than c99: akerror.h uses PATH_MAX, which <limits.h> hides
    # under __STRICT_ANSI__, and the library itself is built with the compiler's
    # default dialect. A stricter flag here would fail on the dependency rather
    # than on the example.
    if "${CC}" -fsyntax-only -std=gnu99 -Wall -Wextra -Werror \
	       "${CFLAGS[@]}" "${unit}" > "${log}" 2>&1; then
	RAN[c]=$((RAN[c] + 1))
    else
	fail "${where}" "does not compile"
	sed -n '1,25p' "${log}" >&2
    fi
}

# A declaration echoed from a header. Compiling it would redefine the type, so
# the check is that it still says what the header says -- which is the drift
# that actually happened, and the one a compile could not have caught.
run_excerpt()
{
    local where="$1" body="$2" path="$3"

    if [ ! -r "${path}" ]; then
	fail "${where}" "excerpt=${path} names no such file"
	return
    fi
    RAN[excerpt]=$((RAN[excerpt] + 1))
    if ! normalise_c "${path}" | grep -qF -- "$(normalise_c "${body}")"; then
	fail "${where}" "no longer appears in ${path}"
	echo "--- the documentation says ---" >&2
	cat "${body}" >&2
	echo "--- ${path} has drifted from it; comments and whitespace are ignored ---" >&2
    fi
}

# A shell command, in a sandbox with the built interpreter reachable at the path
# the documentation tells a reader to use.
run_sh()
{
    local where="$1" body="$2" info="$3" expected="$4"
    local setup sandbox="${WORK}/sh" script="${WORK}/block.sh" got="${WORK}/got"

    rm -rf "${sandbox}"
    mkdir -p "${sandbox}/build"
    ln -s "${BASIC}" "${sandbox}/build/basic"

    setup="$(attr setup "${info}")"
    if [ -n "${setup}" ]; then
	if [ ! -r "tests/docs_setups/${setup}.sh" ]; then
	    fail "${where}" "setup=${setup} names no tests/docs_setups/${setup}.sh"
	    return
	fi
	if ! ( cd "${sandbox}" && bash "${ROOT}/tests/docs_setups/${setup}.sh" ) >/dev/null 2>&1; then
	    fail "${where}" "setup=${setup} failed"
	    return
	fi
    fi

    # A leading "$ " is a prompt in a transcript, not part of the command.
    sed -e 's/^\$ //' "${body}" > "${script}"
    if ! ( cd "${sandbox}" && bash -e "${script}" ) > "${got}" 2>&1; then
	fail "${where}" "the command failed"
	sed -n '1,20p' "${got}" >&2
	return
    fi
    RAN[sh]=$((RAN[sh] + 1))

    if [ -n "${expected}" ]; then
	RAN[output]=$((RAN[output] + 1))
	if ! cmp -s "${expected}" "${got}"; then
	    fail "${where}" "output does not match the block below it"
	    show_diff "${expected}" "${got}"
	fi
    fi
}

# ------------------------------------------------------------------ the pass
for DOC in "${DOCS[@]}"; do
    BLOCKS="${WORK}/blocks"
    rm -rf "${BLOCKS}"
    mkdir -p "${BLOCKS}"
    : > "${BLOCKS}/index"
    extract "${DOC}" "${BLOCKS}"

    # Read the index into arrays first. An `output` block belongs to the block
    # before it, so each iteration has to be able to look one ahead.
    CLAIMED=0
    IDS=(); LINES=(); INFOS=()
    while IFS=$'\t' read -r _id _line _info; do
	IDS+=("${_id}"); LINES+=("${_line}"); INFOS+=("${_info}")
    done < "${BLOCKS}/index"

    for ((n = 0; n < ${#IDS[@]}; n++)); do
	INFO="${INFOS[n]}"
	BODY="${BLOCKS}/${IDS[n]}"
	WHERE="${DOC}:${LINES[n]}"
	KIND="${INFO%% *}"

	# An `output` block belongs to the block above it, which claimed it on the
	# way past. One that nothing claimed is compared against nothing at all --
	# indistinguishable from a passing test, and the exact failure this
	# harness exists to rule out. Note a *skipped* block still claims its
	# output: `norun` and `requires=` are decisions, not accidents.
	if [ "${KIND}" = "output" ]; then
	    if [ "${CLAIMED}" -eq 0 ]; then
		fail "${WHERE}" "an output block with nothing runnable above it"
	    fi
	    CLAIMED=0
	    continue
	fi
	CLAIMED=0

	EXPECTED=""
	case "${KIND}" in
	basic|sh)
	    if [ $((n + 1)) -lt ${#IDS[@]} ] && [ "${INFOS[n + 1]%% *}" = "output" ]; then
		EXPECTED="${BLOCKS}/${IDS[n + 1]}"
		CLAIMED=1
	    fi
	    ;;
	esac

	if has_word norun "${INFO}"; then
	    SKIPPED[norun]=$((SKIPPED[norun] + 1))
	    continue
	fi

	# A handful of examples are about what a *build* does rather than what the
	# language does -- a verb refusing because no device was attached is the
	# clearest one -- so they run in one configuration and are skipped in the
	# other. Both directions are needed, and both are visible in the count.
	REQUIRES="$(attr requires "${INFO}")"
	if [ "${REQUIRES}" = "akgl" ] && [ "${WITH_AKGL}" -eq 0 ]; then
	    SKIPPED[akgl]=$((SKIPPED[akgl] + 1))
	    continue
	fi
	if [ "${REQUIRES}" = "noakgl" ] && [ "${WITH_AKGL}" -eq 1 ]; then
	    SKIPPED[akgl]=$((SKIPPED[akgl] + 1))
	    continue
	fi

	case "${KIND}" in
	basic)
	    run_basic "${WHERE}" "${BODY}" "${INFO}" "${EXPECTED}"
	    ;;
	c)
	    EXCERPT="$(attr excerpt "${INFO}")"
	    if [ -n "${EXCERPT}" ]; then
		run_excerpt "${WHERE}" "${BODY}" "${EXCERPT}"
	    else
		run_c "${WHERE}" "${BODY}" "${INFO}" "${DOC}" "${LINES[n]}"
	    fi
	    ;;
	sh)
	    run_sh "${WHERE}" "${BODY}" "${INFO}" "${EXPECTED}"
	    ;;
	cmake)
	    SKIPPED[cmake]=$((SKIPPED[cmake] + 1))
	    ;;
	# A block diagram, or a stack trace captured from a real run. There is no
	# executable claim in either, so the tag exists to say that out loud rather
	# than to let one sit here untagged and unnoticed.
	text)
	    SKIPPED[text]=$((SKIPPED[text] + 1))
	    ;;
	"")
	    fail "${WHERE}" "fenced block with no info string; tag it (see MAINTENANCE.md)"
	    ;;
	*)
	    fail "${WHERE}" "unknown info string \"${INFO}\" (see MAINTENANCE.md)"
	    ;;
	esac
    done
done

# The count is part of the result, not decoration. A harness that passes because
# a chapter stopped matching the extractor looks exactly like a harness that
# passes because the documentation is correct, and this is what tells them apart.
echo "ran: ${RAN[basic]} programs, ${RAN[repl]} transcripts, ${RAN[output]} output comparisons, ${RAN[c]} C snippets, ${RAN[excerpt]} excerpts, ${RAN[sh]} shell blocks"
echo "skipped: ${SKIPPED[norun]} norun, ${SKIPPED[akgl]} needing akgl, ${SKIPPED[cmake]} cmake, ${SKIPPED[text]} text, ${SKIPPED[nocc]} C (no compiler flags given)"

if [ "${FAILURES}" -eq 0 ]; then
    echo "PASS: every documented example does what the documentation says"
fi
exit "${FAILURES}"