libakstdlib/scripts/consumer_calls.py

#!/usr/bin/env python3
"""
Consumer adoption harness for libakstdlib.

Answers one question about a consumer's source tree: how often does it call
this library, and how often does it reach past this library to the libc
function this library wraps?

The ratio is the only external evidence there is about whether the surface that
got built is the surface anyone wanted. A wrapper nobody calls is a wrapper that
either does not fit or is not discoverable, and both are the library's problem.

The number this reports is only worth something if it is reproducible, which is
why this is a script and not a paragraph. TODO.md's first consumer figure was
recorded without one, and recovering the method afterwards cost more than
writing it down would have.

WHAT COUNTS

  * The corpus is every *.c and *.h directly under the given directory. It does
    not recurse: a consumer's src/ is the thing being measured, not its vendored
    dependencies, and those are usually a subdirectory.

  * Comments and string/character literals are stripped before anything is
    counted, so a function named in prose or inside a format string does not
    score. This matters more than it sounds -- "strlen" appears in doc comments
    throughout a codebase that has been thinking about strlen.

  * A call site is IDENT immediately followed by '(', where IDENT is not
    preceded by an identifier character. Declarations are not distinguished from
    calls; a consumer that declares a function named for a libc entry point will
    over-count by one per declaration, which is visible in --detail.

  * A library call is any IDENT matching ^aksl_.

  * A bypass is any IDENT naming a libc function this library wraps. That set is
    read out of include/akstdlib.h rather than hardcoded, so it grows when the
    library grows and a recount after a release measures the surface that
    release actually shipped.

  * libc functions this library does NOT wrap -- isdigit, exit, qsort -- score
    on neither side. The question is how often a consumer bypasses an available
    wrapper, not how much libc it uses. Adding a wrapper for something and
    having it ignored is a finding; a consumer calling exit() is not.

WHAT IT CANNOT TELL YOU

  One consumer's ratio is evidence, not a plan. A consumer that draws
  everything from fixed pools will never call the allocator no matter how good
  the allocator is, and will weight the string wrappers accordingly. Weight the
  result by what the consumer is, and get a second consumer before treating any
  ranking as settled.

Usage:
    scripts/consumer_calls.py DIR [options]

    DIR                 consumer source directory to measure, e.g.
                        ../akbasic/src
    --header PATH       akstdlib.h to read the wrapped-libc set from
                        (default: include/akstdlib.h beside this script's repo)
    --detail            list the per-function breakdown on both sides
    --per-file          list per-file counts, worst bypass ratio first
    --baseline A/B      compare against a previous count, e.g. --baseline 10/119
    --json              emit the whole result as JSON instead of text
"""

import argparse
import json
import os
import re
import sys
from collections import Counter

# Wrapper families that are this library's own constructs rather than a libc
# function under a new name. aksl_list_append has no libc counterpart, so
# "append" must not become a name a consumer can be scored for bypassing.
LIBRARY_ONLY_PREFIXES = ("hashmap", "list", "tree", "strbuf", "version",
                         "strhash")

# Library-only names whose first underscore-separated word is shared with a real
# libc entry point, so a prefix rule cannot separate them. aksl_realpath wraps
# realpath(3) and must score; aksl_realpath_alloc is this library's own.
LIBRARY_ONLY_NAMES = frozenset(("freep", "realpath_alloc"))

CALL = re.compile(r"(?<![A-Za-z0-9_])([A-Za-z_][A-Za-z0-9_]*)\s*\(")


def wrapped_libc(header):
    """The set of libc names this library wraps, read out of the header."""
    with open(header, encoding="utf-8", errors="replace") as handle:
        text = handle.read()
    names = set()
    for match in re.finditer(r"\baksl_([a-z0-9_]+)\s*\(", text):
        name = match.group(1)
        if name.split("_")[0] in LIBRARY_ONLY_PREFIXES:
            continue
        if name in LIBRARY_ONLY_NAMES:
            continue
        names.add(name)
    return names


def strip_c(src):
    """Remove comments and string/char literals, preserving everything else."""
    out = []
    i, end = 0, len(src)
    while i < end:
        char = src[i]
        if char == "/" and i + 1 < end and src[i + 1] == "/":
            while i < end and src[i] != "\n":
                i += 1
        elif char == "/" and i + 1 < end and src[i + 1] == "*":
            i += 2
            while i + 1 < end and not (src[i] == "*" and src[i + 1] == "/"):
                i += 1
            i += 2
        elif char in ('"', "'"):
            quote = char
            i += 1
            while i < end and src[i] != quote:
                if src[i] == "\\":
                    i += 1
                i += 1
            i += 1
            out.append(" ")
        else:
            out.append(char)
            i += 1
    return "".join(out)


def measure(srcdir, libc):
    """Count library and bypass call sites across one directory."""
    library, bypass, per_file = Counter(), Counter(), {}
    names = sorted(name for name in os.listdir(srcdir)
                   if name.endswith((".c", ".h")))
    for name in names:
        with open(os.path.join(srcdir, name), encoding="utf-8",
                  errors="replace") as handle:
            text = strip_c(handle.read())
        here_lib = here_raw = 0
        for match in CALL.finditer(text):
            ident = match.group(1)
            if ident.startswith("aksl_"):
                library[ident] += 1
                here_lib += 1
            elif ident in libc:
                bypass[ident] += 1
                here_raw += 1
        if here_lib or here_raw:
            per_file[name] = (here_lib, here_raw)
    return library, bypass, per_file


def rate(bypassed, total):
    return 100.0 * bypassed / total if total else 0.0


def main():
    parser = argparse.ArgumentParser(add_help=True)
    parser.add_argument("srcdir")
    parser.add_argument("--header")
    parser.add_argument("--detail", action="store_true")
    parser.add_argument("--per-file", action="store_true")
    parser.add_argument("--baseline")
    parser.add_argument("--json", action="store_true")
    args = parser.parse_args()

    header = args.header or os.path.join(
        os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
        "include", "akstdlib.h")
    if not os.path.isfile(header):
        sys.stderr.write(f"error: no such header: {header}\n")
        return 2
    if not os.path.isdir(args.srcdir):
        sys.stderr.write(f"error: no such directory: {args.srcdir}\n")
        return 2

    libc = wrapped_libc(header)
    library, bypass, per_file = measure(args.srcdir, libc)
    lib_total, raw_total = sum(library.values()), sum(bypass.values())
    total = lib_total + raw_total

    if args.json:
        print(json.dumps({
            "source": os.path.abspath(args.srcdir),
            "header": os.path.abspath(header),
            "wrapped_libc_names": len(libc),
            "library_calls": lib_total,
            "bypass_calls": raw_total,
            "bypass_pct": round(rate(raw_total, total), 1),
            "library_breakdown": dict(library.most_common()),
            "bypass_breakdown": dict(bypass.most_common()),
            "per_file": {k: {"library": v[0], "bypass": v[1]}
                         for k, v in per_file.items()},
        }, indent=2))
        return 0

    print(f"consumer   : {os.path.abspath(args.srcdir)}")
    print(f"measured against: {os.path.abspath(header)} "
          f"({len(libc)} wrapped libc names)")
    print()
    print(f"libakstdlib calls : {lib_total}")
    print(f"bypassed to libc  : {raw_total}")
    print(f"bypass rate       : {rate(raw_total, total):.1f}%  "
          f"({raw_total}/{total})")

    if args.baseline:
        try:
            was_lib, was_raw = (int(part) for part in args.baseline.split("/"))
        except ValueError:
            sys.stderr.write("error: --baseline wants LIBRARY/BYPASS, "
                             "e.g. 10/119\n")
            return 2
        was_total = was_lib + was_raw
        print()
        print(f"baseline          : {was_lib} / {was_raw}  "
              f"({rate(was_raw, was_total):.1f}% bypass)")
        print(f"change            : {lib_total - was_lib:+d} library, "
              f"{raw_total - was_raw:+d} bypass, "
              f"{rate(raw_total, total) - rate(was_raw, was_total):+.1f} pt")
        print()
        print("A bypass rate is only comparable between two counts of the same")
        print("tree. If the consumer grew between them, compare the rate and")
        print("not the totals -- and say which tree each number came from.")

    if args.detail:
        print("\nbypassed to libc")
        for name, count in bypass.most_common():
            print(f"  {name:<22}{count}")
        print("\ncalls into libakstdlib")
        for name, count in library.most_common():
            print(f"  {name:<22}{count}")

    if args.per_file:
        print("\nper file (library, bypass), worst bypass first")
        order = sorted(per_file.items(), key=lambda kv: (-kv[1][1], kv[0]))
        for name, (lib, raw) in order:
            print(f"  {name:<32}{lib:>5}{raw:>6}")

    return 0


if __name__ == "__main__":
    sys.exit(main())