#!/usr/bin/env python3 # Compile-time regression check. # # Times `ail check FILE` and `ail build --opt=-O0 FILE` over a curated # corpus, drops the slowest run, takes the median of the rest, diffs # against bench/baseline_compile.json. Exits 0 if every metric is within # tolerance, 1 on any regression, 2 on infrastructure failure (missing # fixture, ail spawn error). # # Methodology caveats this bench cannot work around: # - Wall time of `ail check` is dominated by subprocess spawn (~5-10ms # on Linux); typechecker work below that scale is invisible. # - Wall time of `ail build` is dominated by clang's link step # (~100ms+); the AILang IR-emit + codegen contribution is a # small slice of the total. # Both are still useful as catastrophe detectors (10x slowdowns visible) # even if subtler regressions need a profiler. The right tool for finer- # grained codegen perf is the runtime bench (bench/run.sh), not this one. # # Usage: # bench/compile_check.py # run bench, exit 0/1/2 # bench/compile_check.py -n 10 # more runs, tighter median # bench/compile_check.py --update-baseline # bench/compile_check.py --baseline path # alternate baseline file from __future__ import annotations import argparse import json import subprocess import sys import tempfile import time from pathlib import Path ROOT = Path(__file__).resolve().parent.parent DEFAULT_BASELINE = ROOT / "bench" / "baseline_compile.json" AIL = ROOT / "target" / "release" / "ail" EXAMPLES = ROOT / "examples" # Curated corpus. Pinned by name so the baseline keys are stable across # example-directory churn. Each entry must have a corresponding # `.ail` under examples/. CORPUS = [ # Surface coverage: each fixture exercises a different feature set. "hello", # IO baseline "list_map_poly", # polymorphism + HOF "local_rec_capture", # let-rec capture, closure path "borrow_own_demo", # explicit modes "nested_pat", # nested pattern matching # Bench fixtures (correlation with bench/check.py runtime baseline). "bench_list_sum", "bench_tree_walk", "bench_closure_chain", "bench_hof_pipeline", "bench_compute_intsum", # 21'd; LLVM-folded at runtime, but compile path is normal "bench_compute_collatz", # 21'd "bench_list_sum_explicit", # 21'f; explicit-mode pair fixture ] def time_one(args: list[str]) -> float: """Run a command, return wall-time in milliseconds. Raises if exit != 0.""" t0 = time.monotonic() proc = subprocess.run(args, capture_output=True) t1 = time.monotonic() if proc.returncode != 0: raise RuntimeError( f"command failed (exit {proc.returncode}): {' '.join(args)}\n" f"stderr: {proc.stderr.decode(errors='replace')}" ) return (t1 - t0) * 1000.0 def median_drop_slowest(runs: list[float]) -> float: """Drop the slowest run, return median of the rest.""" if len(runs) < 2: return runs[0] if runs else 0.0 kept = sorted(runs)[:-1] n = len(kept) if n % 2 == 1: return kept[n // 2] return 0.5 * (kept[n // 2 - 1] + kept[n // 2]) def measure(num_runs: int) -> dict[str, dict[str, float]]: """Return {section: {fixture: ms}}. Aborts on missing fixture / spawn fail.""" if not AIL.is_file(): print(f"missing release ail binary at {AIL}; building...", file=sys.stderr) subprocess.run( ["cargo", "build", "--release", "-p", "ail"], cwd=str(ROOT), check=True, capture_output=True, ) out: dict[str, dict[str, float]] = {"check_ms": {}, "build_O0_ms": {}} with tempfile.NamedTemporaryFile(prefix="bench_compile_", delete=False) as tf: out_path = tf.name try: for fixture in CORPUS: src = EXAMPLES / f"{fixture}.ail" if not src.is_file(): print(f"missing fixture: {src}", file=sys.stderr) sys.exit(2) try: check_runs = [time_one([str(AIL), "check", str(src)]) for _ in range(num_runs)] build_runs = [time_one([str(AIL), "build", "--opt=-O0", str(src), "-o", out_path]) for _ in range(num_runs)] except RuntimeError as e: print(f"spawn error on {fixture}: {e}", file=sys.stderr) sys.exit(2) out["check_ms"][fixture] = median_drop_slowest(check_runs) out["build_O0_ms"][fixture] = median_drop_slowest(build_runs) print(f" {fixture:<32} check={out['check_ms'][fixture]:6.1f}ms " f"build={out['build_O0_ms'][fixture]:6.1f}ms", file=sys.stderr) finally: try: Path(out_path).unlink() except FileNotFoundError: pass return out def diff_report(measured: dict, baseline: dict) -> tuple[str, bool]: rows = [] has_regression = False for section in ("check_ms", "build_O0_ms"): for fixture, spec in baseline.get(section, {}).items(): actual = measured[section].get(fixture) if actual is None: rows.append((f"{section}.{fixture}", spec["baseline_ms"], None, None, spec["tolerance_pct"], "MISSING")) has_regression = True continue base = spec["baseline_ms"] tol = spec["tolerance_pct"] diff = 100.0 * (actual - base) / base if base else 0.0 if diff > tol: status = "REGRESSION" has_regression = True elif diff < -tol: status = "improvement" else: status = "ok" rows.append((f"{section}.{fixture}", base, actual, diff, tol, status)) lines = [] lines.append(f"{'metric':<48} {'baseline':>10} {'actual':>10} {'diff':>9} {'tol':>6} status") lines.append("-" * 100) for metric, base, actual, diff, tol, status in rows: if actual is None: lines.append(f"{metric:<48} {base:>10.1f} {'-':>10} {'-':>9} {tol:>5.1f}% {status}") else: lines.append( f"{metric:<48} {base:>10.1f} {actual:>10.1f} " f"{diff:>+8.2f}% {tol:>5.1f}% {status}" ) regressed = sum(1 for r in rows if r[5] == "REGRESSION") improved = sum(1 for r in rows if r[5] == "improvement") stable = len(rows) - regressed - improved lines.append("") lines.append(f"summary: {len(rows)} metrics; " f"{regressed} regressed, {improved} improved beyond tolerance, " f"{stable} stable") return "\n".join(lines), has_regression def write_baseline(measured: dict, baseline_path: Path) -> None: today = subprocess.check_output(["date", "+%Y-%m-%d"], text=True).strip() if baseline_path.exists(): existing = json.loads(baseline_path.read_text()) check_tols = {f: spec.get("tolerance_pct", 25) for f, spec in existing.get("check_ms", {}).items()} build_tols = {f: spec.get("tolerance_pct", 20) for f, spec in existing.get("build_O0_ms", {}).items()} else: check_tols = {} build_tols = {} new = { "version": 1, "captured": today, "captured_via": "bench/compile_check.py", "note": "Compile-time bench. `check_ms` is `ail check FILE`; `build_O0_ms` is `ail build --opt=-O0 FILE`. Wall-clock includes subprocess spawn (~5-10 ms on Linux) and, for build, the clang link step. Tolerances are tuned for noise on a quiet developer machine, not as the language correctness bar.", "check_ms": {}, "build_O0_ms": {}, } for fixture, ms in measured["check_ms"].items(): new["check_ms"][fixture] = { "baseline_ms": round(ms, 2), "tolerance_pct": check_tols.get(fixture, 25), } for fixture, ms in measured["build_O0_ms"].items(): new["build_O0_ms"][fixture] = { "baseline_ms": round(ms, 2), "tolerance_pct": build_tols.get(fixture, 20), } baseline_path.write_text(json.dumps(new, indent=2) + "\n") print(f">>> wrote new baseline to {baseline_path}", file=sys.stderr) def main() -> int: ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("-n", "--runs", type=int, default=5, help="runs per fixture per op; min 2, default 5") ap.add_argument("--baseline", type=Path, default=DEFAULT_BASELINE) ap.add_argument("--update-baseline", action="store_true", help="re-measure, then overwrite baseline_compile.json") args = ap.parse_args() if args.runs < 2: print("--runs must be >= 2", file=sys.stderr) return 2 print(f">>> measuring {len(CORPUS)} fixtures, {args.runs} runs each " f"(check + build_O0)", file=sys.stderr) measured = measure(args.runs) if args.update_baseline: write_baseline(measured, args.baseline) return 0 if not args.baseline.exists(): print(f"no baseline at {args.baseline}; create one with --update-baseline", file=sys.stderr) return 2 baseline = json.loads(args.baseline.read_text()) report, has_regression = diff_report(measured, baseline) print(report) return 1 if has_regression else 0 if __name__ == "__main__": sys.exit(main())