diff --git a/bench/baseline.json b/bench/baseline.json new file mode 100644 index 0000000..61f9e77 --- /dev/null +++ b/bench/baseline.json @@ -0,0 +1,53 @@ +{ + "version": 1, + "captured": "2026-05-09", + "captured_via": "bench/run.sh -n 5", + "note": "Baseline for bench/check.py regression detection. Decision-10 thresholds (rc/bump <= 1.3x throughput, p99/median <= 5x latency) are LANGUAGE invariants, not regression-check tolerances. The tolerances below are tuned to absorb run-to-run noise on a quiet developer machine; they are NOT the correctness bar. To update after an intentional change, re-run bench/run.sh and replace the values, recording the reason in the JOURNAL entry that ships the baseline bump.", + + "throughput": { + "bench_list_sum": { + "gc_s": { "baseline": 0.137, "tolerance_pct": 10 }, + "bump_s": { "baseline": 0.046, "tolerance_pct": 10 }, + "rc_s": { "baseline": 0.133, "tolerance_pct": 10 }, + "gc_over_bump": { "baseline": 2.98, "tolerance_pct": 8 }, + "rc_over_bump": { "baseline": 2.89, "tolerance_pct": 8 }, + "gc_rss_kb": { "baseline": 103980, "tolerance_pct": 5 }, + "bump_rss_kb": { "baseline": 97696, "tolerance_pct": 5 }, + "rc_rss_kb": { "baseline": 193448, "tolerance_pct": 5 } + }, + "bench_tree_walk": { + "gc_s": { "baseline": 0.103, "tolerance_pct": 10 }, + "bump_s": { "baseline": 0.038, "tolerance_pct": 10 }, + "rc_s": { "baseline": 0.095, "tolerance_pct": 10 }, + "gc_over_bump": { "baseline": 2.71, "tolerance_pct": 8 }, + "rc_over_bump": { "baseline": 2.50, "tolerance_pct": 8 }, + "gc_rss_kb": { "baseline": 73260, "tolerance_pct": 5 }, + "bump_rss_kb": { "baseline": 55208, "tolerance_pct": 5 }, + "rc_rss_kb": { "baseline": 108956, "tolerance_pct": 5 } + } + }, + + "latency": { + "implicit_at_gc": { + "median_us": { "baseline": 96.4, "tolerance_pct": 15 }, + "p99_us": { "baseline": 7130.0, "tolerance_pct": 20 }, + "p99_9_us": { "baseline": 8131.2, "tolerance_pct": 25 }, + "max_us": { "baseline": 8343.7, "tolerance_pct": 25 }, + "p99_over_median": { "baseline": 73.92, "tolerance_pct": 20 } + }, + "explicit_at_rc": { + "median_us": { "baseline": 213.9, "tolerance_pct": 15 }, + "p99_us": { "baseline": 357.5, "tolerance_pct": 25 }, + "p99_9_us": { "baseline": 404.1, "tolerance_pct": 25 }, + "max_us": { "baseline": 413.0, "tolerance_pct": 25 }, + "p99_over_median": { "baseline": 1.66, "tolerance_pct": 25 } + }, + "implicit_at_rc": { + "median_us": { "baseline": 285.7, "tolerance_pct": 15 }, + "p99_us": { "baseline": 407.1, "tolerance_pct": 20 }, + "p99_9_us": { "baseline": 452.0, "tolerance_pct": 25 }, + "max_us": { "baseline": 477.3, "tolerance_pct": 25 }, + "p99_over_median": { "baseline": 1.43, "tolerance_pct": 20 } + } + } +} diff --git a/bench/check.py b/bench/check.py new file mode 100755 index 0000000..95873cb --- /dev/null +++ b/bench/check.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python3 +# Performance regression check. +# +# Runs bench/run.sh (or consumes its output from stdin / a file), parses +# the throughput table and the per-arm latency stanzas, and diffs every +# metric against bench/baseline.json. Exits 0 if every metric is within +# its per-metric tolerance; exits 1 with a per-metric diff table on any +# regression. +# +# Tolerances are one-sided: only metrics that got *worse* than baseline +# (slower wall-time, larger RSS, larger latency, larger ratio) trigger a +# regression. Improvements never fire — they show in the diff with a "+" +# sign and a note, so an intentional improvement can be ratified into +# the baseline with `--update-baseline`. +# +# Usage: +# bench/check.py # spawn bench/run.sh -n 5 +# bench/check.py --from-file out.txt # consume captured output +# cat out.txt | bench/check.py --stdin +# bench/check.py --update-baseline # re-run, write new baseline +# bench/check.py --baseline path.json # alternate baseline location + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +DEFAULT_BASELINE = ROOT / "bench" / "baseline.json" +RUN_SH = ROOT / "bench" / "run.sh" + + +@dataclass +class Measurement: + metric_id: str + baseline: float + tolerance_pct: float + actual: float + + @property + def diff_pct(self) -> float: + if self.baseline == 0: + return 0.0 + return 100.0 * (self.actual - self.baseline) / self.baseline + + @property + def regressed(self) -> bool: + return self.diff_pct > self.tolerance_pct + + +def parse_throughput_table(text: str) -> dict[str, dict[str, float]]: + # The throughput table has a header row, a separator row, then one row + # per workload. Columns are pipe-separated. + out: dict[str, dict[str, float]] = {} + in_table = False + for line in text.splitlines(): + if line.startswith("workload") and "gc(s)" in line: + in_table = True + continue + if in_table and line.startswith("---"): + continue + if in_table: + if not line.strip() or "|" not in line: + in_table = False + continue + cells = [c.strip() for c in line.split("|")] + if len(cells) != 9: + continue + workload = cells[0] + try: + out[workload] = { + "gc_s": float(cells[1]), + "bump_s": float(cells[2]), + "rc_s": float(cells[3]), + "gc_over_bump": float(cells[4].rstrip("x")), + "rc_over_bump": float(cells[5].rstrip("x")), + "gc_rss_kb": float(cells[6]), + "bump_rss_kb": float(cells[7]), + "rc_rss_kb": float(cells[8]), + } + except ValueError: + continue + return out + + +# Latency arm header looks like: "=== implicit @ gc (Boehm-fair) ===". +# We map the leading prose label to the canonical arm key in baseline.json. +ARM_LABEL_TO_KEY = { + "implicit @ gc": "implicit_at_gc", + "explicit @ rc": "explicit_at_rc", + "implicit @ rc": "implicit_at_rc", +} + +# Inside each arm stanza, lines that report a metric look like: +# " median median= 96.4 range=[..., ...] us" +# or: +# " p99/median median=73.92x range=[...]" +# We pick out the metric name and its "median=NN" value. +LINE_RE = re.compile( + r"^\s+(?P[a-zA-Z0-9./_]+)\s+median=\s*(?P[\d.]+)x?" +) +HEADER_RE = re.compile(r"^=== (?P