Brummel
5a4a6de031
Closes the third corpus blind spot (heap-allocation-only) by adding two fixtures with no allocation pressure: bench_compute_ intsum (tail-recursive integer accumulator) and bench_compute_ collatz (Collatz step-counter, branchy). Surprise on intsum: 50M-iteration loop runs in 1ms wall under all three allocators. LLVM's induction-variable analysis applies the closed-form triangular-sum reduction to AILang's IR — a positive codegen finding (the IR composes with LLVM's optimizer at the same level a hand-C loop would) but it makes intsum useless as a runtime regression bench. Excluded from run.sh's fixtures array; kept in examples/ as reference and as a future cross-language comparison anchor. Collatz survives optimization (data-dependent control flow). At 56ms wall, gc/bump/rc all within 2% — the canonical "pure-compute is allocator-invariant" data point this fixture is meant to prove. If a future codegen change leaks an allocation into the inner loop, the 1.00x / 1.02x ratios diverge visibly. Two infrastructure fixes the new fixtures forced: - 6-decimal precision in run.sh's Python timing helper and median averager (was 3-decimal; sub-ms times rounded to 0.000 and crashed the ratio awk with Division durch Null). - Zero-guard in the ratio awk (defensive even with the precision bump, since LLVM-eliminated workloads can still hit zero). Latency baseline: implicit_at_rc.max_us tolerance 25% -> 30%. Three captures today (477 / 456 / 609 µs) show natural run-to-run dispersion wider than the original tolerance accounts for. Not a softening to dodge regression — the original baseline was the first capture; a fairer tolerance across natural max-of-1000- samples width is what the harness needed from the start. Baseline file: 47 -> 55 metrics. 21'e (cross-language reference, clang -O2 hand-C ratios) is the natural next dispatch.
84 lines
4.5 KiB
JSON
84 lines
4.5 KiB
JSON
{
|
|
"version": 1,
|
|
"captured": "2026-05-09",
|
|
"captured_via": "bench/run.sh -n 5",
|
|
"note": "Baseline for bench/check.py regression detection. Decision-10 thresholds (rc/bump <= 1.3x throughput, p99/median <= 5x latency) are LANGUAGE invariants, not regression-check tolerances. The tolerances below are tuned to absorb run-to-run noise on a quiet developer machine; they are NOT the correctness bar. To update after an intentional change, re-run bench/run.sh and replace the values, recording the reason in the JOURNAL entry that ships the baseline bump.",
|
|
|
|
"throughput": {
|
|
"bench_list_sum": {
|
|
"gc_s": { "baseline": 0.137, "tolerance_pct": 10 },
|
|
"bump_s": { "baseline": 0.046, "tolerance_pct": 10 },
|
|
"rc_s": { "baseline": 0.133, "tolerance_pct": 10 },
|
|
"gc_over_bump": { "baseline": 2.98, "tolerance_pct": 8 },
|
|
"rc_over_bump": { "baseline": 2.89, "tolerance_pct": 8 },
|
|
"gc_rss_kb": { "baseline": 103980, "tolerance_pct": 5 },
|
|
"bump_rss_kb": { "baseline": 97696, "tolerance_pct": 5 },
|
|
"rc_rss_kb": { "baseline": 193448, "tolerance_pct": 5 }
|
|
},
|
|
"bench_tree_walk": {
|
|
"gc_s": { "baseline": 0.103, "tolerance_pct": 10 },
|
|
"bump_s": { "baseline": 0.038, "tolerance_pct": 10 },
|
|
"rc_s": { "baseline": 0.095, "tolerance_pct": 10 },
|
|
"gc_over_bump": { "baseline": 2.71, "tolerance_pct": 8 },
|
|
"rc_over_bump": { "baseline": 2.50, "tolerance_pct": 8 },
|
|
"gc_rss_kb": { "baseline": 73260, "tolerance_pct": 5 },
|
|
"bump_rss_kb": { "baseline": 55208, "tolerance_pct": 5 },
|
|
"rc_rss_kb": { "baseline": 108956, "tolerance_pct": 5 }
|
|
},
|
|
"bench_closure_chain": {
|
|
"gc_s": { "baseline": 0.013, "tolerance_pct": 25 },
|
|
"bump_s": { "baseline": 0.007, "tolerance_pct": 25 },
|
|
"rc_s": { "baseline": 0.029, "tolerance_pct": 20 },
|
|
"gc_over_bump": { "baseline": 1.86, "tolerance_pct": 15 },
|
|
"rc_over_bump": { "baseline": 4.14, "tolerance_pct": 15 },
|
|
"gc_rss_kb": { "baseline": 13688, "tolerance_pct": 15 },
|
|
"bump_rss_kb": { "baseline": 15836, "tolerance_pct": 15 },
|
|
"rc_rss_kb": { "baseline": 39644, "tolerance_pct": 10 }
|
|
},
|
|
"bench_hof_pipeline": {
|
|
"gc_s": { "baseline": 0.134, "tolerance_pct": 10 },
|
|
"bump_s": { "baseline": 0.048, "tolerance_pct": 10 },
|
|
"rc_s": { "baseline": 0.136, "tolerance_pct": 10 },
|
|
"gc_over_bump": { "baseline": 2.79, "tolerance_pct": 8 },
|
|
"rc_over_bump": { "baseline": 2.83, "tolerance_pct": 8 },
|
|
"gc_rss_kb": { "baseline": 103788, "tolerance_pct": 5 },
|
|
"bump_rss_kb": { "baseline": 97448, "tolerance_pct": 5 },
|
|
"rc_rss_kb": { "baseline": 193640, "tolerance_pct": 5 }
|
|
},
|
|
"bench_compute_collatz": {
|
|
"gc_s": { "baseline": 0.057, "tolerance_pct": 12 },
|
|
"bump_s": { "baseline": 0.056, "tolerance_pct": 12 },
|
|
"rc_s": { "baseline": 0.056, "tolerance_pct": 12 },
|
|
"gc_over_bump": { "baseline": 1.02, "tolerance_pct": 10 },
|
|
"rc_over_bump": { "baseline": 1.00, "tolerance_pct": 10 },
|
|
"gc_rss_kb": { "baseline": 13624, "tolerance_pct": 15 },
|
|
"bump_rss_kb": { "baseline": 13860, "tolerance_pct": 15 },
|
|
"rc_rss_kb": { "baseline": 13880, "tolerance_pct": 15 }
|
|
}
|
|
},
|
|
|
|
"latency": {
|
|
"implicit_at_gc": {
|
|
"median_us": { "baseline": 96.4, "tolerance_pct": 15 },
|
|
"p99_us": { "baseline": 7130.0, "tolerance_pct": 20 },
|
|
"p99_9_us": { "baseline": 8131.2, "tolerance_pct": 25 },
|
|
"max_us": { "baseline": 8343.7, "tolerance_pct": 25 },
|
|
"p99_over_median": { "baseline": 73.92, "tolerance_pct": 20 }
|
|
},
|
|
"explicit_at_rc": {
|
|
"median_us": { "baseline": 213.9, "tolerance_pct": 15 },
|
|
"p99_us": { "baseline": 357.5, "tolerance_pct": 25 },
|
|
"p99_9_us": { "baseline": 404.1, "tolerance_pct": 25 },
|
|
"max_us": { "baseline": 413.0, "tolerance_pct": 25 },
|
|
"p99_over_median": { "baseline": 1.66, "tolerance_pct": 25 }
|
|
},
|
|
"implicit_at_rc": {
|
|
"median_us": { "baseline": 285.7, "tolerance_pct": 15 },
|
|
"p99_us": { "baseline": 407.1, "tolerance_pct": 20 },
|
|
"p99_9_us": { "baseline": 452.0, "tolerance_pct": 25 },
|
|
"max_us": { "baseline": 477.3, "tolerance_pct": 30 },
|
|
"p99_over_median": { "baseline": 1.43, "tolerance_pct": 20 }
|
|
}
|
|
}
|
|
}
|