diff --git a/bench/baseline.json b/bench/baseline.json index 218c5a1..52415c1 100644 --- a/bench/baseline.json +++ b/bench/baseline.json @@ -1,93 +1,256 @@ { "version": 1, - "captured": "2026-05-09", + "captured": "2026-05-20", "captured_via": "bench/run.sh -n 5", - "note": "Baseline for bench/check.py regression detection. Decision-10 thresholds (rc/bump <= 1.3x throughput, p99/median <= 5x latency) are LANGUAGE invariants, not regression-check tolerances. The tolerances below are tuned to absorb run-to-run noise on a quiet developer machine; they are NOT the correctness bar. To update after an intentional change, re-run bench/run.sh and replace the values, recording the reason in the JOURNAL entry that ships the baseline bump. Tolerance convention: max-of-distribution metrics (e.g. *.max_us) get a wider band than percentile metrics (p99, p99_9) because the maximum of a 1000-sample tail-latency distribution has wider natural run-to-run dispersion than a percentile-of-distribution does — codified after the 21'd implicit_at_rc.max_us 25%->30% recalibration.", - + "note": "Baseline for bench/check.py regression detection. The language-invariant thresholds (rc/bump <= 1.3x throughput, p99/median <= 5x latency) are NOT the regression-check tolerances; the per-metric tolerances below are tuned to absorb run-to-run noise on a quiet developer machine. To update after an intentional change, re-run bench/run.sh and replace the values, recording the reason in the commit body that ships the baseline bump. The latency arms gate only on median / p99 / p99_over_median — max_us and p99_9_us were removed on the 2026-05-20 recapture (Gitea #15 / #16) because tail-of-distribution latency metrics are dominated by OS-level jitter (THP defrag, scheduler preemption, IRQ load), not allocator behaviour, and produced 3+ consecutive false-positive REGRESSION rows on byte-identical no-op milestones. See docs/specs/2026-05-20-bench-harness-recalibration.md.", "throughput": { "bench_list_sum": { - "gc_s": { "baseline": 0.137, "tolerance_pct": 10 }, - "bump_s": { "baseline": 0.046, "tolerance_pct": 10 }, - "rc_s": { "baseline": 0.133, "tolerance_pct": 10 }, - "gc_over_bump": { "baseline": 2.98, "tolerance_pct": 8 }, - "rc_over_bump": { "baseline": 2.89, "tolerance_pct": 8 }, - "gc_rss_kb": { "baseline": 137448, "tolerance_pct": 5 }, - "bump_rss_kb": { "baseline": 97696, "tolerance_pct": 5 }, - "rc_rss_kb": { "baseline": 193448, "tolerance_pct": 5 } + "gc_s": { + "baseline": 0.150576, + "tolerance_pct": 10 + }, + "bump_s": { + "baseline": 0.052977, + "tolerance_pct": 10 + }, + "rc_s": { + "baseline": 0.143404, + "tolerance_pct": 10 + }, + "gc_over_bump": { + "baseline": 2.84, + "tolerance_pct": 8 + }, + "rc_over_bump": { + "baseline": 2.71, + "tolerance_pct": 8 + }, + "gc_rss_kb": { + "baseline": 137636.0, + "tolerance_pct": 5 + }, + "bump_rss_kb": { + "baseline": 97436.0, + "tolerance_pct": 5 + }, + "rc_rss_kb": { + "baseline": 193628.0, + "tolerance_pct": 5 + } }, "bench_tree_walk": { - "gc_s": { "baseline": 0.103, "tolerance_pct": 10 }, - "bump_s": { "baseline": 0.038, "tolerance_pct": 10 }, - "rc_s": { "baseline": 0.095, "tolerance_pct": 10 }, - "gc_over_bump": { "baseline": 2.71, "tolerance_pct": 8 }, - "rc_over_bump": { "baseline": 2.50, "tolerance_pct": 8 }, - "gc_rss_kb": { "baseline": 73260, "tolerance_pct": 5 }, - "bump_rss_kb": { "baseline": 55208, "tolerance_pct": 5 }, - "rc_rss_kb": { "baseline": 108956, "tolerance_pct": 5 } + "gc_s": { + "baseline": 0.101088, + "tolerance_pct": 10 + }, + "bump_s": { + "baseline": 0.038371, + "tolerance_pct": 10 + }, + "rc_s": { + "baseline": 0.098525, + "tolerance_pct": 10 + }, + "gc_over_bump": { + "baseline": 2.63, + "tolerance_pct": 8 + }, + "rc_over_bump": { + "baseline": 2.57, + "tolerance_pct": 8 + }, + "gc_rss_kb": { + "baseline": 73316.0, + "tolerance_pct": 5 + }, + "bump_rss_kb": { + "baseline": 55196.0, + "tolerance_pct": 5 + }, + "rc_rss_kb": { + "baseline": 108764.0, + "tolerance_pct": 5 + } }, "bench_closure_chain": { - "gc_s": { "baseline": 0.013, "tolerance_pct": 25 }, - "bump_s": { "baseline": 0.007, "tolerance_pct": 25 }, - "rc_s": { "baseline": 0.029, "tolerance_pct": 20 }, - "gc_over_bump": { "baseline": 1.86, "tolerance_pct": 15 }, - "rc_over_bump": { "baseline": 4.14, "tolerance_pct": 15 }, - "gc_rss_kb": { "baseline": 13688, "tolerance_pct": 15 }, - "bump_rss_kb": { "baseline": 15836, "tolerance_pct": 15 }, - "rc_rss_kb": { "baseline": 39644, "tolerance_pct": 10 } + "gc_s": { + "baseline": 0.012135, + "tolerance_pct": 25 + }, + "bump_s": { + "baseline": 0.007945, + "tolerance_pct": 25 + }, + "rc_s": { + "baseline": 0.031716, + "tolerance_pct": 20 + }, + "gc_over_bump": { + "baseline": 1.53, + "tolerance_pct": 15 + }, + "rc_over_bump": { + "baseline": 3.99, + "tolerance_pct": 15 + }, + "gc_rss_kb": { + "baseline": 13684.0, + "tolerance_pct": 15 + }, + "bump_rss_kb": { + "baseline": 15836.0, + "tolerance_pct": 15 + }, + "rc_rss_kb": { + "baseline": 39652.0, + "tolerance_pct": 10 + } }, "bench_hof_pipeline": { - "gc_s": { "baseline": 0.134, "tolerance_pct": 10 }, - "bump_s": { "baseline": 0.048, "tolerance_pct": 10 }, - "rc_s": { "baseline": 0.136, "tolerance_pct": 10 }, - "gc_over_bump": { "baseline": 2.79, "tolerance_pct": 8 }, - "rc_over_bump": { "baseline": 2.83, "tolerance_pct": 8 }, - "gc_rss_kb": { "baseline": 137768, "tolerance_pct": 5 }, - "bump_rss_kb": { "baseline": 97448, "tolerance_pct": 5 }, - "rc_rss_kb": { "baseline": 193640, "tolerance_pct": 5 } + "gc_s": { + "baseline": 0.149062, + "tolerance_pct": 10 + }, + "bump_s": { + "baseline": 0.051993, + "tolerance_pct": 10 + }, + "rc_s": { + "baseline": 0.143367, + "tolerance_pct": 10 + }, + "gc_over_bump": { + "baseline": 2.87, + "tolerance_pct": 8 + }, + "rc_over_bump": { + "baseline": 2.76, + "tolerance_pct": 8 + }, + "gc_rss_kb": { + "baseline": 137564.0, + "tolerance_pct": 5 + }, + "bump_rss_kb": { + "baseline": 97628.0, + "tolerance_pct": 5 + }, + "rc_rss_kb": { + "baseline": 193816.0, + "tolerance_pct": 5 + } }, "bench_compute_collatz": { - "gc_s": { "baseline": 0.057, "tolerance_pct": 12 }, - "bump_s": { "baseline": 0.056, "tolerance_pct": 12 }, - "rc_s": { "baseline": 0.056, "tolerance_pct": 12 }, - "gc_over_bump": { "baseline": 1.02, "tolerance_pct": 10 }, - "rc_over_bump": { "baseline": 1.00, "tolerance_pct": 10 }, - "gc_rss_kb": { "baseline": 13624, "tolerance_pct": 15 }, - "bump_rss_kb": { "baseline": 13860, "tolerance_pct": 15 }, - "rc_rss_kb": { "baseline": 13880, "tolerance_pct": 15 } + "gc_s": { + "baseline": 0.056628, + "tolerance_pct": 12 + }, + "bump_s": { + "baseline": 0.056266, + "tolerance_pct": 12 + }, + "rc_s": { + "baseline": 0.05629, + "tolerance_pct": 12 + }, + "gc_over_bump": { + "baseline": 1.01, + "tolerance_pct": 10 + }, + "rc_over_bump": { + "baseline": 1.0, + "tolerance_pct": 10 + }, + "gc_rss_kb": { + "baseline": 13860.0, + "tolerance_pct": 15 + }, + "bump_rss_kb": { + "baseline": 14044.0, + "tolerance_pct": 15 + }, + "rc_rss_kb": { + "baseline": 14012.0, + "tolerance_pct": 15 + } }, "bench_list_sum_explicit": { - "gc_s": { "baseline": 0.139, "tolerance_pct": 10 }, - "bump_s": { "baseline": 0.048, "tolerance_pct": 10 }, - "rc_s": { "baseline": 0.152, "tolerance_pct": 10 }, - "gc_over_bump": { "baseline": 2.89, "tolerance_pct": 8 }, - "rc_over_bump": { "baseline": 3.14, "tolerance_pct": 8 }, - "gc_rss_kb": { "baseline": 137636, "tolerance_pct": 5 }, - "bump_rss_kb": { "baseline": 97636, "tolerance_pct": 5 }, - "rc_rss_kb": { "baseline": 142424, "tolerance_pct": 8 } + "gc_s": { + "baseline": 0.1506, + "tolerance_pct": 10 + }, + "bump_s": { + "baseline": 0.053389, + "tolerance_pct": 10 + }, + "rc_s": { + "baseline": 0.15632, + "tolerance_pct": 10 + }, + "gc_over_bump": { + "baseline": 2.82, + "tolerance_pct": 8 + }, + "rc_over_bump": { + "baseline": 2.93, + "tolerance_pct": 8 + }, + "gc_rss_kb": { + "baseline": 137196.0, + "tolerance_pct": 5 + }, + "bump_rss_kb": { + "baseline": 97628.0, + "tolerance_pct": 5 + }, + "rc_rss_kb": { + "baseline": 141988.0, + "tolerance_pct": 8 + } } }, - "latency": { "implicit_at_gc": { - "median_us": { "baseline": 96.4, "tolerance_pct": 15 }, - "p99_us": { "baseline": 7130.0, "tolerance_pct": 20 }, - "p99_9_us": { "baseline": 8131.2, "tolerance_pct": 25 }, - "max_us": { "baseline": 8343.7, "tolerance_pct": 25 }, - "p99_over_median": { "baseline": 73.92, "tolerance_pct": 20 } + "median_us": { + "baseline": 97.6, + "tolerance_pct": 15 + }, + "p99_us": { + "baseline": 7125.8, + "tolerance_pct": 20 + }, + "p99_over_median": { + "baseline": 72.79, + "tolerance_pct": 20 + } }, "explicit_at_rc": { - "median_us": { "baseline": 213.9, "tolerance_pct": 15 }, - "p99_us": { "baseline": 357.5, "tolerance_pct": 25 }, - "p99_9_us": { "baseline": 404.1, "tolerance_pct": 25 }, - "max_us": { "baseline": 413.0, "tolerance_pct": 25 }, - "p99_over_median": { "baseline": 1.66, "tolerance_pct": 25 } + "median_us": { + "baseline": 221.2, + "tolerance_pct": 15 + }, + "p99_us": { + "baseline": 428.8, + "tolerance_pct": 25 + }, + "p99_over_median": { + "baseline": 1.93, + "tolerance_pct": 25 + } }, "implicit_at_rc": { - "median_us": { "baseline": 285.7, "tolerance_pct": 15 }, - "p99_us": { "baseline": 407.1, "tolerance_pct": 20 }, - "p99_9_us": { "baseline": 452.0, "tolerance_pct": 25 }, - "max_us": { "baseline": 477.3, "tolerance_pct": 30 }, - "p99_over_median": { "baseline": 1.43, "tolerance_pct": 20 } + "median_us": { + "baseline": 301.2, + "tolerance_pct": 15 + }, + "p99_us": { + "baseline": 461.8, + "tolerance_pct": 20 + }, + "p99_over_median": { + "baseline": 1.53, + "tolerance_pct": 20 + } } } } diff --git a/bench/orchestrator-stats/2026-05-20-iter-bench-harness-recalibration.1.json b/bench/orchestrator-stats/2026-05-20-iter-bench-harness-recalibration.1.json new file mode 100644 index 0000000..c54f10d --- /dev/null +++ b/bench/orchestrator-stats/2026-05-20-iter-bench-harness-recalibration.1.json @@ -0,0 +1,30 @@ +{ + "iter_id": "bench-harness-recalibration.1", + "date": "2026-05-20", + "mode": "standard", + "outcome": "DONE", + "tasks_total": 4, + "tasks_completed": 4, + "reloops_per_task": { + "1": 0, + "2": 0, + "3": 0, + "4": 0 + }, + "review_loops_spec": 0, + "review_loops_quality": 0, + "blocked_reason": null, + "bench_results": { + "check_py_exit_task3_replay": 0, + "check_py_summary_task3": "57 metrics; 0 regressed, 0 improved beyond tolerance, 57 stable", + "check_py_exit_task4_injection": 1, + "check_py_injection_row": "throughput.bench_list_sum.bump_s 0.026 0.053 +99.57% 10.0% REGRESSION", + "check_py_exit_task4_restored": 0, + "check_py_summary_task4_restored": "57 metrics; 0 regressed, 2 improved beyond tolerance, 55 stable" + }, + "notes": [ + "Task 3 same-HEAD replay passed on first try (no re-run needed for single-run noise mitigation).", + "Task 4 restored replay shows 2 improvements on latency.explicit_at_rc.p99_us / p99_over_median (-33%) — improvements never gate per bench/check.py header; this is exactly the tail-distribution jitter that motivated dropping max_us / p99_9_us.", + "All four tasks land as a single working-tree diff on bench/baseline.json (single-artefact iter)." + ] +}