Aura/crates/aura-bench/src/main.rs

//! `aura-bench` — the wall-clock benchmark harness with committed baselines
//! (issue #251, phase 1). Report-only: timing drift never fails; a determinism
//! fingerprint mismatch does (exit 1), so a fast-but-wrong binary fails the
//! bench instead of pinning a bogus baseline. Infra errors exit 2.

mod baseline;
mod child;
mod host;
mod surfaces;
mod synth;

use baseline::{compare, exit_code, BaselineDoc, Compared, Measured};
use clap::{Parser, Subcommand};
use std::path::{Path, PathBuf};
use surfaces::Sizing;

const SURFACES: &[&str] =
    &["engine_throughput", "ingest_throughput", "campaign_sweep", "campaign_heavy", "cli_fixed_cost"];
/// The in-process surfaces — selecting only these never resolves (or builds)
/// the release `aura` binary. Keep in sync with `measure_surface`.
const LIBRARY_SURFACES: &[&str] = &["engine_throughput", "ingest_throughput"];
const DEFAULT_REPS: u32 = 3;
/// 1-min loadavg above which the banner warns the box is not quiet.
const LOAD_WARN_1M: f64 = 2.0;

#[derive(Parser)]
#[command(name = "aura-bench", about = "wall-clock benchmarks against committed baselines (#251)")]
struct Cli {
    #[command(subcommand)]
    cmd: Cmd,
}

#[derive(Subcommand)]
enum Cmd {
    /// Measure surfaces and compare against committed baselines (report-only).
    Run {
        /// Only these surfaces (repeatable). Default: all.
        #[arg(long)]
        surface: Vec<String>,
        /// Tiny workloads, results written to --out, committed baselines untouched.
        #[arg(long)]
        quick: bool,
        /// Measured repetitions per surface (after 1 discarded warmup).
        #[arg(long, default_value_t = DEFAULT_REPS)]
        reps: u32,
        /// Write measured results as JSON files into this directory.
        #[arg(long)]
        out: Option<PathBuf>,
    },
    /// Measure surfaces and (re)write the committed baselines.
    Pin {
        /// Only these surfaces (repeatable). Default: all.
        #[arg(long)]
        surface: Vec<String>,
        /// Measured repetitions per surface (after 1 discarded warmup).
        #[arg(long, default_value_t = DEFAULT_REPS)]
        reps: u32,
    },
}

fn baselines_dir(ws_root: &Path) -> PathBuf {
    ws_root.join("crates/aura-bench/baselines")
}

fn measure_surface(
    name: &str,
    sizing: Sizing,
    reps: u32,
    aura_bin: &Path,
) -> Result<Measured, String> {
    match name {
        "engine_throughput" => surfaces::run_reps(name, reps, || surfaces::engine::rep(sizing)),
        "ingest_throughput" => {
            let fixture = surfaces::ingest::setup(sizing)?;
            surfaces::run_reps(name, reps, || surfaces::ingest::rep(&fixture))
        }
        "campaign_sweep" => {
            surfaces::run_reps(name, reps, || surfaces::campaign::sweep_rep(aura_bin, sizing))
        }
        "campaign_heavy" => {
            surfaces::run_reps(name, reps, || surfaces::campaign::heavy_rep(aura_bin, sizing))
        }
        "cli_fixed_cost" => surfaces::run_reps(name, reps, || surfaces::fixed_cost::rep(aura_bin)),
        other => Err(format!("unknown surface: {other}")),
    }
}

fn selected(filter: &[String]) -> Result<Vec<&'static str>, String> {
    if filter.is_empty() {
        return Ok(SURFACES.to_vec());
    }
    let mut out = Vec::new();
    for f in filter {
        let s = SURFACES
            .iter()
            .find(|s| **s == f.as_str())
            .ok_or_else(|| format!("unknown surface {f:?}; known: {SURFACES:?}"))?;
        out.push(*s);
    }
    Ok(out)
}

fn fmt_metric(name: &str, v: f64) -> String {
    if name.ends_with("_ms") {
        format!("{name} {v:.1}ms")
    } else if name == "bars_per_s" {
        format!("{name} {v:.0}")
    } else if name == "wall_s" {
        format!("{name} {v:.3}s")
    } else {
        format!("{name} {v:.1}")
    }
}

fn report_line(m: &Measured, c: Option<&Compared>) -> String {
    let metrics = m
        .metrics
        .iter()
        .map(|(k, v)| fmt_metric(k, *v))
        .collect::<Vec<_>>()
        .join("   ");
    let verdict = match c {
        None => "no baseline (pin to create)".to_string(),
        Some(c) if c.fingerprint_changed => "fingerprint CHANGED".to_string(),
        Some(c) => {
            let drift = c
                .deltas
                .iter()
                .map(|(k, d)| format!("{k} {:+.1}%", d * 100.0))
                .collect::<Vec<_>>()
                .join(" ");
            let notice = if c.notices.is_empty() { "" } else { "  NOTICE" };
            format!("Δ {drift}{notice}   fingerprint OK")
        }
    };
    format!("{:<20} {metrics}   {verdict}", m.surface)
}

fn load_baseline(dir: &Path, surface: &str) -> Option<BaselineDoc> {
    let text = std::fs::read_to_string(dir.join(format!("{surface}.json"))).ok()?;
    serde_json::from_str(&text).ok()
}

/// One `Measured` result plus a host + rep count becomes a `BaselineDoc`,
/// whether it's about to be committed (`pin`) or dropped into `--out` (`run`).
fn to_baseline_doc(m: &Measured, reps: u32, host: &baseline::HostInfo) -> BaselineDoc {
    BaselineDoc {
        surface: m.surface.clone(),
        metrics: m.metrics.clone(),
        fingerprint: m.fingerprint.clone(),
        reps,
        host: host.clone(),
        profile: "release".to_string(),
        commit: host::git_commit(),
        date: host::today(),
    }
}

fn main() {
    if cfg!(debug_assertions) {
        eprintln!("aura-bench measures nothing in a debug build; run via `cargo run --release -p aura-bench`");
        std::process::exit(2);
    }
    let cli = Cli::parse();
    let ws_root = child::workspace_root();
    let code = match cli.cmd {
        Cmd::Run { surface, quick, reps, out } => cmd_run(&ws_root, &surface, quick, reps, out),
        Cmd::Pin { surface, reps } => cmd_pin(&ws_root, &surface, reps),
    };
    std::process::exit(code);
}

fn prepare(ws_root: &Path, filter: &[String]) -> Result<(Vec<&'static str>, PathBuf), String> {
    let names = selected(filter)?;
    let needs_bin = names.iter().any(|n| !LIBRARY_SURFACES.contains(n));
    let bin = if needs_bin {
        child::resolve_aura_bin(ws_root)?
    } else {
        PathBuf::from("aura-unused")
    };
    Ok((names, bin))
}

fn banner(quick: bool) {
    let h = host::host_info();
    let load = host::loadavg_1m().map(|l| format!("{l:.1}")).unwrap_or_else(|| "?".to_string());
    let mode = if quick { " QUICK (mechanics only)" } else { "" };
    println!("aura-bench: host {} ({} cores), load {load}{mode}", h.hostname, h.nproc);
    if let Some(l) = host::loadavg_1m()
        && l > LOAD_WARN_1M
    {
        println!("WARNING: loadavg {l:.1} > {LOAD_WARN_1M} — measurements belong on a quiet box");
    }
}

fn cmd_run(
    ws_root: &Path,
    filter: &[String],
    quick: bool,
    reps: u32,
    out: Option<PathBuf>,
) -> i32 {
    let (names, bin) = match prepare(ws_root, filter) {
        Ok(v) => v,
        Err(e) => return infra(&e),
    };
    banner(quick);
    let sizing = Sizing { quick };
    let reps = if quick { 1 } else { reps };
    let dir = baselines_dir(ws_root);
    let this_host = host::host_info();
    let mut compared: Vec<Option<Compared>> = Vec::new();
    let mut host_mismatch = false;
    for name in &names {
        let m = match measure_surface(name, sizing, reps, &bin) {
            Ok(m) => m,
            Err(e) => return infra(&format!("surface {name}: {e}")),
        };
        let b = if quick { None } else { load_baseline(&dir, name) };
        let c = b.as_ref().map(|b| {
            let c = compare(b, &m, &this_host);
            host_mismatch |= c.host_mismatch;
            c
        });
        println!("{}", report_line(&m, c.as_ref()));
        if let (Some(b), Some(c)) = (&b, &c)
            && c.fingerprint_changed
        {
            println!("  baseline: {}", b.fingerprint);
            println!("  measured: {}", m.fingerprint);
            println!(
                "  The workload computed different results. If a behaviour change is\n  \
                 intended, re-pin (aura-bench pin); otherwise this is a correctness bug."
            );
        }
        if let Some(out_dir) = &out
            && let Err(e) = write_result(out_dir, &m, reps, &this_host)
        {
            return infra(&e);
        }
        compared.push(c);
    }
    if host_mismatch {
        println!("HOST MISMATCH (informational): baselines were pinned on a different host");
    }
    let notices: usize = compared.iter().flatten().map(|c| c.notices.len()).sum();
    if notices > 0 {
        println!(
            "{notices} NOTICE(s) (drift >= 10%) — report-only, nothing failed. Re-pin: aura-bench pin"
        );
    }
    exit_code(&compared)
}

fn cmd_pin(ws_root: &Path, filter: &[String], reps: u32) -> i32 {
    let (names, bin) = match prepare(ws_root, filter) {
        Ok(v) => v,
        Err(e) => return infra(&e),
    };
    banner(false);
    let dir = baselines_dir(ws_root);
    if let Err(e) = std::fs::create_dir_all(&dir) {
        return infra(&format!("create baselines dir: {e}"));
    }
    let this_host = host::host_info();
    for name in &names {
        let m = match measure_surface(name, Sizing { quick: false }, reps, &bin) {
            Ok(m) => m,
            Err(e) => return infra(&format!("surface {name}: {e}")),
        };
        let doc = to_baseline_doc(&m, reps, &this_host);
        let path = dir.join(format!("{name}.json"));
        let json = serde_json::to_string_pretty(&doc).expect("baseline serializes");
        if let Err(e) = std::fs::write(&path, json + "\n") {
            return infra(&format!("write {}: {e}", path.display()));
        }
        println!("pinned {name} -> {}", path.display());
    }
    0
}

fn write_result(out_dir: &Path, m: &Measured, reps: u32, h: &baseline::HostInfo) -> Result<(), String> {
    std::fs::create_dir_all(out_dir).map_err(|e| format!("create out dir: {e}"))?;
    let doc = to_baseline_doc(m, reps, h);
    let json = serde_json::to_string_pretty(&doc).expect("result serializes");
    std::fs::write(out_dir.join(format!("{}.json", m.surface)), json + "\n")
        .map_err(|e| format!("write result: {e}"))
}

fn infra(msg: &str) -> i32 {
    eprintln!("aura-bench: infra error: {msg}");
    2
}

#[cfg(test)]
mod tests {
    use super::*;

    fn measured(surface: &str, metrics: &[(&str, f64)], fp: &str) -> Measured {
        Measured {
            surface: surface.to_string(),
            metrics: metrics.iter().map(|(k, v)| (k.to_string(), *v)).collect(),
            fingerprint: fp.to_string(),
        }
    }

    /// selected() with no filter runs every registered surface, in registration order.
    #[test]
    fn selected_empty_filter_returns_all_surfaces() {
        assert_eq!(selected(&[]).unwrap(), SURFACES.to_vec());
    }

    /// selected() with a known name narrows to exactly that surface.
    #[test]
    fn selected_known_name_returns_that_surface() {
        assert_eq!(selected(&["engine_throughput".to_string()]).unwrap(), vec!["engine_throughput"]);
    }

    /// selected() rejects an unknown surface name instead of silently dropping it,
    /// so a `--surface` typo fails loudly rather than measuring nothing.
    #[test]
    fn selected_unknown_name_is_rejected() {
        let err = selected(&["not_a_surface".to_string()]).unwrap_err();
        assert!(err.contains("not_a_surface"), "error should name the bad surface: {err}");
    }

    /// fmt_metric renders *_ms metrics with millisecond precision and unit suffix.
    #[test]
    fn fmt_metric_formats_ms_suffixed_metrics() {
        assert_eq!(fmt_metric("help_ms", 12.345), "help_ms 12.3ms");
    }

    /// fmt_metric renders bars_per_s as a rounded integer (throughput, not fractional).
    #[test]
    fn fmt_metric_formats_bars_per_s_without_decimals() {
        assert_eq!(fmt_metric("bars_per_s", 812345.6), "bars_per_s 812346");
    }

    /// fmt_metric renders wall_s with millisecond-precision seconds.
    #[test]
    fn fmt_metric_formats_wall_s_with_millis_precision() {
        assert_eq!(fmt_metric("wall_s", 1.23456), "wall_s 1.235s");
    }

    /// fmt_metric falls back to 1-decimal generic formatting for any other metric name.
    #[test]
    fn fmt_metric_falls_back_to_generic_formatting() {
        assert_eq!(fmt_metric("peak_rss_mb", 42.16), "peak_rss_mb 42.2");
    }

    /// report_line without a baseline tells the operator to pin one, rather than
    /// implying a comparison happened.
    #[test]
    fn report_line_without_baseline_prompts_to_pin() {
        let m = measured("s", &[("wall_s", 1.0)], "fp");
        let line = report_line(&m, None);
        assert!(line.contains("no baseline (pin to create)"), "{line}");
    }

    /// report_line surfaces a fingerprint mismatch distinctly from ordinary timing
    /// drift — this is the correctness-vs-performance line the whole harness rests on.
    #[test]
    fn report_line_flags_fingerprint_change() {
        let m = measured("s", &[("wall_s", 1.0)], "new");
        let c = Compared { deltas: vec![], notices: vec![], fingerprint_changed: true, host_mismatch: false };
        let line = report_line(&m, Some(&c));
        assert!(line.contains("fingerprint CHANGED"), "{line}");
    }

    /// report_line prints per-metric drift percentages and an OK fingerprint when
    /// nothing changed.
    #[test]
    fn report_line_reports_drift_and_fingerprint_ok() {
        let m = measured("s", &[("wall_s", 1.0)], "fp");
        let c = Compared {
            deltas: vec![("wall_s".to_string(), 0.05)],
            notices: vec![],
            fingerprint_changed: false,
            host_mismatch: false,
        };
        let line = report_line(&m, Some(&c));
        assert!(line.contains("wall_s +5.0%"), "{line}");
        assert!(line.contains("fingerprint OK"), "{line}");
    }

    /// `prepare` only resolves the real release `aura` binary when a selected
    /// surface actually spawns a child process; a library-only selection
    /// (`engine_throughput`/`ingest_throughput`) must never touch the
    /// filesystem to build or locate it. Protects the fast-iteration path:
    /// a regression here would force a multi-second `cargo build --release`
    /// onto every `aura-bench run --surface engine_throughput`, defeating
    /// the point of the in-process surfaces. Proven by using a nonexistent
    /// `ws_root` — `resolve_aura_bin` would error immediately if `prepare`
    /// ever called it for this filter.
    /// Pins the hand-synced `LIBRARY_SURFACES` list against `measure_surface`
    /// itself: every listed name must be a registered surface AND actually
    /// measurable without a real binary path (quick mode, 1 rep). A surface
    /// added to the list without being binary-free fails here — the silent
    /// failure mode would otherwise be a needless release build on every
    /// library-only invocation.
    #[test]
    fn library_surfaces_measure_without_the_binary() {
        for name in LIBRARY_SURFACES {
            assert!(SURFACES.contains(name), "{name} must be a registered surface");
            let fake_bin = Path::new("/nonexistent-aura-binary-for-library-surface-test");
            measure_surface(name, Sizing { quick: true }, 1, fake_bin)
                .unwrap_or_else(|e| panic!("library surface {name} must run without the binary: {e}"));
        }
    }

    #[test]
    fn prepare_skips_binary_resolution_for_library_only_surfaces() {
        let fake_root = Path::new("/nonexistent-ws-root-for-aura-bench-test");
        let filter = vec!["engine_throughput".to_string(), "ingest_throughput".to_string()];
        let (names, bin) = prepare(fake_root, &filter).expect("must not try to resolve a binary");
        assert_eq!(names, vec!["engine_throughput", "ingest_throughput"]);
        assert_eq!(bin, PathBuf::from("aura-unused"));
    }

    /// `write_result`'s `--out` artifact is schema-identical to a committed
    /// baseline (round-trips through `BaselineDoc`) — the property that makes
    /// a `--quick --out` result directly promotable/comparable, never a
    /// shadow format that silently drifts from what `pin` writes.
    #[test]
    fn write_result_round_trips_measured_into_baseline_shaped_json() {
        let scratch = crate::synth::ScratchDir::new("write-result-test").expect("scratch dir");
        let m = measured("cli_fixed_cost", &[("help_ms", 1.5), ("run_ms", 3.0)], "run_line_fnv=abc");
        let host = baseline::HostInfo { hostname: "h".to_string(), nproc: 8 };
        write_result(&scratch.0, &m, 2, &host).expect("write_result must succeed");
        let doc = load_baseline(&scratch.0, "cli_fixed_cost")
            .expect("write_result's file must be a loadable BaselineDoc");
        assert_eq!(doc.surface, "cli_fixed_cost");
        assert_eq!(doc.metrics, m.metrics);
        assert_eq!(doc.fingerprint, "run_line_fnv=abc");
        assert_eq!(doc.reps, 2);
        assert_eq!(doc.profile, "release");
    }

    /// The full CLI driver glue's core promise: `pin` then `run` against the
    /// baseline it just wrote reports zero fingerprint drift — a scratch
    /// `ws_root` keeps this fully isolated from the real committed
    /// `baselines/` directory (never touches it). If `cmd_pin` and `cmd_run`
    /// ever disagreed on what a surface measures (e.g. different `Sizing`,
    /// different metric keys), a fresh `pin` would immediately fail its own
    /// `run` — exactly the workflow the README documents ("re-pin, the
    /// baseline diff is reviewed like any other commit").
    ///
    /// Ignored in the default suite: it runs the engine surface at full size
    /// (1,000,000 bars, twice) in an unoptimized `dev`-profile test binary —
    /// the suite-wallclock discipline reserves full-size runs for release
    /// builds. Run it explicitly: `cargo test -p aura-bench -- --ignored`.
    #[test]
    #[ignore = "full-size (1M bar) engine surface in a dev-profile build; run via -- --ignored"]
    fn pin_then_run_round_trip_reports_zero_drift() {
        let scratch = crate::synth::ScratchDir::new("pin-run-roundtrip").expect("scratch dir");
        let filter = vec!["engine_throughput".to_string()];
        let pin_code = cmd_pin(&scratch.0, &filter, 1);
        assert_eq!(pin_code, 0, "pin must succeed against an isolated scratch ws_root");
        let run_code = cmd_run(&scratch.0, &filter, false, 1, None);
        assert_eq!(run_code, 0, "running right after pinning must report zero fingerprint drift");
    }
}