Aura/crates/aura-bench/src/baseline.rs

//! Baseline documents and the report-only comparison core. Pure functions —
//! everything here is unit-tested without running a single benchmark.

use serde::{Deserialize, Serialize};
use std::collections::BTreeMap;

/// One committed baseline: `baselines/<surface>.json`.
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct BaselineDoc {
    pub surface: String,
    /// Flat metric map, e.g. `bars_per_s`, `wall_s`, `peak_rss_mb`.
    pub metrics: BTreeMap<String, f64>,
    /// Opaque determinism fingerprint; compared for byte equality.
    pub fingerprint: String,
    pub reps: u32,
    pub host: HostInfo,
    pub profile: String,
    pub commit: String,
    pub date: String,
}

#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
pub struct HostInfo {
    pub hostname: String,
    pub nproc: u32,
}

/// One measurement result, produced by a surface run.
#[derive(Clone, Debug, PartialEq)]
pub struct Measured {
    pub surface: String,
    pub metrics: BTreeMap<String, f64>,
    pub fingerprint: String,
}

/// Timing drift at or beyond this fraction is reported as a NOTICE line.
pub const NOTICE_THRESHOLD: f64 = 0.10;

#[derive(Clone, Debug, PartialEq)]
pub struct Compared {
    /// Per-metric relative delta, `(name, (measured - baseline) / baseline)`.
    /// Metrics missing on either side are skipped.
    pub deltas: Vec<(String, f64)>,
    /// Metric names whose |delta| >= NOTICE_THRESHOLD.
    pub notices: Vec<String>,
    pub fingerprint_changed: bool,
    pub host_mismatch: bool,
}

pub fn compare(baseline: &BaselineDoc, measured: &Measured, host: &HostInfo) -> Compared {
    let mut deltas = Vec::new();
    let mut notices = Vec::new();
    for (name, base) in &baseline.metrics {
        let Some(now) = measured.metrics.get(name) else { continue };
        if *base == 0.0 {
            continue; // no meaningful relative delta
        }
        let d = (now - base) / base;
        if d.abs() >= NOTICE_THRESHOLD {
            notices.push(name.clone());
        }
        deltas.push((name.clone(), d));
    }
    Compared {
        deltas,
        notices,
        fingerprint_changed: baseline.fingerprint != measured.fingerprint,
        host_mismatch: baseline.host != *host,
    }
}

/// The process exit code for a full run: 1 if ANY surface's fingerprint
/// changed, else 0. (Infra errors exit 2 before this is ever consulted;
/// timing drift is report-only by design — the #251 cadence decision.)
pub fn exit_code(compared: &[Option<Compared>]) -> i32 {
    if compared.iter().flatten().any(|c| c.fingerprint_changed) { 1 } else { 0 }
}

#[cfg(test)]
mod tests {
    use super::*;

    fn doc(metrics: &[(&str, f64)], fp: &str) -> BaselineDoc {
        BaselineDoc {
            surface: "s".into(),
            metrics: metrics.iter().map(|(k, v)| (k.to_string(), *v)).collect(),
            fingerprint: fp.into(),
            reps: 3,
            host: HostInfo { hostname: "h".into(), nproc: 24 },
            profile: "release".into(),
            commit: "abc".into(),
            date: "2026-07-17".into(),
        }
    }

    fn measured(metrics: &[(&str, f64)], fp: &str) -> Measured {
        Measured {
            surface: "s".into(),
            metrics: metrics.iter().map(|(k, v)| (k.to_string(), *v)).collect(),
            fingerprint: fp.into(),
        }
    }

    fn host() -> HostInfo {
        HostInfo { hostname: "h".into(), nproc: 24 }
    }

    #[test]
    fn compare_reports_relative_drift_per_metric() {
        let c = compare(&doc(&[("wall_s", 10.0)], "f"), &measured(&[("wall_s", 10.5)], "f"), &host());
        assert_eq!(c.deltas.len(), 1);
        assert_eq!(c.deltas[0].0, "wall_s");
        assert!((c.deltas[0].1 - 0.05).abs() < 1e-12, "5% drift, got {}", c.deltas[0].1);
        assert!(c.notices.is_empty(), "5% is below the 10% notice bar");
        assert!(!c.fingerprint_changed);
        assert!(!c.host_mismatch);
    }

    #[test]
    fn compare_flags_notice_at_ten_percent() {
        let c = compare(&doc(&[("wall_s", 10.0)], "f"), &measured(&[("wall_s", 11.0)], "f"), &host());
        assert_eq!(c.notices, vec!["wall_s".to_string()]);
    }

    #[test]
    fn compare_detects_fingerprint_change() {
        let c = compare(&doc(&[], "old"), &measured(&[], "new"), &host());
        assert!(c.fingerprint_changed);
    }

    #[test]
    fn compare_flags_host_mismatch_informationally() {
        let other = HostInfo { hostname: "elsewhere".into(), nproc: 8 };
        let c = compare(&doc(&[], "f"), &measured(&[], "f"), &other);
        assert!(c.host_mismatch);
        assert!(!c.fingerprint_changed);
    }

    #[test]
    fn compare_skips_metrics_missing_on_either_side() {
        let c = compare(
            &doc(&[("only_base", 1.0)], "f"),
            &measured(&[("only_measured", 2.0)], "f"),
            &host(),
        );
        assert!(c.deltas.is_empty());
    }

    #[test]
    fn exit_code_is_one_only_on_fingerprint_change() {
        let clean = compare(&doc(&[], "f"), &measured(&[], "f"), &host());
        let changed = compare(&doc(&[], "f"), &measured(&[], "g"), &host());
        assert_eq!(exit_code(&[Some(clean.clone()), None]), 0);
        assert_eq!(exit_code(&[Some(clean), Some(changed)]), 1);
    }

    #[test]
    fn baseline_doc_round_trips_through_json() {
        let d = doc(&[("bars_per_s", 812345.0)], "last=0.1 n=5");
        let json = serde_json::to_string_pretty(&d).expect("serialize");
        let back: BaselineDoc = serde_json::from_str(&json).expect("parse");
        assert_eq!(d, back);
    }
}