diff --git a/CLAUDE.md b/CLAUDE.md index 14886c1..aa14e20 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -178,6 +178,8 @@ Everything else is a fixed convention (see `~/dev/skills/docs/conventions.md`). - **Test** — `cargo test --workspace` - **Lint** — `cargo clippy --workspace --all-targets -- -D warnings` - **Doc build** — `cargo doc --workspace --no-deps 2>&1` +- **Bench** — `cargo run --release -p aura-bench -- run` (cycle close; + report-only — see `crates/aura-bench/README.md`) - **Design ledger** — `docs/design/INDEX.md` - **Glossary** — `docs/glossary.md` - **Issue tracker** — Gitea, repo `Brummel/Aura`: diff --git a/crates/aura-bench/README.md b/crates/aura-bench/README.md new file mode 100644 index 0000000..155818a --- /dev/null +++ b/crates/aura-bench/README.md @@ -0,0 +1,47 @@ +# aura-bench + +Wall-clock benchmarks with committed baselines (issue #251, phase 1). Catches +hot-path regressions the E2E suite structurally cannot see: every surface pins +its inputs and asserts a determinism fingerprint, so a fast-but-wrong binary +fails the bench instead of pinning a bogus baseline. + +## Running + +```console +cargo run --release -p aura-bench -- run # measure + compare, report-only +cargo run --release -p aura-bench -- run --surface engine_throughput +cargo run --release -p aura-bench -- pin # rewrite committed baselines +``` + +Debug builds refuse to measure (exit 2) — wall-clock numbers from a debug +build measure the profile, not the code. + +## Surfaces + +| surface | workload | metrics | +|---|---|---| +| `engine_throughput` | 10M synthetic bars through an SMA-cross harness (library) | `wall_s`, `bars_per_s` | +| `ingest_throughput` | 24 synthetic archive months x 20 fresh drains via the ingest window API (library) | `wall_s`, `bars_per_s` | +| `campaign_sweep` | 8-cell sweep-only campaign, release `aura` binary on a scratch project | `wall_s`, `cpu_percent`, `peak_rss_mb` | +| `campaign_heavy` | same cells; sweep → gate → walk-forward → 1000-resample bootstrap | `wall_s`, `cpu_percent`, `peak_rss_mb` | +| `cli_fixed_cost` | spawn floor (`aura --help`) + minimal data-only project run | `help_ms`, `run_ms` | + +## Discipline + +- Quiet box: the driver warns at 1-min loadavg > 2.0. Close heavy jobs first. +- 1 discarded warmup + 3 measured reps (median). Reps must agree on the + fingerprint — cross-rep drift is reported as nondeterminism, not averaged. +- Baselines are per-host context (`host` block inside each JSON): comparison on + a different host prints an informational banner and stays report-only. +- Timing drift NEVER fails the run (report-only phase, #251 cadence decision); + a NOTICE line flags drift >= 10%. A fingerprint mismatch exits 1. +- Re-pin (`pin`) deliberately, after an accepted behaviour or performance + change; the baseline diff is reviewed like any other commit. + +## Deferred (follow-up on the tracker) + +- Instruction-count backend (iai-callgrind) once valgrind is installed on the + measuring host — slots in as a sibling backend without reshaping baselines. +- Native-crate project-load sub-cost (dylib SHA-256) — needs a build step in + the scratch project. +- Thresholded failure, once recorded variance data justifies thresholds. diff --git a/crates/aura-bench/baselines/campaign_heavy.json b/crates/aura-bench/baselines/campaign_heavy.json new file mode 100644 index 0000000..5e08d18 --- /dev/null +++ b/crates/aura-bench/baselines/campaign_heavy.json @@ -0,0 +1,17 @@ +{ + "surface": "campaign_heavy", + "metrics": { + "cpu_percent": 2116.0, + "peak_rss_mb": 98.25, + "wall_s": 5.604210818 + }, + "fingerprint": "cells=8 c0s0w17 c0s1v[0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20.21.22.23.24] c0s3b2883 c1s0w17 c1s1v[0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20.21.22.23.24] c1s3b2883 c2s0w17 c2s1v[0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20.21.22.23.24] c2s3b2883 c3s0w17 c3s1v[0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20.21.22.23.24] c3s3b2883 c4s0w17 c4s1v[0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20.21.22.23.24] c4s3b2883 c5s0w17 c5s1v[0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20.21.22.23.24] c5s3b2883 c6s0w17 c6s1v[0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20.21.22.23.24] c6s3b2883 c7s0w17 c7s1v[0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17.18.19.20.21.22.23.24] c7s3b2883", + "reps": 3, + "host": { + "hostname": "Raki", + "nproc": 24 + }, + "profile": "release", + "commit": "d62dede", + "date": "2026-07-17" +} diff --git a/crates/aura-bench/baselines/campaign_sweep.json b/crates/aura-bench/baselines/campaign_sweep.json new file mode 100644 index 0000000..751a5ab --- /dev/null +++ b/crates/aura-bench/baselines/campaign_sweep.json @@ -0,0 +1,17 @@ +{ + "surface": "campaign_sweep", + "metrics": { + "cpu_percent": 2170.0, + "peak_rss_mb": 98.21875, + "wall_s": 1.418092216 + }, + "fingerprint": "cells=8 c0s0w17 c1s0w17 c2s0w17 c3s0w17 c4s0w17 c5s0w17 c6s0w17 c7s0w17", + "reps": 3, + "host": { + "hostname": "Raki", + "nproc": 24 + }, + "profile": "release", + "commit": "d62dede", + "date": "2026-07-17" +} diff --git a/crates/aura-bench/baselines/cli_fixed_cost.json b/crates/aura-bench/baselines/cli_fixed_cost.json new file mode 100644 index 0000000..d6e90b9 --- /dev/null +++ b/crates/aura-bench/baselines/cli_fixed_cost.json @@ -0,0 +1,16 @@ +{ + "surface": "cli_fixed_cost", + "metrics": { + "help_ms": 1.53201, + "run_ms": 3.37904 + }, + "fingerprint": "run_line_fnv=ef791827583f616b", + "reps": 3, + "host": { + "hostname": "Raki", + "nproc": 24 + }, + "profile": "release", + "commit": "d62dede", + "date": "2026-07-17" +} diff --git a/crates/aura-bench/baselines/engine_throughput.json b/crates/aura-bench/baselines/engine_throughput.json new file mode 100644 index 0000000..b3f2cfe --- /dev/null +++ b/crates/aura-bench/baselines/engine_throughput.json @@ -0,0 +1,16 @@ +{ + "surface": "engine_throughput", + "metrics": { + "bars_per_s": 13591474.080269944, + "wall_s": 0.735755367 + }, + "fingerprint": "last=0.417351 max_dd=8.274577 flips=265308 n=10000000", + "reps": 3, + "host": { + "hostname": "Raki", + "nproc": 24 + }, + "profile": "release", + "commit": "d62dede", + "date": "2026-07-17" +} diff --git a/crates/aura-bench/baselines/ingest_throughput.json b/crates/aura-bench/baselines/ingest_throughput.json new file mode 100644 index 0000000..ddb27f8 --- /dev/null +++ b/crates/aura-bench/baselines/ingest_throughput.json @@ -0,0 +1,16 @@ +{ + "surface": "ingest_throughput", + "metrics": { + "bars_per_s": 12912289.748572402, + "wall_s": 0.388838858 + }, + "fingerprint": "n=251040 xor=0019885debd3bd5e", + "reps": 3, + "host": { + "hostname": "Raki", + "nproc": 24 + }, + "profile": "release", + "commit": "d62dede", + "date": "2026-07-17" +} diff --git a/crates/aura-bench/src/baseline.rs b/crates/aura-bench/src/baseline.rs index 04af3eb..a39e3ba 100644 --- a/crates/aura-bench/src/baseline.rs +++ b/crates/aura-bench/src/baseline.rs @@ -1,12 +1,5 @@ //! Baseline documents and the report-only comparison core. Pure functions — //! everything here is unit-tested without running a single benchmark. -//! -//! The full CLI glue that wires this module's public surface into the -//! surfaces' measurement loop lands in Task 7; until then nothing outside -//! `#[cfg(test)]` calls into it, so the module is exempted from the -//! dead-code lint here rather than left to trip the workspace's -//! `-D warnings` clippy gate. -#![allow(dead_code)] use serde::{Deserialize, Serialize}; use std::collections::BTreeMap; diff --git a/crates/aura-bench/src/child.rs b/crates/aura-bench/src/child.rs index 3519f32..f97f3dc 100644 --- a/crates/aura-bench/src/child.rs +++ b/crates/aura-bench/src/child.rs @@ -3,12 +3,6 @@ //! from the stderr report. `/usr/bin/time` missing is an infra error — the //! campaign surfaces have no fallback RSS probe by design (one tool, one //! parse, the practice proven in the #277 measurement session). -//! -//! The CLI glue that wires this module's public surface into the surfaces' -//! measurement loop lands in Task 7; until then nothing outside `#[cfg(test)]` -//! calls into it, so the module is exempted from the dead-code lint here -//! rather than left to trip the workspace's `-D warnings` clippy gate. -#![allow(dead_code)] use std::path::{Path, PathBuf}; use std::time::Instant; diff --git a/crates/aura-bench/src/host.rs b/crates/aura-bench/src/host.rs index e3f88f8..57129f2 100644 --- a/crates/aura-bench/src/host.rs +++ b/crates/aura-bench/src/host.rs @@ -1,12 +1,6 @@ //! Host context probes. All best-effort: a probe that fails degrades to a //! placeholder value and never fails a bench (the report stays honest via the //! host block committed inside each baseline). -//! -//! The CLI glue that wires this module's public surface into the surfaces' -//! measurement loop lands in Task 7; until then nothing outside `#[cfg(test)]` -//! calls into it, so the module is exempted from the dead-code lint here -//! rather than left to trip the workspace's `-D warnings` clippy gate. -#![allow(dead_code)] use crate::baseline::HostInfo; diff --git a/crates/aura-bench/src/main.rs b/crates/aura-bench/src/main.rs index be8c98f..7f06ab5 100644 --- a/crates/aura-bench/src/main.rs +++ b/crates/aura-bench/src/main.rs @@ -1,7 +1,7 @@ //! `aura-bench` — the wall-clock benchmark harness with committed baselines //! (issue #251, phase 1). Report-only: timing drift never fails; a determinism //! fingerprint mismatch does (exit 1), so a fast-but-wrong binary fails the -//! bench instead of pinning a bogus baseline. +//! bench instead of pinning a bogus baseline. Infra errors exit 2. mod baseline; mod child; @@ -9,12 +9,438 @@ mod host; mod surfaces; mod synth; +use baseline::{compare, exit_code, BaselineDoc, Compared, Measured}; +use clap::{Parser, Subcommand}; +use std::path::{Path, PathBuf}; +use surfaces::Sizing; + +const SURFACES: &[&str] = + &["engine_throughput", "ingest_throughput", "campaign_sweep", "campaign_heavy", "cli_fixed_cost"]; +/// The in-process surfaces — selecting only these never resolves (or builds) +/// the release `aura` binary. Keep in sync with `measure_surface`. +const LIBRARY_SURFACES: &[&str] = &["engine_throughput", "ingest_throughput"]; +const DEFAULT_REPS: u32 = 3; +/// 1-min loadavg above which the banner warns the box is not quiet. +const LOAD_WARN_1M: f64 = 2.0; + +#[derive(Parser)] +#[command(name = "aura-bench", about = "wall-clock benchmarks against committed baselines (#251)")] +struct Cli { + #[command(subcommand)] + cmd: Cmd, +} + +#[derive(Subcommand)] +enum Cmd { + /// Measure surfaces and compare against committed baselines (report-only). + Run { + /// Only these surfaces (repeatable). Default: all. + #[arg(long)] + surface: Vec, + /// Tiny workloads, results written to --out, committed baselines untouched. + #[arg(long)] + quick: bool, + /// Measured repetitions per surface (after 1 discarded warmup). + #[arg(long, default_value_t = DEFAULT_REPS)] + reps: u32, + /// Write measured results as JSON files into this directory. + #[arg(long)] + out: Option, + }, + /// Measure surfaces and (re)write the committed baselines. + Pin { + /// Only these surfaces (repeatable). Default: all. + #[arg(long)] + surface: Vec, + /// Measured repetitions per surface (after 1 discarded warmup). + #[arg(long, default_value_t = DEFAULT_REPS)] + reps: u32, + }, +} + +fn baselines_dir(ws_root: &Path) -> PathBuf { + ws_root.join("crates/aura-bench/baselines") +} + +fn measure_surface( + name: &str, + sizing: Sizing, + reps: u32, + aura_bin: &Path, +) -> Result { + match name { + "engine_throughput" => surfaces::run_reps(name, reps, || surfaces::engine::rep(sizing)), + "ingest_throughput" => { + let fixture = surfaces::ingest::setup(sizing)?; + surfaces::run_reps(name, reps, || surfaces::ingest::rep(&fixture)) + } + "campaign_sweep" => { + surfaces::run_reps(name, reps, || surfaces::campaign::sweep_rep(aura_bin, sizing)) + } + "campaign_heavy" => { + surfaces::run_reps(name, reps, || surfaces::campaign::heavy_rep(aura_bin, sizing)) + } + "cli_fixed_cost" => surfaces::run_reps(name, reps, || surfaces::fixed_cost::rep(aura_bin)), + other => Err(format!("unknown surface: {other}")), + } +} + +fn selected(filter: &[String]) -> Result, String> { + if filter.is_empty() { + return Ok(SURFACES.to_vec()); + } + let mut out = Vec::new(); + for f in filter { + let s = SURFACES + .iter() + .find(|s| **s == f.as_str()) + .ok_or_else(|| format!("unknown surface {f:?}; known: {SURFACES:?}"))?; + out.push(*s); + } + Ok(out) +} + +fn fmt_metric(name: &str, v: f64) -> String { + if name.ends_with("_ms") { + format!("{name} {v:.1}ms") + } else if name == "bars_per_s" { + format!("{name} {v:.0}") + } else if name == "wall_s" { + format!("{name} {v:.3}s") + } else { + format!("{name} {v:.1}") + } +} + +fn report_line(m: &Measured, c: Option<&Compared>) -> String { + let metrics = m + .metrics + .iter() + .map(|(k, v)| fmt_metric(k, *v)) + .collect::>() + .join(" "); + let verdict = match c { + None => "no baseline (pin to create)".to_string(), + Some(c) if c.fingerprint_changed => "fingerprint CHANGED".to_string(), + Some(c) => { + let drift = c + .deltas + .iter() + .map(|(k, d)| format!("{k} {:+.1}%", d * 100.0)) + .collect::>() + .join(" "); + let notice = if c.notices.is_empty() { "" } else { " NOTICE" }; + format!("Δ {drift}{notice} fingerprint OK") + } + }; + format!("{:<20} {metrics} {verdict}", m.surface) +} + +fn load_baseline(dir: &Path, surface: &str) -> Option { + let text = std::fs::read_to_string(dir.join(format!("{surface}.json"))).ok()?; + serde_json::from_str(&text).ok() +} + +/// One `Measured` result plus a host + rep count becomes a `BaselineDoc`, +/// whether it's about to be committed (`pin`) or dropped into `--out` (`run`). +fn to_baseline_doc(m: &Measured, reps: u32, host: &baseline::HostInfo) -> BaselineDoc { + BaselineDoc { + surface: m.surface.clone(), + metrics: m.metrics.clone(), + fingerprint: m.fingerprint.clone(), + reps, + host: host.clone(), + profile: "release".to_string(), + commit: host::git_commit(), + date: host::today(), + } +} + fn main() { - // Full CLI glue lands with the surfaces (Task 7). The skeleton only - // guards the profile: wall-clock numbers from a debug build measure the - // profile, not the code. if cfg!(debug_assertions) { eprintln!("aura-bench measures nothing in a debug build; run via `cargo run --release -p aura-bench`"); std::process::exit(2); } + let cli = Cli::parse(); + let ws_root = child::workspace_root(); + let code = match cli.cmd { + Cmd::Run { surface, quick, reps, out } => cmd_run(&ws_root, &surface, quick, reps, out), + Cmd::Pin { surface, reps } => cmd_pin(&ws_root, &surface, reps), + }; + std::process::exit(code); +} + +fn prepare(ws_root: &Path, filter: &[String]) -> Result<(Vec<&'static str>, PathBuf), String> { + let names = selected(filter)?; + let needs_bin = names.iter().any(|n| !LIBRARY_SURFACES.contains(n)); + let bin = if needs_bin { + child::resolve_aura_bin(ws_root)? + } else { + PathBuf::from("aura-unused") + }; + Ok((names, bin)) +} + +fn banner(quick: bool) { + let h = host::host_info(); + let load = host::loadavg_1m().map(|l| format!("{l:.1}")).unwrap_or_else(|| "?".to_string()); + let mode = if quick { " QUICK (mechanics only)" } else { "" }; + println!("aura-bench: host {} ({} cores), load {load}{mode}", h.hostname, h.nproc); + if let Some(l) = host::loadavg_1m() + && l > LOAD_WARN_1M + { + println!("WARNING: loadavg {l:.1} > {LOAD_WARN_1M} — measurements belong on a quiet box"); + } +} + +fn cmd_run( + ws_root: &Path, + filter: &[String], + quick: bool, + reps: u32, + out: Option, +) -> i32 { + let (names, bin) = match prepare(ws_root, filter) { + Ok(v) => v, + Err(e) => return infra(&e), + }; + banner(quick); + let sizing = Sizing { quick }; + let reps = if quick { 1 } else { reps }; + let dir = baselines_dir(ws_root); + let this_host = host::host_info(); + let mut compared: Vec> = Vec::new(); + let mut host_mismatch = false; + for name in &names { + let m = match measure_surface(name, sizing, reps, &bin) { + Ok(m) => m, + Err(e) => return infra(&format!("surface {name}: {e}")), + }; + let b = if quick { None } else { load_baseline(&dir, name) }; + let c = b.as_ref().map(|b| { + let c = compare(b, &m, &this_host); + host_mismatch |= c.host_mismatch; + c + }); + println!("{}", report_line(&m, c.as_ref())); + if let (Some(b), Some(c)) = (&b, &c) + && c.fingerprint_changed + { + println!(" baseline: {}", b.fingerprint); + println!(" measured: {}", m.fingerprint); + println!( + " The workload computed different results. If a behaviour change is\n \ + intended, re-pin (aura-bench pin); otherwise this is a correctness bug." + ); + } + if let Some(out_dir) = &out + && let Err(e) = write_result(out_dir, &m, reps, &this_host) + { + return infra(&e); + } + compared.push(c); + } + if host_mismatch { + println!("HOST MISMATCH (informational): baselines were pinned on a different host"); + } + let notices: usize = compared.iter().flatten().map(|c| c.notices.len()).sum(); + if notices > 0 { + println!( + "{notices} NOTICE(s) (drift >= 10%) — report-only, nothing failed. Re-pin: aura-bench pin" + ); + } + exit_code(&compared) +} + +fn cmd_pin(ws_root: &Path, filter: &[String], reps: u32) -> i32 { + let (names, bin) = match prepare(ws_root, filter) { + Ok(v) => v, + Err(e) => return infra(&e), + }; + banner(false); + let dir = baselines_dir(ws_root); + if let Err(e) = std::fs::create_dir_all(&dir) { + return infra(&format!("create baselines dir: {e}")); + } + let this_host = host::host_info(); + for name in &names { + let m = match measure_surface(name, Sizing { quick: false }, reps, &bin) { + Ok(m) => m, + Err(e) => return infra(&format!("surface {name}: {e}")), + }; + let doc = to_baseline_doc(&m, reps, &this_host); + let path = dir.join(format!("{name}.json")); + let json = serde_json::to_string_pretty(&doc).expect("baseline serializes"); + if let Err(e) = std::fs::write(&path, json + "\n") { + return infra(&format!("write {}: {e}", path.display())); + } + println!("pinned {name} -> {}", path.display()); + } + 0 +} + +fn write_result(out_dir: &Path, m: &Measured, reps: u32, h: &baseline::HostInfo) -> Result<(), String> { + std::fs::create_dir_all(out_dir).map_err(|e| format!("create out dir: {e}"))?; + let doc = to_baseline_doc(m, reps, h); + let json = serde_json::to_string_pretty(&doc).expect("result serializes"); + std::fs::write(out_dir.join(format!("{}.json", m.surface)), json + "\n") + .map_err(|e| format!("write result: {e}")) +} + +fn infra(msg: &str) -> i32 { + eprintln!("aura-bench: infra error: {msg}"); + 2 +} + +#[cfg(test)] +mod tests { + use super::*; + + fn measured(surface: &str, metrics: &[(&str, f64)], fp: &str) -> Measured { + Measured { + surface: surface.to_string(), + metrics: metrics.iter().map(|(k, v)| (k.to_string(), *v)).collect(), + fingerprint: fp.to_string(), + } + } + + /// selected() with no filter runs every registered surface, in registration order. + #[test] + fn selected_empty_filter_returns_all_surfaces() { + assert_eq!(selected(&[]).unwrap(), SURFACES.to_vec()); + } + + /// selected() with a known name narrows to exactly that surface. + #[test] + fn selected_known_name_returns_that_surface() { + assert_eq!(selected(&["engine_throughput".to_string()]).unwrap(), vec!["engine_throughput"]); + } + + /// selected() rejects an unknown surface name instead of silently dropping it, + /// so a `--surface` typo fails loudly rather than measuring nothing. + #[test] + fn selected_unknown_name_is_rejected() { + let err = selected(&["not_a_surface".to_string()]).unwrap_err(); + assert!(err.contains("not_a_surface"), "error should name the bad surface: {err}"); + } + + /// fmt_metric renders *_ms metrics with millisecond precision and unit suffix. + #[test] + fn fmt_metric_formats_ms_suffixed_metrics() { + assert_eq!(fmt_metric("help_ms", 12.345), "help_ms 12.3ms"); + } + + /// fmt_metric renders bars_per_s as a rounded integer (throughput, not fractional). + #[test] + fn fmt_metric_formats_bars_per_s_without_decimals() { + assert_eq!(fmt_metric("bars_per_s", 812345.6), "bars_per_s 812346"); + } + + /// fmt_metric renders wall_s with millisecond-precision seconds. + #[test] + fn fmt_metric_formats_wall_s_with_millis_precision() { + assert_eq!(fmt_metric("wall_s", 1.23456), "wall_s 1.235s"); + } + + /// fmt_metric falls back to 1-decimal generic formatting for any other metric name. + #[test] + fn fmt_metric_falls_back_to_generic_formatting() { + assert_eq!(fmt_metric("peak_rss_mb", 42.16), "peak_rss_mb 42.2"); + } + + /// report_line without a baseline tells the operator to pin one, rather than + /// implying a comparison happened. + #[test] + fn report_line_without_baseline_prompts_to_pin() { + let m = measured("s", &[("wall_s", 1.0)], "fp"); + let line = report_line(&m, None); + assert!(line.contains("no baseline (pin to create)"), "{line}"); + } + + /// report_line surfaces a fingerprint mismatch distinctly from ordinary timing + /// drift — this is the correctness-vs-performance line the whole harness rests on. + #[test] + fn report_line_flags_fingerprint_change() { + let m = measured("s", &[("wall_s", 1.0)], "new"); + let c = Compared { deltas: vec![], notices: vec![], fingerprint_changed: true, host_mismatch: false }; + let line = report_line(&m, Some(&c)); + assert!(line.contains("fingerprint CHANGED"), "{line}"); + } + + /// report_line prints per-metric drift percentages and an OK fingerprint when + /// nothing changed. + #[test] + fn report_line_reports_drift_and_fingerprint_ok() { + let m = measured("s", &[("wall_s", 1.0)], "fp"); + let c = Compared { + deltas: vec![("wall_s".to_string(), 0.05)], + notices: vec![], + fingerprint_changed: false, + host_mismatch: false, + }; + let line = report_line(&m, Some(&c)); + assert!(line.contains("wall_s +5.0%"), "{line}"); + assert!(line.contains("fingerprint OK"), "{line}"); + } + + /// `prepare` only resolves the real release `aura` binary when a selected + /// surface actually spawns a child process; a library-only selection + /// (`engine_throughput`/`ingest_throughput`) must never touch the + /// filesystem to build or locate it. Protects the fast-iteration path: + /// a regression here would force a multi-second `cargo build --release` + /// onto every `aura-bench run --surface engine_throughput`, defeating + /// the point of the in-process surfaces. Proven by using a nonexistent + /// `ws_root` — `resolve_aura_bin` would error immediately if `prepare` + /// ever called it for this filter. + #[test] + fn prepare_skips_binary_resolution_for_library_only_surfaces() { + let fake_root = Path::new("/nonexistent-ws-root-for-aura-bench-test"); + let filter = vec!["engine_throughput".to_string(), "ingest_throughput".to_string()]; + let (names, bin) = prepare(fake_root, &filter).expect("must not try to resolve a binary"); + assert_eq!(names, vec!["engine_throughput", "ingest_throughput"]); + assert_eq!(bin, PathBuf::from("aura-unused")); + } + + /// `write_result`'s `--out` artifact is schema-identical to a committed + /// baseline (round-trips through `BaselineDoc`) — the property that makes + /// a `--quick --out` result directly promotable/comparable, never a + /// shadow format that silently drifts from what `pin` writes. + #[test] + fn write_result_round_trips_measured_into_baseline_shaped_json() { + let scratch = crate::synth::ScratchDir::new("write-result-test").expect("scratch dir"); + let m = measured("cli_fixed_cost", &[("help_ms", 1.5), ("run_ms", 3.0)], "run_line_fnv=abc"); + let host = baseline::HostInfo { hostname: "h".to_string(), nproc: 8 }; + write_result(&scratch.0, &m, 2, &host).expect("write_result must succeed"); + let doc = load_baseline(&scratch.0, "cli_fixed_cost") + .expect("write_result's file must be a loadable BaselineDoc"); + assert_eq!(doc.surface, "cli_fixed_cost"); + assert_eq!(doc.metrics, m.metrics); + assert_eq!(doc.fingerprint, "run_line_fnv=abc"); + assert_eq!(doc.reps, 2); + assert_eq!(doc.profile, "release"); + } + + /// The full CLI driver glue's core promise: `pin` then `run` against the + /// baseline it just wrote reports zero fingerprint drift — a scratch + /// `ws_root` keeps this fully isolated from the real committed + /// `baselines/` directory (never touches it). If `cmd_pin` and `cmd_run` + /// ever disagreed on what a surface measures (e.g. different `Sizing`, + /// different metric keys), a fresh `pin` would immediately fail its own + /// `run` — exactly the workflow the README documents ("re-pin, the + /// baseline diff is reviewed like any other commit"). + /// + /// Ignored in the default suite: it runs the engine surface at full size + /// (1,000,000 bars, twice) in an unoptimized `dev`-profile test binary — + /// the suite-wallclock discipline reserves full-size runs for release + /// builds. Run it explicitly: `cargo test -p aura-bench -- --ignored`. + #[test] + #[ignore = "full-size (1M bar) engine surface in a dev-profile build; run via -- --ignored"] + fn pin_then_run_round_trip_reports_zero_drift() { + let scratch = crate::synth::ScratchDir::new("pin-run-roundtrip").expect("scratch dir"); + let filter = vec!["engine_throughput".to_string()]; + let pin_code = cmd_pin(&scratch.0, &filter, 1); + assert_eq!(pin_code, 0, "pin must succeed against an isolated scratch ws_root"); + let run_code = cmd_run(&scratch.0, &filter, false, 1, None); + assert_eq!(run_code, 0, "running right after pinning must report zero fingerprint drift"); + } } diff --git a/crates/aura-bench/src/surfaces/campaign.rs b/crates/aura-bench/src/surfaces/campaign.rs index 471902b..35a429b 100644 --- a/crates/aura-bench/src/surfaces/campaign.rs +++ b/crates/aura-bench/src/surfaces/campaign.rs @@ -14,11 +14,17 @@ use std::path::Path; pub const FULL_INSTRUMENTS: &[&str] = &["BSYMA", "BSYMB", "BSYMC", "BSYMD"]; pub const QUICK_INSTRUMENTS: &[&str] = &["BSYMA", "BSYMB"]; -/// Archive span (2024-01..08) and campaign window (Mar-01..Jun-30) mirror the -/// green E2E fixture: ~17 weeks gives the (14d, 7d, 7d) roller a comfortable -/// tiling (research_docs.rs:2440-2441). -pub const ARCHIVE_MONTHS: ((i32, u32), (i32, u32)) = ((2024, 1), (2024, 8)); -pub const WINDOW_MS: (i64, i64) = (1709251200000, 1719791999999); +/// Full-size span: 24 archive months with a ~22-month campaign window +/// (2024-02-01 .. 2025-11-30 UTC), sized so the measured child runs land in +/// whole seconds — sub-second walls put ordinary spawn/scheduler jitter within +/// reach of the 10% NOTICE threshold and make the report flap. +pub const ARCHIVE_MONTHS_FULL: ((i32, u32), (i32, u32)) = ((2024, 1), (2025, 12)); +pub const WINDOW_MS_FULL: (i64, i64) = (1706745600000, 1764547199999); +/// Quick span: the green E2E fixture's shape (archive 2024-01..08, window +/// Mar-01..Jun-30 — ~17 weeks gives the (14d, 7d, 7d) roller a comfortable +/// tiling, research_docs.rs:2440-2441). +pub const ARCHIVE_MONTHS_QUICK: ((i32, u32), (i32, u32)) = ((2024, 1), (2024, 8)); +pub const WINDOW_MS_QUICK: (i64, i64) = (1709251200000, 1719791999999); pub const SWEEP_PROCESS_DOC: &str = r#"{ "format_version": 1, @@ -135,16 +141,24 @@ fn register_process(bin: &Path, dir: &Path, file: &str, doc: &str) -> Result String { +fn campaign_doc(instruments: &[&str], bp_ids: &[String], proc_id: &str, quick: bool) -> String { let instruments_json = instruments.iter().map(|i| format!("\"{i}\"")).collect::>().join(", "); + // 5x5 member grid at full size (the workload lever), the E2E-shaped 3x3 + // in quick mode (mechanics only). + let (fast_vals, slow_vals) = if quick { + ("[2, 3, 4]", "[8, 12, 16]") + } else { + ("[2, 3, 4, 5, 6]", "[8, 10, 12, 14, 16]") + }; + let window = if quick { WINDOW_MS_QUICK } else { WINDOW_MS_FULL }; let strategies_json = bp_ids .iter() .map(|id| { format!( r#"{{ "ref": {{ "content_id": "{id}" }}, - "axes": {{ "fast.length": {{ "kind": "I64", "values": [2, 3, 4] }}, - "slow.length": {{ "kind": "I64", "values": [8, 12, 16] }} }} }}"# + "axes": {{ "fast.length": {{ "kind": "I64", "values": {fast_vals} }}, + "slow.length": {{ "kind": "I64", "values": {slow_vals} }} }} }}"# ) }) .collect::>() @@ -160,8 +174,8 @@ fn campaign_doc(instruments: &[&str], bp_ids: &[String], proc_id: &str) -> Strin "seed": 7, "presentation": {{ "persist_taps": [], "emit": [] }} }}"#, - from = WINDOW_MS.0, - to = WINDOW_MS.1, + from = window.0, + to = window.1, ) } @@ -175,8 +189,9 @@ fn build_scratch(bin: &Path, sizing: Sizing, process_doc: &str) -> Result<(Scrat let data_dir = root.join("data"); std::fs::create_dir_all(&data_dir).map_err(|e| format!("create data dir: {e}"))?; let instruments = if sizing.quick { QUICK_INSTRUMENTS } else { FULL_INSTRUMENTS }; + let months = if sizing.quick { ARCHIVE_MONTHS_QUICK } else { ARCHIVE_MONTHS_FULL }; for sym in instruments { - write_symbol_archive(&data_dir, sym, ARCHIVE_MONTHS.0, ARCHIVE_MONTHS.1); + write_symbol_archive(&data_dir, sym, months.0, months.1); } std::fs::write(root.join("bench_sma_a.json"), BP_SMA_A) .map_err(|e| format!("write blueprint a: {e}"))?; @@ -187,14 +202,19 @@ fn build_scratch(bin: &Path, sizing: Sizing, process_doc: &str) -> Result<(Scrat bp_ids.push(seed_blueprint(bin, root, "bench_sma_b.json", "seed-b")?); } let proc_id = register_process(bin, root, "bench.process.json", process_doc)?; - let doc = campaign_doc(instruments, &bp_ids, &proc_id); + let doc = campaign_doc(instruments, &bp_ids, &proc_id, sizing.quick); std::fs::write(root.join("bench.campaign.json"), doc) .map_err(|e| format!("write campaign doc: {e}"))?; Ok((scratch, "bench.campaign.json".to_string())) } -/// Extract the winner-ordinal fingerprint from the always-on final -/// `{"campaign_run":...}` stdout line. +/// Extract the determinism fingerprint from the always-on final +/// `{"campaign_run":...}` stdout line: per stage the selection's winner +/// ordinal (`w`), the survivor-ordinal list (`v[..]`), and the bootstrap's +/// trade count (`b`, pooled or summed across per-survivor entries). Folding +/// all three realization layers in means a wrong-result regression in the +/// sweep, gate, walk-forward, or bootstrap machinery fails the bench — not +/// just a wrong sweep winner. pub fn winner_fingerprint(stdout: &str) -> Result { let line = stdout .lines() @@ -212,6 +232,21 @@ pub fn winner_fingerprint(stdout: &str) -> Result { if let Some(w) = stage["selection"]["winner_ordinal"].as_u64() { parts.push(format!("c{ci}s{si}w{w}")); } + if let Some(surv) = stage["survivor_ordinals"].as_array() { + let list = surv + .iter() + .filter_map(|x| x.as_u64()) + .map(|x| x.to_string()) + .collect::>() + .join("."); + parts.push(format!("c{ci}s{si}v[{list}]")); + } + if let Some(n) = stage["bootstrap"]["pooled_oos"]["n_trades"].as_u64() { + parts.push(format!("c{ci}s{si}b{n}")); + } else if let Some(per) = stage["bootstrap"]["per_survivor"].as_array() { + let n: u64 = per.iter().filter_map(|pair| pair[1]["n_trades"].as_u64()).sum(); + parts.push(format!("c{ci}s{si}b{n}")); + } } } if parts.is_empty() { @@ -250,7 +285,7 @@ pub fn heavy_rep(bin: &Path, sizing: Sizing) -> Result { campaign_rep(bin, sizing, doc) } -/// Used by `cli_fixed_cost` (Task 7): a minimal one-symbol scratch project. +/// Used by `cli_fixed_cost`: a minimal one-symbol scratch project. pub fn build_minimal_project() -> Result { let scratch = ScratchDir::new("fixedcost").map_err(|e| format!("scratch dir: {e}"))?; let root = &scratch.0; @@ -273,9 +308,20 @@ mod tests { let stdout = "noise\n{\"campaign_run\":{\"campaign\":\"c\",\"process\":\"p\",\"run\":0,\"seed\":7,\ \"cells\":[{\"strategy\":\"s\",\"instrument\":\"I\",\"window_ms\":[1,2],\ \"stages\":[{\"block\":\"std::sweep\",\"selection\":{\"winner_ordinal\":4,\"params\":[],\ - \"selection\":\"argmax\"}},{\"block\":\"std::gate\"}]}]}}\n"; + \"selection\":\"argmax\"}},{\"block\":\"std::gate\",\"survivor_ordinals\":[0,2]},\ + {\"block\":\"std::monte_carlo\",\"bootstrap\":{\"pooled_oos\":{\"n_trades\":42}}}]}]}}\n"; let fp = winner_fingerprint(stdout).expect("parses"); - assert_eq!(fp, "cells=1 c0s0w4"); + assert_eq!(fp, "cells=1 c0s0w4 c0s1v[0.2] c0s2b42"); + } + + #[test] + fn winner_fingerprint_sums_per_survivor_bootstrap_trades() { + let stdout = "{\"campaign_run\":{\"campaign\":\"c\",\"process\":\"p\",\"run\":0,\"seed\":7,\ + \"cells\":[{\"strategy\":\"s\",\"instrument\":\"I\",\"window_ms\":[1,2],\ + \"stages\":[{\"block\":\"std::monte_carlo\",\"bootstrap\":{\"per_survivor\":\ + [[0,{\"n_trades\":10}],[3,{\"n_trades\":5}]]}}]}]}}\n"; + let fp = winner_fingerprint(stdout).expect("parses"); + assert_eq!(fp, "cells=1 c0s0b15"); } #[test] @@ -285,16 +331,28 @@ mod tests { #[test] fn campaign_doc_lists_all_instruments_and_strategies() { - let doc = campaign_doc( - &["BSYMA", "BSYMB"], - &["id-a".to_string(), "id-b".to_string()], - "proc-id", - ); - let v: serde_json::Value = serde_json::from_str(&doc).expect("doc is valid JSON"); - assert_eq!(v["data"]["instruments"].as_array().expect("instruments").len(), 2); - assert_eq!(v["strategies"].as_array().expect("strategies").len(), 2); - assert_eq!(v["seed"], 7); - assert_eq!(v["process"]["ref"]["content_id"], "proc-id"); + for quick in [true, false] { + let doc = campaign_doc( + &["BSYMA", "BSYMB"], + &["id-a".to_string(), "id-b".to_string()], + "proc-id", + quick, + ); + let v: serde_json::Value = serde_json::from_str(&doc).expect("doc is valid JSON"); + assert_eq!(v["data"]["instruments"].as_array().expect("instruments").len(), 2); + assert_eq!(v["strategies"].as_array().expect("strategies").len(), 2); + assert_eq!(v["seed"], 7); + assert_eq!(v["process"]["ref"]["content_id"], "proc-id"); + let grid = v["strategies"][0]["axes"]["fast.length"]["values"] + .as_array() + .expect("fast axis") + .len() + * v["strategies"][0]["axes"]["slow.length"]["values"] + .as_array() + .expect("slow axis") + .len(); + assert_eq!(grid, if quick { 9 } else { 25 }, "member grid, quick={quick}"); + } } #[test] diff --git a/crates/aura-bench/src/surfaces/engine.rs b/crates/aura-bench/src/surfaces/engine.rs index cabea8b..ffd84ef 100644 --- a/crates/aura-bench/src/surfaces/engine.rs +++ b/crates/aura-bench/src/surfaces/engine.rs @@ -11,7 +11,9 @@ use std::collections::BTreeMap; use std::sync::mpsc; use std::time::Instant; -pub const FULL_BARS: usize = 1_000_000; +/// 10M bars lands the release-profile measurement at ~0.7 s wall — big enough +/// that in-process timing jitter sits far under the 10% NOTICE threshold. +pub const FULL_BARS: usize = 10_000_000; pub const QUICK_BARS: usize = 2_000; pub fn rep(sizing: Sizing) -> Result { diff --git a/crates/aura-bench/src/surfaces/fixed_cost.rs b/crates/aura-bench/src/surfaces/fixed_cost.rs new file mode 100644 index 0000000..f64e289 --- /dev/null +++ b/crates/aura-bench/src/surfaces/fixed_cost.rs @@ -0,0 +1,64 @@ +//! CLI fixed cost: the spawn floor (`aura --help`, no fingerprint — nothing is +//! computed) and a minimal data-only project run whose fingerprint is the +//! FNV-1a hash of the run's single stdout JSON line. Fresh scratch project per +//! repetition, so store counters start identical. + +use super::{median, RepOutcome}; +use crate::child::run_timed; +use crate::surfaces::campaign::build_minimal_project; +use crate::synth::fnv1a; +use std::collections::BTreeMap; +use std::path::Path; +use std::time::Instant; + +/// Spawn samples folded into one repetition: single-digit-millisecond spawns +/// jitter well past the 10% NOTICE threshold, so each repetition reports the +/// median over a batch instead of one throw of the scheduler dice. +pub const HELP_SAMPLES: u32 = 20; +pub const RUN_SAMPLES: u32 = 10; + +pub fn rep(bin: &Path) -> Result { + // Spawn floor: median wall over a batch of `aura --help` spawns. + let mut help_walls = Vec::new(); + for _ in 0..HELP_SAMPLES { + let t = Instant::now(); + let help = std::process::Command::new(bin) + .arg("--help") + .output() + .map_err(|e| format!("spawn aura --help: {e}"))?; + help_walls.push(t.elapsed().as_secs_f64() * 1000.0); + if !help.status.success() { + return Err(format!("aura --help exited {:?}", help.status.code())); + } + } + + // Project-load + minimal run: one scratch project, a batch of runs, the + // median wall. The fingerprint is the FIRST run's record line — later + // runs append to the same store, so their lines legitimately differ (run + // counters), while the first run of a fresh scratch is deterministic. + let scratch = build_minimal_project()?; + let mut run_walls = Vec::new(); + let mut first_line = None; + for _ in 0..RUN_SAMPLES { + let timed = run_timed(bin, &["run", "bench_sma_a.json"], &scratch.0)?; + if timed.exit != Some(0) { + return Err(format!("aura run exited {:?}: {}", timed.exit, timed.stdout)); + } + run_walls.push(timed.wall_s * 1000.0); + if first_line.is_none() { + first_line = Some( + timed + .stdout + .lines() + .next() + .ok_or("aura run must print its JSON record line")? + .to_string(), + ); + } + } + let line = first_line.expect("RUN_SAMPLES >= 1 sets the first line"); + let mut metrics = BTreeMap::new(); + metrics.insert("help_ms".to_string(), median(help_walls)); + metrics.insert("run_ms".to_string(), median(run_walls)); + Ok(RepOutcome { metrics, fingerprint: format!("run_line_fnv={:016x}", fnv1a(line.as_bytes())) }) +} diff --git a/crates/aura-bench/src/surfaces/ingest.rs b/crates/aura-bench/src/surfaces/ingest.rs index 836acf2..f9844a1 100644 --- a/crates/aura-bench/src/surfaces/ingest.rs +++ b/crates/aura-bench/src/surfaces/ingest.rs @@ -16,37 +16,56 @@ pub const FULL_MONTHS: ((i32, u32), (i32, u32)) = ((2024, 1), (2025, 12)); pub const QUICK_MONTHS: ((i32, u32), (i32, u32)) = ((2024, 1), (2024, 2)); /// The archive is written once per surface run (setup, unmeasured) and shared -/// by every repetition. +/// by every repetition. `passes` fresh drains per repetition lift the measured +/// wall out of the millisecond regime, where ordinary jitter would reach the +/// 10% NOTICE threshold and flap the report. pub struct IngestFixture { pub dir: ScratchDir, + pub passes: u32, } +pub const FULL_PASSES: u32 = 20; + pub fn setup(sizing: Sizing) -> Result { let dir = ScratchDir::new("ingest").map_err(|e| format!("scratch dir: {e}"))?; let (from, to) = if sizing.quick { QUICK_MONTHS } else { FULL_MONTHS }; write_symbol_archive(&dir.0, SYMBOL, from, to); - Ok(IngestFixture { dir }) + Ok(IngestFixture { dir, passes: if sizing.quick { 1 } else { FULL_PASSES } }) } -pub fn rep(fixture: &IngestFixture) -> Result { - let server = Arc::new(DataServer::new(&fixture.dir.0)); +/// One measured drain: a FRESH `DataServer`, so the pass pays zip inflate + +/// parse instead of hitting the in-memory FileCache. +fn drain_once(dir: &std::path::Path) -> Result<(u64, u64), String> { + let server = Arc::new(DataServer::new(dir)); let mut srcs = open_columns(&server, SYMBOL, None, None, &[M1Field::Close]) .ok_or("bench archive must be present for the close column")?; let mut src = srcs.pop().ok_or("one source for one field")?; - let t = Instant::now(); let mut n: u64 = 0; let mut xor: u64 = 0; while let Some((_, v)) = src.next() { n += 1; xor ^= v.as_f64().to_bits(); } - let wall = t.elapsed().as_secs_f64(); if n == 0 { return Err("the synthetic archive must yield bars".to_string()); } + Ok((n, xor)) +} + +pub fn rep(fixture: &IngestFixture) -> Result { + let t = Instant::now(); + let (n, xor) = drain_once(&fixture.dir.0)?; + for _ in 1..fixture.passes { + let again = drain_once(&fixture.dir.0)?; + if again != (n, xor) { + return Err(format!("ingest passes disagree: {:?} vs {:?}", (n, xor), again)); + } + } + let wall = t.elapsed().as_secs_f64(); + let total = n * u64::from(fixture.passes); let mut metrics = BTreeMap::new(); metrics.insert("wall_s".to_string(), wall); - metrics.insert("bars_per_s".to_string(), n as f64 / wall); + metrics.insert("bars_per_s".to_string(), total as f64 / wall); Ok(RepOutcome { metrics, fingerprint: format!("n={n} xor={xor:016x}") }) } diff --git a/crates/aura-bench/src/surfaces/mod.rs b/crates/aura-bench/src/surfaces/mod.rs index 81b8824..e9cd385 100644 --- a/crates/aura-bench/src/surfaces/mod.rs +++ b/crates/aura-bench/src/surfaces/mod.rs @@ -3,15 +3,10 @@ //! warmup + N measured reps, median per metric) and enforces cross-rep //! fingerprint identity — a surface disagreeing with itself is nondeterminism, //! reported as an infra error, never averaged away. -//! -//! The CLI glue that wires this framework's public surface into a running -//! binary lands in Task 7; until then nothing outside `#[cfg(test)]` calls -//! into it, so the module is exempted from the dead-code lint here rather -//! than left to trip the workspace's `-D warnings` clippy gate. -#![allow(dead_code)] pub mod campaign; pub mod engine; +pub mod fixed_cost; pub mod ingest; use crate::baseline::Measured; @@ -75,12 +70,12 @@ mod tests { assert_eq!(median(vec![1.0, 2.0]), 2.0); } - /// Task 3's actual deliverable: `run_reps` composed with the real engine - /// surface (a re-bootstrapped SMA-cross harness per call), not a mock - /// closure. Protects the wiring end to end — a warmup plus three fresh - /// bootstraps must agree on the determinism fingerprint and the returned - /// metrics map must carry both keys a `BaselineDoc` for this surface - /// needs (`wall_s`, `bars_per_s`), positive and finite. + /// The engine surface's actual deliverable: `run_reps` composed with the + /// real engine surface (a re-bootstrapped SMA-cross harness per call), + /// not a mock closure. Protects the wiring end to end — a warmup plus + /// three fresh bootstraps must agree on the determinism fingerprint and + /// the returned metrics map must carry both keys a `BaselineDoc` for + /// this surface needs (`wall_s`, `bars_per_s`), positive and finite. #[test] fn run_reps_drives_the_real_engine_surface_deterministically() { let measured = run_reps("engine", 3, || engine::rep(Sizing { quick: true })) @@ -91,8 +86,9 @@ mod tests { assert!(measured.metrics.contains_key("wall_s")); } - /// Task 4's actual deliverable: `run_reps` composed with the real ingest - /// surface (a fresh `DataServer` re-opened and re-drained per call). + /// The ingest surface's actual deliverable: `run_reps` composed with the + /// real ingest surface (a fresh `DataServer` re-opened and re-drained + /// per call). /// Protects that a warmup plus three fresh drains of the same synthetic /// archive agree on the fingerprint, and that the metrics map carries /// the schema the ingest `BaselineDoc` needs. @@ -138,6 +134,34 @@ mod tests { ); } + /// `run_reps` composed with the real CLI fixed-cost surface — a genuine + /// `aura --help` spawn-floor wall plus a fresh scratch project and a real + /// `aura run` child process per repetition (not an in-process shortcut). + /// Protects that two independent scratch runs agree on the run-line FNV + /// fingerprint extracted from the child's stdout, and that the metrics + /// map carries the wall-clock keys the fixed_cost `BaselineDoc` needs. + /// + /// Ignored in the default suite: it builds the release `aura` binary and + /// spawns real child processes — the suite-wallclock discipline forbids + /// that cost on every `cargo test`. Run it explicitly: + /// `cargo test -p aura-bench -- --ignored`. + #[test] + #[ignore = "builds the release aura binary + real fixed-cost child runs; run via -- --ignored"] + fn run_reps_drives_the_real_fixed_cost_surface_deterministically() { + let bin = crate::child::resolve_aura_bin(&crate::child::workspace_root()) + .expect("the release aura binary must build"); + let measured = run_reps("cli_fixed_cost", 2, || fixed_cost::rep(&bin)) + .expect("the real fixed_cost surface must agree with itself across independent scratch runs"); + assert_eq!(measured.surface, "cli_fixed_cost"); + assert!(measured.metrics.contains_key("help_ms")); + assert!(measured.metrics.contains_key("run_ms")); + assert!( + measured.fingerprint.starts_with("run_line_fnv="), + "fingerprint: {}", + measured.fingerprint + ); + } + #[test] fn run_reps_rejects_cross_rep_fingerprint_drift() { let mut n = 0; diff --git a/crates/aura-bench/src/synth.rs b/crates/aura-bench/src/synth.rs index d2acc71..b915f10 100644 --- a/crates/aura-bench/src/synth.rs +++ b/crates/aura-bench/src/synth.rs @@ -2,12 +2,6 @@ //! 48-byte M1 zip archive writer (ingest + campaign surfaces), and the scratch //! temp-dir guard. Everything here is a pure function of its arguments — same //! bytes on every host, every run (C1 is what makes the engine benchmarkable). -//! -//! The CLI glue that wires this module's public surface into the surfaces' -//! measurement loop lands in Task 7; until then nothing outside `#[cfg(test)]` -//! calls into it, so the module is exempted from the dead-code lint here -//! rather than left to trip the workspace's `-D warnings` clippy gate. -#![allow(dead_code)] use aura_core::{Scalar, Timestamp}; use std::io::Write; @@ -224,8 +218,9 @@ mod tests { } /// #251: the synthetic archive is the ingest surface's benchmark input - /// (Task 7), so its bytes must be more than internally self-consistent — - /// they must actually parse through the real production M1 reader + /// (fed through the CLI wiring), so its bytes must be more than + /// internally self-consistent — they must actually parse through the + /// real production M1 reader /// (`aura_ingest::DataServer`/`load_m1_window`), the same path a real /// GER40 archive goes through. This is not a round trip against our own /// `pack_record`: it is the production parser recovering `pack_record`'s diff --git a/crates/aura-bench/tests/profile_guard.rs b/crates/aura-bench/tests/profile_guard.rs index 8b8e37e..97d8eb7 100644 --- a/crates/aura-bench/tests/profile_guard.rs +++ b/crates/aura-bench/tests/profile_guard.rs @@ -13,8 +13,8 @@ const BIN: &str = env!("CARGO_BIN_EXE_aura-bench"); /// baseline. This pins the profile guard's observable contract — exit code 2 /// (the usage-error convention aura-bench shares with `aura-cli`) and a /// stderr message naming both the cause ("debug build") and the fix -/// ("--release") — so a future CLI-wiring change (the surfaces land in -/// Task 7) cannot silently drop or weaken the guard. +/// ("--release") — so a future CLI-wiring change (the surfaces glued into +/// the CLI) cannot silently drop or weaken the guard. #[test] fn refuses_to_run_in_a_debug_build() { // `cargo test` builds test-harness binaries (and their bin-target