feat(bench): host probes, child-process runner + the two campaign surfaces

Tasks 5-6 of the bench-harness plan (refs #251). Host context probes
(hostname/nproc/loadavg/commit/date, all best-effort), the child runner
(release aura binary resolved from cargo's compiler-artifact JSON messages;
/usr/bin/time -v wraps the child for CPU% + peak RSS, wall measured
in-process), and the campaign surfaces:

- a scratch data-only project per repetition ([paths] only — no [nodes]; the
  std-vocabulary fallback covers the SMA blueprints), synthetic archives for
  4 instruments, blueprints seeded via tiny synthetic sweeps, process docs
  registered through the real CLI;
- campaign_sweep (sweep-only process — narrow per-cell fan-out) and
  campaign_heavy (sweep -> gate -> walk-forward -> 1000-resample bootstrap —
  wide fan-out): one campaign shape misreads regressions in the other;
- fingerprint = winner ordinals parsed from the always-on final
  {"campaign_run":...} stdout line.

The full campaign-sweep determinism E2E (real release binary, real scratch
runs) is #[ignore]d out of the default suite — it costs a release build plus
child campaign runs, which the suite-wallclock discipline forbids on every
cargo test; run it via `cargo test -p aura-bench -- --ignored`. Held
quality note: the heavy/quick process docs stay two verbatim consts (plan
prescription; a format! builder would trade the uniform &str contract for
cosmetic dedup).
This commit is contained in:
2026-07-17 14:42:13 +02:00
parent e015317183
commit 3a25a4ccba
5 changed files with 571 additions and 0 deletions
+155
View File
@@ -0,0 +1,155 @@
//! Child-process measurement: resolve the release `aura` binary via cargo's
//! JSON messages, run it under `/usr/bin/time -v`, and parse wall/CPU/RSS
//! from the stderr report. `/usr/bin/time` missing is an infra error — the
//! campaign surfaces have no fallback RSS probe by design (one tool, one
//! parse, the practice proven in the #277 measurement session).
//!
//! The CLI glue that wires this module's public surface into the surfaces'
//! measurement loop lands in Task 7; until then nothing outside `#[cfg(test)]`
//! calls into it, so the module is exempted from the dead-code lint here
//! rather than left to trip the workspace's `-D warnings` clippy gate.
#![allow(dead_code)]
use std::path::{Path, PathBuf};
use std::time::Instant;
/// Workspace root: two levels above this crate's manifest dir.
pub fn workspace_root() -> PathBuf {
Path::new(env!("CARGO_MANIFEST_DIR"))
.ancestors()
.nth(2)
.expect("crates/aura-bench sits two levels under the workspace root")
.to_path_buf()
}
/// Build (if needed) and locate the release `aura` binary. Unmeasured setup.
pub fn resolve_aura_bin(ws_root: &Path) -> Result<PathBuf, String> {
let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string());
let out = std::process::Command::new(cargo)
.args(["build", "--release", "-p", "aura-cli", "--message-format=json"])
.current_dir(ws_root)
.output()
.map_err(|e| format!("spawn cargo build: {e}"))?;
if !out.status.success() {
return Err(format!(
"cargo build --release -p aura-cli failed:\n{}",
String::from_utf8_lossy(&out.stderr)
));
}
parse_aura_executable(&String::from_utf8_lossy(&out.stdout))
.ok_or_else(|| "no compiler-artifact message named `aura` with an executable".to_string())
}
/// Extract the `aura` executable path from `--message-format=json` lines.
pub fn parse_aura_executable(stdout: &str) -> Option<PathBuf> {
for line in stdout.lines() {
let Ok(v) = serde_json::from_str::<serde_json::Value>(line) else { continue };
if v["reason"] == "compiler-artifact"
&& v["target"]["name"] == "aura"
&& v["executable"].is_string()
{
return Some(PathBuf::from(v["executable"].as_str().expect("checked string")));
}
}
None
}
pub struct TimedChild {
pub wall_s: f64,
pub cpu_percent: Option<f64>,
pub peak_rss_mb: Option<f64>,
pub stdout: String,
pub stderr: String,
pub exit: Option<i32>,
}
/// Run `argv` in `cwd` under `/usr/bin/time -v`, measuring wall in-process.
pub fn run_timed(bin: &Path, args: &[&str], cwd: &Path) -> Result<TimedChild, String> {
if !Path::new("/usr/bin/time").exists() {
return Err("/usr/bin/time is required for child-process surfaces".to_string());
}
let t = Instant::now();
let out = std::process::Command::new("/usr/bin/time")
.arg("-v")
.arg(bin)
.args(args)
.current_dir(cwd)
.output()
.map_err(|e| format!("spawn {}: {e}", bin.display()))?;
let wall_s = t.elapsed().as_secs_f64();
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
let (cpu_percent, peak_rss_mb) = parse_time_v(&stderr);
Ok(TimedChild {
wall_s,
cpu_percent,
peak_rss_mb,
stdout: String::from_utf8_lossy(&out.stdout).into_owned(),
stderr,
exit: out.status.code(),
})
}
/// Parse `Percent of CPU this job got` and `Maximum resident set size` out of
/// a `/usr/bin/time -v` stderr report.
pub fn parse_time_v(stderr: &str) -> (Option<f64>, Option<f64>) {
let mut cpu = None;
let mut rss = None;
for line in stderr.lines() {
let line = line.trim();
if let Some(v) = line.strip_prefix("Percent of CPU this job got: ") {
cpu = v.trim_end_matches('%').parse::<f64>().ok();
} else if let Some(v) = line.strip_prefix("Maximum resident set size (kbytes): ") {
rss = v.parse::<f64>().ok().map(|kb| kb / 1024.0);
}
}
(cpu, rss)
}
#[cfg(test)]
mod tests {
use super::*;
/// `resolve_aura_bin` hands its `cargo build --release -p aura-cli`
/// invocation a `current_dir`, and that invocation only produces the
/// right binary if `workspace_root` actually resolves to the real
/// workspace root — the `.ancestors().nth(2)` computation is a
/// hardcoded distance that would silently point at the wrong directory
/// (and build nothing, or the wrong crate graph) if `aura-bench` were
/// ever relocated a level deeper or shallower. Pins that it lands on
/// the directory holding the actual workspace manifest, not merely
/// some ancestor.
#[test]
fn workspace_root_resolves_the_real_cargo_workspace() {
let root = workspace_root();
let manifest = std::fs::read_to_string(root.join("Cargo.toml")).unwrap_or_else(|e| {
panic!("workspace_root ({}) must contain Cargo.toml: {e}", root.display())
});
assert!(manifest.contains("[workspace]"), "not the workspace root: {}", root.display());
}
#[test]
fn parse_time_v_extracts_cpu_and_rss() {
let s = "\tCommand being timed: \"aura --help\"\n\
\tPercent of CPU this job got: 2270%\n\
\tMaximum resident set size (kbytes): 1048576\n";
let (cpu, rss) = parse_time_v(s);
assert_eq!(cpu, Some(2270.0));
assert_eq!(rss, Some(1024.0));
}
#[test]
fn parse_aura_executable_reads_compiler_artifact_lines() {
let lines = concat!(
"{\"reason\":\"compiler-artifact\",\"target\":{\"name\":\"aura-bench\"},\"executable\":\"/x/aura-bench\"}\n",
"{\"reason\":\"compiler-artifact\",\"target\":{\"name\":\"aura\"},\"executable\":\"/x/release/aura\"}\n",
"{\"reason\":\"build-finished\",\"success\":true}\n",
);
assert_eq!(parse_aura_executable(lines), Some(PathBuf::from("/x/release/aura")));
}
#[test]
fn parse_aura_executable_ignores_null_executables() {
let lines = "{\"reason\":\"compiler-artifact\",\"target\":{\"name\":\"aura\"},\"executable\":null}\n";
assert_eq!(parse_aura_executable(lines), None);
}
}
+75
View File
@@ -0,0 +1,75 @@
//! Host context probes. All best-effort: a probe that fails degrades to a
//! placeholder value and never fails a bench (the report stays honest via the
//! host block committed inside each baseline).
//!
//! The CLI glue that wires this module's public surface into the surfaces'
//! measurement loop lands in Task 7; until then nothing outside `#[cfg(test)]`
//! calls into it, so the module is exempted from the dead-code lint here
//! rather than left to trip the workspace's `-D warnings` clippy gate.
#![allow(dead_code)]
use crate::baseline::HostInfo;
pub fn host_info() -> HostInfo {
let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname")
.map(|s| s.trim().to_string())
.unwrap_or_else(|_| "unknown".to_string());
let nproc = std::thread::available_parallelism().map(|n| n.get() as u32).unwrap_or(0);
HostInfo { hostname, nproc }
}
/// 1-minute loadavg, or `None` when unreadable.
pub fn loadavg_1m() -> Option<f64> {
let text = std::fs::read_to_string("/proc/loadavg").ok()?;
text.split_whitespace().next()?.parse().ok()
}
/// Short commit of the enclosing worktree, best-effort.
pub fn git_commit() -> String {
std::process::Command::new("git")
.args(["rev-parse", "--short", "HEAD"])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_else(|| "unknown".to_string())
}
/// UTC date `YYYY-MM-DD`, best-effort via `date -u`.
pub fn today() -> String {
std::process::Command::new("date")
.args(["-u", "+%F"])
.output()
.ok()
.filter(|o| o.status.success())
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
.unwrap_or_else(|| "unknown".to_string())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn host_info_reads_something() {
let h = host_info();
assert!(!h.hostname.is_empty());
}
/// #251: every committed baseline records `commit` for provenance (a
/// baseline that can't be tied to the code it measured is useless for
/// drift comparison). This test runs inside a real git worktree, so
/// `git_commit` must shell out and parse successfully — falling silently
/// back to `"unknown"` here would be exactly the failure mode the
/// best-effort fallback exists to hide, and this pins the happy path
/// rather than only the "never panics" degenerate case.
#[test]
fn git_commit_reports_the_real_short_hash_inside_a_git_worktree() {
let commit = git_commit();
assert_ne!(commit, "unknown", "must read the real worktree HEAD, not fall back silently");
assert!(
commit.len() >= 7 && commit.chars().all(|c| c.is_ascii_hexdigit()),
"not a short hex hash: {commit:?}"
);
}
}
+2
View File
@@ -4,6 +4,8 @@
//! bench instead of pinning a bogus baseline.
mod baseline;
mod child;
mod host;
mod surfaces;
mod synth;
+307
View File
@@ -0,0 +1,307 @@
//! The two campaign surfaces: `campaign_sweep` (narrow per-cell fan-out — a
//! sweep-only process) and `campaign_heavy` (wide fan-out — sweep → gate →
//! walk-forward → bootstrap), both run as the release `aura` binary on a
//! scratch data-only project over synthetic archives. Fingerprint: the winner
//! ordinals extracted from the always-on final `{"campaign_run":...}` stdout
//! line. One campaign shape misreads regressions in the other (#251), so both
//! are pinned.
use super::{RepOutcome, Sizing};
use crate::child::{run_timed, TimedChild};
use crate::synth::{write_symbol_archive, ScratchDir};
use std::collections::BTreeMap;
use std::path::Path;
pub const FULL_INSTRUMENTS: &[&str] = &["BSYMA", "BSYMB", "BSYMC", "BSYMD"];
pub const QUICK_INSTRUMENTS: &[&str] = &["BSYMA", "BSYMB"];
/// Archive span (2024-01..08) and campaign window (Mar-01..Jun-30) mirror the
/// green E2E fixture: ~17 weeks gives the (14d, 7d, 7d) roller a comfortable
/// tiling (research_docs.rs:2440-2441).
pub const ARCHIVE_MONTHS: ((i32, u32), (i32, u32)) = ((2024, 1), (2024, 8));
pub const WINDOW_MS: (i64, i64) = (1709251200000, 1719791999999);
pub const SWEEP_PROCESS_DOC: &str = r#"{
"format_version": 1,
"kind": "process",
"name": "bench-sweep-only",
"pipeline": [
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" }
]
}"#;
/// The heavy shape: the E2E walk-forward process with the bootstrap sized up
/// (1000 resamples — the #277 measured workload's count).
pub const HEAVY_PROCESS_DOC: &str = r#"{
"format_version": 1,
"kind": "process",
"name": "bench-heavy",
"pipeline": [
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
{ "block": "std::gate", "all": [ { "metric": "n_trades", "cmp": "ge", "value": 0.0 } ] },
{ "block": "std::walk_forward", "in_sample_ms": 1209600000, "out_of_sample_ms": 604800000,
"step_ms": 604800000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" },
{ "block": "std::monte_carlo", "resamples": 1000, "block_len": 5 }
]
}"#;
pub const HEAVY_PROCESS_DOC_QUICK: &str = r#"{
"format_version": 1,
"kind": "process",
"name": "bench-heavy",
"pipeline": [
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
{ "block": "std::gate", "all": [ { "metric": "n_trades", "cmp": "ge", "value": 0.0 } ] },
{ "block": "std::walk_forward", "in_sample_ms": 1209600000, "out_of_sample_ms": 604800000,
"step_ms": 604800000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" },
{ "block": "std::monte_carlo", "resamples": 50, "block_len": 5 }
]
}"#;
/// The two SMA strategy blueprints: byte-identical to the shipped
/// `crates/aura-cli/examples/r_sma.json` except the second variant's bound
/// lengths (3/6 instead of 2/4), which gives it a distinct content id — two
/// strategies, same axes surface.
pub const BP_SMA_A: &str = r#"{"format_version":1,"blueprint":{"name":"sma_signal","nodes":[{"primitive":{"type":"SMA","name":"fast","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":2}}]}},{"primitive":{"type":"SMA","name":"slow","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":4}}]}},{"primitive":{"type":"Sub"}},{"primitive":{"type":"Bias","name":"bias","bound":[{"pos":0,"name":"scale","kind":"F64","value":{"F64":0.5}}]}}],"edges":[{"from":0,"to":2,"slot":0,"from_field":0},{"from":1,"to":2,"slot":1,"from_field":0},{"from":2,"to":3,"slot":0,"from_field":0}],"input_roles":[{"name":"price","targets":[{"node":0,"slot":0},{"node":1,"slot":0}],"source":"F64"}],"output":[{"node":3,"field":0,"name":"bias"}]}}"#;
pub const BP_SMA_B: &str = r#"{"format_version":1,"blueprint":{"name":"sma_signal","nodes":[{"primitive":{"type":"SMA","name":"fast","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":3}}]}},{"primitive":{"type":"SMA","name":"slow","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":6}}]}},{"primitive":{"type":"Sub"}},{"primitive":{"type":"Bias","name":"bias","bound":[{"pos":0,"name":"scale","kind":"F64","value":{"F64":0.5}}]}}],"edges":[{"from":0,"to":2,"slot":0,"from_field":0},{"from":1,"to":2,"slot":1,"from_field":0},{"from":2,"to":3,"slot":0,"from_field":0}],"input_roles":[{"name":"price","targets":[{"node":0,"slot":0},{"node":1,"slot":0}],"source":"F64"}],"output":[{"node":3,"field":0,"name":"bias"}]}}"#;
fn run_in(bin: &Path, dir: &Path, args: &[&str]) -> Result<(String, Option<i32>), String> {
let out = std::process::Command::new(bin)
.args(args)
.current_dir(dir)
.output()
.map_err(|e| format!("spawn aura {args:?}: {e}"))?;
let text = format!(
"{}{}",
String::from_utf8_lossy(&out.stdout),
String::from_utf8_lossy(&out.stderr)
);
Ok((text, out.status.code()))
}
/// Blueprint stems currently in the scratch store.
fn blueprint_stems(dir: &Path) -> Vec<String> {
std::fs::read_dir(dir.join("runs").join("blueprints"))
.map(|rd| {
rd.filter_map(|e| e.ok())
.filter_map(|e| e.path().file_stem().map(|s| s.to_string_lossy().into_owned()))
.collect()
})
.unwrap_or_default()
}
/// Seed one blueprint into the scratch store via a tiny synthetic sweep (the
/// E2E `seed_blueprint` pattern) and return its content id.
fn seed_blueprint(bin: &Path, dir: &Path, file: &str, name: &str) -> Result<String, String> {
let before = blueprint_stems(dir);
let (out, code) = run_in(
bin,
dir,
&[
"sweep",
file,
"--axis",
"sma_signal.fast.length=2,4",
"--axis",
"sma_signal.slow.length=8,16",
"--name",
name,
],
)?;
if code != Some(0) {
return Err(format!("seed sweep failed ({code:?}): {out}"));
}
blueprint_stems(dir)
.into_iter()
.find(|s| !before.contains(s))
.ok_or_else(|| "seed sweep must store a new blueprint".to_string())
}
fn register_process(bin: &Path, dir: &Path, file: &str, doc: &str) -> Result<String, String> {
std::fs::write(dir.join(file), doc).map_err(|e| format!("write {file}: {e}"))?;
let (out, code) = run_in(bin, dir, &["process", "register", file])?;
if code != Some(0) {
return Err(format!("process register failed ({code:?}): {out}"));
}
out.lines()
.find(|l| l.starts_with("registered process "))
.map(|l| {
l.trim_start_matches("registered process ")
.split(' ')
.next()
.expect("split yields at least one piece")
.trim_start_matches("content:")
.to_string()
})
.ok_or_else(|| format!("no register line in: {out}"))
}
fn campaign_doc(instruments: &[&str], bp_ids: &[String], proc_id: &str) -> String {
let instruments_json =
instruments.iter().map(|i| format!("\"{i}\"")).collect::<Vec<_>>().join(", ");
let strategies_json = bp_ids
.iter()
.map(|id| {
format!(
r#"{{ "ref": {{ "content_id": "{id}" }},
"axes": {{ "fast.length": {{ "kind": "I64", "values": [2, 3, 4] }},
"slow.length": {{ "kind": "I64", "values": [8, 12, 16] }} }} }}"#
)
})
.collect::<Vec<_>>()
.join(",\n ");
format!(
r#"{{
"format_version": 1,
"kind": "campaign",
"name": "bench",
"data": {{ "instruments": [{instruments_json}], "windows": [ {{ "from_ms": {from}, "to_ms": {to} }} ] }},
"strategies": [ {strategies_json} ],
"process": {{ "ref": {{ "content_id": "{proc_id}" }} }},
"seed": 7,
"presentation": {{ "persist_taps": [], "emit": [] }}
}}"#,
from = WINDOW_MS.0,
to = WINDOW_MS.1,
)
}
/// Build a fresh scratch campaign project and return (scratch, campaign doc
/// file name). Setup, unmeasured.
fn build_scratch(bin: &Path, sizing: Sizing, process_doc: &str) -> Result<(ScratchDir, String), String> {
let scratch = ScratchDir::new("campaign").map_err(|e| format!("scratch dir: {e}"))?;
let root = &scratch.0;
std::fs::write(root.join("Aura.toml"), "[paths]\nruns = \"runs\"\ndata = \"data\"\n")
.map_err(|e| format!("write Aura.toml: {e}"))?;
let data_dir = root.join("data");
std::fs::create_dir_all(&data_dir).map_err(|e| format!("create data dir: {e}"))?;
let instruments = if sizing.quick { QUICK_INSTRUMENTS } else { FULL_INSTRUMENTS };
for sym in instruments {
write_symbol_archive(&data_dir, sym, ARCHIVE_MONTHS.0, ARCHIVE_MONTHS.1);
}
std::fs::write(root.join("bench_sma_a.json"), BP_SMA_A)
.map_err(|e| format!("write blueprint a: {e}"))?;
let mut bp_ids = vec![seed_blueprint(bin, root, "bench_sma_a.json", "seed-a")?];
if !sizing.quick {
std::fs::write(root.join("bench_sma_b.json"), BP_SMA_B)
.map_err(|e| format!("write blueprint b: {e}"))?;
bp_ids.push(seed_blueprint(bin, root, "bench_sma_b.json", "seed-b")?);
}
let proc_id = register_process(bin, root, "bench.process.json", process_doc)?;
let doc = campaign_doc(instruments, &bp_ids, &proc_id);
std::fs::write(root.join("bench.campaign.json"), doc)
.map_err(|e| format!("write campaign doc: {e}"))?;
Ok((scratch, "bench.campaign.json".to_string()))
}
/// Extract the winner-ordinal fingerprint from the always-on final
/// `{"campaign_run":...}` stdout line.
pub fn winner_fingerprint(stdout: &str) -> Result<String, String> {
let line = stdout
.lines()
.find(|l| l.starts_with("{\"campaign_run\":"))
.ok_or("no campaign_run record line on stdout")?;
let v: serde_json::Value =
serde_json::from_str(line).map_err(|e| format!("campaign_run line parses: {e}"))?;
let cells = v["campaign_run"]["cells"]
.as_array()
.ok_or("campaign_run.cells is an array")?;
let mut parts = Vec::new();
for (ci, cell) in cells.iter().enumerate() {
let stages = cell["stages"].as_array().ok_or("cell.stages is an array")?;
for (si, stage) in stages.iter().enumerate() {
if let Some(w) = stage["selection"]["winner_ordinal"].as_u64() {
parts.push(format!("c{ci}s{si}w{w}"));
}
}
}
if parts.is_empty() {
return Err("no winner ordinals in the campaign_run record".to_string());
}
Ok(format!("cells={} {}", cells.len(), parts.join(" ")))
}
fn campaign_rep(bin: &Path, sizing: Sizing, process_doc: &str) -> Result<RepOutcome, String> {
let (scratch, doc) = build_scratch(bin, sizing, process_doc)?;
let timed: TimedChild = run_timed(bin, &["campaign", "run", &doc], &scratch.0)?;
if timed.exit != Some(0) {
return Err(format!(
"campaign run exited {:?}\nstdout: {}\nstderr: {}",
timed.exit, timed.stdout, timed.stderr
));
}
let fingerprint = winner_fingerprint(&timed.stdout)?;
let mut metrics = BTreeMap::new();
metrics.insert("wall_s".to_string(), timed.wall_s);
if let Some(c) = timed.cpu_percent {
metrics.insert("cpu_percent".to_string(), c);
}
if let Some(r) = timed.peak_rss_mb {
metrics.insert("peak_rss_mb".to_string(), r);
}
Ok(RepOutcome { metrics, fingerprint })
}
pub fn sweep_rep(bin: &Path, sizing: Sizing) -> Result<RepOutcome, String> {
campaign_rep(bin, sizing, SWEEP_PROCESS_DOC)
}
pub fn heavy_rep(bin: &Path, sizing: Sizing) -> Result<RepOutcome, String> {
let doc = if sizing.quick { HEAVY_PROCESS_DOC_QUICK } else { HEAVY_PROCESS_DOC };
campaign_rep(bin, sizing, doc)
}
/// Used by `cli_fixed_cost` (Task 7): a minimal one-symbol scratch project.
pub fn build_minimal_project() -> Result<ScratchDir, String> {
let scratch = ScratchDir::new("fixedcost").map_err(|e| format!("scratch dir: {e}"))?;
let root = &scratch.0;
std::fs::write(root.join("Aura.toml"), "[paths]\nruns = \"runs\"\ndata = \"data\"\n")
.map_err(|e| format!("write Aura.toml: {e}"))?;
let data_dir = root.join("data");
std::fs::create_dir_all(&data_dir).map_err(|e| format!("create data dir: {e}"))?;
write_symbol_archive(&data_dir, "BSYMA", (2024, 1), (2024, 1));
std::fs::write(root.join("bench_sma_a.json"), BP_SMA_A)
.map_err(|e| format!("write blueprint: {e}"))?;
Ok(scratch)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn winner_fingerprint_extracts_ordinals_from_a_record_line() {
let stdout = "noise\n{\"campaign_run\":{\"campaign\":\"c\",\"process\":\"p\",\"run\":0,\"seed\":7,\
\"cells\":[{\"strategy\":\"s\",\"instrument\":\"I\",\"window_ms\":[1,2],\
\"stages\":[{\"block\":\"std::sweep\",\"selection\":{\"winner_ordinal\":4,\"params\":[],\
\"selection\":\"argmax\"}},{\"block\":\"std::gate\"}]}]}}\n";
let fp = winner_fingerprint(stdout).expect("parses");
assert_eq!(fp, "cells=1 c0s0w4");
}
#[test]
fn winner_fingerprint_rejects_missing_record_line() {
assert!(winner_fingerprint("no record here\n").is_err());
}
#[test]
fn campaign_doc_lists_all_instruments_and_strategies() {
let doc = campaign_doc(
&["BSYMA", "BSYMB"],
&["id-a".to_string(), "id-b".to_string()],
"proc-id",
);
let v: serde_json::Value = serde_json::from_str(&doc).expect("doc is valid JSON");
assert_eq!(v["data"]["instruments"].as_array().expect("instruments").len(), 2);
assert_eq!(v["strategies"].as_array().expect("strategies").len(), 2);
assert_eq!(v["seed"], 7);
assert_eq!(v["process"]["ref"]["content_id"], "proc-id");
}
#[test]
fn process_docs_are_valid_json() {
for doc in [SWEEP_PROCESS_DOC, HEAVY_PROCESS_DOC, HEAVY_PROCESS_DOC_QUICK] {
let v: serde_json::Value = serde_json::from_str(doc).expect("process doc parses");
assert_eq!(v["kind"], "process");
}
}
}
+32
View File
@@ -10,6 +10,7 @@
//! than left to trip the workspace's `-D warnings` clippy gate.
#![allow(dead_code)]
pub mod campaign;
pub mod engine;
pub mod ingest;
@@ -106,6 +107,37 @@ mod tests {
assert!(measured.metrics.contains_key("wall_s"));
}
/// `run_reps` composed with the real campaign sweep surface — a fresh
/// scratch project, a freshly seeded blueprint, and a genuine
/// `aura campaign run` child process spawned per repetition (not an
/// in-process shortcut). Protects that two independently built-and-run
/// scratch campaigns agree on the winner-ordinal fingerprint extracted
/// from the child's stdout — C1's determinism carried through the
/// child-process boundary, which is the whole reason the campaign
/// surfaces exist as a distinct measurement path from the in-process
/// engine/ingest surfaces above — and that the metrics map carries the
/// wall-clock key the campaign `BaselineDoc` needs.
///
/// Ignored in the default suite: it builds the release `aura` binary and
/// spawns real campaign child runs (minutes cold) — the suite-wallclock
/// discipline forbids that cost on every `cargo test`. Run it explicitly:
/// `cargo test -p aura-bench -- --ignored`.
#[test]
#[ignore = "builds the release aura binary + real campaign scratch runs; run via -- --ignored"]
fn run_reps_drives_the_real_campaign_sweep_surface_deterministically() {
let bin = crate::child::resolve_aura_bin(&crate::child::workspace_root())
.expect("the release aura binary must build");
let measured = run_reps("campaign_sweep", 2, || campaign::sweep_rep(&bin, Sizing { quick: true }))
.expect("the real campaign sweep surface must agree with itself across independent scratch runs");
assert_eq!(measured.surface, "campaign_sweep");
assert!(measured.metrics.contains_key("wall_s"));
assert!(
measured.fingerprint.starts_with("cells="),
"fingerprint: {}",
measured.fingerprint
);
}
#[test]
fn run_reps_rejects_cross_rep_fingerprint_drift() {
let mut n = 0;