feat(bench): host probes, child-process runner + the two campaign surfaces
Tasks 5-6 of the bench-harness plan (refs #251). Host context probes (hostname/nproc/loadavg/commit/date, all best-effort), the child runner (release aura binary resolved from cargo's compiler-artifact JSON messages; /usr/bin/time -v wraps the child for CPU% + peak RSS, wall measured in-process), and the campaign surfaces: - a scratch data-only project per repetition ([paths] only — no [nodes]; the std-vocabulary fallback covers the SMA blueprints), synthetic archives for 4 instruments, blueprints seeded via tiny synthetic sweeps, process docs registered through the real CLI; - campaign_sweep (sweep-only process — narrow per-cell fan-out) and campaign_heavy (sweep -> gate -> walk-forward -> 1000-resample bootstrap — wide fan-out): one campaign shape misreads regressions in the other; - fingerprint = winner ordinals parsed from the always-on final {"campaign_run":...} stdout line. The full campaign-sweep determinism E2E (real release binary, real scratch runs) is #[ignore]d out of the default suite — it costs a release build plus child campaign runs, which the suite-wallclock discipline forbids on every cargo test; run it via `cargo test -p aura-bench -- --ignored`. Held quality note: the heavy/quick process docs stay two verbatim consts (plan prescription; a format! builder would trade the uniform &str contract for cosmetic dedup).
This commit is contained in:
@@ -0,0 +1,155 @@
|
||||
//! Child-process measurement: resolve the release `aura` binary via cargo's
|
||||
//! JSON messages, run it under `/usr/bin/time -v`, and parse wall/CPU/RSS
|
||||
//! from the stderr report. `/usr/bin/time` missing is an infra error — the
|
||||
//! campaign surfaces have no fallback RSS probe by design (one tool, one
|
||||
//! parse, the practice proven in the #277 measurement session).
|
||||
//!
|
||||
//! The CLI glue that wires this module's public surface into the surfaces'
|
||||
//! measurement loop lands in Task 7; until then nothing outside `#[cfg(test)]`
|
||||
//! calls into it, so the module is exempted from the dead-code lint here
|
||||
//! rather than left to trip the workspace's `-D warnings` clippy gate.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Instant;
|
||||
|
||||
/// Workspace root: two levels above this crate's manifest dir.
|
||||
pub fn workspace_root() -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.ancestors()
|
||||
.nth(2)
|
||||
.expect("crates/aura-bench sits two levels under the workspace root")
|
||||
.to_path_buf()
|
||||
}
|
||||
|
||||
/// Build (if needed) and locate the release `aura` binary. Unmeasured setup.
|
||||
pub fn resolve_aura_bin(ws_root: &Path) -> Result<PathBuf, String> {
|
||||
let cargo = std::env::var("CARGO").unwrap_or_else(|_| "cargo".to_string());
|
||||
let out = std::process::Command::new(cargo)
|
||||
.args(["build", "--release", "-p", "aura-cli", "--message-format=json"])
|
||||
.current_dir(ws_root)
|
||||
.output()
|
||||
.map_err(|e| format!("spawn cargo build: {e}"))?;
|
||||
if !out.status.success() {
|
||||
return Err(format!(
|
||||
"cargo build --release -p aura-cli failed:\n{}",
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
));
|
||||
}
|
||||
parse_aura_executable(&String::from_utf8_lossy(&out.stdout))
|
||||
.ok_or_else(|| "no compiler-artifact message named `aura` with an executable".to_string())
|
||||
}
|
||||
|
||||
/// Extract the `aura` executable path from `--message-format=json` lines.
|
||||
pub fn parse_aura_executable(stdout: &str) -> Option<PathBuf> {
|
||||
for line in stdout.lines() {
|
||||
let Ok(v) = serde_json::from_str::<serde_json::Value>(line) else { continue };
|
||||
if v["reason"] == "compiler-artifact"
|
||||
&& v["target"]["name"] == "aura"
|
||||
&& v["executable"].is_string()
|
||||
{
|
||||
return Some(PathBuf::from(v["executable"].as_str().expect("checked string")));
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
pub struct TimedChild {
|
||||
pub wall_s: f64,
|
||||
pub cpu_percent: Option<f64>,
|
||||
pub peak_rss_mb: Option<f64>,
|
||||
pub stdout: String,
|
||||
pub stderr: String,
|
||||
pub exit: Option<i32>,
|
||||
}
|
||||
|
||||
/// Run `argv` in `cwd` under `/usr/bin/time -v`, measuring wall in-process.
|
||||
pub fn run_timed(bin: &Path, args: &[&str], cwd: &Path) -> Result<TimedChild, String> {
|
||||
if !Path::new("/usr/bin/time").exists() {
|
||||
return Err("/usr/bin/time is required for child-process surfaces".to_string());
|
||||
}
|
||||
let t = Instant::now();
|
||||
let out = std::process::Command::new("/usr/bin/time")
|
||||
.arg("-v")
|
||||
.arg(bin)
|
||||
.args(args)
|
||||
.current_dir(cwd)
|
||||
.output()
|
||||
.map_err(|e| format!("spawn {}: {e}", bin.display()))?;
|
||||
let wall_s = t.elapsed().as_secs_f64();
|
||||
let stderr = String::from_utf8_lossy(&out.stderr).into_owned();
|
||||
let (cpu_percent, peak_rss_mb) = parse_time_v(&stderr);
|
||||
Ok(TimedChild {
|
||||
wall_s,
|
||||
cpu_percent,
|
||||
peak_rss_mb,
|
||||
stdout: String::from_utf8_lossy(&out.stdout).into_owned(),
|
||||
stderr,
|
||||
exit: out.status.code(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Parse `Percent of CPU this job got` and `Maximum resident set size` out of
|
||||
/// a `/usr/bin/time -v` stderr report.
|
||||
pub fn parse_time_v(stderr: &str) -> (Option<f64>, Option<f64>) {
|
||||
let mut cpu = None;
|
||||
let mut rss = None;
|
||||
for line in stderr.lines() {
|
||||
let line = line.trim();
|
||||
if let Some(v) = line.strip_prefix("Percent of CPU this job got: ") {
|
||||
cpu = v.trim_end_matches('%').parse::<f64>().ok();
|
||||
} else if let Some(v) = line.strip_prefix("Maximum resident set size (kbytes): ") {
|
||||
rss = v.parse::<f64>().ok().map(|kb| kb / 1024.0);
|
||||
}
|
||||
}
|
||||
(cpu, rss)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// `resolve_aura_bin` hands its `cargo build --release -p aura-cli`
|
||||
/// invocation a `current_dir`, and that invocation only produces the
|
||||
/// right binary if `workspace_root` actually resolves to the real
|
||||
/// workspace root — the `.ancestors().nth(2)` computation is a
|
||||
/// hardcoded distance that would silently point at the wrong directory
|
||||
/// (and build nothing, or the wrong crate graph) if `aura-bench` were
|
||||
/// ever relocated a level deeper or shallower. Pins that it lands on
|
||||
/// the directory holding the actual workspace manifest, not merely
|
||||
/// some ancestor.
|
||||
#[test]
|
||||
fn workspace_root_resolves_the_real_cargo_workspace() {
|
||||
let root = workspace_root();
|
||||
let manifest = std::fs::read_to_string(root.join("Cargo.toml")).unwrap_or_else(|e| {
|
||||
panic!("workspace_root ({}) must contain Cargo.toml: {e}", root.display())
|
||||
});
|
||||
assert!(manifest.contains("[workspace]"), "not the workspace root: {}", root.display());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_time_v_extracts_cpu_and_rss() {
|
||||
let s = "\tCommand being timed: \"aura --help\"\n\
|
||||
\tPercent of CPU this job got: 2270%\n\
|
||||
\tMaximum resident set size (kbytes): 1048576\n";
|
||||
let (cpu, rss) = parse_time_v(s);
|
||||
assert_eq!(cpu, Some(2270.0));
|
||||
assert_eq!(rss, Some(1024.0));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_aura_executable_reads_compiler_artifact_lines() {
|
||||
let lines = concat!(
|
||||
"{\"reason\":\"compiler-artifact\",\"target\":{\"name\":\"aura-bench\"},\"executable\":\"/x/aura-bench\"}\n",
|
||||
"{\"reason\":\"compiler-artifact\",\"target\":{\"name\":\"aura\"},\"executable\":\"/x/release/aura\"}\n",
|
||||
"{\"reason\":\"build-finished\",\"success\":true}\n",
|
||||
);
|
||||
assert_eq!(parse_aura_executable(lines), Some(PathBuf::from("/x/release/aura")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_aura_executable_ignores_null_executables() {
|
||||
let lines = "{\"reason\":\"compiler-artifact\",\"target\":{\"name\":\"aura\"},\"executable\":null}\n";
|
||||
assert_eq!(parse_aura_executable(lines), None);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
//! Host context probes. All best-effort: a probe that fails degrades to a
|
||||
//! placeholder value and never fails a bench (the report stays honest via the
|
||||
//! host block committed inside each baseline).
|
||||
//!
|
||||
//! The CLI glue that wires this module's public surface into the surfaces'
|
||||
//! measurement loop lands in Task 7; until then nothing outside `#[cfg(test)]`
|
||||
//! calls into it, so the module is exempted from the dead-code lint here
|
||||
//! rather than left to trip the workspace's `-D warnings` clippy gate.
|
||||
#![allow(dead_code)]
|
||||
|
||||
use crate::baseline::HostInfo;
|
||||
|
||||
pub fn host_info() -> HostInfo {
|
||||
let hostname = std::fs::read_to_string("/proc/sys/kernel/hostname")
|
||||
.map(|s| s.trim().to_string())
|
||||
.unwrap_or_else(|_| "unknown".to_string());
|
||||
let nproc = std::thread::available_parallelism().map(|n| n.get() as u32).unwrap_or(0);
|
||||
HostInfo { hostname, nproc }
|
||||
}
|
||||
|
||||
/// 1-minute loadavg, or `None` when unreadable.
|
||||
pub fn loadavg_1m() -> Option<f64> {
|
||||
let text = std::fs::read_to_string("/proc/loadavg").ok()?;
|
||||
text.split_whitespace().next()?.parse().ok()
|
||||
}
|
||||
|
||||
/// Short commit of the enclosing worktree, best-effort.
|
||||
pub fn git_commit() -> String {
|
||||
std::process::Command::new("git")
|
||||
.args(["rev-parse", "--short", "HEAD"])
|
||||
.output()
|
||||
.ok()
|
||||
.filter(|o| o.status.success())
|
||||
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
}
|
||||
|
||||
/// UTC date `YYYY-MM-DD`, best-effort via `date -u`.
|
||||
pub fn today() -> String {
|
||||
std::process::Command::new("date")
|
||||
.args(["-u", "+%F"])
|
||||
.output()
|
||||
.ok()
|
||||
.filter(|o| o.status.success())
|
||||
.map(|o| String::from_utf8_lossy(&o.stdout).trim().to_string())
|
||||
.unwrap_or_else(|| "unknown".to_string())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn host_info_reads_something() {
|
||||
let h = host_info();
|
||||
assert!(!h.hostname.is_empty());
|
||||
}
|
||||
|
||||
/// #251: every committed baseline records `commit` for provenance (a
|
||||
/// baseline that can't be tied to the code it measured is useless for
|
||||
/// drift comparison). This test runs inside a real git worktree, so
|
||||
/// `git_commit` must shell out and parse successfully — falling silently
|
||||
/// back to `"unknown"` here would be exactly the failure mode the
|
||||
/// best-effort fallback exists to hide, and this pins the happy path
|
||||
/// rather than only the "never panics" degenerate case.
|
||||
#[test]
|
||||
fn git_commit_reports_the_real_short_hash_inside_a_git_worktree() {
|
||||
let commit = git_commit();
|
||||
assert_ne!(commit, "unknown", "must read the real worktree HEAD, not fall back silently");
|
||||
assert!(
|
||||
commit.len() >= 7 && commit.chars().all(|c| c.is_ascii_hexdigit()),
|
||||
"not a short hex hash: {commit:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@
|
||||
//! bench instead of pinning a bogus baseline.
|
||||
|
||||
mod baseline;
|
||||
mod child;
|
||||
mod host;
|
||||
mod surfaces;
|
||||
mod synth;
|
||||
|
||||
|
||||
@@ -0,0 +1,307 @@
|
||||
//! The two campaign surfaces: `campaign_sweep` (narrow per-cell fan-out — a
|
||||
//! sweep-only process) and `campaign_heavy` (wide fan-out — sweep → gate →
|
||||
//! walk-forward → bootstrap), both run as the release `aura` binary on a
|
||||
//! scratch data-only project over synthetic archives. Fingerprint: the winner
|
||||
//! ordinals extracted from the always-on final `{"campaign_run":...}` stdout
|
||||
//! line. One campaign shape misreads regressions in the other (#251), so both
|
||||
//! are pinned.
|
||||
|
||||
use super::{RepOutcome, Sizing};
|
||||
use crate::child::{run_timed, TimedChild};
|
||||
use crate::synth::{write_symbol_archive, ScratchDir};
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
|
||||
pub const FULL_INSTRUMENTS: &[&str] = &["BSYMA", "BSYMB", "BSYMC", "BSYMD"];
|
||||
pub const QUICK_INSTRUMENTS: &[&str] = &["BSYMA", "BSYMB"];
|
||||
/// Archive span (2024-01..08) and campaign window (Mar-01..Jun-30) mirror the
|
||||
/// green E2E fixture: ~17 weeks gives the (14d, 7d, 7d) roller a comfortable
|
||||
/// tiling (research_docs.rs:2440-2441).
|
||||
pub const ARCHIVE_MONTHS: ((i32, u32), (i32, u32)) = ((2024, 1), (2024, 8));
|
||||
pub const WINDOW_MS: (i64, i64) = (1709251200000, 1719791999999);
|
||||
|
||||
pub const SWEEP_PROCESS_DOC: &str = r#"{
|
||||
"format_version": 1,
|
||||
"kind": "process",
|
||||
"name": "bench-sweep-only",
|
||||
"pipeline": [
|
||||
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" }
|
||||
]
|
||||
}"#;
|
||||
|
||||
/// The heavy shape: the E2E walk-forward process with the bootstrap sized up
|
||||
/// (1000 resamples — the #277 measured workload's count).
|
||||
pub const HEAVY_PROCESS_DOC: &str = r#"{
|
||||
"format_version": 1,
|
||||
"kind": "process",
|
||||
"name": "bench-heavy",
|
||||
"pipeline": [
|
||||
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
|
||||
{ "block": "std::gate", "all": [ { "metric": "n_trades", "cmp": "ge", "value": 0.0 } ] },
|
||||
{ "block": "std::walk_forward", "in_sample_ms": 1209600000, "out_of_sample_ms": 604800000,
|
||||
"step_ms": 604800000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" },
|
||||
{ "block": "std::monte_carlo", "resamples": 1000, "block_len": 5 }
|
||||
]
|
||||
}"#;
|
||||
|
||||
pub const HEAVY_PROCESS_DOC_QUICK: &str = r#"{
|
||||
"format_version": 1,
|
||||
"kind": "process",
|
||||
"name": "bench-heavy",
|
||||
"pipeline": [
|
||||
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
|
||||
{ "block": "std::gate", "all": [ { "metric": "n_trades", "cmp": "ge", "value": 0.0 } ] },
|
||||
{ "block": "std::walk_forward", "in_sample_ms": 1209600000, "out_of_sample_ms": 604800000,
|
||||
"step_ms": 604800000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" },
|
||||
{ "block": "std::monte_carlo", "resamples": 50, "block_len": 5 }
|
||||
]
|
||||
}"#;
|
||||
|
||||
/// The two SMA strategy blueprints: byte-identical to the shipped
|
||||
/// `crates/aura-cli/examples/r_sma.json` except the second variant's bound
|
||||
/// lengths (3/6 instead of 2/4), which gives it a distinct content id — two
|
||||
/// strategies, same axes surface.
|
||||
pub const BP_SMA_A: &str = r#"{"format_version":1,"blueprint":{"name":"sma_signal","nodes":[{"primitive":{"type":"SMA","name":"fast","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":2}}]}},{"primitive":{"type":"SMA","name":"slow","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":4}}]}},{"primitive":{"type":"Sub"}},{"primitive":{"type":"Bias","name":"bias","bound":[{"pos":0,"name":"scale","kind":"F64","value":{"F64":0.5}}]}}],"edges":[{"from":0,"to":2,"slot":0,"from_field":0},{"from":1,"to":2,"slot":1,"from_field":0},{"from":2,"to":3,"slot":0,"from_field":0}],"input_roles":[{"name":"price","targets":[{"node":0,"slot":0},{"node":1,"slot":0}],"source":"F64"}],"output":[{"node":3,"field":0,"name":"bias"}]}}"#;
|
||||
pub const BP_SMA_B: &str = r#"{"format_version":1,"blueprint":{"name":"sma_signal","nodes":[{"primitive":{"type":"SMA","name":"fast","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":3}}]}},{"primitive":{"type":"SMA","name":"slow","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":6}}]}},{"primitive":{"type":"Sub"}},{"primitive":{"type":"Bias","name":"bias","bound":[{"pos":0,"name":"scale","kind":"F64","value":{"F64":0.5}}]}}],"edges":[{"from":0,"to":2,"slot":0,"from_field":0},{"from":1,"to":2,"slot":1,"from_field":0},{"from":2,"to":3,"slot":0,"from_field":0}],"input_roles":[{"name":"price","targets":[{"node":0,"slot":0},{"node":1,"slot":0}],"source":"F64"}],"output":[{"node":3,"field":0,"name":"bias"}]}}"#;
|
||||
|
||||
fn run_in(bin: &Path, dir: &Path, args: &[&str]) -> Result<(String, Option<i32>), String> {
|
||||
let out = std::process::Command::new(bin)
|
||||
.args(args)
|
||||
.current_dir(dir)
|
||||
.output()
|
||||
.map_err(|e| format!("spawn aura {args:?}: {e}"))?;
|
||||
let text = format!(
|
||||
"{}{}",
|
||||
String::from_utf8_lossy(&out.stdout),
|
||||
String::from_utf8_lossy(&out.stderr)
|
||||
);
|
||||
Ok((text, out.status.code()))
|
||||
}
|
||||
|
||||
/// Blueprint stems currently in the scratch store.
|
||||
fn blueprint_stems(dir: &Path) -> Vec<String> {
|
||||
std::fs::read_dir(dir.join("runs").join("blueprints"))
|
||||
.map(|rd| {
|
||||
rd.filter_map(|e| e.ok())
|
||||
.filter_map(|e| e.path().file_stem().map(|s| s.to_string_lossy().into_owned()))
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
/// Seed one blueprint into the scratch store via a tiny synthetic sweep (the
|
||||
/// E2E `seed_blueprint` pattern) and return its content id.
|
||||
fn seed_blueprint(bin: &Path, dir: &Path, file: &str, name: &str) -> Result<String, String> {
|
||||
let before = blueprint_stems(dir);
|
||||
let (out, code) = run_in(
|
||||
bin,
|
||||
dir,
|
||||
&[
|
||||
"sweep",
|
||||
file,
|
||||
"--axis",
|
||||
"sma_signal.fast.length=2,4",
|
||||
"--axis",
|
||||
"sma_signal.slow.length=8,16",
|
||||
"--name",
|
||||
name,
|
||||
],
|
||||
)?;
|
||||
if code != Some(0) {
|
||||
return Err(format!("seed sweep failed ({code:?}): {out}"));
|
||||
}
|
||||
blueprint_stems(dir)
|
||||
.into_iter()
|
||||
.find(|s| !before.contains(s))
|
||||
.ok_or_else(|| "seed sweep must store a new blueprint".to_string())
|
||||
}
|
||||
|
||||
fn register_process(bin: &Path, dir: &Path, file: &str, doc: &str) -> Result<String, String> {
|
||||
std::fs::write(dir.join(file), doc).map_err(|e| format!("write {file}: {e}"))?;
|
||||
let (out, code) = run_in(bin, dir, &["process", "register", file])?;
|
||||
if code != Some(0) {
|
||||
return Err(format!("process register failed ({code:?}): {out}"));
|
||||
}
|
||||
out.lines()
|
||||
.find(|l| l.starts_with("registered process "))
|
||||
.map(|l| {
|
||||
l.trim_start_matches("registered process ")
|
||||
.split(' ')
|
||||
.next()
|
||||
.expect("split yields at least one piece")
|
||||
.trim_start_matches("content:")
|
||||
.to_string()
|
||||
})
|
||||
.ok_or_else(|| format!("no register line in: {out}"))
|
||||
}
|
||||
|
||||
fn campaign_doc(instruments: &[&str], bp_ids: &[String], proc_id: &str) -> String {
|
||||
let instruments_json =
|
||||
instruments.iter().map(|i| format!("\"{i}\"")).collect::<Vec<_>>().join(", ");
|
||||
let strategies_json = bp_ids
|
||||
.iter()
|
||||
.map(|id| {
|
||||
format!(
|
||||
r#"{{ "ref": {{ "content_id": "{id}" }},
|
||||
"axes": {{ "fast.length": {{ "kind": "I64", "values": [2, 3, 4] }},
|
||||
"slow.length": {{ "kind": "I64", "values": [8, 12, 16] }} }} }}"#
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(",\n ");
|
||||
format!(
|
||||
r#"{{
|
||||
"format_version": 1,
|
||||
"kind": "campaign",
|
||||
"name": "bench",
|
||||
"data": {{ "instruments": [{instruments_json}], "windows": [ {{ "from_ms": {from}, "to_ms": {to} }} ] }},
|
||||
"strategies": [ {strategies_json} ],
|
||||
"process": {{ "ref": {{ "content_id": "{proc_id}" }} }},
|
||||
"seed": 7,
|
||||
"presentation": {{ "persist_taps": [], "emit": [] }}
|
||||
}}"#,
|
||||
from = WINDOW_MS.0,
|
||||
to = WINDOW_MS.1,
|
||||
)
|
||||
}
|
||||
|
||||
/// Build a fresh scratch campaign project and return (scratch, campaign doc
|
||||
/// file name). Setup, unmeasured.
|
||||
fn build_scratch(bin: &Path, sizing: Sizing, process_doc: &str) -> Result<(ScratchDir, String), String> {
|
||||
let scratch = ScratchDir::new("campaign").map_err(|e| format!("scratch dir: {e}"))?;
|
||||
let root = &scratch.0;
|
||||
std::fs::write(root.join("Aura.toml"), "[paths]\nruns = \"runs\"\ndata = \"data\"\n")
|
||||
.map_err(|e| format!("write Aura.toml: {e}"))?;
|
||||
let data_dir = root.join("data");
|
||||
std::fs::create_dir_all(&data_dir).map_err(|e| format!("create data dir: {e}"))?;
|
||||
let instruments = if sizing.quick { QUICK_INSTRUMENTS } else { FULL_INSTRUMENTS };
|
||||
for sym in instruments {
|
||||
write_symbol_archive(&data_dir, sym, ARCHIVE_MONTHS.0, ARCHIVE_MONTHS.1);
|
||||
}
|
||||
std::fs::write(root.join("bench_sma_a.json"), BP_SMA_A)
|
||||
.map_err(|e| format!("write blueprint a: {e}"))?;
|
||||
let mut bp_ids = vec![seed_blueprint(bin, root, "bench_sma_a.json", "seed-a")?];
|
||||
if !sizing.quick {
|
||||
std::fs::write(root.join("bench_sma_b.json"), BP_SMA_B)
|
||||
.map_err(|e| format!("write blueprint b: {e}"))?;
|
||||
bp_ids.push(seed_blueprint(bin, root, "bench_sma_b.json", "seed-b")?);
|
||||
}
|
||||
let proc_id = register_process(bin, root, "bench.process.json", process_doc)?;
|
||||
let doc = campaign_doc(instruments, &bp_ids, &proc_id);
|
||||
std::fs::write(root.join("bench.campaign.json"), doc)
|
||||
.map_err(|e| format!("write campaign doc: {e}"))?;
|
||||
Ok((scratch, "bench.campaign.json".to_string()))
|
||||
}
|
||||
|
||||
/// Extract the winner-ordinal fingerprint from the always-on final
|
||||
/// `{"campaign_run":...}` stdout line.
|
||||
pub fn winner_fingerprint(stdout: &str) -> Result<String, String> {
|
||||
let line = stdout
|
||||
.lines()
|
||||
.find(|l| l.starts_with("{\"campaign_run\":"))
|
||||
.ok_or("no campaign_run record line on stdout")?;
|
||||
let v: serde_json::Value =
|
||||
serde_json::from_str(line).map_err(|e| format!("campaign_run line parses: {e}"))?;
|
||||
let cells = v["campaign_run"]["cells"]
|
||||
.as_array()
|
||||
.ok_or("campaign_run.cells is an array")?;
|
||||
let mut parts = Vec::new();
|
||||
for (ci, cell) in cells.iter().enumerate() {
|
||||
let stages = cell["stages"].as_array().ok_or("cell.stages is an array")?;
|
||||
for (si, stage) in stages.iter().enumerate() {
|
||||
if let Some(w) = stage["selection"]["winner_ordinal"].as_u64() {
|
||||
parts.push(format!("c{ci}s{si}w{w}"));
|
||||
}
|
||||
}
|
||||
}
|
||||
if parts.is_empty() {
|
||||
return Err("no winner ordinals in the campaign_run record".to_string());
|
||||
}
|
||||
Ok(format!("cells={} {}", cells.len(), parts.join(" ")))
|
||||
}
|
||||
|
||||
fn campaign_rep(bin: &Path, sizing: Sizing, process_doc: &str) -> Result<RepOutcome, String> {
|
||||
let (scratch, doc) = build_scratch(bin, sizing, process_doc)?;
|
||||
let timed: TimedChild = run_timed(bin, &["campaign", "run", &doc], &scratch.0)?;
|
||||
if timed.exit != Some(0) {
|
||||
return Err(format!(
|
||||
"campaign run exited {:?}\nstdout: {}\nstderr: {}",
|
||||
timed.exit, timed.stdout, timed.stderr
|
||||
));
|
||||
}
|
||||
let fingerprint = winner_fingerprint(&timed.stdout)?;
|
||||
let mut metrics = BTreeMap::new();
|
||||
metrics.insert("wall_s".to_string(), timed.wall_s);
|
||||
if let Some(c) = timed.cpu_percent {
|
||||
metrics.insert("cpu_percent".to_string(), c);
|
||||
}
|
||||
if let Some(r) = timed.peak_rss_mb {
|
||||
metrics.insert("peak_rss_mb".to_string(), r);
|
||||
}
|
||||
Ok(RepOutcome { metrics, fingerprint })
|
||||
}
|
||||
|
||||
pub fn sweep_rep(bin: &Path, sizing: Sizing) -> Result<RepOutcome, String> {
|
||||
campaign_rep(bin, sizing, SWEEP_PROCESS_DOC)
|
||||
}
|
||||
|
||||
pub fn heavy_rep(bin: &Path, sizing: Sizing) -> Result<RepOutcome, String> {
|
||||
let doc = if sizing.quick { HEAVY_PROCESS_DOC_QUICK } else { HEAVY_PROCESS_DOC };
|
||||
campaign_rep(bin, sizing, doc)
|
||||
}
|
||||
|
||||
/// Used by `cli_fixed_cost` (Task 7): a minimal one-symbol scratch project.
|
||||
pub fn build_minimal_project() -> Result<ScratchDir, String> {
|
||||
let scratch = ScratchDir::new("fixedcost").map_err(|e| format!("scratch dir: {e}"))?;
|
||||
let root = &scratch.0;
|
||||
std::fs::write(root.join("Aura.toml"), "[paths]\nruns = \"runs\"\ndata = \"data\"\n")
|
||||
.map_err(|e| format!("write Aura.toml: {e}"))?;
|
||||
let data_dir = root.join("data");
|
||||
std::fs::create_dir_all(&data_dir).map_err(|e| format!("create data dir: {e}"))?;
|
||||
write_symbol_archive(&data_dir, "BSYMA", (2024, 1), (2024, 1));
|
||||
std::fs::write(root.join("bench_sma_a.json"), BP_SMA_A)
|
||||
.map_err(|e| format!("write blueprint: {e}"))?;
|
||||
Ok(scratch)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn winner_fingerprint_extracts_ordinals_from_a_record_line() {
|
||||
let stdout = "noise\n{\"campaign_run\":{\"campaign\":\"c\",\"process\":\"p\",\"run\":0,\"seed\":7,\
|
||||
\"cells\":[{\"strategy\":\"s\",\"instrument\":\"I\",\"window_ms\":[1,2],\
|
||||
\"stages\":[{\"block\":\"std::sweep\",\"selection\":{\"winner_ordinal\":4,\"params\":[],\
|
||||
\"selection\":\"argmax\"}},{\"block\":\"std::gate\"}]}]}}\n";
|
||||
let fp = winner_fingerprint(stdout).expect("parses");
|
||||
assert_eq!(fp, "cells=1 c0s0w4");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn winner_fingerprint_rejects_missing_record_line() {
|
||||
assert!(winner_fingerprint("no record here\n").is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn campaign_doc_lists_all_instruments_and_strategies() {
|
||||
let doc = campaign_doc(
|
||||
&["BSYMA", "BSYMB"],
|
||||
&["id-a".to_string(), "id-b".to_string()],
|
||||
"proc-id",
|
||||
);
|
||||
let v: serde_json::Value = serde_json::from_str(&doc).expect("doc is valid JSON");
|
||||
assert_eq!(v["data"]["instruments"].as_array().expect("instruments").len(), 2);
|
||||
assert_eq!(v["strategies"].as_array().expect("strategies").len(), 2);
|
||||
assert_eq!(v["seed"], 7);
|
||||
assert_eq!(v["process"]["ref"]["content_id"], "proc-id");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn process_docs_are_valid_json() {
|
||||
for doc in [SWEEP_PROCESS_DOC, HEAVY_PROCESS_DOC, HEAVY_PROCESS_DOC_QUICK] {
|
||||
let v: serde_json::Value = serde_json::from_str(doc).expect("process doc parses");
|
||||
assert_eq!(v["kind"], "process");
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@
|
||||
//! than left to trip the workspace's `-D warnings` clippy gate.
|
||||
#![allow(dead_code)]
|
||||
|
||||
pub mod campaign;
|
||||
pub mod engine;
|
||||
pub mod ingest;
|
||||
|
||||
@@ -106,6 +107,37 @@ mod tests {
|
||||
assert!(measured.metrics.contains_key("wall_s"));
|
||||
}
|
||||
|
||||
/// `run_reps` composed with the real campaign sweep surface — a fresh
|
||||
/// scratch project, a freshly seeded blueprint, and a genuine
|
||||
/// `aura campaign run` child process spawned per repetition (not an
|
||||
/// in-process shortcut). Protects that two independently built-and-run
|
||||
/// scratch campaigns agree on the winner-ordinal fingerprint extracted
|
||||
/// from the child's stdout — C1's determinism carried through the
|
||||
/// child-process boundary, which is the whole reason the campaign
|
||||
/// surfaces exist as a distinct measurement path from the in-process
|
||||
/// engine/ingest surfaces above — and that the metrics map carries the
|
||||
/// wall-clock key the campaign `BaselineDoc` needs.
|
||||
///
|
||||
/// Ignored in the default suite: it builds the release `aura` binary and
|
||||
/// spawns real campaign child runs (minutes cold) — the suite-wallclock
|
||||
/// discipline forbids that cost on every `cargo test`. Run it explicitly:
|
||||
/// `cargo test -p aura-bench -- --ignored`.
|
||||
#[test]
|
||||
#[ignore = "builds the release aura binary + real campaign scratch runs; run via -- --ignored"]
|
||||
fn run_reps_drives_the_real_campaign_sweep_surface_deterministically() {
|
||||
let bin = crate::child::resolve_aura_bin(&crate::child::workspace_root())
|
||||
.expect("the release aura binary must build");
|
||||
let measured = run_reps("campaign_sweep", 2, || campaign::sweep_rep(&bin, Sizing { quick: true }))
|
||||
.expect("the real campaign sweep surface must agree with itself across independent scratch runs");
|
||||
assert_eq!(measured.surface, "campaign_sweep");
|
||||
assert!(measured.metrics.contains_key("wall_s"));
|
||||
assert!(
|
||||
measured.fingerprint.starts_with("cells="),
|
||||
"fingerprint: {}",
|
||||
measured.fingerprint
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_reps_rejects_cross_rep_fingerprint_drift() {
|
||||
let mut n = 0;
|
||||
|
||||
Reference in New Issue
Block a user