From dd23ea34a5f3ad1a6a32ff34c6a397c1cc37df06 Mon Sep 17 00:00:00 2001 From: Brummel Date: Sat, 11 Jul 2026 05:28:36 +0200 Subject: [PATCH] =?UTF-8?q?test(cli):=20e2e=20evidence=20pin=20=E2=80=94?= =?UTF-8?q?=20a=20project=20node's=20open=20param=20through=20sweep=20and?= =?UTF-8?q?=20campaign?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The defining loop of a genuine external project, proven end-to-end and green on arrival: a fresh aura-new project authors an OPEN blueprint over its own node (knob_lab::Scale, factor unbound); --list-axes shows the axis, a real-data GER40 sweep over it yields two members whose manifests carry the project-node binding, the family persists and reproduces 2/2 bit-identically, and one campaign cell over the same blueprint runs to exit 0 linking a sweep family. No implementation was needed — the loop already worked; this pin is the previously-missing evidence (the deepest exercised path before was a single param-less aura run). The axis-discovery leg is un-gated so the test stays meaningful on a data-less host. closes #235 --- .../aura-cli/tests/project_sweep_campaign.rs | 322 ++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 crates/aura-cli/tests/project_sweep_campaign.rs diff --git a/crates/aura-cli/tests/project_sweep_campaign.rs b/crates/aura-cli/tests/project_sweep_campaign.rs new file mode 100644 index 0000000..a8ad06a --- /dev/null +++ b/crates/aura-cli/tests/project_sweep_campaign.rs @@ -0,0 +1,322 @@ +//! E2E (#235): the defining loop of a genuine external project — sweep MY OWN +//! project-authored node's knob — proven end to end. +//! +//! Every prior sweep/campaign in the suite ran over a std-only blueprint (or, at +//! most, a param-less project node); no project-authored node with an OPEN param +//! had ever gone through sweep or campaign. A fresh `aura new` project ships a +//! project-authored `Scale` node with one open `factor` param (#183); this test +//! authors an OPEN blueprint over that node (factor left unbound) and drives it +//! through the two acceptance boxes of #235: +//! +//! 1. `--list-axes` discovers the project node's own axis -> a real-data sweep +//! over it -> family persisted -> `reproduce` re-derives N/N bit-identically; +//! 2. one campaign cell over the same project blueprint runs to exit 0. +//! +//! Sibling idiom: `project_new.rs` (scaffold via the built binary + cargo-build +//! the project crate in-test — the expensive-but-accepted authoring-loop idiom) +//! and `cli_run.rs`'s `local_data_present()` archive gate for the real-data legs. +//! The axis-discovery leg is NOT gated: it proves the project node's knob is +//! sweepable on every host, data or no data. + +use std::path::{Path, PathBuf}; +use std::process::{Command, Output}; +use std::sync::{Mutex, MutexGuard, OnceLock}; + +const BIN: &str = env!("CARGO_BIN_EXE_aura"); + +// GER40 Sept-2024 window (inclusive Unix-ms) — the same gated real window the +// sibling sweep/campaign e2es drive (cli_run.rs / research_docs.rs). +const GER40_FROM_MS: &str = "1725148800000"; +const GER40_TO_MS: &str = "1727740799999"; + +/// The OPEN blueprint: the scaffold's SMA-cross bias graph, but its terminal +/// node is the project's OWN `knob_lab::Scale` (named `gain`), left UNBOUND — so +/// `factor` is the one open sweep axis (`scaled_signal.gain.factor` wrapped / +/// `gain.factor` raw). `knob_lab` is the snake namespace `aura new knob-lab` +/// derives; `Scale` is the starter node every scaffold emits (#183). +const OPEN_BLUEPRINT: &str = r#"{"format_version":1,"blueprint":{"name":"scaled_signal","nodes":[{"primitive":{"type":"SMA","name":"fast","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":2}}]}},{"primitive":{"type":"SMA","name":"slow","bound":[{"pos":0,"name":"length","kind":"I64","value":{"I64":4}}]}},{"primitive":{"type":"Sub"}},{"primitive":{"type":"Bias","name":"bias","bound":[{"pos":0,"name":"scale","kind":"F64","value":{"F64":0.5}}]}},{"primitive":{"type":"knob_lab::Scale","name":"gain"}}],"edges":[{"from":0,"to":2,"slot":0,"from_field":0},{"from":1,"to":2,"slot":1,"from_field":0},{"from":2,"to":3,"slot":0,"from_field":0},{"from":3,"to":4,"slot":0,"from_field":0}],"input_roles":[{"name":"price","targets":[{"node":0,"slot":0},{"node":1,"slot":0}],"source":"F64"}],"output":[{"node":4,"field":0,"name":"bias"}]}}"#; + +/// The minimal executable process pipeline (one sweep stage) — the campaign +/// cell's methodology reference. +const SWEEP_ONLY_PROCESS: &str = r#"{"format_version":1,"kind":"process","name":"knob-sweep-only","pipeline":[{"block":"std::sweep","metric":"sqn_normalized","select":"argmax"}]}"#; + +fn local_data_present() -> bool { + Path::new(data_server::DEFAULT_DATA_PATH).is_dir() +} + +fn aura_in(dir: &Path, args: &[&str]) -> Output { + Command::new(BIN) + .args(args) + .current_dir(dir) + .output() + .expect("run aura") +} + +/// Scaffold `knob-lab` once via `aura new`, cargo-build its cdylib, and drop the +/// OPEN blueprint into `blueprints/scaled_open.json`. Returns the project dir. +/// `OnceLock` so the expensive scaffold+build happens once per test binary; the +/// project's path-deps resolve into this checkout (the baked engine root), like +/// `project_new.rs`. +fn built_scale_project() -> &'static PathBuf { + static BUILT: OnceLock = OnceLock::new(); + BUILT.get_or_init(|| { + let base = std::env::temp_dir().join(format!("aura-235-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&base); + std::fs::create_dir_all(&base).expect("create scaffold base"); + + let new = aura_in(&base, &["new", "knob-lab"]); + assert!( + new.status.success(), + "aura new knob-lab failed: {}", + String::from_utf8_lossy(&new.stderr) + ); + let proj = base.join("knob-lab"); + + let build = Command::new("cargo") + .arg("build") + .current_dir(&proj) + .output() + .expect("cargo build the scaffolded project"); + assert!( + build.status.success(), + "scaffolded project build failed:\n{}", + String::from_utf8_lossy(&build.stderr) + ); + + std::fs::write(proj.join("blueprints/scaled_open.json"), OPEN_BLUEPRINT) + .expect("write the open blueprint"); + proj + }) +} + +/// Serializes tests that share the one built project (they reset `runs/`, so +/// parallel threads would race). Poison-tolerant. +fn project_lock() -> MutexGuard<'static, ()> { + static LOCK: Mutex<()> = Mutex::new(()); + LOCK.lock().unwrap_or_else(|e| e.into_inner()) +} + +/// Removes the project's `runs/` on drop (including mid-panic) so a +/// content-addressed store never leaks between the shared tests. +struct RunsCleanup(PathBuf); +impl Drop for RunsCleanup { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.0).ok(); + } +} + +/// Enter the shared built project with a FRESH `runs/` store, serialized. Bind +/// the guard tuple for the whole test body: `let (dir, _g) = fresh_project();`. +/// `RunsCleanup` is ordered before the `MutexGuard` so the store removal runs +/// while the lock is still held. +fn fresh_project() -> (&'static PathBuf, (RunsCleanup, MutexGuard<'static, ()>)) { + let lock = project_lock(); + let dir = built_scale_project(); + let runs = dir.join("runs"); + std::fs::remove_dir_all(&runs).ok(); + (dir, (RunsCleanup(runs), lock)) +} + +/// Acceptance box 1 (#235): a project-authored node with an OPEN param is +/// discoverable as a sweep axis, sweeps over real data into a persisted family, +/// and that family reproduces bit-identically — the defining research loop of a +/// genuine external project ("sweep my OWN node's knob"), proven end to end. +#[test] +fn project_node_open_param_sweeps_and_reproduces() { + let (dir, _g) = fresh_project(); + + // (a) `--list-axes` discovers the PROJECT node's own open knob. NOT gated: + // enumerating axes needs no data, so the discoverability half runs on every + // host. The wrapped name is `..`. + let axes = aura_in(dir, &["sweep", "blueprints/scaled_open.json", "--list-axes"]); + assert!( + axes.status.success(), + "--list-axes stderr: {}", + String::from_utf8_lossy(&axes.stderr) + ); + assert_eq!( + String::from_utf8_lossy(&axes.stdout).trim(), + "scaled_signal.gain.factor:F64", + "the project node's own param is the one discoverable sweep axis; stderr: {}", + String::from_utf8_lossy(&axes.stderr) + ); + + // The real-data sweep/reproduce legs are archive-gated. + if !local_data_present() { + eprintln!("skip: no local data at {}", data_server::DEFAULT_DATA_PATH); + return; + } + + // (b) sweep the project node's knob over real GER40 data -> two members. + let sweep = aura_in( + dir, + &[ + "sweep", + "blueprints/scaled_open.json", + "--real", + "GER40", + "--from", + GER40_FROM_MS, + "--to", + GER40_TO_MS, + "--axis", + "scaled_signal.gain.factor=0.5,1.0", + "--name", + "knob", + ], + ); + assert!( + sweep.status.success(), + "sweep stderr: {}", + String::from_utf8_lossy(&sweep.stderr) + ); + let stdout = String::from_utf8_lossy(&sweep.stdout).into_owned(); + let lines: Vec<&str> = stdout.lines().collect(); + assert_eq!(lines.len(), 2, "one member line per factor point: {stdout}"); + + // Each member's manifest carries the swept PROJECT-node param binding under + // its wrapped name — the load-bearing proof that it is MY node's knob that + // varied across the family, not some incidental std axis. + for (line, factor) in lines.iter().zip([0.5_f64, 1.0]) { + let v: serde_json::Value = serde_json::from_str(line).expect("member line parses as JSON"); + assert_eq!( + v["report"]["manifest"]["instrument"].as_str(), + Some("GER40"), + "member manifest stamps the instrument: {line}" + ); + let params = v["report"]["manifest"]["params"] + .as_array() + .expect("manifest.params is an array"); + let bound = params + .iter() + .find(|p| p[0].as_str() == Some("scaled_signal.gain.factor")) + .and_then(|p| p[1]["F64"].as_f64()); + assert_eq!( + bound, + Some(factor), + "member manifest carries the swept project-node param binding: {line}" + ); + } + + // (c) reproduce the persisted family: every member re-derives bit-identically. + let family_id = serde_json::from_str::(lines[0]) + .expect("first member line parses")["family_id"] + .as_str() + .expect("member line carries a family_id") + .to_string(); + let rep = aura_in(dir, &["reproduce", &family_id]); + assert!( + rep.status.success(), + "reproduce stderr: {}", + String::from_utf8_lossy(&rep.stderr) + ); + assert!( + String::from_utf8_lossy(&rep.stdout).contains("reproduced 2/2 members bit-identically"), + "reproduce stdout: {}", + String::from_utf8_lossy(&rep.stdout) + ); +} + +/// Acceptance box 2 (#235): one campaign cell over the same project blueprint. +/// The blueprint (with its project-authored open param) registers, a minimal +/// sweep-only process registers, and a campaign referencing both by content id +/// — with its axis on the project node's own `gain.factor` — runs to exit 0 with +/// a `campaign_run` record whose single cell links a sweep family. +#[test] +fn project_node_open_param_runs_one_campaign_cell() { + let (dir, _g) = fresh_project(); + + // Register the OPEN project blueprint -> content id. + let reg = aura_in(dir, &["graph", "register", "blueprints/scaled_open.json"]); + assert!( + reg.status.success(), + "graph register stderr: {}", + String::from_utf8_lossy(®.stderr) + ); + let reg_out = String::from_utf8_lossy(®.stdout).into_owned(); + let bp_id = reg_out + .lines() + .find(|l| l.starts_with("registered blueprint ")) + .expect("register line") + .trim_start_matches("registered blueprint ") + .split(' ') + .next() + .expect("blueprint id") + .to_string(); + + // Register a minimal sweep-only process -> content id. + std::fs::write(dir.join("knob.process.json"), SWEEP_ONLY_PROCESS).expect("write process doc"); + let preg = aura_in(dir, &["process", "register", "knob.process.json"]); + assert!( + preg.status.success(), + "process register stderr: {}", + String::from_utf8_lossy(&preg.stderr) + ); + let preg_out = String::from_utf8_lossy(&preg.stdout).into_owned(); + let proc_id = preg_out + .lines() + .find(|l| l.starts_with("registered process ")) + .expect("register line") + .trim_start_matches("registered process ") + .split(' ') + .next() + .expect("process id") + .to_string(); + + // Author the campaign cell: one instrument, one window, axis on the project + // node's OWN `gain.factor` (the RAW param-space name campaign axes use). + let campaign = format!( + r#"{{ + "format_version": 1, + "kind": "campaign", + "name": "knob-cell", + "data": {{ "instruments": ["GER40"], "windows": [ {{ "from_ms": {from}, "to_ms": {to} }} ] }}, + "strategies": [ {{ "ref": {{ "content_id": "{bp}" }}, + "axes": {{ "gain.factor": {{ "kind": "F64", "values": [0.5, 1.0] }} }} }} ], + "process": {{ "ref": {{ "content_id": "{proc}" }} }}, + "seed": 7, + "presentation": {{ "persist_taps": [], "emit": ["family_table"] }} +}}"#, + from = GER40_FROM_MS, + to = GER40_TO_MS, + bp = bp_id, + proc = proc_id, + ); + std::fs::write(dir.join("knob.campaign.json"), &campaign).expect("write campaign doc"); + + let run = aura_in(dir, &["campaign", "run", "knob.campaign.json"]); + let out = format!( + "{}{}", + String::from_utf8_lossy(&run.stdout), + String::from_utf8_lossy(&run.stderr) + ); + + // Archive-gated: skip cleanly on a data-less machine (the member-data + // refusal, never a panic) — mirroring campaign_run_real_e2e_*. + if run.status.code() == Some(1) + && (out.contains("no recorded geometry") || out.contains("no data for instrument")) + { + eprintln!("skip: no local GER40 data for the campaign cell"); + return; + } + + assert_eq!(run.status.code(), Some(0), "campaign run: {out}"); + let record = out + .lines() + .find(|l| l.starts_with("{\"campaign_run\":")) + .expect("the always-on final campaign_run line"); + let v: serde_json::Value = + serde_json::from_str(record).expect("campaign_run line parses as JSON"); + let cells = v["campaign_run"]["cells"] + .as_array() + .expect("cells array"); + assert_eq!( + cells.len(), + 1, + "one (strategy, instrument, window) cell over the project blueprint: {record}" + ); + assert!( + cells[0]["stages"][0]["family_id"].as_str().is_some(), + "the sweep stage links a family: {record}" + ); +}