Brummel
aeb0366aed
campaign_run.rs: target resolution (file is register-then-run sugar; bare 64-hex is the canonical id — #198 decision 1), project gate before any store write, zero-fault referential gate, process fetch + v1 preflight, then aura_campaign::execute with the CLI MemberRunner over the shipped loaded-blueprint convention: per-member blueprint reload, reduce-mode wrap, unique suffix-join of raw campaign axis names onto the wrapped param_space, windowed real data via the shipped ms->ns source seam (absent archive/zero-bar windows -> NoData). Emission: family_table / selection_report lines gated on the doc's emit list (names debug_asserted against emit_vocabulary), the campaign_run record line always, persist_taps deferred LOUDLY on stderr before execution (F7 lesson), zero-survivor cells noted on stderr with exit 0. exec_fault_prose/member_fault_prose keep the Debug-free house seam. Seam tests: 8 campaign_run_* e2e tests incl. outside-project, bogus target, unknown id, v1-boundary (mc process registers fine, run refuses), persist_taps ordering, and the gated real-data sweep->gate->walkforward e2e — which ran its full assert path on this host (local GER40 2024-09 archive) rather than the data-less skip. Also: exec.rs campaign-id guard tightened to lowercase hex, matching is_content_id and the store's self-keyed form (review nit). Gates: workspace 975/0, clippy -D warnings clean. refs #198
875 lines
38 KiB
Rust
875 lines
38 KiB
Rust
//! End-to-end coverage for the `aura process` verb family (#188/#189): drives
|
|
//! the built `aura` binary over argv/file, the exact surface a data-level
|
|
//! author uses.
|
|
|
|
use std::path::{Path, PathBuf};
|
|
use std::sync::{Mutex, MutexGuard, OnceLock};
|
|
|
|
/// A fresh, unique working directory for a process test that persists
|
|
/// content-addressed documents under `./runs/` (so `aura process register`
|
|
/// never dirties the repo). Unique per test + per process; no external
|
|
/// tempfile dependency (mirrors `cli_run.rs` / `cli_broken_pipe.rs`).
|
|
fn temp_cwd(name: &str) -> PathBuf {
|
|
let dir = std::env::temp_dir().join(format!("aura-cli-{}-{}", std::process::id(), name));
|
|
let _ = std::fs::remove_dir_all(&dir);
|
|
std::fs::create_dir_all(&dir).expect("create temp cwd");
|
|
dir
|
|
}
|
|
|
|
fn run_code_in(dir: &Path, args: &[&str]) -> (String, Option<i32>) {
|
|
let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura"))
|
|
.args(args)
|
|
.current_dir(dir)
|
|
.output()
|
|
.expect("binary runs");
|
|
let text = format!(
|
|
"{}{}",
|
|
String::from_utf8_lossy(&out.stdout),
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
(text, out.status.code())
|
|
}
|
|
|
|
fn run_code(args: &[&str]) -> (String, Option<i32>) {
|
|
run_code_in(Path::new("."), args)
|
|
}
|
|
|
|
const PROCESS_DOC: &str = r#"{
|
|
"format_version": 1,
|
|
"kind": "process",
|
|
"name": "wf-deflated-screen",
|
|
"pipeline": [
|
|
{ "block": "std::sweep", "metric": "net_expectancy_r", "select": "plateau:worst", "deflate": true },
|
|
{ "block": "std::gate", "all": [
|
|
{ "metric": "net_expectancy_r", "cmp": "gt", "value": 0.0 },
|
|
{ "metric": "overfit_probability", "cmp": "lt", "value": 0.1 } ] },
|
|
{ "block": "std::walk_forward", "in_sample_ms": 4000, "out_of_sample_ms": 1000,
|
|
"step_ms": 1000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" }
|
|
]
|
|
}"#;
|
|
|
|
fn write_doc(dir: &Path, name: &str, text: &str) -> PathBuf {
|
|
let p = dir.join(name);
|
|
std::fs::write(&p, text).expect("write doc");
|
|
p
|
|
}
|
|
|
|
#[test]
|
|
fn process_validate_reports_intrinsic_ok() {
|
|
let dir = temp_cwd("process-validate-ok");
|
|
write_doc(&dir, "p.process.json", PROCESS_DOC);
|
|
let (out, code) = run_code_in(&dir, &["process", "validate", "p.process.json"]);
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
assert!(out.contains("process document valid (intrinsic): 3 pipeline blocks, 2 gate predicates"));
|
|
}
|
|
|
|
#[test]
|
|
fn process_validate_refuses_unknown_metric_as_prose_exit_1() {
|
|
let dir = temp_cwd("process-validate-bad-metric");
|
|
let bad = PROCESS_DOC.replacen(
|
|
"net_expectancy_r\", \"select\": \"plateau:worst",
|
|
"netto_r\", \"select\": \"plateau:worst",
|
|
1,
|
|
);
|
|
write_doc(&dir, "bad.process.json", &bad);
|
|
let (out, code) = run_code_in(&dir, &["process", "validate", "bad.process.json"]);
|
|
assert_eq!(code, Some(1));
|
|
assert!(out.contains("unknown metric \"netto_r\""));
|
|
assert!(!out.contains("UnknownMetric"), "Debug leak: {out}");
|
|
}
|
|
|
|
#[test]
|
|
fn process_validate_refuses_zero_walk_forward_length_as_prose_exit_1() {
|
|
let dir = temp_cwd("process-validate-zero-wf");
|
|
let bad = PROCESS_DOC.replacen("\"step_ms\": 1000", "\"step_ms\": 0", 1);
|
|
write_doc(&dir, "bad.process.json", &bad);
|
|
let (out, code) = run_code_in(&dir, &["process", "validate", "bad.process.json"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(out.contains("pipeline[2]: walk_forward step_ms must be > 0"), "stdout/stderr: {out}");
|
|
assert!(!out.contains("ZeroWalkForwardLength"), "Debug leak: {out}");
|
|
}
|
|
|
|
/// A document authored against the vocabulary `std::walk_forward` carried
|
|
/// before cycle 0107 (`folds`/`in_sample_bars`/`out_of_sample_bars`, none of
|
|
/// which the engine's `WindowRoller` accepts) is refused end-to-end through
|
|
/// the CLI's malformed-document path (a parse failure, distinct from the
|
|
/// validate-fault path exercised above) with prose naming the offending
|
|
/// slot — never a Debug-formatted `DocError`, never a silent accept of the
|
|
/// retired shape.
|
|
#[test]
|
|
fn process_validate_refuses_retired_walk_forward_vocabulary_as_prose_exit_1() {
|
|
let dir = temp_cwd("process-validate-retired-wf-vocab");
|
|
let bad = PROCESS_DOC.replacen(
|
|
"{ \"block\": \"std::walk_forward\", \"in_sample_ms\": 4000, \"out_of_sample_ms\": 1000,\n \"step_ms\": 1000, \"mode\": \"rolling\", \"metric\": \"net_expectancy_r\", \"select\": \"argmax\" }",
|
|
"{ \"block\": \"std::walk_forward\", \"folds\": 4, \"in_sample_bars\": 4000, \"out_of_sample_bars\": 1000, \"metric\": \"net_expectancy_r\", \"select\": \"argmax\" }",
|
|
1,
|
|
);
|
|
assert_ne!(bad, PROCESS_DOC, "replacen must actually match the fixture's walk_forward stage");
|
|
write_doc(&dir, "bad.process.json", &bad);
|
|
let (out, code) = run_code_in(&dir, &["process", "validate", "bad.process.json"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(out.contains("unknown slot \"folds\""), "stdout/stderr: {out}");
|
|
assert!(!out.contains("Malformed"), "Debug leak: {out}");
|
|
}
|
|
|
|
/// The roller's second `RollMode` — `anchored` (growing, origin-fixed
|
|
/// in-sample) — round-trips through the CLI's actual validate path, not
|
|
/// just the `slot_kind_label` description string: a data author who writes
|
|
/// `"mode": "anchored"` gets a valid document, exercising the `WfMode`
|
|
/// serde oracle's non-default variant end to end.
|
|
#[test]
|
|
fn process_validate_accepts_anchored_walk_forward_mode() {
|
|
let dir = temp_cwd("process-validate-anchored-wf-mode");
|
|
let anchored = PROCESS_DOC.replacen("\"mode\": \"rolling\"", "\"mode\": \"anchored\"", 1);
|
|
assert_ne!(anchored, PROCESS_DOC, "replacen must actually match the fixture's mode field");
|
|
write_doc(&dir, "anchored.process.json", &anchored);
|
|
let (out, code) = run_code_in(&dir, &["process", "validate", "anchored.process.json"]);
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
assert!(out.contains("process document valid (intrinsic): 3 pipeline blocks, 2 gate predicates"));
|
|
}
|
|
|
|
#[test]
|
|
fn process_introspect_vocabulary_block_and_content_id() {
|
|
let (out, code) = run_code(&["process", "introspect", "--vocabulary"]);
|
|
assert_eq!(code, Some(0));
|
|
for id in ["std::sweep", "std::gate", "std::walk_forward", "std::monte_carlo", "std::generalize"] {
|
|
assert!(out.contains(id), "vocabulary misses {id}: {out}");
|
|
}
|
|
let (out, code) = run_code(&["process", "introspect", "--block", "std::sweep"]);
|
|
assert_eq!(code, Some(0));
|
|
assert!(out.contains("metric"));
|
|
assert!(out.contains("required"));
|
|
|
|
let (out, code) = run_code(&["process", "introspect", "--block", "std::walk_forward"]);
|
|
assert_eq!(code, Some(0));
|
|
for slot in ["in_sample_ms", "out_of_sample_ms", "step_ms", "mode"] {
|
|
assert!(out.contains(slot), "walk_forward describe misses {slot}: {out}");
|
|
}
|
|
assert!(out.contains("one of: rolling | anchored"), "mode label missing: {out}");
|
|
assert!(!out.contains("folds"), "retired slot still advertised: {out}");
|
|
|
|
let dir = temp_cwd("process-introspect-content-id");
|
|
write_doc(&dir, "p.process.json", PROCESS_DOC);
|
|
let (out, code) = run_code_in(&dir, &["process", "introspect", "--content-id", "p.process.json"]);
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
let line = out.lines().find(|l| l.starts_with("content:")).expect("id line");
|
|
assert_eq!(line.len(), "content:".len() + 64);
|
|
}
|
|
|
|
#[test]
|
|
fn process_introspect_unknown_block_is_prose_exit_1() {
|
|
let (out, code) = run_code(&["process", "introspect", "--block", "std::nope"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(out.contains("unknown block \"std::nope\""), "stdout/stderr: {out}");
|
|
}
|
|
|
|
#[test]
|
|
fn process_introspect_unwired_lists_open_slots() {
|
|
let dir = temp_cwd("process-introspect-unwired");
|
|
let partial = r#"{ "format_version": 1, "kind": "process",
|
|
"pipeline": [ { "block": "std::sweep", "metric": "sqn" } ] }"#;
|
|
write_doc(&dir, "partial.process.json", partial);
|
|
let (out, code) = run_code_in(&dir, &["process", "introspect", "--unwired", "partial.process.json"]);
|
|
assert_eq!(code, Some(0));
|
|
assert!(out.contains("open slot: name"));
|
|
assert!(out.contains("open slot: pipeline[0].select"));
|
|
}
|
|
|
|
#[test]
|
|
fn process_introspect_no_flag_is_usage_exit_2() {
|
|
let (_out, code) = run_code(&["process", "introspect"]);
|
|
assert_eq!(code, Some(2));
|
|
}
|
|
|
|
#[test]
|
|
fn process_register_stores_content_addressed_under_runs_root() {
|
|
let dir = temp_cwd("process-register");
|
|
write_doc(&dir, "p.process.json", PROCESS_DOC);
|
|
let (out, code) = run_code_in(&dir, &["process", "register", "p.process.json"]);
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
let line = out.lines().find(|l| l.starts_with("registered process content:")).expect("line");
|
|
let id = line
|
|
.trim_start_matches("registered process content:")
|
|
.split(' ')
|
|
.next()
|
|
.expect("id");
|
|
assert!(dir.join("runs").join("processes").join(format!("{id}.json")).is_file());
|
|
}
|
|
|
|
/// The demo-project fixture (Aura.toml present), built once — mirrors
|
|
/// `project_load.rs`'s `built_fixture` (a separate test binary, so it needs
|
|
/// its own `OnceLock`, but `cargo build` is idempotent).
|
|
fn built_project() -> &'static PathBuf {
|
|
static BUILT: OnceLock<PathBuf> = OnceLock::new();
|
|
BUILT.get_or_init(|| {
|
|
let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/demo-project");
|
|
let out = std::process::Command::new("cargo")
|
|
.arg("build")
|
|
.current_dir(&dir)
|
|
.output()
|
|
.expect("spawn cargo build for the fixture project");
|
|
assert!(
|
|
out.status.success(),
|
|
"fixture build failed:\n{}",
|
|
String::from_utf8_lossy(&out.stderr)
|
|
);
|
|
dir
|
|
})
|
|
}
|
|
|
|
/// Serializes every test that touches the shared demo-project fixture store
|
|
/// (they remove/re-seed `<fixture>/runs`, so parallel test threads would race
|
|
/// on it). A poisoned lock is taken over: one failed test must not cascade
|
|
/// into unrelated lock panics.
|
|
fn project_lock() -> MutexGuard<'static, ()> {
|
|
static LOCK: Mutex<()> = Mutex::new(());
|
|
LOCK.lock().unwrap_or_else(|e| e.into_inner())
|
|
}
|
|
|
|
/// A scratch filesystem entry this test writes under the git-tracked
|
|
/// demo-project fixture root (only `runs/` there is fixture-gitignored),
|
|
/// removed on drop — including during a mid-test panic — so a failed
|
|
/// assertion never leaks scratch docs into tracked working-tree state.
|
|
enum ScratchPath {
|
|
File(PathBuf),
|
|
Dir(PathBuf),
|
|
}
|
|
|
|
struct ScratchGuard(Vec<ScratchPath>);
|
|
|
|
impl Drop for ScratchGuard {
|
|
fn drop(&mut self) {
|
|
for p in &self.0 {
|
|
match p {
|
|
ScratchPath::File(p) => {
|
|
let _ = std::fs::remove_file(p);
|
|
}
|
|
ScratchPath::Dir(p) => {
|
|
let _ = std::fs::remove_dir_all(p);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
/// The `aura campaign validate` referential tier (the campaign family's
|
|
/// headline addition over the intrinsic-only process family), exercised at
|
|
/// the CLI seam inside a real project: a fully-resolved document prints the
|
|
/// "valid (referential)" success line, and an unresolved reference prints
|
|
/// the "do not resolve" fault block through `ref_fault_prose`. The
|
|
/// constituents (`validate_campaign_refs`, `ref_fault_prose`) are unit-
|
|
/// tested elsewhere; this pins that `aura campaign validate` actually wires
|
|
/// them together end to end.
|
|
#[test]
|
|
fn campaign_validate_in_project_reports_referential_tier_end_to_end() {
|
|
let _fixture = project_lock();
|
|
let dir = built_project();
|
|
let runs_dir = dir.join("runs");
|
|
std::fs::remove_dir_all(&runs_dir).ok();
|
|
let _cleanup = ScratchGuard(vec![
|
|
ScratchPath::Dir(runs_dir.clone()),
|
|
ScratchPath::File(dir.join("seed.process.json")),
|
|
ScratchPath::File(dir.join("ref-ok.campaign.json")),
|
|
ScratchPath::File(dir.join("ref-bad.campaign.json")),
|
|
]);
|
|
|
|
// Seed one real, content-addressed, open-param blueprint into the
|
|
// project's own store via a real sweep (mirrors `project_load.rs`'s
|
|
// anchor test).
|
|
let open_bp = format!("{}/tests/fixtures/sma_signal_open.json", env!("CARGO_MANIFEST_DIR"));
|
|
let (sweep_out, sweep_code) = run_code_in(
|
|
dir,
|
|
&[
|
|
"sweep",
|
|
&open_bp,
|
|
"--axis",
|
|
"sma_signal.fast.length=2,4",
|
|
"--axis",
|
|
"sma_signal.slow.length=8,16",
|
|
"--name",
|
|
"campaign-ref-seed",
|
|
],
|
|
);
|
|
assert_eq!(sweep_code, Some(0), "seed sweep failed: {sweep_out}");
|
|
let bp_id = std::fs::read_dir(runs_dir.join("blueprints"))
|
|
.expect("blueprints dir")
|
|
.next()
|
|
.expect("one stored blueprint")
|
|
.expect("dir entry")
|
|
.path()
|
|
.file_stem()
|
|
.expect("stem")
|
|
.to_string_lossy()
|
|
.into_owned();
|
|
|
|
// Register a valid process into the same project store.
|
|
write_doc(dir, "seed.process.json", PROCESS_DOC);
|
|
let (proc_out, proc_code) = run_code_in(dir, &["process", "register", "seed.process.json"]);
|
|
assert_eq!(proc_code, Some(0), "seed process register failed: {proc_out}");
|
|
let proc_id = proc_out
|
|
.lines()
|
|
.find(|l| l.starts_with("registered process content:"))
|
|
.expect("register line")
|
|
.trim_start_matches("registered process content:")
|
|
.split(' ')
|
|
.next()
|
|
.expect("id")
|
|
.to_string();
|
|
|
|
// "fast.length": the axis name is the RAW composite's `param_space` name.
|
|
// `validate_campaign_refs` loads the stored blueprint bare, unlike the
|
|
// sweep's `wrap_r`-wrapped axis probe, so it does NOT carry the
|
|
// "sma_signal." prefix `aura sweep --axis` binds against.
|
|
let campaign = format!(
|
|
r#"{{
|
|
"format_version": 1,
|
|
"kind": "campaign",
|
|
"name": "ref-check",
|
|
"data": {{ "instruments": ["GER40"], "windows": [ {{ "from_ms": 1, "to_ms": 2 }} ] }},
|
|
"strategies": [ {{ "ref": {{ "content_id": "{bp}" }},
|
|
"axes": {{ "fast.length": {{ "kind": "I64", "values": [2] }} }} }} ],
|
|
"process": {{ "ref": {{ "content_id": "{proc}" }} }},
|
|
"seed": 1,
|
|
"presentation": {{ "persist_taps": [], "emit": ["family_table"] }}
|
|
}}"#,
|
|
bp = bp_id,
|
|
proc = proc_id
|
|
);
|
|
write_doc(dir, "ref-ok.campaign.json", &campaign);
|
|
let (out, code) = run_code_in(dir, &["campaign", "validate", "ref-ok.campaign.json"]);
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
assert!(out.contains(
|
|
"campaign document valid (referential): all references resolve, axes are in the param space"
|
|
));
|
|
|
|
// An unresolved process reference: the fault block + `ref_fault_prose`
|
|
// mapping, prose exit 1, at the same CLI seam.
|
|
let bad = campaign.replace(&proc_id, "0000000000000000000000000000000000000000000000000000000000000000");
|
|
write_doc(dir, "ref-bad.campaign.json", &bad);
|
|
let (out2, code2) = run_code_in(dir, &["campaign", "validate", "ref-bad.campaign.json"]);
|
|
assert_eq!(code2, Some(1));
|
|
assert!(out2.contains("campaign references do not resolve:"), "stdout/stderr: {out2}");
|
|
assert!(out2.contains("not found in the project store"), "stdout/stderr: {out2}");
|
|
|
|
// `_cleanup` (a `ScratchGuard`) removes the scratch docs and `runs/` on
|
|
// drop, at the end of this scope — including on a mid-test panic.
|
|
}
|
|
|
|
const CAMPAIGN_DOC: &str = r#"{
|
|
"format_version": 1,
|
|
"kind": "campaign",
|
|
"name": "screen",
|
|
"data": { "instruments": ["GER40"], "windows": [ { "from_ms": 1, "to_ms": 2 } ] },
|
|
"strategies": [ { "ref": { "content_id": "9f3a" },
|
|
"axes": { "fast": { "kind": "I64", "values": [8] } } } ],
|
|
"process": { "ref": { "content_id": "4e2d" } },
|
|
"seed": 1,
|
|
"presentation": { "persist_taps": [], "emit": ["family_table"] }
|
|
}"#;
|
|
|
|
#[test]
|
|
fn campaign_validate_outside_project_skips_referential_tier() {
|
|
let dir = temp_cwd("campaign-validate-outside-project");
|
|
write_doc(&dir, "c.campaign.json", CAMPAIGN_DOC);
|
|
let (out, code) = run_code_in(&dir, &["campaign", "validate", "c.campaign.json"]);
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
assert!(out.contains("campaign document valid (intrinsic):"));
|
|
assert!(out.contains("referential checks skipped (no Aura.toml found up from "));
|
|
}
|
|
|
|
#[test]
|
|
fn campaign_validate_refuses_empty_axis_prose_exit_1() {
|
|
let dir = temp_cwd("campaign-validate-empty-axis");
|
|
let bad = CAMPAIGN_DOC.replacen(r#""values": [8]"#, r#""values": []"#, 1);
|
|
write_doc(&dir, "bad.campaign.json", &bad);
|
|
let (out, code) = run_code_in(&dir, &["campaign", "validate", "bad.campaign.json"]);
|
|
assert_eq!(code, Some(1));
|
|
assert!(out.contains("axes.fast: an axis is a non-empty finite set"));
|
|
assert!(!out.contains("EmptyAxis"), "Debug leak: {out}");
|
|
}
|
|
|
|
#[test]
|
|
fn campaign_introspect_vocabulary_lists_sections() {
|
|
let (out, code) = run_code(&["campaign", "introspect", "--vocabulary"]);
|
|
assert_eq!(code, Some(0));
|
|
for id in ["std::data", "std::strategy", "std::process_ref", "std::presentation"] {
|
|
assert!(out.contains(id), "vocabulary misses {id}: {out}");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn campaign_introspect_unwired_reports_the_spec_example_slots() {
|
|
let dir = temp_cwd("campaign-introspect-unwired");
|
|
let draft = r#"{ "format_version": 1, "kind": "campaign", "name": "draft",
|
|
"data": { "instruments": ["GER40"], "windows": [ { "from_ms": 1, "to_ms": 2 } ] },
|
|
"strategies": [ { "ref": { "content_id": "9f3a" },
|
|
"axes": { "slow": { "kind": "I64", "values": [] } } } ],
|
|
"seed": 1,
|
|
"presentation": { "persist_taps": [], "emit": [] } }"#;
|
|
write_doc(&dir, "draft.campaign.json", draft);
|
|
let (out, code) = run_code_in(&dir, &["campaign", "introspect", "--unwired", "draft.campaign.json"]);
|
|
assert_eq!(code, Some(0));
|
|
assert!(out.contains("open slot: process.ref (required, content id of a process document)"));
|
|
assert!(out.contains("open slot: strategies[0].axes.slow (axis declared empty — a P1 axis is a non-empty finite set)"));
|
|
}
|
|
|
|
/// The campaign twin of `process_introspect_vocabulary_block_and_content_id`'s
|
|
/// `--content-id` branch: pins that `campaign introspect --content-id` wires
|
|
/// `parse_valid_campaign` + `campaign_to_json` + `content_id_of` end to end
|
|
/// (the process branch was covered; this one was not).
|
|
#[test]
|
|
fn campaign_introspect_content_id_prints_hash() {
|
|
let dir = temp_cwd("campaign-introspect-content-id");
|
|
write_doc(&dir, "c.campaign.json", CAMPAIGN_DOC);
|
|
let (out, code) = run_code_in(&dir, &["campaign", "introspect", "--content-id", "c.campaign.json"]);
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
let line = out.lines().find(|l| l.starts_with("content:")).expect("id line");
|
|
assert_eq!(line.len(), "content:".len() + 64);
|
|
}
|
|
|
|
#[test]
|
|
fn campaign_register_stores_content_addressed_under_runs_root() {
|
|
let dir = temp_cwd("campaign-register");
|
|
write_doc(&dir, "c.campaign.json", CAMPAIGN_DOC);
|
|
let (out, code) = run_code_in(&dir, &["campaign", "register", "c.campaign.json"]);
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
let line = out.lines().find(|l| l.starts_with("registered campaign content:")).expect("line");
|
|
let id = line
|
|
.trim_start_matches("registered campaign content:")
|
|
.split(' ')
|
|
.next()
|
|
.expect("id");
|
|
assert!(dir.join("runs").join("campaigns").join(format!("{id}.json")).is_file());
|
|
}
|
|
|
|
/// `guard_one_mode` is shared, family-parameterized code (`process` and
|
|
/// `campaign` both route through it). The process family already pins the
|
|
/// zero-flags arm; this pins that the campaign family gets its own family
|
|
/// name threaded into the usage line, not a copy-pasted "process" string.
|
|
#[test]
|
|
fn campaign_introspect_no_flag_usage_names_the_campaign_family() {
|
|
let (out, code) = run_code(&["campaign", "introspect"]);
|
|
assert_eq!(code, Some(2), "stdout/stderr: {out}");
|
|
assert!(
|
|
out.contains("Usage: aura campaign introspect --vocabulary | --block <ID> | --unwired <FILE> | --content-id <FILE>"),
|
|
"stdout/stderr: {out}"
|
|
);
|
|
}
|
|
|
|
/// `guard_one_mode` requires *exactly* one mode: the zero-flags arm is
|
|
/// pinned elsewhere, but nothing previously exercised the over-selection
|
|
/// arm (`selected_modes() != 1` is not just `== 0`). Two modes at once must
|
|
/// refuse with the same usage-exit-2 idiom, not silently pick one.
|
|
#[test]
|
|
fn campaign_introspect_two_flags_is_usage_exit_2() {
|
|
let (out, code) = run_code(&["campaign", "introspect", "--vocabulary", "--block", "std::data"]);
|
|
assert_eq!(code, Some(2), "stdout/stderr: {out}");
|
|
assert!(out.contains("Usage: aura campaign introspect"), "stdout/stderr: {out}");
|
|
}
|
|
|
|
/// Register must be a gate, not a passthrough: an invalid document is
|
|
/// refused with prose (exit 1) and — the property that matters for a
|
|
/// content-addressed store — no file is ever written under `runs/campaigns/`
|
|
/// for it. A store that could contain unvalidated documents would make
|
|
/// every downstream reader re-derive the checks register was supposed to
|
|
/// have already made.
|
|
#[test]
|
|
fn campaign_register_refuses_invalid_document_and_writes_nothing() {
|
|
let dir = temp_cwd("campaign-register-invalid");
|
|
let bad = CAMPAIGN_DOC.replacen(r#""values": [8]"#, r#""values": []"#, 1);
|
|
write_doc(&dir, "bad.campaign.json", &bad);
|
|
let (out, code) = run_code_in(&dir, &["campaign", "register", "bad.campaign.json"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(out.contains("refusing to register:"), "stdout/stderr: {out}");
|
|
assert!(out.contains("axes.fast: an axis is a non-empty finite set"), "stdout/stderr: {out}");
|
|
assert!(
|
|
!dir.join("runs").join("campaigns").exists(),
|
|
"register must not create a store entry for an invalid document"
|
|
);
|
|
}
|
|
|
|
/// Seed one open-param blueprint into the built demo project's store via a
|
|
/// real sweep and return its content id (the referential test's recipe).
|
|
fn seed_blueprint(dir: &Path, name: &str) -> String {
|
|
let open_bp = format!("{}/tests/fixtures/sma_signal_open.json", env!("CARGO_MANIFEST_DIR"));
|
|
let (out, code) = run_code_in(
|
|
dir,
|
|
&[
|
|
"sweep",
|
|
&open_bp,
|
|
"--axis",
|
|
"sma_signal.fast.length=2,4",
|
|
"--axis",
|
|
"sma_signal.slow.length=8,16",
|
|
"--name",
|
|
name,
|
|
],
|
|
);
|
|
assert_eq!(code, Some(0), "seed sweep failed: {out}");
|
|
std::fs::read_dir(dir.join("runs").join("blueprints"))
|
|
.expect("blueprints dir")
|
|
.next()
|
|
.expect("one stored blueprint")
|
|
.expect("dir entry")
|
|
.path()
|
|
.file_stem()
|
|
.expect("stem")
|
|
.to_string_lossy()
|
|
.into_owned()
|
|
}
|
|
|
|
/// Register `doc` as a process document in the project store; returns its id.
|
|
/// Asserts register exits 0 — an intrinsically valid document (an mc-bearing
|
|
/// one included) must always register; only `campaign run` draws the v1 line.
|
|
fn register_process_doc(dir: &Path, file: &str, doc: &str) -> String {
|
|
write_doc(dir, file, doc);
|
|
let (out, code) = run_code_in(dir, &["process", "register", file]);
|
|
assert_eq!(code, Some(0), "process register failed: {out}");
|
|
out.lines()
|
|
.find(|l| l.starts_with("registered process content:"))
|
|
.expect("register line")
|
|
.trim_start_matches("registered process content:")
|
|
.split(' ')
|
|
.next()
|
|
.expect("id")
|
|
.to_string()
|
|
}
|
|
|
|
/// A referentially-resolving campaign over the seeded blueprint. Axes name
|
|
/// the RAW `param_space` names (`fast.length` / `slow.length` — see the
|
|
/// naming note in the referential test above). `persist_taps`/`emit` are
|
|
/// spliced verbatim (pass `""` for empty, `"\"family_table\""` etc.).
|
|
fn campaign_doc_json(
|
|
bp_id: &str,
|
|
proc_id: &str,
|
|
window: (i64, i64),
|
|
persist_taps: &str,
|
|
emit: &str,
|
|
) -> String {
|
|
format!(
|
|
r#"{{
|
|
"format_version": 1,
|
|
"kind": "campaign",
|
|
"name": "run-seam",
|
|
"data": {{ "instruments": ["GER40"], "windows": [ {{ "from_ms": {from}, "to_ms": {to} }} ] }},
|
|
"strategies": [ {{ "ref": {{ "content_id": "{bp_id}" }},
|
|
"axes": {{ "fast.length": {{ "kind": "I64", "values": [2, 4] }},
|
|
"slow.length": {{ "kind": "I64", "values": [8, 16] }} }} }} ],
|
|
"process": {{ "ref": {{ "content_id": "{proc_id}" }} }},
|
|
"seed": 7,
|
|
"presentation": {{ "persist_taps": [{persist_taps}], "emit": [{emit}] }}
|
|
}}"#,
|
|
from = window.0,
|
|
to = window.1,
|
|
)
|
|
}
|
|
|
|
/// An mc-bearing process: intrinsically VALID (register accepts it) but past
|
|
/// the executable v1 boundary (`campaign run` refuses it at preflight).
|
|
const MC_PROCESS_DOC: &str = r#"{
|
|
"format_version": 1,
|
|
"kind": "process",
|
|
"name": "mc-screen",
|
|
"pipeline": [
|
|
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
|
|
{ "block": "std::monte_carlo", "resamples": 100, "block_len": 5 }
|
|
]
|
|
}"#;
|
|
|
|
/// The minimal executable v1 pipeline (one sweep stage).
|
|
const SWEEP_ONLY_PROCESS_DOC: &str = r#"{
|
|
"format_version": 1,
|
|
"kind": "process",
|
|
"name": "sweep-only",
|
|
"pipeline": [ { "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" } ]
|
|
}"#;
|
|
|
|
/// The full v1 shape for the gated e2e: sweep -> gate -> walk_forward. The
|
|
/// gate (`n_trades ge 0`) passes every member, so walk-forward always has
|
|
/// survivors. Roller: 14d IS / 7d OOS / 7d step in epoch-ms, tiling the
|
|
/// ~30-day GER40 Sept-2024 campaign window.
|
|
const WF_PROCESS_DOC: &str = r#"{
|
|
"format_version": 1,
|
|
"kind": "process",
|
|
"name": "screen-then-walkforward",
|
|
"pipeline": [
|
|
{ "block": "std::sweep", "metric": "sqn_normalized", "select": "argmax" },
|
|
{ "block": "std::gate", "all": [ { "metric": "n_trades", "cmp": "ge", "value": 0.0 } ] },
|
|
{ "block": "std::walk_forward", "in_sample_ms": 1209600000, "out_of_sample_ms": 604800000,
|
|
"step_ms": 604800000, "mode": "rolling", "metric": "net_expectancy_r", "select": "argmax" }
|
|
]
|
|
}"#;
|
|
|
|
/// `campaign run` outside a project refuses up front — before target
|
|
/// resolution, so not even the file-sugar registration touches a store.
|
|
#[test]
|
|
fn campaign_run_outside_project_refuses() {
|
|
let dir = temp_cwd("campaign-run-outside-project");
|
|
write_doc(&dir, "c.campaign.json", CAMPAIGN_DOC);
|
|
let (out, code) = run_code_in(&dir, &["campaign", "run", "c.campaign.json"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(out.contains("campaign run needs a project"), "stdout/stderr: {out}");
|
|
assert!(
|
|
!dir.join("runs").exists(),
|
|
"a refused run must not create a store outside a project"
|
|
);
|
|
}
|
|
|
|
/// A target that is neither a readable file nor a 64-hex id refuses naming
|
|
/// both readings (inside the project, past the project gate).
|
|
#[test]
|
|
fn campaign_run_bogus_target_refuses() {
|
|
let _fixture = project_lock();
|
|
let dir = built_project();
|
|
let (out, code) = run_code_in(dir, &["campaign", "run", "no-such-target"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(
|
|
out.contains("'no-such-target' is neither a readable .json file nor a 64-hex content id"),
|
|
"stdout/stderr: {out}"
|
|
);
|
|
}
|
|
|
|
/// A well-formed but unknown content id refuses with the store-miss prose.
|
|
#[test]
|
|
fn campaign_run_unknown_id_refuses() {
|
|
let _fixture = project_lock();
|
|
let dir = built_project();
|
|
let id = "0".repeat(64);
|
|
let (out, code) = run_code_in(dir, &["campaign", "run", &id]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(
|
|
out.contains(&format!("no campaign {id} in the project store")),
|
|
"stdout/stderr: {out}"
|
|
);
|
|
}
|
|
|
|
/// The v1 boundary: `process register` ACCEPTS an mc-bearing document (it is
|
|
/// intrinsically valid — asserted inside `register_process_doc`); only
|
|
/// `campaign run` refuses it, at preflight, before any member runs (so no
|
|
/// data is needed), with path-addressed Debug-free prose.
|
|
#[test]
|
|
fn campaign_run_v1_boundary_refuses_mc_process() {
|
|
let _fixture = project_lock();
|
|
let dir = built_project();
|
|
let runs_dir = dir.join("runs");
|
|
std::fs::remove_dir_all(&runs_dir).ok();
|
|
let _cleanup = ScratchGuard(vec![
|
|
ScratchPath::Dir(runs_dir.clone()),
|
|
ScratchPath::File(dir.join("mc.process.json")),
|
|
ScratchPath::File(dir.join("mc.campaign.json")),
|
|
]);
|
|
let bp_id = seed_blueprint(dir, "campaign-run-mc-seed");
|
|
let proc_id = register_process_doc(dir, "mc.process.json", MC_PROCESS_DOC);
|
|
write_doc(dir, "mc.campaign.json", &campaign_doc_json(&bp_id, &proc_id, (1, 2), "", ""));
|
|
let (out, code) = run_code_in(dir, &["campaign", "run", "mc.campaign.json"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(out.contains("not executable in v1"), "stdout/stderr: {out}");
|
|
assert!(out.contains("std::monte_carlo"), "the prose names the block: {out}");
|
|
assert!(!out.contains("UnsupportedStage"), "Debug leak: {out}");
|
|
}
|
|
|
|
/// Non-empty persist_taps defers LOUDLY, and the note's ORDER is pinned: it
|
|
/// prints before member execution, so it is asserted here on a run
|
|
/// that refuses at the member-data seam. The [1, 2] epoch-ms window (1970)
|
|
/// makes that refusal deterministic on every machine: a data-less host
|
|
/// refuses on missing geometry, a data-ful host on a window no archive file
|
|
/// overlaps — exit 1 either way, tap note already on stderr.
|
|
#[test]
|
|
fn campaign_run_persist_taps_deferred_loudly() {
|
|
let _fixture = project_lock();
|
|
let dir = built_project();
|
|
let runs_dir = dir.join("runs");
|
|
std::fs::remove_dir_all(&runs_dir).ok();
|
|
let _cleanup = ScratchGuard(vec![
|
|
ScratchPath::Dir(runs_dir.clone()),
|
|
ScratchPath::File(dir.join("taps.process.json")),
|
|
ScratchPath::File(dir.join("taps.campaign.json")),
|
|
]);
|
|
let bp_id = seed_blueprint(dir, "campaign-run-taps-seed");
|
|
let proc_id = register_process_doc(dir, "taps.process.json", SWEEP_ONLY_PROCESS_DOC);
|
|
write_doc(
|
|
dir,
|
|
"taps.campaign.json",
|
|
&campaign_doc_json(&bp_id, &proc_id, (1, 2), "\"r_record\"", ""),
|
|
);
|
|
let (out, code) = run_code_in(dir, &["campaign", "run", "taps.campaign.json"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(
|
|
out.contains("aura: persist_taps not yet honored (1 tap(s) ignored)"),
|
|
"the tap note must precede the member-data refusal: {out}"
|
|
);
|
|
assert!(
|
|
out.contains("no recorded geometry") || out.contains("no data for instrument"),
|
|
"the refusal names the data condition: {out}"
|
|
);
|
|
}
|
|
|
|
/// Gated real-data e2e (the `cli_run.rs` skip idiom): a full
|
|
/// sweep -> gate -> walk_forward campaign over GER40 Sept-2024 where the
|
|
/// local archive is present — exit 0, emit-gated family/selection lines, a
|
|
/// final line parseable as JSON with top-level `campaign_run` linking a sweep
|
|
/// family id and a walk-forward family id, and the `campaign_runs.jsonl`
|
|
/// sibling store written. Skips with a note elsewhere so
|
|
/// `cargo test --workspace` stays green on a data-less machine.
|
|
#[test]
|
|
fn campaign_run_real_e2e_sweep_gate_walkforward() {
|
|
let _fixture = project_lock();
|
|
let dir = built_project();
|
|
let runs_dir = dir.join("runs");
|
|
std::fs::remove_dir_all(&runs_dir).ok();
|
|
let _cleanup = ScratchGuard(vec![
|
|
ScratchPath::Dir(runs_dir.clone()),
|
|
ScratchPath::File(dir.join("wf.process.json")),
|
|
ScratchPath::File(dir.join("wf.campaign.json")),
|
|
]);
|
|
let bp_id = seed_blueprint(dir, "campaign-run-e2e-seed");
|
|
let proc_id = register_process_doc(dir, "wf.process.json", WF_PROCESS_DOC);
|
|
// The GER40 Sept-2024 window (inclusive Unix-ms) — the same gated window
|
|
// cli_run.rs drives; ~30 days, so the (14d, 7d, 7d) roller tiles it.
|
|
write_doc(
|
|
dir,
|
|
"wf.campaign.json",
|
|
&campaign_doc_json(
|
|
&bp_id,
|
|
&proc_id,
|
|
(1725148800000, 1727740799999),
|
|
"",
|
|
"\"family_table\", \"selection_report\"",
|
|
),
|
|
);
|
|
let (out, code) = run_code_in(dir, &["campaign", "run", "wf.campaign.json"]);
|
|
|
|
// Skip on a data-less machine: the member-data refusal, never a panic.
|
|
if code == Some(1)
|
|
&& (out.contains("no recorded geometry") || out.contains("no data for instrument"))
|
|
{
|
|
eprintln!("skip: no local GER40 data for the campaign e2e");
|
|
return;
|
|
}
|
|
|
|
assert_eq!(code, Some(0), "stdout/stderr: {out}");
|
|
let record_line = out
|
|
.lines()
|
|
.find(|l| l.starts_with("{\"campaign_run\":"))
|
|
.expect("the always-on final campaign_run line");
|
|
let v: serde_json::Value =
|
|
serde_json::from_str(record_line).expect("campaign_run line parses as JSON");
|
|
let cells = v["campaign_run"]["cells"].as_array().expect("cells array");
|
|
assert_eq!(cells.len(), 1, "one (strategy, instrument, window) cell: {record_line}");
|
|
let stages = cells[0]["stages"].as_array().expect("stages array");
|
|
assert_eq!(stages.len(), 3, "sweep + gate + walk_forward realized: {record_line}");
|
|
assert!(
|
|
stages[0]["family_id"].as_str().is_some(),
|
|
"sweep stage links a family: {record_line}"
|
|
);
|
|
assert_eq!(
|
|
stages[1]["survivor_ordinals"].as_array().map(|a| a.len()),
|
|
Some(4),
|
|
"the always-true gate keeps all four members: {record_line}"
|
|
);
|
|
assert!(
|
|
stages[2]["family_id"].as_str().is_some(),
|
|
"walk-forward stage links a family: {record_line}"
|
|
);
|
|
// Emit-gated lines: per-member family_table lines (4 sweep members plus
|
|
// the walk-forward OOS members) and at least one selection_report line.
|
|
assert!(
|
|
out.lines().filter(|l| l.starts_with("{\"family_id\":")).count() >= 4,
|
|
"family_table member lines emitted: {out}"
|
|
);
|
|
assert!(
|
|
out.lines().any(|l| l.starts_with("{\"selection_report\":")),
|
|
"selection_report line emitted: {out}"
|
|
);
|
|
// The registry's new sibling store carries the realization record.
|
|
assert!(
|
|
runs_dir.join("campaign_runs.jsonl").is_file(),
|
|
"campaign_runs.jsonl written beside runs.jsonl"
|
|
);
|
|
}
|
|
|
|
/// `run_campaign` resolves a target two ways — a readable file (register-then-
|
|
/// run sugar) or a bare content id (direct store address) — then funnels BOTH
|
|
/// through one shared post-resolution path: "fetch the stored canonical bytes
|
|
/// by id ... so file addressing and id addressing produce the same
|
|
/// realization by construction" (the driver's own doc comment). This pins
|
|
/// that property observably: registering a document and then running it once
|
|
/// by FILE and once by its own resulting content ID must refuse with the
|
|
/// byte-identical prose — proof the two addressing modes are not two
|
|
/// independently drifting code paths. No real data needed: both runs refuse
|
|
/// at the (data-free) v1-pipeline-shape preflight.
|
|
#[test]
|
|
fn campaign_run_by_content_id_matches_file_sugar_refusal() {
|
|
let _fixture = project_lock();
|
|
let dir = built_project();
|
|
let runs_dir = dir.join("runs");
|
|
std::fs::remove_dir_all(&runs_dir).ok();
|
|
let _cleanup = ScratchGuard(vec![
|
|
ScratchPath::Dir(runs_dir.clone()),
|
|
ScratchPath::File(dir.join("mc2.process.json")),
|
|
ScratchPath::File(dir.join("mc2.campaign.json")),
|
|
]);
|
|
let bp_id = seed_blueprint(dir, "campaign-run-id-seed");
|
|
let proc_id = register_process_doc(dir, "mc2.process.json", MC_PROCESS_DOC);
|
|
write_doc(dir, "mc2.campaign.json", &campaign_doc_json(&bp_id, &proc_id, (1, 2), "", ""));
|
|
|
|
let (file_out, file_code) = run_code_in(dir, &["campaign", "run", "mc2.campaign.json"]);
|
|
assert_eq!(file_code, Some(1), "stdout/stderr: {file_out}");
|
|
|
|
let (reg_out, reg_code) = run_code_in(dir, &["campaign", "register", "mc2.campaign.json"]);
|
|
assert_eq!(reg_code, Some(0), "register failed: {reg_out}");
|
|
let id = reg_out
|
|
.lines()
|
|
.find(|l| l.starts_with("registered campaign content:"))
|
|
.expect("register line")
|
|
.trim_start_matches("registered campaign content:")
|
|
.split(' ')
|
|
.next()
|
|
.expect("id")
|
|
.to_string();
|
|
let (id_out, id_code) = run_code_in(dir, &["campaign", "run", &id]);
|
|
assert_eq!(id_code, Some(1), "stdout/stderr: {id_out}");
|
|
|
|
let file_line = file_out.lines().find(|l| l.starts_with("aura: ")).expect("file refusal line");
|
|
let id_line = id_out.lines().find(|l| l.starts_with("aura: ")).expect("id refusal line");
|
|
assert_eq!(file_line, id_line, "file- and id-addressed runs must refuse identically");
|
|
}
|
|
|
|
/// `campaign run`'s file-sugar branch calls `parse_valid_campaign` directly
|
|
/// (unwrapped), the SAME intrinsic validator `campaign register` calls but
|
|
/// wraps in its own "refusing to register:" prefix. This pins that `run`
|
|
/// does NOT inherit register's wrapper — an intrinsically invalid document
|
|
/// refuses with the bare "campaign document invalid:" prose — and, like
|
|
/// register, never touches the store for it. No project scaffolding, no
|
|
/// data, no seeded blueprint needed: the refusal fires before any of that is
|
|
/// read.
|
|
#[test]
|
|
fn campaign_run_invalid_file_refuses_before_touching_store() {
|
|
// Run inside the built project (campaign run needs one, per the project
|
|
// gate); the document fails intrinsic validation before any store or
|
|
// referential check is reached, so the project's own store is untouched.
|
|
let _fixture = project_lock();
|
|
let dir = built_project();
|
|
let runs_dir = dir.join("runs");
|
|
std::fs::remove_dir_all(&runs_dir).ok();
|
|
let bad = CAMPAIGN_DOC.replacen(r#""values": [8]"#, r#""values": []"#, 1);
|
|
let bad_path = write_doc(dir, "bad.campaign.json", &bad);
|
|
let _cleanup = ScratchGuard(vec![ScratchPath::Dir(runs_dir.clone()), ScratchPath::File(bad_path)]);
|
|
|
|
let (out, code) = run_code_in(dir, &["campaign", "run", "bad.campaign.json"]);
|
|
assert_eq!(code, Some(1), "stdout/stderr: {out}");
|
|
assert!(
|
|
out.contains("aura: campaign document invalid:"),
|
|
"run must surface the bare doc-tier prose, not register's wrapper: {out}"
|
|
);
|
|
assert!(
|
|
out.contains("axes.fast: an axis is a non-empty finite set"),
|
|
"the doc-tier fault names the offending axis: {out}"
|
|
);
|
|
assert!(!out.contains("refusing to register:"), "run must not reuse register's prefix: {out}");
|
|
assert!(
|
|
!dir.join("runs").join("campaigns").exists(),
|
|
"an invalid document must not create a store entry"
|
|
);
|
|
}
|