diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index 39a5bff..18214c0 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -2569,6 +2569,15 @@ fn reproduce_family_in(reg: &Registry, id: &str, data: &DataSource) -> Reproduce let w = window_of(&s).expect("non-empty synthetic walk"); (s, w) } + FamilyKind::WalkForward => { + // each member is one OOS window: rebuild its windowed slice from the + // stored window bounds; the winner params come from the shared + // manifest->cells recovery below (as Sweep members do). + let (from, to) = stored.manifest.window; + let s = data.windowed_sources(from, to); + let w = window_of(&s).expect("non-empty OOS window"); + (s, w) + } _ => (data.run_sources(), window), }; let rerun = run_blueprint_member( @@ -3230,6 +3239,87 @@ fn blueprint_sweep_family( }) } +/// Sweep the LOADED blueprint over the user `--axis` grid on an in-sample window +/// `[from,to]` — the windowed, lattice-carrying twin of `stage1_r_sweep_over` and +/// `blueprint_sweep_family`. `sweep_with_lattice` gives the grid lattice `--select +/// plateau` needs. An unknown/kind-mismatched axis surfaces as `BindError` at the +/// sweep terminal (no panic, no hidden exit) for the caller to render. +fn blueprint_sweep_over( + doc: &str, axes: &[(String, Vec)], from: Timestamp, to: Timestamp, data: &DataSource, +) -> Result<(SweepFamily, Vec), BindError> { + let reload = |d: &str| { + blueprint_from_json(d, &|t| std_vocabulary(t)) + .expect("doc parse-validated at the dispatch boundary; reload is infallible") + }; + let pip = data.pip_size(); + let topo = topology_hash(&reload(doc)); + let probe = blueprint_axis_probe(doc); + let space = probe.param_space(); + let mut iter = axes.iter(); + let (first_name, first_vals) = iter.next().expect("a blueprint walk-forward declares >= 1 axis"); + let mut binder = probe.axis(first_name, first_vals.clone()); + for (n, vals) in iter { + binder = binder.axis(n, vals.clone()); + } + binder.sweep_with_lattice(|point| { + let sources = data.windowed_sources(from, to); + let window = window_of(&sources).expect("non-empty in-sample window"); + run_blueprint_member(reload(doc), point, &space, sources, window, 0, pip, &topo) + }) +} + +/// Run the winner params over an out-of-sample window `[from,to]` on the loaded +/// blueprint — the loaded-member analog of `run_oos_r`. The reduce-mode member +/// (`run_blueprint_member`) retains R-metrics, not a raw pip curve, so the stitching +/// segment is empty (an empty segment leaves the stitched curve unbroken). +fn run_oos_blueprint( + doc: &str, params: &[Cell], space: &[ParamSpec], from: Timestamp, to: Timestamp, + topo: &str, data: &DataSource, +) -> (Vec<(Timestamp, f64)>, RunReport) { + let reload = blueprint_from_json(doc, &|t| std_vocabulary(t)) + .expect("doc parse-validated at the dispatch boundary; reload is infallible"); + let pip = data.pip_size(); + let sources = data.windowed_sources(from, to); + let window = window_of(&sources).expect("non-empty out-of-sample window"); + let report = run_blueprint_member(reload, params, space, sources, window, 0, pip, topo); + (Vec::new(), report) +} + +/// The loaded-blueprint IS-refit walk-forward: per IS window, re-optimize the +/// blueprint over the user `--axis` grid, select by `sqn_normalized`, run the +/// winner OOS. Structural twin of `walkforward_family`'s `Stage1R` arm — the +/// generic `walk_forward` driver + `select_winner` reused; only the per-window +/// sweep/OOS source the loaded blueprint. In-closure errors (a bad `--axis`) +/// `exit(2)` with the sweep terminal's message, as the hard-wired arm does. +fn blueprint_walkforward_family( + doc: &str, axes: &[(String, Vec)], data: &DataSource, select: Selection, +) -> WalkForwardResult { + let span = data.wf_full_span(); + let (is_len, oos_len, step) = data.wf_window_sizes(); + let roller = match WindowRoller::new(span, is_len, oos_len, step, RollMode::Rolling) { + Ok(r) => r, + Err(e) => { + eprintln!("aura: walk-forward window too short for one IS+OOS span: {e:?}"); + std::process::exit(2); + } + }; + let space = blueprint_axis_probe(doc).param_space(); + let topo = topology_hash(&blueprint_from_json(doc, &|t| std_vocabulary(t)) + .expect("doc parse-validated at the dispatch boundary; reload is infallible")); + walk_forward(roller, space.clone(), |w: WindowBounds| { + let (is_family, lattice) = blueprint_sweep_over(doc, axes, w.is.0, w.is.1, data) + .unwrap_or_else(|e| { eprintln!("aura: {e:?}"); std::process::exit(2); }); + let (best, selection) = match select_winner(&is_family, "sqn_normalized", select, Some(&lattice)) { + Ok(v) => v, + Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); } + }; + let (oos_equity, mut oos_report) = + run_oos_blueprint(doc, &best.params, &space, w.oos.0, w.oos.1, &topo, data); + oos_report.manifest.selection = Some(selection); + WindowRun { chosen_params: best.params, oos_equity, oos_report } + }) +} + /// A fresh seeded synthetic price walk for one Monte-Carlo draw — the `mc_family` /// pattern (a distinct realization per seed). A FIXED `SyntheticSpec` shared by the /// `aura mc ` persist path AND the reproduce MonteCarlo branch, so the @@ -3336,6 +3426,36 @@ fn run_blueprint_sweep(doc: &str, axes: &[(String, Vec)], name: &str, pe } } +/// `aura walkforward --axis …`: build the loaded-blueprint +/// IS-refit walk-forward, store the canonical blueprint ONCE keyed by the shared +/// `topology_hash` (the C18 hook, so `aura reproduce` re-derives it), record it as +/// a `FamilyKind::WalkForward` family, and print each OOS member line + the summary. +/// Mirrors `run_blueprint_sweep` (content-addressed family verb; no ensure_name_free). +fn run_blueprint_walkforward(doc: &str, axes: &[(String, Vec)], name: &str, data: DataSource, select: Selection) { + let result = blueprint_walkforward_family(doc, axes, &data, select); + let reg = default_registry(); + let topo = result.windows[0] + .run + .oos_report + .manifest + .topology_hash + .clone() + .expect("a blueprint walk-forward stamps every member's topology_hash"); + let canonical = blueprint_to_json( + &blueprint_from_json(doc, &|t| std_vocabulary(t)).expect("doc parse-validated at the dispatch boundary"), + ) + .expect("a loaded blueprint re-serializes"); + reg.put_blueprint(&topo, &canonical).unwrap_or_else(|e| { eprintln!("aura: {e}"); std::process::exit(2); }); + let id = match reg.append_family(name, FamilyKind::WalkForward, &walkforward_member_reports(&result)) { + Ok(id) => id, + Err(e) => { eprintln!("aura: {e}"); std::process::exit(2); } + }; + for w in &result.windows { + println!("{}", family_member_line(&id, &w.run.oos_report)); + } + println!("{}", walkforward_summary_json(&result)); +} + /// `aura mc --seeds N`: build a Monte-Carlo family from a loaded CLOSED /// blueprint (the World/C21 verb), store the canonical blueprint ONCE keyed by the shared /// `topology_hash` (the 0094 hook, so `aura reproduce` re-derives it), record it as a @@ -3985,6 +4105,43 @@ fn parse_sweep_blueprint_args(rest: &[&str]) -> Result)>, + name: String, + select: Selection, +} + +/// Parse the `aura walkforward …` tail: repeatable `--axis +/// =` (the per-window re-fit grid, >= 1 required), `--select +/// ` (default argmax), `--name ` (default +/// "walkforward"). Reduce-mode + content-addressed, so no `--trace`. +fn parse_walkforward_blueprint_args(rest: &[&str]) -> Result { + let usage = || "walkforward --axis = [--axis …] [--select ] [--name ]".to_string(); + let mut axes: Vec<(String, Vec)> = Vec::new(); + let mut name: Option = None; + let mut select = Selection::Argmax; + let mut tail = rest; + while let Some((flag, t)) = tail.split_first() { + let (value, t) = t.split_first().ok_or_else(usage)?; + match *flag { + "--axis" => { + let (n, csv) = value.split_once('=').ok_or_else(usage)?; + if n.is_empty() || axes.iter().any(|(a, _)| a == n) { return Err(usage()); } + let vals = parse_scalar_csv(csv).ok_or_else(usage)?; + axes.push((n.to_string(), vals)); + } + "--select" => select = parse_select(value).map_err(|()| usage())?, + "--name" if name.is_none() => name = Some((*value).to_string()), + _ => return Err(usage()), + } + tail = t; + } + if axes.is_empty() { + return Err("walkforward requires >= 1 --axis to re-fit per window".to_string()); + } + Ok(WfBlueprintArgs { axes, name: name.unwrap_or_else(|| "walkforward".to_string()), select }) +} + /// Parsed tail of `aura mc …`: the seed count + the family name. /// Synthetic-only this cycle (real-data MC rides #172), so `--real`/`--from`/`--to` /// are rejected as unknown flags. Mirrors `parse_sweep_blueprint_args`' shape. @@ -4172,15 +4329,36 @@ fn main() { } } } - ["walkforward", rest @ ..] => match parse_walkforward_args(rest) { - Ok((strategy, name, persist, choice, grid, select)) => { - run_walkforward(strategy, &name, persist, DataSource::from_choice(choice), &grid, select) + ["walkforward", rest @ ..] => { + if let Some(path) = + rest.first().filter(|a| a.ends_with(".json") && std::path::Path::new(a).is_file()) + { + let doc = std::fs::read_to_string(path).unwrap_or_else(|e| { + eprintln!("aura: {path}: {e}"); + std::process::exit(2); + }); + if let Err(e) = blueprint_from_json(&doc, &|t| std_vocabulary(t)) { + eprintln!("aura: {path}: {e:?}"); + std::process::exit(2); + } + let WfBlueprintArgs { axes, name, select } = + parse_walkforward_blueprint_args(&rest[1..]).unwrap_or_else(|msg| { + eprintln!("aura: {msg}"); + std::process::exit(2); + }); + run_blueprint_walkforward(&doc, &axes, &name, DataSource::Synthetic, select); + return; } - Err(msg) => { - eprintln!("aura: {msg}"); - std::process::exit(2); + match parse_walkforward_args(rest) { + Ok((strategy, name, persist, choice, grid, select)) => { + run_walkforward(strategy, &name, persist, DataSource::from_choice(choice), &grid, select) + } + Err(msg) => { + eprintln!("aura: {msg}"); + std::process::exit(2); + } } - }, + } ["generalize", rest @ ..] => match parse_generalize_args(rest) { Ok((name, symbols, grid, metric, from_ms, to_ms)) => { run_generalize(&name, &symbols, &grid, &metric, from_ms, to_ms) @@ -5339,6 +5517,32 @@ mod tests { assert!(parse_sweep_blueprint_args(&["--axis", "x=1", "--name", "a", "--trace", "b"]).is_err()); // name + trace together } + /// Property: the `aura walkforward ` grammar requires >= 1 + /// repeatable `--axis =` re-fit axis, parses the `--select` + /// argmax/plateau selector (default argmax), names the family via `--name` + /// (default "walkforward"), and rejects a walk-forward with nothing to re-fit + /// (no axis) or an unknown `--select` value — each a strict usage error. + #[test] + fn parse_walkforward_blueprint_args_grammar() { + let a = parse_walkforward_blueprint_args(&[ + "--axis", "stage1_signal.fast.length=2,3", "--select", "plateau:mean", "--name", "wf", + ]).expect("valid tail parses"); + assert_eq!(a.axes.len(), 1); + assert_eq!(a.name, "wf"); + assert!(matches!(a.select, Selection::Plateau(_))); + // a walk-forward with nothing to re-fit is rejected: + assert!(parse_walkforward_blueprint_args(&[]).is_err()); + assert!(parse_walkforward_blueprint_args(&["--name", "wf"]).is_err()); + // an unknown --select value is rejected: + assert!(parse_walkforward_blueprint_args(&["--axis", "a=1,2", "--select", "bogus"]).is_err()); + // a repeated --name is rejected (mirrors the sweep/mc blueprint parsers): + assert!(parse_walkforward_blueprint_args(&["--axis", "a=1,2", "--name", "a", "--name", "b"]).is_err()); + // default name + default argmax: + let b = parse_walkforward_blueprint_args(&["--axis", "a=1,2"]).expect("minimal tail parses"); + assert_eq!(b.name, "walkforward"); + assert!(matches!(b.select, Selection::Argmax)); + } + /// Property: `--list-axes` is a standalone query sub-mode — it parses alone /// (setting `list_axes` and binding no axes), is mutually exclusive with a real /// sweep in either flag order, and its absence leaves `list_axes` false while a @@ -5455,6 +5659,25 @@ mod tests { assert!(blueprint_axis_probe(closed).param_space().is_empty()); } + #[test] + fn blueprint_walkforward_family_refits_each_window() { + // The open fixture's two SMA lengths are re-fit per IS window over a 2x2 grid. + let doc = include_str!("../tests/fixtures/stage1_signal_open.json"); + let axes = vec![ + ("stage1_signal.fast.length".to_string(), vec![Scalar::i64(2), Scalar::i64(3)]), + ("stage1_signal.slow.length".to_string(), vec![Scalar::i64(4), Scalar::i64(6)]), + ]; + let result = blueprint_walkforward_family(doc, &axes, &DataSource::Synthetic, Selection::Argmax); + // 24/12/12 over the 60-bar synthetic span -> 3 rolling windows. + assert_eq!(result.windows.len(), 3, "three rolling IS/OOS windows"); + for w in &result.windows { + assert_eq!(w.run.chosen_params.len(), 2, "both axes re-fit each window"); + assert!(w.run.oos_report.metrics.r.is_some(), "OOS record is R-metrics"); + } + // reduce-mode retains no raw pip curve -> the stitched pip-equity is empty. + assert!(result.stitched_oos_equity.is_empty(), "no raw pip curve in reduce-mode"); + } + #[test] fn blueprint_mc_family_seeds_differ() { // MC over a CLOSED signal (both SMA knobs bound): 3 seeds -> 3 draws, one shared diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index 2a6cdff..f6dc88a 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -3533,6 +3533,111 @@ fn aura_mc_over_a_blueprint_reproduces_bit_identically() { let _ = std::fs::remove_dir_all(&cwd); } +/// E2E (#173): an IS-refit walk-forward over a loaded blueprint persists a +/// content-addressed WalkForward family that `aura reproduce`s bit-identically — +/// each OOS member's windowed slice + winner params are recovered from its manifest. +#[test] +fn aura_walkforward_over_a_blueprint_reproduces_bit_identically() { + let cwd = temp_cwd("blueprint-walkforward-reproduce"); + let bp = format!("{}/tests/fixtures/stage1_signal_open.json", env!("CARGO_MANIFEST_DIR")); + let run = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args(["walkforward", &bp, "--axis", "stage1_signal.fast.length=2,3", + "--axis", "stage1_signal.slow.length=4,6", "--name", "wfr"]) + .current_dir(&cwd) + .output() + .expect("spawn aura walkforward"); + assert_eq!(run.status.code(), Some(0), "stderr: {}", String::from_utf8_lossy(&run.stderr)); + let repro = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args(["reproduce", "wfr-0"]) + .current_dir(&cwd) + .output() + .expect("spawn aura reproduce"); + assert_eq!(repro.status.code(), Some(0)); + let out = String::from_utf8(repro.stdout).expect("utf-8"); + assert!(out.contains("reproduced 3/3 members bit-identically"), "stdout: {out}"); + let _ = std::fs::remove_dir_all(&cwd); +} + +/// E2E (#173): the loaded-blueprint walk-forward runs + persists a WalkForward +/// family (3 OOS members) with a {"walkforward":…} summary carrying oos_r, and +/// stores exactly one content-addressed blueprint. +#[test] +fn aura_walkforward_over_a_blueprint_persists_a_family() { + let cwd = temp_cwd("blueprint-walkforward-persist"); + let bp = format!("{}/tests/fixtures/stage1_signal_open.json", env!("CARGO_MANIFEST_DIR")); + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args(["walkforward", &bp, "--axis", "stage1_signal.fast.length=2,3", + "--axis", "stage1_signal.slow.length=4,6", "--name", "wfx"]) + .current_dir(&cwd) + .output() + .expect("spawn aura walkforward"); + assert_eq!(out.status.code(), Some(0), "stderr: {}", String::from_utf8_lossy(&out.stderr)); + let stdout = String::from_utf8(out.stdout).expect("utf-8"); + assert_eq!(stdout.matches("wfx-").count(), 3, "3 OOS member lines: {stdout}"); + assert!(stdout.contains("\"walkforward\""), "summary line: {stdout}"); + assert!(stdout.contains("\"oos_r\""), "R-measured summary: {stdout}"); + let families = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args(["runs", "families"]).current_dir(&cwd).output().expect("spawn families"); + let fo = String::from_utf8(families.stdout).expect("utf-8"); + assert!(fo.contains("\"kind\":\"WalkForward\""), "families: {fo}"); + let n = std::fs::read_dir(cwd.join("runs").join("blueprints")).map(|d| d.count()).unwrap_or(0); + assert_eq!(n, 1, "exactly one content-addressed blueprint stored"); + let _ = std::fs::remove_dir_all(&cwd); +} + +/// E2E (#173): an OPEN blueprint with no --axis (nothing to re-fit) is a usage +/// error (exit 2), naming that >= 1 --axis is required. +#[test] +fn aura_walkforward_over_a_blueprint_rejects_no_axis() { + let bp = format!("{}/tests/fixtures/stage1_signal_open.json", env!("CARGO_MANIFEST_DIR")); + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args(["walkforward", &bp]) + .output() + .expect("spawn aura walkforward (no axis)"); + assert_eq!(out.status.code(), Some(2)); + let stderr = String::from_utf8(out.stderr).expect("utf-8"); + assert!(stderr.contains("requires >= 1 --axis"), "stderr: {stderr}"); +} + +/// E2E (#173, safety): a malformed blueprint is rejected at the dispatch boundary +/// (exit 2, UnknownNodeType) rather than panicking. +#[test] +fn aura_walkforward_over_a_blueprint_rejects_malformed() { + let bp = format!("{}/tests/fixtures/unknown_node.json", env!("CARGO_MANIFEST_DIR")); + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args(["walkforward", &bp, "--axis", "stage1_signal.fast.length=2,3"]) + .output() + .expect("spawn aura walkforward (malformed)"); + assert_eq!(out.status.code(), Some(2), "malformed must be rejected, not panic"); + let stderr = String::from_utf8(out.stderr).expect("utf-8"); + assert!(stderr.contains("UnknownNodeType"), "stderr: {stderr}"); + assert!(!stderr.contains("panicked"), "must reject cleanly: {stderr}"); +} + +/// E2E (#173, panic-safety): a `--axis` naming a knob that is not in the loaded +/// blueprint's param_space fails clean (named BindError, exit 2, no panic). This +/// pins the in-closure fallible-bind path — `blueprint_sweep_over` returns a +/// `BindError` from inside the `walk_forward` window closure — which is a DISTINCT +/// branch from the dispatch-boundary blueprint-parse rejection (the malformed test): +/// the blueprint here is well-formed and passes dispatch, then the per-window +/// re-fit's bind fails. A regression to `.expect()` on that bind would panic here +/// while leaving the malformed test green. +#[test] +fn aura_walkforward_over_a_blueprint_rejects_an_unknown_axis() { + let bp = format!("{}/tests/fixtures/stage1_signal_open.json", env!("CARGO_MANIFEST_DIR")); + let out = std::process::Command::new(env!("CARGO_BIN_EXE_aura")) + .args(["walkforward", &bp, "--axis", "nope=1,2"]) + .output() + .expect("spawn aura walkforward (unknown axis)"); + assert_eq!(out.status.code(), Some(2), "an unknown axis must fail clean, not panic"); + let stderr = String::from_utf8(out.stderr).expect("utf-8"); + assert!( + stderr.contains("Knob") && stderr.contains("nope"), + "names the unresolved axis via BindError: {stderr}" + ); + assert!(!stderr.contains("panicked"), "must reject cleanly, never panic: {stderr}"); +} + /// E2E (acc 2): `aura mc ` is rejected with a named error + exit 2 /// (NOT a panic / exit 101) — MC binds no axis, so a free knob has no binder. #[test]