From bd5a08d71f944add6fcc49f5cc8c1266531ff256 Mon Sep 17 00:00:00 2001 From: Brummel Date: Sun, 21 Jun 2026 14:17:49 +0200 Subject: [PATCH] plan: 0060 real-data source path for family runs Five compile-gated tasks for spec 0060: (1) additive DataSource/DataChoice provider + ns roller constants, (2) pip-parametrize the two blueprints + thread all callers, (3) thread &DataSource into the five family builders + run_sweep/ run_walkforward (dispatch passes Synthetic), (4) --real parser grammar + build-or-refuse dispatch + USAGE, (5) gated real-path integration tests + full suite + clippy. Each task threads its signature change with all callers so `cargo build -p aura-cli --all-targets` is green per task; the byte-unchanged synthetic path is the regression guard. refs #106 --- docs/plans/0060-real-data-family-runs.md | 685 +++++++++++++++++++++++ 1 file changed, 685 insertions(+) create mode 100644 docs/plans/0060-real-data-family-runs.md diff --git a/docs/plans/0060-real-data-family-runs.md b/docs/plans/0060-real-data-family-runs.md new file mode 100644 index 0000000..f6a8742 --- /dev/null +++ b/docs/plans/0060-real-data-family-runs.md @@ -0,0 +1,685 @@ +# Real-data source path for family runs — Implementation Plan + +> **Parent spec:** `docs/specs/0060-real-data-family-runs.md` +> +> **For agentic workers:** REQUIRED SUB-SKILL: use the `implement` skill to run +> this plan. Steps use `- [ ]` checkboxes for tracking. + +**Goal:** Add opt-in `--real [--from ] [--to ]` to `aura sweep` +and `aura walkforward` so each family member streams real M1 close bars (the #71 +`M1FieldSource` seam) instead of the synthetic stream. Without `--real`, +byte-unchanged. MC excluded. + +**Architecture:** One CLI-side `DataSource` provider (synthetic|real) supplies a +member's source, pip, window, and (walk-forward) roller sizes, replacing the +hardcoded `VecSource` / `SYNTHETIC_PIP_SIZE`. Engine, ingest, registry untouched. + +**Tech Stack:** `crates/aura-cli/src/main.rs` only (+ its `tests/cli_run.rs`). +`aura-cli` is binary-only: test via `cargo test -p aura-cli --bin aura ` +(src `#[cfg(test)]`) and `cargo test -p aura-cli --test cli_run ` +(integration). NEVER `--lib`. The authoritative caller set for every signature +change is `cargo build -p aura-cli --all-targets` (the compiler enumerates the +`#[cfg(test)]` callers a grep misses). + +**Files this plan creates or modifies:** + +- Modify: `crates/aura-cli/src/main.rs` — all production + unit-test changes. +- Test: `crates/aura-cli/tests/cli_run.rs` — real-path integration tests. + +All anchors below are current as of HEAD `9637730` (the spec commit added only +`docs/`, so `main.rs` line numbers are unchanged from recon). + +--- + +### Task 1: `DataSource` provider type, `DataChoice`, ns constants (additive) + +Pure addition — nothing calls it yet, so the tree stays green. Establishes the +provider the later tasks thread. + +**Files:** +- Modify: `crates/aura-cli/src/main.rs` (new items near `SYNTHETIC_PIP_SIZE` :38 + and before the family builders) + +- [ ] **Step 1: Add the real walk-forward roller-size constants** beside + `SYNTHETIC_PIP_SIZE` (after `main.rs:38`): + +```rust +/// Real walk-forward roller sizes (Fork D/F). `WindowRoller` takes sizes in the +/// stream's epoch-unit; for real M1 that is nanoseconds. A classic 3-month +/// in-sample / 1-month out-of-sample / 1-month step (contiguous OOS tiling). +const WF_DAY_NS: i64 = 86_400_000_000_000; +const WF_REAL_IS_NS: i64 = 90 * WF_DAY_NS; +const WF_REAL_OOS_NS: i64 = 30 * WF_DAY_NS; +const WF_REAL_STEP_NS: i64 = 30 * WF_DAY_NS; +``` + +- [ ] **Step 2: Add `DataChoice` (parsed) and `DataSource` (opened) + the refuse + helper.** Place after `parse_real_args` (ends ~:430), before the family + builders. Uses the same qualified paths as `run_sample_real` (`std::sync::Arc`, + `data_server::*`, `aura_ingest::*`, `aura_engine::Source`): + +```rust +/// What `--real` parsing yields: the synthetic default, or a real symbol + an +/// optional window (parsed, not yet opened). Pure, so the grammar is unit-testable. +#[derive(Debug, Clone, PartialEq)] +enum DataChoice { + Synthetic, + Real { symbol: String, from_ms: Option, to_ms: Option }, +} + +/// The source provider threaded into the family builders: synthetic built-in +/// streams, or real M1 close bars from the data-server archive. Replaces the +/// hardcoded `VecSource` so a member's source, pip, window, and roller sizes come +/// from one place (Fork B/D/F). +enum DataSource { + Synthetic, + Real { + server: std::sync::Arc, + symbol: String, + from_ms: Option, + to_ms: Option, + pip: f64, + }, +} + +/// No-local-data refusal — stderr + exit(2), mirroring `run_sample_real`. +fn no_real_data(symbol: &str) -> ! { + eprintln!("aura: no local data for symbol '{symbol}' at {}", data_server::DEFAULT_DATA_PATH); + std::process::exit(2) +} + +impl DataSource { + /// Build a provider from a parsed choice, or refuse (stderr + exit 2) on an + /// un-vetted symbol / absent data — both BEFORE any member runs (Fork C/G), + /// mirroring `run_sample_real:323-341`. + fn from_choice(choice: DataChoice) -> DataSource { + match choice { + DataChoice::Synthetic => DataSource::Synthetic, + DataChoice::Real { symbol, from_ms, to_ms } => { + let spec = match aura_ingest::instrument_spec(&symbol) { + Some(s) => s, + None => { + eprintln!("aura: no vetted pip/instrument spec for symbol '{symbol}' — refusing to run a real instrument with a guessed pip (add it to the instrument table)"); + std::process::exit(2); + } + }; + let server = std::sync::Arc::new(data_server::DataServer::new(data_server::DEFAULT_DATA_PATH)); + if !server.has_symbol(&symbol) { + no_real_data(&symbol); + } + DataSource::Real { server, symbol, from_ms, to_ms, pip: spec.pip_size } + } + } + } + + fn pip_size(&self) -> f64 { + match self { + DataSource::Synthetic => SYNTHETIC_PIP_SIZE, + DataSource::Real { pip, .. } => *pip, + } + } + + /// The full run window, probed once. Synthetic: the showcase span. Real: drain + /// a separate probe source for first/last ts (a Source is single-pass), as + /// `run_sample_real` does. + fn full_window(&self) -> (Timestamp, Timestamp) { + match self { + DataSource::Synthetic => { + let s: Vec> = vec![Box::new(VecSource::new(showcase_prices()))]; + window_of(&s).expect("non-empty showcase stream") + } + DataSource::Real { server, symbol, from_ms, to_ms, .. } => { + let mut probe = aura_ingest::M1FieldSource::open(server, symbol, *from_ms, *to_ms, aura_ingest::M1Field::Close) + .unwrap_or_else(|| no_real_data(symbol)); + let first = aura_engine::Source::peek(&probe).unwrap_or_else(|| no_real_data(symbol)); + let mut last = first; + while let Some((t, _)) = aura_engine::Source::next(&mut probe) { + last = t; + } + (first, last) + } + } + } + + /// A fresh full-window source per member (single-pass). Synthetic: showcase. + fn run_sources(&self) -> Vec> { + match self { + DataSource::Synthetic => vec![Box::new(VecSource::new(showcase_prices()))], + DataSource::Real { server, symbol, from_ms, to_ms, .. } => vec![Box::new( + aura_ingest::M1FieldSource::open(server, symbol, *from_ms, *to_ms, aura_ingest::M1Field::Close) + .unwrap_or_else(|| no_real_data(symbol)), + )], + } + } + + /// A fresh windowed source for an IS/OOS sub-window (walk-forward). Synthetic: + /// `walkforward_window_source`. Real: `open_window` (ns-native `Timestamp`). + fn windowed_sources(&self, from: Timestamp, to: Timestamp) -> Vec> { + match self { + DataSource::Synthetic => vec![Box::new(walkforward_window_source(from, to))], + DataSource::Real { server, symbol, .. } => vec![Box::new( + aura_ingest::M1FieldSource::open_window(server, symbol, Some(from), Some(to), aura_ingest::M1Field::Close) + .unwrap_or_else(|| no_real_data(symbol)), + )], + } + } + + /// WindowRoller sizes per data kind (Fork F): bar-index for synthetic (24/12/12 + /// over the 60-bar span), calendar-ns for real. + fn wf_window_sizes(&self) -> (i64, i64, i64) { + match self { + DataSource::Synthetic => (24, 12, 12), + DataSource::Real { .. } => (WF_REAL_IS_NS, WF_REAL_OOS_NS, WF_REAL_STEP_NS), + } + } +} +``` + +- [ ] **Step 3: Add unit tests** for the synthetic arm + the constants (in the + `main.rs` `#[cfg(test)] mod tests`). These also serve as the RED→GREEN for the + additive type (def + tests land in one diff per the project's RED-in-one-diff + convention): + +```rust +#[test] +fn data_source_synthetic_pip_and_window_match_the_built_ins() { + let d = DataSource::Synthetic; + assert_eq!(d.pip_size(), SYNTHETIC_PIP_SIZE); + assert!(!d.run_sources().is_empty()); + assert_eq!(d.wf_window_sizes(), (24, 12, 12)); + // full_window equals window_of over the showcase stream (byte-unchanged source) + let s: Vec> = vec![Box::new(VecSource::new(showcase_prices()))]; + assert_eq!(d.full_window(), window_of(&s).unwrap()); +} + +#[test] +fn wf_real_roller_sizes_are_90_30_30_days_in_ns() { + assert_eq!(WF_REAL_IS_NS, 90 * 86_400_000_000_000); + assert_eq!(WF_REAL_OOS_NS, 30 * 86_400_000_000_000); + assert_eq!(WF_REAL_STEP_NS, 30 * 86_400_000_000_000); +} +``` + +- [ ] **Step 4: Build + run the new tests.** + +Run: `cargo build -p aura-cli --all-targets` +Expected: builds clean (additive; `DataSource` may warn-unused until Task 3 — if +`-D warnings` is not set on build it is fine; the final clippy gate is Task 5). + +Run: `cargo test -p aura-cli --bin aura data_source_synthetic_pip_and_window` +Expected: PASS + +Run: `cargo test -p aura-cli --bin aura wf_real_roller_sizes` +Expected: PASS + +--- + +### Task 2: Pip-parametrize the two blueprints + thread all callers + +Signature change with a compile-gate: `sample_blueprint_with_sinks` and +`momentum_blueprint_with_sinks` gain `pip_size: f64`; EVERY caller is threaded in +this same task (all still pass `SYNTHETIC_PIP_SIZE` — no behaviour change). + +**Files:** +- Modify: `crates/aura-cli/src/main.rs` + +- [ ] **Step 1: Add the `pip_size` param to `sample_blueprint_with_sinks`** + (:465). Signature `fn sample_blueprint_with_sinks(pip_size: f64) -> (...)`; the + broker line at :475 becomes `let broker = g.add(SimBroker::builder(pip_size));`. + +- [ ] **Step 2: Add the `pip_size` param to `momentum_blueprint_with_sinks`** + (:551). Same shape; broker at :566 becomes `SimBroker::builder(pip_size)`. + +- [ ] **Step 3: Thread every caller — pass `SYNTHETIC_PIP_SIZE`.** The compiler + (Step 4) enumerates them; the known set (all in `main.rs`): + - `build_sample` (:491): `sample_blueprint_with_sinks(SYNTHETIC_PIP_SIZE)`. + - `sweep_family` (:506 + closure :520): `sample_blueprint_with_sinks(SYNTHETIC_PIP_SIZE)`. + - `momentum_sweep_family` (:587 + closure :596): `momentum_blueprint_with_sinks(SYNTHETIC_PIP_SIZE)`. + - `walkforward_family` (:735): `sample_blueprint_with_sinks(SYNTHETIC_PIP_SIZE).0.param_space()`. + - `sweep_over` (:753 + closure :764): `sample_blueprint_with_sinks(SYNTHETIC_PIP_SIZE)`. + - `run_oos` (:790): `sample_blueprint_with_sinks(SYNTHETIC_PIP_SIZE)`. + - test `sample_blueprint_with_sinks_bootstraps_runs_and_drains` (:1274): pass `SYNTHETIC_PIP_SIZE`. + - test `momentum_param_space_*` (:1660): `momentum_blueprint_with_sinks(SYNTHETIC_PIP_SIZE).0.param_space()`. + +- [ ] **Step 4: Compile-gate — all callers threaded.** + +Run: `cargo build -p aura-cli --all-targets` +Expected: 0 errors. (If the compiler names any caller not in Step 3's list, thread +it the same way — `SYNTHETIC_PIP_SIZE` — before proceeding.) + +- [ ] **Step 5: Prove byte-unchanged.** + +Run: `cargo test -p aura-cli --bin aura sweep_report_renders_four_points` +Expected: PASS + +Run: `cargo test -p aura-cli --test cli_run sweep_prints_four_family_json_lines_and_exits_zero` +Expected: PASS + +--- + +### Task 3: Thread `&DataSource` into the family builders + run functions + +The core change. Signature change + compile-gate: the five family builders and the +two `run_*` functions gain a `DataSource`; the dispatch passes `DataSource::Synthetic` +(the `--real` parser is Task 4), so behaviour stays synthetic and byte-unchanged. + +**Files:** +- Modify: `crates/aura-cli/src/main.rs` + +- [ ] **Step 1: `sweep_family`** (:505) → `fn sweep_family(trace: Option<&str>, data: &DataSource) -> SweepFamily`. + Replace the body's source/pip/window: + +```rust +fn sweep_family(trace: Option<&str>, data: &DataSource) -> SweepFamily { + let pip = data.pip_size(); + let window = data.full_window(); + let bp = sample_blueprint_with_sinks(pip).0; + let space = bp.param_space(); + let binder = bp + .axis("signals.trend.fast.length", [2, 3]) + .axis("signals.trend.slow.length", [4, 5]) + .axis("signals.momentum.fast.length", [2]) + .axis("signals.momentum.slow.length", [4]) + .axis("signals.momentum.signal.length", [3]) + .axis("signals.blend.weights[0]", [1.0]) + .axis("signals.blend.weights[1]", [1.0]) + .axis("exposure.scale", [0.5]); + let varying: HashSet = binder.varying_axes().into_iter().collect(); + binder + .sweep(|point| { + let (bp, rx_eq, rx_ex) = sample_blueprint_with_sinks(pip); + let mut h = bp + .bootstrap_with_cells(point) + .expect("grid points are kind-checked against param_space"); + let sources = data.run_sources(); + h.run(sources); + let eq_rows = rx_eq.try_iter().collect::>(); + let ex_rows = rx_ex.try_iter().collect::>(); + let named = zip_params(&space, point); + let key = member_key(&named, &varying); + let manifest = sim_optimal_manifest(named, window, 0, pip); + if let Some(name) = trace { + persist_traces(&format!("{name}/{key}"), &manifest, &eq_rows, &ex_rows); + } + let equity = f64_field(&eq_rows, 0); + let exposure = f64_field(&ex_rows, 0); + RunReport { manifest, metrics: summarize(&equity, &exposure) } + }) + .expect("the built-in named grid matches the sample param-space") +} +``` + +- [ ] **Step 2: `momentum_sweep_family`** (:586) → same transform: add `data: &DataSource`, + `let pip = data.pip_size(); let window = data.full_window();`, build with + `momentum_blueprint_with_sinks(pip)`, source `data.run_sources()`, manifest + `sim_optimal_manifest(named, window, 0, pip)`. + +- [ ] **Step 3: `walkforward_family`** (:729) → add `data: &DataSource`; span + + roller-sizes from the provider; replace the `.expect` with a typed exit(2) gate: + +```rust +fn walkforward_family(trace: Option<&str>, data: &DataSource) -> WalkForwardResult { + let span = data.full_window(); + let (is_len, oos_len, step) = data.wf_window_sizes(); + let roller = match WindowRoller::new(span, is_len, oos_len, step, RollMode::Rolling) { + Ok(r) => r, + Err(e) => { + eprintln!("aura: walk-forward window too short for one IS+OOS span: {e}"); + std::process::exit(2); + } + }; + let space = sample_blueprint_with_sinks(data.pip_size()).0.param_space(); + walk_forward(roller, space, |w: WindowBounds| { + let is_family = sweep_over(w.is.0, w.is.1, data); + let best = optimize(&is_family, "total_pips").expect("total_pips is a known metric"); + let (oos_equity, oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data); + WindowRun { chosen_params: best.params, oos_equity, oos_report } + }) +} +``` + +- [ ] **Step 4: `sweep_over`** (:752) → `fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> SweepFamily`. + `let pip = data.pip_size();` build with `sample_blueprint_with_sinks(pip)`; in + the closure `let sources = data.windowed_sources(from, to);` (keep + `let window = window_of(&sources).expect("non-empty in-sample window");`); + manifest `sim_optimal_manifest(zip_params(&space, point), window, 0, pip)`. + +- [ ] **Step 5: `run_oos`** (:784) → `fn run_oos(params: &[Cell], from: Timestamp, to: Timestamp, trace: Option<&str>, data: &DataSource) -> (...)`. + `let pip = data.pip_size();` build with `sample_blueprint_with_sinks(pip)`; + `let sources = data.windowed_sources(from, to);` (keep its `window_of`); manifest + pip `pip`; the trace key `format!("{name}/oos{}", from.0)` is unchanged. + +- [ ] **Step 6: `run_sweep`** (:681) → `fn run_sweep(strategy: Strategy, name: &str, persist: bool, data: DataSource)`. + Pass `&data` into the two builders: + `Strategy::SmaCross => sweep_family(persist.then_some(name), &data)`, + `Strategy::Momentum => momentum_sweep_family(persist.then_some(name), &data)`. + +- [ ] **Step 7: `run_walkforward`** (:707) → `fn run_walkforward(name: &str, persist: bool, data: DataSource)`; + `walkforward_family(persist.then_some(name), &data)`. + +- [ ] **Step 8: Thread the synthetic dispatch + test callers.** The compiler + (Step 9) names them; known set: + - dispatch sweep (:1150-1152): `run_sweep(strategy, &name, persist, DataSource::Synthetic)`. + - dispatch walkforward (:1157-1159, three exact arms): pass `DataSource::Synthetic` + to `run_walkforward(...)` (the arms stay as-is this task; Task 4 rewrites them). + - test helper `sweep_report` (:624): `sweep_family(None, &DataSource::Synthetic)`. + - test helper `walkforward_report` (:855): `walkforward_family(None, &DataSource::Synthetic)`. + - tests `families_*` (:1349, :1358): `sweep_family(None, &DataSource::Synthetic)` / + `walkforward_family(None, &DataSource::Synthetic)`. + - tests momentum determinism (:1673, :1674): `momentum_sweep_family(None, &DataSource::Synthetic)`. + +- [ ] **Step 9: Compile-gate.** + +Run: `cargo build -p aura-cli --all-targets` +Expected: 0 errors. (Thread any caller the compiler names not in Step 8 with +`&DataSource::Synthetic` / `DataSource::Synthetic`.) + +- [ ] **Step 10: Prove byte-unchanged across the synthetic family paths.** + +Run: `cargo test -p aura-cli --bin aura walkforward_report_has_one_oos_line_per_window` +Expected: PASS + +Run: `cargo test -p aura-cli --test cli_run sweep_trace_persists_a_member_dir_per_grid_point` +Expected: PASS + +Run: `cargo test -p aura-cli --test cli_run walkforward_trace_persists_a_member_dir_per_oos_window` +Expected: PASS + +--- + +### Task 4: `--real` parser grammar + dispatch wiring + USAGE + +Wire `--real` end to end: the parsers yield a `DataChoice`, the dispatch builds the +`DataSource` (build-or-refuse), USAGE documents the tails. + +**Files:** +- Modify: `crates/aura-cli/src/main.rs` + +- [ ] **Step 1: `parse_sweep_args`** (:651) → return type grows a 4th element: + `Result<(Strategy, String, bool, DataChoice), String>`. Add `--real ` + with optional `--from ` / `--to ` (ms parsed as `i64`); default + `DataChoice::Synthetic`. Keep `--strategy` / `--name` / `--trace` and their + mutual-exclusion. New shape: + +```rust +fn parse_sweep_args(rest: &[&str]) -> Result<(Strategy, String, bool, DataChoice), String> { + let usage = || "sweep [--strategy ] [--real [--from ] [--to ]] [--name | --trace ]".to_string(); + let mut strategy = Strategy::SmaCross; + let mut name: Option<(String, bool)> = None; + let mut real_symbol: Option = None; + let mut from_ms: Option = None; + let mut to_ms: Option = None; + let mut it = rest.iter(); + while let Some(tok) = it.next() { + match *tok { + "--strategy" => { + let v = it.next().ok_or_else(usage)?; + strategy = match *v { + "sma" => Strategy::SmaCross, + "momentum" => Strategy::Momentum, + _ => return Err(usage()), + }; + } + "--real" => { + let v = it.next().ok_or_else(usage)?; + real_symbol = Some((*v).to_string()); + } + "--from" => from_ms = Some(it.next().ok_or_else(usage)?.parse().map_err(|_| usage())?), + "--to" => to_ms = Some(it.next().ok_or_else(usage)?.parse().map_err(|_| usage())?), + "--name" if name.is_none() => name = Some((it.next().ok_or_else(usage)?.to_string(), false)), + "--trace" if name.is_none() => name = Some((it.next().ok_or_else(usage)?.to_string(), true)), + _ => return Err(usage()), + } + } + // --from / --to without --real is a usage error (no synthetic window knob). + if real_symbol.is_none() && (from_ms.is_some() || to_ms.is_some()) { + return Err(usage()); + } + let (name, persist) = name.unwrap_or_else(|| ("sweep".to_string(), false)); + let data = match real_symbol { + Some(symbol) => DataChoice::Real { symbol, from_ms, to_ms }, + None => DataChoice::Synthetic, + }; + Ok((strategy, name, persist, data)) +} +``` + +(Note: the existing parser uses a `value` binding pattern; the implementer may +keep that style as long as the grammar + return tuple match. Preserve the existing +`--name`/`--trace` mutual exclusion semantics.) + +- [ ] **Step 2: Add `parse_walkforward_args`** mirroring it, no `--strategy`: + +```rust +/// Parse the `walkforward` tail: `[--real [--from ] [--to ]] [--name | --trace ]`. +/// Defaults: synthetic, name "walkforward", no persist. `--name`/`--trace` are +/// mutually exclusive; `--from`/`--to` require `--real`. Pure (no I/O / exit). +fn parse_walkforward_args(rest: &[&str]) -> Result<(String, bool, DataChoice), String> { + let usage = || "walkforward [--real [--from ] [--to ]] [--name | --trace ]".to_string(); + let mut name: Option<(String, bool)> = None; + let mut real_symbol: Option = None; + let mut from_ms: Option = None; + let mut to_ms: Option = None; + let mut it = rest.iter(); + while let Some(tok) = it.next() { + match *tok { + "--real" => real_symbol = Some(it.next().ok_or_else(usage)?.to_string()), + "--from" => from_ms = Some(it.next().ok_or_else(usage)?.parse().map_err(|_| usage())?), + "--to" => to_ms = Some(it.next().ok_or_else(usage)?.parse().map_err(|_| usage())?), + "--name" if name.is_none() => name = Some((it.next().ok_or_else(usage)?.to_string(), false)), + "--trace" if name.is_none() => name = Some((it.next().ok_or_else(usage)?.to_string(), true)), + _ => return Err(usage()), + } + } + if real_symbol.is_none() && (from_ms.is_some() || to_ms.is_some()) { + return Err(usage()); + } + let (name, persist) = name.unwrap_or_else(|| ("walkforward".to_string(), false)); + let data = match real_symbol { + Some(symbol) => DataChoice::Real { symbol, from_ms, to_ms }, + None => DataChoice::Synthetic, + }; + Ok((name, persist, data)) +} +``` + +- [ ] **Step 3: Rewrite the dispatch arms** in `main()`. Sweep (:1150-1156) builds + the `DataSource` from the `DataChoice` (which may refuse): + +```rust + ["sweep", rest @ ..] => match parse_sweep_args(rest) { + Ok((strategy, name, persist, choice)) => { + run_sweep(strategy, &name, persist, DataSource::from_choice(choice)) + } + Err(m) => { + eprintln!("aura: {m}"); + std::process::exit(2); + } + }, +``` + +Walkforward (:1157-1159, replace the three exact arms with one): + +```rust + ["walkforward", rest @ ..] => match parse_walkforward_args(rest) { + Ok((name, persist, choice)) => { + run_walkforward(&name, persist, DataSource::from_choice(choice)) + } + Err(m) => { + eprintln!("aura: {m}"); + std::process::exit(2); + } + }, +``` + +(The `mc` arms at :1160-1162 are unchanged.) + +- [ ] **Step 4: Update `USAGE`** (:1126) — change the `sweep` and `walkforward` + clauses to: + `aura sweep [--strategy ] [--real [--from ] [--to ]] [--name |--trace ]` + and + `aura walkforward [--real [--from ] [--to ]] [--name |--trace ]`. + +- [ ] **Step 5: Update + add parser unit tests.** Extend the existing + `parse_sweep_args_defaults_selects_and_rejects` (:1680) so every `assert_eq!` + expects the 4-tuple (append `, DataChoice::Synthetic` to the synthetic + expectations), and add `--real` cases + the walkforward parser test: + +```rust +#[test] +fn parse_sweep_args_accepts_real_symbol_and_window() { + assert_eq!( + parse_sweep_args(&["--real", "EURUSD", "--from", "100", "--to", "200", "--trace", "s"]), + Ok((Strategy::SmaCross, "s".to_string(), true, + DataChoice::Real { symbol: "EURUSD".to_string(), from_ms: Some(100), to_ms: Some(200) })) + ); + assert!(parse_sweep_args(&["--real"]).is_err()); // --real needs a symbol + assert!(parse_sweep_args(&["--from", "100"]).is_err()); // --from without --real +} + +#[test] +fn parse_walkforward_args_defaults_and_accepts_real() { + assert_eq!(parse_walkforward_args(&[]), Ok(("walkforward".to_string(), false, DataChoice::Synthetic))); + assert_eq!( + parse_walkforward_args(&["--real", "EURUSD", "--trace", "w"]), + Ok(("w".to_string(), true, DataChoice::Real { symbol: "EURUSD".to_string(), from_ms: None, to_ms: None })) + ); + assert!(parse_walkforward_args(&["--name", "a", "--trace", "b"]).is_err()); + assert!(parse_walkforward_args(&["--real"]).is_err()); +} +``` + +- [ ] **Step 6: Compile-gate + parser tests.** + +Run: `cargo build -p aura-cli --all-targets` +Expected: 0 errors. + +Run: `cargo test -p aura-cli --bin aura parse_sweep_args` +Expected: PASS (both the updated default test and the new real test) + +Run: `cargo test -p aura-cli --bin aura parse_walkforward_args` +Expected: PASS + +--- + +### Task 5: Real-path integration tests + full-suite + lint gate + +Prove the real path end to end (gated on local data), the refusal (not gated), and +the whole-workspace green + clippy. + +**Files:** +- Test: `crates/aura-cli/tests/cli_run.rs` + +- [ ] **Step 1: Add the real-path integration tests.** Follow the existing gated + real test pattern (skip when `/mnt/tickdata` absent) and the existing + `temp_cwd` / `BIN` helpers. Use a vetted symbol (EURUSD, pip 0.0001) and a + bounded window (e.g. one month of 2024): + +```rust +// EURUSD 2024-06 (UTC inclusive ms): a bounded vetted real window. +const EURUSD_JUN2024_FROM_MS: i64 = 1_717_200_000_000; +const EURUSD_JUN2024_TO_MS: i64 = 1_719_791_999_999; + +fn local_data_present() -> bool { + std::path::Path::new("/mnt/tickdata/Pepperstone").is_dir() +} + +#[test] +fn sweep_real_persists_portable_member_dirs_over_real_bars() { + if !local_data_present() { + eprintln!("skip: no local data at /mnt/tickdata/Pepperstone"); + return; + } + let dir = temp_cwd("sweep_real"); + let out = std::process::Command::new(BIN) + .args(["sweep", "--real", "EURUSD", + "--from", &EURUSD_JUN2024_FROM_MS.to_string(), + "--to", &EURUSD_JUN2024_TO_MS.to_string(), + "--trace", "swpr"]) + .current_dir(&dir).output().unwrap(); + assert!(out.status.success(), "stderr: {}", String::from_utf8_lossy(&out.stderr)); + let members: Vec<_> = std::fs::read_dir(dir.join("runs/traces/swpr")).unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + assert_eq!(members.len(), 4, "four sweep members: {members:?}"); + assert!(members.iter().all(|k| k.chars().all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-'))), + "all keys filesystem-portable: {members:?}"); + // one member charts as a uPlot page + let key = &members[0]; + let chart = std::process::Command::new(BIN) + .args(["chart", &format!("swpr/{key}")]).current_dir(&dir).output().unwrap(); + assert!(out.status.success()); + assert!(String::from_utf8_lossy(&chart.stdout).contains("uPlot")); +} + +#[test] +fn walkforward_real_persists_one_oos_member_per_window() { + if !local_data_present() { + eprintln!("skip: no local data at /mnt/tickdata/Pepperstone"); + return; + } + let dir = temp_cwd("wf_real"); + // a full year so 90/30/30-day rolling fits several windows + let out = std::process::Command::new(BIN) + .args(["walkforward", "--real", "EURUSD", + "--from", "1704067200000", "--to", "1735689599999", + "--trace", "wfr"]) + .current_dir(&dir).output().unwrap(); + assert!(out.status.success(), "stderr: {}", String::from_utf8_lossy(&out.stderr)); + let members: Vec<_> = std::fs::read_dir(dir.join("runs/traces/wfr")).unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()).collect(); + assert!(members.iter().all(|k| k.starts_with("oos")), "oos member dirs: {members:?}"); + assert!(members.len() >= 2, "several OOS windows over a year: {members:?}"); +} + +#[test] +fn sweep_real_unspecced_symbol_refuses_with_exit_2() { + // not gated: the pip refusal happens before any data access. + let dir = temp_cwd("sweep_real_refuse"); + let out = std::process::Command::new(BIN) + .args(["sweep", "--real", "AAPL.US"]).current_dir(&dir).output().unwrap(); + assert_eq!(out.status.code(), Some(2)); + assert!(String::from_utf8_lossy(&out.stderr).contains("no vetted pip")); +} + +#[test] +fn sweep_real_is_byte_deterministic_across_runs() { + if !local_data_present() { + eprintln!("skip: no local data at /mnt/tickdata/Pepperstone"); + return; + } + let run = || { + let dir = temp_cwd("sweep_real_det"); + let o = std::process::Command::new(BIN) + .args(["sweep", "--real", "EURUSD", + "--from", &EURUSD_JUN2024_FROM_MS.to_string(), + "--to", &EURUSD_JUN2024_TO_MS.to_string()]) + .current_dir(&dir).output().unwrap(); + // family_id increments per registry; compare the report bodies, not the id. + String::from_utf8_lossy(&o.stdout).replace("sweep-0", "sweep-N") + }; + assert_eq!(run(), run(), "same real sweep twice must be byte-identical (C1)"); +} +``` + +(If `temp_cwd` / `BIN` / a `local_data_present`-equivalent already exist in +`cli_run.rs`, reuse them rather than redefining. Verify the existing real test's +skip idiom and match it.) + +- [ ] **Step 2: Run the new integration tests.** + +Run: `cargo test -p aura-cli --test cli_run sweep_real` +Expected: PASS (the gated ones run if `/mnt/tickdata` is present, else print skip +and pass; `sweep_real_unspecced_symbol_refuses_with_exit_2` always runs) + +Run: `cargo test -p aura-cli --test cli_run walkforward_real` +Expected: PASS + +- [ ] **Step 3: Full-suite gate.** + +Run: `cargo test --workspace` +Expected: all PASS (no synthetic test regressed; real tests pass-or-skip) + +- [ ] **Step 4: Lint gate.** + +Run: `cargo clippy --workspace --all-targets -- -D warnings` +Expected: clean (0 warnings — `DataSource` is now used, so no dead-code warning)