From c192dfd344e5c167a98f33697c87a6ef47c7ec64 Mon Sep 17 00:00:00 2001 From: Brummel Date: Fri, 26 Jun 2026 12:19:07 +0200 Subject: [PATCH] =?UTF-8?q?audit(0075):=20cycle=20close=20=E2=80=94=20drif?= =?UTF-8?q?t-clean;=20doc-link=20+=20trade=5Frs=20tripwire=20tidy;=20drop?= =?UTF-8?q?=20ephemera?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Architect drift review (c99e7b8..2e77ab1) against the design ledger + CLAUDE.md invariants: drift-clean, debt only — no contract or spec violation. Verified intact: C18 wire shape (trade_rs is #[serde(skip)] + excluded from the hand-written PartialEq; runs.jsonl/families.jsonl bytes and digests unmoved; mc_r_bootstrap is a printed line, never a record), C1 determinism (seeded r_bootstrap + disjoint walkforward windows, both pinned by byte-identical-rerun e2e tests), C7 frictionless Stage-1 (round_trip_cost stays 0.0), C8/C12 streaming reduction (IS sweep folds O(trades)/member, OOS run non-reduce). The verbatim summarize_r / r_metrics_from_rs float duplication is guarded by the cross-reducer equality test — sound. Tidy applied this close: - doc-link drift: the new E[R] notation in r_bootstrap / RBootstrap / run_mc_r_bootstrap doc-comments was parsed by rustdoc as an intra-doc link `[R]` (4 warnings). Wrapped in backticks; doc build now clean. - coverage micro-gap [low, architect]: added a tripwire asserting a populated trade_rs is absent from the serialized JSON (the serde(skip) C18-preservation property had no direct test — only Vec::new() was serialized before). Carry-on (benign debt, not fixed): the mc usage literal carries a `usage:` prefix the sibling sweep/walkforward usages omit — required because the new `["mc", rest @ ..]` arm now handles the `--real` rejection and the preserved golden asserts stderr.contains("usage"); churning the golden to drop the prefix is not worth it. Removed the cycle's ephemeral spec + plans (docs/specs/0075, docs/plans/0075, docs/plans/0076) per the lifecycle rule. Verified: cargo test --workspace 610 passed / 0 failed, clippy -D warnings clean, cargo doc --no-deps clean. closes #139 --- crates/aura-cli/src/main.rs | 4 +- crates/aura-engine/src/mc.rs | 2 +- crates/aura-engine/src/report.rs | 10 + .../plans/0075-walkforward-mc-r-validation.md | 768 ------------------ docs/plans/0076-walkforward-mc-r-bootstrap.md | 487 ----------- .../specs/0075-walkforward-mc-r-validation.md | 359 -------- 6 files changed, 13 insertions(+), 1617 deletions(-) delete mode 100644 docs/plans/0075-walkforward-mc-r-validation.md delete mode 100644 docs/plans/0076-walkforward-mc-r-bootstrap.md delete mode 100644 docs/specs/0075-walkforward-mc-r-validation.md diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index 3f2a8a8..975b4b6 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -2101,14 +2101,14 @@ fn parse_mc_args(rest: &[&str]) -> Result { /// `aura mc --strategy stage1-r [--real ]`: run the stage1-r walk-forward, pool /// every OOS window's per-trade R series in roll order, and print one moving-block -/// bootstrap `mc_r_bootstrap` line (E[R] distribution + P(E[R] <= 0)). Frictionless +/// bootstrap `mc_r_bootstrap` line (`E[R]` distribution + P(`E[R]` <= 0)). Frictionless /// Stage-1 (no costs); deterministic given `seed` (C1). fn run_mc_r_bootstrap(data: DataSource, grid: &Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64) { println!("{}", mc_r_bootstrap_report(&data, grid, block_len, n_resamples, seed)); } /// Assemble the `mc` R-bootstrap line: run the stage1-r walk-forward, pool the OOS -/// per-trade R series in roll order, bootstrap E[R], and render the `mc_r_bootstrap` +/// per-trade R series in roll order, bootstrap `E[R]`, and render the `mc_r_bootstrap` /// line. The body of `run_mc_r_bootstrap` minus the `println!`, so the full real-R /// wiring (walk-forward -> non-empty pooling -> bootstrap -> render) is reachable /// over synthetic data in a `#[cfg(test)]` unit, mirroring `mc_report` / diff --git a/crates/aura-engine/src/mc.rs b/crates/aura-engine/src/mc.rs index c43af5e..a6edaf5 100644 --- a/crates/aura-engine/src/mc.rs +++ b/crates/aura-engine/src/mc.rs @@ -153,7 +153,7 @@ where McFamily { draws, aggregate } } -/// Distribution of E[R] under a moving-block bootstrap of an OOS per-trade R series +/// Distribution of `E[R]` under a moving-block bootstrap of an OOS per-trade R series /// (#139). `block_len == 1` is the i.i.d. trade-shuffle; `block_len > 1` resamples /// contiguous runs, preserving the serial correlation of sequential trades a pure /// shuffle would erase. Deterministic (C1) given `seed`. diff --git a/crates/aura-engine/src/report.rs b/crates/aura-engine/src/report.rs index 627dcd0..81129a9 100644 --- a/crates/aura-engine/src/report.rs +++ b/crates/aura-engine/src/report.rs @@ -1477,6 +1477,16 @@ mod tests { assert_eq!(a, b); } + #[test] + fn populated_trade_rs_is_absent_from_serialized_json() { + // the in-memory conduit must never reach the wire: a populated trade_rs is + // dropped by #[serde(skip)], so the C18 on-disk shape is byte-unperturbed (#139). + let m = summarize_r(&pm_record_two_closed_trades(), 0.0); + assert_eq!(m.trade_rs, vec![2.0, -1.0], "precondition: trade_rs is populated"); + let json = serde_json::to_string(&m).expect("RMetrics serializes"); + assert!(!json.contains("trade_rs"), "trade_rs must not reach the wire: {json}"); + } + /// A minimal dense PositionManagement record with two closed trades at R = +2, -1. /// Columns per `r_col` (CLOSED=0, REALIZED_R=1, DIRECTION=4, ENTRY_PRICE=6, /// STOP_PRICE=7, CONVICTION_AT_ENTRY=9, SIZE=10, OPEN=11, UNREALIZED_R=12); width 13. diff --git a/docs/plans/0075-walkforward-mc-r-validation.md b/docs/plans/0075-walkforward-mc-r-validation.md deleted file mode 100644 index 53470c3..0000000 --- a/docs/plans/0075-walkforward-mc-r-validation.md +++ /dev/null @@ -1,768 +0,0 @@ -# Walk-forward strategy-selectable + R-reporting (iteration 1) — Implementation Plan - -> **Parent spec:** `docs/specs/0075-walkforward-mc-r-validation.md` -> -> **For agentic workers:** REQUIRED SUB-SKILL: use the `implement` skill to run -> this plan. Steps use `- [ ]` checkboxes for tracking. - -**Goal:** Make `aura walkforward --strategy stage1-r --real ` roll IS→OOS -windows, sweep the stage1-r grid in-sample, pick the best by an R metric, run the -chosen params OOS, and report per-window + across-window-pooled OOS R-metrics — -while the bare SMA `walkforward` path stays byte-identical. - -**Architecture:** Two additive library changes in `aura-engine` (`RMetrics` carries -an in-memory `trade_rs` vector excluded from serde + equality; a new -`r_metrics_from_rs` pooled reducer), then `aura-cli` gains `--strategy` + grid -flags on `walkforward`, a strategy-dispatched `walkforward_family` (SMA arm -verbatim), windowed stage1-r helpers (`stage1_r_sweep_over` reduce-mode IS sweep, -`run_oos_r` non-reduce OOS run, `stage1_r_space`), and a pooled `oos_r` summary -block. The Monte-Carlo bootstrap half is iteration 2 (separate plan). - -**Tech Stack:** Rust workspace — `aura-engine` (`report.rs`), `aura-registry` -(`lib.rs` test helper), `aura-cli` (`main.rs`). Test command: -`cargo test --workspace` (one positional filter per invocation). - ---- - -**Files this plan creates or modifies:** - -- Modify: `crates/aura-engine/src/report.rs:44-62` — `RMetrics` gains - `#[serde(skip)] trade_rs: Vec`; drop `PartialEq` from the derive, hand-write - it (ignoring `trade_rs`). -- Modify: `crates/aura-engine/src/report.rs:89-219` — `summarize_r` sets `trade_rs` - in both return arms; add new `pub fn r_metrics_from_rs`. -- Modify: `crates/aura-registry/src/lib.rs:255` — `report_with_r` test helper - `RMetrics { .. }` literal gains `trade_rs: Vec::new()`. -- Modify: `crates/aura-cli/src/main.rs:1526-1546` — `parse_walkforward_args` 3-tuple - → 5-tuple (`--strategy` + four grid flags). -- Modify: `crates/aura-cli/src/main.rs:1615-1666` — `run_walkforward` + - `walkforward_family` gain `strategy`/`grid`; strategy dispatch. -- Modify: `crates/aura-cli/src/main.rs:1734-1744` — `walkforward_summary_json` gains - the `oos_r` pooled block. -- Modify: `crates/aura-cli/src/main.rs:2591-2592` — USAGE walkforward fragment. -- Modify: `crates/aura-cli/src/main.rs:2637-2645` — walkforward dispatch arm - destructures the 5-tuple. -- Create (new fns in `crates/aura-cli/src/main.rs`): `stage1_r_sweep_over`, - `run_oos_r`, `stage1_r_space`. -- Test: `crates/aura-engine/src/report.rs` (#cfg test) — `summarize_r` populates - `trade_rs`; `RMetrics` round-trip equal despite skipped `trade_rs`; - `r_metrics_from_rs` goldens. -- Test: `crates/aura-cli/src/main.rs` (#cfg test) — `parse_walkforward_args` - 5-tuple + `--strategy`/grid; update the two existing assertions at 3653-3656. -- Test: `crates/aura-cli/tests/cli_run.rs` — `aura walkforward --strategy stage1-r` - emits per-window `r` blocks + an `oos_r` summary (synthetic). - ---- - -### Task 1: `RMetrics.trade_rs` — in-memory per-trade R series - -**Files:** -- Modify: `crates/aura-engine/src/report.rs:44-62, 89-135, 206-219` -- Modify: `crates/aura-registry/src/lib.rs:255` -- Test: `crates/aura-engine/src/report.rs` (#cfg test) - -- [ ] **Step 1: Write the failing tests** — append to the `#[cfg(test)] mod tests` - in `crates/aura-engine/src/report.rs`: - -```rust -#[test] -fn summarize_r_populates_trade_rs_in_trade_order() { - // two closed trades (R = +2.0, then -1.0) over a minimal PositionManagement - // record; trade_rs must carry [2.0, -1.0] in trade order. - let rec = pm_record_two_closed_trades(); // helper below - let m = summarize_r(&rec, 0.0); - assert_eq!(m.trade_rs, vec![2.0, -1.0]); - assert_eq!(m.n_trades, 2); -} - -#[test] -fn summarize_r_empty_record_has_empty_trade_rs() { - let m = summarize_r(&[], 0.0); - assert!(m.trade_rs.is_empty()); - assert_eq!(m.n_trades, 0); -} - -#[test] -fn rmetrics_partial_eq_ignores_trade_rs() { - // two RMetrics equal in every metric but differing in trade_rs compare EQUAL - // (trade_rs is an in-memory conduit, excluded from equality) — this is what - // keeps serialize->deserialize round-trips equal (trade_rs is serde-skipped, - // so it deserializes empty). - let a = summarize_r(&pm_record_two_closed_trades(), 0.0); - let mut b = a.clone(); - b.trade_rs = Vec::new(); - assert_eq!(a, b); -} - -/// A minimal dense PositionManagement record with two closed trades at R = +2, -1. -/// Columns per `r_col` (CLOSED=0, REALIZED_R=1, DIRECTION=4, ENTRY_PRICE=6, -/// STOP_PRICE=7, CONVICTION_AT_ENTRY=9, SIZE=10, OPEN=11, UNREALIZED_R=12); width 13. -fn pm_record_two_closed_trades() -> Vec<(Timestamp, Vec)> { - let row = |closed: bool, r: f64, open: bool| { - let mut c = vec![Scalar::f64(0.0); 13]; - c[0] = Scalar::bool(closed); - c[1] = Scalar::f64(r); - c[6] = Scalar::f64(1.0); // entry - c[7] = Scalar::f64(0.5); // stop -> latched 0.5 - c[9] = Scalar::f64(0.3); // conviction - c[11] = Scalar::bool(open); - c - }; - vec![ - (Timestamp(1), row(true, 2.0, false)), - (Timestamp(2), row(true, -1.0, false)), - ] -} -``` - -- [ ] **Step 2: Run to verify it fails** - -Run: `cargo test --workspace summarize_r_populates_trade_rs_in_trade_order` -Expected: FAIL — compile error `no field `trade_rs` on type `RMetrics`` (and the -two siblings fail to compile likewise). - -- [ ] **Step 3: Add the field + hand-written `PartialEq`** — in - `crates/aura-engine/src/report.rs`, change the `RMetrics` derive (line 44) to drop - `PartialEq`, add the field after `conviction_terciles_r` (line 61), and add the - `impl PartialEq` directly after the struct: - -```rust -// line 44: drop PartialEq from the derive -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub struct RMetrics { - // ... all existing fields unchanged ... - pub conviction_terciles_r: [f64; 3], - /// Realised R per closed trade, in trade order — an in-memory conduit for the - /// OOS R-series bootstrap (#139). Excluded from serde (`skip`) so the C18 wire - /// shape is unchanged, and from `PartialEq` (below) so every existing - /// `RMetrics`/`RunReport` equality assertion and round-trip stays green. - #[serde(skip)] - pub trade_rs: Vec, -} - -impl PartialEq for RMetrics { - fn eq(&self, o: &Self) -> bool { - self.expectancy_r == o.expectancy_r - && self.n_trades == o.n_trades - && self.win_rate == o.win_rate - && self.avg_win_r == o.avg_win_r - && self.avg_loss_r == o.avg_loss_r - && self.profit_factor == o.profit_factor - && self.max_r_drawdown == o.max_r_drawdown - && self.n_open_at_end == o.n_open_at_end - && self.sqn == o.sqn - && self.sqn_normalized == o.sqn_normalized - && self.net_expectancy_r == o.net_expectancy_r - && self.conviction_terciles_r == o.conviction_terciles_r - // trade_rs deliberately excluded - } -} -``` - -- [ ] **Step 4: Populate `trade_rs` in both `summarize_r` arms** — in - `crates/aura-engine/src/report.rs`: the empty-input arm (line 122-135 literal) adds - `trade_rs: Vec::new(),`, and the main return literal (line 206-219) adds - `trade_rs: rs,` (the `rs` vector built at line 137 is the per-trade realised R in - trade order; it is last-read by the sqn block before the return, so the move is - valid): - -```rust -// empty-input arm (line 122) — add as the last field: - return RMetrics { - // ... existing zero fields ... - conviction_terciles_r: [0.0; 3], - trade_rs: Vec::new(), - }; -// main return (line 206) — add as the last field: - RMetrics { - // ... existing fields ... - conviction_terciles_r, - trade_rs: rs, - } -``` - -- [ ] **Step 5: Thread the other two `RMetrics { .. }` literal sites** (the only - remaining struct literals workspace-wide — both test helpers): - -```rust -// crates/aura-engine/src/report.rs — the runmetrics_with_r_block_round_trips test -// literal (~line 1135): add `trade_rs: Vec::new(),` as the last field. -// crates/aura-registry/src/lib.rs:255 — the report_with_r helper literal: add -// `trade_rs: Vec::new(),` as the last field. -``` - -- [ ] **Step 6: Build gate (all literal sites threaded)** - -Run: `cargo build --workspace` -Expected: 0 errors (every `RMetrics { .. }` literal now carries `trade_rs`). - -- [ ] **Step 7: Run the new + back-compat tests** - -Run: `cargo test --workspace summarize_r_populates_trade_rs_in_trade_order` -Expected: PASS -Run: `cargo test --workspace rmetrics_partial_eq_ignores_trade_rs` -Expected: PASS -Run: `cargo test --workspace runmetrics_with_r_block_round_trips` -Expected: PASS (the serde-skip + hand-written eq keep the round-trip equal) - -### Task 2: `r_metrics_from_rs` — pooled reducer over a flat R series - -**Files:** -- Modify: `crates/aura-engine/src/report.rs` (new `pub fn` after `summarize_r`) -- Test: `crates/aura-engine/src/report.rs` (#cfg test) - -- [ ] **Step 1: Write the failing test** — append to the report.rs test module: - -```rust -#[test] -fn r_metrics_from_rs_folds_a_flat_series() { - // pooled across-window R series [2.0, -1.0, 1.0]: expectancy = 2/3, 2 wins of 3, - // profit_factor = (2+1)/1 = 3. At cost 0 (frictionless Stage-1) net == gross. - // conviction terciles are not pooled -> [0,0,0]; n_open_at_end is not a pooled - // concept -> 0. - let m = r_metrics_from_rs(&[2.0, -1.0, 1.0]); - assert_eq!(m.n_trades, 3); - assert!((m.expectancy_r - 2.0 / 3.0).abs() < 1e-12); - assert!((m.win_rate - 2.0 / 3.0).abs() < 1e-12); - assert!((m.profit_factor - 3.0).abs() < 1e-12); - assert_eq!(m.net_expectancy_r, m.expectancy_r); - assert_eq!(m.conviction_terciles_r, [0.0; 3]); - assert_eq!(m.n_open_at_end, 0); -} - -#[test] -fn r_metrics_from_rs_empty_is_all_zero() { - let m = r_metrics_from_rs(&[]); - assert_eq!(m.n_trades, 0); - assert_eq!(m.expectancy_r, 0.0); - assert_eq!(m.sqn, 0.0); -} -``` - -- [ ] **Step 2: Run to verify it fails** - -Run: `cargo test --workspace r_metrics_from_rs_folds_a_flat_series` -Expected: FAIL — `cannot find function `r_metrics_from_rs` in this scope`. - -- [ ] **Step 3: Add the reducer** — in `crates/aura-engine/src/report.rs`, directly - after `summarize_r` (line 220). The R-distribution arithmetic is copied - **verbatim** from `summarize_r` (lines 137-177) — the byte-pinned float - expressions must not be algebraically refactored (a shared factoring would shift - the golden by ~1 ULP); `summarize_r` itself is left untouched: - -```rust -/// Reduce a flat per-trade R series (e.g. the pooled across-window OOS series of a -/// walk-forward) into `RMetrics`. The R-distribution fields use the same arithmetic -/// as [`summarize_r`] (copied verbatim — the pinned floats are not refactored). The -/// fields a flat R series cannot carry are set honestly: `n_open_at_end = 0`, -/// `net_expectancy_r = expectancy_r` (exact under the Stage-1 cost = 0 invariant), -/// `conviction_terciles_r = [0,0,0]` (per-trade conviction is not pooled). Empty -/// input -> a well-defined all-zero `RMetrics`. -pub fn r_metrics_from_rs(rs: &[f64]) -> RMetrics { - let n = rs.len() as u64; - if n == 0 { - return RMetrics { - expectancy_r: 0.0, n_trades: 0, win_rate: 0.0, avg_win_r: 0.0, - avg_loss_r: 0.0, profit_factor: 0.0, max_r_drawdown: 0.0, - n_open_at_end: 0, sqn: 0.0, sqn_normalized: 0.0, net_expectancy_r: 0.0, - conviction_terciles_r: [0.0; 3], trade_rs: Vec::new(), - }; - } - let sum: f64 = rs.iter().sum(); - let mean = sum / n as f64; - let wins: Vec = rs.iter().copied().filter(|&r| r > 0.0).collect(); - let losses: Vec = rs.iter().copied().filter(|&r| r <= 0.0).collect(); - let sum_win: f64 = wins.iter().sum(); - let sum_loss: f64 = losses.iter().sum(); - let avg = |v: &[f64]| if v.is_empty() { 0.0 } else { v.iter().sum::() / v.len() as f64 }; - let mut peak = f64::NEG_INFINITY; - let mut cum = 0.0; - let mut max_dd = 0.0_f64; - for &r in rs { - cum += r; - if cum > peak { peak = cum; } - let dd = peak - cum; - if dd > max_dd { max_dd = dd; } - } - let (sqn, sqn_normalized) = if n < 2 { - (0.0, 0.0) - } else { - let var = rs.iter().map(|&r| (r - mean).powi(2)).sum::() / (n as f64 - 1.0); - let sd = var.sqrt(); - if sd > 0.0 { - ((n as f64).sqrt() * mean / sd, (n.min(SQN_CAP) as f64).sqrt() * mean / sd) - } else { - (0.0, 0.0) - } - }; - RMetrics { - expectancy_r: mean, - n_trades: n, - win_rate: wins.len() as f64 / n as f64, - avg_win_r: avg(&wins), - avg_loss_r: avg(&losses), - profit_factor: if sum_loss < 0.0 { sum_win / (-sum_loss) } else { 0.0 }, - max_r_drawdown: max_dd, - n_open_at_end: 0, - sqn, - sqn_normalized, - net_expectancy_r: mean, // cost = 0 -> net == gross (Stage-1 frictionless) - conviction_terciles_r: [0.0; 3], - trade_rs: Vec::new(), - } -} -``` - -- [ ] **Step 4: Run to verify it passes** - -Run: `cargo test --workspace r_metrics_from_rs_folds_a_flat_series` -Expected: PASS -Run: `cargo test --workspace r_metrics_from_rs_empty_is_all_zero` -Expected: PASS - -### Task 3: `walkforward` signature plumbing — `--strategy` + grid flags - -This task threads the strategy/grid through `parse_walkforward_args` → -`run_walkforward` → `walkforward_family` (SMA arm **verbatim**; stage1-r arm a -graceful error stub filled in Task 4) and every call site, so the workspace -compiles and all SMA goldens stay green. - -**Files:** -- Modify: `crates/aura-cli/src/main.rs:1526-1546, 1607-1666, 2591-2592, 2637-2645` -- Test: `crates/aura-cli/src/main.rs` (#cfg test, lines 3651-3660 updated + a new - case) - -- [ ] **Step 1: Write/adjust the failing parse test** — in the report-less - `crates/aura-cli/src/main.rs` test module, replace the body of - `parse_walkforward_args_defaults_and_accepts_real` (3652-3660) to expect the - 5-tuple, and add a stage1-r case: - -```rust -#[test] -fn parse_walkforward_args_defaults_and_accepts_real() { - assert_eq!( - parse_walkforward_args(&[]), - Ok((Strategy::SmaCross, "walkforward".to_string(), false, DataChoice::Synthetic, Stage1RGrid::default())) - ); - assert_eq!( - parse_walkforward_args(&["--real", "EURUSD", "--trace", "w"]), - Ok((Strategy::SmaCross, "w".to_string(), true, - DataChoice::Real { symbol: "EURUSD".to_string(), from_ms: None, to_ms: None }, - Stage1RGrid::default())) - ); - assert!(parse_walkforward_args(&["--name", "a", "--trace", "b"]).is_err()); - assert!(parse_walkforward_args(&["--real"]).is_err()); -} - -#[test] -fn parse_walkforward_args_accepts_strategy_and_grid_flags() { - let r = parse_walkforward_args(&["--strategy", "stage1-r", "--fast", "5,10", "--stop-k", "2.0,3.0"]); - let (strategy, _, _, _, grid) = r.expect("valid stage1-r walkforward args"); - assert_eq!(strategy, Strategy::Stage1R); - assert_eq!(grid.fast, vec![5, 10]); - assert_eq!(grid.stop_k, vec![2.0, 3.0]); - assert!(parse_walkforward_args(&["--strategy", "bogus"]).is_err()); -} -``` - -- [ ] **Step 2: Run to verify it fails** - -Run: `cargo test --workspace parse_walkforward_args_accepts_strategy_and_grid_flags` -Expected: FAIL — compile error (the 5-tuple / `Strategy` return type does not exist -yet; existing 3-tuple assertions also fail to compile). - -- [ ] **Step 3: Extend `parse_walkforward_args`** — replace lines 1523-1546 with the - 5-tuple grammar mirroring `parse_sweep_args` (1480-1521): - -```rust -/// Parse the `walkforward` tail: -/// `[--strategy ] [--real [--from ] [--to ]] [--name | --trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--channel ] [--window ] [--band-k ]`. -/// Defaults: SMA-cross, synthetic, name "walkforward", no persist. `--name`/`--trace` -/// mutually exclusive; `--from`/`--to` require `--real`. Pure (no I/O / exit). -fn parse_walkforward_args( - rest: &[&str], -) -> Result<(Strategy, String, bool, DataChoice, Stage1RGrid), String> { - let usage = || "walkforward [--strategy ] [--real [--from ] [--to ]] [--name | --trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--channel ] [--window ] [--band-k ]".to_string(); - let mut strategy = Strategy::SmaCross; - let mut name: Option<(String, bool)> = None; - let mut real = RealWindowGrammar::default(); - let mut grid = Stage1RGrid::default(); - let mut tail = rest; - while let Some((flag, t)) = tail.split_first() { - let (value, t) = t.split_first().ok_or_else(usage)?; - if real.accept(flag, value, &usage)? { - tail = t; - continue; - } - match *flag { - "--strategy" => { - strategy = match *value { - "sma" => Strategy::SmaCross, - "momentum" => Strategy::Momentum, - "stage1-r" => Strategy::Stage1R, - "stage1-breakout" => Strategy::Stage1Breakout, - "stage1-meanrev" => Strategy::Stage1MeanRev, - _ => return Err(usage()), - }; - } - "--name" if name.is_none() => name = Some(((*value).to_string(), false)), - "--trace" if name.is_none() => name = Some(((*value).to_string(), true)), - "--fast" => grid.fast = parse_csv_list(value).map_err(|()| usage())?, - "--slow" => grid.slow = parse_csv_list(value).map_err(|()| usage())?, - "--stop-length" => grid.stop_length = parse_csv_list(value).map_err(|()| usage())?, - "--stop-k" => grid.stop_k = parse_csv_list(value).map_err(|()| usage())?, - "--channel" => grid.channel = parse_csv_list(value).map_err(|()| usage())?, - "--window" => grid.window = parse_csv_list(value).map_err(|()| usage())?, - "--band-k" => grid.band_k = parse_csv_list(value).map_err(|()| usage())?, - _ => return Err(usage()), - } - tail = t; - } - let (name, persist) = name.unwrap_or_else(|| ("walkforward".to_string(), false)); - Ok((strategy, name, persist, real.finish(&usage)?, grid)) -} -``` - -- [ ] **Step 4: Thread `run_walkforward` + `walkforward_family`** — change - `run_walkforward` (1615) and `walkforward_family` (1643) signatures and add the - dispatch (SMA arm is today's body verbatim; stage1-r arm errors for now): - -```rust -fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSource, grid: &Stage1RGrid) { - if persist - && let Err(e) = TraceStore::open("runs").ensure_name_free(name, WriteKind::Family) - { - eprintln!("aura: {e}"); - std::process::exit(2); - } - let reg = default_registry(); - let result = walkforward_family(strategy, persist.then_some(name), &data, grid); - // ... rest of run_walkforward body unchanged (append_family / print loop / summary) ... -} - -fn walkforward_family( - strategy: Strategy, trace: Option<&str>, data: &DataSource, grid: &Stage1RGrid, -) -> WalkForwardResult { - let span = data.wf_full_span(); - let (is_len, oos_len, step) = data.wf_window_sizes(); - let roller = match WindowRoller::new(span, is_len, oos_len, step, RollMode::Rolling) { - Ok(r) => r, - Err(e) => { - eprintln!("aura: walk-forward window too short for one IS+OOS span: {e:?}"); - std::process::exit(2); - } - }; - match strategy { - Strategy::SmaCross => { - let space = sample_blueprint_with_sinks(data.pip_size()).0.param_space(); - walk_forward(roller, space, |w: WindowBounds| { - let is_family = sweep_over(w.is.0, w.is.1, data); - let best = optimize(&is_family, "total_pips").expect("total_pips is a known metric"); - let (oos_equity, oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data); - WindowRun { chosen_params: best.params, oos_equity, oos_report } - }) - } - Strategy::Stage1R => { - // filled in Task 4 (stage1_r_sweep_over / run_oos_r / stage1_r_space). - eprintln!("aura: walkforward --strategy stage1-r not yet wired"); - std::process::exit(2); - } - _ => { - eprintln!("aura: walkforward has no form for the selected strategy"); - std::process::exit(2); - } - } -} -``` - -- [ ] **Step 5: Update the dispatch arm + USAGE** - -```rust -// dispatch arm (main.rs:2637) -["walkforward", rest @ ..] => match parse_walkforward_args(rest) { - Ok((strategy, name, persist, choice, grid)) => { - run_walkforward(strategy, &name, persist, DataSource::from_choice(choice), &grid) - } - Err(msg) => { - eprintln!("aura: {msg}"); - std::process::exit(2); - } -}, -// USAGE (main.rs:2592) — replace the walkforward fragment with: -// aura walkforward [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] -``` - -- [ ] **Step 6: Thread the remaining `walkforward_family` test callers** (1775, - 3151) so the workspace compiles — both pass the new args: - -```rust -// main.rs:1775 (walkforward_report helper) and :3151 (member-reports test): -// walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default()) -``` - -- [ ] **Step 7: Build gate** - -Run: `cargo build --workspace` -Expected: 0 errors (every `parse_walkforward_args` / `run_walkforward` / -`walkforward_family` caller threaded). - -- [ ] **Step 8: Run parse tests + the SMA walkforward goldens** - -Run: `cargo test --workspace parse_walkforward_args_accepts_strategy_and_grid_flags` -Expected: PASS -Run: `cargo test --workspace parse_walkforward_args_defaults_and_accepts_real` -Expected: PASS -Run: `cargo test --workspace walkforward_report_has_one_oos_line_per_window_plus_summary` -Expected: PASS (the SMA arm is byte-verbatim) - -### Task 4: stage1-r walk-forward arm — windowed helpers + dispatch + `oos_r` - -**Files:** -- Modify: `crates/aura-cli/src/main.rs` — add `stage1_r_space`, - `stage1_r_sweep_over`, `run_oos_r`; fill the `Strategy::Stage1R` arm in - `walkforward_family`; extend `walkforward_summary_json`. -- Test: `crates/aura-cli/tests/cli_run.rs` — synthetic stage1-r walkforward E2E. - -- [ ] **Step 1: Write the failing E2E test** — append to - `crates/aura-cli/tests/cli_run.rs` (model on the existing walkforward + stage1-r - shell tests; runs the built binary on synthetic data): - -```rust -#[test] -fn walkforward_strategy_stage1_r_reports_oos_r() { - // `aura walkforward --strategy stage1-r` on synthetic data: every per-window - // line carries an `r` block, and the summary line carries a pooled `oos_r` - // block. Deterministic: a second run is byte-identical. - let out = run_aura(&["walkforward", "--strategy", "stage1-r"]); - let lines: Vec<&str> = out.trim().lines().collect(); - let summary: serde_json::Value = serde_json::from_str(lines.last().unwrap()).unwrap(); - assert!(summary["walkforward"]["oos_r"].is_object(), "summary carries oos_r"); - assert!(summary["walkforward"]["oos_r"]["n_trades"].is_number()); - // each per-window member line carries metrics.r - for l in &lines[..lines.len() - 1] { - let v: serde_json::Value = serde_json::from_str(l).unwrap(); - assert!(v["report"]["metrics"]["r"].is_object(), "per-window r block present"); - } - let out2 = run_aura(&["walkforward", "--strategy", "stage1-r"]); - assert_eq!(out, out2, "stage1-r walkforward is deterministic"); -} -``` - -(If `run_aura` is not the existing harness name in `cli_run.rs`, use that file's -established binary-invocation helper — the same one -`walkforward_real_persists_one_oos_member_per_window` uses.) - -- [ ] **Step 2: Run to verify it fails** - -Run: `cargo test --workspace walkforward_strategy_stage1_r_reports_oos_r` -Expected: FAIL — the process exits 2 ("not yet wired") so the output has no lines / -the assertion on `oos_r` fails. - -- [ ] **Step 3: Add `stage1_r_space`** — after `stage1_r_sweep_family` (main.rs:1275): - -```rust -/// The param-space of the OPEN stage1-r blueprint (all four knobs free) — the kinds -/// the per-window `WindowRun::chosen_params` are read against (C7). Mirrors the -/// throwaway-build param_space resolution inside `stage1_r_sweep_family`. -fn stage1_r_space() -> Vec { - let (tx_eq, _) = mpsc::channel(); - let (tx_ex, _) = mpsc::channel(); - let (tx_r, _) = mpsc::channel(); - let (tx_req, _) = mpsc::channel(); - stage1_r_graph(tx_eq, tx_ex, tx_r, tx_req, None, None, true, true).param_space() -} -``` - -- [ ] **Step 4: Add `stage1_r_sweep_over`** — the windowed, reduce-mode IS sweep. It - is `stage1_r_sweep_family` (main.rs:1180-1275) with two changes: it always folds - (reduce = true, never persists), and it sources `data.windowed_sources(from, to)` - instead of `data.run_sources()`: - -```rust -/// Windowed reduce-mode stage1-r sweep over `[from, to]` — the in-sample leg of the -/// stage1-r walk-forward. Identical grid/axis/fold logic to `stage1_r_sweep_family`, -/// but windowed (`windowed_sources`) and always folded (O(trades)/member): each -/// member's RunReport carries `metrics.r = Some(summarize_r(..))`, so the family is -/// rankable by an R metric. -fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid: &Stage1RGrid) -> SweepFamily { - let pip = data.pip_size(); - let (tx_eq, _) = mpsc::channel(); - let (tx_ex, _) = mpsc::channel(); - let (tx_r, _) = mpsc::channel(); - let (tx_req, _) = mpsc::channel(); - let bp = stage1_r_graph(tx_eq, tx_ex, tx_r, tx_req, None, None, true, true); - let space = bp.param_space(); - let stop_length_axis = space.iter().map(|p| p.name.clone()) - .find(|n| n.ends_with(STOP_LENGTH_SUFFIX)).expect("open stage1-r vol-stop exposes a stop_length axis"); - let stop_k_axis = space.iter().map(|p| p.name.clone()) - .find(|n| n.ends_with(STOP_K_SUFFIX)).expect("open stage1-r vol-stop exposes a stop_k axis"); - let binder = bp - .axis("fast.length", grid.fast.clone()) - .axis("slow.length", grid.slow.clone()) - .axis(&stop_length_axis, grid.stop_length.clone()) - .axis(&stop_k_axis, grid.stop_k.clone()); - let varying: HashSet = binder.varying_axes().into_iter() - .map(|n| stage1_r_friendly_name(&n)).collect(); - binder - .sweep(|point| { - let (tx_eq, rx_eq) = mpsc::channel(); - let (tx_ex, rx_ex) = mpsc::channel(); - let (tx_r, rx_r) = mpsc::channel(); - let (tx_req, _rx_req) = mpsc::channel(); - let mut h = stage1_r_graph(tx_eq, tx_ex, tx_r, tx_req, None, None, true, true) - .bootstrap_with_cells(point) - .expect("stage1-r grid points are kind-checked against param_space"); - let sources = data.windowed_sources(from, to); - let window = window_of(&sources).expect("non-empty in-sample window"); - h.run(sources); - let mut named: Vec<(String, Scalar)> = zip_params(&space, point).into_iter() - .map(|(n, v)| (stage1_r_friendly_name(&n), v)).collect(); - named.push(("bias_scale".to_string(), Scalar::f64(0.5))); - let key = member_key(&named, &varying); - let _ = &key; // member key not persisted here (in-memory IS family) - let mut manifest = sim_optimal_manifest(named, window, 0, pip); - manifest.broker = stage1_r_broker_label(pip); - let r_rows: Vec<(Timestamp, Vec)> = rx_r.try_iter().collect(); - let (total_pips, max_drawdown) = rx_eq.try_iter().next() - .map(|(_, row)| (row[0].as_f64(), row[1].as_f64())).unwrap_or((0.0, 0.0)); - let bias_sign_flips = rx_ex.try_iter().next().map(|(_, row)| row[2].as_i64() as u64).unwrap_or(0); - let mut m = RunMetrics { total_pips, max_drawdown, bias_sign_flips, r: None }; - m.r = Some(summarize_r(&r_rows, 0.0)); - RunReport { manifest, metrics: m } - }) - .expect("the stage1-r named grid matches the stage1-r param-space") -} -``` - -- [ ] **Step 5: Add `run_oos_r`** — the windowed, non-reduce OOS run (the pip-equity - curve is needed for `stitch`; folds `summarize` + `summarize_r`): - -```rust -/// Run the chosen stage1-r params over an OOS window; return the recorded pip-equity -/// segment (for stitching) and the OOS RunReport whose `metrics.r` carries both the -/// R metrics and the per-trade `trade_rs`. Non-reduce (raw recorders): one window's -/// curve is bounded, and `stitch` needs the full pip-equity series. -fn run_oos_r( - params: &[Cell], from: Timestamp, to: Timestamp, trace: Option<&str>, data: &DataSource, -) -> (Vec<(Timestamp, f64)>, RunReport) { - let pip = data.pip_size(); - let (tx_eq, rx_eq) = mpsc::channel(); - let (tx_ex, rx_ex) = mpsc::channel(); - let (tx_r, rx_r) = mpsc::channel(); - let (tx_req, rx_req) = mpsc::channel(); - let bp = stage1_r_graph(tx_eq, tx_ex, tx_r, tx_req, None, None, true, false); - let space = bp.param_space(); - let mut h = bp.bootstrap_with_cells(params) - .expect("chosen params pre-validated by the in-sample GridSpace::new"); - let sources = data.windowed_sources(from, to); - let window = window_of(&sources).expect("non-empty out-of-sample window"); - h.run(sources); - let eq_rows = rx_eq.try_iter().collect::>(); - let ex_rows = rx_ex.try_iter().collect::>(); - let r_rows: Vec<(Timestamp, Vec)> = rx_r.try_iter().collect(); - let req_rows: Vec<(Timestamp, Vec)> = rx_req.try_iter().collect(); - let mut named: Vec<(String, Scalar)> = zip_params(&space, params).into_iter() - .map(|(n, v)| (stage1_r_friendly_name(&n), v)).collect(); - named.push(("bias_scale".to_string(), Scalar::f64(0.5))); - let mut manifest = sim_optimal_manifest(named, window, 0, pip); - manifest.broker = stage1_r_broker_label(pip); - if let Some(name) = trace { - persist_traces_r(&format!("{name}/oos{}", from.0), &manifest, &eq_rows, &ex_rows, &req_rows); - } - let equity = f64_field(&eq_rows, 0); - let exposure = f64_field(&ex_rows, 0); - let mut metrics = summarize(&equity, &exposure); - metrics.r = Some(summarize_r(&r_rows, 0.0)); - (equity, RunReport { manifest, metrics }) -} -``` - -- [ ] **Step 6: Fill the `Strategy::Stage1R` arm** in `walkforward_family` (replace - the Task-3 error stub): - -```rust - Strategy::Stage1R => { - let space = stage1_r_space(); - walk_forward(roller, space, |w: WindowBounds| { - let is_family = stage1_r_sweep_over(w.is.0, w.is.1, data, grid); - let best = optimize(&is_family, "sqn_normalized").expect("sqn_normalized is a known metric"); - let (oos_equity, oos_report) = run_oos_r(&best.params, w.oos.0, w.oos.1, trace, data); - WindowRun { chosen_params: best.params, oos_equity, oos_report } - }) - } -``` - -- [ ] **Step 7: Extend `walkforward_summary_json`** to add the pooled `oos_r` block - (only when at least one window carries an `r` block — the SMA path's windows have - `r: None`, so its summary line stays byte-identical): - -```rust -fn walkforward_summary_json(result: &WalkForwardResult) -> String { - let total = result.stitched_oos_equity.last().map(|&(_, v)| v).unwrap_or(0.0); - // pool the per-window OOS per-trade R series in roll order, then reduce. - let pooled_rs: Vec = result.windows.iter() - .flat_map(|w| w.run.oos_report.metrics.r.as_ref().map(|r| r.trade_rs.clone()).unwrap_or_default()) - .collect(); - let mut obj = serde_json::json!({ - "windows": result.windows.len(), - "stitched_total_pips": total, - "param_stability": param_stability(result), - }); - if result.windows.iter().any(|w| w.run.oos_report.metrics.r.is_some()) { - // RMetrics serializes its scalar fields (trade_rs is serde-skipped, so the - // oos_r block is the clean R-metric summary of the pooled series). - obj["oos_r"] = serde_json::to_value(r_metrics_from_rs(&pooled_rs)) - .expect("RMetrics serializes"); - } - serde_json::json!({ "walkforward": obj }).to_string() -} -``` - -(Import `r_metrics_from_rs` from `aura_engine` alongside the existing `summarize_r` -/ `summarize` imports at the top of `main.rs`.) - -- [ ] **Step 8: Build + run the E2E test + the determinism/golden guards** - -Run: `cargo build --workspace` -Expected: 0 errors -Run: `cargo test --workspace walkforward_strategy_stage1_r_reports_oos_r` -Expected: PASS -Run: `cargo test --workspace walkforward_report_has_one_oos_line_per_window_plus_summary` -Expected: PASS (SMA summary unchanged — no `oos_r` key, windows have `r: None`) - -- [ ] **Step 9: Full-suite regression gate** - -Run: `cargo test --workspace` -Expected: PASS — all pre-existing goldens (SMA walkforward, synthetic mc, stage1-r -single-run `stage1_r_single_run_output_golden`, C18 round-trips) green; the new -tests green. - -Run: `cargo clippy --workspace --all-targets -- -D warnings` -Expected: 0 warnings. - ---- - -## Self-review - -- **Spec coverage:** Changes 1 (RMetrics.trade_rs / summarize_r — Task 1), the - pooled reducer (Task 2), Change 3 walkforward dispatch (Tasks 3+4), Change 5 USAGE - (Task 3). Monte-Carlo (spec Changes 2+4) is iteration 2 — out of this plan's scope - by design. ✓ -- **Placeholder scan:** no TBD/TODO; the Task-3 stage1-r error stub is an explicit, - compiling intermediate filled in Task 4 (named, not a placeholder). ✓ -- **Type/name consistency:** `RMetrics.trade_rs`, `r_metrics_from_rs`, - `stage1_r_sweep_over`, `run_oos_r`, `stage1_r_space`, `Strategy::Stage1R`, - `Stage1RGrid` used identically across tasks. ✓ -- **Step granularity:** each step is one edit or one command. ✓ -- **No commit steps.** ✓ -- **Compile-gate ordering (rule 7):** Task 1 threads all 4 `RMetrics { .. }` literal - sites before its build gate; Task 3 threads all `parse_walkforward_args` / - `walkforward_family` / `run_walkforward` callers (dispatch arm + 2 test helpers + - parse-test assertions) before its build gate, with the stage1-r arm a compiling - error-stub so the gate is satisfiable. ✓ -- **Filter strings (rule 8):** every `cargo test` filter names a real test — - pre-existing (`runmetrics_with_r_block_round_trips`, - `walkforward_report_has_one_oos_line_per_window_plus_summary`, - `parse_walkforward_args_defaults_and_accepts_real`, - `stage1_r_single_run_output_golden`) or introduced in the same task; the final - gate is the unfiltered `cargo test --workspace`. ✓ diff --git a/docs/plans/0076-walkforward-mc-r-bootstrap.md b/docs/plans/0076-walkforward-mc-r-bootstrap.md deleted file mode 100644 index 2825938..0000000 --- a/docs/plans/0076-walkforward-mc-r-bootstrap.md +++ /dev/null @@ -1,487 +0,0 @@ -# Monte-Carlo R-bootstrap (cycle 0075, iteration 2) — Implementation Plan - -> **Parent spec:** `docs/specs/0075-walkforward-mc-r-validation.md` -> -> **For agentic workers:** REQUIRED SUB-SKILL: use the `implement` skill to run -> this plan. Steps use `- [ ]` checkboxes for tracking. - -**Goal:** `aura mc --strategy stage1-r [--real ]` runs the stage1-r -walk-forward, pools its OOS per-trade R series, and reports a moving-block -bootstrap `E[R]` distribution / confidence interval (`mc_r_bootstrap` line) — while -the bare synthetic `aura mc` path stays byte-identical. - -**Architecture:** A new deterministic engine primitive `r_bootstrap` (moving-block -bootstrap over a flat R series, reusing `MetricStats`/`SplitMix64`), then a CLI -`parse_mc_args`/`McArgs` split routing the bare/`--name`/`--trace` forms to today's -`run_mc` (unchanged) and the `--strategy stage1-r` form to a new -`run_mc_r_bootstrap` that reuses iteration 1's `walkforward_family` + -`RMetrics.trade_rs`. The mc real-R path accepts only `--strategy stage1-r` (the -sole R-reporting walk-forward strategy) — a bare `--real` without it stays a usage -error. - -**Tech Stack:** Rust workspace — `aura-engine` (`mc.rs`, `lib.rs`), `aura-cli` -(`main.rs`, `tests/cli_run.rs`). Test command: `cargo test --workspace` (one -positional filter per invocation). - ---- - -**Files this plan creates or modifies:** - -- Modify: `crates/aura-engine/src/mc.rs:111` — add `pub struct RBootstrap` + - `pub fn r_bootstrap`; `use crate::harness::SplitMix64;` at the module head. -- Modify: `crates/aura-engine/src/lib.rs:72` — re-export `RBootstrap`, `r_bootstrap` - from the `pub use mc::{..}` block. -- Modify: `crates/aura-cli/src/main.rs:2811-2813` — replace the three literal `mc` - dispatch arms with one `["mc", rest @ ..]` arm. -- Modify: `crates/aura-cli/src/main.rs` (new fns near the mc cluster ~2045) — - `parse_mc_args`, `enum McArgs`, `run_mc_r_bootstrap`, `mc_r_bootstrap_json`. -- Modify: `crates/aura-cli/src/main.rs:2756-2757` — USAGE mc fragment. -- Test: `crates/aura-engine/src/mc.rs` (#cfg test ~155) — `r_bootstrap` goldens. -- Test: `crates/aura-cli/src/main.rs` (#cfg test) — `parse_mc_args` units. -- Test: `crates/aura-cli/tests/cli_run.rs` — `aura mc --strategy stage1-r` E2E; the - `mc_rejects_real_flag_with_usage_exit_2` doc-comment refresh (assertions stay). - ---- - -### Task 1: `r_bootstrap` — moving-block bootstrap over a flat R series - -**Files:** -- Modify: `crates/aura-engine/src/mc.rs:1-13` (module-head `use`), `:111` (new - struct + fn) -- Modify: `crates/aura-engine/src/lib.rs:72` -- Test: `crates/aura-engine/src/mc.rs` (#cfg test, ~155) - -- [ ] **Step 1: Write the failing tests** — append to the `mod tests` in - `crates/aura-engine/src/mc.rs` (`use super::*` is already in scope): - -```rust -#[test] -fn r_bootstrap_empty_series_is_all_zero() { - let b = r_bootstrap(&[], 100, 1, 7); - assert_eq!(b.n_trades, 0); - assert_eq!(b.e_r.mean, 0.0); - assert_eq!(b.prob_le_zero, 0.0); -} - -#[test] -fn r_bootstrap_single_block_equals_full_series_mean() { - // block_len == n: the only valid start is 0, so every resample IS the full - // series -> every resample mean == the series mean -> zero spread. - let rs = [1.0, -2.0, 3.0]; // mean = 2/3 - let b = r_bootstrap(&rs, 500, rs.len(), 7); - let mean = 2.0 / 3.0; - assert!((b.e_r.mean - mean).abs() < 1e-12); - assert!((b.e_r.p5 - mean).abs() < 1e-12); - assert!((b.e_r.p95 - mean).abs() < 1e-12); - assert_eq!(b.prob_le_zero, 0.0); // mean > 0 - assert_eq!(b.n_trades, 3); - assert_eq!(b.block_len, 3); - assert_eq!(b.n_resamples, 500); -} - -#[test] -fn r_bootstrap_is_deterministic_given_seed() { - let rs = [0.5, -1.0, 2.0, -0.5, 1.5, -2.0, 0.25]; - let a = r_bootstrap(&rs, 1000, 1, 42); - let b = r_bootstrap(&rs, 1000, 1, 42); - assert_eq!(a, b, "same rs+seed+params -> identical RBootstrap (C1)"); - let c = r_bootstrap(&rs, 1000, 1, 43); - assert_ne!(a.e_r.p5, c.e_r.p5, "a different seed reshuffles differently"); -} - -#[test] -fn r_bootstrap_block_len_is_clamped_to_series_len() { - // block_len > n clamps to n -> single-block behaviour (no panic, no OOB). - let rs = [1.0, 2.0]; - let b = r_bootstrap(&rs, 10, 99, 1); - assert_eq!(b.block_len, 2); - assert!((b.e_r.mean - 1.5).abs() < 1e-12); -} -``` - -- [ ] **Step 2: Run to verify it fails** - -Run: `cargo test --workspace r_bootstrap_single_block_equals_full_series_mean` -Expected: FAIL — `cannot find function `r_bootstrap` in this scope`. - -- [ ] **Step 3: Add the module-head import** — in `crates/aura-engine/src/mc.rs`, - after the existing `use crate::{RunMetrics, RunReport, Scalar};` (line 13): - -```rust -use crate::harness::SplitMix64; -``` - -- [ ] **Step 4: Add `RBootstrap` + `r_bootstrap`** — in - `crates/aura-engine/src/mc.rs`, after the `quantile` fn (line 110), before - `#[cfg(test)] mod tests`: - -```rust -/// Distribution of E[R] under a moving-block bootstrap of an OOS per-trade R series -/// (#139). `block_len == 1` is the i.i.d. trade-shuffle; `block_len > 1` resamples -/// contiguous runs, preserving the serial correlation of sequential trades a pure -/// shuffle would erase. Deterministic (C1) given `seed`. -#[derive(Clone, Debug, PartialEq)] -pub struct RBootstrap { - pub e_r: MetricStats, // mean + p5/p25/p50/p75/p95 of the resampled E[R] - pub prob_le_zero: f64, // fraction of resamples whose mean R <= 0 - pub n_trades: usize, - pub block_len: usize, - pub n_resamples: usize, -} - -/// Moving-block bootstrap of `rs` (non-circular; the final block of each resample is -/// truncated so the resample has exactly `n` values). `block_len` is clamped to -/// `[1, n]`. Empty `rs` or zero resamples -> an all-zero `RBootstrap`. Pure given -/// `seed` (drives the existing `SplitMix64`). -pub fn r_bootstrap(rs: &[f64], n_resamples: usize, block_len: usize, seed: u64) -> RBootstrap { - let n = rs.len(); - if n == 0 || n_resamples == 0 { - return RBootstrap { - e_r: MetricStats { mean: 0.0, p5: 0.0, p25: 0.0, p50: 0.0, p75: 0.0, p95: 0.0 }, - prob_le_zero: 0.0, - n_trades: n, - block_len: block_len.clamp(1, n.max(1)), - n_resamples, - }; - } - let block_len = block_len.clamp(1, n); - let mut rng = SplitMix64::new(seed); - let mut means: Vec = Vec::with_capacity(n_resamples); - for _ in 0..n_resamples { - let mut sample: Vec = Vec::with_capacity(n); - while sample.len() < n { - let start = (rng.next_u64() % (n - block_len + 1) as u64) as usize; - let take = block_len.min(n - sample.len()); - sample.extend_from_slice(&rs[start..start + take]); - } - means.push(sample.iter().sum::() / n as f64); - } - let e_r = MetricStats::from_values(&means); - let prob_le_zero = means.iter().filter(|&&m| m <= 0.0).count() as f64 / n_resamples as f64; - RBootstrap { e_r, prob_le_zero, n_trades: n, block_len, n_resamples } -} -``` - -- [ ] **Step 5: Re-export from the crate root** — in - `crates/aura-engine/src/lib.rs:72`, add `RBootstrap` and `r_bootstrap` to the - `pub use mc::{..}` list: - -```rust -pub use mc::{monte_carlo, r_bootstrap, McAggregate, McDraw, McFamily, MetricStats, RBootstrap}; -``` - -- [ ] **Step 6: Run to verify they pass** - -Run: `cargo test --workspace r_bootstrap_single_block_equals_full_series_mean` -Expected: PASS -Run: `cargo test --workspace r_bootstrap_is_deterministic_given_seed` -Expected: PASS -Run: `cargo test --workspace r_bootstrap_empty_series_is_all_zero` -Expected: PASS -Run: `cargo test --workspace r_bootstrap_block_len_is_clamped_to_series_len` -Expected: PASS - -### Task 2: `parse_mc_args` + `McArgs` + `run_mc_r_bootstrap` + dispatch - -**Files:** -- Modify: `crates/aura-cli/src/main.rs` — new `enum McArgs`, `parse_mc_args`, - `run_mc_r_bootstrap`, `mc_r_bootstrap_json` (near the mc cluster); dispatch arm - (2811-2813); USAGE (2757). -- Test: `crates/aura-cli/src/main.rs` (#cfg test) — `parse_mc_args` units. - -- [ ] **Step 1: Write the failing parse tests** — append to the `mod tests` in - `crates/aura-cli/src/main.rs`: - -```rust -#[test] -fn parse_mc_args_bare_and_name_route_to_synthetic() { - assert_eq!(parse_mc_args(&[]), Ok(McArgs::Synthetic { name: "mc".to_string(), persist: false })); - assert_eq!(parse_mc_args(&["--name", "m"]), Ok(McArgs::Synthetic { name: "m".to_string(), persist: false })); - assert_eq!(parse_mc_args(&["--trace", "m"]), Ok(McArgs::Synthetic { name: "m".to_string(), persist: true })); -} - -#[test] -fn parse_mc_args_stage1_r_routes_to_real_r_with_defaults_and_overrides() { - assert_eq!( - parse_mc_args(&["--strategy", "stage1-r"]), - Ok(McArgs::RealR { - choice: DataChoice::Synthetic, grid: Stage1RGrid::default(), - block_len: 1, n_resamples: 1000, seed: 1, - }) - ); - let r = parse_mc_args(&["--strategy", "stage1-r", "--real", "USDJPY", - "--block-len", "5", "--resamples", "200", "--seed", "9", "--fast", "5,10"]); - let McArgs::RealR { choice, grid, block_len, n_resamples, seed } = r.expect("valid real-R mc args") - else { panic!("expected RealR") }; - assert_eq!(choice, DataChoice::Real { symbol: "USDJPY".to_string(), from_ms: None, to_ms: None }); - assert_eq!((block_len, n_resamples, seed), (5, 200, 9)); - assert_eq!(grid.fast, vec![5, 10]); -} - -#[test] -fn parse_mc_args_real_without_stage1_r_is_usage_error() { - // a bare --real (no R candidate) is still rejected — the synthetic seed-resweep - // is undefined over real bars; the R path needs --strategy stage1-r. - assert!(parse_mc_args(&["--real", "EURUSD"]).is_err()); - assert!(parse_mc_args(&["--strategy", "sma"]).is_err()); - assert!(parse_mc_args(&["--strategy", "stage1-r", "--name", "x"]).is_err()); // name invalid on R path -} -``` - -- [ ] **Step 2: Run to verify it fails** - -Run: `cargo test --workspace parse_mc_args_bare_and_name_route_to_synthetic` -Expected: FAIL — `cannot find function `parse_mc_args` / type `McArgs``. - -- [ ] **Step 3: Add `McArgs` + `parse_mc_args`** — in `crates/aura-cli/src/main.rs`, - after `run_mc` (~line 2045): - -```rust -/// Parsed form of the `mc` tail: either the synthetic seed-resweep (today's path, -/// preserved byte-for-byte) or the real-candidate R-bootstrap. The R path accepts -/// ONLY `--strategy stage1-r` (the sole R-reporting walk-forward strategy); a bare -/// `--real` without it is a usage error (the synthetic seed-resweep is undefined -/// over real bars). -#[derive(Clone, Debug, PartialEq)] -enum McArgs { - Synthetic { name: String, persist: bool }, - RealR { choice: DataChoice, grid: Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64 }, -} - -fn parse_mc_args(rest: &[&str]) -> Result { - let usage = || "mc [--name |--trace ] | mc --strategy stage1-r [--real [--from ] [--to ]] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--block-len ] [--resamples ] [--seed ]".to_string(); - let mut strategy: Option = None; - let mut name: Option<(String, bool)> = None; - let mut real = RealWindowGrammar::default(); - let mut grid = Stage1RGrid::default(); - let mut block_len: usize = 1; - let mut n_resamples: usize = 1000; - let mut seed: u64 = 1; - let mut r_path = false; - let mut tail = rest; - while let Some((flag, t)) = tail.split_first() { - let (value, t) = t.split_first().ok_or_else(usage)?; - if real.accept(flag, value, &usage)? { - r_path = true; - tail = t; - continue; - } - match *flag { - "--strategy" => { - strategy = Some(match *value { - "stage1-r" => Strategy::Stage1R, - _ => return Err(usage()), - }); - r_path = true; - } - "--name" if name.is_none() => name = Some(((*value).to_string(), false)), - "--trace" if name.is_none() => name = Some(((*value).to_string(), true)), - "--fast" => { grid.fast = parse_csv_list(value).map_err(|()| usage())?; r_path = true; } - "--slow" => { grid.slow = parse_csv_list(value).map_err(|()| usage())?; r_path = true; } - "--stop-length" => { grid.stop_length = parse_csv_list(value).map_err(|()| usage())?; r_path = true; } - "--stop-k" => { grid.stop_k = parse_csv_list(value).map_err(|()| usage())?; r_path = true; } - "--block-len" => { block_len = value.parse().map_err(|_| usage())?; r_path = true; } - "--resamples" => { n_resamples = value.parse().map_err(|_| usage())?; r_path = true; } - "--seed" => { seed = value.parse().map_err(|_| usage())?; r_path = true; } - _ => return Err(usage()), - } - tail = t; - } - if r_path { - if strategy != Some(Strategy::Stage1R) { - return Err(usage()); - } - if name.is_some() { - return Err(usage()); - } - let choice = real.finish(&usage)?; - Ok(McArgs::RealR { choice, grid, block_len, n_resamples, seed }) - } else { - let (name, persist) = name.unwrap_or_else(|| ("mc".to_string(), false)); - Ok(McArgs::Synthetic { name, persist }) - } -} -``` - -- [ ] **Step 4: Add `run_mc_r_bootstrap` + `mc_r_bootstrap_json`** — after - `parse_mc_args`: - -```rust -/// `aura mc --strategy stage1-r [--real ]`: run the stage1-r walk-forward, pool -/// every OOS window's per-trade R series in roll order, and print one moving-block -/// bootstrap `mc_r_bootstrap` line (E[R] distribution + P(E[R] <= 0)). Frictionless -/// Stage-1 (no costs); deterministic given `seed` (C1). -fn run_mc_r_bootstrap(data: DataSource, grid: &Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64) { - let result = walkforward_family(Strategy::Stage1R, None, &data, grid); - let pooled: Vec = result - .windows - .iter() - .flat_map(|w| w.run.oos_report.metrics.r.as_ref().map(|r| r.trade_rs.clone()).unwrap_or_default()) - .collect(); - let boot = r_bootstrap(&pooled, n_resamples, block_len, seed); - println!("{}", mc_r_bootstrap_json(&boot)); -} - -/// Render an `RBootstrap` as one canonical JSON line (`MetricStats` serializes; the -/// scalar fields are spliced in), mirroring `mc_aggregate_json`. -fn mc_r_bootstrap_json(b: &RBootstrap) -> String { - serde_json::json!({ - "mc_r_bootstrap": { - "n_trades": b.n_trades, - "block_len": b.block_len, - "n_resamples": b.n_resamples, - "e_r": b.e_r, - "prob_le_zero": b.prob_le_zero, - } - }) - .to_string() -} -``` - -(Add `r_bootstrap`, `RBootstrap` to the `use aura_engine::{..}` import block at the -top of `main.rs`, alongside the existing `monte_carlo` / `McFamily` imports.) - -- [ ] **Step 5: Replace the dispatch arms** — in `crates/aura-cli/src/main.rs`, - replace the three literal arms (2811-2813): - -```rust -["mc", rest @ ..] => match parse_mc_args(rest) { - Ok(McArgs::Synthetic { name, persist }) => run_mc(&name, persist), - Ok(McArgs::RealR { choice, grid, block_len, n_resamples, seed }) => { - run_mc_r_bootstrap(DataSource::from_choice(choice), &grid, block_len, n_resamples, seed) - } - Err(msg) => { - eprintln!("aura: {msg}"); - std::process::exit(2); - } -}, -``` - -- [ ] **Step 6: Extend USAGE** — in `crates/aura-cli/src/main.rs:2757`, replace the - `aura mc [--name |--trace ]` fragment with: - -``` -aura mc [--name |--trace ] | aura mc --strategy stage1-r [--real [--from ] [--to ]] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--block-len ] [--resamples ] [--seed ] -``` - -- [ ] **Step 7: Build gate** - -Run: `cargo build --workspace` -Expected: 0 errors. - -- [ ] **Step 8: Run the parse units + the preserved synthetic mc goldens** - -Run: `cargo test --workspace parse_mc_args_bare_and_name_route_to_synthetic` -Expected: PASS -Run: `cargo test --workspace parse_mc_args_stage1_r_routes_to_real_r_with_defaults_and_overrides` -Expected: PASS -Run: `cargo test --workspace parse_mc_args_real_without_stage1_r_is_usage_error` -Expected: PASS -Run: `cargo test --workspace mc_runs_persists_a_monte_carlo_family_and_lists_it` -Expected: PASS (bare `aura mc` unchanged — routed to `run_mc` byte-for-byte) -Run: `cargo test --workspace mc_rejects_real_flag_with_usage_exit_2` -Expected: PASS (a bare `--real` without `--strategy stage1-r` still exits 2 + usage) - -### Task 3: stage1-r mc E2E + doc-comment refresh - -**Files:** -- Test: `crates/aura-cli/tests/cli_run.rs` — new `aura mc --strategy stage1-r` E2E; - refresh the `mc_rejects_real_flag_with_usage_exit_2` doc-comment (1298-1304). - -- [ ] **Step 1: Write the failing E2E test** — append to - `crates/aura-cli/tests/cli_run.rs` (uses the established `Command::new(BIN)` + - `temp_cwd` harness): - -```rust -/// Property: `aura mc --strategy stage1-r` runs the stage1-r walk-forward on -/// synthetic data, bootstraps the pooled OOS per-trade R series, and prints exactly -/// one canonical `mc_r_bootstrap` line carrying an `e_r` distribution block and -/// `prob_le_zero`. Deterministic: a second run with the same (default) seed is -/// byte-identical (C1). Frictionless Stage-1 — no costs, no `runs/` family write. -#[test] -fn mc_strategy_stage1_r_prints_a_bootstrap_line_deterministically() { - let dir = temp_cwd("mc_stage1r_boot"); - let run = || { - Command::new(BIN) - .args(["mc", "--strategy", "stage1-r", "--resamples", "200"]) - .current_dir(&dir) - .output() - .expect("spawn aura mc --strategy stage1-r") - }; - let out = run(); - assert!(out.status.success(), "exit: {:?} stderr: {}", out.status, String::from_utf8_lossy(&out.stderr)); - let stdout = String::from_utf8(out.stdout).expect("utf-8 stdout"); - assert_eq!(stdout.lines().count(), 1, "exactly one mc_r_bootstrap line: {stdout:?}"); - let v: serde_json::Value = serde_json::from_str(stdout.trim()).expect("canonical JSON"); - assert!(v["mc_r_bootstrap"]["e_r"]["mean"].is_number(), "e_r block present: {stdout}"); - assert!(v["mc_r_bootstrap"]["prob_le_zero"].is_number(), "prob_le_zero present: {stdout}"); - assert_eq!(v["mc_r_bootstrap"]["n_resamples"], 200); - - let out2 = run(); - assert_eq!(out.stdout, out2.stdout, "same seed -> byte-identical line (C1)"); - let _ = std::fs::remove_dir_all(&dir); -} -``` - -- [ ] **Step 2: Run to verify it fails** - -(Without Task 2 this would fail; with Tasks 1-2 already applied in the same iter, -this verifies the wired path.) - -Run: `cargo test --workspace mc_strategy_stage1_r_prints_a_bootstrap_line_deterministically` -Expected: PASS once Tasks 1-2 are in the tree (the implementer applies tasks in -order; if run before, FAIL with exit 2 / no `mc_r_bootstrap` line). - -- [ ] **Step 3: Refresh the stale doc-comment** — the - `mc_rejects_real_flag_with_usage_exit_2` assertions stay (a bare `--real` is still - rejected), but its rationale prose (1298-1304) overclaims that mc carves out the - real path entirely. Replace that doc-comment block with: - -```rust -/// The synthetic seed-resweep `mc` is undefined over real bars (one realization -> -/// identical members), so a bare `aura mc --real EURUSD` — with no R candidate — is -/// REJECTED (usage on stderr, exit 2) at the binary boundary. The real path is -/// reachable ONLY via `--strategy stage1-r` (the R-bootstrap over the pooled OOS R -/// series, `mc_strategy_stage1_r_prints_a_bootstrap_line_deterministically`); a -/// `--real` without it stays a usage error. NOT gated — the refusal precedes any -/// data access, so it is CI-safe on every machine. -``` - -- [ ] **Step 4: Full-suite regression gate** - -Run: `cargo test --workspace` -Expected: PASS — the new engine + parse + E2E tests green; all pre-existing goldens -(bare `aura mc`, `aura walkforward` SMA, stage1-r single-run, C18 round-trips) -green. - -Run: `cargo clippy --workspace --all-targets -- -D warnings` -Expected: 0 warnings. - ---- - -## Self-review - -- **Spec coverage:** spec Change 2 (`RBootstrap`/`r_bootstrap` — Task 1) and Change 4 - (`parse_mc_args`/`McArgs`/`run_mc_r_bootstrap`/dispatch/USAGE — Task 2; E2E + - doc-refresh — Task 3). Both in-scope sections covered. ✓ -- **Placeholder scan:** no TBD/TODO/"similar to". ✓ -- **Type/name consistency:** `RBootstrap`, `r_bootstrap`, `McArgs`, - `parse_mc_args`, `run_mc_r_bootstrap`, `mc_r_bootstrap_json`, `Strategy::Stage1R`, - `Stage1RGrid`, `DataChoice` used identically across tasks; the pooled field path - is `w.run.oos_report.metrics.r.trade_rs` (per recon, not the spec's abbreviated - `w.oos_report...`). ✓ -- **Step granularity:** each step is one edit or one command. ✓ -- **No commit steps.** ✓ -- **Compile-gate ordering (rule 7):** Task 1 is purely additive (new symbol + its - re-export + tests) — no caller breakage. Task 2 replaces the dispatch arms and - adds `parse_mc_args`/`McArgs`/`run_mc_r_bootstrap` together (the arm references - both, so they land in one task) before its build gate; the only `run_mc` caller - becomes the `McArgs::Synthetic` branch, threaded in the same task. ✓ -- **Filter strings (rule 8):** every `cargo test` filter names a real test — - pre-existing (`mc_runs_persists_a_monte_carlo_family_and_lists_it`, - `mc_rejects_real_flag_with_usage_exit_2`) or introduced in the same task; the - final gate is the unfiltered `cargo test --workspace`. ✓ -- **mc-R restriction (derived, recorded on #139):** the R path accepts only - `--strategy stage1-r`; a bare `--real` stays rejected, so - `mc_rejects_real_flag_with_usage_exit_2`'s assertions are preserved (only its - prose is refreshed). ✓ diff --git a/docs/specs/0075-walkforward-mc-r-validation.md b/docs/specs/0075-walkforward-mc-r-validation.md deleted file mode 100644 index f20c8fa..0000000 --- a/docs/specs/0075-walkforward-mc-r-validation.md +++ /dev/null @@ -1,359 +0,0 @@ -# Walk-forward + Monte-Carlo OOS validation for R candidates — Design Spec - -**Date:** 2026-06-26 -**Status:** Draft — awaiting user spec review -**Authors:** orchestrator + Claude -**Issue:** #139 (reference issue; design forks recorded in its comment thread) - -## Goal - -Make the World layer's two out-of-sample validation verbs work on an -**R-based candidate** on real data, not just the SMA sample blueprint: - -1. **`aura walkforward --strategy stage1-r --real `** — roll IS→OOS windows, - sweep the stage1-r grid in-sample, pick the best by an **R metric**, run the - chosen params out-of-sample, and report **per-window and across-window-pooled - OOS R-metrics** (the `r` block), not pip-only metrics. -2. **`aura mc --strategy stage1-r --real `** — bootstrap the walk-forward's - pooled OOS per-trade R series to a confidence interval on `E[R]`, so a positive - OOS edge can be shown (un)likely under resampling. - -Both are **frictionless Stage-1 R** (C7): costs are Stage-2, out of scope here. -The existing SMA/`total_pips` `walkforward` path and the synthetic-seed `mc` path -are **preserved byte-for-byte** (additive change; `--strategy`/`--real` select the -new behaviour, their absence keeps today's). - -Non-goals (recorded on #139): no genericisation of the orchestration over a metric -type (that is #136, which absorbs these call sites later); no cross-*symbol* -pooling (needs a multi-symbol CLI grammar — deferred follow-on); no costs. - -## Architecture - -The library substrate is already candidate- and metric-generic and stays -unchanged in signature: - -- `walk_forward(roller, space, run_window)` (`aura-engine/src/walkforward.rs:185`) - is generic over the `run_window` closure; it rolls bounds, runs each window - disjointly (C1), and stitches the OOS pip-equity. **No change.** -- `WindowRoller::new(span, is_len, oos_len, step, RollMode)` - (`walkforward.rs:73`). **No change.** -- `optimize(family, metric)` / `metric_cmp` (`aura-registry/src/lib.rs:186/125`) - already resolve the R metric names (`sqn`, `sqn_normalized`, `expectancy_r`, - `net_expectancy_r`), sorting an `r: None` member to `NEG_INFINITY`. **No change.** -- `monte_carlo(base, seeds, run_one)` (`mc.rs:122`) is MC-over-seeds — the **wrong - axis** for this work; it is left untouched for the synthetic path. - -Two library changes, one CLI surface change: - -- **`RMetrics` gains an in-memory-only per-trade R vector** (`report.rs:45`), - populated by `summarize_r`, excluded from serde and equality (fork A on #139). -- **A new deterministic block-bootstrap primitive** `r_bootstrap` in - `aura-engine/src/mc.rs`, over a `&[f64]` R series (fork on #139). -- **`walkforward` and `mc` become strategy-selectable** in `aura-cli/src/main.rs`, - dispatching the per-window/per-draw work onto the stage1-r blueprint and folding - via `summarize_r`; the stage1-r IS sweep uses reduce-mode folded sinks, the OOS - run uses raw recorders (fork B on #139). - -The varying dimension of `walk_forward` is the data window (C12 axis 3); MC here is -not axis 4 (seeds) but a post-hoc resample of an existing R sample. - -## Concrete code shapes - -### User-facing surface (the acceptance-criterion evidence) - -A researcher validating a candidate runs exactly: - -```console -$ aura walkforward --strategy stage1-r --real USDJPY --from 1672531200000 --to 1735689599000 -{"family_id":"wf-...","report":{...,"metrics":{"total_pips":...,"max_drawdown":..., - "bias_sign_flips":...,"r":{"expectancy_r":0.12,"sqn_normalized":0.34,"n_trades":21,...}}}} -... one line per OOS window, each carrying the r block ... -{"walkforward":{"windows":21,"stitched_total_pips":-171.7, - "oos_r":{"expectancy_r":-0.04,"sqn_normalized":-0.11,"n_trades":438,...}, - "param_stability":[...]}} -``` - -```console -$ aura mc --strategy stage1-r --real USDJPY --resamples 2000 --block-len 5 --seed 1 -{"mc_r_bootstrap":{"n_trades":438,"block_len":5,"n_resamples":2000, - "e_r":{"mean":-0.04,"p5":-0.18,"p25":-0.10,"p50":-0.04,"p75":0.02,"p95":0.10}, - "prob_le_zero":0.71}} -``` - -The bare forms are unchanged: - -```console -$ aura walkforward # SMA sample, pip-only — byte-identical to today -$ aura mc # synthetic 3-seed resweep — byte-identical to today -$ aura mc --name foo # unchanged -``` - -What the criterion demands and this shows: the intended audience (a researcher -proving OOS edge) reaches for these two commands directly; the output is the -instrument-agnostic R block + a resampled `E[R]` CI; no cost/friction enters -(C7 Stage-1); determinism (C1) holds because the roller, the sweep, and the -seeded bootstrap are all pure. - -### Change 1 — `RMetrics` carries the per-trade R series (in-memory only) - -`crates/aura-engine/src/report.rs:45` - -```rust -// BEFORE -#[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] -pub struct RMetrics { - pub expectancy_r: f64, - // ... all existing fields ... - pub conviction_terciles_r: [f64; 3], -} - -// AFTER — `trade_rs` is excluded from serde (skip) AND equality, so the C18 wire -// shape and every existing RMetrics/RunReport equality assertion are unchanged. -// It is a pure in-memory conduit: summarize_r already builds this vector (the -// local `rs`, report.rs:137) and drops it; now it is retained for the bootstrap. -#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] -pub struct RMetrics { - pub expectancy_r: f64, - // ... all existing fields ... - pub conviction_terciles_r: [f64; 3], - #[serde(skip)] - pub trade_rs: Vec, // realised R per closed trade, in trade order; empty after deserialize -} - -// PartialEq hand-written to ignore `trade_rs` (so a serialize->deserialize -// round-trip still compares equal, and the all-zero `RMetrics` literal in -// summarize_r's empty-input arm needs no R-vector to match). -impl PartialEq for RMetrics { - fn eq(&self, o: &Self) -> bool { - self.expectancy_r == o.expectancy_r - && /* ... all existing fields, trade_rs excluded ... */ - && self.conviction_terciles_r == o.conviction_terciles_r - } -} -``` - -`summarize_r` (`report.rs:89`) sets `trade_rs: rs.clone()` (or moves `rs`) into the -returned `RMetrics` on the non-empty path; the empty-input arm sets -`trade_rs: Vec::new()`. - -### Change 2 — the block-bootstrap primitive - -`crates/aura-engine/src/mc.rs` (new function + result struct, reusing the existing -`MetricStats`/`quantile`) - -```rust -/// Distribution of E[R] under a moving-block bootstrap of an OOS per-trade R -/// series. `block_len == 1` is the i.i.d. trade-shuffle; `block_len > 1` preserves -/// the serial correlation of sequential trades (winner/loser streaks), which a -/// pure shuffle would erase and so overstate confidence. Deterministic (C1) given -/// `seed` via the existing SplitMix64. -#[derive(Clone, Debug, PartialEq)] -pub struct RBootstrap { - pub e_r: MetricStats, // mean + p5/p25/p50/p75/p95 of the resampled E[R] - pub prob_le_zero: f64, // fraction of resamples whose mean R <= 0 - pub n_trades: usize, - pub block_len: usize, - pub n_resamples: usize, -} - -/// Moving-block bootstrap (non-circular; the final block is truncated so each -/// resample has exactly `n` trades). Empty `rs` -> all-zero RBootstrap. `block_len` -/// is clamped to `[1, n]`. -pub fn r_bootstrap(rs: &[f64], n_resamples: usize, block_len: usize, seed: u64) -> RBootstrap { - let n = rs.len(); - // n == 0 guard -> all-zero result; block_len = block_len.clamp(1, n.max(1)). - let mut rng = SplitMix64::new(seed); - let mut means = Vec::with_capacity(n_resamples); - for _ in 0..n_resamples { - let mut sample = Vec::with_capacity(n); - while sample.len() < n { - let start = (rng.next_u64() % (n - block_len + 1) as u64) as usize; - for j in start..(start + block_len).min(start + (n - sample.len())) { - sample.push(rs[j]); - } - } - means.push(sample.iter().sum::() / n as f64); - } - RBootstrap { - e_r: MetricStats::from_values(&means), - prob_le_zero: means.iter().filter(|&&m| m <= 0.0).count() as f64 / n_resamples as f64, - n_trades: n, block_len, n_resamples, - } -} -``` - -`SplitMix64` (`harness.rs:139`) is the existing deterministic RNG; if not already -crate-visible, it is promoted to `pub(crate)` (the planner's mechanic). - -### Change 3 — `walkforward` strategy dispatch - -`crates/aura-cli/src/main.rs` — `parse_walkforward_args` gains `--strategy` + the -four grid flags (mirroring `parse_sweep_args:1480`); `walkforward_family` branches -on the strategy. - -```rust -// BEFORE (main.rs:1526) -fn parse_walkforward_args(rest: &[&str]) -> Result<(String, bool, DataChoice), String> { ... } - -// AFTER — same RealWindowGrammar + name/trace handling, plus --strategy and the -// stage1-r grid flags reusing parse_csv_list + Stage1RGrid (verbatim from sweep). -fn parse_walkforward_args( - rest: &[&str], -) -> Result<(Strategy, String, bool, DataChoice, Stage1RGrid), String> { ... } -``` - -```rust -// BEFORE (main.rs:1643) — hardwired SMA, optimize by total_pips, summarize (r:None) -fn walkforward_family(trace: Option<&str>, data: &DataSource) -> WalkForwardResult { - let space = sample_blueprint_with_sinks(data.pip_size()).0.param_space(); - walk_forward(roller, space, |w| { - let is_family = sweep_over(w.is.0, w.is.1, data); - let best = optimize(&is_family, "total_pips").expect(...); - let (oos_equity, oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data); - WindowRun { chosen_params: best.params, oos_equity, oos_report } - }) -} - -// AFTER — strategy-dispatched. SMA arm is verbatim today's code (goldens hold). -fn walkforward_family( - strategy: Strategy, trace: Option<&str>, data: &DataSource, grid: &Stage1RGrid, -) -> WalkForwardResult { - match strategy { - Strategy::SmaCross => { /* exactly today's body */ } - Strategy::Stage1R => { - let space = stage1_r_space(); // param_space of the open stage1_r_graph - walk_forward(roller, space, |w| { - let is_family = stage1_r_sweep_over(w.is.0, w.is.1, data, grid); // reduce-mode fold - let best = optimize(&is_family, "sqn_normalized").expect("known R metric"); - let (oos_equity, oos_report) = run_oos_r(&best.params, w.oos.0, w.oos.1, trace, data); - WindowRun { chosen_params: best.params, oos_equity, oos_report } - }) - } - // other stage1 strategies: out of scope this cycle -> usage error or SmaCross-style guard - _ => { /* return a clear "strategy has no walkforward form" error, like run_dispatch */ } - } -} -``` - -New helpers (windowed siblings of the SMA `sweep_over`/`run_oos`, reusing the -stage1-r machinery at `main.rs:1180`/`2112`): - -- `stage1_r_sweep_over(from, to, data, grid) -> SweepFamily` — the in-sample - stage1-r sweep over `data.windowed_sources(from, to)`, **reduce-mode** (folded - `SeriesReducer`/`GatedRecorder`, O(trades)/member, `m.r = Some(summarize_r(..))`) - so the per-window sweep footprint stays bounded. -- `run_oos_r(params, from, to, trace, data) -> (Vec<(Timestamp,f64)>, RunReport)` — - the chosen-params OOS run over the windowed sources, **non-reduce** raw recorders - (the pip-equity curve is needed for `stitch`); folds `summarize` for the pip - block and `summarize_r` for `metrics.r` (which now carries `trade_rs`). - -`walkforward_summary_json` (`main.rs:1734`) gains an `oos_r` block: the pooled -across-windows R-metrics, computed by concatenating each window's -`oos_report.metrics.r.trade_rs` in roll order and folding (the pooled trade series -→ `summarize`-style R reduction, or a thin `RMetrics`-from-`trade_rs` reducer). The -SMA path emits no `oos_r` (its windows have `r: None`), so its summary line is -unchanged. - -### Change 4 — `mc` strategy dispatch + bootstrap - -`crates/aura-cli/src/main.rs` — replace the three literal `mc` dispatch arms with a -single `["mc", rest @ ..]` routed through a new `parse_mc_args`. - -```rust -// BEFORE (main.rs:2646) -["mc"] => run_mc("mc", false), -["mc", "--name", n] => run_mc(n, false), -["mc", "--trace", n] => run_mc(n, true), - -// AFTER -["mc", rest @ ..] => match parse_mc_args(rest) { - Ok(McArgs::Synthetic { name, persist }) => run_mc(&name, persist), // today's path, unchanged - Ok(McArgs::RealR { strategy, choice, grid, block_len, n_resamples, seed }) => - run_mc_r_bootstrap(strategy, DataSource::from_choice(choice), &grid, block_len, n_resamples, seed), - Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); } -}, -``` - -`run_mc_r_bootstrap` reuses `walkforward_family(strategy, None, &data, grid)` to -obtain the pooled OOS R series, then `r_bootstrap(&pooled, n_resamples, block_len, -seed)`, and prints one `mc_r_bootstrap` JSON line (shape above). Defaults: -`--resamples 1000`, `--block-len 1`, `--seed 1` (recorded; derivable). - -### Change 5 — USAGE - -`main.rs:2591` — extend the `walkforward` and `mc` fragments with -`[--strategy <...>] [--fast ] ...` and (for mc) `[--block-len ] -[--resamples ] [--seed ]`, matching the new grammars. - -## Components - -| Component | Location | Change | -|---|---|---| -| `RMetrics.trade_rs` + hand-written `PartialEq` | `aura-engine/report.rs:45` | new in-memory field, serde+eq excluded | -| `summarize_r` | `aura-engine/report.rs:89` | retain `rs` into `trade_rs` (was dropped) | -| `r_bootstrap` + `RBootstrap` | `aura-engine/mc.rs` | new deterministic moving-block bootstrap | -| `SplitMix64` visibility | `aura-engine/harness.rs:139` | `pub(crate)` if needed | -| `parse_walkforward_args` | `aura-cli/main.rs:1526` | `--strategy` + grid flags | -| `walkforward_family` | `aura-cli/main.rs:1643` | strategy dispatch; SMA arm verbatim | -| `stage1_r_sweep_over`, `run_oos_r`, `stage1_r_space` | `aura-cli/main.rs` (new) | windowed reduce-mode IS sweep + non-reduce OOS run | -| `walkforward_summary_json` | `aura-cli/main.rs:1734` | `oos_r` pooled block (stage1-r only) | -| `parse_mc_args` + `McArgs` + `run_mc_r_bootstrap` | `aura-cli/main.rs` (new) | mc real-candidate path | -| `mc` dispatch arms, USAGE | `aura-cli/main.rs:2646/2591` | single `["mc", rest @ ..]`, usage text | - -## Data flow - -1. **walk-forward (stage1-r):** roller → per window: windowed IS sources → - stage1-r reduce-mode sweep → `optimize(.., "sqn_normalized")` → chosen params → - windowed OOS run (non-reduce) → `(oos_equity, RunReport{metrics.r{..,trade_rs}})` - → `WindowRun`. `walk_forward` stitches pip-equity; summary folds the pooled - `trade_rs` into `oos_r`. -2. **mc (stage1-r):** run (1) → concat `windows[*].oos_report.metrics.r.trade_rs` - in roll order → `r_bootstrap` → `RBootstrap` → one JSON line. - -No look-ahead (C2: `WindowRoller` guarantees `oos.0 > is.1`). Disjoint windows run -in parallel (C1). The bootstrap is a pure post-run reduction (C12). - -## Error handling - -- Malformed grid list / unknown strategy / flag without value → the subcommand - `usage()` string (strict, like `parse_sweep_args`). -- A strategy with no walk-forward/mc form (e.g. `momentum`) → a clear error string - (mirroring `run_dispatch`'s `HarnessKind` guards), not a panic. -- Span too short for one IS+OOS window → today's `WindowRoller::new` error path - (`main.rs:1646` exit 2), unchanged. -- Empty pooled R series (no OOS trades) → `r_bootstrap` returns an all-zero - `RBootstrap` (well-defined), not a panic. - -## Testing strategy - -- **Goldens preserved (add, don't break):** existing `aura walkforward` (SMA) and - `aura mc` (synthetic) output goldens must stay byte-identical — covered by - running the bare forms; the SMA `walkforward_family` arm and `mc_family` are - untouched. C18 `runs.jsonl` shape unchanged (`trade_rs` is `serde(skip)`). -- **`r_bootstrap` unit goldens:** fixed `rs` + fixed seed → pinned `RBootstrap` - quantiles; `block_len = 1` vs `> 1` both deterministic; `block_len = n` (one - block) → every resample equals the full series mean; empty `rs` → all-zero. -- **`RMetrics` round-trip:** serialize → deserialize → `PartialEq` still equal - (the hand-written `eq` ignores `trade_rs`), pinning fork-A back-compat. -- **walk-forward stage1-r E2E:** `aura walkforward --strategy stage1-r --real ` - over a small fixed window emits per-window `r` blocks and an `oos_r` summary; - determinism (same input → same output) pinned. -- **mc stage1-r E2E:** `aura mc --strategy stage1-r --real ` emits a - `mc_r_bootstrap` line; same seed → same line. -- **arg-parsing units:** `parse_walkforward_args` / `parse_mc_args` accept the new - flags and reject malformed input, mirroring `parse_sweep_args` unit tests. - -## Acceptance criteria - -1. `aura walkforward --strategy stage1-r --real ` reports per-window and - pooled OOS **R-metrics** (the `r` block + `oos_r` summary), not pip-only. -2. `aura mc --strategy stage1-r --real ` reports a bootstrap `E[R]` - distribution / CI over the pooled OOS per-trade R series. -3. The four stage1-r grid flags (`--fast/--slow/--stop-length/--stop-k`) drive the - per-window IS sweep, optimised by an R metric. -4. Bare `aura walkforward` and `aura mc` (and `--name`/`--trace`) are - byte-identical to today; C18 wire shape unchanged. -5. Frictionless Stage-1 R: no cost enters the path. Determinism (C1) holds: - same input + same seed → same output. -6. Per-member walk-forward footprint stays O(trades) — the IS sweep folds - (reduce-mode), not O(cycles).