From 2e77ab1dda8d5802182176b7b3d590628cef0542 Mon Sep 17 00:00:00 2001 From: Brummel Date: Fri, 26 Jun 2026 12:12:06 +0200 Subject: [PATCH] feat(0075): Monte-Carlo R-bootstrap for OOS validation (iter 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `aura mc --strategy stage1-r [--real ]`: run the stage1-r walk-forward, pool its OOS per-trade R series, and report a moving-block bootstrap E[R] distribution / CI (one `mc_r_bootstrap` line with an `e_r` quantile block + P(E[R]<=0)). Completes spec 0075's Monte-Carlo half on top of iter 1's walk-forward R-reporting. Engine: `r_bootstrap(rs, n_resamples, block_len, seed) -> RBootstrap` — a deterministic moving-block bootstrap (non-circular, final block truncated; block_len=1 = i.i.d. trade-shuffle; clamped to [1,n]; empty -> all-zero), reusing the existing MetricStats/quantile and SplitMix64. CLI: parse_mc_args/McArgs split — bare/`--name`/`--trace` route to the unchanged synthetic seed-resweep `run_mc` (byte-for-byte preserved); the R path accepts ONLY `--strategy stage1-r`, so a bare `--real` stays a usage error (mc_rejects_real_flag_with_usage_exit_2 preserved, doc-comment refreshed). run_mc_r_bootstrap reuses walkforward_family + RMetrics.trade_rs. Corrective fixes over the plan snippets, both verified: the mc usage string carries a `usage:` token (the new `["mc", rest @ ..]` arm now handles the `--real` rejection, and the preserved golden asserts `stderr.contains ("usage")`); `#[allow(clippy::large_enum_variant)]` on McArgs keeps the unboxed Stage1RGrid field layout. Scope: single `--real ` per invocation; cross-symbol pooling is a deferred follow-on (needs a multi-symbol grammar). Verified: cargo build clean, full `cargo test --workspace` green (609 passed), clippy -D warnings clean; bare `aura mc`, SMA walkforward, stage1-r single-run, and C18 round-trip goldens all preserved. refs #139 --- crates/aura-cli/src/main.rs | 229 +++++++++++++++++++++++++++++-- crates/aura-cli/tests/cli_run.rs | 115 +++++++++++++++- crates/aura-engine/src/lib.rs | 2 +- crates/aura-engine/src/mc.rs | 146 ++++++++++++++++++++ 4 files changed, 471 insertions(+), 21 deletions(-) diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index 2881822..3f2a8a8 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -17,11 +17,11 @@ use render::{ChartData, ChartMeta, ChartMode, ReduceKind, Series}; use aura_core::{zip_params, Cell, Firing, ParamSpec, Scalar, ScalarKind, Timestamp}; use aura_composites::{risk_executor, risk_executor_vol_open, StopRule}; use aura_engine::{ - f64_field, join_on_ts, monte_carlo, param_stability, r_metrics_from_rs, summarize, summarize_r, - walk_forward, window_of, ColumnarTrace, Composite, Edge, FlatGraph, GraphBuilder, Harness, - JoinedRow, McAggregate, McFamily, RollMode, RunManifest, RunMetrics, RunReport, SourceSpec, - SweepFamily, SweepPoint, SyntheticSpec, Target, VecSource, WalkForwardResult, WindowBounds, - WindowRoller, WindowRun, + f64_field, join_on_ts, monte_carlo, param_stability, r_bootstrap, r_metrics_from_rs, summarize, + summarize_r, walk_forward, window_of, ColumnarTrace, Composite, Edge, FlatGraph, GraphBuilder, + Harness, JoinedRow, McAggregate, McFamily, RBootstrap, RollMode, RunManifest, RunMetrics, + RunReport, SourceSpec, SweepFamily, SweepPoint, SyntheticSpec, Target, VecSource, + WalkForwardResult, WindowBounds, WindowRoller, WindowRun, }; use aura_registry::{ group_families, mc_member_reports, optimize, rank_by, sweep_member_reports, @@ -1885,15 +1885,24 @@ fn run_oos( (equity, report) } +/// Pool every OOS window's per-trade R series into one flat vector, in roll order +/// (window order, then within-window trade order). Windows with no `r` block +/// contribute nothing. The single home of the pooling-in-roll-order semantics — +/// both the walk-forward `oos_r` summary and the `mc` R-bootstrap reduce this. +fn pooled_oos_trade_rs(result: &WalkForwardResult) -> Vec { + result + .windows + .iter() + .flat_map(|w| w.run.oos_report.metrics.r.as_ref().map(|r| r.trade_rs.clone()).unwrap_or_default()) + .collect() +} + /// The walk-forward summary line: window count, stitched OOS total pips (the last /// stitched-curve value), and the on-demand per-param stability. Canonical JSON /// (C14). fn walkforward_summary_json(result: &WalkForwardResult) -> String { let total = result.stitched_oos_equity.last().map(|&(_, v)| v).unwrap_or(0.0); - // pool the per-window OOS per-trade R series in roll order, then reduce. - let pooled_rs: Vec = result.windows.iter() - .flat_map(|w| w.run.oos_report.metrics.r.as_ref().map(|r| r.trade_rs.clone()).unwrap_or_default()) - .collect(); + let pooled_rs = pooled_oos_trade_rs(result); let mut obj = serde_json::json!({ "windows": result.windows.len(), "stitched_total_pips": total, @@ -2022,6 +2031,110 @@ fn run_mc(name: &str, persist: bool) { println!("{}", mc_aggregate_json(&family.aggregate)); } +/// Parsed form of the `mc` tail: either the synthetic seed-resweep (today's path, +/// preserved byte-for-byte) or the real-candidate R-bootstrap. The R path accepts +/// ONLY `--strategy stage1-r` (the sole R-reporting walk-forward strategy); a bare +/// `--real` without it is a usage error (the synthetic seed-resweep is undefined +/// over real bars). +// Parsed once at the dispatch boundary and immediately destructured into the run +// call; never stored or collected, so the inter-variant size gap is free here. +#[allow(clippy::large_enum_variant)] +#[derive(Clone, Debug, PartialEq)] +enum McArgs { + Synthetic { name: String, persist: bool }, + RealR { choice: DataChoice, grid: Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64 }, +} + +fn parse_mc_args(rest: &[&str]) -> Result { + let usage = || "usage: mc [--name |--trace ] | mc --strategy stage1-r [--real [--from ] [--to ]] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--block-len ] [--resamples ] [--seed ]".to_string(); + let mut strategy: Option = None; + let mut name: Option<(String, bool)> = None; + let mut real = RealWindowGrammar::default(); + let mut grid = Stage1RGrid::default(); + let mut block_len: usize = 1; + let mut n_resamples: usize = 1000; + let mut seed: u64 = 1; + let mut r_path = false; + let mut tail = rest; + while let Some((flag, t)) = tail.split_first() { + let (value, t) = t.split_first().ok_or_else(usage)?; + if real.accept(flag, value, &usage)? { + r_path = true; + tail = t; + continue; + } + match *flag { + "--strategy" => { + strategy = Some(match *value { + "stage1-r" => Strategy::Stage1R, + _ => return Err(usage()), + }); + r_path = true; + } + "--name" if name.is_none() => name = Some(((*value).to_string(), false)), + "--trace" if name.is_none() => name = Some(((*value).to_string(), true)), + "--fast" => { grid.fast = parse_csv_list(value).map_err(|()| usage())?; r_path = true; } + "--slow" => { grid.slow = parse_csv_list(value).map_err(|()| usage())?; r_path = true; } + "--stop-length" => { grid.stop_length = parse_csv_list(value).map_err(|()| usage())?; r_path = true; } + "--stop-k" => { grid.stop_k = parse_csv_list(value).map_err(|()| usage())?; r_path = true; } + "--block-len" => { block_len = value.parse().map_err(|_| usage())?; r_path = true; } + "--resamples" => { n_resamples = value.parse().map_err(|_| usage())?; r_path = true; } + "--seed" => { seed = value.parse().map_err(|_| usage())?; r_path = true; } + _ => return Err(usage()), + } + tail = t; + } + if r_path { + if strategy != Some(Strategy::Stage1R) { + return Err(usage()); + } + if name.is_some() { + return Err(usage()); + } + let choice = real.finish(&usage)?; + Ok(McArgs::RealR { choice, grid, block_len, n_resamples, seed }) + } else { + let (name, persist) = name.unwrap_or_else(|| ("mc".to_string(), false)); + Ok(McArgs::Synthetic { name, persist }) + } +} + +/// `aura mc --strategy stage1-r [--real ]`: run the stage1-r walk-forward, pool +/// every OOS window's per-trade R series in roll order, and print one moving-block +/// bootstrap `mc_r_bootstrap` line (E[R] distribution + P(E[R] <= 0)). Frictionless +/// Stage-1 (no costs); deterministic given `seed` (C1). +fn run_mc_r_bootstrap(data: DataSource, grid: &Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64) { + println!("{}", mc_r_bootstrap_report(&data, grid, block_len, n_resamples, seed)); +} + +/// Assemble the `mc` R-bootstrap line: run the stage1-r walk-forward, pool the OOS +/// per-trade R series in roll order, bootstrap E[R], and render the `mc_r_bootstrap` +/// line. The body of `run_mc_r_bootstrap` minus the `println!`, so the full real-R +/// wiring (walk-forward -> non-empty pooling -> bootstrap -> render) is reachable +/// over synthetic data in a `#[cfg(test)]` unit, mirroring `mc_report` / +/// `walkforward_report` / `sweep_report`. +fn mc_r_bootstrap_report(data: &DataSource, grid: &Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64) -> String { + let result = walkforward_family(Strategy::Stage1R, None, data, grid); + let pooled = pooled_oos_trade_rs(&result); + let boot = r_bootstrap(&pooled, n_resamples, block_len, seed); + mc_r_bootstrap_json(&boot) +} + +/// Render an `RBootstrap` as one canonical JSON line (`MetricStats` serializes; the +/// scalar fields are spliced in), mirroring `mc_aggregate_json`. +fn mc_r_bootstrap_json(b: &RBootstrap) -> String { + serde_json::json!({ + "mc_r_bootstrap": { + "n_trades": b.n_trades, + "block_len": b.block_len, + "n_resamples": b.n_resamples, + "e_r": b.e_r, + "prob_le_zero": b.prob_le_zero, + } + }) + .to_string() +} + /// Render the built-in Monte-Carlo family as the per-draw `RunReport` lines plus /// the aggregate line — the `run_mc` shape minus registry persistence (no /// `family_id`, which is store-assigned). Test helper, mirroring `sweep_report` / @@ -2754,7 +2867,7 @@ fn run_dispatch(args: RunArgs) -> Result { } const USAGE: &str = - "usage: aura run [--harness ] [--real [--from ] [--to ]] [--trace ] | aura chart [--tap ] [--panels] | aura graph | aura sweep [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] | aura mc [--name |--trace ] | aura walkforward [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] | aura runs families | aura runs family [rank ]"; + "usage: aura run [--harness ] [--real [--from ] [--to ]] [--trace ] | aura chart [--tap ] [--panels] | aura graph | aura sweep [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] | aura mc [--name |--trace ] | aura mc --strategy stage1-r [--real [--from ] [--to ]] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--block-len ] [--resamples ] [--seed ] | aura walkforward [--strategy ] [--real [--from ] [--to ]] [--name |--trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] | aura runs families | aura runs family [rank ]"; fn main() { // Collect argv and match the whole vector: every accepted form is exhaustive, @@ -2808,9 +2921,16 @@ fn main() { std::process::exit(2); } }, - ["mc"] => run_mc("mc", false), - ["mc", "--name", n] => run_mc(n, false), - ["mc", "--trace", n] => run_mc(n, true), + ["mc", rest @ ..] => match parse_mc_args(rest) { + Ok(McArgs::Synthetic { name, persist }) => run_mc(&name, persist), + Ok(McArgs::RealR { choice, grid, block_len, n_resamples, seed }) => { + run_mc_r_bootstrap(DataSource::from_choice(choice), &grid, block_len, n_resamples, seed) + } + Err(msg) => { + eprintln!("aura: {msg}"); + std::process::exit(2); + } + }, ["runs", "families"] => runs_families(), ["runs", "family", id] => runs_family(id, None), ["runs", "family", id, "rank", metric] => runs_family(id, Some(metric)), @@ -3925,4 +4045,87 @@ mod tests { assert!(parse_run_args(&["--real", "GER40", "--from"]).is_err()); // flag missing its value assert!(parse_run_args(&["bogus"]).is_err()); // unknown trailing token } + + #[test] + fn parse_mc_args_bare_and_name_route_to_synthetic() { + assert_eq!(parse_mc_args(&[]), Ok(McArgs::Synthetic { name: "mc".to_string(), persist: false })); + assert_eq!(parse_mc_args(&["--name", "m"]), Ok(McArgs::Synthetic { name: "m".to_string(), persist: false })); + assert_eq!(parse_mc_args(&["--trace", "m"]), Ok(McArgs::Synthetic { name: "m".to_string(), persist: true })); + } + + #[test] + fn parse_mc_args_stage1_r_routes_to_real_r_with_defaults_and_overrides() { + assert_eq!( + parse_mc_args(&["--strategy", "stage1-r"]), + Ok(McArgs::RealR { + choice: DataChoice::Synthetic, grid: Stage1RGrid::default(), + block_len: 1, n_resamples: 1000, seed: 1, + }) + ); + let r = parse_mc_args(&["--strategy", "stage1-r", "--real", "USDJPY", + "--block-len", "5", "--resamples", "200", "--seed", "9", "--fast", "5,10"]); + let McArgs::RealR { choice, grid, block_len, n_resamples, seed } = r.expect("valid real-R mc args") + else { panic!("expected RealR") }; + assert_eq!(choice, DataChoice::Real { symbol: "USDJPY".to_string(), from_ms: None, to_ms: None }); + assert_eq!((block_len, n_resamples, seed), (5, 200, 9)); + assert_eq!(grid.fast, vec![5, 10]); + } + + #[test] + fn parse_mc_args_real_without_stage1_r_is_usage_error() { + // a bare --real (no R candidate) is still rejected — the synthetic seed-resweep + // is undefined over real bars; the R path needs --strategy stage1-r. + assert!(parse_mc_args(&["--real", "EURUSD"]).is_err()); + assert!(parse_mc_args(&["--strategy", "sma"]).is_err()); + assert!(parse_mc_args(&["--strategy", "stage1-r", "--name", "x"]).is_err()); // name invalid on R path + } + + #[test] + fn mc_r_bootstrap_json_carries_every_bootstrap_field_under_the_mc_r_bootstrap_key() { + // Property: the `mc_r_bootstrap` output line is the full RBootstrap shape — + // each field is named and value-faithful, so a renamed/dropped field here + // breaks a test instead of silently shipping. Pins the user-visible wire + // shape of the mc R-bootstrap render (the parser + engine primitive are + // covered elsewhere; this is the output-shape layer). + let boot = r_bootstrap(&[1.0, -0.5, 2.0, -1.0], 64, 2, 7); + let line = mc_r_bootstrap_json(&boot); + let v: serde_json::Value = serde_json::from_str(&line).expect("canonical json line"); + let obj = &v["mc_r_bootstrap"]; + assert_eq!(obj["n_trades"], serde_json::json!(boot.n_trades)); + assert_eq!(obj["block_len"], serde_json::json!(boot.block_len)); + assert_eq!(obj["n_resamples"], serde_json::json!(boot.n_resamples)); + assert_eq!(obj["prob_le_zero"], serde_json::json!(boot.prob_le_zero)); + // e_r is the nested MetricStats block (mean + quantiles), not a flat scalar. + assert_eq!(obj["e_r"], serde_json::to_value(&boot.e_r).expect("MetricStats serializes")); + assert!(obj["e_r"]["mean"].is_number(), "e_r should nest the MetricStats block: {line}"); + } + + #[test] + fn mc_r_bootstrap_report_pools_a_non_empty_oos_r_series_over_synthetic() { + // Property: the full real-R assembly path — walkforward_family(Stage1R) -> + // pooled_oos_trade_rs -> r_bootstrap -> mc_r_bootstrap_json — wires up and + // reduces a NON-EMPTY pooled OOS R series (the synthetic stage1-r walk-forward + // closes >= 1 trade across its windows). Guards the wiring + the non-empty + // pooling branch the parser/primitive unit tests cannot reach; mirrors + // `mc_report` / `walkforward_report`. Deterministic (C1). + let result = walkforward_family(Strategy::Stage1R, None, &DataSource::Synthetic, &Stage1RGrid::default()); + let pooled = pooled_oos_trade_rs(&result); + assert!(!pooled.is_empty(), "synthetic stage1-r walk-forward must pool >= 1 OOS trade R"); + + let line = mc_r_bootstrap_report(&DataSource::Synthetic, &Stage1RGrid::default(), 1, 256, 1); + let v: serde_json::Value = serde_json::from_str(&line).expect("canonical json line"); + let obj = &v["mc_r_bootstrap"]; + // n_trades is the pooled-series length the bootstrap actually saw — non-zero + // proves the assembled pooling fed the primitive, not an empty fallback. + assert_eq!(obj["n_trades"], serde_json::json!(pooled.len())); + assert!(obj["n_trades"].as_u64().expect("n_trades is an integer") >= 1); + assert_eq!(obj["n_resamples"], serde_json::json!(256)); + assert!(obj["e_r"]["mean"].as_f64().expect("e_r.mean is a number").is_finite()); + + // C1 at the CLI edge: the assembled real-R line is byte-identical on re-run. + assert_eq!( + line, + mc_r_bootstrap_report(&DataSource::Synthetic, &Stage1RGrid::default(), 1, 256, 1), + ); + } } diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index ab76d44..d958a35 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -1294,13 +1294,12 @@ fn sweep_real_is_byte_deterministic_across_runs() { assert_eq!(run(), run(), "same real sweep twice must be byte-identical (C1)"); } -/// Property (spec-0060 MC exclusion): `aura mc` admits NO `--real` flag. The -/// spec carves MC out of the real-data path — its seed varies a *synthetic* -/// price-walk realization, undefined over real bars (one realization -> identical -/// members). The observable contract is that `aura mc --real EURUSD` is REJECTED -/// (usage on stderr, exit 2) at the binary boundary, never silently accepted as a -/// real run. This pins the exclusion: if an `["mc", rest @ ..]` arm ever wired -/// `--real` into MC, this test fails loudly. NOT gated — the refusal precedes any +/// The synthetic seed-resweep `mc` is undefined over real bars (one realization -> +/// identical members), so a bare `aura mc --real EURUSD` — with no R candidate — is +/// REJECTED (usage on stderr, exit 2) at the binary boundary. The real path is +/// reachable ONLY via `--strategy stage1-r` (the R-bootstrap over the pooled OOS R +/// series, `mc_strategy_stage1_r_prints_a_bootstrap_line_deterministically`); a +/// `--real` without it stays a usage error. NOT gated — the refusal precedes any /// data access, so it is CI-safe on every machine. #[test] fn mc_rejects_real_flag_with_usage_exit_2() { @@ -2407,3 +2406,105 @@ fn walkforward_unsupported_strategy_exits_2_naming_the_token() { assert!(out.stdout.is_empty(), "no family summary on the rejected path: {:?}", out.stdout); let _ = std::fs::remove_dir_all(&cwd); } + +/// Property: `aura mc --strategy stage1-r` runs the stage1-r walk-forward on +/// synthetic data, bootstraps the pooled OOS per-trade R series, and prints exactly +/// one canonical `mc_r_bootstrap` line carrying an `e_r` distribution block and +/// `prob_le_zero`. Deterministic: a second run with the same (default) seed is +/// byte-identical (C1). Frictionless Stage-1 — no costs, no `runs/` family write. +#[test] +fn mc_strategy_stage1_r_prints_a_bootstrap_line_deterministically() { + let dir = temp_cwd("mc_stage1r_boot"); + let run = || { + Command::new(BIN) + .args(["mc", "--strategy", "stage1-r", "--resamples", "200"]) + .current_dir(&dir) + .output() + .expect("spawn aura mc --strategy stage1-r") + }; + let out = run(); + assert!(out.status.success(), "exit: {:?} stderr: {}", out.status, String::from_utf8_lossy(&out.stderr)); + let stdout = String::from_utf8(out.stdout).expect("utf-8 stdout"); + assert_eq!(stdout.lines().count(), 1, "exactly one mc_r_bootstrap line: {stdout:?}"); + let v: serde_json::Value = serde_json::from_str(stdout.trim()).expect("canonical JSON"); + assert!(v["mc_r_bootstrap"]["e_r"]["mean"].is_number(), "e_r block present: {stdout}"); + assert!(v["mc_r_bootstrap"]["prob_le_zero"].is_number(), "prob_le_zero present: {stdout}"); + assert_eq!(v["mc_r_bootstrap"]["n_resamples"], 200); + + let out2 = run(); + assert_eq!(stdout.as_bytes(), out2.stdout.as_slice(), "same seed -> byte-identical line (C1)"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Property: `--block-len` is the moving-block knob, and it is observable AND +/// load-bearing at the binary boundary — it round-trips into the emitted +/// `block_len` field, and a different `block-len` (over the same seed + pooled R +/// series) yields a genuinely different E[R] distribution. This is what separates +/// the moving-block bootstrap from a plain i.i.d. trade shuffle: contiguous runs +/// preserve serial correlation a `block-len 1` shuffle erases. Without this test a +/// regression that dropped `--block-len` on the floor (or wired it to `block_len 1` +/// always) would still pass the happy-path E2E. Same-seed determinism (C1) pins +/// that the difference is the block length, not RNG noise. Not gated — synthetic. +#[test] +fn mc_stage1_r_block_len_is_observable_and_changes_the_distribution() { + let dir = temp_cwd("mc_stage1r_block"); + let run = |block_len: &str| { + let out = Command::new(BIN) + .args(["mc", "--strategy", "stage1-r", "--resamples", "256", "--seed", "1", "--block-len", block_len]) + .current_dir(&dir) + .output() + .expect("spawn aura mc --strategy stage1-r --block-len"); + assert!(out.status.success(), "exit: {:?} stderr: {}", out.status, String::from_utf8_lossy(&out.stderr)); + String::from_utf8(out.stdout).expect("utf-8 stdout") + }; + + let one = run("1"); + let three = run("3"); + + let v1: serde_json::Value = serde_json::from_str(one.trim()).expect("canonical JSON (block-len 1)"); + let v3: serde_json::Value = serde_json::from_str(three.trim()).expect("canonical JSON (block-len 3)"); + // the flag round-trips verbatim into the emitted line. + assert_eq!(v1["mc_r_bootstrap"]["block_len"], 1, "block_len 1 must echo: {one}"); + assert_eq!(v3["mc_r_bootstrap"]["block_len"], 3, "block_len 3 must echo: {three}"); + // same seed, same pooled series, different block length -> different E[R] spread. + // (a no-op `--block-len` would make these byte-identical.) + assert_ne!(one, three, "block-len must change the resampled distribution, not be ignored"); + + // C1 at the CLI edge: the block-len-1 line is byte-identical on re-run. + assert_eq!(one, run("1"), "same seed+block-len -> byte-identical line (C1)"); + let _ = std::fs::remove_dir_all(&dir); +} + +/// Property: a `--block-len` at or above the pooled-series length is CLAMPED to the +/// series length at the binary boundary (no panic, no out-of-bounds), and the +/// degenerate single-block case collapses the bootstrap — when every resample IS +/// the full series, the E[R] distribution becomes a single point (all quantiles +/// equal). The emitted `block_len` reports the clamped value, not the raw flag. +/// This pins the clamp the engine primitive guards in-crate, now at the observable +/// CLI edge where an off-by-one in the index arithmetic would otherwise surface as +/// a process crash. Not gated — synthetic stage1-r pools a fixed, non-empty series. +#[test] +fn mc_stage1_r_oversized_block_len_clamps_and_collapses() { + let dir = temp_cwd("mc_stage1r_clamp"); + let out = Command::new(BIN) + .args(["mc", "--strategy", "stage1-r", "--resamples", "200", "--seed", "1", "--block-len", "9999"]) + .current_dir(&dir) + .output() + .expect("spawn aura mc --strategy stage1-r --block-len 9999"); + assert!(out.status.success(), "must not crash on oversized block-len: {:?} stderr: {}", + out.status, String::from_utf8_lossy(&out.stderr)); + let stdout = String::from_utf8(out.stdout).expect("utf-8 stdout"); + let v: serde_json::Value = serde_json::from_str(stdout.trim()).expect("canonical JSON"); + let obj = &v["mc_r_bootstrap"]; + let n_trades = obj["n_trades"].as_u64().expect("n_trades is an integer"); + assert!(n_trades >= 1, "synthetic stage1-r must pool a non-empty R series: {stdout}"); + // block_len is reported clamped to the pooled-series length, never the raw 9999. + assert_eq!(obj["block_len"], serde_json::json!(n_trades), + "oversized block-len must clamp to n_trades: {stdout}"); + // single-block degenerate: every resample is the full series, so the E[R] + // distribution is a single point — p5 == p95 (zero spread). + let p5 = obj["e_r"]["p5"].as_f64().expect("p5 is a number"); + let p95 = obj["e_r"]["p95"].as_f64().expect("p95 is a number"); + assert!((p5 - p95).abs() < 1e-12, "single-block bootstrap must have zero spread: p5={p5} p95={p95}"); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/aura-engine/src/lib.rs b/crates/aura-engine/src/lib.rs index b076933..eb1fc8b 100644 --- a/crates/aura-engine/src/lib.rs +++ b/crates/aura-engine/src/lib.rs @@ -69,7 +69,7 @@ pub use report::{ pub use sweep::{ sweep, GridSpace, ParamRange, RandomSpace, Space, SweepError, SweepFamily, SweepPoint, }; -pub use mc::{monte_carlo, McAggregate, McDraw, McFamily, MetricStats}; +pub use mc::{monte_carlo, r_bootstrap, McAggregate, McDraw, McFamily, MetricStats, RBootstrap}; pub use walkforward::{ param_stability, walk_forward, RollMode, WalkForwardError, WalkForwardResult, WindowBounds, WindowOutcome, WindowRoller, WindowRun, diff --git a/crates/aura-engine/src/mc.rs b/crates/aura-engine/src/mc.rs index 3eb9c97..c43af5e 100644 --- a/crates/aura-engine/src/mc.rs +++ b/crates/aura-engine/src/mc.rs @@ -11,6 +11,7 @@ use crate::sweep::run_indexed; use crate::{RunMetrics, RunReport, Scalar}; +use crate::harness::SplitMix64; /// One Monte-Carlo realization: the seed that drove it and the full `RunReport`. /// Self-describing, analog to [`SweepPoint`](crate::SweepPoint). @@ -152,6 +153,51 @@ where McFamily { draws, aggregate } } +/// Distribution of E[R] under a moving-block bootstrap of an OOS per-trade R series +/// (#139). `block_len == 1` is the i.i.d. trade-shuffle; `block_len > 1` resamples +/// contiguous runs, preserving the serial correlation of sequential trades a pure +/// shuffle would erase. Deterministic (C1) given `seed`. +#[derive(Clone, Debug, PartialEq)] +pub struct RBootstrap { + pub e_r: MetricStats, // mean + p5/p25/p50/p75/p95 of the resampled E[R] + pub prob_le_zero: f64, // fraction of resamples whose mean R <= 0 + pub n_trades: usize, + pub block_len: usize, + pub n_resamples: usize, +} + +/// Moving-block bootstrap of `rs` (non-circular; the final block of each resample is +/// truncated so the resample has exactly `n` values). `block_len` is clamped to +/// `[1, n]`. Empty `rs` or zero resamples -> an all-zero `RBootstrap`. Pure given +/// `seed` (drives the existing `SplitMix64`). +pub fn r_bootstrap(rs: &[f64], n_resamples: usize, block_len: usize, seed: u64) -> RBootstrap { + let n = rs.len(); + if n == 0 || n_resamples == 0 { + return RBootstrap { + e_r: MetricStats { mean: 0.0, p5: 0.0, p25: 0.0, p50: 0.0, p75: 0.0, p95: 0.0 }, + prob_le_zero: 0.0, + n_trades: n, + block_len: block_len.clamp(1, n.max(1)), + n_resamples, + }; + } + let block_len = block_len.clamp(1, n); + let mut rng = SplitMix64::new(seed); + let mut means: Vec = Vec::with_capacity(n_resamples); + for _ in 0..n_resamples { + let mut sample: Vec = Vec::with_capacity(n); + while sample.len() < n { + let start = (rng.next_u64() % (n - block_len + 1) as u64) as usize; + let take = block_len.min(n - sample.len()); + sample.extend_from_slice(&rs[start..start + take]); + } + means.push(sample.iter().sum::() / n as f64); + } + let e_r = MetricStats::from_values(&means); + let prob_le_zero = means.iter().filter(|&&m| m <= 0.0).count() as f64 / n_resamples as f64; + RBootstrap { e_r, prob_le_zero, n_trades: n, block_len, n_resamples } +} + #[cfg(test)] mod tests { use super::*; @@ -312,4 +358,104 @@ mod tests { let back: MetricStats = serde_json::from_str(&json).expect("deserialize MetricStats"); assert_eq!(back, s); } + + #[test] + fn r_bootstrap_empty_series_is_all_zero() { + let b = r_bootstrap(&[], 100, 1, 7); + assert_eq!(b.n_trades, 0); + assert_eq!(b.e_r.mean, 0.0); + assert_eq!(b.prob_le_zero, 0.0); + } + + #[test] + fn r_bootstrap_single_block_equals_full_series_mean() { + // block_len == n: the only valid start is 0, so every resample IS the full + // series -> every resample mean == the series mean -> zero spread. + let rs = [1.0, -2.0, 3.0]; // mean = 2/3 + let b = r_bootstrap(&rs, 500, rs.len(), 7); + let mean = 2.0 / 3.0; + assert!((b.e_r.mean - mean).abs() < 1e-12); + assert!((b.e_r.p5 - mean).abs() < 1e-12); + assert!((b.e_r.p95 - mean).abs() < 1e-12); + assert_eq!(b.prob_le_zero, 0.0); // mean > 0 + assert_eq!(b.n_trades, 3); + assert_eq!(b.block_len, 3); + assert_eq!(b.n_resamples, 500); + } + + #[test] + fn r_bootstrap_is_deterministic_given_seed() { + let rs = [0.5, -1.0, 2.0, -0.5, 1.5, -2.0, 0.25]; + let a = r_bootstrap(&rs, 1000, 1, 42); + let b = r_bootstrap(&rs, 1000, 1, 42); + assert_eq!(a, b, "same rs+seed+params -> identical RBootstrap (C1)"); + let c = r_bootstrap(&rs, 1000, 1, 43); + assert_ne!(a.e_r.p5, c.e_r.p5, "a different seed reshuffles differently"); + } + + #[test] + fn r_bootstrap_block_len_is_clamped_to_series_len() { + // block_len > n clamps to n -> single-block behaviour (no panic, no OOB). + let rs = [1.0, 2.0]; + let b = r_bootstrap(&rs, 10, 99, 1); + assert_eq!(b.block_len, 2); + assert!((b.e_r.mean - 1.5).abs() < 1e-12); + } + + #[test] + fn r_bootstrap_moving_block_resamples_are_contiguous_truncated_runs() { + // The serial-correlation-preserving branch: 1 < block_len < n with n NOT a + // multiple of block_len, so the final block of every resample is truncated + // (`take = block_len.min(n - sample.len())`). Every resample is a sequence of + // contiguous runs of `rs` (the last one cut short to reach exactly n values), + // so each resample mean must lie in the finite set of legal block-composition + // means. Distinct powers of ten make every distinct multiset of contributions + // a distinct sum -> a faithful membership check. n=5, block_len=2 -> blocks of + // length 2, 2, 1. + let rs = [1.0, 10.0, 100.0, 1000.0, 10000.0]; + let n = rs.len(); + let block_len = 2; + + // Enumerate every legal resample sum: pick a start in [0, n-block_len] for each + // of the three blocks (lengths 2, 2, 1), summing min(block_len, n-filled) values + // from that start. Membership-by-sum, mirroring r_bootstrap's own construction. + let starts: Vec = (0..=n - block_len).collect(); + let mut legal_means: Vec = Vec::new(); + for &s0 in &starts { + for &s1 in &starts { + for &s2 in &starts { + let mut sum = 0.0; + let mut filled = 0usize; + for &start in &[s0, s1, s2] { + let take = block_len.min(n - filled); + if take == 0 { + break; + } + sum += rs[start..start + take].iter().sum::(); + filled += take; + } + legal_means.push(sum / n as f64); + } + } + } + + // Drive enough resamples that the truncated final block is exercised many + // times; assert every produced mean is a legal block-composition mean. + let b = r_bootstrap(&rs, 2000, block_len, 7); + assert_eq!(b.block_len, 2); + assert_eq!(b.n_trades, 5); + // The mean of resample means is a weighted average of legal means, so it is + // bounded by their min and max (not itself a single legal mean). + let lo = legal_means.iter().cloned().fold(f64::INFINITY, f64::min); + let hi = legal_means.iter().cloned().fold(f64::NEG_INFINITY, f64::max); + assert!(lo - 1e-6 <= b.e_r.mean && b.e_r.mean <= hi + 1e-6); + // Each quantile of the resampled E[R] is an order statistic of the resample + // means, hence itself a legal contiguous-block-composition mean. + for &m in &[b.e_r.p5, b.e_r.p25, b.e_r.p50, b.e_r.p75, b.e_r.p95] { + assert!( + legal_means.iter().any(|&lm| (lm - m).abs() < 1e-6), + "resampled quantile {m} is not a legal contiguous-block-composition mean", + ); + } + } }