feat(0075): Monte-Carlo R-bootstrap for OOS validation (iter 2)
Add `aura mc --strategy stage1-r [--real <SYM>]`: run the stage1-r
walk-forward, pool its OOS per-trade R series, and report a moving-block
bootstrap E[R] distribution / CI (one `mc_r_bootstrap` line with an `e_r`
quantile block + P(E[R]<=0)). Completes spec 0075's Monte-Carlo half on top
of iter 1's walk-forward R-reporting.
Engine: `r_bootstrap(rs, n_resamples, block_len, seed) -> RBootstrap` — a
deterministic moving-block bootstrap (non-circular, final block truncated;
block_len=1 = i.i.d. trade-shuffle; clamped to [1,n]; empty -> all-zero),
reusing the existing MetricStats/quantile and SplitMix64.
CLI: parse_mc_args/McArgs split — bare/`--name`/`--trace` route to the
unchanged synthetic seed-resweep `run_mc` (byte-for-byte preserved); the R
path accepts ONLY `--strategy stage1-r`, so a bare `--real` stays a usage
error (mc_rejects_real_flag_with_usage_exit_2 preserved, doc-comment
refreshed). run_mc_r_bootstrap reuses walkforward_family + RMetrics.trade_rs.
Corrective fixes over the plan snippets, both verified: the mc usage string
carries a `usage:` token (the new `["mc", rest @ ..]` arm now handles the
`--real` rejection, and the preserved golden asserts `stderr.contains
("usage")`); `#[allow(clippy::large_enum_variant)]` on McArgs keeps the
unboxed Stage1RGrid field layout.
Scope: single `--real <SYM>` per invocation; cross-symbol pooling is a
deferred follow-on (needs a multi-symbol grammar). Verified: cargo build
clean, full `cargo test --workspace` green (609 passed), clippy -D warnings
clean; bare `aura mc`, SMA walkforward, stage1-r single-run, and C18
round-trip goldens all preserved.
refs #139
This commit is contained in:
+216
-13
@@ -17,11 +17,11 @@ use render::{ChartData, ChartMeta, ChartMode, ReduceKind, Series};
|
||||
use aura_core::{zip_params, Cell, Firing, ParamSpec, Scalar, ScalarKind, Timestamp};
|
||||
use aura_composites::{risk_executor, risk_executor_vol_open, StopRule};
|
||||
use aura_engine::{
|
||||
f64_field, join_on_ts, monte_carlo, param_stability, r_metrics_from_rs, summarize, summarize_r,
|
||||
walk_forward, window_of, ColumnarTrace, Composite, Edge, FlatGraph, GraphBuilder, Harness,
|
||||
JoinedRow, McAggregate, McFamily, RollMode, RunManifest, RunMetrics, RunReport, SourceSpec,
|
||||
SweepFamily, SweepPoint, SyntheticSpec, Target, VecSource, WalkForwardResult, WindowBounds,
|
||||
WindowRoller, WindowRun,
|
||||
f64_field, join_on_ts, monte_carlo, param_stability, r_bootstrap, r_metrics_from_rs, summarize,
|
||||
summarize_r, walk_forward, window_of, ColumnarTrace, Composite, Edge, FlatGraph, GraphBuilder,
|
||||
Harness, JoinedRow, McAggregate, McFamily, RBootstrap, RollMode, RunManifest, RunMetrics,
|
||||
RunReport, SourceSpec, SweepFamily, SweepPoint, SyntheticSpec, Target, VecSource,
|
||||
WalkForwardResult, WindowBounds, WindowRoller, WindowRun,
|
||||
};
|
||||
use aura_registry::{
|
||||
group_families, mc_member_reports, optimize, rank_by, sweep_member_reports,
|
||||
@@ -1885,15 +1885,24 @@ fn run_oos(
|
||||
(equity, report)
|
||||
}
|
||||
|
||||
/// Pool every OOS window's per-trade R series into one flat vector, in roll order
|
||||
/// (window order, then within-window trade order). Windows with no `r` block
|
||||
/// contribute nothing. The single home of the pooling-in-roll-order semantics —
|
||||
/// both the walk-forward `oos_r` summary and the `mc` R-bootstrap reduce this.
|
||||
fn pooled_oos_trade_rs(result: &WalkForwardResult) -> Vec<f64> {
|
||||
result
|
||||
.windows
|
||||
.iter()
|
||||
.flat_map(|w| w.run.oos_report.metrics.r.as_ref().map(|r| r.trade_rs.clone()).unwrap_or_default())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// The walk-forward summary line: window count, stitched OOS total pips (the last
|
||||
/// stitched-curve value), and the on-demand per-param stability. Canonical JSON
|
||||
/// (C14).
|
||||
fn walkforward_summary_json(result: &WalkForwardResult) -> String {
|
||||
let total = result.stitched_oos_equity.last().map(|&(_, v)| v).unwrap_or(0.0);
|
||||
// pool the per-window OOS per-trade R series in roll order, then reduce.
|
||||
let pooled_rs: Vec<f64> = result.windows.iter()
|
||||
.flat_map(|w| w.run.oos_report.metrics.r.as_ref().map(|r| r.trade_rs.clone()).unwrap_or_default())
|
||||
.collect();
|
||||
let pooled_rs = pooled_oos_trade_rs(result);
|
||||
let mut obj = serde_json::json!({
|
||||
"windows": result.windows.len(),
|
||||
"stitched_total_pips": total,
|
||||
@@ -2022,6 +2031,110 @@ fn run_mc(name: &str, persist: bool) {
|
||||
println!("{}", mc_aggregate_json(&family.aggregate));
|
||||
}
|
||||
|
||||
/// Parsed form of the `mc` tail: either the synthetic seed-resweep (today's path,
|
||||
/// preserved byte-for-byte) or the real-candidate R-bootstrap. The R path accepts
|
||||
/// ONLY `--strategy stage1-r` (the sole R-reporting walk-forward strategy); a bare
|
||||
/// `--real` without it is a usage error (the synthetic seed-resweep is undefined
|
||||
/// over real bars).
|
||||
// Parsed once at the dispatch boundary and immediately destructured into the run
|
||||
// call; never stored or collected, so the inter-variant size gap is free here.
|
||||
#[allow(clippy::large_enum_variant)]
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
enum McArgs {
|
||||
Synthetic { name: String, persist: bool },
|
||||
RealR { choice: DataChoice, grid: Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64 },
|
||||
}
|
||||
|
||||
fn parse_mc_args(rest: &[&str]) -> Result<McArgs, String> {
|
||||
let usage = || "usage: mc [--name <n>|--trace <n>] | mc --strategy stage1-r [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] [--block-len <n>] [--resamples <n>] [--seed <n>]".to_string();
|
||||
let mut strategy: Option<Strategy> = None;
|
||||
let mut name: Option<(String, bool)> = None;
|
||||
let mut real = RealWindowGrammar::default();
|
||||
let mut grid = Stage1RGrid::default();
|
||||
let mut block_len: usize = 1;
|
||||
let mut n_resamples: usize = 1000;
|
||||
let mut seed: u64 = 1;
|
||||
let mut r_path = false;
|
||||
let mut tail = rest;
|
||||
while let Some((flag, t)) = tail.split_first() {
|
||||
let (value, t) = t.split_first().ok_or_else(usage)?;
|
||||
if real.accept(flag, value, &usage)? {
|
||||
r_path = true;
|
||||
tail = t;
|
||||
continue;
|
||||
}
|
||||
match *flag {
|
||||
"--strategy" => {
|
||||
strategy = Some(match *value {
|
||||
"stage1-r" => Strategy::Stage1R,
|
||||
_ => return Err(usage()),
|
||||
});
|
||||
r_path = true;
|
||||
}
|
||||
"--name" if name.is_none() => name = Some(((*value).to_string(), false)),
|
||||
"--trace" if name.is_none() => name = Some(((*value).to_string(), true)),
|
||||
"--fast" => { grid.fast = parse_csv_list(value).map_err(|()| usage())?; r_path = true; }
|
||||
"--slow" => { grid.slow = parse_csv_list(value).map_err(|()| usage())?; r_path = true; }
|
||||
"--stop-length" => { grid.stop_length = parse_csv_list(value).map_err(|()| usage())?; r_path = true; }
|
||||
"--stop-k" => { grid.stop_k = parse_csv_list(value).map_err(|()| usage())?; r_path = true; }
|
||||
"--block-len" => { block_len = value.parse().map_err(|_| usage())?; r_path = true; }
|
||||
"--resamples" => { n_resamples = value.parse().map_err(|_| usage())?; r_path = true; }
|
||||
"--seed" => { seed = value.parse().map_err(|_| usage())?; r_path = true; }
|
||||
_ => return Err(usage()),
|
||||
}
|
||||
tail = t;
|
||||
}
|
||||
if r_path {
|
||||
if strategy != Some(Strategy::Stage1R) {
|
||||
return Err(usage());
|
||||
}
|
||||
if name.is_some() {
|
||||
return Err(usage());
|
||||
}
|
||||
let choice = real.finish(&usage)?;
|
||||
Ok(McArgs::RealR { choice, grid, block_len, n_resamples, seed })
|
||||
} else {
|
||||
let (name, persist) = name.unwrap_or_else(|| ("mc".to_string(), false));
|
||||
Ok(McArgs::Synthetic { name, persist })
|
||||
}
|
||||
}
|
||||
|
||||
/// `aura mc --strategy stage1-r [--real <SYM>]`: run the stage1-r walk-forward, pool
|
||||
/// every OOS window's per-trade R series in roll order, and print one moving-block
|
||||
/// bootstrap `mc_r_bootstrap` line (E[R] distribution + P(E[R] <= 0)). Frictionless
|
||||
/// Stage-1 (no costs); deterministic given `seed` (C1).
|
||||
fn run_mc_r_bootstrap(data: DataSource, grid: &Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64) {
|
||||
println!("{}", mc_r_bootstrap_report(&data, grid, block_len, n_resamples, seed));
|
||||
}
|
||||
|
||||
/// Assemble the `mc` R-bootstrap line: run the stage1-r walk-forward, pool the OOS
|
||||
/// per-trade R series in roll order, bootstrap E[R], and render the `mc_r_bootstrap`
|
||||
/// line. The body of `run_mc_r_bootstrap` minus the `println!`, so the full real-R
|
||||
/// wiring (walk-forward -> non-empty pooling -> bootstrap -> render) is reachable
|
||||
/// over synthetic data in a `#[cfg(test)]` unit, mirroring `mc_report` /
|
||||
/// `walkforward_report` / `sweep_report`.
|
||||
fn mc_r_bootstrap_report(data: &DataSource, grid: &Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64) -> String {
|
||||
let result = walkforward_family(Strategy::Stage1R, None, data, grid);
|
||||
let pooled = pooled_oos_trade_rs(&result);
|
||||
let boot = r_bootstrap(&pooled, n_resamples, block_len, seed);
|
||||
mc_r_bootstrap_json(&boot)
|
||||
}
|
||||
|
||||
/// Render an `RBootstrap` as one canonical JSON line (`MetricStats` serializes; the
|
||||
/// scalar fields are spliced in), mirroring `mc_aggregate_json`.
|
||||
fn mc_r_bootstrap_json(b: &RBootstrap) -> String {
|
||||
serde_json::json!({
|
||||
"mc_r_bootstrap": {
|
||||
"n_trades": b.n_trades,
|
||||
"block_len": b.block_len,
|
||||
"n_resamples": b.n_resamples,
|
||||
"e_r": b.e_r,
|
||||
"prob_le_zero": b.prob_le_zero,
|
||||
}
|
||||
})
|
||||
.to_string()
|
||||
}
|
||||
|
||||
/// Render the built-in Monte-Carlo family as the per-draw `RunReport` lines plus
|
||||
/// the aggregate line — the `run_mc` shape minus registry persistence (no
|
||||
/// `family_id`, which is store-assigned). Test helper, mirroring `sweep_report` /
|
||||
@@ -2754,7 +2867,7 @@ fn run_dispatch(args: RunArgs) -> Result<RunReport, String> {
|
||||
}
|
||||
|
||||
const USAGE: &str =
|
||||
"usage: aura run [--harness <sma|macd|stage1-r>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--trace <name>] | aura chart <name> [--tap <t>] [--panels] | aura graph | aura sweep [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n>|--trace <n>] | aura mc [--name <n>|--trace <n>] | aura walkforward [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n>|--trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] | aura runs families | aura runs family <id> [rank <metric>]";
|
||||
"usage: aura run [--harness <sma|macd|stage1-r>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--trace <name>] | aura chart <name> [--tap <t>] [--panels] | aura graph | aura sweep [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n>|--trace <n>] | aura mc [--name <n>|--trace <n>] | aura mc --strategy stage1-r [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] [--block-len <n>] [--resamples <n>] [--seed <n>] | aura walkforward [--strategy <sma|momentum|stage1-r|stage1-breakout|stage1-meanrev>] [--real <SYMBOL> [--from <ms>] [--to <ms>]] [--name <n>|--trace <n>] [--fast <csv>] [--slow <csv>] [--stop-length <csv>] [--stop-k <csv>] | aura runs families | aura runs family <id> [rank <metric>]";
|
||||
|
||||
fn main() {
|
||||
// Collect argv and match the whole vector: every accepted form is exhaustive,
|
||||
@@ -2808,9 +2921,16 @@ fn main() {
|
||||
std::process::exit(2);
|
||||
}
|
||||
},
|
||||
["mc"] => run_mc("mc", false),
|
||||
["mc", "--name", n] => run_mc(n, false),
|
||||
["mc", "--trace", n] => run_mc(n, true),
|
||||
["mc", rest @ ..] => match parse_mc_args(rest) {
|
||||
Ok(McArgs::Synthetic { name, persist }) => run_mc(&name, persist),
|
||||
Ok(McArgs::RealR { choice, grid, block_len, n_resamples, seed }) => {
|
||||
run_mc_r_bootstrap(DataSource::from_choice(choice), &grid, block_len, n_resamples, seed)
|
||||
}
|
||||
Err(msg) => {
|
||||
eprintln!("aura: {msg}");
|
||||
std::process::exit(2);
|
||||
}
|
||||
},
|
||||
["runs", "families"] => runs_families(),
|
||||
["runs", "family", id] => runs_family(id, None),
|
||||
["runs", "family", id, "rank", metric] => runs_family(id, Some(metric)),
|
||||
@@ -3925,4 +4045,87 @@ mod tests {
|
||||
assert!(parse_run_args(&["--real", "GER40", "--from"]).is_err()); // flag missing its value
|
||||
assert!(parse_run_args(&["bogus"]).is_err()); // unknown trailing token
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_mc_args_bare_and_name_route_to_synthetic() {
|
||||
assert_eq!(parse_mc_args(&[]), Ok(McArgs::Synthetic { name: "mc".to_string(), persist: false }));
|
||||
assert_eq!(parse_mc_args(&["--name", "m"]), Ok(McArgs::Synthetic { name: "m".to_string(), persist: false }));
|
||||
assert_eq!(parse_mc_args(&["--trace", "m"]), Ok(McArgs::Synthetic { name: "m".to_string(), persist: true }));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_mc_args_stage1_r_routes_to_real_r_with_defaults_and_overrides() {
|
||||
assert_eq!(
|
||||
parse_mc_args(&["--strategy", "stage1-r"]),
|
||||
Ok(McArgs::RealR {
|
||||
choice: DataChoice::Synthetic, grid: Stage1RGrid::default(),
|
||||
block_len: 1, n_resamples: 1000, seed: 1,
|
||||
})
|
||||
);
|
||||
let r = parse_mc_args(&["--strategy", "stage1-r", "--real", "USDJPY",
|
||||
"--block-len", "5", "--resamples", "200", "--seed", "9", "--fast", "5,10"]);
|
||||
let McArgs::RealR { choice, grid, block_len, n_resamples, seed } = r.expect("valid real-R mc args")
|
||||
else { panic!("expected RealR") };
|
||||
assert_eq!(choice, DataChoice::Real { symbol: "USDJPY".to_string(), from_ms: None, to_ms: None });
|
||||
assert_eq!((block_len, n_resamples, seed), (5, 200, 9));
|
||||
assert_eq!(grid.fast, vec![5, 10]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_mc_args_real_without_stage1_r_is_usage_error() {
|
||||
// a bare --real (no R candidate) is still rejected — the synthetic seed-resweep
|
||||
// is undefined over real bars; the R path needs --strategy stage1-r.
|
||||
assert!(parse_mc_args(&["--real", "EURUSD"]).is_err());
|
||||
assert!(parse_mc_args(&["--strategy", "sma"]).is_err());
|
||||
assert!(parse_mc_args(&["--strategy", "stage1-r", "--name", "x"]).is_err()); // name invalid on R path
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mc_r_bootstrap_json_carries_every_bootstrap_field_under_the_mc_r_bootstrap_key() {
|
||||
// Property: the `mc_r_bootstrap` output line is the full RBootstrap shape —
|
||||
// each field is named and value-faithful, so a renamed/dropped field here
|
||||
// breaks a test instead of silently shipping. Pins the user-visible wire
|
||||
// shape of the mc R-bootstrap render (the parser + engine primitive are
|
||||
// covered elsewhere; this is the output-shape layer).
|
||||
let boot = r_bootstrap(&[1.0, -0.5, 2.0, -1.0], 64, 2, 7);
|
||||
let line = mc_r_bootstrap_json(&boot);
|
||||
let v: serde_json::Value = serde_json::from_str(&line).expect("canonical json line");
|
||||
let obj = &v["mc_r_bootstrap"];
|
||||
assert_eq!(obj["n_trades"], serde_json::json!(boot.n_trades));
|
||||
assert_eq!(obj["block_len"], serde_json::json!(boot.block_len));
|
||||
assert_eq!(obj["n_resamples"], serde_json::json!(boot.n_resamples));
|
||||
assert_eq!(obj["prob_le_zero"], serde_json::json!(boot.prob_le_zero));
|
||||
// e_r is the nested MetricStats block (mean + quantiles), not a flat scalar.
|
||||
assert_eq!(obj["e_r"], serde_json::to_value(&boot.e_r).expect("MetricStats serializes"));
|
||||
assert!(obj["e_r"]["mean"].is_number(), "e_r should nest the MetricStats block: {line}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mc_r_bootstrap_report_pools_a_non_empty_oos_r_series_over_synthetic() {
|
||||
// Property: the full real-R assembly path — walkforward_family(Stage1R) ->
|
||||
// pooled_oos_trade_rs -> r_bootstrap -> mc_r_bootstrap_json — wires up and
|
||||
// reduces a NON-EMPTY pooled OOS R series (the synthetic stage1-r walk-forward
|
||||
// closes >= 1 trade across its windows). Guards the wiring + the non-empty
|
||||
// pooling branch the parser/primitive unit tests cannot reach; mirrors
|
||||
// `mc_report` / `walkforward_report`. Deterministic (C1).
|
||||
let result = walkforward_family(Strategy::Stage1R, None, &DataSource::Synthetic, &Stage1RGrid::default());
|
||||
let pooled = pooled_oos_trade_rs(&result);
|
||||
assert!(!pooled.is_empty(), "synthetic stage1-r walk-forward must pool >= 1 OOS trade R");
|
||||
|
||||
let line = mc_r_bootstrap_report(&DataSource::Synthetic, &Stage1RGrid::default(), 1, 256, 1);
|
||||
let v: serde_json::Value = serde_json::from_str(&line).expect("canonical json line");
|
||||
let obj = &v["mc_r_bootstrap"];
|
||||
// n_trades is the pooled-series length the bootstrap actually saw — non-zero
|
||||
// proves the assembled pooling fed the primitive, not an empty fallback.
|
||||
assert_eq!(obj["n_trades"], serde_json::json!(pooled.len()));
|
||||
assert!(obj["n_trades"].as_u64().expect("n_trades is an integer") >= 1);
|
||||
assert_eq!(obj["n_resamples"], serde_json::json!(256));
|
||||
assert!(obj["e_r"]["mean"].as_f64().expect("e_r.mean is a number").is_finite());
|
||||
|
||||
// C1 at the CLI edge: the assembled real-R line is byte-identical on re-run.
|
||||
assert_eq!(
|
||||
line,
|
||||
mc_r_bootstrap_report(&DataSource::Synthetic, &Stage1RGrid::default(), 1, 256, 1),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1294,13 +1294,12 @@ fn sweep_real_is_byte_deterministic_across_runs() {
|
||||
assert_eq!(run(), run(), "same real sweep twice must be byte-identical (C1)");
|
||||
}
|
||||
|
||||
/// Property (spec-0060 MC exclusion): `aura mc` admits NO `--real` flag. The
|
||||
/// spec carves MC out of the real-data path — its seed varies a *synthetic*
|
||||
/// price-walk realization, undefined over real bars (one realization -> identical
|
||||
/// members). The observable contract is that `aura mc --real EURUSD` is REJECTED
|
||||
/// (usage on stderr, exit 2) at the binary boundary, never silently accepted as a
|
||||
/// real run. This pins the exclusion: if an `["mc", rest @ ..]` arm ever wired
|
||||
/// `--real` into MC, this test fails loudly. NOT gated — the refusal precedes any
|
||||
/// The synthetic seed-resweep `mc` is undefined over real bars (one realization ->
|
||||
/// identical members), so a bare `aura mc --real EURUSD` — with no R candidate — is
|
||||
/// REJECTED (usage on stderr, exit 2) at the binary boundary. The real path is
|
||||
/// reachable ONLY via `--strategy stage1-r` (the R-bootstrap over the pooled OOS R
|
||||
/// series, `mc_strategy_stage1_r_prints_a_bootstrap_line_deterministically`); a
|
||||
/// `--real` without it stays a usage error. NOT gated — the refusal precedes any
|
||||
/// data access, so it is CI-safe on every machine.
|
||||
#[test]
|
||||
fn mc_rejects_real_flag_with_usage_exit_2() {
|
||||
@@ -2407,3 +2406,105 @@ fn walkforward_unsupported_strategy_exits_2_naming_the_token() {
|
||||
assert!(out.stdout.is_empty(), "no family summary on the rejected path: {:?}", out.stdout);
|
||||
let _ = std::fs::remove_dir_all(&cwd);
|
||||
}
|
||||
|
||||
/// Property: `aura mc --strategy stage1-r` runs the stage1-r walk-forward on
|
||||
/// synthetic data, bootstraps the pooled OOS per-trade R series, and prints exactly
|
||||
/// one canonical `mc_r_bootstrap` line carrying an `e_r` distribution block and
|
||||
/// `prob_le_zero`. Deterministic: a second run with the same (default) seed is
|
||||
/// byte-identical (C1). Frictionless Stage-1 — no costs, no `runs/` family write.
|
||||
#[test]
|
||||
fn mc_strategy_stage1_r_prints_a_bootstrap_line_deterministically() {
|
||||
let dir = temp_cwd("mc_stage1r_boot");
|
||||
let run = || {
|
||||
Command::new(BIN)
|
||||
.args(["mc", "--strategy", "stage1-r", "--resamples", "200"])
|
||||
.current_dir(&dir)
|
||||
.output()
|
||||
.expect("spawn aura mc --strategy stage1-r")
|
||||
};
|
||||
let out = run();
|
||||
assert!(out.status.success(), "exit: {:?} stderr: {}", out.status, String::from_utf8_lossy(&out.stderr));
|
||||
let stdout = String::from_utf8(out.stdout).expect("utf-8 stdout");
|
||||
assert_eq!(stdout.lines().count(), 1, "exactly one mc_r_bootstrap line: {stdout:?}");
|
||||
let v: serde_json::Value = serde_json::from_str(stdout.trim()).expect("canonical JSON");
|
||||
assert!(v["mc_r_bootstrap"]["e_r"]["mean"].is_number(), "e_r block present: {stdout}");
|
||||
assert!(v["mc_r_bootstrap"]["prob_le_zero"].is_number(), "prob_le_zero present: {stdout}");
|
||||
assert_eq!(v["mc_r_bootstrap"]["n_resamples"], 200);
|
||||
|
||||
let out2 = run();
|
||||
assert_eq!(stdout.as_bytes(), out2.stdout.as_slice(), "same seed -> byte-identical line (C1)");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Property: `--block-len` is the moving-block knob, and it is observable AND
|
||||
/// load-bearing at the binary boundary — it round-trips into the emitted
|
||||
/// `block_len` field, and a different `block-len` (over the same seed + pooled R
|
||||
/// series) yields a genuinely different E[R] distribution. This is what separates
|
||||
/// the moving-block bootstrap from a plain i.i.d. trade shuffle: contiguous runs
|
||||
/// preserve serial correlation a `block-len 1` shuffle erases. Without this test a
|
||||
/// regression that dropped `--block-len` on the floor (or wired it to `block_len 1`
|
||||
/// always) would still pass the happy-path E2E. Same-seed determinism (C1) pins
|
||||
/// that the difference is the block length, not RNG noise. Not gated — synthetic.
|
||||
#[test]
|
||||
fn mc_stage1_r_block_len_is_observable_and_changes_the_distribution() {
|
||||
let dir = temp_cwd("mc_stage1r_block");
|
||||
let run = |block_len: &str| {
|
||||
let out = Command::new(BIN)
|
||||
.args(["mc", "--strategy", "stage1-r", "--resamples", "256", "--seed", "1", "--block-len", block_len])
|
||||
.current_dir(&dir)
|
||||
.output()
|
||||
.expect("spawn aura mc --strategy stage1-r --block-len");
|
||||
assert!(out.status.success(), "exit: {:?} stderr: {}", out.status, String::from_utf8_lossy(&out.stderr));
|
||||
String::from_utf8(out.stdout).expect("utf-8 stdout")
|
||||
};
|
||||
|
||||
let one = run("1");
|
||||
let three = run("3");
|
||||
|
||||
let v1: serde_json::Value = serde_json::from_str(one.trim()).expect("canonical JSON (block-len 1)");
|
||||
let v3: serde_json::Value = serde_json::from_str(three.trim()).expect("canonical JSON (block-len 3)");
|
||||
// the flag round-trips verbatim into the emitted line.
|
||||
assert_eq!(v1["mc_r_bootstrap"]["block_len"], 1, "block_len 1 must echo: {one}");
|
||||
assert_eq!(v3["mc_r_bootstrap"]["block_len"], 3, "block_len 3 must echo: {three}");
|
||||
// same seed, same pooled series, different block length -> different E[R] spread.
|
||||
// (a no-op `--block-len` would make these byte-identical.)
|
||||
assert_ne!(one, three, "block-len must change the resampled distribution, not be ignored");
|
||||
|
||||
// C1 at the CLI edge: the block-len-1 line is byte-identical on re-run.
|
||||
assert_eq!(one, run("1"), "same seed+block-len -> byte-identical line (C1)");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
/// Property: a `--block-len` at or above the pooled-series length is CLAMPED to the
|
||||
/// series length at the binary boundary (no panic, no out-of-bounds), and the
|
||||
/// degenerate single-block case collapses the bootstrap — when every resample IS
|
||||
/// the full series, the E[R] distribution becomes a single point (all quantiles
|
||||
/// equal). The emitted `block_len` reports the clamped value, not the raw flag.
|
||||
/// This pins the clamp the engine primitive guards in-crate, now at the observable
|
||||
/// CLI edge where an off-by-one in the index arithmetic would otherwise surface as
|
||||
/// a process crash. Not gated — synthetic stage1-r pools a fixed, non-empty series.
|
||||
#[test]
|
||||
fn mc_stage1_r_oversized_block_len_clamps_and_collapses() {
|
||||
let dir = temp_cwd("mc_stage1r_clamp");
|
||||
let out = Command::new(BIN)
|
||||
.args(["mc", "--strategy", "stage1-r", "--resamples", "200", "--seed", "1", "--block-len", "9999"])
|
||||
.current_dir(&dir)
|
||||
.output()
|
||||
.expect("spawn aura mc --strategy stage1-r --block-len 9999");
|
||||
assert!(out.status.success(), "must not crash on oversized block-len: {:?} stderr: {}",
|
||||
out.status, String::from_utf8_lossy(&out.stderr));
|
||||
let stdout = String::from_utf8(out.stdout).expect("utf-8 stdout");
|
||||
let v: serde_json::Value = serde_json::from_str(stdout.trim()).expect("canonical JSON");
|
||||
let obj = &v["mc_r_bootstrap"];
|
||||
let n_trades = obj["n_trades"].as_u64().expect("n_trades is an integer");
|
||||
assert!(n_trades >= 1, "synthetic stage1-r must pool a non-empty R series: {stdout}");
|
||||
// block_len is reported clamped to the pooled-series length, never the raw 9999.
|
||||
assert_eq!(obj["block_len"], serde_json::json!(n_trades),
|
||||
"oversized block-len must clamp to n_trades: {stdout}");
|
||||
// single-block degenerate: every resample is the full series, so the E[R]
|
||||
// distribution is a single point — p5 == p95 (zero spread).
|
||||
let p5 = obj["e_r"]["p5"].as_f64().expect("p5 is a number");
|
||||
let p95 = obj["e_r"]["p95"].as_f64().expect("p95 is a number");
|
||||
assert!((p5 - p95).abs() < 1e-12, "single-block bootstrap must have zero spread: p5={p5} p95={p95}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
@@ -69,7 +69,7 @@ pub use report::{
|
||||
pub use sweep::{
|
||||
sweep, GridSpace, ParamRange, RandomSpace, Space, SweepError, SweepFamily, SweepPoint,
|
||||
};
|
||||
pub use mc::{monte_carlo, McAggregate, McDraw, McFamily, MetricStats};
|
||||
pub use mc::{monte_carlo, r_bootstrap, McAggregate, McDraw, McFamily, MetricStats, RBootstrap};
|
||||
pub use walkforward::{
|
||||
param_stability, walk_forward, RollMode, WalkForwardError, WalkForwardResult,
|
||||
WindowBounds, WindowOutcome, WindowRoller, WindowRun,
|
||||
|
||||
@@ -11,6 +11,7 @@
|
||||
|
||||
use crate::sweep::run_indexed;
|
||||
use crate::{RunMetrics, RunReport, Scalar};
|
||||
use crate::harness::SplitMix64;
|
||||
|
||||
/// One Monte-Carlo realization: the seed that drove it and the full `RunReport`.
|
||||
/// Self-describing, analog to [`SweepPoint`](crate::SweepPoint).
|
||||
@@ -152,6 +153,51 @@ where
|
||||
McFamily { draws, aggregate }
|
||||
}
|
||||
|
||||
/// Distribution of E[R] under a moving-block bootstrap of an OOS per-trade R series
|
||||
/// (#139). `block_len == 1` is the i.i.d. trade-shuffle; `block_len > 1` resamples
|
||||
/// contiguous runs, preserving the serial correlation of sequential trades a pure
|
||||
/// shuffle would erase. Deterministic (C1) given `seed`.
|
||||
#[derive(Clone, Debug, PartialEq)]
|
||||
pub struct RBootstrap {
|
||||
pub e_r: MetricStats, // mean + p5/p25/p50/p75/p95 of the resampled E[R]
|
||||
pub prob_le_zero: f64, // fraction of resamples whose mean R <= 0
|
||||
pub n_trades: usize,
|
||||
pub block_len: usize,
|
||||
pub n_resamples: usize,
|
||||
}
|
||||
|
||||
/// Moving-block bootstrap of `rs` (non-circular; the final block of each resample is
|
||||
/// truncated so the resample has exactly `n` values). `block_len` is clamped to
|
||||
/// `[1, n]`. Empty `rs` or zero resamples -> an all-zero `RBootstrap`. Pure given
|
||||
/// `seed` (drives the existing `SplitMix64`).
|
||||
pub fn r_bootstrap(rs: &[f64], n_resamples: usize, block_len: usize, seed: u64) -> RBootstrap {
|
||||
let n = rs.len();
|
||||
if n == 0 || n_resamples == 0 {
|
||||
return RBootstrap {
|
||||
e_r: MetricStats { mean: 0.0, p5: 0.0, p25: 0.0, p50: 0.0, p75: 0.0, p95: 0.0 },
|
||||
prob_le_zero: 0.0,
|
||||
n_trades: n,
|
||||
block_len: block_len.clamp(1, n.max(1)),
|
||||
n_resamples,
|
||||
};
|
||||
}
|
||||
let block_len = block_len.clamp(1, n);
|
||||
let mut rng = SplitMix64::new(seed);
|
||||
let mut means: Vec<f64> = Vec::with_capacity(n_resamples);
|
||||
for _ in 0..n_resamples {
|
||||
let mut sample: Vec<f64> = Vec::with_capacity(n);
|
||||
while sample.len() < n {
|
||||
let start = (rng.next_u64() % (n - block_len + 1) as u64) as usize;
|
||||
let take = block_len.min(n - sample.len());
|
||||
sample.extend_from_slice(&rs[start..start + take]);
|
||||
}
|
||||
means.push(sample.iter().sum::<f64>() / n as f64);
|
||||
}
|
||||
let e_r = MetricStats::from_values(&means);
|
||||
let prob_le_zero = means.iter().filter(|&&m| m <= 0.0).count() as f64 / n_resamples as f64;
|
||||
RBootstrap { e_r, prob_le_zero, n_trades: n, block_len, n_resamples }
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -312,4 +358,104 @@ mod tests {
|
||||
let back: MetricStats = serde_json::from_str(&json).expect("deserialize MetricStats");
|
||||
assert_eq!(back, s);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_empty_series_is_all_zero() {
|
||||
let b = r_bootstrap(&[], 100, 1, 7);
|
||||
assert_eq!(b.n_trades, 0);
|
||||
assert_eq!(b.e_r.mean, 0.0);
|
||||
assert_eq!(b.prob_le_zero, 0.0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_single_block_equals_full_series_mean() {
|
||||
// block_len == n: the only valid start is 0, so every resample IS the full
|
||||
// series -> every resample mean == the series mean -> zero spread.
|
||||
let rs = [1.0, -2.0, 3.0]; // mean = 2/3
|
||||
let b = r_bootstrap(&rs, 500, rs.len(), 7);
|
||||
let mean = 2.0 / 3.0;
|
||||
assert!((b.e_r.mean - mean).abs() < 1e-12);
|
||||
assert!((b.e_r.p5 - mean).abs() < 1e-12);
|
||||
assert!((b.e_r.p95 - mean).abs() < 1e-12);
|
||||
assert_eq!(b.prob_le_zero, 0.0); // mean > 0
|
||||
assert_eq!(b.n_trades, 3);
|
||||
assert_eq!(b.block_len, 3);
|
||||
assert_eq!(b.n_resamples, 500);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_is_deterministic_given_seed() {
|
||||
let rs = [0.5, -1.0, 2.0, -0.5, 1.5, -2.0, 0.25];
|
||||
let a = r_bootstrap(&rs, 1000, 1, 42);
|
||||
let b = r_bootstrap(&rs, 1000, 1, 42);
|
||||
assert_eq!(a, b, "same rs+seed+params -> identical RBootstrap (C1)");
|
||||
let c = r_bootstrap(&rs, 1000, 1, 43);
|
||||
assert_ne!(a.e_r.p5, c.e_r.p5, "a different seed reshuffles differently");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_block_len_is_clamped_to_series_len() {
|
||||
// block_len > n clamps to n -> single-block behaviour (no panic, no OOB).
|
||||
let rs = [1.0, 2.0];
|
||||
let b = r_bootstrap(&rs, 10, 99, 1);
|
||||
assert_eq!(b.block_len, 2);
|
||||
assert!((b.e_r.mean - 1.5).abs() < 1e-12);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn r_bootstrap_moving_block_resamples_are_contiguous_truncated_runs() {
|
||||
// The serial-correlation-preserving branch: 1 < block_len < n with n NOT a
|
||||
// multiple of block_len, so the final block of every resample is truncated
|
||||
// (`take = block_len.min(n - sample.len())`). Every resample is a sequence of
|
||||
// contiguous runs of `rs` (the last one cut short to reach exactly n values),
|
||||
// so each resample mean must lie in the finite set of legal block-composition
|
||||
// means. Distinct powers of ten make every distinct multiset of contributions
|
||||
// a distinct sum -> a faithful membership check. n=5, block_len=2 -> blocks of
|
||||
// length 2, 2, 1.
|
||||
let rs = [1.0, 10.0, 100.0, 1000.0, 10000.0];
|
||||
let n = rs.len();
|
||||
let block_len = 2;
|
||||
|
||||
// Enumerate every legal resample sum: pick a start in [0, n-block_len] for each
|
||||
// of the three blocks (lengths 2, 2, 1), summing min(block_len, n-filled) values
|
||||
// from that start. Membership-by-sum, mirroring r_bootstrap's own construction.
|
||||
let starts: Vec<usize> = (0..=n - block_len).collect();
|
||||
let mut legal_means: Vec<f64> = Vec::new();
|
||||
for &s0 in &starts {
|
||||
for &s1 in &starts {
|
||||
for &s2 in &starts {
|
||||
let mut sum = 0.0;
|
||||
let mut filled = 0usize;
|
||||
for &start in &[s0, s1, s2] {
|
||||
let take = block_len.min(n - filled);
|
||||
if take == 0 {
|
||||
break;
|
||||
}
|
||||
sum += rs[start..start + take].iter().sum::<f64>();
|
||||
filled += take;
|
||||
}
|
||||
legal_means.push(sum / n as f64);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Drive enough resamples that the truncated final block is exercised many
|
||||
// times; assert every produced mean is a legal block-composition mean.
|
||||
let b = r_bootstrap(&rs, 2000, block_len, 7);
|
||||
assert_eq!(b.block_len, 2);
|
||||
assert_eq!(b.n_trades, 5);
|
||||
// The mean of resample means is a weighted average of legal means, so it is
|
||||
// bounded by their min and max (not itself a single legal mean).
|
||||
let lo = legal_means.iter().cloned().fold(f64::INFINITY, f64::min);
|
||||
let hi = legal_means.iter().cloned().fold(f64::NEG_INFINITY, f64::max);
|
||||
assert!(lo - 1e-6 <= b.e_r.mean && b.e_r.mean <= hi + 1e-6);
|
||||
// Each quantile of the resampled E[R] is an order statistic of the resample
|
||||
// means, hence itself a legal contiguous-block-composition mean.
|
||||
for &m in &[b.e_r.p5, b.e_r.p25, b.e_r.p50, b.e_r.p75, b.e_r.p95] {
|
||||
assert!(
|
||||
legal_means.iter().any(|&lm| (lm - m).abs() < 1e-6),
|
||||
"resampled quantile {m} is not a legal contiguous-block-composition mean",
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user