From cb32658fd1fd2598b92a1b404b3cfeddb128b30b Mon Sep 17 00:00:00 2001 From: Brummel Date: Fri, 26 Jun 2026 16:38:18 +0200 Subject: [PATCH] feat(0077): prefer a robust parameter plateau over the in-sample peak MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an opt-in plateau selection objective for walk-forward, recorded on the same RunManifest.selection carrier #144 introduced. Default argmax is byte-identical (C23); plateau is reached only via a new --select flag. What landed: - FamilySelection / SelectionMode reshape (report.rs): the selection RULE (mode) and its ANNOTATION are orthogonal — deflation fields become Option (present iff Argmax), plus PlateauMean/PlateauWorst variants and neighbourhood_score/n_neighbours (present iff Plateau*). Legacy lines still load: the deflation scalars deserialize from bare values via serde default (C14/C18). compat.rs embeds FamilySelection by value — reshape flows through untouched. - GridSpace::axis_lens() + SweepBinder::sweep_with_lattice (engine): the grid radixes in param_space()/odometer order. sweep() delegates to sweep_with_lattice with the lattice dropped, so every existing .sweep() caller is byte-unchanged. - optimize_plateau + PlateauMode + closed_neighbourhood (aura-registry, C9): each member scores as the mean/worst of its closed mixed-radix grid neighbourhood's metric_value; the winner is the smoothed argmax by the metric's direction (earliest-odometer tie, as optimize). Pure, no RNG (C1); in-sample only (C2). A higher_is_better helper now sources the per-metric direction once, shared by metric_cmp and optimize_plateau. - CLI --select (default argmax; unknown token exits 2). walkforward_family dispatches via a select_winner helper; sweep_over/stage1_r_sweep_over carry the lattice as Option>. runs-family display gains a plateau()= over cells line. RandomSpace-refuse: walkforward has no --random producer, so the plateau-without-lattice guard is structural — select_winner returns the refuse on a None lattice (the caller prints the message and exits 2), unit-tested at the helper, not via a --random E2E (decided + recorded on #145 comment 1974, with the lattice-seam fork). Tests: plateau-picks-plateau-not-spike, mean-vs-worst, lower-is-better direction-flip, closed-neighbourhood golden, single-member degeneracy, C1 determinism; CLI select-parse, select_winner refuse; E2E argmax-byte-identical (C23), plateau:mean and plateau:worst provenance. Full workspace suite green; clippy --all-targets -D warnings clean. closes #145 --- crates/aura-cli/src/main.rs | 178 ++++++++++++++---- crates/aura-cli/tests/cli_run.rs | 124 +++++++++++++ crates/aura-engine/src/blueprint.rs | 30 ++- crates/aura-engine/src/report.rs | 65 +++++-- crates/aura-engine/src/sweep.rs | 21 +++ crates/aura-registry/src/lib.rs | 277 ++++++++++++++++++++++++++-- 6 files changed, 620 insertions(+), 75 deletions(-) diff --git a/crates/aura-cli/src/main.rs b/crates/aura-cli/src/main.rs index ca2d6a8..2256403 100644 --- a/crates/aura-cli/src/main.rs +++ b/crates/aura-cli/src/main.rs @@ -18,15 +18,15 @@ use aura_core::{zip_params, Cell, Firing, ParamSpec, Scalar, ScalarKind, Timesta use aura_composites::{risk_executor, risk_executor_vol_open, StopRule}; use aura_engine::{ f64_field, join_on_ts, monte_carlo, param_stability, r_bootstrap, r_metrics_from_rs, summarize, - summarize_r, walk_forward, window_of, ColumnarTrace, Composite, Edge, FlatGraph, GraphBuilder, - Harness, JoinedRow, McAggregate, McFamily, RBootstrap, RollMode, RunManifest, RunMetrics, - RunReport, SourceSpec, SweepFamily, SweepPoint, SyntheticSpec, Target, VecSource, - WalkForwardResult, WindowBounds, WindowRoller, WindowRun, + summarize_r, walk_forward, window_of, ColumnarTrace, Composite, Edge, FamilySelection, FlatGraph, + GraphBuilder, Harness, JoinedRow, McAggregate, McFamily, RBootstrap, RollMode, RunManifest, + RunMetrics, RunReport, SelectionMode, SourceSpec, SweepFamily, SweepPoint, SyntheticSpec, Target, + VecSource, WalkForwardResult, WindowBounds, WindowRoller, WindowRun, }; use aura_registry::{ - group_families, mc_member_reports, optimize_deflated, rank_by, sweep_member_reports, - walkforward_member_reports, FamilyKind, FamilyMember, NameKind, Registry, RunTraces, TraceStore, - WriteKind, + group_families, mc_member_reports, optimize_deflated, optimize_plateau, rank_by, + sweep_member_reports, walkforward_member_reports, FamilyKind, FamilyMember, NameKind, + PlateauMode, Registry, RunTraces, TraceStore, WriteKind, }; use aura_std::{ Add, Bias, Delay, Ema, GatedRecorder, Gt, Latch, LinComb, LongOnly, Mul, Recorder, RollingMax, @@ -1298,7 +1298,7 @@ fn stage1_r_space() -> Vec { /// but windowed (`windowed_sources`) and always folded (O(trades)/member): each /// member's RunReport carries `metrics.r = Some(summarize_r(..))`, so the family is /// rankable by an R metric. -fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid: &Stage1RGrid) -> SweepFamily { +fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid: &Stage1RGrid) -> (SweepFamily, Option>) { let pip = data.pip_size(); let (tx_eq, _) = mpsc::channel(); let (tx_ex, _) = mpsc::channel(); @@ -1315,7 +1315,7 @@ fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid: .axis("slow.length", grid.slow.clone()) .axis(&stop_length_axis, grid.stop_length.clone()) .axis(&stop_k_axis, grid.stop_k.clone()) - .sweep(|point| { + .sweep_with_lattice(|point| { let (tx_eq, rx_eq) = mpsc::channel(); let (tx_ex, rx_ex) = mpsc::channel(); let (tx_r, rx_r) = mpsc::channel(); @@ -1339,6 +1339,7 @@ fn stage1_r_sweep_over(from: Timestamp, to: Timestamp, data: &DataSource, grid: m.r = Some(summarize_r(&r_rows, 0.0)); RunReport { manifest, metrics: m } }) + .map(|(fam, lat)| (fam, Some(lat))) .expect("the stage1-r named grid matches the stage1-r param-space") } @@ -1556,6 +1557,26 @@ enum Strategy { Stage1MeanRev, } +/// In-sample winner-selection objective for walk-forward (cycle 0077). `Argmax` is +/// the bare-best pick deflated for trials (#144, the default); `Plateau` argmaxes +/// the neighbourhood-smoothed surface instead (opt-in via `--select`). +#[derive(Clone, Copy)] +enum Selection { + Argmax, + Plateau(PlateauMode), +} + +/// Parse a `--select` token: `argmax` | `plateau:mean` | `plateau:worst`. Unknown +/// tokens are a usage error (the caller maps `Err(())` to exit 2). +fn parse_select(s: &str) -> Result { + match s { + "argmax" => Ok(Selection::Argmax), + "plateau:mean" => Ok(Selection::Plateau(PlateauMode::Mean)), + "plateau:worst" => Ok(Selection::Plateau(PlateauMode::Worst)), + _ => Err(()), + } +} + impl Strategy { /// The CLI `--strategy` token this variant parses from — the inverse of the /// `parse_*_args` match arms. Used to echo the offending strategy in error @@ -1642,17 +1663,18 @@ fn parse_sweep_args( } /// Parse the `walkforward` tail: -/// `[--strategy ] [--real [--from ] [--to ]] [--name | --trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ]`. +/// `[--strategy ] [--real [--from ] [--to ]] [--name | --trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--select ]`. /// Defaults: SMA-cross, synthetic, name "walkforward", no persist. `--name`/`--trace` /// mutually exclusive; `--from`/`--to` require `--real`. Pure (no I/O / exit). fn parse_walkforward_args( rest: &[&str], -) -> Result<(Strategy, String, bool, DataChoice, Stage1RGrid), String> { - let usage = || "walkforward [--strategy ] [--real [--from ] [--to ]] [--name | --trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ]".to_string(); +) -> Result<(Strategy, String, bool, DataChoice, Stage1RGrid, Selection), String> { + let usage = || "walkforward [--strategy ] [--real [--from ] [--to ]] [--name | --trace ] [--fast ] [--slow ] [--stop-length ] [--stop-k ] [--select ]".to_string(); let mut strategy = Strategy::SmaCross; let mut name: Option<(String, bool)> = None; let mut real = RealWindowGrammar::default(); let mut grid = Stage1RGrid::default(); + let mut select = Selection::Argmax; let mut tail = rest; while let Some((flag, t)) = tail.split_first() { let (value, t) = t.split_first().ok_or_else(usage)?; @@ -1677,12 +1699,13 @@ fn parse_walkforward_args( "--slow" => grid.slow = parse_csv_list(value).map_err(|()| usage())?, "--stop-length" => grid.stop_length = parse_csv_list(value).map_err(|()| usage())?, "--stop-k" => grid.stop_k = parse_csv_list(value).map_err(|()| usage())?, + "--select" => select = parse_select(value).map_err(|()| usage())?, _ => return Err(usage()), } tail = t; } let (name, persist) = name.unwrap_or_else(|| ("walkforward".to_string(), false)); - Ok((strategy, name, persist, real.finish(&usage)?, grid)) + Ok((strategy, name, persist, real.finish(&usage)?, grid, select)) } /// Render a family-member stdout line: the assigned `family_id` plus the embedded @@ -1752,7 +1775,7 @@ fn run_sweep(strategy: Strategy, name: &str, persist: bool, data: DataSource, gr /// print each carrying the assigned id, then the stitched summary line. With /// `--trace`, also persist each OOS member's streams under /// `runs/traces//oos/` (opt-in). Deterministic (C1). -fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSource, grid: &Stage1RGrid) { +fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSource, grid: &Stage1RGrid, select: Selection) { if persist && let Err(e) = TraceStore::open("runs").ensure_name_free(name, WriteKind::Family) { @@ -1760,7 +1783,7 @@ fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSour std::process::exit(2); } let reg = default_registry(); - let result = walkforward_family(strategy, persist.then_some(name), &data, grid); + let result = walkforward_family(strategy, persist.then_some(name), &data, grid, select); let id = match reg.append_family(name, FamilyKind::WalkForward, &walkforward_member_reports(&result)) { @@ -1776,6 +1799,31 @@ fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSour println!("{}", walkforward_summary_json(&result)); } +/// Resolve the in-sample winner under the chosen selection objective. `Argmax` +/// defers to the trials-deflation pick (#144). `Plateau` argmaxes the smoothed grid +/// surface — it needs the grid lattice, so a sweep with no lattice (a future random +/// walk-forward producer) is refused rather than silently argmaxed. The metric is +/// always known at the call sites, so a metric error is unreachable (`expect`); the +/// only fallible outcome is the plateau-without-lattice refusal, returned as +/// `Err(message)` for the caller to print and exit 2. +fn select_winner( + family: &SweepFamily, metric: &str, select: Selection, lattice: Option<&[usize]>, +) -> Result<(SweepPoint, FamilySelection), String> { + match select { + Selection::Argmax => Ok(optimize_deflated( + family, metric, DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED, + ).expect("walk-forward metrics are known")), + Selection::Plateau(mode) => match lattice { + Some(lens) => Ok(optimize_plateau(family, lens, metric, mode) + .expect("walk-forward metrics are known")), + None => Err( + "--select plateau requires a grid sweep; a random sweep has no parameter lattice" + .to_string(), + ), + }, + } +} + /// The built-in rolling walk-forward: 24-bar in-sample, 12-bar out-of-sample, /// stepping 12 (contiguous OOS tiling), over the 60-bar synthetic span -> 3 /// windows. Each window sweeps a grid in-sample, optimizes by a metric, and runs @@ -1785,6 +1833,7 @@ fn run_walkforward(strategy: Strategy, name: &str, persist: bool, data: DataSour /// Other strategies have no walk-forward form yet (exit 2). fn walkforward_family( strategy: Strategy, trace: Option<&str>, data: &DataSource, grid: &Stage1RGrid, + select: Selection, ) -> WalkForwardResult { let span = data.wf_full_span(); let (is_len, oos_len, step) = data.wf_window_sizes(); @@ -1799,10 +1848,11 @@ fn walkforward_family( Strategy::SmaCross => { let space = sample_blueprint_with_sinks(data.pip_size()).0.param_space(); walk_forward(roller, space, |w: WindowBounds| { - let is_family = sweep_over(w.is.0, w.is.1, data); - let (best, selection) = optimize_deflated( - &is_family, "total_pips", DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED, - ).expect("total_pips is a known metric"); + let (is_family, lattice) = sweep_over(w.is.0, w.is.1, data); + let (best, selection) = match select_winner(&is_family, "total_pips", select, lattice.as_deref()) { + Ok(v) => v, + Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); } + }; let (oos_equity, mut oos_report) = run_oos(&best.params, w.oos.0, w.oos.1, trace, data); oos_report.manifest.selection = Some(selection); WindowRun { @@ -1817,10 +1867,11 @@ fn walkforward_family( Strategy::Stage1R => { let space = stage1_r_space(); walk_forward(roller, space, |w: WindowBounds| { - let is_family = stage1_r_sweep_over(w.is.0, w.is.1, data, grid); - let (best, selection) = optimize_deflated( - &is_family, "sqn_normalized", DEFLATION_N_RESAMPLES, DEFLATION_BLOCK_LEN, DEFLATION_SEED, - ).expect("sqn_normalized is a known metric"); + let (is_family, lattice) = stage1_r_sweep_over(w.is.0, w.is.1, data, grid); + let (best, selection) = match select_winner(&is_family, "sqn_normalized", select, lattice.as_deref()) { + Ok(v) => v, + Err(msg) => { eprintln!("aura: {msg}"); std::process::exit(2); } + }; let (oos_equity, mut oos_report) = run_oos_r(&best.params, w.oos.0, w.oos.1, trace, data); oos_report.manifest.selection = Some(selection); WindowRun { chosen_params: best.params, oos_equity, oos_report } @@ -1838,7 +1889,7 @@ fn walkforward_family( /// Sweep the built-in named grid over an in-sample window, sourcing the in-memory /// windowed stream. Mirrors `sweep_family`, but windowed by `[from, to]`. -fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> SweepFamily { +fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> (SweepFamily, Option>) { let pip = data.pip_size(); let bp = sample_blueprint_with_sinks(pip).0; let space = bp.param_space(); @@ -1851,7 +1902,7 @@ fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> SweepFamily .axis("signals.blend.weights[0]", [1.0]) .axis("signals.blend.weights[1]", [1.0]) .axis("bias.scale", [0.5]) - .sweep(|point| { + .sweep_with_lattice(|point| { let (bp, rx_eq, rx_ex) = sample_blueprint_with_sinks(pip); let mut h = bp .bootstrap_with_cells(point) @@ -1866,6 +1917,7 @@ fn sweep_over(from: Timestamp, to: Timestamp, data: &DataSource) -> SweepFamily metrics: summarize(&equity, &exposure), } }) + .map(|(fam, lat)| (fam, Some(lat))) .expect("the built-in named grid matches the sample param-space") } @@ -1960,7 +2012,7 @@ fn walkforward_window_source(from: Timestamp, to: Timestamp) -> VecSource { /// helper (mirrors `sweep_report`). #[cfg(test)] fn walkforward_report() -> String { - let result = walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default()); + let result = walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default(), Selection::Argmax); let mut out = String::new(); for w in &result.windows { out.push_str(&w.run.oos_report.to_json()); @@ -2128,7 +2180,7 @@ fn run_mc_r_bootstrap(data: DataSource, grid: &Stage1RGrid, block_len: usize, n_ /// over synthetic data in a `#[cfg(test)]` unit, mirroring `mc_report` / /// `walkforward_report` / `sweep_report`. fn mc_r_bootstrap_report(data: &DataSource, grid: &Stage1RGrid, block_len: usize, n_resamples: usize, seed: u64) -> String { - let result = walkforward_family(Strategy::Stage1R, None, data, grid); + let result = walkforward_family(Strategy::Stage1R, None, data, grid, Selection::Argmax); let pooled = pooled_oos_trade_rs(&result); let boot = r_bootstrap(&pooled, n_resamples, block_len, seed); mc_r_bootstrap_json(&boot) @@ -2215,9 +2267,24 @@ fn runs_family(id: &str, rank: Option<&str>) { for report in &ordered { println!("{}", report.to_json()); if let Some(sel) = &report.manifest.selection { - match sel.overfit_probability { - Some(p) => println!(" deflated={:.4} P(overfit)={:.4}", sel.deflated_score, p), - None => println!(" deflated={:.4}", sel.deflated_score), + match sel.mode { + // `deflated_score` is `None` only on an Argmax record with no + // deflation run (report.rs); guard it, symmetric with the + // plateau branch, so a from-disk record cannot panic here. When + // present (the sole producer always stamps it), the bytes are + // unchanged. + SelectionMode::Argmax => if let Some(deflated) = sel.deflated_score { + match sel.overfit_probability { + Some(p) => println!(" deflated={deflated:.4} P(overfit)={p:.4}"), + None => println!(" deflated={deflated:.4}"), + } + }, + SelectionMode::PlateauMean | SelectionMode::PlateauWorst => { + let label = if matches!(sel.mode, SelectionMode::PlateauMean) { "mean" } else { "worst" }; + if let (Some(score), Some(n)) = (sel.neighbourhood_score, sel.n_neighbours) { + println!(" plateau({label})={score:.4} over {n} cells"); + } + } } } } @@ -2933,8 +3000,8 @@ fn main() { } }, ["walkforward", rest @ ..] => match parse_walkforward_args(rest) { - Ok((strategy, name, persist, choice, grid)) => { - run_walkforward(strategy, &name, persist, DataSource::from_choice(choice), &grid) + Ok((strategy, name, persist, choice, grid, select)) => { + run_walkforward(strategy, &name, persist, DataSource::from_choice(choice), &grid, select) } Err(msg) => { eprintln!("aura: {msg}"); @@ -2965,6 +3032,29 @@ fn main() { mod tests { use super::*; + #[test] + fn select_winner_refuses_plateau_without_a_lattice() { + // A plateau request with no lattice (a random sweep would yield None) is + // refused, never silently argmaxed. The refuse short-circuits before the + // family is read, so an empty family is fine here. + let fam = SweepFamily { space: vec![], points: vec![] }; + let err = select_winner(&fam, "total_pips", Selection::Plateau(PlateauMode::Mean), None) + .unwrap_err(); + assert!(err.contains("requires a grid sweep"), "refuse message: {err}"); + } + + #[test] + fn parse_walkforward_select_flag() { + let argmax = parse_walkforward_args(&["--strategy", "stage1-r"]).unwrap(); + assert!(matches!(argmax.5, Selection::Argmax), "default is argmax"); + let mean = parse_walkforward_args(&["--select", "plateau:mean"]).unwrap(); + assert!(matches!(mean.5, Selection::Plateau(PlateauMode::Mean))); + let worst = parse_walkforward_args(&["--select", "plateau:worst"]).unwrap(); + assert!(matches!(worst.5, Selection::Plateau(PlateauMode::Worst))); + assert!(parse_walkforward_args(&["--select", "bogus"]).is_err(), + "unknown --select token is a usage error"); + } + fn cmp_member(key: &str, ts: &[i64], vals: &[f64]) -> FamilyMember { cmp_member_win(key, ts, vals, (0, 0)) } @@ -3453,7 +3543,7 @@ mod tests { .append_family( "walkforward", FamilyKind::WalkForward, - &walkforward_member_reports(&walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default())), + &walkforward_member_reports(&walkforward_family(Strategy::SmaCross, None, &DataSource::Synthetic, &Stage1RGrid::default(), Selection::Argmax)), ) .expect("walkforward family"); assert_eq!((sid.as_str(), mid.as_str(), wid.as_str()), ("sweep-0", "mc-0", "walkforward-0")); @@ -3955,15 +4045,23 @@ mod tests { /// and rejects two name flags or a `--real` missing its symbol. #[test] fn parse_walkforward_args_defaults_and_accepts_real() { + // Selection is not PartialEq/Debug (it carries the opaque PlateauMode), so + // the comparable fields are asserted via the 5-tuple prefix and `select` + // separately via `matches!`. + let (strategy, name, persist, choice, grid, select) = + parse_walkforward_args(&[]).expect("defaults parse"); assert_eq!( - parse_walkforward_args(&[]), - Ok((Strategy::SmaCross, "walkforward".to_string(), false, DataChoice::Synthetic, Stage1RGrid::default())) + (strategy, name, persist, choice, grid), + (Strategy::SmaCross, "walkforward".to_string(), false, DataChoice::Synthetic, Stage1RGrid::default()) ); + assert!(matches!(select, Selection::Argmax), "default selection is argmax"); + let (strategy, name, persist, choice, grid, _) = + parse_walkforward_args(&["--real", "EURUSD", "--trace", "w"]).expect("real/trace parse"); assert_eq!( - parse_walkforward_args(&["--real", "EURUSD", "--trace", "w"]), - Ok((Strategy::SmaCross, "w".to_string(), true, + (strategy, name, persist, choice, grid), + (Strategy::SmaCross, "w".to_string(), true, DataChoice::Real { symbol: "EURUSD".to_string(), from_ms: None, to_ms: None }, - Stage1RGrid::default())) + Stage1RGrid::default()) ); assert!(parse_walkforward_args(&["--name", "a", "--trace", "b"]).is_err()); assert!(parse_walkforward_args(&["--real"]).is_err()); @@ -3972,7 +4070,7 @@ mod tests { #[test] fn parse_walkforward_args_accepts_strategy_and_grid_flags() { let r = parse_walkforward_args(&["--strategy", "stage1-r", "--fast", "5,10", "--stop-k", "2.0,3.0"]); - let (strategy, _, _, _, grid) = r.expect("valid stage1-r walkforward args"); + let (strategy, _, _, _, grid, _) = r.expect("valid stage1-r walkforward args"); assert_eq!(strategy, Strategy::Stage1R); assert_eq!(grid.fast, vec![5, 10]); assert_eq!(grid.stop_k, vec![2.0, 3.0]); @@ -4128,7 +4226,7 @@ mod tests { // closes >= 1 trade across its windows). Guards the wiring + the non-empty // pooling branch the parser/primitive unit tests cannot reach; mirrors // `mc_report` / `walkforward_report`. Deterministic (C1). - let result = walkforward_family(Strategy::Stage1R, None, &DataSource::Synthetic, &Stage1RGrid::default()); + let result = walkforward_family(Strategy::Stage1R, None, &DataSource::Synthetic, &Stage1RGrid::default(), Selection::Argmax); let pooled = pooled_oos_trade_rs(&result); assert!(!pooled.is_empty(), "synthetic stage1-r walk-forward must pool >= 1 OOS trade R"); diff --git a/crates/aura-cli/tests/cli_run.rs b/crates/aura-cli/tests/cli_run.rs index bef80f2..6871c28 100644 --- a/crates/aura-cli/tests/cli_run.rs +++ b/crates/aura-cli/tests/cli_run.rs @@ -2387,6 +2387,130 @@ fn runs_family_rank_shows_deflated_line() { ); } +/// Property: a stage1-r walk-forward run with `--select plateau:mean` selects the +/// neighbourhood-smoothed winner and stamps each OOS manifest with the plateau +/// provenance (`mode = PlateauMean`, `neighbourhood_score`, `n_neighbours`); `runs +/// family … rank` renders the `plateau(mean)=… over N cells` line, and the +/// deflation fields are omitted (orthogonal annotation). End-to-end witness that +/// the opt-in selection rule reaches disk (C18) and the display path renders it. +#[test] +fn walkforward_plateau_select_stamps_plateau_provenance() { + let cwd = temp_cwd("runs-plateau"); + let wf = Command::new(BIN) + .args([ + "walkforward", "--strategy", "stage1-r", "--select", "plateau:mean", + "--fast", "50,100", "--slow", "200,400", + "--stop-length", "14,21", "--stop-k", "2.0,3.0", + "--name", "wf-plat", + ]) + .current_dir(&cwd) + .output() + .expect("spawn walkforward --select plateau:mean"); + assert!( + wf.status.success(), + "walkforward exit: {:?}; stderr: {}", + wf.status, + String::from_utf8_lossy(&wf.stderr) + ); + + let rank = Command::new(BIN) + .args(["runs", "family", "wf-plat-0", "rank", "sqn_normalized"]) + .current_dir(&cwd) + .output() + .expect("spawn runs family wf-plat-0 rank sqn_normalized"); + assert!(rank.status.success(), "rank exit: {:?}", rank.status); + let rank_out = String::from_utf8(rank.stdout).expect("utf-8"); + let _ = std::fs::remove_dir_all(&cwd); + + assert!(rank_out.contains("plateau(mean)="), "rank output missing plateau(mean)=: {rank_out:?}"); + assert!(rank_out.contains(" cells"), "rank output missing 'over N cells': {rank_out:?}"); + assert!(rank_out.contains("\"mode\":\"PlateauMean\""), "manifest carries PlateauMean: {rank_out:?}"); + assert!(!rank_out.contains("\"deflated_score\""), "plateau run omits deflated_score: {rank_out:?}"); +} + +/// Property (C23, the opt-in feature's headline guarantee): `--select argmax` is a +/// no-op against the default. A stage1-r walk-forward run with an EXPLICIT +/// `--select argmax` produces stdout byte-for-byte identical to the same run with +/// no `--select` flag at all, AND keeps the trials-deflation provenance +/// (`mode == Argmax`, the deflated annotation) on each OOS manifest. Pins both +/// halves of "plateau is strictly opt-in": the flag's default IS argmax (no +/// divergence), and threading the new `Selection` did not silently demote argmax to +/// a bare pick (the #144 deflation path survives). A regression where `--select` +/// perturbed the default path, or where the default switched away from argmax, +/// passes every other test but fails this byte-equality. +#[test] +fn walkforward_select_argmax_is_byte_identical_to_default() { + let run = |args: &[&str]| { + let cwd = temp_cwd("wf-argmax-noop"); + let out = Command::new(BIN) + .arg("walkforward") + .args(args) + .current_dir(&cwd) + .output() + .expect("spawn aura walkforward"); + let _ = std::fs::remove_dir_all(&cwd); + assert!( + out.status.success(), + "walkforward exit: {:?}; stderr: {}", + out.status, + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8(out.stdout).expect("utf-8") + }; + let default = run(&["--strategy", "stage1-r"]); + let explicit = run(&["--strategy", "stage1-r", "--select", "argmax"]); + assert_eq!(default, explicit, "explicit --select argmax must be a no-op vs the default"); + // argmax stayed the deflation path (not a bare argmax): the per-window member + // JSON carries the Argmax mode and its deflation annotation. + assert!(default.contains("\"mode\":\"Argmax\""), "argmax manifest mode: {default:?}"); + assert!(default.contains("\"deflated_score\""), "argmax keeps the deflation annotation: {default:?}"); + assert!(!default.contains("\"neighbourhood_score\""), "argmax omits the plateau annotation: {default:?}"); +} + +/// Property: `--select plateau:worst` reaches disk and the display path through the +/// DISTINCT worst-case branch — a separate `SelectionMode::PlateauWorst` enum +/// variant on the wire and a separate `plateau(worst)=` display label. The landed +/// happy-path E2E exercises only `plateau:mean`; the worst arm's serialization +/// variant and its display label are otherwise untouched end-to-end, so a +/// regression that mislabelled worst as mean (or failed to round-trip the variant) +/// would pass the mean test. Same fixture grid as the mean E2E, only the rule +/// differs. +#[test] +fn walkforward_plateau_worst_stamps_worst_variant_and_label() { + let cwd = temp_cwd("runs-plateau-worst"); + let wf = Command::new(BIN) + .args([ + "walkforward", "--strategy", "stage1-r", "--select", "plateau:worst", + "--fast", "50,100", "--slow", "200,400", + "--stop-length", "14,21", "--stop-k", "2.0,3.0", + "--name", "wf-worst", + ]) + .current_dir(&cwd) + .output() + .expect("spawn walkforward --select plateau:worst"); + assert!( + wf.status.success(), + "walkforward exit: {:?}; stderr: {}", + wf.status, + String::from_utf8_lossy(&wf.stderr) + ); + + let rank = Command::new(BIN) + .args(["runs", "family", "wf-worst-0", "rank", "sqn_normalized"]) + .current_dir(&cwd) + .output() + .expect("spawn runs family wf-worst-0 rank sqn_normalized"); + assert!(rank.status.success(), "rank exit: {:?}", rank.status); + let rank_out = String::from_utf8(rank.stdout).expect("utf-8"); + let _ = std::fs::remove_dir_all(&cwd); + + assert!(rank_out.contains("plateau(worst)="), "rank output missing plateau(worst)=: {rank_out:?}"); + assert!(rank_out.contains("\"mode\":\"PlateauWorst\""), "manifest carries PlateauWorst: {rank_out:?}"); + // the worst label must not be rendered or stamped as mean + assert!(!rank_out.contains("plateau(mean)="), "worst run must not render the mean label: {rank_out:?}"); + assert!(!rank_out.contains("\"mode\":\"PlateauMean\""), "worst run must not stamp PlateauMean: {rank_out:?}"); +} + /// Property: the bare `aura walkforward` (default SMA-cross) summary is preserved /// byte-for-byte by the new `--strategy`/grid plumbing — it still carries exactly /// `windows`, `stitched_total_pips`, `param_stability` and NOT the stage1-r-only diff --git a/crates/aura-engine/src/blueprint.rs b/crates/aura-engine/src/blueprint.rs index 91b7c2b..83351c1 100644 --- a/crates/aura-engine/src/blueprint.rs +++ b/crates/aura-engine/src/blueprint.rs @@ -397,8 +397,20 @@ impl SweepBinder { } /// Resolve the named axes against `param_space()` into a positional grid and - /// run the disjoint sweep. + /// run the disjoint sweep. `sweep` is [`SweepBinder::sweep_with_lattice`] with + /// the lattice dropped, so every existing caller is byte-unchanged. pub fn sweep(self, run_one: F) -> Result + where + F: Fn(&[Cell]) -> RunReport + Sync, + { + self.sweep_with_lattice(run_one).map(|(family, _lattice)| family) + } + + /// As [`SweepBinder::sweep`], plus the grid's per-axis radixes (`axis_lens`, + /// in `param_space()` / odometer order). The lattice is what a plateau + /// neighbourhood walk needs (cycle 0077); only the engine's post-`resolve_axes` + /// grid holds it in the correct order. + pub fn sweep_with_lattice(self, run_one: F) -> Result<(SweepFamily, Vec), BindError> where F: Fn(&[Cell]) -> RunReport + Sync, { @@ -407,7 +419,8 @@ impl SweepBinder { let ordered = resolve_axes(&space, &self.axes)?; let grid = GridSpace::new(&space, ordered) .expect("named layer pre-validates arity/kind/non-empty"); - Ok(sweep(&grid, run_one)) + let lattice = grid.axis_lens(); + Ok((sweep(&grid, run_one), lattice)) } } @@ -938,6 +951,19 @@ mod tests { } } + #[test] + fn sweep_with_lattice_surfaces_grid_radixes_in_param_space_order() { + let bp = composite_sma_cross_harness().0; + let (fam, lattice) = bp + .axis("sma_cross.fast.length", vec![Scalar::i64(2), Scalar::i64(3)]) // 2 values + .axis("sma_cross.slow.length", vec![Scalar::i64(4), Scalar::i64(5)]) // 2 values + .axis("bias.scale", vec![Scalar::f64(0.5)]) // 1 value + .sweep_with_lattice(run_point) + .expect("named binding resolves and runs"); + assert_eq!(lattice, vec![2, 2, 1], "radixes in param_space slot order"); + assert_eq!(lattice.iter().product::(), fam.points.len()); // 4 + } + /// Property (the reason `RandomBinder` exists): building a random sweep **by /// name** is order-independent and binds the same knob to the same range /// regardless of `.range(...)` call order — exactly the safety the grid's diff --git a/crates/aura-engine/src/report.rs b/crates/aura-engine/src/report.rs index b0bd5e8..9abb318 100644 --- a/crates/aura-engine/src/report.rs +++ b/crates/aura-engine/src/report.rs @@ -39,30 +39,46 @@ pub struct RunMetrics { } /// Which selection objective produced the record (additive provenance, C23). -/// `Argmax` is the bare-best pick (cycle 0076). The enum is deliberately left -/// open: cycle 0145 adds a `Plateau*` variant on this same field, so peak-vs- -/// plateau runs stay distinguishable. +/// `Argmax` is the bare-best pick (cycle 0076), deflated for the number of trials. +/// `PlateauMean` / `PlateauWorst` (cycle 0077) argmax the neighbourhood-smoothed +/// surface instead, so peak-vs-plateau runs stay distinguishable on this field. #[derive(Clone, Copy, Debug, PartialEq, serde::Serialize, serde::Deserialize)] pub enum SelectionMode { Argmax, + PlateauMean, + PlateauWorst, } -/// Selection-provenance for a sweep winner: how its metric was deflated for the -/// number of configurations it beat. Additive — recorded, never re-ranking (C23). -/// `overfit_probability` is the empirical data-snooping p-value (R arm only); -/// `None` on the `total_pips` dispersion-floor arm. +/// Selection-provenance for a sweep winner. The selection RULE (`mode`) and its +/// ANNOTATION are orthogonal: `Argmax` carries the trials-deflation annotation +/// (`deflated_score` / `overfit_probability` / `n_resamples` / `block_len` / +/// `seed`); `Plateau*` carries the smoothing annotation (`neighbourhood_score` / +/// `n_neighbours`). Each annotation is `Option`, present iff its rule produced the +/// record; all are additive — recorded, never re-ranking (C23). A legacy line +/// (pre-0077) carries the deflation scalars as bare values that deserialize to +/// `Some` (serde default), so it still loads (C14/C18). #[derive(Clone, Debug, PartialEq, serde::Serialize, serde::Deserialize)] pub struct FamilySelection { pub selection_metric: String, pub n_trials: usize, pub raw_winner_metric: f64, - pub deflated_score: f64, + pub mode: SelectionMode, + // deflation annotation (present iff mode == Argmax with a deflation run) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub deflated_score: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub overfit_probability: Option, - pub mode: SelectionMode, - pub n_resamples: usize, - pub block_len: usize, - pub seed: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub n_resamples: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub block_len: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub seed: Option, + // plateau annotation (present iff mode is Plateau*) + #[serde(default, skip_serializing_if = "Option::is_none")] + pub neighbourhood_score: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub n_neighbours: Option, } /// R-based signal-quality metrics (Stage-1), reduced from a `PositionManagement` dense @@ -763,8 +779,10 @@ mod tests { fn family_selection_round_trips_on_the_manifest() { let sel = FamilySelection { selection_metric: "sqn_normalized".into(), n_trials: 4, raw_winner_metric: 1.83, - deflated_score: 0.21, overfit_probability: Some(0.06), mode: SelectionMode::Argmax, - n_resamples: 1000, block_len: 5, seed: 42, + mode: SelectionMode::Argmax, + deflated_score: Some(0.21), overfit_probability: Some(0.06), + n_resamples: Some(1000), block_len: Some(5), seed: Some(42), + neighbourhood_score: None, n_neighbours: None, }; let m = RunManifest { commit: "c".into(), params: vec![], window: (Timestamp(0), Timestamp(0)), @@ -774,6 +792,25 @@ mod tests { assert_eq!(back.selection, Some(sel)); } + #[test] + fn family_selection_plateau_shape_round_trips() { + let sel = FamilySelection { + selection_metric: "sqn_normalized".into(), n_trials: 16, raw_winner_metric: 1.42, + mode: SelectionMode::PlateauMean, + deflated_score: None, overfit_probability: None, + n_resamples: None, block_len: None, seed: None, + neighbourhood_score: Some(1.27), n_neighbours: Some(5), + }; + let json = serde_json::to_string(&sel).unwrap(); + // plateau annotation present; deflation fields omitted (skip_serializing_if) + assert!(json.contains("\"neighbourhood_score\":1.27"), "json: {json}"); + assert!(json.contains("\"n_neighbours\":5"), "json: {json}"); + assert!(!json.contains("deflated_score"), "deflation fields omitted under plateau: {json}"); + assert!(!json.contains("n_resamples"), "json: {json}"); + let back: FamilySelection = serde_json::from_str(&json).unwrap(); + assert_eq!(back, sel); + } + #[test] fn position_action_round_trips_through_i64() { for a in [PositionAction::Buy, PositionAction::Sell, PositionAction::Close] { diff --git a/crates/aura-engine/src/sweep.rs b/crates/aura-engine/src/sweep.rs index aa34f7f..6840d93 100644 --- a/crates/aura-engine/src/sweep.rs +++ b/crates/aura-engine/src/sweep.rs @@ -64,6 +64,13 @@ impl GridSpace { false } + /// Per-axis cardinalities in `param_space()` order (the odometer radixes, + /// last-axis-fastest). `∏ axis_lens() == len()`. The lattice shape a plateau + /// neighbourhood walks (cycle 0077). + pub fn axis_lens(&self) -> Vec { + self.axes.iter().map(Vec::len).collect() + } + /// The cartesian product, in odometer order: the **last** axis varies /// fastest. Deterministic — the same grid yields the same point sequence. pub fn points(&self) -> Vec> { @@ -597,6 +604,20 @@ mod tests { assert!(!grid.is_empty()); } + #[test] + fn grid_axis_lens_are_the_per_axis_radixes() { + let space = vec![ + ParamSpec { name: "a".into(), kind: ScalarKind::I64 }, + ParamSpec { name: "b".into(), kind: ScalarKind::I64 }, + ]; + let grid = GridSpace::new(&space, vec![ + vec![Scalar::i64(10), Scalar::i64(20)], // axis 0: 2 values + vec![Scalar::i64(1), Scalar::i64(2), Scalar::i64(3)], // axis 1: 3 values + ]).expect("2x3 grid"); + assert_eq!(grid.axis_lens(), vec![2, 3]); + assert_eq!(grid.axis_lens().iter().product::(), grid.len()); + } + #[test] fn arity_mismatch_is_an_error() { let space = i64_space(2); diff --git a/crates/aura-registry/src/lib.rs b/crates/aura-registry/src/lib.rs index b6c8807..cffb409 100644 --- a/crates/aura-registry/src/lib.rs +++ b/crates/aura-registry/src/lib.rs @@ -155,21 +155,20 @@ fn metric_value(rep: &RunReport, m: Metric) -> f64 { /// deterministically without panicking. An unknown metric name is a /// `RegistryError::UnknownMetric`. /// -/// The single source of truth for "best" — both [`rank_by`] (sort by it) and -/// [`optimize`] (argmax by it) call this, so the per-metric direction lives in -/// exactly one place. The metric name is resolved once via `resolve_metric`; the -/// returned closure carries the resolved `Metric`, so per-comparison work is just -/// reading each value (`metric_value`) and the key compare. +/// The single source of best-first *ordering* — both [`rank_by`] (sort by it) and +/// [`optimize`] (argmax by it) call this. The per-metric direction itself lives in +/// `higher_is_better` (which this and `optimize_plateau` both read); `metric_cmp` +/// turns that direction into a `total_cmp` over the read values. The metric name is +/// resolved once via `resolve_metric`; the returned closure carries the resolved +/// `Metric`, so per-comparison work is just reading each value (`metric_value`) and +/// the key compare. fn metric_cmp(metric: &str) -> Result Ordering, RegistryError> { let m = resolve_metric(metric)?; + let hib = higher_is_better(m); Ok(move |a: &RunReport, b: &RunReport| { let (va, vb) = (metric_value(a, m), metric_value(b, m)); - match m { - // lower-is-better - Metric::MaxDrawdown | Metric::BiasSignFlips => va.total_cmp(&vb), - // higher-is-better - _ => vb.total_cmp(&va), - } + // higher-is-better ranks the larger value first; lower-is-better the smaller. + if hib { vb.total_cmp(&va) } else { va.total_cmp(&vb) } }) } @@ -206,6 +205,43 @@ fn is_r_metric(m: Metric) -> bool { matches!(m, Metric::Sqn | Metric::SqnNormalized | Metric::ExpectancyR | Metric::NetExpectancyR) } +/// The metric's optimisation direction: `true` if a larger value is better +/// (`total_pips`, the four R keys), `false` for the lower-is-better keys +/// (`max_drawdown`, `bias_sign_flips`). The single direction source — `metric_cmp` +/// (best-first ordering) and `optimize_plateau` (smoothed argmax + worst-neighbour) +/// both read it, so the per-metric sense lives in exactly one place. +fn higher_is_better(m: Metric) -> bool { + !matches!(m, Metric::MaxDrawdown | Metric::BiasSignFlips) +} + +/// The closed grid neighbourhood of flat index `i` over a mixed-radix lattice +/// (`axis_lens`, last axis fastest — the `GridSpace` odometer convention): `{i}` +/// plus each in-range ±1-per-axis cell. Pure index math, deterministic. The order +/// is `i` first then per-axis neighbours; callers use the result only as a set and +/// for its length, so the order is not load-bearing. +fn closed_neighbourhood(i: usize, axis_lens: &[usize]) -> Vec { + // decompose i to per-axis coords (last axis fastest) + let mut coords = vec![0usize; axis_lens.len()]; + let mut rem = i; + for k in (0..axis_lens.len()).rev() { + coords[k] = rem % axis_lens[k]; + rem /= axis_lens[k]; + } + // recompose a coord vector to its flat index (Horner, last axis fastest) + let recompose = |c: &[usize]| c.iter().zip(axis_lens).fold(0usize, |flat, (&ck, &len)| flat * len + ck); + let mut out = vec![i]; + for k in 0..axis_lens.len() { + for delta in [-1isize, 1] { + let ck = coords[k] as isize + delta; + if ck < 0 || ck as usize >= axis_lens[k] { continue; } + let mut nb = coords.clone(); + nb[k] = ck as usize; + out.push(recompose(&nb)); + } + } + out +} + fn member_trade_rs(rep: &RunReport) -> &[f64] { rep.metrics.r.as_ref().map(|r| r.trade_rs.as_slice()).unwrap_or(&[]) } @@ -255,6 +291,84 @@ fn member_sd(family: &SweepFamily, m: Metric) -> f64 { (vals.iter().map(|v| (v - mean).powi(2)).sum::() / (n - 1) as f64).sqrt() } +/// Plateau-selection aggregate: how a member's grid-neighbourhood metric is +/// reduced to one smoothed score before the argmax. `Mean` averages the closed +/// neighbourhood; `Worst` takes the most-pessimistic neighbour by the metric's +/// direction (biasing toward interior cells, which keep more neighbours). +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum PlateauMode { + Mean, + Worst, +} + +/// `optimize`'s argmax over the NEIGHBOURHOOD-SMOOTHED surface, plus its plateau +/// provenance. Each member scores as the mean (or worst-case) of its closed grid +/// neighbourhood's `metric_value`; the winner is the best smoothed score by the +/// metric's own direction (earliest-odometer tie, as `optimize`). `axis_lens` are +/// the grid radixes (`param_space()` order, last axis fastest). Pure, +/// deterministic (C1). The returned `SweepPoint` is the winning grid CELL; +/// `raw_winner_metric` is that cell's own metric, `neighbourhood_score` the +/// smoothed value the argmax maximised. +pub fn optimize_plateau( + family: &SweepFamily, axis_lens: &[usize], metric: &str, mode: PlateauMode, +) -> Result<(SweepPoint, FamilySelection), RegistryError> { + let m = resolve_metric(metric)?; + let n = family.points.len(); + debug_assert_eq!( + axis_lens.iter().product::(), n, + "axis_lens product must equal the family size (a valid grid lattice)", + ); + let hib = higher_is_better(m); + // smoothed score + closed-neighbourhood size per member, in odometer order + let scored: Vec<(usize, f64, usize)> = (0..n) + .map(|i| { + let nbrs = closed_neighbourhood(i, axis_lens); + let vals: Vec = nbrs.iter().map(|&j| metric_value(&family.points[j].report, m)).collect(); + let score = match mode { + PlateauMode::Mean => vals.iter().sum::() / vals.len() as f64, + PlateauMode::Worst => { + if hib { + vals.iter().copied().fold(f64::INFINITY, f64::min) + } else { + vals.iter().copied().fold(f64::NEG_INFINITY, f64::max) + } + } + }; + (i, score, nbrs.len()) + }) + .collect(); + // argmax the smoothed surface by direction; earliest-odometer tie (only a + // strictly-better later cell displaces the incumbent, as `optimize`). + let &(wi, wscore, wn) = scored + .iter() + .reduce(|best, cur| { + let better = if hib { cur.1 > best.1 } else { cur.1 < best.1 }; + if better { cur } else { best } + }) + .expect("a SweepFamily is non-empty by construction (EmptyAxis is rejected upstream)"); + let winner = family.points[wi].clone(); + let raw = metric_value(&winner.report, m); + Ok(( + winner, + FamilySelection { + selection_metric: metric.to_string(), + n_trials: n, + raw_winner_metric: raw, + mode: match mode { + PlateauMode::Mean => SelectionMode::PlateauMean, + PlateauMode::Worst => SelectionMode::PlateauWorst, + }, + deflated_score: None, + overfit_probability: None, + n_resamples: None, + block_len: None, + seed: None, + neighbourhood_score: Some(wscore), + n_neighbours: Some(wn), + }, + )) +} + /// `optimize`'s argmax winner PLUS its trials-deflation provenance. The returned /// `SweepPoint` is byte-identical to `optimize(family, metric)` (additive, C23). /// R arm: a centred moving-block reality-check (`overfit_probability` = @@ -307,8 +421,10 @@ pub fn optimize_deflated( Ok((winner.clone(), FamilySelection { selection_metric: metric.to_string(), n_trials: k, raw_winner_metric: raw, - deflated_score, overfit_probability, mode: SelectionMode::Argmax, - n_resamples, block_len, seed, + mode: SelectionMode::Argmax, + deflated_score: Some(deflated_score), overfit_probability, + n_resamples: Some(n_resamples), block_len: Some(block_len), seed: Some(seed), + neighbourhood_score: None, n_neighbours: None, })) } @@ -746,7 +862,7 @@ mod tests { let sel = optimize_deflated(&fam, "total_pips", 100, 3, 1).unwrap().1; assert!(sel.overfit_probability.is_none()); assert_eq!(sel.selection_metric, "total_pips"); - assert!(sel.deflated_score <= sel.raw_winner_metric); // dispersion floor ≤ raw + assert!(sel.deflated_score.unwrap() <= sel.raw_winner_metric); // dispersion floor ≤ raw } /// Sibling-contract parity with `r_bootstrap`, which defines `n_resamples == @@ -759,8 +875,8 @@ mod tests { fn optimize_deflated_zero_resamples_floors_like_r_bootstrap() { let fam = fixture_family_with_r(); let sel = optimize_deflated(&fam, "expectancy_r", 0, 3, 1).unwrap().1; - assert_eq!(sel.n_resamples, 0); - assert_eq!(sel.deflated_score, sel.raw_winner_metric, "p95 of an empty null is 0"); + assert_eq!(sel.n_resamples, Some(0)); + assert_eq!(sel.deflated_score.unwrap(), sel.raw_winner_metric, "p95 of an empty null is 0"); assert_eq!(sel.overfit_probability, Some(1.0), "(0+1)/(0+1) Laplace floor"); } @@ -779,8 +895,8 @@ mod tests { } } let sel = optimize_deflated(&fam, "expectancy_r", 500, 3, 1).unwrap().1; - assert!(sel.deflated_score.is_finite(), "deflated_score must be finite, got {}", sel.deflated_score); - assert_eq!(sel.deflated_score, sel.raw_winner_metric); + assert!(sel.deflated_score.unwrap().is_finite(), "deflated_score must be finite, got {:?}", sel.deflated_score); + assert_eq!(sel.deflated_score.unwrap(), sel.raw_winner_metric); assert_eq!(sel.overfit_probability, Some(1.0)); } @@ -791,7 +907,7 @@ mod tests { let fam = fixture_family_one_strong_edge(); let sel = optimize_deflated(&fam, "expectancy_r", 1000, 1, 9).unwrap().1; assert!(sel.overfit_probability.unwrap() < 0.2, "p={:?}", sel.overfit_probability); - assert!(sel.deflated_score > 0.0); + assert!(sel.deflated_score.unwrap() > 0.0); } /// Every member shares the SAME strong +R edge. An *uncentred* null (each @@ -856,4 +972,127 @@ mod tests { let m = resolve_metric("sqn_normalized").unwrap(); assert_eq!(sel.raw_winner_metric, metric_value(&winner.report, m)); } + + /// A 1-D grid (7 cells, odometer order) with two end spikes and a broad middle + /// plateau in `total_pips`. Bare argmax takes an end spike; the plateau-mean + /// argmax takes the interior plateau centre. + fn fixture_grid_spike_vs_plateau() -> SweepFamily { + let pips = [80.0, 5.0, 48.0, 50.0, 49.0, 5.0, 80.0]; + SweepFamily { space: vec![], points: pips.iter().map(|&p| member(p, vec![1.0])).collect() } + } + + #[test] + fn closed_neighbourhood_2x3_golden() { + // 2x3 lattice, odometer (last axis fastest): + // (0,0)=0 (0,1)=1 (0,2)=2 + // (1,0)=3 (1,1)=4 (1,2)=5 + let sorted = |i: usize| { let mut v = closed_neighbourhood(i, &[2, 3]); v.sort_unstable(); v }; + assert_eq!(sorted(0), vec![0, 1, 3]); // corner (0,0): self + 2 + assert_eq!(sorted(4), vec![1, 3, 4, 5]); // interior (1,1): self + 3 + assert_eq!(sorted(2), vec![1, 2, 5]); // corner (0,2): self + 2 + } + + #[test] + fn plateau_picks_the_plateau_not_the_spike() { + let fam = fixture_grid_spike_vs_plateau(); + let spike = optimize(&fam, "total_pips").unwrap(); + let (centre, sel) = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Mean).unwrap(); + assert_ne!(centre.report.metrics.total_pips, spike.report.metrics.total_pips); + assert_eq!(centre.report.metrics.total_pips, 50.0, "plateau centre is the middle cell"); + assert_eq!(sel.mode, SelectionMode::PlateauMean); + assert_eq!(sel.raw_winner_metric, 50.0); + assert!(sel.neighbourhood_score.unwrap() < spike.report.metrics.total_pips, + "smoothed score is below the spike's raw peak"); + assert_eq!(sel.n_neighbours, Some(3), "interior cell has self + 2 neighbours"); + assert!(sel.deflated_score.is_none() && sel.n_resamples.is_none(), + "deflation fields omitted under plateau"); + } + + #[test] + fn plateau_worst_is_more_conservative_than_mean() { + let fam = fixture_grid_spike_vs_plateau(); + let (_, mean_sel) = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Mean).unwrap(); + let (_, worst_sel) = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Worst).unwrap(); + // Worst scores each cell by its weakest neighbour, never above the mean. + assert!(worst_sel.neighbourhood_score.unwrap() <= mean_sel.neighbourhood_score.unwrap()); + assert_eq!(worst_sel.mode, SelectionMode::PlateauWorst); + assert!(worst_sel.n_neighbours.unwrap() >= 2, "winner is an interior cell"); + } + + #[test] + fn plateau_single_member_equals_optimize() { + let fam = SweepFamily { space: vec![], points: vec![member(42.0, vec![1.0])] }; + let plain = optimize(&fam, "total_pips").unwrap(); + let (winner, sel) = optimize_plateau(&fam, &[1], "total_pips", PlateauMode::Mean).unwrap(); + assert_eq!(winner.report.metrics.total_pips, plain.report.metrics.total_pips); + assert_eq!(sel.n_neighbours, Some(1), "a 1-cell grid's closed neighbourhood is itself"); + assert_eq!(sel.neighbourhood_score, Some(42.0)); + assert_eq!(sel.raw_winner_metric, 42.0); + } + + #[test] + fn optimize_plateau_is_deterministic() { + let fam = fixture_grid_spike_vs_plateau(); + let a = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Worst).unwrap().1; + let b = optimize_plateau(&fam, &[7], "total_pips", PlateauMode::Worst).unwrap().1; + assert_eq!(a, b); + } + + /// A 1-D grid (7 cells, odometer order) in `max_drawdown` — the canonical + /// lower-is-better key. Two end *dips* (drawdown 1.0, the lure for a bare + /// lower-is-better argmax) flank a broad middle plateau of moderately-low + /// drawdown (≈30), with worse shoulders. A member carries only `max_drawdown`; + /// `total_pips`/`r` are inert here. + fn fixture_grid_dip_vs_plateau_lower_is_better() -> SweepFamily { + let dds = [1.0, 60.0, 30.0, 30.0, 31.0, 60.0, 1.0]; + let point = |dd: f64| SweepPoint { + params: vec![], + report: report_with(0.0, dd, 0), + }; + SweepFamily { space: vec![], points: dds.iter().map(|&dd| point(dd)).collect() } + } + + /// Direction-flip coverage: on a lower-is-better metric (`max_drawdown`), + /// plateau selection exercises the `else` (smaller-is-better) argmax branch AND + /// the `Worst` arm's `NEG_INFINITY`/`max` fold. Bare argmax takes an end dip + /// (the global minimum drawdown); the plateau-mean argmax takes the interior + /// plateau centre, whose neighbourhood mean is the lowest. This is the + /// lower-is-better companion the higher-is-better plateau tests do not reach. + #[test] + fn plateau_lower_is_better_picks_the_plateau_not_the_dip() { + let fam = fixture_grid_dip_vs_plateau_lower_is_better(); + let dip = optimize(&fam, "max_drawdown").unwrap(); + assert_eq!(dip.report.metrics.max_drawdown, 1.0, "bare argmax takes the end dip"); + + let (centre, mean_sel) = + optimize_plateau(&fam, &[7], "max_drawdown", PlateauMode::Mean).unwrap(); + // the plateau centre is an interior cell, not the end dip + assert_ne!(centre.report.metrics.max_drawdown, 1.0); + assert_eq!(mean_sel.mode, SelectionMode::PlateauMean); + assert_eq!(mean_sel.n_neighbours, Some(3), "interior cell has self + 2 neighbours"); + // the winning cell's neighbourhood mean beats the end dip's (whose 60.0 + // shoulder drags its mean up), so the smaller-is-better argmax did not pick + // the dip. + assert!( + mean_sel.neighbourhood_score.unwrap() < (1.0 + 60.0) / 2.0, + "plateau mean is below the end dip's neighbourhood mean; score={:?}", + mean_sel.neighbourhood_score, + ); + + // Worst arm: each cell scores by its highest (worst) drawdown neighbour + // (the NEG_INFINITY/max fold). It is never below the mean for a + // lower-is-better metric, and still avoids the dip whose worst neighbour is + // 60.0. + let (_, worst_sel) = + optimize_plateau(&fam, &[7], "max_drawdown", PlateauMode::Worst).unwrap(); + assert_eq!(worst_sel.mode, SelectionMode::PlateauWorst); + assert!( + worst_sel.neighbourhood_score.unwrap() >= mean_sel.neighbourhood_score.unwrap(), + "worst (max drawdown) ≥ mean for a lower-is-better metric", + ); + assert!( + worst_sel.neighbourhood_score.unwrap() < 60.0, + "the plateau interior's worst neighbour is below the dip's 60.0 shoulder", + ); + } }